Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

114 KB
Raw Blame
"""soup train — the main training command."""

from __future__ import annotations

import contextlib
import json
import logging
import os
import re
import sys
from pathlib import Path
from typing import TYPE_CHECKING, Annotated, Any

import typer
from rich.console import Console
from rich.markup import escape as markup_escape
from rich.panel import Panel

from soup_cli.config.loader import load_config
from soup_cli.data.loader import (
    PREFERENCE_PAIR_TASKS,
    data_config_for_task,
    load_dataset,
    task_preserves_source_columns,
)
from soup_cli.monitoring.display import TrainingDisplay
from soup_cli.trainer.classifier import CLASSIFICATION_TASKS
from soup_cli.utils.gpu import (
    configure_cuda_alloc_hint,
    configure_mps_async_load,
    detect_device,
    get_gpu_info,
    resolve_quantization,
)

if TYPE_CHECKING:  # pragma: no cover - type hints only, no runtime import
    from soup_cli.config.schema import SoupConfig
    from soup_cli.utils.energy import EnergyMeasurement

logger = logging.getLogger(__name__)
console = Console()

# Optimizers the analytical hardware-fit predictor understands (mirror of
# hardware_fit._VALID_OPTIMIZERS); an unknown optimizer maps to the
# highest-state default so the estimate stays conservative.
_HW_FIT_OPTIMIZERS = frozenset({
    "adamw_torch", "adamw_torch_fused", "adafactor", "sgd",
    "adamw_bnb_8bit", "paged_adamw_8bit", "lion_8bit",
    "lomo", "adalomo", "schedule_free_adamw",
})

# training.quantization -> the hardware-fit predictor's name for it. A value
# that is not here (``hqq:*``) has no memory model, and the pre-flight says it
# skipped the run (#1652).
_HW_FIT_QUANT = {
    "none": "none",
    "4bit": "4bit",
    "8bit": "8bit",
    # Stay packed on the card.
    "gptq": "gptq",
    "awq": "awq",
    "aqlm": "aqlm",
    "eetq": "eetq",
    # Dequantized on load, so priced (and gated) like "none" (#1631, #1652).
    "mxfp4": "mxfp4",
    "fp8": "fp8",
}

# The formats above that train in bf16, and how the gate's panel names them.
_HW_FIT_DEQUANTIZED_ON_LOAD = {"mxfp4": "MXFP4", "fp8": "FP8"}

_UNWIRED_TRAINING_TUNABLES = (
    # Group B tunables (forgetting_eval_steps, forgetting_benchmark, forgetting_stop,
    # checkpoint_eval_steps, checkpoint_eval_metric, checkpoint_eval_tasks,
    # checkpoint_keep_top, convergence_window, convergence_rel_tol) moved to
    # config/staged_fields.py (#808), and early_stop_patience moved in #761:
    # the loader warns about them with the refusal date, so they are not listed here.
    "forgetting_threshold",
)


def _stdin_isatty() -> bool:
    """True when standard input is an interactive terminal."""
    return sys.stdin.isatty()


def _nondefault_unwired_training_settings(training_config) -> list[str]:
    """Return staged training settings whose value differs from the schema default."""
    fields = type(training_config).model_fields
    enabled_flags = [
        name
        for name, enabled in (
            ("forgetting_detection", training_config.forgetting_detection),
            ("checkpoint_intelligence", training_config.checkpoint_intelligence),
            ("early_stop_on_regression", training_config.early_stop_on_regression),
            ("convergence_detection", training_config.convergence_detection),
        )
        if enabled
    ]
    changed_tunables = [
        name
        for name in _UNWIRED_TRAINING_TUNABLES
        if getattr(training_config, name) != fields[name].default
    ]
    return enabled_flags + changed_tunables


UNSUPPORTED_RESUME_TASKS: frozenset[str] = frozenset({"unlearn"})


def _format_training_complete_loss(result: dict) -> str:
    """Render only a loss comparison that the trainer actually measured."""
    summary_kind = result.get("loss_summary_kind")
    if summary_kind == "unavailable":
        return "Loss: [bold]unavailable[/]"
    loss_key = result.get("loss_key")
    label = f" [dim]({loss_key})[/]" if isinstance(loss_key, str) else ""
    if summary_kind in {"mean", "single"} or (
        summary_kind is None and result["initial_loss"] == result["final_loss"]
    ):
        return f"Loss: [bold]{result['final_loss']:.4f}[/]{label}"
    return f"Loss: [bold]{result['initial_loss']:.4f} -> {result['final_loss']:.4f}[/]{label}"


def _format_duration_display(result: dict) -> str:
    """Duration line for the completion panel (#1529).

    Every trainer wrapper returns both a pre-formatted ``duration`` string and
    the raw ``duration_secs`` — except unlearn, which until #1529 returned only
    the seconds, so the panel's ``result['duration']`` raised ``KeyError``
    AFTER the adapter was saved and the run was otherwise complete. Read the
    string when present and fall back to formatting the seconds (``unknown``
    when both are missing — a measured ``0m`` stays ``0m``, but an absent
    measurement should not read as a zero) so no wrapper can lose a finished
    run's summary.
    """
    duration = result.get("duration")
    if duration:
        return duration
    duration_secs = result.get("duration_secs")
    if duration_secs is None:
        return "unknown"
    hours = int(duration_secs // 3600)
    minutes = int((duration_secs % 3600) // 60)
    return f"{hours}h {minutes}m" if hours > 0 else f"{minutes}m"


def _run_auto_eval_after_training(
    eval_config,
    output_dir: str,
    run_id: str,
    *,
    trust_remote_code: bool = False,
) -> None:
    """Run configured evaluation once after the trained model is saved.

    ``trust_remote_code`` is the run's own ``--trust-remote-code`` flag (CLI only;
    there is no soup.yaml field). It reaches only the loads of the trained output
    and its base: ``benchmark`` and ``custom``.
    """
    if not eval_config or not getattr(eval_config, "auto_eval", False):
        return
    if not output_dir or not _should_run_diagnose_gate_on_rank():
        return

    console.print("\n[bold blue]Running auto-eval...[/]")

    benchmarks = getattr(eval_config, "benchmarks", None) or []
    custom_tasks = getattr(eval_config, "custom_tasks", None)

    if benchmarks:
        try:
            from soup_cli.commands.eval import benchmark

            benchmark(
                model=output_dir,
                benchmarks=",".join(benchmarks),
                num_fewshot=None,
                batch_size=8,
                run_id=run_id,
                device=None,
                trust_remote_code=trust_remote_code,
            )
        except typer.Exit:
            logger.debug("Auto-eval benchmark skipped", exc_info=True)
            console.print(
                "[yellow]Auto-eval benchmark skipped (see the message above)[/]"
            )
        except Exception as exc:
            logger.exception("Auto-eval benchmark failed")
            console.print(
                f"[yellow]Auto-eval benchmark failed: {markup_escape(str(exc))}[/]"
            )

    if custom_tasks:
        try:
            from soup_cli.commands.eval import custom

            custom(
                tasks=custom_tasks,
                model=output_dir,
                run_id=run_id,
                attach_to_registry=None,
                output=None,
                trust_remote_code=trust_remote_code,
            )
        except typer.Exit:
            logger.debug("Auto-eval custom skipped", exc_info=True)
            console.print(
                "[yellow]Auto-eval custom skipped (see the message above)[/]"
            )
        except Exception as exc:
            logger.exception("Auto-eval custom failed")
            console.print(
                f"[yellow]Auto-eval custom failed: {markup_escape(str(exc))}[/]"
            )


def _train_sample_count(dcfg, dataset) -> int:
    """Rows training will actually consume (#1054).

    For ``format: pre_tokenized`` the rows come from the Arrow cache at
    ``data.tokenized_path`` (loaded later, in the trainer), not from
    ``load_dataset`` — which sees the ORIGINAL ``data.train`` file and drops
    every row for want of an ``input_ids`` column, printing "0 train samples"
    for a run that then trains on the whole cache. Read the count preprocess
    recorded instead, falling back to the loader's view if it is unavailable or
    not a plausible count (a negative ``row_count`` is a corrupt or hand-edited
    metadata.json, and printing "Loaded: -3 train samples" helps nobody).
    """
    rows = len(dataset.get("train", []))
    if dcfg.format != "pre_tokenized" or not dcfg.tokenized_path:
        return rows
    try:
        with open(
            os.path.join(dcfg.tokenized_path, "metadata.json"), encoding="utf-8"
        ) as f:
            count = json.load(f).get("row_count")
    except (OSError, ValueError):
        return rows
    valid = isinstance(count, int) and not isinstance(count, bool) and count >= 0
    return count if valid else rows


def _validate_classification_dataset_if_applicable(cfg: Any, dataset: dict) -> None:
    """Validate sequence classification rows upfront if task is in CLASSIFICATION_TASKS."""
    if cfg.task in CLASSIFICATION_TASKS:
        from rich.markup import escape

        from soup_cli.trainer.classifier import validate_classification_dataset

        try:
            validate_classification_dataset(cfg, dataset)
        except (ValueError, TypeError) as exc:
            console.print(
                f"[red]Error validating {cfg.task} dataset:[/] {escape(str(exc))}"
            )
            raise typer.Exit(1) from exc


def _validate_csm_dataset_if_applicable(cfg: Any, dataset: dict) -> None:
    if cfg.task != "tts" or cfg.training.tts_family != "sesame_csm":
        return
    from rich.markup import escape

    from soup_cli.utils.csm import validate_csm_dataset

    try:
        validate_csm_dataset(dataset)
    except ValueError as exc:
        console.print(f"[red]Error validating CSM dataset:[/] {escape(str(exc))}")
        raise typer.Exit(1) from exc


def _validate_preference_dataset_if_applicable(cfg: Any, dataset: dict) -> None:
    """Reject a lossy format choice before constructing any preference trainer."""
    if cfg.task not in PREFERENCE_PAIR_TASKS:
        return
    from soup_cli.data.loader import last_load_outcome

    required = {"prompt", "chosen", "rejected"}
    outcome = last_load_outcome()
    fmt = outcome.fmt or cfg.data.format
    for split in ("train", "val"):
        rows = dataset.get(split, [])
        for index, row in enumerate(rows, 1):
            if isinstance(row, dict) and required.issubset(row):
                continue
            console.print(
                f"[red]Invalid {cfg.task} dataset:[/] {split} row {index} loaded as "
                f"format {markup_escape(repr(fmt))} lacks prompt, chosen or rejected. "
                "Use data.format: dpo with preference-pair rows."
            )
            raise typer.Exit(1)
    if not dataset.get("train") and outcome.first_drop is not None:
        console.print(
            f"[red]No training rows:[/] preference data loaded as format "
            f"{markup_escape(repr(fmt))} must provide prompt, chosen and rejected. "
            "Use data.format: dpo with preference-pair rows."
        )
        raise typer.Exit(1)


def _refuse_empty_train(dcfg, dataset) -> None:
    """Stop a run whose data loaded zero training rows (#1217).

    The loader drops rows its format cannot convert (one bad line must not
    abort a load), so a format mismatch loads nothing and still returns
    cleanly. Before this, ``--dry-run`` then printed "Data OK: 0 train
    samples" and "Config valid. Ready to train!", and the real run loaded the
    model before failing on an empty dataset. Names the format and the first
    drop reason, which is what the user has to change.
    """
    from rich.markup import escape

    from soup_cli.data.loader import last_load_outcome

    if _train_sample_count(dcfg, dataset) > 0:
        return
    outcome = last_load_outcome()
    fmt = outcome.fmt or dcfg.format
    message = (
        f"[red]No training rows:[/] {escape(str(dcfg.train))} loaded 0 train "
        f"samples as format {escape(repr(fmt))}."
    )
    if outcome.first_drop is not None:
        drop_fmt, index, reason, _source = outcome.first_drop
        message += (
            f"\nEvery row that failed to convert was dropped. First: row "
            f"{index} (as {escape(repr(drop_fmt))}): {escape(reason)}."
            "\nFix those rows, or set data.format to the format they are in."
        )
    else:
        message += (
            "\nNo row failed to convert: the data is empty, every row went to "
            "validation (data.val_split), or the rows were skipped for missing "
            "media files."
        )
    console.print(message)
    raise typer.Exit(1)


def _refuse_over_judge_budget(cfg, dataset) -> None:
    """Refuse a ``judge_rank`` run that could exceed its judge-call budget (#2047).

    The rows are known only once the dataset is loaded, and the trainer loads
    the model before it reads them, so this runs between the two.
    """
    from rich.markup import escape

    from soup_cli.trainer.grpo_judge import check_judge_rank_budget

    try:
        check_judge_rank_budget(cfg, dataset)
    except ValueError as exc:
        console.print(f"[red]Judge budget:[/] {escape(str(exc))}")
        raise typer.Exit(1) from exc


def _build_hardware_fit_input(cfg):
    """Best-effort ``HardwareFitInput`` from a ``SoupConfig``.

    Returns ``None`` when the run is not statically predictable (batch_size
    ``"auto"``, unknown model size, unsupported quant, out-of-range dims), in
    which case the caller skips the gate rather than guess.
    """
    from soup_cli.utils.gpu import model_size_from_name
    from soup_cli.utils.hardware_fit import HardwareFitInput, resolve_peft_mode

    tcfg = cfg.training
    bs = getattr(tcfg, "batch_size", None)
    if not isinstance(bs, int) or isinstance(bs, bool):
        return None  # "auto" resolves later — can't predict yet
    params_b = model_size_from_name(getattr(cfg, "base", "") or "")
    if not isinstance(params_b, (int, float)) or params_b <= 0:
        return None
    seq_len = getattr(cfg.data, "max_length", None)
    if not isinstance(seq_len, int) or isinstance(seq_len, bool):
        return None
    quant = _HW_FIT_QUANT.get(str(getattr(tcfg, "quantization", "none") or "none"))
    if quant is None:
        return None
    peft = resolve_peft_mode(cfg, quant=quant)
    optimizer = str(getattr(tcfg, "optimizer", "adamw_torch") or "adamw_torch")
    if optimizer not in _HW_FIT_OPTIMIZERS:
        optimizer = "adamw_torch"
    gc = bool(getattr(tcfg, "gradient_checkpointing", False))
    try:
        return HardwareFitInput(
            params_b=float(params_b),
            seq_len=int(seq_len),
            batch_size=int(bs),
            optimizer=optimizer,
            quant=quant,
            peft=peft,
            gradient_checkpointing=gc,
        )
    except (ValueError, TypeError):
        return None  # dims out of the predictor's supported range


def _hardware_fit_preflight(cfg, gpu_info, *, allow_oom_attempt: bool) -> None:
    """Refuse (or warn) before launch when the predicted peak VRAM won't fit.

    Skips silently on CPU / when VRAM is unknown / when the run isn't
    statically predictable, so CI and small runs are unaffected. A quantization
    the predictor has no figure for is skipped with one line (#1652). Honors
    the documented ``--allow-oom-attempt`` opt-out.
    """
    # v0.72.0 — layer streaming bounds peak VRAM by ONE decoder layer, so the
    # resident prediction (full weights + optimizer + grads on the card) is the
    # wrong model entirely: it refuses exactly the runs streaming exists to
    # enable. The streaming path runs its own pre-flight instead (RAM-tier fit
    # + the plan panel in _setup_streaming_transformers).
    if getattr(cfg.training, "stream_layers", False):
        return
    # MLX uses Apple unified memory and its own runtime allocator, so the
    # CUDA-shaped analytical VRAM predictor is skipped.  On a non-Apple host,
    # ``backend: mlx`` still skips harmlessly: ``resolve_trainer`` fails on
    # the ``mlx_lm`` import before training starts, so there is no silent
    # hazard from bypassing the gate.
    if getattr(cfg, "backend", None) == "mlx":
        return

    total_bytes = 0
    try:
        total_bytes = int(gpu_info.get("memory_total_bytes", 0) or 0)
    except (AttributeError, TypeError, ValueError):
        return
    if total_bytes <= 0:
        return  # no CUDA VRAM to predict against
    inp = _build_hardware_fit_input(cfg)
    if inp is None:
        quantization = str(getattr(cfg.training, "quantization", "none") or "none")
        if quantization not in _HW_FIT_QUANT:
            # #1652: silence here read as "fits".
            console.print(
                "[yellow]Pre-flight skipped:[/] no memory model for "
                f"quantization: {quantization}"
            )
        return
    from soup_cli.utils.hardware_fit import VRAM_SAFETY_MARGIN, decide_hardware_fit

    report = decide_hardware_fit(inp, available_vram_gb=total_bytes / 1e9)
    if report.ok:
        return
    b = report.breakdown
    # #1631: a "quantized" run priced like an unquantized one needs a word why.
    dequantized = _HW_FIT_DEQUANTIZED_ON_LOAD.get(inp.quant)
    note = (
        f"An {dequantized} base is dequantized on load, so this estimate prices "
        "the bf16 model.\n"
        if dequantized
        else ""
    )
    tail = (
        "[yellow]--allow-oom-attempt set: launching anyway.[/]"
        if allow_oom_attempt
        else "Reduce batch_size / max_length, enable gradient_checkpointing or "
        "quantization, or pass [bold]--allow-oom-attempt[/] to try anyway."
    )
    console.print(
        Panel(
            f"Predicted peak VRAM [bold]{report.peak_vram_gb:.1f} GB[/] "
            f"(+{int(VRAM_SAFETY_MARGIN * 100)}% margin = "
            f"{report.required_with_margin_gb:.1f} GB) exceeds "
            f"{report.available_vram_gb:.1f} GB available.\n"
            f"weights {b.weights_gb:.1f} | optim {b.optimizer_gb:.1f} | "
            f"grads {b.gradients_gb:.1f} | activations {b.activations_gb:.1f} "
            f"| overhead {b.overhead_gb:.1f} GB\n" + note + "\n" + tail,
            title=(
                "[yellow]Hardware-fit warning[/]"
                if allow_oom_attempt
                else "[bold red]Hardware-fit gate[/]"
            ),
            border_style="yellow" if allow_oom_attempt else "red",
        )
    )
    if not allow_oom_attempt:
        raise typer.Exit(1)


def _loads_on_unsloth(cfg) -> bool:
    """#1736 — would this config still load with ``backend: unsloth``?

    The tip that suggests the backend used to look at the backend alone, so it
    was also printed for configs the schema refuses on unsloth (a task with no
    unsloth setup, ``freeze_layers``, ``moe_lora``...). The schema is asked
    instead of a list kept here: the dumped config is validated again with the
    backend changed. Only that pass's ``ValidationError`` means no; anything
    else is a bug and must surface.
    """
    from pydantic import ValidationError

    from soup_cli.config.schema import SoupConfig

    try:
        SoupConfig.model_validate({**cfg.model_dump(), "backend": "unsloth"})
    except ValidationError:
        return False
    return True


def _unsloth_tip_applies(cfg) -> bool:
    """Whether to suggest ``backend: unsloth``: only to a run on transformers,
    with unsloth installed, whose config would load with it (#1736). The
    installed check comes before the second validation, so a machine without
    unsloth validates nothing twice."""
    if cfg.backend != "transformers":
        return False
    from soup_cli.utils.unsloth import is_unsloth_available

    return is_unsloth_available() and _loads_on_unsloth(cfg)


def _apply_replay_overrides(cfg, *, replay, replay_ratio, replay_seed=None):
    """Apply the ``--replay*`` flags, then RE-VALIDATE.

    Re-validation is the point: a CLI override must clear the same
    cross-validators as YAML, or ``--replay`` on ``task='dpo'`` would slip
    past ``_validate_replay_compat``. Rebuilding the model (rather than
    mutating in place) is what re-runs them, and leaves the caller's config
    untouched.
    """
    if replay is None and replay_ratio is None and replay_seed is None:
        return cfg
    payload = cfg.model_dump()
    if replay is not None:
        payload["data"]["replay"] = replay
    if replay_ratio is not None:
        payload["data"]["replay_ratio"] = replay_ratio
    if replay_seed is not None:
        payload["data"]["replay_seed"] = replay_seed
    return type(cfg)(**payload)


def _describe_exception_for_tracker(exc: BaseException) -> str:
    """Format ``exc`` for ``ExperimentTracker.fail_run``'s ``error=`` column.

    A handled setup failure (a bad ``--tracker`` value, a hub download
    error, a hub-cache path-containment refusal) exits via
    ``raise typer.Exit(...)``, sometimes chained with ``from exc`` and
    sometimes not. ``typer.Exit`` carries no message of its own, so
    formatting it directly writes ``"Exit: "`` to the row and loses the
    reason a human needs. Unwrapping to ``__cause__`` recovers it for the
    chained sites; recording ``exit_code`` covers the one site that
    raises bare, so all three read as something more useful than
    ``"Exit: "`` (#764/#767 review).

    The unwrap is gated to ``typer.Exit`` specifically — this helper also
    runs on ordinary training failures (``raise X from Y`` deep inside a
    library), where the outer exception X is the operator-facing reason
    and the inner cause Y is just the mechanism. Unwrapping unconditionally
    would discard X and keep only Y, which is less informative than before
    this fix existed — the exact regression the same review caught.
    """
    if isinstance(exc, typer.Exit):
        if exc.__cause__ is not None:
            cause = exc.__cause__
            return f"{type(cause).__name__}: {cause} (exit code {exc.exit_code})"
        return f"Exit(code={exc.exit_code})"
    return f"{type(exc).__name__}: {exc}"


def train(
    config: str = typer.Option(
        "soup.yaml",
        "--config",
        "-c",
        help="Path to soup.yaml config file",
    ),
    name: str = typer.Option(
        None,
        "--name",
        "-n",
        help="Experiment name (auto-generated if not set)",
    ),
    dry_run: bool = typer.Option(
        False,
        "--dry-run",
        help="Validate config and data without training",
    ),
    resume: str = typer.Option(
        None,
        "--resume",
        "-r",
        help="Resume from checkpoint: path to checkpoint dir ('auto' for latest); "
        "on the MLX backend, a path to a .safetensors adapter file instead. "
        "Not supported when training.relora_steps is configured",
    ),
    wandb: bool = typer.Option(
        False,
        "--wandb",
        help="Enable Weights & Biases logging",
    ),
    tensorboard: bool = typer.Option(
        False,
        "--tensorboard",
        help="Enable TensorBoard logging (logs to output_dir/runs/)",
    ),
    tracker: str = typer.Option(
        None,
        "--tracker",
        help=(
            "Experiment tracker: mlflow / swanlab / trackio (v0.43.0). "
            "Mutually exclusive with --wandb / --tensorboard."
        ),
    ),
    deepspeed: str = typer.Option(
        None,
        "--deepspeed",
        help=(
            "Enable DeepSpeed: zero2, zero3, zero2_offload, zero3_offload "
            "(stage 3 + CPU parameter offload), zero++ (ZeRO++), "
            "or path to config JSON"
        ),
    ),
    fsdp: str = typer.Option(
        None,
        "--fsdp",
        help="Enable FSDP2: full_shard, shard_grad, or full_offload",
    ),
    gpus: str = typer.Option(
        None,
        "--gpus",
        help="GPUs per node for distributed training ('auto' or integer)",
    ),
    nodes: Annotated[int, typer.Option(
        "--nodes", help="Number of machines; each must use the same --gpus count",
    )] = 1,
    node_rank: Annotated[int | None, typer.Option(
        "--node-rank", help="This machine's rank, from 0 to nodes minus 1 (default 0)",
    )] = None,
    master_addr: Annotated[str | None, typer.Option(
        "--master-addr",
        help="Rank 0 hostname or IP reachable by all nodes; required with --nodes > 1",
    )] = None,
    master_port: Annotated[int | None, typer.Option(
        "--master-port", help="Shared coordinator port for multiple nodes (default 29500)",
    )] = None,
    no_reexec: bool = typer.Option(
        False,
        "--no-reexec",
        help=(
            "Print the distributed launch command instead of launching it"
        ),
    ),
    gate: str = typer.Option(
        None,
        "--gate",
        help=(
            "Enable eval-gated training with a suite file "
            "(shortcut for training.eval_gate.enabled=true + suite=<path>)"
        ),
    ),
    push_as: str = typer.Option(
        None,
        "--push-as",
        help=(
            "Auto-push each save_steps checkpoint to HF Hub as "
            "'checkpoint-<step>' branch of the given repo (e.g. user/my-model)"
        ),
    ),
    hf_resume: bool = typer.Option(
        False,
        "--hf-resume",
        help=(
            "Download the latest checkpoint branch from the --push-as repo "
            "and resume from it. Requires --push-as; not supported when "
            "training.relora_steps is configured."
        ),
    ),
    find_lr: bool = typer.Option(
        False,
        "--find-lr",
        help=(
            "LR range finder (v0.32.0): run a short geometric LR sweep, write "
            "a JSON report with the recommended LR, then exit without training."
        ),
    ),
    find_lr_start: float = typer.Option(
        1e-7,
        "--find-lr-start",
        help="LR range finder: starting LR (default 1e-7)",
    ),
    find_lr_end: float = typer.Option(
        1e-1,
        "--find-lr-end",
        help="LR range finder: ending LR (default 1e-1)",
    ),
    find_lr_steps: int = typer.Option(
        100,
        "--find-lr-steps",
        help="LR range finder: number of sweep steps (default 100)",
    ),
    find_lr_output: str = typer.Option(
        "lr_finder.json",
        "--find-lr-output",
        help="LR range finder: JSON report path (default ./lr_finder.json)",
    ),
    yes: bool = typer.Option(
        False,
        "--yes",
        "-y",
        help=(
            "Skip the start-training confirmation. Required when stdin is "
            "not a TTY (scripts, CI, pipes)."
        ),
    ),
    trust_remote_code: bool = typer.Option(
        False,
        "--trust-remote-code",
        help=(
            "Allow loading models that ship custom Python via auto_map. "
            "Default deny (v0.36.0). Only enable if you trust the source."
        ),
    ),
    echo_trap_tokenizer_aware: bool = typer.Option(
        False,
        "--echo-trap-tokenizer-aware",
        help=(
            "Use tokenizer-id n-grams for echo-trap scoring. Requires "
            "training.echo_trap_enabled=true on grpo/ppo."
        ),
    ),
    reward_hack_detector: str = typer.Option(
        None,
        "--reward-hack-detector",
        help=(
            "Reward-hacking detector for GRPO/PPO: info_rm | rm_ensemble. "
            "Overrides training.reward_hack_detector. (v0.71.26)"
        ),
    ),
    reward_hack_halt: bool = typer.Option(
        False,
        "--reward-hack-halt",
        help=(
            "Auto-halt training on a HACK verdict. Requires "
            "--reward-hack-detector (or training.reward_hack_detector). (v0.71.26)"
        ),
    ),
    replay: str = typer.Option(
        None, "--replay",
        help=(
            "Old dataset to interleave as continual-learning rehearsal, so "
            "training on the new task does not erase the previous one. "
            "sft/pretrain only; incompatible with packing/multipack. "
            "Overrides data.replay. (v0.71.36)"
        ),
    ),
    replay_ratio: float = typer.Option(
        None, "--replay-ratio",
        help=(
            "Fraction of the FINAL mixed train set that is replay rows "
            "(default 0.1). Overrides data.replay_ratio. (v0.71.36)"
        ),
    ),
    replay_seed: int = typer.Option(
        None, "--replay-seed",
        help=(
            "Seed for the replay sample + interleave. Overrides "
            "data.replay_seed. (v0.71.36)"
        ),
    ),
    reward_hack_mitigation: str = typer.Option(
        None,
        "--reward-hack-mitigation",
        help=(
            "Closed-loop reward-hacking mitigation mode: off | log_only | "
            "kl_control | pid_lagrangian. Requires training.reward_hack_detector "
            "on grpo/ppo. Overrides training.reward_hack_mitigation. (v0.71.26)"
        ),
    ),
    minillm_on_policy: bool = typer.Option(
        False,
        "--minillm-on-policy",
        help=(
            "Use the TRUE on-policy MiniLLM teacher-mixed rollout (v0.71.18 "
            "#257) instead of the offline distribution blend. Requires "
            "training.minillm_enabled=true on task='distill'."
        ),
    ),
    profile_run: bool = typer.Option(
        False,
        "--profile",
        help=(
            "Record a torch.profiler trace (Chrome trace JSON) during early "
            "training steps. Output: <output>/profiles/<run_id>.trace.json"
        ),
    ),
    allow_oom_attempt: bool = typer.Option(
        False,
        "--allow-oom-attempt",
        help=(
            "Bypass the analytical hardware-fit VRAM gate and launch even when "
            "the run is predicted to run out of GPU memory (opt-out)."
        ),
    ),
    diagnose_gate: str = typer.Option(
        None,
        "--diagnose-gate",
        help=(
            "After training, run `soup diagnose` against the supplied evidence "
            "JSON (or scratch evidence). Refuses to mark the run successful "
            "if any failure mode returns MAJOR (exit 2) or NOT_RUN (exit 3)."
        ),
    ),
    annex_xi: str = typer.Option(
        None,
        "--annex-xi",
        help=(
            "After training, render an EU AI Act Annex XI/XII auto-doc to the "
            "given output path (cwd-contained). Markdown body now; PDF in v0.59.1."
        ),
    ),
    repro_receipt: str = typer.Option(
        None,
        "--repro-receipt",
        help=(
            "After training, write an SR 11-7-style reproducibility receipt "
            "(seeds + kernel versions + GPU + OS) to the given path. v0.59.0."
        ),
    ),
    capture_activations: str = typer.Option(
        None,
        "--capture-activations",
        help=(
            "After training, capture residual-stream activations at the named "
            "decoder layer (e.g. model.layers.5) on --capture-prompts and write "
            "them to <output>/activations/activations.json for soup probe "
            "sae-diff / sleeper. v0.71.8 #219."
        ),
    ),
    capture_prompts: str = typer.Option(
        None,
        "--capture-prompts",
        help=(
            "JSONL (or .txt) of prompts to run for --capture-activations "
            "(one prompt per line; 'prompt'/'text' field or raw text)."
        ),
    ),
    track_energy: bool = typer.Option(
        False,
        "--track-energy",
        help=(
            "Measure the training window's energy + CO2 via codecarbon "
            "(offline; requires `pip install soup-cli\\[carbon]`). Feeds the "
            "kWh / CO2 into --annex-xi. v0.71.3."
        ),
    ),
    energy_country: str = typer.Option(
        "USA",
        "--energy-country",
        help=(
            "ISO 3166-1 alpha-3 country code for the CO2 grid-intensity "
            "estimate used by --track-energy (default USA)."
        ),
    ),
    energy_out: str = typer.Option(
        None,
        "--energy-out",
        help=(
            "Write the --track-energy measurement to this JSON file (cwd-"
            "contained) so `soup bom emit --energy <path>` can consume it. "
            "v0.71.15."
        ),
    ),
    cloud: str = typer.Option(
        None,
        "--cloud",
        help=(
            "Train on a cloud GPU instead of locally (v0.71.18 #16). Supported: "
            "modal, lambda, runpod. Renders a cloud app stub from the config "
            "(plan-only); use --cloud-submit to submit live."
        ),
    ),
    gpu: str = typer.Option(
        "a100",
        "--gpu",
        help=(
            "Cloud GPU type for --cloud (t4 / l4 / a10 / a10g / a100 / a100-80gb / "
            "l40s / h100 / a6000). Default a100. Provider-specific allowlists apply."
        ),
    ),
    cloud_submit: bool = typer.Option(
        False,
        "--cloud-submit",
        help=(
            "With --cloud, submit the rendered run live via the cloud's SDK or API "
            "(gated on respective provider token/API key). Default is plan-only "
            "(render + print the command). Lambda requires a registered SSH key."
        ),
    ),
):
    """Start training from a soup.yaml config."""
    from soup_cli.utils.launcher import validate_multi_node_options

    try:
        validate_multi_node_options(nodes, node_rank, master_addr, master_port)
        if nodes > 1:
            if not gpus:
                raise ValueError("--nodes > 1 requires --gpus (GPUs per node)")
            if cloud or find_lr:
                raise ValueError("--nodes > 1 cannot be combined with --cloud or --find-lr")
    except ValueError as exc:
        message = str(exc)
        for field, flag in (
            ("num_machines", "--nodes"), ("machine_rank", "--node-rank"),
            ("main_process_ip", "--master-addr"), ("main_process_port", "--master-port"),
        ):
            message = message.replace(field, flag)
        console.print(f"[red]Invalid distributed launch:[/] {markup_escape(message)}")
        raise typer.Exit(1) from exc

    from soup_cli.utils.terminal import for_terminal

    config_path = Path(config)
    if not config_path.exists():
        console.print(f"[red]Config not found: {for_terminal(config_path)}[/]")
        console.print("Run [bold]soup init[/] to create one.")
        raise typer.Exit(1)

    # --- LR range finder fast path ---
    if find_lr:
        from soup_cli.utils.lr_finder import (
            LrSweepUnavailableError,
            SweepTooShortError,
            compute_lr_schedule,
            save_lr_finder_report,
        )

        try:
            schedule = compute_lr_schedule(
                start_lr=find_lr_start,
                end_lr=find_lr_end,
                num_steps=find_lr_steps,
            )
        except ValueError as exc:
            console.print(f"[red]Invalid --find-lr range:[/] {exc}")
            raise typer.Exit(1) from exc
        # v0.33.0 #56: live LR-sweep training loop. #1203: when it cannot run, the
        # command refuses rather than writing a curve nobody measured.
        # #1189: the sweep returns the LRs it actually ran, which can be fewer than
        # --find-lr-steps (a short dataset, or a loss that went non-finite).
        try:
            lrs, losses_for_report = _run_live_lr_sweep(
                config_path, schedule, trust_remote_code=trust_remote_code,
            )
        except (SweepTooShortError, LrSweepUnavailableError) as exc:
            # The refusal can quote the config path and text from the file.
            console.print(f"[red]{for_terminal(str(exc))}[/]")
            raise typer.Exit(1) from exc
        try:
            save_lr_finder_report(lrs, losses_for_report, find_lr_output)
        except ValueError as exc:
            console.print(f"[red]Invalid --find-lr-output:[/] {exc}")
            raise typer.Exit(1) from exc
        console.print(f"[green]LR finder report written to:[/] {find_lr_output}")
        raise typer.Exit(0)

    # Load & validate config
    console.print(f"[dim]Loading config from {for_terminal(config_path)}...[/]")
    cfg = load_config(
        config_path,
        training_overrides=(
            {"minillm_on_policy": True} if minillm_on_policy else None
        ),
    )

    # --- v0.71.36 replay passthrough ---
    try:
        cfg = _apply_replay_overrides(
            cfg,
            replay=replay,
            replay_ratio=replay_ratio,
            replay_seed=replay_seed,
        )
    except Exception as exc:  # noqa: BLE001 — pydantic ValidationError et al.
        console.print(f"[red]{markup_escape(str(exc))}[/]")
        raise typer.Exit(code=2) from exc

    if cfg.training.relora_steps is not None and (resume is not None or hf_resume):
        console.print(
            "[red]ReLoRA runs cannot resume from checkpoints.[/] "
            "Remove training.relora_steps or start a fresh run."
        )
        raise typer.Exit(1)

    # An unregistered data.chat_template name raises KeyError in the trainer,
    # after the model has loaded. Check it before anything is downloaded.
    from soup_cli.data.chat_templates import resolve_chat_template

    try:
        resolve_chat_template(cfg.data.chat_template)
    except KeyError as exc:
        console.print(f"[red]Invalid data.chat_template:[/] {for_terminal(exc.args[0])}")
        raise typer.Exit(1) from exc

    # v0.72.3 — --resume / --hf-resume now work with layer streaming. v0.72.0-.2
    # refused them because a streamed model's `named_parameters()` carry an
    # `.inner.` segment that `load_state_dict` narrows away, so PEFT matched
    # NOTHING and silently continued with a freshly initialised adapter (measured:
    # 0 of 12 tensors, and a resumed loss curve byte-identical to a from-scratch
    # one). `StreamedDecoderLayer` now redirects canonical keys at load time,
    # mirroring the v0.72.1 save-side delegation.

    # --- RA-DIT generator-stage auto-link (v0.71.10 #200) ---
    # When a generator stage has no retriever model set, splice in the latest
    # RA-DIT retriever output from the Registry. A manual value always wins.
    if getattr(cfg.training, "ra_dit_stage", None) == "generator":
        from soup_cli.utils.ra_dit_run import autolink_generator_retriever

        advisory = autolink_generator_retriever(cfg)
        if advisory:
            # `advisory` embeds a Registry-derived `output` path — escape it
            # before printing into the Rich-markup console (security MEDIUM).
            console.print(f"[yellow]RA-DIT:[/] {markup_escape(advisory)}")

    # --- Echo-trap tokenizer-aware shortcut ---
    if echo_trap_tokenizer_aware:
        if not cfg.training.echo_trap_enabled:
            console.print(
                "[red]--echo-trap-tokenizer-aware requires "
                "training.echo_trap_enabled=true[/]"
            )
            raise typer.Exit(1)
        cfg.training.echo_trap_tokenizer_aware = True
        console.print("[green]Echo-trap tokenizer-aware scoring enabled[/]")

    # --- Reward-hack detector / halt shortcut (v0.71.26) ---
    if reward_hack_detector is not None:
        if reward_hack_detector not in ("info_rm", "rm_ensemble"):
            console.print(
                "[red]--reward-hack-detector must be info_rm or rm_ensemble[/]"
            )
            raise typer.Exit(1)
        cfg.training.reward_hack_detector = reward_hack_detector
        console.print(f"[green]Reward-hack detector:[/] {reward_hack_detector}")
    if reward_hack_halt:
        if cfg.training.reward_hack_detector is None:
            console.print(
                "[red]--reward-hack-halt requires --reward-hack-detector "
                "(or training.reward_hack_detector)[/]"
            )
            raise typer.Exit(1)
        cfg.training.reward_hack_halt = True
        console.print("[green]Reward-hack auto-halt enabled[/]")

    # --- Reward-hack mitigation shortcut (v0.71.26) ---
    if reward_hack_mitigation is not None:
        valid_modes = ("off", "log_only", "kl_control", "pid_lagrangian")
        if reward_hack_mitigation not in valid_modes:
            console.print(
                "[red]--reward-hack-mitigation must be one of "
                f"{', '.join(valid_modes)}[/]"
            )
            raise typer.Exit(1)
        if (
            reward_hack_mitigation != "off"
            and cfg.training.reward_hack_detector is None
        ):
            console.print(
                "[red]--reward-hack-mitigation requires "
                "training.reward_hack_detector to be set (the signal source)[/]"
            )
            raise typer.Exit(1)
        cfg.training.reward_hack_mitigation = reward_hack_mitigation
        console.print(
            f"[green]Reward-hack mitigation:[/] {reward_hack_mitigation}"
        )

    # --- MiniLLM on-policy rollout shortcut (v0.71.18 #257) ---
    # Applied before SoupConfig validation via load_config(training_overrides=),
    # so --minillm-on-policy can still select student-only sampling when the
    # YAML leaves mix at 0 (#692 / #977). The assignment below is therefore a
    # no-op when the flag was set; it stays so a later reader sees the flag
    # take effect on cfg.training.
    if minillm_on_policy:
        if not cfg.training.minillm_enabled:
            console.print(
                "[red]--minillm-on-policy requires "
                "training.minillm_enabled=true (task='distill')[/]"
            )
            raise typer.Exit(1)
        cfg.training.minillm_on_policy = True
        console.print("[green]MiniLLM on-policy rollout enabled[/]")

    # --- Cloud GPU training (v0.71.18 #16, v0.71.22 #264) ---
    if cloud:
        from soup_cli import __version__ as _soup_version

        cloud = cloud.lower()
        if cloud == "runpod":
            from soup_cli.cloud import runpod as cloud_mod
        elif cloud == "modal":
            from soup_cli.cloud import modal as cloud_mod
        elif cloud == "lambda":
            from soup_cli.cloud import lambda_labs as cloud_mod
        else:
            console.print(
                f"[red]Invalid --cloud:[/] {markup_escape(cloud)}. "
                "Supported: modal, lambda, runpod."
            )
            raise typer.Exit(2)

        try:
            cloud_mod.validate_cloud(cloud)
            cloud_mod.validate_gpu(gpu)
        except ValueError as exc:
            console.print(
                f"[red]Invalid --cloud / --gpu:[/] {markup_escape(str(exc))}"
            )
            raise typer.Exit(2) from exc
        try:
            # We call the generic-shaped plan function dynamically
            plan_func = getattr(cloud_mod, f"plan_{cloud}_run", None)
            if plan_func is None:
                raise ValueError(f"cloud backend {cloud!r} has no plan function")
            plan = plan_func(
                str(config_path),
                gpu=gpu,
                output_dir=cfg.output,
                soup_version=_soup_version,
            )
            stub_realpath = cloud_mod.write_stub(plan)
        except (ValueError, TypeError) as exc:
            console.print(f"[red]Cloud plan failed:[/] {markup_escape(str(exc))}")
            raise typer.Exit(2) from exc

        console.print(
            Panel(
                f"Cloud:    [bold]{markup_escape(cloud)}[/]\n"
                f"GPU:      [bold]{markup_escape(plan.gpu)}[/]\n"
                f"Stub:     [bold]{markup_escape(os.path.relpath(stub_realpath))}[/]\n"
                f"Output:   [bold]{markup_escape(plan.output_dir)}[/]\n\n"
                f"[bold]Run:[/] {markup_escape(plan.run_command)}",
                title=f"[bold green]soup train --cloud {markup_escape(cloud)}[/]",
            )
        )
        if cloud_submit:
            try:
                submit_func = getattr(cloud_mod, f"submit_{cloud}_run", None)
                if submit_func is None:
                    raise RuntimeError(f"cloud backend {cloud!r} has no submit function")
                rc = submit_func(plan)
            except RuntimeError as exc:
                console.print(
                    f"[yellow]{cloud.title()} submit unavailable:[/] "
                    f"{markup_escape(str(exc))}"
                )
                raise typer.Exit(1) from exc
            raise typer.Exit(rc)
        console.print(
            f"[yellow]Note:[/] plan-only. Authenticate with {cloud}, then run the "
            "command above (or re-run with --cloud-submit)."
        )
        raise typer.Exit(0)

    # --- --push-as / --hf-resume validation ---
    if push_as:
        from soup_cli.utils.hf import validate_repo_id

        try:
            validate_repo_id(push_as)
        except ValueError as exc:
            console.print(f"[red]Invalid --push-as repo id:[/] {exc}")
            raise typer.Exit(1) from exc
    if hf_resume and not push_as:
        console.print("[red]--hf-resume requires --push-as <repo>[/]")
        raise typer.Exit(1)
    if hf_resume and cfg.task in UNSUPPORTED_RESUME_TASKS:
        console.print(f"[red]--hf-resume is not supported for task {cfg.task!r}[/]")
        raise typer.Exit(1)

    # --- Eval-gate shortcut: --gate <path> sets training.eval_gate ---
    if gate:
        from soup_cli.config.schema import EvalGateConfig
        from soup_cli.eval.gate import load_suite

        try:
            # Validate the suite path up-front (path containment + parse).
            load_suite(gate)
        except (FileNotFoundError, ValueError) as exc:
            console.print(f"[red]Invalid --gate suite: {for_terminal(str(exc))}[/]")
            raise typer.Exit(1) from exc
        cfg.training.eval_gate = EvalGateConfig(enabled=True, suite=gate)
        console.print(f"[green]Eval gate enabled[/] with suite: {gate}")

    # Honesty guard: these staged flags (plus forgetting_threshold) are accepted
    # but not enforced mid-training in this build. Their other tuning knobs are
    # reported by the loader with refusal dates, so only active flags and
    # non-default forgetting_threshold are reported here.
    _unwired_gates = _nondefault_unwired_training_settings(cfg.training)
    if _unwired_gates:
        console.print(
            "[yellow]Note:[/] "
            + ", ".join(_unwired_gates)
            + " are set but not enforced during training in this build. "
            "Use [bold]--gate <suite.yaml>[/] for a live eval gate, or run "
            "[bold]soup eval[/] / [bold]soup diagnose[/] after training."
        )

    # --- Resolve resume checkpoint (fail fast before heavy operations) ---
    resume_from = _resolve_resume_or_exit(resume, cfg)

    # --- HF auto-resume: pull latest checkpoint branch into output dir ---
    if hf_resume and push_as and resume_from is None:
        from soup_cli.monitoring.hf_push import prepare_hf_resume
        from soup_cli.utils.hf import resolve_endpoint, resolve_token

        try:
            hf_endpoint = resolve_endpoint()
        except ValueError as exc:
            console.print(f"[red]--hf-resume: {exc}[/]")
            raise typer.Exit(1) from exc
        hf_token = resolve_token()
        if hf_token is None:
            console.print(
                "[yellow]--hf-resume: no HF token available; skipping auto-resume[/]"
            )
        else:
            local_ckpt = prepare_hf_resume(
                repo_id=push_as,
                output_dir=cfg.output,
                token=hf_token,
                endpoint=hf_endpoint,
            )
            if local_ckpt:
                resume_from = local_ckpt
                console.print(f"[green]Resumed from HF:[/] {local_ckpt}")
            else:
                console.print(
                    "[yellow]--hf-resume: no checkpoint branch found; starting fresh[/]"
                )

    # --- Validate logging flags ---
    if wandb and tensorboard:
        console.print(
            "[red]Cannot use --wandb and --tensorboard together. Pick one.[/]"
        )
        raise typer.Exit(1)

    # --- TensorBoard setup ---
    if tensorboard:
        try:
            import tensorboard  # noqa: F401

            console.print("[green]TensorBoard logging enabled[/]")
        except ImportError:
            console.print(
                "[red]TensorBoard not installed.[/]\n"
                "Run: [bold]pip install tensorboard[/]"
            )
            raise typer.Exit(1)

    # --- W&B setup (fail fast if wandb not installed) ---
    if wandb:
        try:
            import wandb as _wandb  # noqa: F401

            console.print("[green]W&B logging enabled[/]")
        except ImportError:
            console.print(
                "[red]wandb not installed.[/]\n"
                "Run: [bold]pip install \"soup-cli\\[wandb]\"[/]"
            )
            raise typer.Exit(1)
        except Exception as wandb_err:
            console.print(
                f"[red]wandb import error:[/] {wandb_err}\n"
                "Try: [bold]pip install 'wandb>=0.15.0,<0.18.0'[/]"
            )
            raise typer.Exit(1)

    # --- DeepSpeed setup ---
    ds_config_path = None
    if deepspeed:
        ds_config_path = _resolve_deepspeed(deepspeed)
        if ds_config_path:
            console.print(f"[green]DeepSpeed enabled:[/] {deepspeed}")

    # --- FSDP2 setup ---
    fsdp_kwargs = None
    if fsdp:
        from soup_cli.utils.fsdp import FSDP_CONFIGS, get_fsdp_training_args

        if fsdp not in FSDP_CONFIGS:
            console.print(
                f"[red]Invalid FSDP preset: {fsdp}[/]\n"
                f"Options: {', '.join(FSDP_CONFIGS.keys())}"
            )
            raise typer.Exit(1)
        fsdp_kwargs = get_fsdp_training_args(fsdp)
        console.print(f"[green]FSDP2 enabled:[/] {fsdp}")

    # #350 — BNB's default uint8 quant storage is not merely slow under FSDP:
    # FSDP cannot flatten it. Resolve storage to the exact BNB compute dtype
    # before any trainer builds its BitsAndBytesConfig. This updates the
    # effective config shared by every wrapper and the reproducibility receipt.
    from soup_cli.utils.quant_menu import resolve_fsdp_qlora_quant_storage

    original_quant_storage = cfg.training.bnb_4bit_quant_storage
    resolved_training = resolve_fsdp_qlora_quant_storage(
        cfg.training,
        fsdp=bool(fsdp),
    )
    if resolved_training is not cfg.training:
        cfg = cfg.model_copy(update={"training": resolved_training})
        action = "selected" if original_quant_storage is None else "overrode"
        console.print(
            f"[green]FSDP QLoRA:[/] {action} bnb_4bit_quant_storage="
            f"{resolved_training.bnb_4bit_quant_storage} to match compute dtype"
        )

    # --- v0.38.0 Quant Menu × multi-GPU compatibility check ---
    from soup_cli.utils.quant_menu import check_quant_distributed_compat

    quant_problems = check_quant_distributed_compat(
        quantization=cfg.training.quantization,
        deepspeed=deepspeed,
        fsdp=bool(fsdp),
        bnb_4bit_quant_storage=cfg.training.bnb_4bit_quant_storage,
    )
    if quant_problems:
        hard = [p for p in quant_problems if not p.lower().startswith("warning")]
        warn = [p for p in quant_problems if p.lower().startswith("warning")]
        for problem in hard:
            console.print(f"[red]Quant compat:[/] {problem}")
        for problem in warn:
            console.print(f"[yellow]{problem}[/]")
        if hard:
            raise typer.Exit(1)

    # --- Multi-GPU topology + --gpus resolution ---
    num_gpus = None
    if gpus:
        from soup_cli.utils.topology import detect_topology, resolve_num_gpus

        try:
            num_gpus = resolve_num_gpus(gpus)
        except ValueError as exc:
            console.print(f"[red]Invalid --gpus:[/] {exc}")
            raise typer.Exit(1) from exc
        topo = detect_topology()
        if num_gpus is not None and num_gpus < 1:
            if nodes > 1:
                console.print("[red]Multi-node training requires at least one GPU per node.[/]")
                raise typer.Exit(1)
            # --gpus auto on CPU / no-CUDA box — explicit, not silent.
            console.print(
                "[yellow]--gpus auto detected 0 GPUs; continuing as a "
                "single-process CPU run.[/]"
            )
        elif num_gpus is not None and num_gpus * nodes > 1:
            from soup_cli.utils.launcher import (
                build_accelerate_argv,
                build_train_reexec_argv,
                collect_reexec_passthrough,
                format_advice,
                hint_argv_from_reexec,
                is_in_distributed,
                run_launcher,
            )

            num_processes = num_gpus * nodes
            if not is_in_distributed():
                # v0.33.0 #37 — auto-reexec under accelerate launch unless
                # --no-reexec was passed. On POSIX the accelerate process
                # replaces this one (os.execvp): no leftover PID tree, stdio
                # passes through unchanged. Windows cannot do that, so there
                # this process waits for the launcher (see run_launcher).
                # #372 — one argv builder for both the re-exec and the printed
                # hint, so they cannot drift. collect_reexec_passthrough is the
                # only list of "flags the user typed" that survive a launch.
                script_args = build_train_reexec_argv(
                    config,
                    collect_reexec_passthrough(
                        name=name,
                        fsdp=fsdp,
                        deepspeed=deepspeed,
                        resume=resume,
                        wandb=wandb,
                        tensorboard=tensorboard,
                        echo_trap_tokenizer_aware=echo_trap_tokenizer_aware,
                        reward_hack_detector=reward_hack_detector,
                        reward_hack_halt=reward_hack_halt,
                        reward_hack_mitigation=reward_hack_mitigation,
                        gate=gate,
                        push_as=push_as,
                        hf_resume=hf_resume,
                        trust_remote_code=trust_remote_code,
                        tracker=tracker,
                        diagnose_gate=diagnose_gate,
                        annex_xi=annex_xi,
                        repro_receipt=repro_receipt,
                        profile_run=profile_run,
                        allow_oom_attempt=allow_oom_attempt,
                        track_energy=track_energy,
                        energy_country=energy_country,
                        energy_out=energy_out,
                        yes=yes,
                        minillm_on_policy=minillm_on_policy,
                        capture_activations=capture_activations,
                        capture_prompts=capture_prompts,
                        replay=replay,
                        replay_ratio=replay_ratio,
                        replay_seed=replay_seed,
                    ),
                )
                if nodes > 1:
                    console.print(
                        "[dim]Multi-node NCCL default: NCCL_IB_DISABLE=0. "
                        "Set NCCL_IB_DISABLE=1 for TCP-only networks. "
                        "Existing environment settings are preserved.[/]"
                    )
                if (dry_run and nodes > 1) or (no_reexec and not dry_run):
                    # Multi-node advice is the executable module-form argv,
                    # exactly as passed to execvp, including the child guard.
                    hint_args = script_args if nodes > 1 else hint_argv_from_reexec(script_args)
                    console.print(
                        Panel(
                            markup_escape(format_advice(
                                num_processes, hint_args, num_machines=nodes,
                                machine_rank=node_rank, main_process_ip=master_addr,
                                main_process_port=master_port,
                            )),
                            title="[yellow]Multi-GPU launch required[/]",
                        )
                    )
                    console.print(
                        f"[dim]Detected topology: {topo['gpu_count']} GPUs, "
                        f"{topo['interconnect']}[/]"
                    )
                    if not dry_run:
                        raise typer.Exit(1)

                if dry_run:
                    console.print(
                        f"[dim]--dry-run: skipping accelerate re-exec "
                        f"({num_processes} GPUs, {topo['interconnect']}).[/]"
                    )
                else:
                    argv = build_accelerate_argv(
                        num_processes=num_processes, script_args=script_args,
                        num_machines=nodes, machine_rank=node_rank,
                        main_process_ip=master_addr, main_process_port=master_port,
                    )
                    if nodes > 1:
                        from soup_cli.utils.topology import suggest_nccl_env

                        for key, val in suggest_nccl_env(
                            gpu_count=num_gpus, interconnect=topo["interconnect"],
                            num_machines=nodes,
                        ).items():
                            os.environ.setdefault(key, val)
                    console.print(
                        f"[green]Auto-reexec under accelerate "
                        f"({num_processes} GPUs, {topo['interconnect']})[/]"
                    )
                    console.print(f"[dim]argv: {markup_escape(' '.join(argv))}[/]")
                    # The launcher takes over; the child carries --no-reexec. On
                    # POSIX run_launcher execs and never returns. On Windows,
                    # where exec would return 0 at once and leave the launcher
                    # running, it waits and returns the launcher's exit code.
                    try:
                        launcher_code = run_launcher(argv)
                    except OSError as exc:
                        console.print(
                            f"[red]accelerate launch failed:[/] {markup_escape(str(exc))}\n"
                            "Use [bold]--no-reexec[/] to print the launch command."
                        )
                        raise typer.Exit(1) from exc
                    raise typer.Exit(launcher_code)
            elif not dry_run:
                # Already a launched rank — announce + apply NCCL hints. (The
                # dry_run branch above intentionally does neither.)
                console.print(
                    f"[green]Distributed run detected[/] "
                    f"({num_processes} procs, {topo['interconnect']} interconnect)"
                )
                # Apply NCCL env hints. All current keys (``NCCL_P2P_DISABLE`` /
                # ``NCCL_IB_DISABLE`` / ``NCCL_NVLS_ENABLE``) are rank-idempotent
                # string literals so it is safe to run on every rank. If a
                # rank-sensitive key is ever added to ``suggest_nccl_env``, this
                # loop must be gated to ``LOCAL_RANK == 0``. ``setdefault`` keeps
                # user / launcher overrides winning over our suggestions.
                from soup_cli.utils.topology import suggest_nccl_env

                for key, val in suggest_nccl_env(
                    gpu_count=num_gpus, interconnect=topo["interconnect"], num_machines=nodes,
                ).items():
                    os.environ.setdefault(key, val)

    # Before detect_device, which creates the CUDA context the allocator reads it at (#2149).
    configure_cuda_alloc_hint(cfg.training.stream_layers)

    # Detect hardware with backend awareness
    device, device_name = detect_device(backend=cfg.backend)
    gpu_info = get_gpu_info(backend=cfg.backend)

    configure_mps_async_load(device)

    # Quantization guard: explicit decision per #423.  See resolve_quantization()
    # docstring for the full rationale — MLX 4-bit uses pre-quantized mlx-community
    # weights (not bitsandbytes NF4), CPU cannot run bitsandbytes at all.
    resolved_quant, quant_warning = resolve_quantization(
        device=device,
        backend=cfg.backend,
        quantization=cfg.training.quantization,
    )
    if quant_warning:
        console.print(f"[yellow]{quant_warning}[/]")
    cfg.training.quantization = resolved_quant

    # Hardware-fit preflight: refuse (unless --allow-oom-attempt) when the
    # analytical VRAM predictor says the run won't fit. Skips silently on CPU
    # or when the config isn't statically predictable (e.g. batch_size='auto');
    # says so when the quantization has no memory model (hqq:*).
    _hardware_fit_preflight(cfg, gpu_info, allow_oom_attempt=allow_oom_attempt)

    backend_label = cfg.backend
    if cfg.backend == "unsloth":
        backend_label = "unsloth [green](fast mode)[/]"

    quant_label = cfg.training.quantization
    if cfg.training.quantization_aware == "quest":
        quant_label = "mixed W4/A4+A16 (QuEST fake quant)"
    elif cfg.training.quantization_aware:
        quant_label += " + QAT"

    # v0.53.2 review-fix: classifier-family tasks train a sequence-classification
    # head, not a causal-LM LoRA — render "head" instead of LoRA r/alpha.
    if cfg.task in CLASSIFICATION_TASKS:
        # v0.71.12 #146 — render BOTH the head line AND a LoRA line when the
        # opt-in classifier LoRA path is active.
        head_line = (
            f"Head:    [bold]num_labels={cfg.training.num_labels}, "
            f"kind={cfg.training.classifier_kind}[/]"
        )
        if getattr(cfg.training, "classifier_lora", False) and cfg.training.lora.r > 0:
            peft_line = (
                head_line
                + f"\nLoRA:    [bold]r={cfg.training.lora.r}, "
                + f"alpha={cfg.training.lora.alpha} (SEQ_CLS)[/]"
            )
        else:
            peft_line = head_line
    else:
        peft_line = (
            f"LoRA:    [bold]r={cfg.training.lora.r}, "
            f"alpha={cfg.training.lora.alpha}[/]"
        )
    # #353 review-fix: a wrong seed is invisible, which is how `training.seed`
    # reaching one wrapper out of nineteen survived releases. Report the value
    # the run actually trains at, and say when it is the unset default, so
    # "did my seed take?" is answered by looking rather than by reading source.
    from soup_cli.utils.seeding import resolve_training_seed

    seed_label = str(resolve_training_seed(cfg.training))
    if cfg.training.seed is None:
        seed_label += " [dim](unset default)[/]"
    if cfg.training.data_seed is not None:
        seed_label += f", data_seed={cfg.training.data_seed}"

    console.print(
        Panel(
            f"Device:  [bold]{device_name}[/]\n"
            f"Memory:  [bold]{gpu_info['memory_total']}[/]\n"
            f"Model:   [bold]{cfg.base}[/]\n"
            f"Task:    [bold]{cfg.task}[/]\n"
            f"Backend: [bold]{backend_label}[/]\n"
            f"{peft_line}\n"
            f"Quant:   [bold]{quant_label}[/]\n"
            f"Seed:    [bold]{seed_label}[/]",
            title="Training Setup",
        )
    )

    # Validate GaLore configuration
    if cfg.training.use_galore:
        from soup_cli.utils.galore import validate_galore_config

        galore_errors = validate_galore_config(
            cfg.training.use_galore, cfg.training.quantization, cfg.backend,
        )
        for err in galore_errors:
            console.print(f"[red]GaLore error:[/] {for_terminal(err)}")
        if galore_errors:
            raise typer.Exit(1)

    # Validate QAT configuration
    if (
        cfg.training.quantization_aware is True
        or cfg.training.quantization_aware == "fp8"
    ):
        from soup_cli.utils.qat import validate_qat_config

        qat_errors = validate_qat_config(
            cfg.training.quantization, cfg.backend, cfg.modality,
            quantization_aware=cfg.training.quantization_aware,
            fp8_recipe=cfg.training.fp8_recipe,
            check_card=not dry_run,
        )
        for err in qat_errors:
            console.print(f"[red]QAT error:[/] {for_terminal(err)}")
        if dry_run and cfg.training.quantization_aware == "fp8" and cfg.backend != "unsloth":
            # #1154 review: a dry run validates the config, and FP8 configs are
            # routinely written on a laptop for a remote card, so the local card
            # is a note here -- printed before any error exit, so a dry run that
            # also lacks torchao still shows the whole picture. The real run
            # still stops on the card. (Unsloth: its refusal is the answer.)
            from soup_cli.utils.fp8 import fp8_training_supported

            card_ok, card_reason = fp8_training_supported(cfg.training.fp8_recipe)
            if not card_ok:
                console.print(
                    "[yellow]Note:[/] this machine could not run it: "
                    f"{markup_escape(card_reason)}"
                )
        if qat_errors:
            raise typer.Exit(1)

    # Validate FSDP configuration
    if fsdp:
        from soup_cli.utils.fsdp import validate_fsdp_config

        fsdp_errors = validate_fsdp_config(
            fsdp_preset=fsdp,
            deepspeed_config=ds_config_path,
            backend=cfg.backend,
            device=device,
        )
        for err in fsdp_errors:
            console.print(f"[red]FSDP error:[/] {for_terminal(err)}")
        if fsdp_errors:
            raise typer.Exit(1)

    # Validate FSDP2 + torch.compile (v0.27.0 Part D)
    if cfg.training.use_fsdp2_compile:
        from soup_cli.utils.fsdp import validate_fsdp2_compile_config

        compile_errors = validate_fsdp2_compile_config(
            use_compile=cfg.training.use_fsdp2_compile,
            fsdp_preset=fsdp,
            backend=cfg.backend,
            device=device,
            deepspeed_config=ds_config_path,
        )
        for err in compile_errors:
            console.print(f"[red]FSDP2 + torch.compile error:[/] {for_terminal(err)}")
        if compile_errors:
            raise typer.Exit(1)

    # Validate pipeline parallelism (v0.27.0 Part F)
    if cfg.training.parallelism == "pipeline":
        from soup_cli.utils.pipeline import validate_pipeline_config

        pp_errors = validate_pipeline_config(
            parallelism=cfg.training.parallelism,
            pipeline_stages=cfg.training.pipeline_stages,
            device=device,
            gpu_count=gpu_info.get("gpu_count", 0),
        )
        for err in pp_errors:
            console.print(f"[red]Pipeline parallel error:[/] {for_terminal(err)}")
        if pp_errors:
            raise typer.Exit(1)
        console.print(
            Panel(
                (
                    f"Pipeline parallelism is configured "
                    f"({cfg.training.pipeline_stages} stages) but live "
                    f"execution wiring ships in v0.27.1. Your config is "
                    f"validated and the trainer will run in data-parallel "
                    f"mode for now."
                ),
                title="[yellow]Pipeline parallelism (deferred execution)[/]",
                border_style="yellow",
            )
        )

    # Validate Liger Kernel configuration
    if cfg.training.use_liger:
        from soup_cli.utils.liger import validate_liger_config

        liger_errors = validate_liger_config(
            cfg.training.use_liger, cfg.backend, device,
        )
        for err in liger_errors:
            console.print(f"[red]Liger error:[/] {for_terminal(err)}")
        if liger_errors:
            raise typer.Exit(1)

    # Validate FlashAttention configuration
    if cfg.training.use_flash_attn:
        from soup_cli.utils.flash_attn import validate_flash_attn_config

        fa_errors = validate_flash_attn_config(
            cfg.training.use_flash_attn, cfg.backend, device,
        )
        for err in fa_errors:
            console.print(f"[red]FlashAttention error:[/] {for_terminal(err)}")
        if fa_errors:
            raise typer.Exit(1)

    # Validate Ring FlashAttention configuration
    if cfg.training.use_ring_attention:
        from soup_cli.utils.ring_attention import validate_ring_attention_config

        ring_errors = validate_ring_attention_config(
            cfg.training.use_ring_attention, device, cfg.data.max_length,
        )
        for err in ring_errors:
            console.print(f"[red]Ring Attention error:[/] {for_terminal(err)}")
        if ring_errors:
            raise typer.Exit(1)

    # Validate long-context configuration
    if cfg.training.rope_scaling_type:
        from soup_cli.utils.long_context import validate_long_context_config

        ctx_errors = validate_long_context_config(
            cfg.data.max_length,
            cfg.training.rope_scaling_type,
            cfg.training.gradient_checkpointing,
        )
        for err in ctx_errors:
            console.print(f"[yellow]Long-context warning:[/] {err}")

    # Suggest unsloth if available but not being used, and only to a config the
    # loader would take with it (#1736).
    if _unsloth_tip_applies(cfg):
        console.print(
            "[dim]Tip: unsloth is installed. Add [bold]backend: unsloth[/dim]"
            "[dim] to soup.yaml for 2-5x faster training.[/]"
        )

    # #1613: cheap stripe roots validation ahead of confirmation, --dry-run,
    # dataset loading, and run creation.
    from soup_cli.utils.stripe_roots import STRIPE_DIRS_ENV

    if getattr(getattr(cfg, "training", None), "stream_layers", False) and os.environ.get(
        STRIPE_DIRS_ENV
    ):
        from soup_cli.utils.errors import format_friendly_error
        from soup_cli.utils.layer_shard import resolve_cache_root
        from soup_cli.utils.stripe_roots import StripeRootError, validate_early_stripe_roots

        try:
            validate_early_stripe_roots(resolve_cache_root())
        except StripeRootError as exc:
            format_friendly_error(exc)
            raise typer.Exit(1) from exc

    if not dry_run and not yes:
        # #1913: on a non-TTY stdin the confirm can never be answered — a pipe
        # hangs until its writer closes (EOF aborts via typer) and an
        # unattended service waits forever. Fail with the escape hatch named.
        from soup_cli.utils.exit_codes import EXIT_USAGE_ERROR

        if not _stdin_isatty():
            console.print(
                "[red]stdin is not a TTY, so the start-training confirmation "
                "cannot be answered.[/] Pass --yes to start without the "
                "confirmation."
            )
            raise typer.Exit(EXIT_USAGE_ERROR)
        if not typer.confirm("Start training?", default=True):
            console.print("[yellow]Cancelled.[/]")
            raise typer.Exit()

    # #1223: a split nothing evaluates is not withheld -- generation tasks train
    # on every row unless training.eval_steps asks for evaluation.
    from soup_cli.utils.eval_schedule import loader_data_config, validation_notice

    run_data_config = loader_data_config(cfg)
    val_notice = validation_notice(cfg)

    if dry_run:
        console.print("[yellow]Dry run - validating data...[/]")
        if val_notice:
            console.print(f"[yellow]Note:[/] {val_notice}")
        dataset = load_dataset(
            data_config_for_task(run_data_config, cfg.task),
            preserve_source_columns=task_preserves_source_columns(cfg.task),
        )
        _validate_preference_dataset_if_applicable(cfg, dataset)
        _refuse_empty_train(cfg.data, dataset)
        _validate_classification_dataset_if_applicable(cfg, dataset)
        _validate_csm_dataset_if_applicable(cfg, dataset)
        _refuse_over_judge_budget(cfg, dataset)
        console.print(
            f"[green]Data OK:[/] {_train_sample_count(cfg.data, dataset)} train samples"
        )
        if "val" in dataset:
            console.print(f"[green]Val:[/] {len(dataset['val'])} samples")
        console.print("[green]Config valid. Ready to train![/]")
        raise typer.Exit()

    # Capture the source before loading/transformation, never at verdict/registration time.
    from soup_cli.utils.training_identity import capture_training_data_hash

    training_data_sha256 = capture_training_data_hash(cfg)

    # Load data
    console.print("[dim]Loading dataset...[/]")
    if val_notice:
        console.print(f"[yellow]Note:[/] {val_notice}")
    dataset = load_dataset(
        data_config_for_task(run_data_config, cfg.task),
        preserve_source_columns=task_preserves_source_columns(cfg.task),
    )
    _validate_preference_dataset_if_applicable(cfg, dataset)
    _refuse_empty_train(cfg.data, dataset)
    _validate_classification_dataset_if_applicable(cfg, dataset)
    _validate_csm_dataset_if_applicable(cfg, dataset)
    _refuse_over_judge_budget(cfg, dataset)
    console.print(
        f"[green]Loaded:[/] {_train_sample_count(cfg.data, dataset)} train samples"
    )

    # Capture the --tracker CLI value BEFORE the local ExperimentTracker
    # shadows it (v0.43.0 review fix — name-collision regression).
    tracker_backend = tracker

    # Start experiment tracking
    from soup_cli.experiment.tracker import ExperimentTracker

    tracker = ExperimentTracker()
    run_base = cfg.base
    experiment_name = cfg.experiment_name or name
    run_id = tracker.start_run(
        config_dict=cfg.model_dump(),
        device=device,
        device_name=device_name,
        gpu_info=gpu_info,
        experiment_name=experiment_name,
        run_id=os.environ.get("SOUP_MCP_RUN_ID") or None,
    )
    # #764: without a pid, a SIGKILL'd or Ctrl+C'd run has no way for
    # _reconcile_orphaned_run to ever notice the process is gone — the
    # rescue path #401 built for MCP-spawned runs was otherwise inert here.
    tracker.mark_running(run_id, pid=os.getpid())
    tracker.record_training_data_hash(run_id, training_data_sha256)
    console.print(f"[dim]Run ID: {run_id}[/]")

    try:
        # Build trainer based on task type
        from soup_cli.utils.trackers import resolve_report_to

        try:
            report_to = resolve_report_to(
                wandb=wandb, tensorboard=tensorboard, tracker=tracker_backend
            )
        except ValueError as exc:
            from rich.markup import escape as _esc

            console.print(f"[red]{_esc(str(exc))}[/]")
            raise typer.Exit(code=2) from exc
        console.print("[dim]Setting up model + trainer...[/]")
        # v0.53.8 #130 — pre-fetch model from non-HF hub into a local cache and
        # rewrite cfg.base to point at the local snapshot. The trainer wrappers
        # still use transformers.from_pretrained, which reads HF Hub by default;
        # by snapshotting first we keep every wrapper unchanged.
        hub_name = getattr(cfg.training, "hub", "hf") or "hf"
        if hub_name != "hf":
            import re as _re

            from rich.markup import escape as _markup_escape

            from soup_cli.utils.hubs import download_repo
            from soup_cli.utils.paths import is_under_cwd

            # Sanitise cache subdir name — strip every path-separator and
            # `..` segment so a crafted ``base: ../../etc`` cannot escape the
            # cache root (Windows ``\\`` and POSIX ``/`` both blocked).
            safe_slug = _re.sub(r"[^A-Za-z0-9._-]+", "__", cfg.base).strip("._-") or "model"
            cache_dir = (Path.cwd() / ".soup_hub_cache" / safe_slug).resolve()
            if not is_under_cwd(str(cache_dir)):
                console.print(
                    "[red]Resolved hub cache dir escaped the current working "
                    "directory; refusing to download.[/]"
                )
                raise typer.Exit(code=1)
            try:
                # Idempotency: if the cache dir already has a config.json, skip
                # the re-download (modelscope/openmind-hub also short-circuit on
                # match but having an explicit probe lets us print a clear hint).
                existing_cfg = cache_dir / "config.json"
                if existing_cfg.is_file():
                    local_path = str(cache_dir)
                    console.print(
                        f"[dim]Using cached snapshot at {local_path}[/]"
                    )
                else:
                    local_path = download_repo(
                        hub_name,
                        cfg.base,
                        local_dir=str(cache_dir),
                    )
                    console.print(
                        f"[dim]Fetched {cfg.base} from hub={hub_name} → "
                        f"{local_path}[/]"
                    )
                # model_copy replaces base without validation and leaves the
                # original config untouched. local_path comes from download_repo
                # under the cache directory checked above, so no revalidation is needed.
                cfg = cfg.model_copy(update={"base": local_path})
            except ImportError as exc:
                console.print(f"[red]{_markup_escape(str(exc))}[/]")
                raise typer.Exit(code=1) from exc

        # v0.53.8 #89 — friendly missing-dep advisory for `--tracker <name>`.
        if report_to and report_to not in ("none", "wandb", "tensorboard"):
            from soup_cli.utils.trackers import tracker_missing_dep_message

            msg = tracker_missing_dep_message(report_to)
            if msg:
                console.print(f"[yellow]{msg}[/]")
        trainer_kwargs = {
            "device": device,
            "report_to": report_to,
            "deepspeed_config": ds_config_path,
            "fsdp_config": fsdp_kwargs,
        }
        # v0.40.4 #63 — every transformer-backend trainer now threads
        # --trust-remote-code through the wrapper (closes the v0.36.0 Part B gap).
        trainer_kwargs = dict(trainer_kwargs, trust_remote_code=trust_remote_code)
        # #1213 — one dispatch, shared with `soup sweep`: the two commands used
        # to keep separate chains and sweep's copy fell ten tasks and the MLX
        # route behind. See soup_cli/trainer/dispatch.py.
        from soup_cli.trainer.dispatch import build_trainer
        from soup_cli.utils.run_provenance import (
            BaseRevision,
            base_revision_scope,
            confirm_distributed_base_revision,
            synchronize_base_revision,
        )

        base_identity = BaseRevision(
            run_base,
            binding_supported=(cfg.backend == "transformers" and hub_name == "hf"),
        )
        synchronize_base_revision(base_identity, cpu=device == "cpu")
        with base_revision_scope(base_identity):
            trainer_wrapper = build_trainer(cfg, **trainer_kwargs)
            trainer_wrapper.setup(dataset)
        confirm_distributed_base_revision(
            base_identity, model=getattr(trainer_wrapper, "model", None),
        )
        tracker.record_base_revision(
            run_id, revision=base_identity.revision, source=base_identity.source,
        )
        if base_identity.source == "hub_commit":
            console.print(f"[dim]Base revision: {base_identity.revision} (bound Hub commit)[/]")
        elif base_identity.source == "cache_path":
            console.print("[dim]Base revision: cache-path metadata; local bytes are unverified[/]")
        else:
            console.print("[dim]Base revision: unknown (existing loading behavior preserved)[/]")

        # #350 — PEFT promotes newly-created adapters to fp32. FSDP cannot flatten
        # those beside bf16/float16 BNB storage, so align every trainable floating
        # tensor after setup creates PEFT modules and before train() wraps the model.
        aligned_fsdp_params = 0
        if fsdp and cfg.training.quantization == "4bit":
            from soup_cli.utils.gpu import get_compute_dtype
            from soup_cli.utils.mixed_precision import align_trainable_dtype_for_fsdp_qlora

            aligned_fsdp_params = align_trainable_dtype_for_fsdp_qlora(
                getattr(trainer_wrapper, "model", None),
                fsdp=True,
                quantization=cfg.training.quantization,
                compute_dtype=get_compute_dtype(),
            )
        if aligned_fsdp_params:
            console.print(
                f"[green]FSDP QLoRA:[/] aligned {aligned_fsdp_params} trainable "
                "parameter tensor(s) to the compute dtype"
            )

        # --- HF auto-push callback (Part B of v0.29.0) ---
        if push_as:
            from soup_cli.monitoring.hf_push import build_push_callback

            push_cb = build_push_callback(
                repo_id=push_as,
                output_dir=cfg.output,
                private=False,
            )
            if push_cb is None:
                console.print(
                    "[yellow]--push-as: no HF token available; skipping auto-push[/]"
                )
            else:
                hf_trainer = getattr(trainer_wrapper, "trainer", None)
                if hf_trainer is not None and hasattr(hf_trainer, "add_callback"):
                    hf_trainer.add_callback(push_cb)
                    console.print(
                        f"[green]HF auto-push enabled[/] -> {push_as} "
                        "(one branch per save_steps)"
                    )
                else:
                    console.print(
                        "[yellow]--push-as: trainer does not expose add_callback; "
                        "auto-push disabled for this run[/]"
                    )

        # Train with live display and experiment tracking
        display = TrainingDisplay(cfg, device_name=device_name)
        console.print("[bold green]Training started![/]\n")

        profiler_ctx = contextlib.nullcontext()
        if profile_run:
            from soup_cli.utils.profiling import profile_training

            profiler_ctx = profile_training(output_dir=Path(cfg.output), run_id=run_id)
            console.print(
                "[cyan]--profile:[/] writing torch.profiler trace to "
                f"{cfg.output}/profiles/{run_id}.trace.json (early-steps window)"
            )

        # v0.71.3 #180 — optional codecarbon energy/CO2 measurement around the
        # training window. Lazy-built; a graceful no-op when codecarbon is absent.
        energy_ctx = contextlib.nullcontext()
        energy_tracker = None
        if track_energy:
            try:
                from soup_cli.utils.energy import EnergyTracker

                energy_tracker = EnergyTracker(country_iso_code=energy_country)
                energy_ctx = energy_tracker
            except ValueError as exc:
                console.print(f"[yellow]--track-energy disabled:[/] {exc}")
                energy_tracker = None
                energy_ctx = contextlib.nullcontext()

    except Exception as exc:
        tracker.fail_run(run_id, error=_describe_exception_for_tracker(exc))
        raise

    try:
        with profiler_ctx as profiler, energy_ctx, base_revision_scope(base_identity):
            if profile_run and profiler is not None:
                from soup_cli.utils.profiling import attach_profile_callback

                if not attach_profile_callback(trainer_wrapper, profiler):
                    console.print(
                        "[yellow]--profile:[/] this trainer has no step hook, "
                        "so no trace will be written"
                    )
            result = trainer_wrapper.train(
                display=display, tracker=tracker, run_id=run_id,
                resume_from_checkpoint=resume_from,
            )

        from soup_cli.utils.run_provenance import write_run_provenance

        confirm_distributed_base_revision(
            base_identity, model=getattr(trainer_wrapper, "model", None),
        )
        tracker.record_base_revision(
            run_id, revision=base_identity.revision, source=base_identity.source,
        )
        # Only the saving rank writes portable metadata beside the final artifact.
        if _should_run_diagnose_gate_on_rank():
            metadata = base_identity.metadata(run_id)
            recorded_run = tracker.get_run(run_id)
            if isinstance(recorded_run, dict):
                metadata["training_data_sha256"] = recorded_run.get("training_data_sha256")
            write_run_provenance(result["output_dir"], metadata)

        # Save completion to tracker
        tracker.finish_run(
            run_id=run_id,
            initial_loss=result["initial_loss"],
            final_loss=result["final_loss"],
            total_steps=result["total_steps"],
            duration_secs=result["duration_secs"],
            output_dir=result["output_dir"],
        )

        _run_auto_eval_after_training(
            cfg.eval,
            result["output_dir"],
            run_id,
            trust_remote_code=trust_remote_code,
        )
    except Exception as exc:
        tracker.fail_run(run_id, error=_describe_exception_for_tracker(exc))
        # v0.34.0 Part D — write a .crash bundle next to the run for triage.
        try:
            from soup_cli.utils.crash import build_crash_bundle, write_crash_bundle

            metrics = tracker.get_metrics(run_id)
            bundle = build_crash_bundle(
                error=exc,
                config=cfg.model_dump() if hasattr(cfg, "model_dump") else None,
                metrics=metrics,
                run_id=run_id,
                output_dir=getattr(cfg, "output", None),
            )
            crash_path = write_crash_bundle(bundle)
            console.print(
                f"[yellow]Crash bundle written:[/] {crash_path}\n"
                "[dim]Attach this file when reporting the failure.[/]"
            )
        except Exception as crash_err:
            # Never let the crash reporter mask the original error, but tell
            # the user the bundle is missing so they don't hunt for it.
            console.print(
                f"[dim]Could not write crash bundle: {crash_err}[/]"
            )
        raise

    # Report
    merge_hint = (
        "ReLoRA output is already dense; no soup merge is needed"
        if cfg.training.relora_steps is not None
        else f"[bold]Merge LoRA:[/]  soup merge --adapter {result['output_dir']}"
    )

    console.print(
        Panel(
            f"{_format_training_complete_loss(result)}\n"
            f"Duration: [bold]{_format_duration_display(result)}[/]\n"
            f"Output: [bold]{result['output_dir']}[/]\n"
            f"Run ID: [bold]{run_id}[/]\n\n"
            f"Quick test:  [bold]soup chat --model {result['output_dir']}[/]\n"
            f"Push to HF:  [bold]soup push --model {result['output_dir']}[/]\n"
            f"{merge_hint}\n"
            f"Export GGUF: [bold]soup export --model {result['output_dir']}[/]\n"
            f"Run details: [bold]soup runs show {run_id}[/]",
            title=("[bold yellow]Training Stopped — Loss watchdog[/]"
                   if result.get("stop_reason") == "loss_watchdog"
                   else "[bold green]Training Complete![/]"),
        )
    )

    # --- v0.56.0 --diagnose-gate: post-training failure-mode check ---
    if diagnose_gate and _should_run_diagnose_gate_on_rank():
        _run_diagnose_gate_or_exit(
            diagnose_gate, run_id, cfg.base, result["output_dir"]
        )

    # --- v0.71.3 #180 --track-energy: print the measured energy/CO2 -------
    energy_measurement = (
        energy_tracker.measurement if energy_tracker is not None else None
    )
    if track_energy:
        if energy_measurement is not None:
            console.print(
                f"[cyan]--track-energy:[/] {energy_measurement.energy_kwh:.4f} kWh"
                f" / {energy_measurement.co2_kg:.4f} kg CO2"
                f" (grid {energy_measurement.grid_intensity_g_per_kwh:.0f} g/kWh,"
                f" PUE {energy_measurement.pue})"
            )
        else:
            console.print(
                "[yellow]--track-energy:[/] no reading "
                "(install `pip install soup-cli\\[carbon]`)"
            )

    # --- v0.71.15 #244 --energy-out: persist for `soup bom emit --energy` -
    if energy_out and _should_run_diagnose_gate_on_rank():
        if energy_measurement is not None:
            try:
                _write_energy_json(energy_out, energy_measurement)
                console.print(
                    "[cyan]--energy-out:[/] wrote "
                    f"{os.path.basename(energy_out)} "
                    "(feed to `soup bom emit --energy`)"
                )
            except (OSError, ValueError) as exc:
                console.print(
                    f"[yellow]--energy-out skipped:[/] {type(exc).__name__}: {exc}"
                )
        else:
            console.print(
                "[yellow]--energy-out skipped:[/] no energy reading "
                "(set --track-energy + install `pip install soup-cli\\[carbon]`)"
            )

    # --- v0.59.0 --annex-xi: Annex XI/XII auto-doc -----------------------
    if annex_xi and _should_run_diagnose_gate_on_rank():
        try:
            _write_annex_xi(annex_xi, run_id, cfg, energy=energy_measurement)
        except typer.Exit:
            raise
        except (OSError, ValueError) as exc:
            console.print(
                f"[yellow]--annex-xi skipped:[/] {type(exc).__name__}: {exc}"
            )

    # --- v0.59.0 --repro-receipt: SR 11-7 receipt ------------------------
    if repro_receipt and _should_run_diagnose_gate_on_rank():
        try:
            _write_repro_receipt(repro_receipt, run_id, cfg)
        except typer.Exit:
            raise
        except (OSError, ValueError) as exc:
            console.print(
                f"[yellow]--repro-receipt skipped:[/] {type(exc).__name__}: {exc}"
            )

    # --- v0.71.8 #219 --capture-activations: SAE-diff-ready snapshot ------
    if capture_activations and _should_run_diagnose_gate_on_rank():
        try:
            written = _capture_activations(
                capture_activations,
                capture_prompts,
                cfg.base,
                result["output_dir"],
                trust_remote_code=trust_remote_code,
            )
            console.print(f"[green]--capture-activations[/] -> {written}")
        except (OSError, ValueError, RuntimeError, ImportError) as exc:
            console.print(
                f"[yellow]--capture-activations skipped:[/] "
                f"{type(exc).__name__}: {exc}"
            )


def _write_annex_xi(out_path: str, run_id: str, cfg, *, energy=None) -> None:
    """Render an Annex XI doc using values from the resolved soup.yaml.

    v0.71.3 #180: the optional ``energy`` measurement populates the kWh / CO2
    fields. v0.71.3 #184: the top crawled domains are auto-extracted from the
    training JSONL. v0.71.3 #181: a ``.pdf`` output path renders a PDF.
    """
    from datetime import datetime, timezone

    from soup_cli import __version__
    from soup_cli.utils.annex_xi import (
        AnnexXIData,
        load_top_domains_from_jsonl,
        write_annex_doc,
    )

    modality = getattr(cfg, "modality", "text") or "text"
    # #1446: without --track-energy nothing was measured; say so instead of 0.000.
    energy_kwh = float(getattr(energy, "energy_kwh", 0.0)) if energy is not None else None
    co2_kg = float(getattr(energy, "co2_kg", 0.0)) if energy is not None else None
    raw_train = getattr(cfg.data, "train", "") or ""
    # #443 — pass the raw str|list through so top-domain extraction
    # aggregates across every interleaved dataset, instead of stringifying
    # a list into a nonexistent path (pre-fix: `str(getattr(...) or "")`
    # applied `or ""` to the getattr result BEFORE str(), so a non-empty
    # list became its own Python repr string, e.g. "['a.jsonl', 'b.jsonl']"
    # — neither a valid path nor useful doc text).
    train_display = ", ".join(raw_train) if isinstance(raw_train, list) else str(raw_train)
    # #184 — best-effort extract the top crawled domains from the training data.
    top_domains = load_top_domains_from_jsonl(raw_train)
    fmt = "pdf" if out_path.lower().endswith(".pdf") else "markdown"
    data = AnnexXIData(
        model_name=str(getattr(cfg, "output", run_id) or run_id),
        base_model=str(cfg.base),
        task=str(cfg.task),
        dataset_summary=train_display,
        modalities=(modality,),
        train_compute_flops=None,  # #1446: Soup does not measure FLOPs; never claim 0
        train_energy_kwh=energy_kwh,
        train_co2_kg=co2_kg,
        top_domains=top_domains,
        soup_version=__version__,
        run_id=run_id,
        created_at=datetime.now(tz=timezone.utc).isoformat(),
    )
    written = write_annex_doc(data, "xi", out_path, fmt=fmt)
    console.print(f"[green]--annex-xi[/] -> {written}")


def _write_repro_receipt(out_path: str, run_id: str, cfg) -> None:
    """Render an SR 11-7 receipt from the resolved soup.yaml."""
    from soup_cli.utils.repro_receipt import build_repro_receipt, write_repro_receipt

    seeds: dict[str, int] = {}
    seed = getattr(cfg.training, "seed", None)
    if isinstance(seed, int) and not isinstance(seed, bool):
        seeds["torch"] = seed
        seeds["numpy"] = seed
        seeds["python"] = seed
    receipt = build_repro_receipt(seeds=seeds, run_id=run_id)
    written = write_repro_receipt(receipt, out_path)
    console.print(f"[green]--repro-receipt[/] -> {written}")


def _load_capture_prompts(canonical_path: str) -> list[str]:
    """Read prompts for --capture-activations (JSONL objects, JSON strings, or raw lines)."""
    import json

    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
    fd = os.open(canonical_path, flags)
    # File-size cap on the SAME fd (no TOCTOU) bounds a pathological
    # newline-free multi-GB line before the `for line in handle` read.
    if os.fstat(fd).st_size > 64 * 1024 * 1024:
        os.close(fd)
        raise ValueError("--capture-prompts file exceeds 64 MiB")
    prompts: list[str] = []
    with os.fdopen(fd, encoding="utf-8") as handle:
        for line in handle:
            line = line.strip()
            if not line:
                continue
            try:
                obj = json.loads(line)
            except json.JSONDecodeError:
                prompts.append(line)  # raw-text line
            else:
                if isinstance(obj, str) and obj:
                    prompts.append(obj)
                elif isinstance(obj, dict):
                    for key in ("prompt", "text", "instruction", "input", "question"):
                        val = obj.get(key)
                        if isinstance(val, str) and val:
                            prompts.append(val)
                            break
            if len(prompts) >= 256:
                break
    return prompts


def _capture_activations(
    layer: str,
    prompts_path: str,
    base: str,
    output_dir: str,
    *,
    trust_remote_code: bool = False,
) -> str:
    """Capture residual-stream activations from the trained model (v0.71.8 #219).

    Loads the trained model at ``output_dir`` (or ``base`` + adapter when the
    output is a LoRA adapter), runs ``prompts_path`` through a forward hook on
    ``layer``, and writes the per-token activations to
    ``<output_dir>/activations/activations.json`` in the
    ``{"activations": [[...]], "layer", "num_tokens", "hidden_dim"}`` shape that
    ``soup probe sae-diff`` / ``sleeper`` consume directly.
    """
    import json

    from soup_cli.utils import live_eval
    from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink

    if not prompts_path:
        raise ValueError("--capture-activations requires --capture-prompts <jsonl>")
    if not isinstance(layer, str) or not layer.strip():
        raise ValueError("--capture-activations layer must be a non-empty string")
    canonical = enforce_under_cwd_and_no_symlink(prompts_path, "--capture-prompts")
    prompts = _load_capture_prompts(canonical)
    if not prompts:
        raise ValueError("--capture-prompts has no usable prompts")

    # When the output is a LoRA adapter dir, load base + adapter; else load the
    # full fine-tuned model from output_dir directly.
    is_adapter = os.path.isfile(os.path.join(output_dir, "adapter_config.json"))
    has_full = os.path.isfile(os.path.join(output_dir, "config.json"))
    if not is_adapter and not has_full:
        raise ValueError(
            f"output dir {os.path.basename(output_dir)} has neither "
            "adapter_config.json nor config.json — cannot capture activations"
        )
    adapter = output_dir if is_adapter else None
    model_id = base if adapter else output_dir

    model, tokenizer, dev = live_eval.load_model_and_tokenizer(
        model_id, adapter=adapter, trust_remote_code=trust_remote_code
    )
    acts = live_eval.extract_layer_activations(
        model, tokenizer, prompts, layer=layer, device=dev, pool="none"
    )

    acts_dir = os.path.join(output_dir, "activations")
    # Refuse a pre-placed `activations` symlink so the write cannot be
    # redirected outside output_dir (defence-in-depth — output_dir is config-
    # derived/trusted but the symlink check is cheap).
    if os.path.islink(acts_dir):
        raise ValueError("activations subdir is a symlink — refusing to write")
    os.makedirs(acts_dir, exist_ok=True)
    out_path = os.path.join(acts_dir, "activations.json")
    payload = {
        "layer": layer,
        "num_tokens": int(acts.shape[0]),
        "hidden_dim": int(acts.shape[1]),
        "activations": acts.tolist(),
    }
    with open(out_path, "w", encoding="utf-8") as handle:
        json.dump(payload, handle)
    return out_path


def _write_energy_json(path: str, measurement: "EnergyMeasurement") -> None:
    """Persist an ``EnergyMeasurement`` as JSON for ``soup bom emit --energy``.

    Writes exactly the five fields ``EnergyMeasurement(**parsed)`` expects so
    the producer (``soup train --track-energy --energy-out``) and the consumer
    (``soup bom emit --energy`` — v0.71.3 #256) round-trip cleanly. Atomic +
    cwd-contained + symlink-rejected via the shared helper (v0.71.15 #244).
    """
    import json
    from dataclasses import asdict

    from soup_cli.utils.paths import atomic_write_text

    payload = json.dumps(asdict(measurement), indent=2)
    atomic_write_text(payload, path, field="--energy-out")


def _should_run_diagnose_gate_on_rank() -> bool:
    """Return True only for the single chief worker in a distributed launch.

    The diagnose gate (and the --annex-xi / --repro-receipt / capture hooks)
    write one report and, on a shared filesystem, read one shared output dir
    -- so they should fire exactly once per *cluster*, not once per node.

    Resolution (v0.71.15 #170 — fixes the v0.56.0 limitation where a shared-FS
    multi-node run fired the gate once per node):
      * If RANK is set (multi-node / torchrun launch) -> gate only on the
        global chief, RANK == 0. On a single-node torchrun this is equivalent
        to LOCAL_RANK == 0 (both 0 only for the one chief process).
      * Else fall back to LOCAL_RANK == 0 (plain single-node accelerate launch
        that doesn't export RANK) so a one-box multi-GPU run still gates once.

    Defaults to True (run gate) on any parse error: a malformed env var is
    safer to over-run than to silently skip.
    """
    rank = os.environ.get("RANK")
    if rank:  # non-empty -> multi-node / torchrun sets a global RANK
        try:
            return int(rank) == 0
        except ValueError:
            return True
    try:
        return int(os.environ.get("LOCAL_RANK", "0")) == 0
    except ValueError:
        return True


def _run_diagnose_gate(
    evidence_path: str, run_id: str, base: str, adapter: str
) -> None:
    """Post-training failure-mode gate (v0.56.0).

    Loads a JSON ``evidence`` file with optional per-mode scores and
    refuses to mark the run successful if any mode comes back MAJOR (exit 2)
    or NOT_RUN (exit 3, #1435). Missing modes fall back to a neutral OK score so partial evidence
    still produces a useful report card. The train command only calls
    this helper on the chief worker (RANK==0 in a multi-node launch, else
    LOCAL_RANK==0) so distributed runs execute the gate once per cluster,
    not once per worker (v0.71.15 #170).
    """
    import json

    from soup_cli.utils.diagnose.report import FAILURE_MODES, FailureScore
    from soup_cli.utils.diagnose.runner import build_report
    from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink
    from soup_cli.utils.terminal import for_terminal

    enforce_under_cwd_and_no_symlink(evidence_path, "--diagnose-gate evidence")
    # 16 MiB cap on evidence JSON (security review HIGH — symmetric with
    # `commands/diagnose._MAX_EVIDENCE_BYTES`; prevents `/dev/zero` /
    # multi-GB symlink-pointed OOM at json.load time).
    if os.path.getsize(evidence_path) > 16 * 1024 * 1024:
        raise ValueError(
            "--diagnose-gate evidence exceeds 16 MiB"
        )
    with open(evidence_path, encoding="utf-8") as handle:
        payload = json.load(handle)
    if not isinstance(payload, dict):
        raise ValueError("evidence file must contain a JSON object")
    raw_scores = payload.get("scores") or {}
    if not isinstance(raw_scores, dict):
        raise ValueError("evidence.scores must be an object")

    from soup_cli.utils.diagnose.report import classify_score, evidence_default_score
    from soup_cli.utils.exit_codes import EXIT_USAGE_ERROR

    scores: dict = {}
    for mode in FAILURE_MODES:
        entry = raw_scores.get(mode)
        if entry is None:
            continue
        if not isinstance(entry, dict):
            raise ValueError(f"scores.{mode} must be an object")
        score = entry.get("score", evidence_default_score(entry))
        if isinstance(score, bool) or not isinstance(score, (int, float)):
            raise ValueError(f"scores.{mode}.score must be a number")
        verdict = entry.get("verdict") or classify_score(score)
        scores[mode] = FailureScore(
            mode=mode,
            score=float(score),
            verdict=verdict,
            evidence=str(entry.get("evidence", "supplied by --diagnose-gate")),
        )

    report = build_report(
        run_id=run_id, base=base, adapter=adapter, scores=scores
    )
    if report.overall == "MAJOR":
        console.print(
            "[red]--diagnose-gate: MAJOR regression in one or more modes.[/]"
        )
        for mode in FAILURE_MODES:
            sc = report.scores[mode]
            if sc.verdict == "MAJOR":
                console.print(f"  [red]MAJOR[/] {mode}: {for_terminal(sc.evidence)}")
        raise typer.Exit(2)
    if report.overall == "NOT_RUN":
        # An unmeasured mode is not a pass (#1435); same exit as `soup diagnose`.
        console.print(
            "[yellow]--diagnose-gate: one or more modes did not run (NOT_RUN); "
            "the run is not verified.[/]"
        )
        for mode in FAILURE_MODES:
            sc = report.scores[mode]
            if sc.verdict == "NOT_RUN":
                console.print(f"  [yellow]NOT_RUN[/] {mode}: {for_terminal(sc.evidence)}")
        raise typer.Exit(EXIT_USAGE_ERROR)
    console.print(
        f"[green]--diagnose-gate: {report.overall}[/] across "
        f"{len(FAILURE_MODES)} modes."
    )


def _run_diagnose_gate_or_exit(
    evidence_path: str, run_id: str, base: str, adapter: str
) -> None:
    """Run the gate; an unreadable or refused evidence file is reported and exits 1."""
    from soup_cli.utils.terminal import for_terminal

    try:
        _run_diagnose_gate(evidence_path, run_id, base, adapter)
    except typer.Exit:
        raise
    except (OSError, ValueError) as exc:
        console.print(
            f"[red]--diagnose-gate failed:[/] {type(exc).__name__}: {for_terminal(exc)}"
        )
        raise typer.Exit(1) from exc


def _resolve_deepspeed(deepspeed: str) -> str:
    """Resolve DeepSpeed config: named preset or path to JSON file."""
    import soup_cli.utils.deepspeed as ds

    # Named preset
    if deepspeed in ds.CONFIGS:
        return ds.write_deepspeed_config(deepspeed)

    # Path to config file. Resolved the same way a preset is (#359): a config
    # that needs no run-dependent rewrite comes back by this very path, and one
    # that copied the ZeRO++ placeholders is repaired into a temp copy with the
    # change printed. The user's file is never modified.
    ds_path = Path(deepspeed)
    if ds_path.exists() and ds_path.suffix == ".json":
        return ds.resolve_user_deepspeed_file(str(ds_path))

    console.print(
        f"[red]Invalid DeepSpeed config: {deepspeed}[/]\n"
        f"Options: {', '.join(ds.CONFIGS.keys())} or path to JSON file."
    )
    raise typer.Exit(1)


_MLX_CHECKPOINT_RE = re.compile(r"^(\d+)_adapters\.safetensors$")


def _highest_numbered_mlx_checkpoint(paths) -> Path | None:
    """Pick the highest step-numbered ``NNNNNNN_adapters.safetensors`` file.

    ``max()`` over the parsed step number: not a sort over whatever order
    the filesystem's ``iterdir()`` happened to enumerate (that order isn't
    guaranteed), and not a comparison of the filenames as strings (that
    would only agree with numeric order if every step number in a run
    happened to be zero-padded to the same width, which nothing here
    enforces). Keying on the parsed int makes both mistakes fail instead
    of coincidentally still returning the right file.
    """
    numbered: list[tuple[int, Path]] = []
    for path in paths:
        if not path.is_file():
            continue
        match = _MLX_CHECKPOINT_RE.match(path.name)
        if match:
            numbered.append((int(match.group(1)), path))
    if not numbered:
        return None
    return max(numbered, key=lambda pair: pair[0])[1]


def _resolve_mlx_checkpoint(resume: str, output_dir: str) -> str | None:
    """MLX-style checkpoint resolution (#634).

    mlx-lm's tuner saves step-numbered adapter snapshots
    (``NNNNNNN_adapters.safetensors``) plus a final ``adapters.safetensors``
    with no step prefix — files, not the ``checkpoint-N`` directories the
    transformers/unsloth backends write. "auto" picks the highest-numbered
    snapshot if one exists, else the final file. A direct ``resume`` value
    must point at one of these files directly.

    Takes no ``experiment_name``, unlike the transformers/unsloth resolver
    below: ``mlx_sft.py``'s ``output_dir`` is always ``Path(cfg.output)``,
    flat, never nested under it the way ``trainer/sft.py`` nests under
    ``output_dir / cfg.experiment_name``. Nesting here would look inside a
    directory MLX never writes to, and "auto" would report no checkpoint
    found on every run that sets ``experiment_name`` — the exact symptom
    #634 reported, reintroduced for a config the original fix didn't cover.
    """
    if resume.lower() == "auto":
        base = Path(output_dir)

        if not base.is_dir():
            return None

        highest = _highest_numbered_mlx_checkpoint(base.iterdir())
        if highest is not None:
            return str(highest)

        final = base / "adapters.safetensors"
        if final.is_file():
            return str(final)
        return None

    # Direct path to a specific adapter file
    checkpoint_path = Path(resume)
    if checkpoint_path.exists() and checkpoint_path.is_file():
        return str(checkpoint_path)
    return None


def _resolve_checkpoint(
    resume: str,
    output_dir: str,
    experiment_name: str | None = None,
    *,
    backend: str = "transformers",
) -> str | None:
    """Resolve the checkpoint path from --resume argument.

    If resume == "auto", find the latest checkpoint in the output directory.
    Otherwise, treat it as a direct path to a checkpoint directory.

    ``backend="mlx"`` dispatches to :func:`_resolve_mlx_checkpoint`, since MLX
    writes step-numbered adapter files rather than ``checkpoint-N``
    directories (#634) — the shape below never matches an MLX run's output.
    """
    if backend == "mlx":
        return _resolve_mlx_checkpoint(resume, output_dir)

    if resume.lower() == "auto":
        base = Path(output_dir)
        if experiment_name:
            base = base / experiment_name

        if not base.exists():
            return None

        checkpoints = sorted(
            [d for d in base.iterdir() if d.is_dir() and d.name.startswith("checkpoint-")],
            key=lambda d: int(d.name.split("-")[-1]) if d.name.split("-")[-1].isdigit() else 0,
        )
        if checkpoints:
            return str(checkpoints[-1])
        return None

    # Direct path
    checkpoint_path = Path(resume)
    if checkpoint_path.exists() and checkpoint_path.is_dir():
        return str(checkpoint_path)
    return None


def _resolve_resume_or_exit(resume: str, cfg: "SoupConfig") -> str | None:
    """Resolve ``--resume`` against ``cfg``, printing status and exiting on
    failure. Extracted out of ``train()`` (#634 review) so the
    ``backend=cfg.backend`` wiring — the entire seam the MLX checkpoint fix
    depends on — is directly testable without invoking the rest of the
    command, which needs real hardware/model loading this can be exercised
    without. Returns ``None`` only when ``resume`` itself is falsy; a
    ``resume`` value that fails to resolve exits rather than returning.
    """
    if not resume:
        return None
    if cfg.task in UNSUPPORTED_RESUME_TASKS:
        console.print(f"[red]--resume is not supported for task {cfg.task!r}[/]")
        raise typer.Exit(1)
    resume_from = _resolve_checkpoint(resume, cfg.output, cfg.experiment_name, backend=cfg.backend)
    if resume_from:
        console.print(f"[green]Resuming from:[/] {resume_from}")
    else:
        console.print("[red]No checkpoint found to resume from.[/]")
        raise typer.Exit(1)
    return resume_from


def _run_live_lr_sweep(
    config_path: str, schedule: list[float], trust_remote_code: bool = False,
) -> tuple[list[float], list[float]]:
    """v0.33.0 #56 — run the in-process LR sweep over the config's own model.

    Returns ``(lrs, losses)`` of equal length.

    #1203: every reason the sweep cannot run is a refusal.
    :class:`LrSweepUnavailableError` covers a config that will not load and a
    model or dataset that cannot be swept; :class:`SweepTooShortError` covers
    fewer than ``MIN_NUM_STEPS`` rows, and a loss that is non-finite from the
    first step or turns non-finite before 4 steps have run.
    This used to answer all of them with a synthetic curve shaped by the LR
    schedule alone, so a base model that does not exist exited 0 with a
    ``recommended_lr`` that described no model and no data.
    """
    from soup_cli.utils.lr_finder import LrSweepUnavailableError, SweepTooShortError

    try:
        # The refusal below quotes the reason, so the loader hands it over
        # instead of printing its own line and exiting.
        cfg = load_config(config_path, raise_unreadable=True)
    except Exception as exc:  # noqa: BLE001 — one refusal line, not a traceback
        raise LrSweepUnavailableError(
            f"--find-lr: config load failed ({exc}); no LR report was written."
        ) from exc

    try:
        return _live_lr_sweep_from_config(
            cfg, schedule, trust_remote_code=trust_remote_code,
        )
    except SweepTooShortError:
        raise
    except Exception as exc:  # noqa: BLE001 — the cause is the useful half
        raise LrSweepUnavailableError(
            f"--find-lr: live sweep unavailable ({exc}); no LR report was written."
        ) from exc


def _lr_finder_dataset_path(train) -> str:
    """#443 — LR-finder samples one representative dataset; it already
    bypasses load_dataset()/_finalize() for a lightweight sweep, so full
    interleave fidelity is out of this issue's scope. Falls back to the
    first dataset rather than crashing on a list. Extracted as its own
    function so it can be exercised directly by
    tests/test_issue443_interleave_wiring.py's enumerating test.
    """
    return train[0] if isinstance(train, list) else train


def _live_lr_sweep_from_config(
    cfg, schedule: list[float], trust_remote_code: bool = False,
) -> tuple[list[float], list[float]]:
    """Build a tiny in-process loop: load model + tokenizer + a slice of
    the train dataset, then call :func:`run_lr_sweep`. Returns the
    ``(lrs, losses)`` the sweep actually ran."""
    # v0.40.1 Part C / G12 — fix broken `load_local` import that previously
    # made the live sweep fail every time. The actual exported symbol
    # is ``load_raw_data`` (path-only loader) — we use that.
    from pathlib import Path as _Path

    import torch
    from transformers import AutoModelForCausalLM, AutoTokenizer

    from soup_cli.data.loader import load_raw_data
    from soup_cli.utils.lr_finder import (
        MIN_NUM_STEPS,
        SweepTooShortError,
        compute_lr_schedule,
        run_lr_sweep,
    )

    # #1189: one row is one sweep step, so size the sweep before the model loads.
    lr_finder_train_path = _lr_finder_dataset_path(cfg.data.train)
    rows = list(load_raw_data(_Path(lr_finder_train_path)))[: len(schedule)]
    if len(rows) < MIN_NUM_STEPS:
        row_count = f"{len(rows)} row" + ("" if len(rows) == 1 else "s")
        raise SweepTooShortError(
            f"--find-lr: the training set has {row_count}, but the sweep needs at "
            f"least {MIN_NUM_STEPS} (one row per step; --find-lr-steps {len(schedule)})"
        )
    if len(rows) < len(schedule):
        # Rebuilt over the same range: keeping the first rows' worth of the original
        # schedule would never reach the high LRs where training diverges.
        console.print(
            f"[yellow]--find-lr: the training set has {len(rows)} rows, fewer than "
            f"--find-lr-steps {len(schedule)}; sweeping {len(rows)} steps over the "
            f"same {schedule[0]:.3g} to {schedule[-1]:.3g} range.[/]"
        )
        schedule = compute_lr_schedule(schedule[0], schedule[-1], len(rows))

    device = "cuda" if torch.cuda.is_available() else "cpu"
    tokenizer = AutoTokenizer.from_pretrained(
        cfg.base, trust_remote_code=trust_remote_code,
    )
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    model = AutoModelForCausalLM.from_pretrained(
        cfg.base, trust_remote_code=trust_remote_code,
    ).to(device)
    model.train()

    def _tokenize(row):
        text = row.get("text") or row.get("prompt") or ""
        if not text and "messages" in row:
            text = " ".join(m.get("content", "") for m in row["messages"])
        enc = tokenizer(
            text or " ", return_tensors="pt", truncation=True,
            max_length=min(cfg.data.max_length or 256, 256),
            padding="max_length",
        )
        enc["labels"] = enc["input_ids"].clone()
        return {k: v.squeeze(0) for k, v in enc.items()}

    def _batched_loader():
        for row in rows:
            tok = _tokenize(row)
            yield {k: v.unsqueeze(0) for k, v in tok.items()}

    losses = run_lr_sweep(
        model=model,
        dataloader=_batched_loader(),
        schedule=schedule,
        optimizer_factory=lambda params: torch.optim.AdamW(params, lr=schedule[0]),
        device=device,
    )
    # With a row for every step, a short result means the loss went non-finite.
    if len(losses) < len(schedule):
        diverged_lr = schedule[len(losses)]
        if not losses:
            # No update has run yet, so the learning rate cannot be the cause.
            raise SweepTooShortError(
                f"--find-lr: the loss was non-finite on the first step (lr {diverged_lr:.3g}), "
                "before any update; check the training data, dtype and model rather "
                "than the LR range"
            )
        if len(losses) < MIN_NUM_STEPS:
            steps_run = f"{len(losses)} step" + ("" if len(losses) == 1 else "s")
            raise SweepTooShortError(
                f"--find-lr: the loss became non-finite at lr {diverged_lr:.3g} after "
                f"{steps_run}, too few for a recommendation (need "
                f"{MIN_NUM_STEPS}); lower --find-lr-start / --find-lr-end"
            )
        console.print(
            f"[yellow]--find-lr: the loss became non-finite at lr {diverged_lr:.3g}; "
            f"the report covers the {len(losses)} steps before it.[/]"
        )
    return schedule[: len(losses)], losses