Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

59.3 KB
Raw Blame
"""MLX SFT trainer — Apple Silicon fine-tuning via mlx-lm.

Rewritten for mlx-lm >= 0.31 (v0.73.0 fix, PR #362):
- ``mlx_lm.tuner.trainer.train`` no longer takes a ``tokenizer`` argument;
  datasets are built with ``mlx_lm.tuner.datasets.create_dataset``, wrapped in
  ``CacheDataset`` (bare ``ChatDataset`` lacks ``__len__``/``__getitem__``),
  and loss is reported through ``TrainingCallback.on_train_loss_report``.
- LoRA layers are applied with ``mlx_lm.tuner.utils.linear_to_lora_layers``
  after ``model.freeze()``: ``mlx_lm.load`` does NOT freeze, and without it
  every parameter is trainable and the saved "adapter" is actually a full
  fine-tune (172 tensors vs 24 LoRA tensors on a 1.2B model).
- Implements the CLI trainer contract: ``train()`` accepts the common kwargs
  (display/tracker/run_id/resume_from_checkpoint) and returns the metrics
  dict that ``commands/train.py`` expects.
"""

from __future__ import annotations

import json
import math
import time
from dataclasses import dataclass
from pathlib import Path

from rich.console import Console

from soup_cli.config.schema import SoupConfig
from soup_cli.trainer.loss_summary import summarize_training_loss

console = Console()

# Chat key for the masked dataset, matching upstream's `chat_feature` default.
_CHAT_KEY = "messages"


class _LossWatchdogStop(Exception):  # noqa: N818 — a control signal, not a training error
    """Internal control signal, raised only after a completed optimizer update."""


@dataclass
class _LossWatchdog:
    """Count consecutive measured reports strictly above an absolute threshold."""

    threshold: float
    patience: int
    counter: int = 0

    def observe(self, loss: float | None) -> bool:
        if loss is None:
            return False
        if not math.isfinite(loss):
            raise FloatingPointError("MLX loss watchdog: non-finite training loss")
        self.counter = self.counter + 1 if loss > self.threshold else 0
        return self.counter >= self.patience


def _count_safetensors_tensors(path: str) -> int:
    """Number of tensors declared in a ``.safetensors`` file's own header.

    Parsed directly from the format (an 8-byte little-endian length prefix
    followed by that many bytes of JSON metadata) rather than via mlx or
    the ``safetensors`` package, so this runs on any machine regardless of
    whether either is installed. ``__metadata__`` is the one header key
    that isn't a tensor.

    ``path`` is untrusted here: the ``--resume`` direct-path branch accepts
    any existing file, and ``--hf-resume`` can point at a directory. The
    length prefix is bounded against the file's own size before it's used
    to size a read, and every failure mode (garbage length, truncated or
    non-JSON content, a directory instead of a file) collapses to one
    ``ValueError`` naming the path, instead of a raw ``MemoryError``,
    ``JSONDecodeError``, or ``IsADirectoryError`` reaching the caller.
    """
    try:
        file_size = Path(path).stat().st_size
        with open(path, "rb") as f:
            header_len = int.from_bytes(f.read(8), "little")
            if not 0 < header_len <= file_size:
                raise ValueError(
                    f"declared header length {header_len} is invalid for a "
                    f"{file_size}-byte file"
                )
            header = json.loads(f.read(header_len))
    except (OSError, ValueError, json.JSONDecodeError) as exc:
        raise ValueError(
            f"MLX checkpoint {path} is not a readable safetensors file: {exc}"
        ) from exc
    return sum(1 for key in header if key != "__metadata__")


#: What `target_modules: auto` means on MLX. peft resolves `auto` per
#: architecture; mlx-lm has no equivalent, so the streamed-down default is
#: attention Q/V — the modules `_apply_lora` has always actually trained.
MLX_DEFAULT_TARGET_KEYS = ["self_attn.q_proj", "self_attn.v_proj"]


def resolve_mlx_target_keys(lora_cfg: object) -> list[str]:
    """THE answer to "which modules does this MLX run train?" (#392).

    Both the trainer and the ``adapter_config.json`` writer must use it. They
    used to resolve separately — ``_apply_lora`` into a local, the writer not at
    all — so a default run trained Q/V and then shipped ``{"keys": ["auto"]}``.
    ``linear_to_lora_layers`` matches nothing against that and
    ``load_weights(strict=False)`` drops every LoRA tensor in silence, leaving
    an adapter whose generations are bit-identical to the base model.
    """
    raw = getattr(lora_cfg, "target_modules", None)
    if not raw or raw in (["auto"], "auto"):
        return list(MLX_DEFAULT_TARGET_KEYS)
    return list(raw) if isinstance(raw, list) else [raw]


class _MlxRewind:
    """The flight recorder attached to one MLX run (see ``trainer/rewind_mlx.py``)."""

    def __init__(self, log: object, state: object, loss: object) -> None:
        self.log = log
        self.state = state
        self.loss = loss

    def wrap(self, train_dataset: object) -> object:
        # Training dataset only: a validation dataset wrapped with the same state
        # would flush the pending training ids against the previous iteration.
        from soup_cli.trainer.rewind_mlx import wrap_dataset

        return wrap_dataset(train_dataset, self.state)

    def finish(self) -> None:
        self.state.flush()
        self.log.close()
        notes = []
        if self.state.dropped:
            notes.append(f"rewind: {self.state.dropped} iteration(s) dropped")
        if self.log.dropped:
            notes.append(f"rewind: {self.log.dropped} malformed record(s) not written")
        for note in notes:
            console.print(f"[yellow]{note}[/]")


def _start_rewind(
    *,
    output_dir: Path,
    rows: list,
    optimizer: object,
    batch_size: int,
    grad_accum: int,
    masked: bool,
):
    """Create the rewind log and recorder for an MLX run, or return None.

    ``optimizer.state`` gains the recorder's two keys BEFORE mlx-lm captures it
    for ``mx.compile``; ``Optimizer.init`` only adds keys for parameters, so they
    survive the lazy init inside the first update. ``batch_size`` is per worker.
    Setup can never stop training: any failure is one line and no recorder.
    """
    try:
        import mlx.core as mx

        world = mx.distributed.init().size()
        if world > 1:
            console.print(
                "[yellow]Rewind log off:[/] the recorder is single-process; "
                f"this run has {world} workers"
            )
            return None

        from soup_cli.monitoring.rewind_log import RewindLog, dataset_fingerprint
        from soup_cli.trainer.rewind_mlx import MlxRewindState, make_rewind_loss

        log = RewindLog(
            output_dir / RewindLog.FILENAME,
            backend="mlx",
            task="sft",
            n_rows=len(rows),
            batch_size=batch_size,
            grad_accum=grad_accum,
            dataset_fingerprint=dataset_fingerprint(rows),
        )
        state = MlxRewindState(
            log,
            optimizer_state=optimizer.state,
            batch_size=batch_size,
            grad_accum=grad_accum,
        )
        loss = make_rewind_loss(state, kind="masked" if masked else "span")
    except Exception as exc:  # noqa: BLE001 — a recorder must never stop a run
        from rich.markup import escape

        console.print(
            f"[yellow]Rewind log off:[/] {escape(type(exc).__name__)}: {escape(str(exc))}"
        )
        return None
    console.print(f"[green]Rewind log:[/] {log.path}")
    return _MlxRewind(log, state, loss)


class _GradientClippingOptimizer:
    """An MLX optimizer that clips the global gradient norm before applying it.

    ``training.max_grad_norm`` is honoured by every transformers trainer and
    reached nothing on this backend: no MLX file read it, ``mlx_lm``'s
    ``TrainingArgs`` has no such field, and its trainer clips nowhere -- so the
    same config trained clipped on one backend and unclipped on the other,
    silently.

    Upstream applies gradients through exactly one call,
    ``optimizer.update(model, grad)`` (``mlx_lm/tuner/trainer.py:259``), so
    clipping is reachable by handing ``train()`` an optimizer that clips first,
    without forking the training loop. That call site is inside a function
    compiled with ``mx.compile(inputs=state, outputs=state)``
    (``trainer.py:246-248``); this proxy was measured through that same
    compiled shape rather than assumed to survive tracing.

    Everything other than ``update`` delegates, because upstream reads
    ``optimizer.state`` (``trainer.py:246``) and ``optimizer.learning_rate``
    (``trainer.py:337``) off the object it is given -- including the callable
    schedule, whose ``.step`` counter lives on the wrapped optimizer.
    """

    def __init__(self, inner: object, max_norm: float) -> None:
        # Assigned through __dict__ so __getattr__ cannot recurse on them.
        self.__dict__["_inner"] = inner
        self.__dict__["_max_norm"] = float(max_norm)

    def update(self, model: object, gradients: object) -> object:
        import mlx.optimizers as optim  # heavy: imported at call time

        gradients, _total_norm = optim.clip_grad_norm(gradients, self._max_norm)
        return self._inner.update(model, gradients)

    def __getattr__(self, name: str) -> object:
        return getattr(self.__dict__["_inner"], name)

    def __setattr__(self, name: str, value: object) -> None:
        setattr(self.__dict__["_inner"], name, value)


def _clipping_optimizer(inner: object, max_norm: float) -> object:
    """Wrap ``inner`` so gradients are clipped at ``max_norm`` before they are
    applied. ``max_grad_norm`` is schema-validated ``gt=0``, so there is no
    "clipping off" configuration to represent."""
    return _GradientClippingOptimizer(inner, max_norm)


def mlx_lora_scale(lora_cfg: object) -> float:
    """The LoRA ``scale`` mlx-lm multiplies the adapter update by.

    ``alpha / sqrt(r)`` when ``lora.use_rslora`` is set (rank-stabilised LoRA,
    the value PEFT computes for the transformers backend), ``alpha / r``
    otherwise. One function for both the ``adapter_config.json`` writer and
    ``_apply_lora``, so the file and the trained modules cannot disagree
    (#1853). Before this the flag loaded on ``backend: mlx`` and was never
    read: with the default ``r: 64, alpha: 16`` an rsLoRA run trained at 0.25
    instead of 2.0, an 8x smaller update than the config asked for.
    """
    rank = max(1.0, float(getattr(lora_cfg, "r", 0)))
    alpha = float(getattr(lora_cfg, "alpha", 0.0))
    if getattr(lora_cfg, "use_rslora", False):
        return alpha / math.sqrt(rank)
    return alpha / rank


def build_mlx_adapter_config(lora_cfg: object, *, adapter_path: str, **extra: object) -> dict:
    """The ``lora_parameters`` block, keyed off the RESOLVED module list.

    Separate from the writer so a test can assert the file's ``keys`` against
    :func:`resolve_mlx_target_keys` without running a training loop — the
    disagreement between those two values is the whole of #392.
    """
    rank = int(getattr(lora_cfg, "r", 0))
    config: dict = {
        "fine_tune_type": "dora" if getattr(lora_cfg, "use_dora", False) else "lora",
        "adapter_path": adapter_path,
        "lora_parameters": {
            "rank": rank,
            "scale": mlx_lora_scale(lora_cfg),
            "dropout": float(getattr(lora_cfg, "dropout", 0.0) or 0.0),
            "keys": resolve_mlx_target_keys(lora_cfg),
        },
    }
    config.update(extra)
    return config


def _resolved_batch_size(tcfg: object) -> int:
    """``training.batch_size`` as MLX runs it: anything but an int (``auto``) is 1."""
    value = getattr(tcfg, "batch_size", 1)
    return int(value) if isinstance(value, int) else 1


def scheduled_eval_updates(total_updates: int, every: int) -> list[int]:
    """#1990: the completed optimizer updates an explicit ``eval_steps`` evaluates at.

    Every multiple of ``every`` up to the budget, plus the final update when it
    is not one of them; ``every`` past the budget is just the final update.
    """
    total = max(1, int(total_updates))
    every = max(1, int(every))
    updates = list(range(every, total + 1, every))
    if not updates or updates[-1] != total:
        updates.append(total)
    return updates


def evaluate_all_rows(model, dataset, *, batch_size: int, max_seq_length: int,
                      loss_fn, iterate_fn) -> float:
    """#1990: validation loss over **every** row, token-weighted.

    mlx-lm's own evaluation reads at most ``val_batches`` (25 by default) whole
    batches and drops a tail smaller than one batch. Here the rows are cut into
    consecutive chunks of ``batch_size`` (the last may be shorter) and each chunk
    is one batch, so every supervised token counts once; the result is
    ``sum(loss * tokens) / sum(tokens)``, the reduction mlx-lm uses per batch.

    The batching helpers draw from numpy's global RNG even for a single batch;
    its state is saved and restored, so an evaluation never moves the training
    order (#1850). The model's training mode is restored too, and no gradient
    or optimizer state is touched.
    """
    import mlx.core as mx
    import numpy as np
    from mlx_lm.tuner.datasets import CacheDataset

    class _Rows(CacheDataset):
        def __init__(self, base, indices):  # noqa: D401 — a view, not a cache
            self._base, self._indices = base, list(indices)

        def __len__(self):
            return len(self._indices)

        def __getitem__(self, i):
            return self._base[self._indices[i]]

        def itemlen(self, i):
            return self._base.itemlen(self._indices[i])

    was_training = bool(getattr(model, "training", True))
    rng_state = np.random.get_state()
    total_loss = mx.array(0.0)
    total_tokens = mx.array(0)
    model.eval()
    try:
        n = len(dataset)
        for start in range(0, n, batch_size):
            chunk = _Rows(dataset, range(start, min(n, start + batch_size)))
            for batch in iterate_fn(
                dataset=chunk, batch_size=len(chunk), max_seq_length=max_seq_length
            ):
                losses, tokens = loss_fn(model, *batch)
                total_loss = total_loss + losses * tokens
                total_tokens = total_tokens + tokens
                mx.eval(total_loss, total_tokens)
    finally:
        np.random.set_state(rng_state)
        if was_training:
            model.train()
    tokens = int(total_tokens.item())
    if tokens == 0:
        raise ValueError(
            "MLX backend: the validation split has no supervised token to evaluate "
            "(every row is masked or empty); training.eval_steps needs one"
        )
    return float((total_loss / total_tokens).item())


class MLXSFTTrainerWrapper:
    """High-level wrapper for MLX supervised fine-tuning."""

    def __init__(self, config: SoupConfig, **kwargs) -> None:
        self.config = config
        # Accepted for CLI-contract parity (trainer_kwargs forward). Stored
        # intentionally unused: trust_remote_code has no MLX meaning and
        # load_mlx_model does not take it.
        self.extra_kwargs = kwargs
        self.model = None
        self.tokenizer = None
        self.trainer = None

    def _require_mlx(self) -> None:
        try:
            import mlx  # noqa: F401
            import mlx.core  # noqa: F401
        except ImportError as exc:
            raise ImportError(
                "MLX backend requires the 'mlx' and 'mlx-lm' packages. "
                "Install with: pip install \"soup-cli[mlx]\""
            ) from exc

    def _check_unsupported(self) -> None:
        tcfg = self.config.training
        dcfg = self.config.data
        unsupported = []
        # #683 review: the per-message `train` field has no MLX equivalent --
        # `MaskedChatDataset` supervises every assistant turn and reads no
        # per-message flag. It looked rejected only because the mutual-
        # exclusion validator fires while `train_on_responses_only` is at its
        # `true` default; set that to false and the field was dropped in
        # silence, which is the shape of the defect #683 reports.
        if getattr(dcfg, "train_on_messages_with_train_field", False):
            unsupported.append(
                "data.train_on_messages_with_train_field (MLX supervises "
                "every assistant turn; the per-message flag is not read)"
            )
        # #761: mask_history narrows the transformers label builder
        # (data/loss_mask.py) to the last assistant turn. MLX SFT builds its own
        # mask and never goes through it, so without this line the same soup.yaml
        # trained the last turn on transformers and every turn here, in silence.
        if getattr(dcfg, "mask_history", False):
            unsupported.append(
                "data.mask_history (MLX supervises every assistant turn, not "
                "only the last)"
            )
        if getattr(dcfg, "train_on_prompt", False):
            unsupported.append(
                "data.train_on_prompt (MLX masks the prompt or supervises the "
                "whole sequence; there is no per-field switch)"
            )
        if tcfg.quantization == "8bit":
            unsupported.append("quantization=8bit (use mlx-community 4bit models)")
        if tcfg.use_galore:
            unsupported.append("GaLore")
        if getattr(tcfg, "use_lorafa", False):
            unsupported.append(
                "training.use_lorafa (LoRA-FA has no MLX implementation)"
            )
        if tcfg.use_ring_attention:
            unsupported.append("Ring Attention")
        if tcfg.use_flash_attn:
            unsupported.append("FlashAttention (MLX has its own attention kernels)")
        if tcfg.use_liger:
            unsupported.append(
                "training.use_liger (Liger fused kernels have no MLX implementation)"
            )
        if tcfg.neftune_alpha is not None:
            unsupported.append(
                "training.neftune_alpha (NEFT noise is applied on the "
                "transformers training path, not MLX)"
            )
        if tcfg.use_mod:
            unsupported.append(
                "training.use_mod (Mixture-of-Depths routing is wired on the "
                "transformers path, not MLX)"
            )
        if tcfg.moe_lora:
            unsupported.append(
                "training.moe_lora (ScatterMoE LoRA targets expert layers on the "
                "transformers path; no MLX implementation)"
            )
        if tcfg.quantization_aware:
            unsupported.append(
                "training.quantization_aware (QAT/FP8 prepare runs on the "
                "transformers path, not MLX)"
            )
        if tcfg.use_fsdp2_compile:
            unsupported.append(
                "training.use_fsdp2_compile (torch.compile on FSDP2 requires "
                "CUDA and the transformers backend)"
            )
        if isinstance(tcfg.gradient_checkpointing, str):
            unsupported.append(
                f"gradient_checkpointing tier {tcfg.gradient_checkpointing!r} "
                "(MLX has a single on/off switch; enabling it)"
            )
        if getattr(tcfg, "loss_spike_recovery", False):
            unsupported.append(
                "training.loss_spike_recovery "
                "(MLX does not implement checkpoint rollback or recovery hints)"
            )
        if getattr(tcfg, "grad_accum_auto_tune", False):
            unsupported.append(
                "training.grad_accum_auto_tune "
                "(there is no VRAM total to measure pressure against on unified memory)"
            )
        if unsupported:
            console.print(
                "[yellow]MLX backend ignores: " + ", ".join(unsupported) + "[/]"
            )

    def _check_train_split_size(self, dataset: dict) -> None:
        """#1851: a training split smaller than one batch cannot train at all.

        mlx-lm's ``iterate_batches`` -- like ``masked_iterate_batches`` -- raises
        on a dataset smaller than one batch, which surfaced after the model had
        loaded. Refused here, before the load. (A short *validation* split is
        skipped with a warning in ``train()`` instead.)
        """
        batch_size = _resolved_batch_size(self.config.training)
        train_rows = len(dataset.get("train") or [])
        if 0 < train_rows < batch_size:
            raise ValueError(
                f"MLX backend: the training split has {train_rows} row(s), fewer "
                f"than training.batch_size ({batch_size}), and mlx-lm trains in "
                f"whole batches. Lower training.batch_size to at most {train_rows} "
                "or add rows."
            )

    def _check_eval_steps_has_rows(self, dataset: dict) -> None:
        """#1990: an explicit ``training.eval_steps`` needs validation rows.

        The schema refuses ``val_split: 0`` on a source that cannot carry its
        own split; a source that can (an HF-hub dataset, a cache) is known only
        after loading, so the rows that arrived are checked here, before the
        model loads.
        """
        if self.config.training.eval_steps is None:
            return
        if not (dataset.get("val") or []):
            raise ValueError(
                f"MLX backend: training.eval_steps is "
                f"{self.config.training.eval_steps}, but the validation split is "
                "empty after loading. Set data.val_split above 0, supply a "
                "validation split, or remove training.eval_steps."
            )

    def setup(self, dataset: dict) -> None:
        """Load MLX model, configure LoRA, prepare dataset."""
        self._require_mlx()
        self._check_unsupported()
        self._check_train_split_size(dataset)
        self._check_eval_steps_has_rows(dataset)

        from soup_cli.utils.mlx import load_mlx_model

        cfg = self.config
        console.print(f"[dim]Loading MLX model: {cfg.base}[/]")
        self.model, self.tokenizer = load_mlx_model(
            cfg.base, quantization=cfg.training.quantization
        )
        console.print(
            f"[green]MLX model loaded:[/] {cfg.base} "
            f"(task={cfg.task}, lora_r={cfg.training.lora.r})"
        )
        self._dataset = dataset

    def _apply_lora(self, model) -> None:
        """Convert the configured linear layers to LoRA (mlx-lm >= 0.31).

        The model must be frozen first: ``mlx_lm.load`` does NOT freeze, and
        without it every parameter is trainable and the saved "adapter" is
        actually a full fine-tune (172 tensors instead of ~64 LoRA weights).
        """
        from mlx_lm.tuner.utils import linear_to_lora_layers

        cfg = self.config
        lora_cfg = cfg.training.lora
        target_keys = resolve_mlx_target_keys(lora_cfg)
        if target_keys == MLX_DEFAULT_TARGET_KEYS and lora_cfg.target_modules in (
            None,
            [],
            "auto",
            ["auto"],
        ):
            console.print(
                "[yellow]MLX backend: target_modules: auto cannot be resolved for "
                f"all architectures; defaulting to {target_keys}[/]"
            )
        keys = set(target_keys)
        use_dora = bool(lora_cfg.use_dora)
        if use_dora:
            import mlx.nn as nn

            # Upstream also converts embeddings and switch layers. This first
            # DoRA path qualifies dense linear targets only. Check every target
            # before freezing/converting so a typo or unsupported layer cannot
            # silently train a subset or leave a partly converted model.
            matches = [
                (name, module)
                for parent in [*model.layers, model]
                for name, module in parent.named_modules()
                if name in keys
            ]
            missing = keys - {name for name, _ in matches}
            unsupported = [
                name for name, module in matches
                if not isinstance(module, (nn.Linear, nn.QuantizedLinear))
                or (isinstance(module, nn.QuantizedLinear) and module.bits != 4)
            ]
            if missing or unsupported:
                raise ValueError(
                    "MLX DoRA requires dense linear targets (unquantized or 4-bit); "
                    f"unmatched targets: {sorted(missing)}; "
                    f"unsupported targets: {sorted(set(unsupported))}."
                )
        model.freeze()
        num_layers = len(getattr(model, "layers", []))
        linear_to_lora_layers(
            model,
            num_layers,
            {
                "rank": int(lora_cfg.r),
                "scale": mlx_lora_scale(lora_cfg),
                "dropout": float(getattr(lora_cfg, "dropout", 0.0) or 0.0),
                "keys": keys,
            },
            **({"use_dora": True} if use_dora else {}),
        )

    def _load_checkpoint_weights(self, checkpoint_path: str) -> None:
        """Warm-start LoRA weights from a saved MLX checkpoint (#634).

        Must run after ``_apply_lora`` — the saved file holds only the
        LoRA-shaped tensors, which don't exist on the model until the linear
        layers have been converted.

        This restores adapter WEIGHTS only. mlx-lm's LoRA trainer exposes no
        optimizer state and no step/iteration count, so it is a warm start,
        not a resume of training state: the step count and data position
        both restart from zero regardless of how far the checkpoint got.
        Say so rather than implying a full resume. Replaying the dataset
        from the saved iteration is a separate, harder claim. The order is
        reproducible from training.seed/data_seed since #1850, but mlx-lm
        exposes no way to start its batch iterator part-way, so it is out of
        scope here.

        ``strict=False`` means a checkpoint saved under a different
        ``lora.r`` or ``target_modules`` drops every tensor in silence —
        exactly the #392 failure mode this file's own
        ``resolve_mlx_target_keys`` docstring records. What's checked here
        is the narrower, MLX-independent half of that: the checkpoint FILE
        itself declares at least one tensor before ``load_weights`` ever
        runs, so an empty or corrupt checkpoint fails loudly instead of
        producing a warm start from nothing that still prints two green
        messages. Confirming that the declared tensors actually match this
        model's LoRA-shaped parameter names — the other half — needs mlx
        itself to introspect, which isn't available to verify here.
        """
        tensor_count = _count_safetensors_tensors(checkpoint_path)
        if tensor_count == 0:
            raise ValueError(
                f"MLX checkpoint {checkpoint_path} declares no tensors; refusing "
                "to warm-start from an empty or corrupt checkpoint file"
            )
        if self.config.training.lora.use_dora:
            import mlx.core as mx
            from mlx.utils import tree_flatten

            # load_weights(strict=False) is needed for adapter-only files, but
            # also silently ignores missing/unexpected tensors. Validate the
            # complete DoRA adapter before any model state is changed.
            weights = mx.load(checkpoint_path)
            expected = dict(tree_flatten(self.model.trainable_parameters()))
            if weights.keys() != expected.keys() or any(
                weights[name].shape != expected[name].shape for name in weights.keys() & expected
            ):
                raise ValueError(
                    f"DoRA checkpoint {checkpoint_path} is incompatible with the "
                    "configured adapter targets/rank; all DoRA tensors, including "
                    "magnitudes, must match."
                )
        console.print(
            f"[green]MLX: loading checkpoint weights from[/] {checkpoint_path} "
            f"({tensor_count} tensors)"
        )
        self.model.load_weights(checkpoint_path, strict=False)

    def train(self, display=None, tracker=None, run_id=None, resume_from_checkpoint=None) -> dict:
        """Run MLX training loop via mlx-lm (mlx-lm >= 0.31 API).

        ``display`` / ``tracker`` / ``run_id`` drive Soup's live dashboard the
        way they do on the transformers path (#23). This is an adapter, not a
        reuse of ``SoupTrainerCallback``: that is a HuggingFace
        ``TrainerCallback`` wanting ``args, state, control``, while mlx-lm
        offers only ``on_train_loss_report`` / ``on_val_loss_report``, so
        bridging through it would couple this path to HF trainer internals.
        """

        self._require_mlx()

        from mlx_lm.tuner.callbacks import TrainingCallback
        from mlx_lm.tuner.datasets import CacheDataset, create_dataset
        from mlx_lm.tuner.trainer import TrainingArgs, train  # type: ignore

        cfg = self.config
        output_dir = Path(cfg.output)
        output_dir.mkdir(parents=True, exist_ok=True)

        if self.model is None or self.tokenizer is None:
            raise RuntimeError(
                "MLX backend: setup(dataset) must be called before train()"
            )

        # #1850: mlx-lm's `train()` shuffles through numpy's global RNG
        # (`iterate_batches` is called without a seed) and LoRA `A` init and
        # dropout draw from `mx.random`. The `mlx_lm.lora` CLI seeds both before
        # training; calling `train()` directly seeded neither, so every run
        # trained on a different batch order. `mx.random` is seeded here, before
        # the adapter exists; numpy right before `train()` below, so nothing in
        # between can move the batch order.
        import mlx.core as mx

        from soup_cli.utils.seeding import resolve_training_seed

        seed = resolve_training_seed(cfg.training)
        data_seed = (
            seed if cfg.training.data_seed is None else int(cfg.training.data_seed)
        )
        mx.random.seed(seed)

        self._apply_lora(self.model)

        if resume_from_checkpoint is not None:
            self._load_checkpoint_weights(resume_from_checkpoint)

        batch_size = _resolved_batch_size(cfg.training)
        train_rows = list(self._dataset.get("train", []))
        val_rows = list(self._dataset.get("val", []))
        eval_every = cfg.training.eval_steps
        if 0 < len(val_rows) < batch_size and eval_every is None:
            # #1851: mlx-lm evaluates in whole batches of `batch_size` and raises
            # on a smaller split. Evaluating nothing is the no-val branch below;
            # the warning keeps it from being #1223's silent hold-out.
            console.print(
                f"[yellow]MLX backend: the validation split has {len(val_rows)} "
                f"row(s), fewer than training.batch_size ({batch_size}); mlx-lm "
                "evaluates in whole batches, so it is not evaluated and val_loss "
                "will be absent. Lower batch_size, raise data.val_split, or set "
                "data.val_split: 0 to train on those rows.[/]"
            )
            val_rows = []
        iters = int(
            cfg.training.epochs * max(1, math.ceil(len(train_rows) / batch_size))
        )

        max_seq_length = int(getattr(cfg.data, "max_length", 2048) or 2048)
        steps_per_report = int(getattr(cfg.training, "logging_steps", 10) or 10)
        steps_per_save = int(getattr(cfg.training, "save_steps", 0) or 0)
        if steps_per_save <= 0:
            steps_per_save = iters
        # Real eval cadence when a val split exists; otherwise the value is
        # irrelevant (val_dataset stays None and mlx-lm skips evaluation).
        steps_per_eval = max(1, iters // 4) if val_rows else max(1000, iters + 1)
        # #1990: an explicit eval_steps is honoured by Soup (scheduled_eval_updates
        # below), not by mlx-lm, whose evaluation runs *before* the step at
        # iteration `it`: under accumulation it would miss the update that
        # completes at `it`, its "final" pass precedes the last update, and it
        # reads at most 25 whole batches. Its own cadence is pushed past the run.
        if eval_every is not None:
            steps_per_eval = max(1000, iters + 1)
        # mlx-lm's TrainingArgs.grad_checkpoint is a single bool with no concept
        # of a granularity tier; bool() is the same coercion commands/train.py's
        # hardware-fit predictor and layer_stream.should_enable_hf_gradient_checkpointing
        # already apply to this field on every other backend, so any non-empty
        # tier string ("selective"/"medium"/"full"/"auto") resolves to True here too.
        grad_checkpoint = bool(cfg.training.gradient_checkpointing)
        grad_accumulation_steps = int(cfg.training.gradient_accumulation_steps)
        watchdog = (
            _LossWatchdog(cfg.training.loss_watchdog_threshold,
                          cfg.training.loss_watchdog_patience)
            if cfg.training.loss_watchdog else None
        )
        if watchdog is not None:
            # mlx-lm reports after executing a micro-batch. An enabled watchdog
            # reports every logging_steps *updates*, so a stop never discards a
            # partial accumulation group. The disabled cadence is unchanged.
            steps_per_report *= grad_accumulation_steps
        # mlx-lm updates the optimizer only when it % accum == 0 (trainer.py) and
        # never flushes a partial group, so round iters down to a whole number of
        # groups, keeping at least one group so a small dataset does not train
        # for zero optimizer steps.
        if grad_accumulation_steps > 1:
            iters = max(
                grad_accumulation_steps,
                iters - (iters % grad_accumulation_steps),
            )
        args = TrainingArgs(
            batch_size=batch_size,
            iters=iters,
            max_seq_length=max_seq_length,
            steps_per_report=steps_per_report,
            steps_per_eval=steps_per_eval,
            steps_per_save=steps_per_save,
            adapter_file=str(output_dir / "adapters.safetensors"),
            grad_checkpoint=grad_checkpoint,
            grad_accumulation_steps=grad_accumulation_steps,
        )

        # #683: `data.train_on_responses_only` defaults to True and was reaching
        # nothing here -- no mask was passed, so every MLX SFT run trained on
        # system and user turns against the documented default.
        #
        # The route in is an attribute set rather than a constructor kwarg:
        # upstream reads `getattr(config, "mask_prompt", False)`
        # (`datasets.py:180`) off the args object, and `TrainingArgs` has no
        # such field, so `TrainingArgs(mask_prompt=...)` raises TypeError.
        #
        # But that only gives the right answer for prompt/completion rows.
        # `ChatDataset` masks a single prefix before `messages[-1]`, so on
        # multi-turn chat it supervises the last assistant turn and silently
        # drops the earlier ones -- a different wrong distribution, not a fix.
        # Chat rows therefore go through Soup's own per-token mask, injected
        # via `train(loss=..., iterate_batches=...)`.
        from soup_cli.trainer.mlx_masking import plan_response_masking

        responses_only = bool(getattr(cfg.data, "train_on_responses_only", False))
        plan = plan_response_masking(
            responses_only, train_rows[0] if train_rows else {}
        )
        use_token_mask = plan.token_mask
        args.mask_prompt = plan.mask_prompt
        if plan.warning:
            console.print(f"[yellow]MLX backend ignores: {plan.warning}[/]")

        if use_token_mask:
            from soup_cli.trainer.mlx_masking import (
                MaskedChatDataset,
                ResponseMaskError,
                masked_iterate_batches,
                masked_loss,
            )

            masked_train = MaskedChatDataset(
                train_rows, self.tokenizer, chat_key=_CHAT_KEY
            )
            # Probe row 0 now. `process` is otherwise called lazily by
            # `CacheDataset` from inside `train()`, so a template this cannot
            # mask -- Qwen3's, which injects its thinking block only for the
            # last assistant turn and is therefore not prefix-stable at any
            # earlier one -- surfaced after the model had loaded, LoRA was
            # applied and "Starting training..." had printed. The refusal is
            # correct; its timing was not.
            if train_rows:
                try:
                    masked_train.process(train_rows[0])
                except ResponseMaskError as exc:
                    raise ResponseMaskError(
                        f"{exc}. Set `data.train_on_responses_only: false` to "
                        "train on the full sequence on this model, or run the "
                        "recipe on the transformers backend."
                    ) from exc

            train_dataset = CacheDataset(masked_train)
            val_dataset = (
                CacheDataset(
                    MaskedChatDataset(val_rows, self.tokenizer, chat_key=_CHAT_KEY)
                )
                if val_rows
                else None
            )
            train_hooks = {"loss": masked_loss, "iterate_batches": masked_iterate_batches}
        else:
            train_dataset = CacheDataset(create_dataset(train_rows, self.tokenizer, args))
            val_dataset = (
                CacheDataset(create_dataset(val_rows, self.tokenizer, args))
                if val_rows
                else None
            )
            train_hooks = {}

        # #686: the optimizer was `AdamW(learning_rate=<scalar>)` and nothing
        # else, so `warmup_ratio`, `scheduler`, `weight_decay` and `optimizer`
        # were validated, accepted and dropped -- an MLX run silently trained a
        # different recipe from the configured one.
        #
        # The schedule counts OPTIMIZER UPDATES, not iterations: MLX calls a
        # callable learning_rate with `optimizer.step`, which advances once per
        # `optimizer.update()`, and mlx-lm calls that only every
        # `grad_accumulation_steps` iterations. Building against `iters` would
        # stretch the warmup by that factor and never reach the cosine floor.
        from soup_cli.trainer.mlx_optim import build_optimizer, plan_optimizer

        total_updates = max(1, iters // max(1, grad_accumulation_steps))
        optimizer_plan = plan_optimizer(
            lr=float(cfg.training.lr),
            optimizer=str(getattr(cfg.training, "optimizer", "adamw_torch")),
            scheduler=str(getattr(cfg.training, "scheduler", "cosine")),
            warmup_ratio=float(getattr(cfg.training, "warmup_ratio", 0.0) or 0.0),
            weight_decay=float(getattr(cfg.training, "weight_decay", 0.0) or 0.0),
            total_updates=total_updates,
        )
        for _warning in optimizer_plan.warnings:
            console.print(f"[yellow]MLX backend: {_warning}[/]")
        # #749: mlx-lm never clips, so the optimizer #686 built is wrapped in
        # one that clips the global gradient norm before delegating. Applied
        # here rather than inside build_optimizer so the plan stays a pure
        # description of the schedule and the two fixes stay separable.
        optimizer = _clipping_optimizer(
            build_optimizer(optimizer_plan), float(cfg.training.max_grad_norm)
        )

        captured: dict = {}
        total_epochs = float(cfg.training.epochs)

        class _Callback(TrainingCallback):
            """mlx-lm's two hooks, adapted onto Soup's display and tracker.

            Keys are mlx-lm's own, built at ``mlx_lm/tuner/trainer.py``:
            ``iteration``, ``train_loss``, ``learning_rate``,
            ``iterations_per_second``, ``peak_memory``. ``speed`` reads
            ``iterations_per_second`` and **not** ``tokens_per_second``: the
            display hard-labels that field ``it/s``, so the token figure would
            render as a ~44x overstatement. mlx-lm reports both; only one
            belongs here. There is deliberately no
            ``grad_norm`` — mlx-lm does not compute one for the callback, and a
            dashboard field reading a plausible 0.0 on this backend while
            carrying a real value on another is worse than an absent one.
            """

            # #23: the last MEASURED validation loss, for the panel only.
            # An evaluation happens every `steps_per_eval` iterations, so the
            # panel must keep showing the last one between passes. It must NOT
            # be written to the tracker or the SSE wire on training steps --
            # that fabricates measurements that never happened, which is the
            # defect #713 was blocked on (9 persisted rows for 2 evaluations).
            _sticky_val_loss = None
            #: Last measured training values, carried into the row an
            #: evaluation creates so it reports no number that was not
            #: measured somewhere. `_last_loss` starts as None, not 0.0
            #: (#1225): an evaluation before the first training step has no
            #: loss to carry, and a 0.0 there would become the lowest point of
            #: the run's loss series. The transformers callback records None
            #: at the same point.
            _last_loss = None
            _last_lr = 0.0
            _last_speed = 0.0

            @staticmethod
            def _as_float(value):
                """mlx-lm hands back a Python float (`evaluate()` calls
                `.item()`), but a future build returning an mx scalar must not
                put an unserialisable object on the SSE wire."""
                if value is None:
                    return None
                try:
                    return float(value)
                except (TypeError, ValueError):
                    return None

            def on_val_loss_report(self, val_info: dict) -> None:
                """mlx-lm's validation hook — the #23 checklist item.

                Payload is built at ``mlx_lm/tuner/trainer.py:310-315`` as
                ``{"iteration": it - 1, "val_loss": float, "val_time": float}``.
                Note ``it - 1``: upstream reports validation one behind the
                training counter. That is recorded as sent rather than
                corrected, so a row can be matched to upstream's own log line.
                """
                val_loss = self._as_float(val_info.get("val_loss"))
                if val_loss is None:
                    # A payload without the key is not an evaluation; recording
                    # a None row would be indistinguishable from a real one.
                    return
                type(self)._sticky_val_loss = val_loss

                # NOTE the value this can take: upstream's `it - 1` at
                # `it = 0` makes the initial evaluation report **step -1**, and
                # that negative step is written to the tracker and the wire as
                # sent. Recording upstream's own counter is deliberate -- a row
                # can be matched to its log line -- but it does surface in a
                # table users read, so it is said here rather than discovered.
                step = int(val_info.get("iteration", 0) or 0)
                epoch = (step / iters * total_epochs) if iters else 0.0

                if display is not None:
                    display.update(
                        step=step,
                        epoch=epoch,
                        # None before the first training loss (#1225): the panel shows "-"
                        loss=captured["losses"][-1] if captured.get("losses") else None,
                        lr=cfg.training.lr,
                        val_loss=val_loss,
                    )
                if tracker is not None and run_id:
                    try:
                        # `log_metrics` defaults loss/lr/speed to 0.0, not None,
                        # so a bare val row writes three training columns that
                        # no step measured -- over half the `loss` series at a
                        # realistic cadence. The merged transformers producer
                        # carries `_last_loss` / `_last_lr` into the row an
                        # evaluation creates (callback.py:205-207); this mirrors
                        # it, so the two backends fabricate the same nothing.
                        #
                        # An evaluation before the first training step has no
                        # measured loss to carry yet, so its row records None
                        # (#1225), as the transformers callback does at the
                        # same point.
                        tracker.log_metrics(
                            run_id=run_id,
                            step=step,
                            epoch=epoch,
                            loss=type(self)._last_loss,
                            lr=type(self)._last_lr,
                            speed=type(self)._last_speed,
                            val_loss=val_loss,
                        )
                    except Exception:  # noqa: BLE001 — telemetry must not kill a run
                        pass
                try:
                    from soup_cli.utils.sse_train_stream import TrainEvent
                    from soup_cli.utils.train_event_buffer import push_train_event

                    push_train_event(
                        TrainEvent(
                            type="metric",
                            step=step,
                            epoch=float(epoch),
                            val_loss=val_loss,
                        )
                    )
                except Exception:  # noqa: BLE001 — telemetry must not kill a run
                    pass

            def on_train_loss_report(self, train_info: dict) -> None:
                should_stop = (
                    watchdog.observe(train_info.get("train_loss"))
                    if watchdog is not None else False
                )
                self._report_train_loss(train_info)
                if should_stop:
                    iteration = int(train_info["iteration"])
                    if iteration <= 0 or iteration % grad_accumulation_steps:
                        raise RuntimeError("MLX watchdog report is not at an optimizer boundary")
                    # Verify the native optimizer too, rather than trusting a
                    # callback index if a future mlx-lm changes loop ordering.
                    if int(optimizer.step.item()) != iteration // grad_accumulation_steps:
                        raise RuntimeError("MLX watchdog report precedes its optimizer update")
                    captured["completed_iterations"] = iteration
                    raise _LossWatchdogStop

            def _report_train_loss(self, train_info: dict) -> None:
                loss = train_info.get("train_loss", 0.0)
                captured.setdefault("losses", []).append(loss)
                # Carried for the validation row, which has no training numbers
                # of its own. Recorded BEFORE the display guard on purpose: a
                # run with no display still reaches this hook, and reading them
                # after the early return would leave every carried value at its
                # initial state for exactly the configuration that has no panel
                # to notice. (The same early-return trap that made the SSE test
                # below vacuous.)
                type(self)._last_loss = loss
                type(self)._last_lr = train_info.get("learning_rate", cfg.training.lr)
                type(self)._last_speed = train_info.get("iterations_per_second", 0.0)
                if display is None:
                    return

                step = int(train_info.get("iteration", 0) or 0)
                epoch = (step / iters * total_epochs) if iters else 0.0
                lr_value = train_info.get("learning_rate", cfg.training.lr)
                # TrainingDisplay hard-labels this "it/s" (display.py:116), and
                # the transformers path feeds it train_steps_per_second. Feeding
                # tokens_per_second here renders a ~50x number under an it/s label.
                speed = train_info.get("iterations_per_second", 0.0)
                peak = train_info.get("peak_memory")
                gpu_mem = f"{peak:.3f} GB" if isinstance(peak, (int, float)) else ""

                display.update(
                    step=step,
                    epoch=epoch,
                    loss=loss,
                    lr=lr_value,
                    speed=speed,
                    gpu_mem=gpu_mem,
                    # Sticky, and deliberately only here: the panel keeps the
                    # last measured validation loss between evaluations, while
                    # the tracker and the wire below receive nothing, so no row
                    # or event claims a measurement that did not happen.
                    val_loss=type(self)._sticky_val_loss,
                )
                if tracker is not None and run_id:
                    tracker.log_metrics(
                        run_id=run_id,
                        step=step,
                        epoch=epoch,
                        loss=loss,
                        lr=lr_value,
                        speed=speed,
                        gpu_mem=gpu_mem,
                    )

                # Feed the SSE buffer so `soup ui` and GET /api/train/stream
                # show an MLX run, not just the terminal panel. Best-effort in
                # the same shape as the transformers path: any exception in
                # here must never take down training. grad_norm stays None --
                # mlx-lm does not compute one, and the event field is Optional.
                try:
                    from soup_cli.utils.sse_train_stream import TrainEvent
                    from soup_cli.utils.train_event_buffer import push_train_event

                    push_train_event(
                        TrainEvent(
                            type="metric",
                            step=step,
                            epoch=float(epoch),
                            loss=float(loss) if loss is not None else None,
                            lr=float(lr_value) if lr_value is not None else None,
                            grad_norm=None,
                        )
                    )
                except Exception:  # noqa: BLE001 — telemetry must not kill a run
                    pass

        callback = _Callback()
        eval_log: list = []
        self._eval_log = eval_log
        if eval_every is not None:
            from mlx_lm.tuner.trainer import default_loss
            from mlx_lm.tuner.trainer import iterate_batches as default_iterate

            eval_dataset, val_dataset = val_dataset, None
            eval_loss = train_hooks.get("loss", default_loss)
            eval_iterate = train_hooks.get("iterate_batches", default_iterate)
            accum = max(1, grad_accumulation_steps)
            schedule = scheduled_eval_updates(iters // accum, int(eval_every))
            console.print(
                f"[dim]MLX: evaluating the validation split ({len(val_rows)} rows) "
                f"after optimizer updates {schedule[:6]}{' ...' if len(schedule) > 6 else ''}"
                f" (every {eval_every} update(s) = {int(eval_every) * accum} "
                f"iteration(s), and after the last of {iters // accum})[/]"
            )

            drift_reported: list = []

            def run_eval(update: int) -> None:
                tic = time.time()
                value = evaluate_all_rows(
                    self.model, eval_dataset, batch_size=batch_size,
                    max_seq_length=max_seq_length, loss_fn=eval_loss,
                    iterate_fn=eval_iterate,
                )
                step_state = getattr(optimizer, "state", {}).get("step")
                optimizer_step = int(step_state.item()) if step_state is not None else None
                if optimizer_step not in (None, update) and not drift_reported:
                    # The trigger assumes mlx-lm fetches the batch for
                    # iteration `it` only after `it - 1` ran; a release that
                    # prefetches would move the schedule, so say so, once.
                    drift_reported.append(update)
                    console.print(
                        f"[yellow]MLX: the evaluation for update {update} ran after "
                        f"{optimizer_step} optimizer step(s); this mlx-lm fetches "
                        "batches ahead of the step, so training.eval_steps is "
                        "approximate here.[/]"
                    )
                eval_log.append({
                    "update": update,
                    "optimizer_step": optimizer_step,
                    "val_loss": value,
                })
                callback.on_val_loss_report({
                    "iteration": update * accum, "val_loss": value,
                    "val_time": time.time() - tic,
                })

            def scheduled_iterate(**kwargs):
                """The training stream, evaluating when update N has just
                completed: mlx-lm fetches the batch for iteration `it` after
                iterations 1..it-1 ran, so `consumed` here is completed work."""
                stream = eval_iterate(**kwargs)
                if not kwargs.get("loop"):
                    yield from stream
                    return
                due = set(schedule)
                consumed = 0
                for batch in stream:
                    if consumed and consumed % accum == 0 and consumed // accum in due:
                        run_eval(consumed // accum)
                    yield batch
                    consumed += 1

            train_hooks = {**train_hooks, "iterate_batches": scheduled_iterate}

        if display is not None:
            display.start(iters)

        rewind = None
        if cfg.training.rewind_log:
            rewind = _start_rewind(
                output_dir=output_dir,
                rows=train_rows,
                optimizer=optimizer,
                batch_size=batch_size,
                grad_accum=grad_accumulation_steps,
                masked=use_token_mask,
            )
            if rewind is not None:
                train_dataset = rewind.wrap(train_dataset)
                train_hooks = {**train_hooks, "loss": rewind.loss}

        import numpy as np

        np.random.seed(data_seed)
        t0 = time.time()
        completed_iterations = iters
        stopped = False
        try:
            try:
                train(
                    model=self.model,
                    optimizer=optimizer,
                    train_dataset=train_dataset,
                    val_dataset=val_dataset,
                    args=args,
                    training_callback=callback,
                    **train_hooks,
                )
            except _LossWatchdogStop:
                import mlx.core as mx
                from mlx.utils import tree_flatten

                stopped = True
                completed_iterations = captured["completed_iterations"]
                # Raising out of mlx-lm bypasses its trailing final save. Save
                # precisely the completed state; a save error remains an error.
                if mx.distributed.init().rank() == 0:
                    mx.save_safetensors(
                        args.adapter_file, dict(tree_flatten(self.model.trainable_parameters()))
                    )
            last_update = completed_iterations // max(1, grad_accumulation_steps)
            if eval_every is not None and (not eval_log or eval_log[-1]["update"] != last_update):
                # The last update has no next fetch to trigger it.
                run_eval(last_update)
        finally:
            # The final iteration has no next fetch to flush it, and it is the
            # one that matters most when a non-finite loss ends the run.
            try:
                if rewind is not None:
                    rewind.finish()
            finally:
                # Even a recorder/save failure must release the live display.
                if display is not None:
                    display.stop()
        duration = time.time() - t0
        outcome = ({
            "training_outcome": "stopped" if stopped else "completed",
            "stop_reason": "loss_watchdog" if stopped else "budget_completed",
            "completed_iterations": completed_iterations,
            "completed_updates": last_update,
            "watchdog_report_unit": "optimizer updates",
        } if watchdog is not None else {})

        # mlx-lm's tuner only saves adapters.safetensors; write the
        # adapter_config.json so the output dir is directly loadable with
        # mlx_lm.load(..., adapter_path=output).
        lora_cfg = cfg.training.lora
        (output_dir / "adapter_config.json").write_text(
            json.dumps(
                {
                    **build_mlx_adapter_config(lora_cfg, adapter_path=str(output_dir)),
                    "model": str(cfg.base),
                    "num_layers": len(getattr(self.model, "layers", [])),
                    "batch_size": batch_size,
                    "iters": iters,
                    **outcome,
                    "learning_rate": float(cfg.training.lr),
                    "steps_per_report": steps_per_report,
                    "steps_per_eval": steps_per_eval,
                    # #1990: an explicit interval, in completed optimizer updates.
                    **({"eval_steps": int(eval_every), "eval_unit": "optimizer updates"}
                       if eval_every is not None else {}),
                    "steps_per_save": steps_per_save,
                    "max_seq_length": max_seq_length,
                    # #683: the EFFECTIVE masking, not a hardcoded False.
                    # `mask_prompt` stays upstream's meaning (a single masked
                    # prefix); `response_token_mask` is Soup's per-token mask,
                    # which is what a multi-turn chat run actually used.
                    "mask_prompt": bool(args.mask_prompt),
                    "response_token_mask": use_token_mask,
                    "train_on_responses_only": responses_only,
                    "grad_checkpoint": grad_checkpoint,
                    "grad_accumulation_steps": grad_accumulation_steps,
                    # #686: the EFFECTIVE optimizer and schedule, so an adapter
                    # records the recipe that ran rather than the one requested.
                    **optimizer_plan.as_metadata(),
                    # #749: the norm gradients were actually clipped at.
                    # Recorded because MLX honours it through a Soup-side
                    # wrapper rather than through anything mlx-lm writes, so
                    # the output dir is the only place a finished run says
                    # whether it clipped.
                    "max_grad_norm": float(cfg.training.max_grad_norm),
                },
                indent=2,
            )
        )

        loss_summary = summarize_training_loss(
            [{"loss": loss} for loss in captured.get("losses", [])]
        )
        result = {
            **loss_summary,
            "total_steps": completed_iterations,
            **outcome,
            "duration_secs": duration,
            "duration": f"{duration:.0f}s",
            "output_dir": str(output_dir),
        }
        if eval_log:
            result["val_loss"] = eval_log[-1]["val_loss"]
            result["eval_updates"] = [entry["update"] for entry in eval_log]
        if stopped:
            console.print(
                f"[yellow]Loss watchdog stopped MLX training after {last_update} "
                f"optimizer updates ({completed_iterations} iterations); "
                f"loss exceeded {watchdog.threshold} for {watchdog.patience} "
                f"consecutive reports. Last adapter saved:[/] {output_dir}"
            )
        else:
            console.print(f"[green]MLX training complete:[/] {output_dir}")
        return result