Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

51.8 KB
Raw Blame
"""Knowledge-distillation trainer (v0.53.2 #133).

``task='distill'`` — student model learns from a frozen teacher.

The training loss is a mix of:

* the standard SFT cross-entropy on the student logits, AND
* a token-level divergence loss between student and teacher logits, scaled
  by ``training.distill_temperature`` (T) following Hinton et al. 2015.

Four divergence options (mirroring axolotl's distill plugin):

* ``kl`` (alias of ``forward_kl``) — KL(teacher || student); standard
  distillation.
* ``reverse_kl`` — KL(student || teacher); mode-seeking.
* ``js`` — Jensen-Shannon (symmetric).

The teacher is loaded once, frozen (``requires_grad_(False)``), and
evaluated under ``torch.no_grad()`` to keep VRAM bounded. Student wears
LoRA per the standard PEFT pipeline.
"""

from __future__ import annotations

import logging
import math
import time
from pathlib import Path
from typing import TYPE_CHECKING, Any

from rich.console import Console

from soup_cli.config.schema import SoupConfig
from soup_cli.trainer.loss_summary import summarize_training_loss
from soup_cli.utils.eval_schedule import training_eval_kwargs
from soup_cli.utils.gpu import bf16_fp16_flags, resolve_device_map
from soup_cli.utils.seeding import apply_training_seed, training_seed_kwargs

if TYPE_CHECKING:
    import torch as _torch_typ

from soup_cli.utils.mixed_precision import align_trainable_dtype_for_fp16
from soup_cli.utils.run_provenance import base_revision_kwargs

logger = logging.getLogger(__name__)
console = Console()

# 50/50 CE / distillation blend — matches Hinton et al. 2015.
# Promote to a schema field (distill_ce_weight) in a follow-up patch.
_CE_WEIGHT: float = 0.5
_DISTILL_WEIGHT: float = 1.0 - _CE_WEIGHT


def _compute_distill_term(
    student_logits: "_torch_typ.Tensor",
    teacher_logits: "_torch_typ.Tensor",
    divergence: str,
    temperature: float,
    labels: "_torch_typ.Tensor | None" = None,
    attention_mask: "_torch_typ.Tensor | None" = None,
    chunk_size: int | None = None,
    use_checkpoint: bool = False,
) -> "_torch_typ.Tensor":
    """Pure tensor kernel: divergence between student and teacher logits.

    Both logits are ``(batch, seq, vocab)``. Temperature softens the
    distributions before the divergence is computed (Hinton). The result is
    a scalar mean over the token-level divergences, restricted to the trained
    tokens: ``labels != -100`` when ``labels`` is given (excludes padding AND
    prompt), else ``attention_mask`` (excludes padding), else all positions.

    Supports token chunking and non-reentrant activation checkpointing via
    ``chunk_size`` and ``use_checkpoint`` to drastically reduce the peak
    memory of autograd saved tensors across large vocabularies and sequences (#722).

    Raises:
        TypeError: ``temperature`` not numeric or is bool; ``chunk_size`` not int
            or is bool; ``use_checkpoint`` not bool.
        ValueError: ``temperature`` non-finite or non-positive; ``chunk_size < 1``;
            ``divergence`` outside the supported set.
    """
    import torch

    if isinstance(temperature, bool):
        raise TypeError(f"temperature must not be bool, got {temperature!r}")
    if not isinstance(temperature, (int, float)):
        raise TypeError(
            f"temperature must be float, got {type(temperature).__name__}"
        )
    if not math.isfinite(float(temperature)) or float(temperature) <= 0:
        raise ValueError(
            f"temperature must be finite and positive, got {temperature!r}"
        )
    if divergence not in ("forward_kl", "reverse_kl", "js"):
        raise ValueError(f"Unknown divergence {divergence!r}")
    if chunk_size is not None:
        if isinstance(chunk_size, bool):
            raise TypeError(f"chunk_size must not be bool, got {chunk_size!r}")
        if not isinstance(chunk_size, int):
            raise TypeError(
                f"chunk_size must be int, got {type(chunk_size).__name__}"
            )
        if chunk_size < 1:
            raise ValueError(f"chunk_size must be >= 1, got {chunk_size}")
    if not isinstance(use_checkpoint, bool):
        raise TypeError(
            f"use_checkpoint must be bool, got {type(use_checkpoint).__name__}"
        )

    # Causal-LM alignment: logits at position i predict token i+1, so the CE
    # term shifts (logits[:, :-1] vs labels[:, 1:]). The KD term must shift the
    # SAME way — otherwise the trained-token mask (labels != -100) is applied
    # one position off: it drops each assistant span's first predicted token and
    # leaks the boundary token just before the span. Shift here so both terms
    # measure the same positions.
    if labels is not None or attention_mask is not None:
        student_logits = student_logits[:, :-1, :]
        teacher_logits = teacher_logits[:, :-1, :]
        if labels is not None:
            labels = labels[:, 1:]
        if attention_mask is not None:
            attention_mask = attention_mask[:, 1:]

    # Keep the divergence kernel in FP32. In lower precision, valid logits at
    # low temperatures readily underflow probabilities to zero; target-side
    # derivatives in torch.kl_div then become non-finite even when the reduced
    # loss is finite (#719).
    temp = float(temperature)
    s = student_logits.float() / temp
    t = teacher_logits.float() / temp

    if labels is not None:
        mask = labels != -100
    elif attention_mask is not None:
        mask = attention_mask.bool()
    else:
        mask = None

    if mask is not None:
        if not mask.any():
            return (student_logits.sum() * 0.0).float()
        s_flat = s[mask]
        t_flat = t[mask]
        denom = mask.sum().float()
    else:
        s_flat = s.reshape(-1, s.size(-1))
        t_flat = t.reshape(-1, t.size(-1))
        denom = torch.tensor(s_flat.size(0), dtype=torch.float32, device=s.device)

    def _chunk_kernel(s_c: "_torch_typ.Tensor", t_c: "_torch_typ.Tensor") -> "_torch_typ.Tensor":
        log_s = torch.log_softmax(s_c, dim=-1)
        log_t = torch.log_softmax(t_c, dim=-1)
        if divergence == "forward_kl":
            p_t = log_t.exp()
            return (p_t * (log_t - log_s)).sum()
        if divergence == "reverse_kl":
            p_s = log_s.exp()
            return (p_s * (log_s - log_t)).sum()
        if divergence == "js":
            p_s = log_s.exp()
            p_t = log_t.exp()
            log_m = torch.logaddexp(log_s, log_t) - math.log(2.0)
            kl_pm = (p_s * (log_s - log_m)).sum()
            kl_qm = (p_t * (log_t - log_m)).sum()
            return 0.5 * (kl_pm + kl_qm)
        raise ValueError(f"Unknown divergence {divergence!r}")

    n_tokens = s_flat.size(0)
    c_size = chunk_size if (chunk_size is not None and chunk_size > 0) else n_tokens

    if c_size >= n_tokens and not use_checkpoint:
        total_sum = _chunk_kernel(s_flat, t_flat).float()
    else:
        total_sum = torch.tensor(0.0, device=s.device, dtype=torch.float32)
        if use_checkpoint:
            from torch.utils.checkpoint import checkpoint

            for i in range(0, n_tokens, c_size):
                s_chunk = s_flat[i : i + c_size]
                t_chunk = t_flat[i : i + c_size]
                if t_chunk.requires_grad:
                    chunk_sum = checkpoint(
                        _chunk_kernel, s_chunk, t_chunk, use_reentrant=False
                    )
                else:
                    def _step(s_in: "_torch_typ.Tensor", _t=t_chunk) -> "_torch_typ.Tensor":
                        return _chunk_kernel(s_in, _t)

                    chunk_sum = checkpoint(_step, s_chunk, use_reentrant=False)
                total_sum = total_sum + chunk_sum.float()
        else:
            for i in range(0, n_tokens, c_size):
                s_chunk = s_flat[i : i + c_size]
                t_chunk = t_flat[i : i + c_size]
                chunk_sum = _chunk_kernel(s_chunk, t_chunk)
                total_sum = total_sum + chunk_sum.float()

    loss = (total_sum / denom.float()) * (temp * temp)
    return loss.to(dtype=s.dtype)


def _require_uld_id_compatible_tokenizers(
    strategy: str, student_tokenizer: Any, teacher_tokenizer: Any
) -> None:
    """Refuse ``wasserstein`` / ``topk_align`` on tokenizers that disagree.

    Both strategies forward the student's own ``input_ids`` straight to the
    teacher (see the caller), which is only correct when a given id decodes
    to the same text in both tokenizers. ``wasserstein_aligned`` handles the
    general case by re-tokenizing and aligning; these two do not, so an
    incompatible pair must fail here rather than train on logits the teacher
    computed for the wrong text (#681).

    Reuses :func:`soup_cli.utils.draft.same_tokenizer`, the same probe this
    codebase already trusts to decide whether a draft tokenizer is
    interchangeable with a target one for speculative decoding (#304).
    """
    from soup_cli.utils.draft import same_tokenizer

    if strategy in ("wasserstein", "topk_align") and not same_tokenizer(
        student_tokenizer, teacher_tokenizer
    ):
        raise ValueError(
            f"training.uld_strategy={strategy!r} forwards the student's "
            "token ids to the teacher unchanged, which only holds when the "
            "two tokenizers are interchangeable. This student/teacher pair "
            "is not, so the teacher would be conditioned on the wrong text "
            "with no visible failure (a finite loss either way). Use "
            "training.uld_strategy='wasserstein_aligned', which "
            "re-tokenizes and aligns text for mismatched tokenizers."
        )


class DistillNonfiniteTracker:
    """Tracks consecutive non-finite distillation terms without per-step device sync (#887).

    A broken teacher (e.g. corrupt weights, unsupported dtype underflow, or mismatched
    tokenizer) produces inf/nan logits every step. GradScaler then skips every step,
    allowing training to complete silently while learning nothing.

    This tracker accumulates consecutive non-finite steps directly on device using
    asynchronous PyTorch operations (``torch.isfinite``, ``torch.where``) with zero host
    synchronization on the per-step path. It synchronizes to the host only periodically
    (every ``check_interval`` steps) or at train end, and emits a single actionable
    warning once the consecutive non-finite count reaches ``threshold``.
    """

    def __init__(
        self,
        teacher_name: str = "teacher",
        temperature: float | None = None,
        threshold: int = 3,
        check_interval: int = 5,
        console: Console | None = None,
    ) -> None:
        if isinstance(threshold, bool) or not isinstance(threshold, int) or threshold < 1:
            raise ValueError(f"threshold must be an int >= 1, got {threshold!r}")
        if (
            isinstance(check_interval, bool)
            or not isinstance(check_interval, int)
            or check_interval < 1
        ):
            raise ValueError(f"check_interval must be an int >= 1, got {check_interval!r}")

        self.teacher_name = str(teacher_name or "teacher")
        self.temperature = temperature
        self.threshold = threshold
        self.check_interval = check_interval
        self.console = console
        self._device_counter: _torch_typ.Tensor | None = None
        self._step_count: int = 0
        self._warned: bool = False

    @property
    def warned(self) -> bool:
        """Whether the diagnostic warning has been emitted."""
        return self._warned

    def record_step(self, distill_loss: "_torch_typ.Tensor") -> None:
        """Record a step's distillation loss on device without per-step host sync.

        Accumulates consecutive non-finite steps asynchronously. If the loss is
        finite, the on-device counter is reset to 0 with no host sync.
        Host sync occurs only every ``check_interval`` steps while not yet warned.
        """
        if self._warned:
            return

        import torch

        with torch.no_grad():
            if (
                self._device_counter is None
                or self._device_counter.device != distill_loss.device
            ):
                self._device_counter = torch.zeros(
                    (), dtype=torch.int32, device=distill_loss.device
                )

            is_finite = torch.isfinite(distill_loss).all()
            self._device_counter = torch.where(
                is_finite,
                torch.zeros_like(self._device_counter),
                self._device_counter + 1,
            )

        self._step_count += 1
        if self._step_count % self.check_interval == 0:
            self.check_and_warn()

    def check_and_warn(self) -> bool:
        """Check the on-device counter and emit a diagnostic warning if threshold reached.

        Returns True if a warning was emitted, False otherwise.
        """
        if self._warned or self._device_counter is None:
            return False

        count = int(self._device_counter.item())
        if count >= self.threshold:
            self._warn(count)
            self._warned = True
            return True
        return False

    def _warn(self, consecutive_count: int) -> None:
        import warnings

        from rich.markup import escape
        from rich.panel import Panel

        temp_str = (
            f" (distill_temperature={self.temperature})"
            if self.temperature is not None
            else ""
        )
        teacher_display = escape(str(self.teacher_name))
        msg = (
            f"Distillation teacher {self.teacher_name!r} produced non-finite logits "
            f"({consecutive_count} consecutive non-finite steps detected{temp_str}). "
            "GradScaler will skip optimizer steps and training may learn nothing. "
            "Likely causes:\n"
            "  1. Teacher model saved/loaded in an unsupported dtype (e.g. float16 underflow).\n"
            "  2. Mismatched tokenizer between student and teacher producing garbage logits.\n"
            "  3. training.distill_temperature is set too small."
        )
        logger.warning(
            "Distillation teacher %r produced non-finite logits "
            "(%d consecutive non-finite steps%s). "
            "Likely causes: unsupported dtype underflow, mismatched tokenizer, "
            "or distill_temperature too small.",
            self.teacher_name,
            consecutive_count,
            temp_str,
        )
        warnings.warn(msg, UserWarning, stacklevel=2)

        if self.console is not None:
            try:
                body = (
                    "[bold yellow]Warning: Persistently non-finite distillation loss[/]\n\n"
                    f"Teacher [bold cyan]{teacher_display}[/] produced non-finite logits "
                    f"for [bold red]{consecutive_count}[/] consecutive steps{temp_str}.\n"
                    "Training will skip optimizer steps and may learn nothing.\n\n"
                    "[bold]Likely causes:[/]\n"
                    "  1. Teacher model saved/loaded in an unsupported dtype "
                    "(e.g. float16 underflow)\n"
                    "  2. Mismatched tokenizer between student and teacher producing "
                    "garbage logits\n"
                    "  3. training.distill_temperature is set too small"
                )
                panel = Panel(
                    body,
                    title="[bold red]Distillation Warning[/bold red]",
                    border_style="yellow",
                )
                self.console.print(panel)
            except (RuntimeError, ValueError, TypeError):
                pass


def _map_distill_rows(
    rows: Any,
    *,
    format_row: Any,
    split: str,
    max_length: int,
    trust_source_index: bool = False,
) -> Any:
    """Tokenize distill rows and attach human-facing row number to causal-loss
    target failures (#1242).
    """
    from datasets import Dataset

    from soup_cli.data.loss_mask import (
        NoCausalLossTargetError,
        ensure_causal_loss_target,
    )

    def checked_format_row(example: dict, row_index: int) -> dict:
        try:
            formatted = format_row(example)
            labels = formatted.get("labels")
            if labels is not None:
                ensure_causal_loss_target(labels, max_length=max_length)
            return formatted
        except NoCausalLossTargetError as exc:
            source_idx = example.get("_source_index") if trust_source_index else None
            if source_idx is not None:
                row_num = int(source_idx) + 1
                msg = str(exc)
                if "(generated by the teacher)" not in msg:
                    msg = msg.replace(
                        "the assistant response is",
                        "the assistant response (generated by the teacher) is",
                    )
                raise ValueError(f"{split} row {row_num}: {msg}") from exc
            row_num = row_index + 1
            raise ValueError(f"{split} row {row_num}: {exc}") from exc

    raw_ds = Dataset.from_list(rows) if isinstance(rows, list) else rows
    return raw_ds.map(
        checked_format_row,
        with_indices=True,
        remove_columns=raw_ds.column_names,
    )


class DistillTrainerWrapper:
    """High-level wrapper for student/teacher distillation.

    Mirrors :class:`BCOTrainerWrapper` lifecycle (``__init__`` → ``setup`` →
    ``train``). Teacher loads once in ``setup``; SFT-shaped dataset is
    formatted via the standard ``build_format_row`` factory so the student
    sees ``{input_ids, labels, attention_mask}`` rows. The custom
    ``compute_loss`` injects the distillation term.
    """

    def __init__(
        self,
        config: SoupConfig,
        device: str = "cuda",
        report_to: str = "none",
        deepspeed_config: str | None = None,
        fsdp_config: dict | None = None,
        trust_remote_code: bool = False,
    ) -> None:
        self.config = config
        self.device = device
        self.report_to = report_to
        self.deepspeed_config = deepspeed_config
        self.fsdp_config = fsdp_config
        # Raw user-supplied flag — kept so teacher trust_remote_code can be
        # resolved separately against its own model id during setup().
        self._raw_trust_remote_code = trust_remote_code

        from soup_cli.utils.trust_remote import (
            model_requires_trust_remote_code,
            resolve_trust_remote_code,
        )

        requires = model_requires_trust_remote_code(config.base) or False
        self._trust_remote_code = resolve_trust_remote_code(
            config.base,
            requested=trust_remote_code,
            console=console,
            requires_remote_code=requires,
        )

        self.model: Any = None
        self.teacher: Any = None
        self.tokenizer: Any = None
        self.trainer: Any = None
        self.nonfinite_tracker: DistillNonfiniteTracker | None = None
        self._output_dir: str | None = None

    def setup(self, dataset: dict) -> None:
        """Load student + teacher, build distillation Trainer."""
        from peft import TaskType, get_peft_model
        from transformers import (
            AutoModelForCausalLM,
            AutoTokenizer,
            Trainer,
            TrainingArguments,
        )

        from soup_cli.data.sft_format import build_format_row
        from soup_cli.utils.distill import validate_divergence

        cfg = self.config
        tcfg = cfg.training

        # #353: seed before the model and any adapter are built.
        apply_training_seed(tcfg)

        if tcfg.teacher_model is None:
            raise ValueError(
                "task='distill' requires training.teacher_model to be set"
            )
        divergence = validate_divergence(tcfg.distill_divergence or "forward_kl")
        temperature = float(tcfg.distill_temperature or 2.0)

        # v0.71.12 #145 — sequence-level KD vs token-level logit KD.
        sequence_mode = getattr(tcfg, "distill_mode", "token") == "sequence"
        if sequence_mode and (
            tcfg.uld_strategy is not None or tcfg.minillm_enabled
        ):
            raise ValueError(
                "distill_mode='sequence' is incompatible with uld_strategy / "
                "minillm_enabled (those are token/logit-level distillation). "
                "Sequence-level KD trains the student with plain CE on "
                "teacher-generated text — drop uld_strategy / minillm_enabled."
            )

        console.print(f"[dim]Loading tokenizer (student/teacher shared): {cfg.base}[/]")
        self.tokenizer = AutoTokenizer.from_pretrained(
            cfg.base, trust_remote_code=self._trust_remote_code,
            **base_revision_kwargs(cfg.base),
        )
        if self.tokenizer.pad_token is None:
            self.tokenizer.pad_token = self.tokenizer.eos_token

        console.print(f"[dim]Loading student: {cfg.base}[/]")
        dev_map = resolve_device_map(self.device)
        self.model = AutoModelForCausalLM.from_pretrained(
            cfg.base,
            trust_remote_code=self._trust_remote_code,
            device_map=dev_map,
            **base_revision_kwargs(cfg.base),
        )

        # LoRA on the student — bracket with v0.40.6 #67 surgical PEFT patches.
        from soup_cli.utils.peft_wiring import (
            build_lora_config,
            resolve_lora_target_modules,
        )

        target_modules = resolve_lora_target_modules(self.model, tcfg.lora.target_modules, console)
        # #1151: moe_lora picks the expert-FFN targets; see sft.py.
        from soup_cli.utils.moe import resolve_moe_lora_targets

        target_modules = resolve_moe_lora_targets(
            self.model, tcfg, target_modules, console
        )
        lora_config = build_lora_config(
            tcfg.lora,
            target_modules=target_modules,
            task_type=TaskType.CAUSAL_LM,
        )
        from soup_cli.utils.peft_wiring import (
            apply_post_lora_patches,
            apply_pre_lora_patches,
        )
        apply_pre_lora_patches(self.model, cfg.base)
        self.model = get_peft_model(self.model, lora_config)
        apply_post_lora_patches(self.model)

        # Teacher trust_remote_code resolved INDEPENDENTLY against the teacher
        # model id (code-review HIGH-1 fix — student's resolution must not
        # auto-trust the teacher).
        from soup_cli.utils.trust_remote import (
            model_requires_trust_remote_code as _req,
        )
        from soup_cli.utils.trust_remote import (
            resolve_trust_remote_code as _resolve,
        )

        teacher_requires = _req(tcfg.teacher_model) or False
        teacher_trc = _resolve(
            tcfg.teacher_model,
            requested=self._raw_trust_remote_code,
            console=console,
            requires_remote_code=teacher_requires,
        )

        console.print(f"[dim]Loading teacher (frozen): {tcfg.teacher_model}[/]")
        self.teacher = AutoModelForCausalLM.from_pretrained(
            tcfg.teacher_model,
            trust_remote_code=teacher_trc,
            device_map=dev_map,
        )
        self.teacher.eval()
        for p in self.teacher.parameters():
            p.requires_grad_(False)

        teacher_vocab = getattr(self.teacher.config, "vocab_size", None)
        student_vocab = getattr(self.model.config, "vocab_size", None)
        uld_projection = None
        uld_teacher_tokenizer = None
        minillm_cb = None

        if sequence_mode:
            # v0.71.12 #145 — sequence-level KD. The teacher generates a
            # completion per prompt using ITS OWN tokenizer; the student then
            # trains with plain CE on the re-tokenised teacher output. This
            # works across ANY tokenizer pair, so the vocab-mismatch gate and
            # the token-level ULD / MiniLLM paths are skipped entirely.
            from soup_cli.utils.distill import build_sequence_distill_rows

            teacher_tokenizer = AutoTokenizer.from_pretrained(
                tcfg.teacher_model, trust_remote_code=teacher_trc
            )
            if (
                teacher_tokenizer.pad_token is None
                and teacher_tokenizer.eos_token is not None
            ):
                teacher_tokenizer.pad_token = teacher_tokenizer.eos_token
            seq_budget = min(int(cfg.data.max_length), 256)
            console.print(
                "[green]Sequence-level KD[/] — generating teacher "
                f"completions (max_new_tokens={seq_budget})"
            )
            dataset = dict(dataset)
            raw_train = list(dataset["train"])
            dataset["train"] = build_sequence_distill_rows(
                raw_train,
                self.teacher,
                teacher_tokenizer,
                max_new_tokens=seq_budget,
            )
            kept_train = {
                r["_source_index"]
                for r in dataset["train"]
                if isinstance(r, dict) and "_source_index" in r
            }
            skipped_train = [
                i for i in range(len(raw_train)) if i not in kept_train
            ]
            if skipped_train:
                pos_list = [str(i + 1) for i in skipped_train]
                sample_pos = ", ".join(pos_list[:5])
                if len(pos_list) > 5:
                    sample_pos += ", ..."
                console.print(
                    f"[yellow]Skipped {len(skipped_train)} train row(s) with no prompt turn[/] "
                    f"(positions: {sample_pos})"
                )
            if len(dataset["train"]) == 0:
                pos_desc = f" (positions: {sample_pos})" if skipped_train else ""
                raise ValueError(
                    f"train split has 0 usable rows after skipping {len(skipped_train)} "
                    f"row(s) with no prompt turn{pos_desc}"
                )

            if dataset.get("val"):
                raw_val = list(dataset["val"])
                dataset["val"] = build_sequence_distill_rows(
                    raw_val,
                    self.teacher,
                    teacher_tokenizer,
                    max_new_tokens=seq_budget,
                )
                kept_val = {
                    r["_source_index"]
                    for r in dataset["val"]
                    if isinstance(r, dict) and "_source_index" in r
                }
                skipped_val = [
                    i for i in range(len(raw_val)) if i not in kept_val
                ]
                if skipped_val:
                    pos_list_val = [str(i + 1) for i in skipped_val]
                    sample_pos_val = ", ".join(pos_list_val[:5])
                    if len(pos_list_val) > 5:
                        sample_pos_val += ", ..."
                    console.print(
                        f"[yellow]Skipped {len(skipped_val)} val row(s) with no prompt turn[/] "
                        f"(positions: {sample_pos_val})"
                    )
                if len(dataset["val"]) == 0:
                    pos_desc_val = f" (positions: {sample_pos_val})" if skipped_val else ""
                    raise ValueError(
                        f"val split has 0 usable rows after skipping {len(skipped_val)} "
                        f"row(s) with no prompt turn{pos_desc_val}"
                    )
            # Free the teacher — sequence-level KD does NOT need it in the
            # student loss loop (keeps the 4 GB VRAM budget honest).
            self.teacher = None
            try:
                import torch as _torch

                if _torch.cuda.is_available():
                    _torch.cuda.empty_cache()
            except Exception:  # noqa: BLE001 — best-effort cleanup
                pass
        elif tcfg.uld_strategy is not None:
            # v0.71.11 #236 — cross-tokenizer ULD projection. When uld_strategy
            # is set, a vocab-size mismatch is EXPECTED (that's the whole point)
            # and the ULD loss handles it; otherwise vocab sizes must match so
            # the column-wise KL is well-defined.
            from soup_cli.utils.uld import ULDConfig, build_uld_projection

            uld_projection = build_uld_projection(
                ULDConfig(
                    strategy=tcfg.uld_strategy,
                    student_vocab_size=int(student_vocab or 32000),
                    teacher_vocab_size=int(teacher_vocab or 32000),
                    top_k=tcfg.uld_top_k,
                )
            )
            # v0.71.18 #258 — the aligned strategy forwards the teacher on ITS
            # OWN tokenization of the same text, so it needs the teacher's
            # tokenizer to re-encode + decode per-token strings for alignment.
            if tcfg.uld_strategy == "wasserstein_aligned":
                uld_teacher_tokenizer = AutoTokenizer.from_pretrained(
                    tcfg.teacher_model, trust_remote_code=teacher_trc
                )
                if (
                    uld_teacher_tokenizer.pad_token is None
                    and uld_teacher_tokenizer.eos_token is not None
                ):
                    uld_teacher_tokenizer.pad_token = uld_teacher_tokenizer.eos_token
            else:
                uld_teacher_tokenizer = None
                # #681: wasserstein / topk_align reuse the student's ids as
                # the teacher's, so fail before training starts if this pair
                # can't support that.
                _teacher_tok_for_check = AutoTokenizer.from_pretrained(
                    tcfg.teacher_model, trust_remote_code=teacher_trc
                )
                _require_uld_id_compatible_tokenizers(
                    tcfg.uld_strategy, self.tokenizer, _teacher_tok_for_check
                )
            console.print(
                f"[green]Cross-tokenizer ULD enabled[/] "
                f"(strategy={tcfg.uld_strategy})"
            )
        elif (
            teacher_vocab is not None
            and student_vocab is not None
            and teacher_vocab != student_vocab
        ):
            raise ValueError(
                f"Teacher vocab size ({teacher_vocab}) != student vocab "
                f"size ({student_vocab}). Cross-tokenizer distillation needs "
                "training.uld_strategy (wasserstein / topk_align); use a "
                "teacher that shares the student tokenizer family, or set "
                "uld_strategy."
            )

        # v0.71.11 #237 — MiniLLM on-policy distillation modifier. Skipped in
        # sequence mode (mutually exclusive — rejected earlier in setup).
        if not sequence_mode and tcfg.minillm_enabled:
            from soup_cli.utils.minillm import MiniLLMConfig, build_minillm_callback

            # v0.71.18 #257 — honour an explicit training.minillm_rollout_length
            # when set; otherwise derive a tractable length from max_length,
            # capped at 32 so the growing-sequence autoregressive loop (re-
            # forwards the full prefix each step, ~O(L^2) graph) stays within
            # the consumer-GPU budget.
            if tcfg.minillm_rollout_length is not None:
                rollout_len = int(tcfg.minillm_rollout_length)
            else:
                rollout_len = max(1, min(int(cfg.data.max_length), 32))
            minillm_cb = build_minillm_callback(
                MiniLLMConfig(
                    teacher_mix_ratio=float(tcfg.minillm_teacher_mix_ratio),
                    length_normalize=bool(tcfg.minillm_length_normalize),
                    pretrain_anchor_weight=float(
                        tcfg.minillm_pretrain_anchor_weight
                    ),
                    pretrain_anchor_path=tcfg.minillm_pretrain_anchor_path,
                    on_policy=bool(tcfg.minillm_on_policy),
                    rollout_length=rollout_len,
                ),
                tokenizer=self.tokenizer,
                temperature=temperature,
            )
            mode = "on-policy rollout" if tcfg.minillm_on_policy else "offline blend"
            console.print(
                f"[green]MiniLLM distillation enabled[/] ({mode}, "
                f"teacher_mix={tcfg.minillm_teacher_mix_ratio}, "
                f"anchor={tcfg.minillm_pretrain_anchor_weight})"
            )

        # Dataset prep — reuse the SFT formatter so distill sees
        # {input_ids, labels, attention_mask}. v0.53.2 #137: pass training_cfg
        # so reasoning_effort + train_on_eot are honored on task='distill'.
        format_row = build_format_row(
            tokenizer=self.tokenizer,
            data_cfg=cfg.data,
            console=console,
            training_cfg=tcfg,
        )
        raw_train = dataset["train"]
        train_ds = _map_distill_rows(
            raw_train,
            format_row=format_row,
            split="train",
            max_length=int(cfg.data.max_length),
            trust_source_index=sequence_mode,
        )
        eval_ds = None
        if "val" in dataset and dataset["val"]:
            eval_ds = _map_distill_rows(
                dataset["val"],
                format_row=format_row,
                split="val",
                max_length=int(cfg.data.max_length),
                trust_source_index=sequence_mode,
            )

        output_dir = Path(cfg.output)
        if cfg.experiment_name:
            output_dir = output_dir / cfg.experiment_name
        output_dir.mkdir(parents=True, exist_ok=True)

        batch_size = tcfg.batch_size if tcfg.batch_size != "auto" else 4
        # --- Calculate warmup steps from ratio (#1431) ---
        from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
        from soup_cli.utils.warmup import resolve_trainer_warmup_steps

        warmup_steps = resolve_trainer_warmup_steps(tcfg.warmup_ratio)

        _bf16, _fp16 = bf16_fp16_flags(self.device)
        args = TrainingArguments(
            output_dir=str(output_dir),
            num_train_epochs=tcfg.epochs,
            per_device_train_batch_size=batch_size,
            gradient_accumulation_steps=tcfg.gradient_accumulation_steps,
            learning_rate=tcfg.lr,
            warmup_steps=warmup_steps,
            gradient_checkpointing=should_enable_hf_gradient_checkpointing(
                tcfg.gradient_checkpointing, stream_layers=tcfg.stream_layers
            ),
            weight_decay=tcfg.weight_decay,
            max_grad_norm=tcfg.max_grad_norm,
            optim=tcfg.optimizer,
            lr_scheduler_type=tcfg.scheduler,
            logging_steps=tcfg.logging_steps,
            save_steps=tcfg.save_steps,
            save_total_limit=3,
            bf16=_bf16,
            fp16=_fp16,
            report_to=self.report_to,
            remove_unused_columns=False,
            deepspeed=self.deepspeed_config,
            **training_seed_kwargs(tcfg),
            **training_eval_kwargs(cfg, eval_ds, batch_size=batch_size),
            **(self.fsdp_config or {}),
        )

        teacher_ref = self.teacher
        _uld_projection = uld_projection
        _minillm_cb = minillm_cb
        _sequence_mode = sequence_mode
        # v0.71.18 #257 — when on-policy, the rollout does its own teacher
        # forwards, so the batch-level teacher forward below is skipped.
        _minillm_on_policy = minillm_cb is not None and minillm_cb.config.on_policy
        # v0.71.18 #258 — aligned ULD re-encodes the text with the teacher
        # tokenizer (different vocab + boundaries) and aligns the sequences.
        _uld_aligned = tcfg.uld_strategy == "wasserstein_aligned"
        _uld_teacher_tokenizer = uld_teacher_tokenizer
        _student_tokenizer = self.tokenizer
        _distill_chunk_size = tcfg.distill_chunk_size
        _distill_checkpoint = bool(tcfg.distill_checkpoint)

        # #887: Track persistently non-finite distillation loss without per-step sync.
        nonfinite_tracker = (
            DistillNonfiniteTracker(
                teacher_name=str(tcfg.teacher_model or "teacher"),
                temperature=temperature,
                threshold=3,
                check_interval=5,
                console=console,
            )
            if not sequence_mode
            else None
        )
        self.nonfinite_tracker = nonfinite_tracker

        class _DistillTrainer(Trainer):
            def __init__(
                self, *args, tracker: DistillNonfiniteTracker | None = None, **kwargs
            ):
                super().__init__(*args, **kwargs)
                # compute_loss consumes Trainer's full accumulation-window
                # target count. Keeping this True makes Trainer collect that
                # count and skip its fixed 1 / gradient_accumulation_steps
                # fallback, which would weight unequal microbatches equally.
                self.model_accepts_loss_kwargs = True
                self.nonfinite_tracker = tracker

            def compute_loss(
                self,
                model,
                inputs,
                return_outputs: bool = False,
                num_items_in_batch=None,
            ):
                import torch

                labels = inputs.get("labels")
                outputs = model(**{k: v for k, v in inputs.items() if k != "labels"})
                student_logits = outputs.logits

                def _token_weighted_accumulation(loss):
                    """Turn this microbatch mean into its share of the window mean."""
                    if num_items_in_batch is None or labels is None:
                        return loss
                    local_items = labels[..., 1:].ne(-100).sum().to(
                        device=loss.device, dtype=loss.dtype
                    )
                    if torch.is_tensor(num_items_in_batch):
                        window_items = num_items_in_batch.to(
                            device=loss.device, dtype=loss.dtype
                        )
                    else:
                        window_items = loss.new_tensor(num_items_in_batch)
                    weighted = loss * local_items / window_items.clamp(min=1)

                    # Match Trainer.compute_loss when it gathers one global
                    # token count: DDP averages gradients, so compensate for
                    # that average after normalising by the global denominator.
                    if self.args.average_tokens_across_devices:
                        loss_scale = self.accelerator.num_processes
                        parallelism = getattr(self.accelerator, "parallelism_config", None)
                        if parallelism is not None:
                            loss_scale //= parallelism.tp_size
                        weighted *= loss_scale if self.args.n_gpu <= 1 else self.args.n_gpu
                    return weighted

                ce_loss = torch.tensor(0.0, device=student_logits.device)
                if labels is not None:
                    shift_logits = student_logits[:, :-1, :].contiguous()
                    shift_labels = labels[:, 1:].contiguous()
                    if shift_labels.ne(-100).any():
                        ce_loss = torch.nn.functional.cross_entropy(
                            shift_logits.view(-1, shift_logits.size(-1)),
                            shift_labels.view(-1),
                            ignore_index=-100,
                        )
                    else:
                        ce_loss = (student_logits.sum() * 0.0).to(
                            dtype=student_logits.dtype
                        )

                # v0.71.12 #145 — sequence-level KD trains the student with
                # plain CE on teacher-generated text. The teacher has already
                # been used (and freed) during dataset construction, so there
                # is no teacher forward / logit term here.
                if _sequence_mode:
                    ce_loss = _token_weighted_accumulation(ce_loss)
                    return (ce_loss, outputs) if return_outputs else ce_loss

                # v0.71.18 #257 — true on-policy MiniLLM rollout. Samples a
                # fresh teacher-mixed rollout (doing its own teacher forwards)
                # so the batch-level teacher forward below is skipped entirely.
                if _minillm_on_policy:
                    rollout_loss = _minillm_cb.on_policy_term(
                        model,
                        teacher_ref,
                        inputs["input_ids"],
                        inputs.get("attention_mask"),
                    )
                    _tracker = getattr(self, "nonfinite_tracker", None)
                    if _tracker is not None:
                        _tracker.record_step(rollout_loss)
                    anchor = _minillm_cb.anchor_term(model)
                    total = _CE_WEIGHT * ce_loss + _DISTILL_WEIGHT * rollout_loss
                    if anchor is not None:
                        total = total + anchor
                    total = _token_weighted_accumulation(total)
                    return (total, outputs) if return_outputs else total

                # v0.71.18 #258 — aligned ULD. Decode the student ids to text,
                # re-encode with the teacher tokenizer (different vocab AND
                # boundaries), forward the teacher on its own ids, then align
                # the two token sequences over their decoded character spans.
                if _uld_aligned and _uld_teacher_tokenizer is not None:
                    from soup_cli.utils.uld import uld_aligned_loss

                    student_ids = inputs["input_ids"]
                    s_mask = inputs.get("attention_mask")
                    texts = _student_tokenizer.batch_decode(
                        student_ids, skip_special_tokens=True
                    )
                    t_enc = _uld_teacher_tokenizer(
                        texts,
                        return_tensors="pt",
                        padding=True,
                        truncation=True,
                        max_length=int(student_ids.shape[1]),
                    )
                    try:
                        t_dev = next(teacher_ref.parameters()).device
                    except StopIteration:
                        t_dev = student_logits.device
                    t_ids = t_enc["input_ids"].to(t_dev)
                    t_mask = t_enc["attention_mask"]
                    with torch.no_grad():
                        t_out = teacher_ref(
                            input_ids=t_ids,
                            attention_mask=t_mask.to(t_dev),
                        )
                        aligned_teacher_logits = t_out.logits.to(
                            student_logits.device
                        )
                    # Per-token decoded strings, trimmed to the real (non-pad)
                    # length so pad tokens don't pollute the alignment. Special
                    # tokens decode to "" as on the teacher side (#1426), so
                    # they align to nothing and stay out of the loss.
                    s_strings = []
                    s_ids_list = student_ids.tolist()
                    for bi, row in enumerate(s_ids_list):
                        s_len = (
                            int(s_mask[bi].sum()) if s_mask is not None else len(row)
                        )
                        s_strings.append(
                            [
                                _student_tokenizer.decode(
                                    [int(i)], skip_special_tokens=True
                                )
                                for i in row[:s_len]
                            ]
                        )
                    t_strings = []
                    for bi, row in enumerate(t_ids.tolist()):
                        t_len = int(t_mask[bi].sum())
                        t_strings.append(
                            [
                                _uld_teacher_tokenizer.decode([int(i)])
                                for i in row[:t_len]
                            ]
                        )
                    distill_loss = uld_aligned_loss(
                        student_logits,
                        aligned_teacher_logits,
                        s_strings,
                        t_strings,
                        config=_uld_projection.config,
                        attention_mask=s_mask,
                        labels=labels,
                    )
                    _tracker = getattr(self, "nonfinite_tracker", None)
                    if _tracker is not None:
                        _tracker.record_step(distill_loss)
                    total = _CE_WEIGHT * ce_loss + _DISTILL_WEIGHT * distill_loss
                    total = _token_weighted_accumulation(total)
                    return (total, outputs) if return_outputs else total

                # Bridge devices: HF Trainer may auto-move the student to
                # CUDA while the frozen teacher stays on CPU (or vice versa).
                # Move teacher inputs onto the teacher's device, then move
                # teacher_logits back onto the student's device for the
                # KL kernel.
                try:
                    teacher_device = next(teacher_ref.parameters()).device
                except StopIteration:
                    teacher_device = student_logits.device
                teacher_inputs = {
                    k: (v.to(teacher_device) if hasattr(v, "to") else v)
                    for k, v in inputs.items()
                    if k != "labels"
                }
                with torch.no_grad():
                    teacher_out = teacher_ref(**teacher_inputs)
                    teacher_logits = teacher_out.logits.to(student_logits.device)

                anchor = None
                if _uld_projection is not None:
                    # v0.71.11 #236 — cross-tokenizer ULD distillation loss.
                    # #682: pass labels so masking matches the CE term,
                    # trained tokens only, not every attended position.
                    distill_loss = _uld_projection(
                        student_logits,
                        teacher_logits,
                        attention_mask=inputs.get("attention_mask"),
                        labels=labels,
                    )
                elif _minillm_cb is not None:
                    # v0.71.11 #237 — MiniLLM teacher-mixed reverse-KL +
                    # pretrain anchor.
                    distill_loss = _minillm_cb.distill_term(
                        student_logits, teacher_logits, labels
                    )
                    anchor = _minillm_cb.anchor_term(model)
                else:
                    # Mask padding + prompt tokens so the divergence is measured
                    # only over the completion tokens (parity with the ULD path).
                    distill_loss = _compute_distill_term(
                        student_logits, teacher_logits, divergence, temperature,
                        labels=labels,
                        attention_mask=inputs.get("attention_mask"),
                        chunk_size=_distill_chunk_size,
                        use_checkpoint=_distill_checkpoint,
                    )
                _tracker = getattr(self, "nonfinite_tracker", None)
                if _tracker is not None:
                    _tracker.record_step(distill_loss)
                total = _CE_WEIGHT * ce_loss + _DISTILL_WEIGHT * distill_loss
                if anchor is not None:
                    total = total + anchor
                total = _token_weighted_accumulation(total)
                return (total, outputs) if return_outputs else total

        # ``DataCollatorForSeq2Seq`` pads ``input_ids`` and ``attention_mask``
        # via the tokenizer AND pads ``labels`` with ``label_pad_token_id``
        # (-100 = IGNORE_INDEX). ``DataCollatorForLanguageModeling`` does
        # NOT pad labels — incorrect for our pre-tokenised loss-masked rows.
        from transformers import DataCollatorForSeq2Seq, TrainerCallback

        self.trainer = _DistillTrainer(
            model=self.model,
            args=args,
            train_dataset=train_ds,
            eval_dataset=eval_ds,
            processing_class=self.tokenizer,
            data_collator=DataCollatorForSeq2Seq(
                tokenizer=self.tokenizer,
                label_pad_token_id=-100,
                padding=True,
            ),
            tracker=nonfinite_tracker,
        )

        class _NonfiniteTrackerCallback(TrainerCallback):
            def __init__(self, tracker: DistillNonfiniteTracker | None = None) -> None:
                super().__init__()
                self.tracker = tracker

            def on_train_end(self, args, state, control, **kwargs):
                if self.tracker is not None:
                    self.tracker.check_and_warn()

        self.trainer.add_callback(_NonfiniteTrackerCallback(tracker=nonfinite_tracker))

        # #359 - the same exposure #336 fixed in sft.py: with LoRA the
        # no-decay optimizer group is empty, DeepSpeed drops it, and the LR
        # scheduler keeps two base_lrs until torch's strict zip raises at the
        # first step. The guard prunes inside create_optimizer, i.e. before
        # the scheduler is built. No-op for full fine-tuning, and only under
        # DeepSpeed so the ordinary path keeps its own optimizer.
        if self.deepspeed_config:
            from soup_cli.utils.deepspeed import attach_empty_param_group_guard

            attach_empty_param_group_guard(self.trainer)

        # v0.71.11 #237 — attach the MiniLLM callback for lifecycle (the
        # loss terms are applied directly in compute_loss above).
        if minillm_cb is not None:
            self.trainer.add_callback(minillm_cb)

        # v0.40.6 #67 — ReLoRA callback.
        from soup_cli.utils.peft_wiring import (
            attach_curriculum_callback,
            attach_loraplus_optimizer,
            attach_plugin_callback,
            attach_relora_callback,
        )
        # LoRA+ optimizer (#724/#745) — build and attach now that the trainer exists.
        attach_loraplus_optimizer(self.trainer, tcfg)
        attach_relora_callback(self.trainer, tcfg)
        # v0.53.5 #114/#115 — dynamic curriculum live callback.
        attach_curriculum_callback(self.trainer, tcfg, str(output_dir), console)
        # v0.53.6 #101 — Soup plugin TrainerCallback.
        attach_plugin_callback(self.trainer, console)

        self._output_dir = str(output_dir)
        self._batch_size = batch_size

    def train(
        self,
        display: object | None = None,
        tracker: object | None = None,
        run_id: str = "",
        resume_from_checkpoint: str | None = None,
    ) -> dict:
        if self.trainer is None:
            raise RuntimeError(
                "DistillTrainerWrapper.train() called before setup(). "
                "Call setup(dataset) first."
            )
        start = time.time()
        if display is not None:
            from soup_cli.monitoring.callback import build_soup_trainer_callback

            self.trainer.add_callback(
                build_soup_trainer_callback(
                    display,
                    config=self.config,
                    tracker=tracker,
                    run_id=run_id,
                    batch_size=self._batch_size,
                    output_dir=self._output_dir,
                )
            )
        align_trainable_dtype_for_fp16(
            self.trainer.model,
            fp16=getattr(self.trainer.args, "fp16", False),
            bf16=getattr(self.trainer.args, "bf16", False),
        )
        self.trainer.train(resume_from_checkpoint=resume_from_checkpoint)
        if getattr(self, "nonfinite_tracker", None) is not None:
            self.nonfinite_tracker.check_and_warn()
        duration = time.time() - start

        if self.config.training.relora_steps is None:
            self.trainer.save_model(self._output_dir)
        else:
            from soup_cli.utils.peft_wiring import save_model_with_relora

            save_model_with_relora(
                self.trainer, self._output_dir, self.config.training.relora_steps
            )
        self.tokenizer.save_pretrained(self._output_dir)

        logs = self.trainer.state.log_history
        loss_summary = summarize_training_loss(logs)

        hours = int(duration // 3600)
        minutes = int((duration % 3600) // 60)
        duration_str = f"{hours}h {minutes}m" if hours > 0 else f"{minutes}m"
        return {
            **loss_summary,
            "duration": duration_str,
            "duration_secs": duration,
            "output_dir": self._output_dir,
            "total_steps": self.trainer.state.global_step,
        }