Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Knowledge-distillation trainer (v0.53.2 #133).
``task='distill'`` — student model learns from a frozen teacher.
The training loss is a mix of:
* the standard SFT cross-entropy on the student logits, AND
* a token-level divergence loss between student and teacher logits, scaled
by ``training.distill_temperature`` (T) following Hinton et al. 2015.
Four divergence options (mirroring axolotl's distill plugin):
* ``kl`` (alias of ``forward_kl``) — KL(teacher || student); standard
distillation.
* ``reverse_kl`` — KL(student || teacher); mode-seeking.
* ``js`` — Jensen-Shannon (symmetric).
The teacher is loaded once, frozen (``requires_grad_(False)``), and
evaluated under ``torch.no_grad()`` to keep VRAM bounded. Student wears
LoRA per the standard PEFT pipeline.
"""
from __future__ import annotations
import logging
import math
import time
from pathlib import Path
from typing import TYPE_CHECKING, Any
from rich.console import Console
from soup_cli.config.schema import SoupConfig
from soup_cli.trainer.loss_summary import summarize_training_loss
from soup_cli.utils.eval_schedule import training_eval_kwargs
from soup_cli.utils.gpu import bf16_fp16_flags, resolve_device_map
from soup_cli.utils.seeding import apply_training_seed, training_seed_kwargs
if TYPE_CHECKING:
import torch as _torch_typ
from soup_cli.utils.mixed_precision import align_trainable_dtype_for_fp16
from soup_cli.utils.run_provenance import base_revision_kwargs
logger = logging.getLogger(__name__)
console = Console()
# 50/50 CE / distillation blend — matches Hinton et al. 2015.
# Promote to a schema field (distill_ce_weight) in a follow-up patch.
_CE_WEIGHT: float = 0.5
_DISTILL_WEIGHT: float = 1.0 - _CE_WEIGHT
def _compute_distill_term(
student_logits: "_torch_typ.Tensor",
teacher_logits: "_torch_typ.Tensor",
divergence: str,
temperature: float,
labels: "_torch_typ.Tensor | None" = None,
attention_mask: "_torch_typ.Tensor | None" = None,
chunk_size: int | None = None,
use_checkpoint: bool = False,
) -> "_torch_typ.Tensor":
"""Pure tensor kernel: divergence between student and teacher logits.
Both logits are ``(batch, seq, vocab)``. Temperature softens the
distributions before the divergence is computed (Hinton). The result is
a scalar mean over the token-level divergences, restricted to the trained
tokens: ``labels != -100`` when ``labels`` is given (excludes padding AND
prompt), else ``attention_mask`` (excludes padding), else all positions.
Supports token chunking and non-reentrant activation checkpointing via
``chunk_size`` and ``use_checkpoint`` to drastically reduce the peak
memory of autograd saved tensors across large vocabularies and sequences (#722).
Raises:
TypeError: ``temperature`` not numeric or is bool; ``chunk_size`` not int
or is bool; ``use_checkpoint`` not bool.
ValueError: ``temperature`` non-finite or non-positive; ``chunk_size < 1``;
``divergence`` outside the supported set.
"""
import torch
if isinstance(temperature, bool):
raise TypeError(f"temperature must not be bool, got {temperature!r}")
if not isinstance(temperature, (int, float)):
raise TypeError(
f"temperature must be float, got {type(temperature).__name__}"
)
if not math.isfinite(float(temperature)) or float(temperature) <= 0:
raise ValueError(
f"temperature must be finite and positive, got {temperature!r}"
)
if divergence not in ("forward_kl", "reverse_kl", "js"):
raise ValueError(f"Unknown divergence {divergence!r}")
if chunk_size is not None:
if isinstance(chunk_size, bool):
raise TypeError(f"chunk_size must not be bool, got {chunk_size!r}")
if not isinstance(chunk_size, int):
raise TypeError(
f"chunk_size must be int, got {type(chunk_size).__name__}"
)
if chunk_size < 1:
raise ValueError(f"chunk_size must be >= 1, got {chunk_size}")
if not isinstance(use_checkpoint, bool):
raise TypeError(
f"use_checkpoint must be bool, got {type(use_checkpoint).__name__}"
)
# Causal-LM alignment: logits at position i predict token i+1, so the CE
# term shifts (logits[:, :-1] vs labels[:, 1:]). The KD term must shift the
# SAME way — otherwise the trained-token mask (labels != -100) is applied
# one position off: it drops each assistant span's first predicted token and
# leaks the boundary token just before the span. Shift here so both terms
# measure the same positions.
if labels is not None or attention_mask is not None:
student_logits = student_logits[:, :-1, :]
teacher_logits = teacher_logits[:, :-1, :]
if labels is not None:
labels = labels[:, 1:]
if attention_mask is not None:
attention_mask = attention_mask[:, 1:]
# Keep the divergence kernel in FP32. In lower precision, valid logits at
# low temperatures readily underflow probabilities to zero; target-side
# derivatives in torch.kl_div then become non-finite even when the reduced
# loss is finite (#719).
temp = float(temperature)
s = student_logits.float() / temp
t = teacher_logits.float() / temp
if labels is not None:
mask = labels != -100
elif attention_mask is not None:
mask = attention_mask.bool()
else:
mask = None
if mask is not None:
if not mask.any():
return (student_logits.sum() * 0.0).float()
s_flat = s[mask]
t_flat = t[mask]
denom = mask.sum().float()
else:
s_flat = s.reshape(-1, s.size(-1))
t_flat = t.reshape(-1, t.size(-1))
denom = torch.tensor(s_flat.size(0), dtype=torch.float32, device=s.device)
def _chunk_kernel(s_c: "_torch_typ.Tensor", t_c: "_torch_typ.Tensor") -> "_torch_typ.Tensor":
log_s = torch.log_softmax(s_c, dim=-1)
log_t = torch.log_softmax(t_c, dim=-1)
if divergence == "forward_kl":
p_t = log_t.exp()
return (p_t * (log_t - log_s)).sum()
if divergence == "reverse_kl":
p_s = log_s.exp()
return (p_s * (log_s - log_t)).sum()
if divergence == "js":
p_s = log_s.exp()
p_t = log_t.exp()
log_m = torch.logaddexp(log_s, log_t) - math.log(2.0)
kl_pm = (p_s * (log_s - log_m)).sum()
kl_qm = (p_t * (log_t - log_m)).sum()
return 0.5 * (kl_pm + kl_qm)
raise ValueError(f"Unknown divergence {divergence!r}")
n_tokens = s_flat.size(0)
c_size = chunk_size if (chunk_size is not None and chunk_size > 0) else n_tokens
if c_size >= n_tokens and not use_checkpoint:
total_sum = _chunk_kernel(s_flat, t_flat).float()
else:
total_sum = torch.tensor(0.0, device=s.device, dtype=torch.float32)
if use_checkpoint:
from torch.utils.checkpoint import checkpoint
for i in range(0, n_tokens, c_size):
s_chunk = s_flat[i : i + c_size]
t_chunk = t_flat[i : i + c_size]
if t_chunk.requires_grad:
chunk_sum = checkpoint(
_chunk_kernel, s_chunk, t_chunk, use_reentrant=False
)
else:
def _step(s_in: "_torch_typ.Tensor", _t=t_chunk) -> "_torch_typ.Tensor":
return _chunk_kernel(s_in, _t)
chunk_sum = checkpoint(_step, s_chunk, use_reentrant=False)
total_sum = total_sum + chunk_sum.float()
else:
for i in range(0, n_tokens, c_size):
s_chunk = s_flat[i : i + c_size]
t_chunk = t_flat[i : i + c_size]
chunk_sum = _chunk_kernel(s_chunk, t_chunk)
total_sum = total_sum + chunk_sum.float()
loss = (total_sum / denom.float()) * (temp * temp)
return loss.to(dtype=s.dtype)
def _require_uld_id_compatible_tokenizers(
strategy: str, student_tokenizer: Any, teacher_tokenizer: Any
) -> None:
"""Refuse ``wasserstein`` / ``topk_align`` on tokenizers that disagree.
Both strategies forward the student's own ``input_ids`` straight to the
teacher (see the caller), which is only correct when a given id decodes
to the same text in both tokenizers. ``wasserstein_aligned`` handles the
general case by re-tokenizing and aligning; these two do not, so an
incompatible pair must fail here rather than train on logits the teacher
computed for the wrong text (#681).
Reuses :func:`soup_cli.utils.draft.same_tokenizer`, the same probe this
codebase already trusts to decide whether a draft tokenizer is
interchangeable with a target one for speculative decoding (#304).
"""
from soup_cli.utils.draft import same_tokenizer
if strategy in ("wasserstein", "topk_align") and not same_tokenizer(
student_tokenizer, teacher_tokenizer
):
raise ValueError(
f"training.uld_strategy={strategy!r} forwards the student's "
"token ids to the teacher unchanged, which only holds when the "
"two tokenizers are interchangeable. This student/teacher pair "
"is not, so the teacher would be conditioned on the wrong text "
"with no visible failure (a finite loss either way). Use "
"training.uld_strategy='wasserstein_aligned', which "
"re-tokenizes and aligns text for mismatched tokenizers."
)
class DistillNonfiniteTracker:
"""Tracks consecutive non-finite distillation terms without per-step device sync (#887).
A broken teacher (e.g. corrupt weights, unsupported dtype underflow, or mismatched
tokenizer) produces inf/nan logits every step. GradScaler then skips every step,
allowing training to complete silently while learning nothing.
This tracker accumulates consecutive non-finite steps directly on device using
asynchronous PyTorch operations (``torch.isfinite``, ``torch.where``) with zero host
synchronization on the per-step path. It synchronizes to the host only periodically
(every ``check_interval`` steps) or at train end, and emits a single actionable
warning once the consecutive non-finite count reaches ``threshold``.
"""
def __init__(
self,
teacher_name: str = "teacher",
temperature: float | None = None,
threshold: int = 3,
check_interval: int = 5,
console: Console | None = None,
) -> None:
if isinstance(threshold, bool) or not isinstance(threshold, int) or threshold < 1:
raise ValueError(f"threshold must be an int >= 1, got {threshold!r}")
if (
isinstance(check_interval, bool)
or not isinstance(check_interval, int)
or check_interval < 1
):
raise ValueError(f"check_interval must be an int >= 1, got {check_interval!r}")
self.teacher_name = str(teacher_name or "teacher")
self.temperature = temperature
self.threshold = threshold
self.check_interval = check_interval
self.console = console
self._device_counter: _torch_typ.Tensor | None = None
self._step_count: int = 0
self._warned: bool = False
@property
def warned(self) -> bool:
"""Whether the diagnostic warning has been emitted."""
return self._warned
def record_step(self, distill_loss: "_torch_typ.Tensor") -> None:
"""Record a step's distillation loss on device without per-step host sync.
Accumulates consecutive non-finite steps asynchronously. If the loss is
finite, the on-device counter is reset to 0 with no host sync.
Host sync occurs only every ``check_interval`` steps while not yet warned.
"""
if self._warned:
return
import torch
with torch.no_grad():
if (
self._device_counter is None
or self._device_counter.device != distill_loss.device
):
self._device_counter = torch.zeros(
(), dtype=torch.int32, device=distill_loss.device
)
is_finite = torch.isfinite(distill_loss).all()
self._device_counter = torch.where(
is_finite,
torch.zeros_like(self._device_counter),
self._device_counter + 1,
)
self._step_count += 1
if self._step_count % self.check_interval == 0:
self.check_and_warn()
def check_and_warn(self) -> bool:
"""Check the on-device counter and emit a diagnostic warning if threshold reached.
Returns True if a warning was emitted, False otherwise.
"""
if self._warned or self._device_counter is None:
return False
count = int(self._device_counter.item())
if count >= self.threshold:
self._warn(count)
self._warned = True
return True
return False
def _warn(self, consecutive_count: int) -> None:
import warnings
from rich.markup import escape
from rich.panel import Panel
temp_str = (
f" (distill_temperature={self.temperature})"
if self.temperature is not None
else ""
)
teacher_display = escape(str(self.teacher_name))
msg = (
f"Distillation teacher {self.teacher_name!r} produced non-finite logits "
f"({consecutive_count} consecutive non-finite steps detected{temp_str}). "
"GradScaler will skip optimizer steps and training may learn nothing. "
"Likely causes:\n"
" 1. Teacher model saved/loaded in an unsupported dtype (e.g. float16 underflow).\n"
" 2. Mismatched tokenizer between student and teacher producing garbage logits.\n"
" 3. training.distill_temperature is set too small."
)
logger.warning(
"Distillation teacher %r produced non-finite logits "
"(%d consecutive non-finite steps%s). "
"Likely causes: unsupported dtype underflow, mismatched tokenizer, "
"or distill_temperature too small.",
self.teacher_name,
consecutive_count,
temp_str,
)
warnings.warn(msg, UserWarning, stacklevel=2)
if self.console is not None:
try:
body = (
"[bold yellow]Warning: Persistently non-finite distillation loss[/]\n\n"
f"Teacher [bold cyan]{teacher_display}[/] produced non-finite logits "
f"for [bold red]{consecutive_count}[/] consecutive steps{temp_str}.\n"
"Training will skip optimizer steps and may learn nothing.\n\n"
"[bold]Likely causes:[/]\n"
" 1. Teacher model saved/loaded in an unsupported dtype "
"(e.g. float16 underflow)\n"
" 2. Mismatched tokenizer between student and teacher producing "
"garbage logits\n"
" 3. training.distill_temperature is set too small"
)
panel = Panel(
body,
title="[bold red]Distillation Warning[/bold red]",
border_style="yellow",
)
self.console.print(panel)
except (RuntimeError, ValueError, TypeError):
pass
def _map_distill_rows(
rows: Any,
*,
format_row: Any,
split: str,
max_length: int,
trust_source_index: bool = False,
) -> Any:
"""Tokenize distill rows and attach human-facing row number to causal-loss
target failures (#1242).
"""
from datasets import Dataset
from soup_cli.data.loss_mask import (
NoCausalLossTargetError,
ensure_causal_loss_target,
)
def checked_format_row(example: dict, row_index: int) -> dict:
try:
formatted = format_row(example)
labels = formatted.get("labels")
if labels is not None:
ensure_causal_loss_target(labels, max_length=max_length)
return formatted
except NoCausalLossTargetError as exc:
source_idx = example.get("_source_index") if trust_source_index else None
if source_idx is not None:
row_num = int(source_idx) + 1
msg = str(exc)
if "(generated by the teacher)" not in msg:
msg = msg.replace(
"the assistant response is",
"the assistant response (generated by the teacher) is",
)
raise ValueError(f"{split} row {row_num}: {msg}") from exc
row_num = row_index + 1
raise ValueError(f"{split} row {row_num}: {exc}") from exc
raw_ds = Dataset.from_list(rows) if isinstance(rows, list) else rows
return raw_ds.map(
checked_format_row,
with_indices=True,
remove_columns=raw_ds.column_names,
)
class DistillTrainerWrapper:
"""High-level wrapper for student/teacher distillation.
Mirrors :class:`BCOTrainerWrapper` lifecycle (``__init__`` → ``setup`` →
``train``). Teacher loads once in ``setup``; SFT-shaped dataset is
formatted via the standard ``build_format_row`` factory so the student
sees ``{input_ids, labels, attention_mask}`` rows. The custom
``compute_loss`` injects the distillation term.
"""
def __init__(
self,
config: SoupConfig,
device: str = "cuda",
report_to: str = "none",
deepspeed_config: str | None = None,
fsdp_config: dict | None = None,
trust_remote_code: bool = False,
) -> None:
self.config = config
self.device = device
self.report_to = report_to
self.deepspeed_config = deepspeed_config
self.fsdp_config = fsdp_config
# Raw user-supplied flag — kept so teacher trust_remote_code can be
# resolved separately against its own model id during setup().
self._raw_trust_remote_code = trust_remote_code
from soup_cli.utils.trust_remote import (
model_requires_trust_remote_code,
resolve_trust_remote_code,
)
requires = model_requires_trust_remote_code(config.base) or False
self._trust_remote_code = resolve_trust_remote_code(
config.base,
requested=trust_remote_code,
console=console,
requires_remote_code=requires,
)
self.model: Any = None
self.teacher: Any = None
self.tokenizer: Any = None
self.trainer: Any = None
self.nonfinite_tracker: DistillNonfiniteTracker | None = None
self._output_dir: str | None = None
def setup(self, dataset: dict) -> None:
"""Load student + teacher, build distillation Trainer."""
from peft import TaskType, get_peft_model
from transformers import (
AutoModelForCausalLM,
AutoTokenizer,
Trainer,
TrainingArguments,
)
from soup_cli.data.sft_format import build_format_row
from soup_cli.utils.distill import validate_divergence
cfg = self.config
tcfg = cfg.training
# #353: seed before the model and any adapter are built.
apply_training_seed(tcfg)
if tcfg.teacher_model is None:
raise ValueError(
"task='distill' requires training.teacher_model to be set"
)
divergence = validate_divergence(tcfg.distill_divergence or "forward_kl")
temperature = float(tcfg.distill_temperature or 2.0)
# v0.71.12 #145 — sequence-level KD vs token-level logit KD.
sequence_mode = getattr(tcfg, "distill_mode", "token") == "sequence"
if sequence_mode and (
tcfg.uld_strategy is not None or tcfg.minillm_enabled
):
raise ValueError(
"distill_mode='sequence' is incompatible with uld_strategy / "
"minillm_enabled (those are token/logit-level distillation). "
"Sequence-level KD trains the student with plain CE on "
"teacher-generated text — drop uld_strategy / minillm_enabled."
)
console.print(f"[dim]Loading tokenizer (student/teacher shared): {cfg.base}[/]")
self.tokenizer = AutoTokenizer.from_pretrained(
cfg.base, trust_remote_code=self._trust_remote_code,
**base_revision_kwargs(cfg.base),
)
if self.tokenizer.pad_token is None:
self.tokenizer.pad_token = self.tokenizer.eos_token
console.print(f"[dim]Loading student: {cfg.base}[/]")
dev_map = resolve_device_map(self.device)
self.model = AutoModelForCausalLM.from_pretrained(
cfg.base,
trust_remote_code=self._trust_remote_code,
device_map=dev_map,
**base_revision_kwargs(cfg.base),
)
# LoRA on the student — bracket with v0.40.6 #67 surgical PEFT patches.
from soup_cli.utils.peft_wiring import (
build_lora_config,
resolve_lora_target_modules,
)
target_modules = resolve_lora_target_modules(self.model, tcfg.lora.target_modules, console)
# #1151: moe_lora picks the expert-FFN targets; see sft.py.
from soup_cli.utils.moe import resolve_moe_lora_targets
target_modules = resolve_moe_lora_targets(
self.model, tcfg, target_modules, console
)
lora_config = build_lora_config(
tcfg.lora,
target_modules=target_modules,
task_type=TaskType.CAUSAL_LM,
)
from soup_cli.utils.peft_wiring import (
apply_post_lora_patches,
apply_pre_lora_patches,
)
apply_pre_lora_patches(self.model, cfg.base)
self.model = get_peft_model(self.model, lora_config)
apply_post_lora_patches(self.model)
# Teacher trust_remote_code resolved INDEPENDENTLY against the teacher
# model id (code-review HIGH-1 fix — student's resolution must not
# auto-trust the teacher).
from soup_cli.utils.trust_remote import (
model_requires_trust_remote_code as _req,
)
from soup_cli.utils.trust_remote import (
resolve_trust_remote_code as _resolve,
)
teacher_requires = _req(tcfg.teacher_model) or False
teacher_trc = _resolve(
tcfg.teacher_model,
requested=self._raw_trust_remote_code,
console=console,
requires_remote_code=teacher_requires,
)
console.print(f"[dim]Loading teacher (frozen): {tcfg.teacher_model}[/]")
self.teacher = AutoModelForCausalLM.from_pretrained(
tcfg.teacher_model,
trust_remote_code=teacher_trc,
device_map=dev_map,
)
self.teacher.eval()
for p in self.teacher.parameters():
p.requires_grad_(False)
teacher_vocab = getattr(self.teacher.config, "vocab_size", None)
student_vocab = getattr(self.model.config, "vocab_size", None)
uld_projection = None
uld_teacher_tokenizer = None
minillm_cb = None
if sequence_mode:
# v0.71.12 #145 — sequence-level KD. The teacher generates a
# completion per prompt using ITS OWN tokenizer; the student then
# trains with plain CE on the re-tokenised teacher output. This
# works across ANY tokenizer pair, so the vocab-mismatch gate and
# the token-level ULD / MiniLLM paths are skipped entirely.
from soup_cli.utils.distill import build_sequence_distill_rows
teacher_tokenizer = AutoTokenizer.from_pretrained(
tcfg.teacher_model, trust_remote_code=teacher_trc
)
if (
teacher_tokenizer.pad_token is None
and teacher_tokenizer.eos_token is not None
):
teacher_tokenizer.pad_token = teacher_tokenizer.eos_token
seq_budget = min(int(cfg.data.max_length), 256)
console.print(
"[green]Sequence-level KD[/] — generating teacher "
f"completions (max_new_tokens={seq_budget})"
)
dataset = dict(dataset)
raw_train = list(dataset["train"])
dataset["train"] = build_sequence_distill_rows(
raw_train,
self.teacher,
teacher_tokenizer,
max_new_tokens=seq_budget,
)
kept_train = {
r["_source_index"]
for r in dataset["train"]
if isinstance(r, dict) and "_source_index" in r
}
skipped_train = [
i for i in range(len(raw_train)) if i not in kept_train
]
if skipped_train:
pos_list = [str(i + 1) for i in skipped_train]
sample_pos = ", ".join(pos_list[:5])
if len(pos_list) > 5:
sample_pos += ", ..."
console.print(
f"[yellow]Skipped {len(skipped_train)} train row(s) with no prompt turn[/] "
f"(positions: {sample_pos})"
)
if len(dataset["train"]) == 0:
pos_desc = f" (positions: {sample_pos})" if skipped_train else ""
raise ValueError(
f"train split has 0 usable rows after skipping {len(skipped_train)} "
f"row(s) with no prompt turn{pos_desc}"
)
if dataset.get("val"):
raw_val = list(dataset["val"])
dataset["val"] = build_sequence_distill_rows(
raw_val,
self.teacher,
teacher_tokenizer,
max_new_tokens=seq_budget,
)
kept_val = {
r["_source_index"]
for r in dataset["val"]
if isinstance(r, dict) and "_source_index" in r
}
skipped_val = [
i for i in range(len(raw_val)) if i not in kept_val
]
if skipped_val:
pos_list_val = [str(i + 1) for i in skipped_val]
sample_pos_val = ", ".join(pos_list_val[:5])
if len(pos_list_val) > 5:
sample_pos_val += ", ..."
console.print(
f"[yellow]Skipped {len(skipped_val)} val row(s) with no prompt turn[/] "
f"(positions: {sample_pos_val})"
)
if len(dataset["val"]) == 0:
pos_desc_val = f" (positions: {sample_pos_val})" if skipped_val else ""
raise ValueError(
f"val split has 0 usable rows after skipping {len(skipped_val)} "
f"row(s) with no prompt turn{pos_desc_val}"
)
# Free the teacher — sequence-level KD does NOT need it in the
# student loss loop (keeps the 4 GB VRAM budget honest).
self.teacher = None
try:
import torch as _torch
if _torch.cuda.is_available():
_torch.cuda.empty_cache()
except Exception: # noqa: BLE001 — best-effort cleanup
pass
elif tcfg.uld_strategy is not None:
# v0.71.11 #236 — cross-tokenizer ULD projection. When uld_strategy
# is set, a vocab-size mismatch is EXPECTED (that's the whole point)
# and the ULD loss handles it; otherwise vocab sizes must match so
# the column-wise KL is well-defined.
from soup_cli.utils.uld import ULDConfig, build_uld_projection
uld_projection = build_uld_projection(
ULDConfig(
strategy=tcfg.uld_strategy,
student_vocab_size=int(student_vocab or 32000),
teacher_vocab_size=int(teacher_vocab or 32000),
top_k=tcfg.uld_top_k,
)
)
# v0.71.18 #258 — the aligned strategy forwards the teacher on ITS
# OWN tokenization of the same text, so it needs the teacher's
# tokenizer to re-encode + decode per-token strings for alignment.
if tcfg.uld_strategy == "wasserstein_aligned":
uld_teacher_tokenizer = AutoTokenizer.from_pretrained(
tcfg.teacher_model, trust_remote_code=teacher_trc
)
if (
uld_teacher_tokenizer.pad_token is None
and uld_teacher_tokenizer.eos_token is not None
):
uld_teacher_tokenizer.pad_token = uld_teacher_tokenizer.eos_token
else:
uld_teacher_tokenizer = None
# #681: wasserstein / topk_align reuse the student's ids as
# the teacher's, so fail before training starts if this pair
# can't support that.
_teacher_tok_for_check = AutoTokenizer.from_pretrained(
tcfg.teacher_model, trust_remote_code=teacher_trc
)
_require_uld_id_compatible_tokenizers(
tcfg.uld_strategy, self.tokenizer, _teacher_tok_for_check
)
console.print(
f"[green]Cross-tokenizer ULD enabled[/] "
f"(strategy={tcfg.uld_strategy})"
)
elif (
teacher_vocab is not None
and student_vocab is not None
and teacher_vocab != student_vocab
):
raise ValueError(
f"Teacher vocab size ({teacher_vocab}) != student vocab "
f"size ({student_vocab}). Cross-tokenizer distillation needs "
"training.uld_strategy (wasserstein / topk_align); use a "
"teacher that shares the student tokenizer family, or set "
"uld_strategy."
)
# v0.71.11 #237 — MiniLLM on-policy distillation modifier. Skipped in
# sequence mode (mutually exclusive — rejected earlier in setup).
if not sequence_mode and tcfg.minillm_enabled:
from soup_cli.utils.minillm import MiniLLMConfig, build_minillm_callback
# v0.71.18 #257 — honour an explicit training.minillm_rollout_length
# when set; otherwise derive a tractable length from max_length,
# capped at 32 so the growing-sequence autoregressive loop (re-
# forwards the full prefix each step, ~O(L^2) graph) stays within
# the consumer-GPU budget.
if tcfg.minillm_rollout_length is not None:
rollout_len = int(tcfg.minillm_rollout_length)
else:
rollout_len = max(1, min(int(cfg.data.max_length), 32))
minillm_cb = build_minillm_callback(
MiniLLMConfig(
teacher_mix_ratio=float(tcfg.minillm_teacher_mix_ratio),
length_normalize=bool(tcfg.minillm_length_normalize),
pretrain_anchor_weight=float(
tcfg.minillm_pretrain_anchor_weight
),
pretrain_anchor_path=tcfg.minillm_pretrain_anchor_path,
on_policy=bool(tcfg.minillm_on_policy),
rollout_length=rollout_len,
),
tokenizer=self.tokenizer,
temperature=temperature,
)
mode = "on-policy rollout" if tcfg.minillm_on_policy else "offline blend"
console.print(
f"[green]MiniLLM distillation enabled[/] ({mode}, "
f"teacher_mix={tcfg.minillm_teacher_mix_ratio}, "
f"anchor={tcfg.minillm_pretrain_anchor_weight})"
)
# Dataset prep — reuse the SFT formatter so distill sees
# {input_ids, labels, attention_mask}. v0.53.2 #137: pass training_cfg
# so reasoning_effort + train_on_eot are honored on task='distill'.
format_row = build_format_row(
tokenizer=self.tokenizer,
data_cfg=cfg.data,
console=console,
training_cfg=tcfg,
)
raw_train = dataset["train"]
train_ds = _map_distill_rows(
raw_train,
format_row=format_row,
split="train",
max_length=int(cfg.data.max_length),
trust_source_index=sequence_mode,
)
eval_ds = None
if "val" in dataset and dataset["val"]:
eval_ds = _map_distill_rows(
dataset["val"],
format_row=format_row,
split="val",
max_length=int(cfg.data.max_length),
trust_source_index=sequence_mode,
)
output_dir = Path(cfg.output)
if cfg.experiment_name:
output_dir = output_dir / cfg.experiment_name
output_dir.mkdir(parents=True, exist_ok=True)
batch_size = tcfg.batch_size if tcfg.batch_size != "auto" else 4
# --- Calculate warmup steps from ratio (#1431) ---
from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
from soup_cli.utils.warmup import resolve_trainer_warmup_steps
warmup_steps = resolve_trainer_warmup_steps(tcfg.warmup_ratio)
_bf16, _fp16 = bf16_fp16_flags(self.device)
args = TrainingArguments(
output_dir=str(output_dir),
num_train_epochs=tcfg.epochs,
per_device_train_batch_size=batch_size,
gradient_accumulation_steps=tcfg.gradient_accumulation_steps,
learning_rate=tcfg.lr,
warmup_steps=warmup_steps,
gradient_checkpointing=should_enable_hf_gradient_checkpointing(
tcfg.gradient_checkpointing, stream_layers=tcfg.stream_layers
),
weight_decay=tcfg.weight_decay,
max_grad_norm=tcfg.max_grad_norm,
optim=tcfg.optimizer,
lr_scheduler_type=tcfg.scheduler,
logging_steps=tcfg.logging_steps,
save_steps=tcfg.save_steps,
save_total_limit=3,
bf16=_bf16,
fp16=_fp16,
report_to=self.report_to,
remove_unused_columns=False,
deepspeed=self.deepspeed_config,
**training_seed_kwargs(tcfg),
**training_eval_kwargs(cfg, eval_ds, batch_size=batch_size),
**(self.fsdp_config or {}),
)
teacher_ref = self.teacher
_uld_projection = uld_projection
_minillm_cb = minillm_cb
_sequence_mode = sequence_mode
# v0.71.18 #257 — when on-policy, the rollout does its own teacher
# forwards, so the batch-level teacher forward below is skipped.
_minillm_on_policy = minillm_cb is not None and minillm_cb.config.on_policy
# v0.71.18 #258 — aligned ULD re-encodes the text with the teacher
# tokenizer (different vocab + boundaries) and aligns the sequences.
_uld_aligned = tcfg.uld_strategy == "wasserstein_aligned"
_uld_teacher_tokenizer = uld_teacher_tokenizer
_student_tokenizer = self.tokenizer
_distill_chunk_size = tcfg.distill_chunk_size
_distill_checkpoint = bool(tcfg.distill_checkpoint)
# #887: Track persistently non-finite distillation loss without per-step sync.
nonfinite_tracker = (
DistillNonfiniteTracker(
teacher_name=str(tcfg.teacher_model or "teacher"),
temperature=temperature,
threshold=3,
check_interval=5,
console=console,
)
if not sequence_mode
else None
)
self.nonfinite_tracker = nonfinite_tracker
class _DistillTrainer(Trainer):
def __init__(
self, *args, tracker: DistillNonfiniteTracker | None = None, **kwargs
):
super().__init__(*args, **kwargs)
# compute_loss consumes Trainer's full accumulation-window
# target count. Keeping this True makes Trainer collect that
# count and skip its fixed 1 / gradient_accumulation_steps
# fallback, which would weight unequal microbatches equally.
self.model_accepts_loss_kwargs = True
self.nonfinite_tracker = tracker
def compute_loss(
self,
model,
inputs,
return_outputs: bool = False,
num_items_in_batch=None,
):
import torch
labels = inputs.get("labels")
outputs = model(**{k: v for k, v in inputs.items() if k != "labels"})
student_logits = outputs.logits
def _token_weighted_accumulation(loss):
"""Turn this microbatch mean into its share of the window mean."""
if num_items_in_batch is None or labels is None:
return loss
local_items = labels[..., 1:].ne(-100).sum().to(
device=loss.device, dtype=loss.dtype
)
if torch.is_tensor(num_items_in_batch):
window_items = num_items_in_batch.to(
device=loss.device, dtype=loss.dtype
)
else:
window_items = loss.new_tensor(num_items_in_batch)
weighted = loss * local_items / window_items.clamp(min=1)
# Match Trainer.compute_loss when it gathers one global
# token count: DDP averages gradients, so compensate for
# that average after normalising by the global denominator.
if self.args.average_tokens_across_devices:
loss_scale = self.accelerator.num_processes
parallelism = getattr(self.accelerator, "parallelism_config", None)
if parallelism is not None:
loss_scale //= parallelism.tp_size
weighted *= loss_scale if self.args.n_gpu <= 1 else self.args.n_gpu
return weighted
ce_loss = torch.tensor(0.0, device=student_logits.device)
if labels is not None:
shift_logits = student_logits[:, :-1, :].contiguous()
shift_labels = labels[:, 1:].contiguous()
if shift_labels.ne(-100).any():
ce_loss = torch.nn.functional.cross_entropy(
shift_logits.view(-1, shift_logits.size(-1)),
shift_labels.view(-1),
ignore_index=-100,
)
else:
ce_loss = (student_logits.sum() * 0.0).to(
dtype=student_logits.dtype
)
# v0.71.12 #145 — sequence-level KD trains the student with
# plain CE on teacher-generated text. The teacher has already
# been used (and freed) during dataset construction, so there
# is no teacher forward / logit term here.
if _sequence_mode:
ce_loss = _token_weighted_accumulation(ce_loss)
return (ce_loss, outputs) if return_outputs else ce_loss
# v0.71.18 #257 — true on-policy MiniLLM rollout. Samples a
# fresh teacher-mixed rollout (doing its own teacher forwards)
# so the batch-level teacher forward below is skipped entirely.
if _minillm_on_policy:
rollout_loss = _minillm_cb.on_policy_term(
model,
teacher_ref,
inputs["input_ids"],
inputs.get("attention_mask"),
)
_tracker = getattr(self, "nonfinite_tracker", None)
if _tracker is not None:
_tracker.record_step(rollout_loss)
anchor = _minillm_cb.anchor_term(model)
total = _CE_WEIGHT * ce_loss + _DISTILL_WEIGHT * rollout_loss
if anchor is not None:
total = total + anchor
total = _token_weighted_accumulation(total)
return (total, outputs) if return_outputs else total
# v0.71.18 #258 — aligned ULD. Decode the student ids to text,
# re-encode with the teacher tokenizer (different vocab AND
# boundaries), forward the teacher on its own ids, then align
# the two token sequences over their decoded character spans.
if _uld_aligned and _uld_teacher_tokenizer is not None:
from soup_cli.utils.uld import uld_aligned_loss
student_ids = inputs["input_ids"]
s_mask = inputs.get("attention_mask")
texts = _student_tokenizer.batch_decode(
student_ids, skip_special_tokens=True
)
t_enc = _uld_teacher_tokenizer(
texts,
return_tensors="pt",
padding=True,
truncation=True,
max_length=int(student_ids.shape[1]),
)
try:
t_dev = next(teacher_ref.parameters()).device
except StopIteration:
t_dev = student_logits.device
t_ids = t_enc["input_ids"].to(t_dev)
t_mask = t_enc["attention_mask"]
with torch.no_grad():
t_out = teacher_ref(
input_ids=t_ids,
attention_mask=t_mask.to(t_dev),
)
aligned_teacher_logits = t_out.logits.to(
student_logits.device
)
# Per-token decoded strings, trimmed to the real (non-pad)
# length so pad tokens don't pollute the alignment. Special
# tokens decode to "" as on the teacher side (#1426), so
# they align to nothing and stay out of the loss.
s_strings = []
s_ids_list = student_ids.tolist()
for bi, row in enumerate(s_ids_list):
s_len = (
int(s_mask[bi].sum()) if s_mask is not None else len(row)
)
s_strings.append(
[
_student_tokenizer.decode(
[int(i)], skip_special_tokens=True
)
for i in row[:s_len]
]
)
t_strings = []
for bi, row in enumerate(t_ids.tolist()):
t_len = int(t_mask[bi].sum())
t_strings.append(
[
_uld_teacher_tokenizer.decode([int(i)])
for i in row[:t_len]
]
)
distill_loss = uld_aligned_loss(
student_logits,
aligned_teacher_logits,
s_strings,
t_strings,
config=_uld_projection.config,
attention_mask=s_mask,
labels=labels,
)
_tracker = getattr(self, "nonfinite_tracker", None)
if _tracker is not None:
_tracker.record_step(distill_loss)
total = _CE_WEIGHT * ce_loss + _DISTILL_WEIGHT * distill_loss
total = _token_weighted_accumulation(total)
return (total, outputs) if return_outputs else total
# Bridge devices: HF Trainer may auto-move the student to
# CUDA while the frozen teacher stays on CPU (or vice versa).
# Move teacher inputs onto the teacher's device, then move
# teacher_logits back onto the student's device for the
# KL kernel.
try:
teacher_device = next(teacher_ref.parameters()).device
except StopIteration:
teacher_device = student_logits.device
teacher_inputs = {
k: (v.to(teacher_device) if hasattr(v, "to") else v)
for k, v in inputs.items()
if k != "labels"
}
with torch.no_grad():
teacher_out = teacher_ref(**teacher_inputs)
teacher_logits = teacher_out.logits.to(student_logits.device)
anchor = None
if _uld_projection is not None:
# v0.71.11 #236 — cross-tokenizer ULD distillation loss.
# #682: pass labels so masking matches the CE term,
# trained tokens only, not every attended position.
distill_loss = _uld_projection(
student_logits,
teacher_logits,
attention_mask=inputs.get("attention_mask"),
labels=labels,
)
elif _minillm_cb is not None:
# v0.71.11 #237 — MiniLLM teacher-mixed reverse-KL +
# pretrain anchor.
distill_loss = _minillm_cb.distill_term(
student_logits, teacher_logits, labels
)
anchor = _minillm_cb.anchor_term(model)
else:
# Mask padding + prompt tokens so the divergence is measured
# only over the completion tokens (parity with the ULD path).
distill_loss = _compute_distill_term(
student_logits, teacher_logits, divergence, temperature,
labels=labels,
attention_mask=inputs.get("attention_mask"),
chunk_size=_distill_chunk_size,
use_checkpoint=_distill_checkpoint,
)
_tracker = getattr(self, "nonfinite_tracker", None)
if _tracker is not None:
_tracker.record_step(distill_loss)
total = _CE_WEIGHT * ce_loss + _DISTILL_WEIGHT * distill_loss
if anchor is not None:
total = total + anchor
total = _token_weighted_accumulation(total)
return (total, outputs) if return_outputs else total
# ``DataCollatorForSeq2Seq`` pads ``input_ids`` and ``attention_mask``
# via the tokenizer AND pads ``labels`` with ``label_pad_token_id``
# (-100 = IGNORE_INDEX). ``DataCollatorForLanguageModeling`` does
# NOT pad labels — incorrect for our pre-tokenised loss-masked rows.
from transformers import DataCollatorForSeq2Seq, TrainerCallback
self.trainer = _DistillTrainer(
model=self.model,
args=args,
train_dataset=train_ds,
eval_dataset=eval_ds,
processing_class=self.tokenizer,
data_collator=DataCollatorForSeq2Seq(
tokenizer=self.tokenizer,
label_pad_token_id=-100,
padding=True,
),
tracker=nonfinite_tracker,
)
class _NonfiniteTrackerCallback(TrainerCallback):
def __init__(self, tracker: DistillNonfiniteTracker | None = None) -> None:
super().__init__()
self.tracker = tracker
def on_train_end(self, args, state, control, **kwargs):
if self.tracker is not None:
self.tracker.check_and_warn()
self.trainer.add_callback(_NonfiniteTrackerCallback(tracker=nonfinite_tracker))
# #359 - the same exposure #336 fixed in sft.py: with LoRA the
# no-decay optimizer group is empty, DeepSpeed drops it, and the LR
# scheduler keeps two base_lrs until torch's strict zip raises at the
# first step. The guard prunes inside create_optimizer, i.e. before
# the scheduler is built. No-op for full fine-tuning, and only under
# DeepSpeed so the ordinary path keeps its own optimizer.
if self.deepspeed_config:
from soup_cli.utils.deepspeed import attach_empty_param_group_guard
attach_empty_param_group_guard(self.trainer)
# v0.71.11 #237 — attach the MiniLLM callback for lifecycle (the
# loss terms are applied directly in compute_loss above).
if minillm_cb is not None:
self.trainer.add_callback(minillm_cb)
# v0.40.6 #67 — ReLoRA callback.
from soup_cli.utils.peft_wiring import (
attach_curriculum_callback,
attach_loraplus_optimizer,
attach_plugin_callback,
attach_relora_callback,
)
# LoRA+ optimizer (#724/#745) — build and attach now that the trainer exists.
attach_loraplus_optimizer(self.trainer, tcfg)
attach_relora_callback(self.trainer, tcfg)
# v0.53.5 #114/#115 — dynamic curriculum live callback.
attach_curriculum_callback(self.trainer, tcfg, str(output_dir), console)
# v0.53.6 #101 — Soup plugin TrainerCallback.
attach_plugin_callback(self.trainer, console)
self._output_dir = str(output_dir)
self._batch_size = batch_size
def train(
self,
display: object | None = None,
tracker: object | None = None,
run_id: str = "",
resume_from_checkpoint: str | None = None,
) -> dict:
if self.trainer is None:
raise RuntimeError(
"DistillTrainerWrapper.train() called before setup(). "
"Call setup(dataset) first."
)
start = time.time()
if display is not None:
from soup_cli.monitoring.callback import build_soup_trainer_callback
self.trainer.add_callback(
build_soup_trainer_callback(
display,
config=self.config,
tracker=tracker,
run_id=run_id,
batch_size=self._batch_size,
output_dir=self._output_dir,
)
)
align_trainable_dtype_for_fp16(
self.trainer.model,
fp16=getattr(self.trainer.args, "fp16", False),
bf16=getattr(self.trainer.args, "bf16", False),
)
self.trainer.train(resume_from_checkpoint=resume_from_checkpoint)
if getattr(self, "nonfinite_tracker", None) is not None:
self.nonfinite_tracker.check_and_warn()
duration = time.time() - start
if self.config.training.relora_steps is None:
self.trainer.save_model(self._output_dir)
else:
from soup_cli.utils.peft_wiring import save_model_with_relora
save_model_with_relora(
self.trainer, self._output_dir, self.config.training.relora_steps
)
self.tokenizer.save_pretrained(self._output_dir)
logs = self.trainer.state.log_history
loss_summary = summarize_training_loss(logs)
hours = int(duration // 3600)
minutes = int((duration % 3600) // 60)
duration_str = f"{hours}h {minutes}m" if hours > 0 else f"{minutes}m"
return {
**loss_summary,
"duration": duration_str,
"duration_secs": duration,
"output_dir": self._output_dir,
"total_steps": self.trainer.state.global_step,
}