Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

fix(train): honor gradient checkpointing in distill, classifier, asr and unlearn (#797) (#2140)

Yang Bai99e3e86b79b9
diff --git a/changelog.d/0.76.0/2140.fixed.md b/changelog.d/0.76.0/2140.fixed.md
new file mode 100644
index 00000000..20f6086c
--- /dev/null
+++ b/changelog.d/0.76.0/2140.fixed.md
@@ -0,0 +1 @@
+- Honor `training.gradient_checkpointing` in distill, classifier (`classifier`, `reranker`, `cross_encoder`), ASR and unlearn, which ignored it and always trained with checkpointing off. Distill, classifier and ASR now pass it to their HF training arguments; unlearn, which has its own training loop, turns checkpointing on for its policy model. Every enabled tier is full checkpointing on these tasks, and the default `false` is unchanged. (#797 by @ybai08 in #2140)
diff --git a/docs/performance-and-quantization.md b/docs/performance-and-quantization.md
index 39d4b3c1..6b06ee8e 100644
--- a/docs/performance-and-quantization.md
+++ b/docs/performance-and-quantization.md
@@ -316,6 +316,14 @@ forward this setting to their TRL/HF training arguments: `false` disables it
and `true` enables it, including unquantized runs (`quantization: none`). These
wrappers use full checkpointing for any enabled tier.
+Distillation (`task: distill`), the classifier wrapper (`classifier`,
+`reranker`, `cross_encoder`) and Whisper ASR (`asr`) forward it to their HF
+training arguments the same way, and unlearning (`task: unlearn`), which trains
+in its own loop rather than through a `Trainer`, turns checkpointing on for its
+LoRA policy model when the setting is enabled. Before, these four ignored the
+setting and always trained with checkpointing off. Every enabled tier is full
+checkpointing on them, and the default `false` keeps it off.
+
## Kernel Auto-Composition
diff --git a/docs/training.md b/docs/training.md
index 00a2b8f4..9047551a 100644
--- a/docs/training.md
+++ b/docs/training.md
@@ -595,7 +595,8 @@ soup eval unlearning <run-id> --benchmark tofu --evidence evidence.json --output
`task: unlearn` is live (v0.71.9): it loads a LoRA-wrapped policy, a frozen reference copy (NPO / RMU), and the forget / retain JSONL sets, then optimises the per-method loss — NPO's `(2/β)·mean(-logσ(-β·(π_logp − ref_logp)))` drives the policy's forget-set log-prob below the reference (= forgetting), while the retain set anchors capability. Run NPO/SimNPO **with** a `retain_set` — without one the policy has no utility anchor and Soup warns loudly.
Unlearning honors `training.optimizer`, `scheduler`, `warmup_ratio`, `weight_decay`,
-`max_grad_norm`, `batch_size` and `gradient_accumulation_steps`. The default
+`max_grad_norm`, `batch_size`, `gradient_accumulation_steps` and
+`gradient_checkpointing` (any enabled tier checkpoints the whole policy model). The default
`batch_size: auto` resolves to 1 for unlearning rather than to the memory estimate
the other trainers use, so a config makes the same number of updates on any card. This
memory-constrained loop processes one example at a time and accumulates gradients
diff --git a/src/soup_cli/trainer/asr.py b/src/soup_cli/trainer/asr.py
index c809295b..f08c1142 100644
--- a/src/soup_cli/trainer/asr.py
+++ b/src/soup_cli/trainer/asr.py
@@ -322,6 +322,7 @@ class AsrTrainerWrapper:
batch_size = tcfg.batch_size if tcfg.batch_size != "auto" else 8
# --- Calculate warmup steps from ratio (#1431) ---
+ from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
from soup_cli.utils.warmup import resolve_trainer_warmup_steps
warmup_steps = resolve_trainer_warmup_steps(tcfg.warmup_ratio)
@@ -342,6 +343,9 @@ class AsrTrainerWrapper:
gradient_accumulation_steps=tcfg.gradient_accumulation_steps,
learning_rate=tcfg.lr,
warmup_steps=warmup_steps,
+ gradient_checkpointing=should_enable_hf_gradient_checkpointing(
+ tcfg.gradient_checkpointing, stream_layers=tcfg.stream_layers
+ ),
weight_decay=tcfg.weight_decay,
max_grad_norm=tcfg.max_grad_norm,
optim=tcfg.optimizer,
diff --git a/src/soup_cli/trainer/classifier.py b/src/soup_cli/trainer/classifier.py
index 4b021975..d132817d 100644
--- a/src/soup_cli/trainer/classifier.py
+++ b/src/soup_cli/trainer/classifier.py
@@ -395,6 +395,7 @@ class ClassifierTrainerWrapper:
batch_size = tcfg.batch_size if tcfg.batch_size != "auto" else 8
# --- Calculate warmup steps from ratio (#1431) ---
+ from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
from soup_cli.utils.warmup import resolve_trainer_warmup_steps
warmup_steps = resolve_trainer_warmup_steps(tcfg.warmup_ratio)
@@ -407,6 +408,9 @@ class ClassifierTrainerWrapper:
gradient_accumulation_steps=tcfg.gradient_accumulation_steps,
learning_rate=tcfg.lr,
warmup_steps=warmup_steps,
+ gradient_checkpointing=should_enable_hf_gradient_checkpointing(
+ tcfg.gradient_checkpointing, stream_layers=tcfg.stream_layers
+ ),
weight_decay=tcfg.weight_decay,
max_grad_norm=tcfg.max_grad_norm,
optim=tcfg.optimizer,
diff --git a/src/soup_cli/trainer/distill.py b/src/soup_cli/trainer/distill.py
index d24437af..608da2a9 100644
--- a/src/soup_cli/trainer/distill.py
+++ b/src/soup_cli/trainer/distill.py
@@ -807,6 +807,7 @@ class DistillTrainerWrapper:
batch_size = tcfg.batch_size if tcfg.batch_size != "auto" else 4
# --- Calculate warmup steps from ratio (#1431) ---
+ from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
from soup_cli.utils.warmup import resolve_trainer_warmup_steps
warmup_steps = resolve_trainer_warmup_steps(tcfg.warmup_ratio)
@@ -819,6 +820,9 @@ class DistillTrainerWrapper:
gradient_accumulation_steps=tcfg.gradient_accumulation_steps,
learning_rate=tcfg.lr,
warmup_steps=warmup_steps,
+ gradient_checkpointing=should_enable_hf_gradient_checkpointing(
+ tcfg.gradient_checkpointing, stream_layers=tcfg.stream_layers
+ ),
weight_decay=tcfg.weight_decay,
max_grad_norm=tcfg.max_grad_norm,
optim=tcfg.optimizer,
diff --git a/src/soup_cli/trainer/unlearn.py b/src/soup_cli/trainer/unlearn.py
index 7fbf2919..ac72f450 100644
--- a/src/soup_cli/trainer/unlearn.py
+++ b/src/soup_cli/trainer/unlearn.py
@@ -213,6 +213,17 @@ class UnlearnTrainerWrapper:
self.model = get_peft_model(self.model, lora_cfg)
self.model.train()
+ # This loop has no Trainer, so nothing else switches checkpointing on.
+ # Do what Trainer.train() does for the other wrappers (#797): every
+ # enabled tier is full checkpointing, with Transformers' default
+ # (non-reentrant) kwargs, and HF enables input grads for the frozen base.
+ from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
+
+ if should_enable_hf_gradient_checkpointing(
+ tcfg.gradient_checkpointing, stream_layers=tcfg.stream_layers
+ ):
+ self.model.gradient_checkpointing_enable()
+
# NPO + RMU need a frozen reference copy of the base.
if method in ("npo", "rmu"):
ref, _, _ = load_model_and_tokenizer(
diff --git a/tests/test_distill_classifier_asr_unlearn_checkpointing.py b/tests/test_distill_classifier_asr_unlearn_checkpointing.py
new file mode 100644
index 00000000..e80e4073
--- /dev/null
+++ b/tests/test_distill_classifier_asr_unlearn_checkpointing.py
@@ -0,0 +1,310 @@
+"""#797, item 3: distill, classifier, asr and unlearn honour the YAML setting.
+
+These four wrappers never read ``training.gradient_checkpointing``, so ``true`` and
+every tier did nothing. Each test drives the real ``setup`` on CPU: a tiny random
+Llama (``tests/_tiny_hf_models.py``) or a tiny random Whisper built here, a real
+tokenizer or processor, PEFT where the task uses it, and the installed HF
+``Trainer``. Only the audio decode is replaced, so the ASR rows need no
+``soundfile``.
+
+The checks are the argument the HF trainer holds, the model's
+``is_gradient_checkpointing`` after one real optimizer step, and that a decoder
+layer runs again in the backward pass (the recompute is what checkpointing is).
+``unlearn`` trains in its own loop with no ``Trainer``, so it is checked through
+``setup`` and one step of that loop.
+"""
+
+from __future__ import annotations
+
+import importlib
+import json
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from soup_cli.config.loader import load_config_from_string
+from tests._tiny_hf_models import tiny_model_dir
+
+_TIERS = ("selective", "medium", "full", "auto")
+_STATES = pytest.mark.parametrize(
+ "checkpointing", [False, True], ids=["disabled", "enabled"]
+)
+
+_WRAPPERS = {
+ "distill": ("soup_cli.trainer.distill", "DistillTrainerWrapper"),
+ "classifier": ("soup_cli.trainer.classifier", "ClassifierTrainerWrapper"),
+ "asr": ("soup_cli.trainer.asr", "AsrTrainerWrapper"),
+}
+
+
[email protected](autouse=True)
+def train_stack(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+ for name in ("torch", "transformers", "peft", "datasets"):
+ pytest.importorskip(name, reason=f"{name} requires the train extra")
+ import torch
+ from transformers import TrainingArguments
+
+ threads = torch.get_num_threads()
+ torch.set_num_threads(1)
+ # ACCELERATE_USE_CPU alone still lets HF pick MPS on a Mac; set the public flag
+ # before HF initialises, without touching the checkpointing value.
+ real_post_init = TrainingArguments.__post_init__
+
+ def cpu_post_init(args: TrainingArguments) -> None:
+ args.use_cpu = True
+ real_post_init(args)
+
+ monkeypatch.setattr(TrainingArguments, "__post_init__", cpu_post_init)
+ yield
+ torch.set_num_threads(threads)
+
+
+def _tiny_whisper_dir(root: Path) -> str:
+ """A random Whisper small enough to train a step on CPU, with no download.
+
+ One-second feature windows (100 mel frames, 50 encoder positions) and a
+ byte-level tokenizer carrying the special tokens Whisper's prefix needs.
+ """
+ import torch
+ from transformers import (
+ WhisperConfig,
+ WhisperFeatureExtractor,
+ WhisperForConditionalGeneration,
+ WhisperProcessor,
+ WhisperTokenizer,
+ )
+ from transformers.convert_slow_tokenizer import bytes_to_unicode
+
+ vocab = {char: index for index, char in enumerate(bytes_to_unicode().values())}
+ tokenizer = WhisperTokenizer(vocab=vocab, merges=[], pad_token="<|endoftext|>")
+ tokenizer.add_special_tokens({
+ "additional_special_tokens": [
+ "<|endoftext|>",
+ "<|startoftranscript|>",
+ "<|en|>",
+ "<|translate|>",
+ "<|transcribe|>",
+ "<|notimestamps|>",
+ ]
+ })
+ target = root / "whisper"
+ processor = WhisperProcessor(
+ feature_extractor=WhisperFeatureExtractor(feature_size=8, chunk_length=1),
+ tokenizer=tokenizer,
+ )
+ processor.save_pretrained(str(target))
+ # Reload so the special-token ids are the ones the wrapper will see.
+ tokenizer = WhisperProcessor.from_pretrained(str(target)).tokenizer
+ start = tokenizer.convert_tokens_to_ids("<|startoftranscript|>")
+ config = WhisperConfig(
+ vocab_size=len(tokenizer),
+ num_mel_bins=8,
+ d_model=16,
+ encoder_layers=1,
+ decoder_layers=1,
+ encoder_attention_heads=2,
+ decoder_attention_heads=2,
+ encoder_ffn_dim=32,
+ decoder_ffn_dim=32,
+ max_source_positions=50,
+ max_target_positions=32,
+ pad_token_id=tokenizer.eos_token_id,
+ bos_token_id=tokenizer.eos_token_id,
+ eos_token_id=tokenizer.eos_token_id,
+ decoder_start_token_id=start,
+ )
+ torch.manual_seed(7)
+ WhisperForConditionalGeneration(config).save_pretrained(str(target))
+ return str(target)
+
+
+def _config(task: str, base: str, out: Path, checkpointing: bool | str, **training: Any):
+ import yaml
+
+ body: dict[str, Any] = {
+ "quantization": "none",
+ "gradient_checkpointing": checkpointing,
+ "batch_size": 2,
+ "gradient_accumulation_steps": 1,
+ "epochs": 1,
+ "warmup_ratio": 0.0,
+ "save_steps": 1000,
+ "seed": 7,
+ "lora": {"r": 4, "alpha": 8, "target_modules": ["q_proj", "v_proj"]},
+ }
+ body.update(training)
+ data: dict[str, Any] = {"train": "train.jsonl", "max_length": 64}
+ if task == "distill":
+ data["chat_template"] = "chatml"
+ if task == "asr":
+ data["format"] = "asr"
+ if task == "unlearn":
+ data.update(forget_set="forget.jsonl", retain_set="retain.jsonl")
+ return load_config_from_string(yaml.safe_dump({
+ "base": base,
+ "task": task,
+ "backend": "transformers",
+ "data": data,
+ "training": body,
+ "output": str(out),
+ }))
+
+
+def _build(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, task: str, checkpointing: bool | str
+) -> tuple[Any, Any]:
+ """Run the wrapper's real ``setup`` and return it with the HF trainer it built."""
+ import numpy as np
+
+ monkeypatch.chdir(tmp_path)
+ llama = tiny_model_dir("llama")
+ if task == "distill":
+ cfg = _config(task, llama, tmp_path / "out", checkpointing, teacher_model=llama)
+ rows = [
+ {"messages": [
+ {"role": "user", "content": "hello"},
+ {"role": "assistant", "content": "world friend"},
+ ]}
+ ] * 4
+ elif task == "classifier":
+ # batch 1: the tiny checkpoint has no pad id, and a sequence-classification
+ # head refuses batches > 1 without one.
+ cfg = _config(
+ task, llama, tmp_path / "out", checkpointing, num_labels=2, batch_size=1
+ )
+ rows = [{"text": "hello world", "label": index % 2} for index in range(4)]
+ else:
+ import soup_cli.utils.tts_codec as tts_codec
+
+ def sine(path: str, *, target_sr: int) -> Any:
+ seconds = np.arange(target_sr // 2, dtype=np.float32) / target_sr
+ return (0.1 * np.sin(2 * np.pi * 220.0 * seconds)).astype(np.float32)
+
+ monkeypatch.setattr(tts_codec, "load_audio_mono", sine)
+ cfg = _config(task, _tiny_whisper_dir(tmp_path), tmp_path / "out", checkpointing)
+ rows = [{"audio": f"clip{index}.wav", "text": "hello world"} for index in range(4)]
+ module_name, class_name = _WRAPPERS[task]
+ wrapper = getattr(importlib.import_module(module_name), class_name)(cfg, device="cpu")
+ wrapper.setup({"train": rows})
+ trainer = wrapper.trainer
+ assert trainer.args.device.type == "cpu"
+ assert next(trainer.model.parameters()).device.type == "cpu"
+ return wrapper, trainer
+
+
+def _count_layer_calls(model: Any) -> list[int]:
+ """Count calls of the first checkpointable layer; recompute runs it twice a step."""
+ from transformers.modeling_layers import GradientCheckpointingLayer
+
+ layer = next(m for m in model.modules() if isinstance(m, GradientCheckpointingLayer))
+ calls = [0]
+
+ def hook(*_args: Any) -> None:
+ calls[0] += 1
+
+ layer.register_forward_pre_hook(hook)
+ return calls
+
+
[email protected]("task", _WRAPPERS)
+@_STATES
+def test_yaml_checkpointing_reaches_the_trainer_arguments(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, task: str, checkpointing: bool
+) -> None:
+ _, trainer = _build(tmp_path, monkeypatch, task, checkpointing)
+ assert trainer.args.gradient_checkpointing is checkpointing, task
+
+
[email protected]("task", _WRAPPERS)
+@_STATES
+def test_checkpointing_state_after_one_cpu_training_step(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, task: str, checkpointing: bool
+) -> None:
+ _, trainer = _build(tmp_path, monkeypatch, task, checkpointing)
+ trainer.args.max_steps = 1
+ trainer.args.save_strategy = "no"
+ trainer.args.disable_tqdm = True
+ calls = _count_layer_calls(trainer.model)
+ trainer.train()
+ assert trainer.state.global_step == 1
+ assert trainer.model.is_gradient_checkpointing is checkpointing, task
+ # One forward per step, plus the backward recompute when checkpointing is on.
+ assert calls[0] == (2 if checkpointing else 1), task
+
+
[email protected]("task", _WRAPPERS)
[email protected]("tier", _TIERS)
+def test_enabled_tiers_are_full_checkpointing(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, task: str, tier: str
+) -> None:
+ # No selective hooks on these wrappers: every enabled tier is full checkpointing.
+ _, trainer = _build(tmp_path, monkeypatch, task, tier)
+ assert trainer.args.gradient_checkpointing is True, (task, tier)
+
+
+# --- unlearn: its own loop, no HF Trainer -------------------------------------
+
+
+def _unlearn(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ checkpointing: bool | str,
+ method: str = "simnpo",
+) -> Any:
+ from soup_cli.trainer.unlearn import UnlearnTrainerWrapper
+
+ monkeypatch.chdir(tmp_path)
+ row = {"prompt": "hello", "completion": "world friend"}
+ (tmp_path / "forget.jsonl").write_text(json.dumps(row) + "\n", encoding="utf-8")
+ (tmp_path / "retain.jsonl").write_text(
+ json.dumps({"prompt": "hi", "completion": "there friend"}) + "\n", encoding="utf-8"
+ )
+ cfg = _config(
+ "unlearn", tiny_model_dir("llama"), Path("out"), checkpointing,
+ unlearn_method=method, batch_size=1,
+ )
+ wrapper = UnlearnTrainerWrapper(cfg, device="cpu")
+ wrapper.setup()
+ assert wrapper._dev == "cpu"
+ return wrapper
+
+
+@_STATES
+def test_unlearn_setup_applies_yaml_checkpointing(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, checkpointing: bool
+) -> None:
+ wrapper = _unlearn(tmp_path, monkeypatch, checkpointing)
+ assert wrapper.model.training is True
+ assert wrapper.model.is_gradient_checkpointing is checkpointing
+
+
[email protected]("tier", _TIERS)
+def test_unlearn_enabled_tiers_are_full_checkpointing(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, tier: str
+) -> None:
+ wrapper = _unlearn(tmp_path, monkeypatch, tier)
+ assert wrapper.model.is_gradient_checkpointing is True, tier
+
+
[email protected]("method", ["simnpo", "npo", "rmu"])
+@_STATES
+def test_unlearn_loop_step_recomputes_only_when_enabled(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, method: str, checkpointing: bool
+) -> None:
+ import torch
+
+ wrapper = _unlearn(tmp_path, monkeypatch, checkpointing, method)
+ lora = [p for n, p in wrapper.model.named_parameters() if "lora_B" in n]
+ before = [p.detach().clone() for p in lora]
+ calls = _count_layer_calls(wrapper.model)
+ result = wrapper.train()
+ assert result["total_steps"] == 1
+ assert wrapper.model.is_gradient_checkpointing is checkpointing
+ # The single forget example takes two policy forwards with a retain set (NPO and
+ # SimNPO: forget + retain CE; RMU: forget + retain activations). Checkpointing
+ # runs each again in the backward pass.
+ assert calls[0] == (4 if checkpointing else 2), method
+ # The step still trained the adapter through the checkpointed layers.
+ assert any(not torch.equal(old, new.detach()) for old, new in zip(before, lora))