Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""PPO (Proximal Policy Optimization) trainer — wraps trl.PPOTrainer.
PPO is the classic RLHF alignment method: generate completions, score them
with a reward model (or reward function), then optimize the policy using
clipped surrogate objectives with a KL penalty against a frozen reference model.
Full RLHF pipeline: SFT → Reward Model → PPO
"""
import math
import os
import time
from pathlib import Path
from typing import Any, Mapping, Optional
from rich.console import Console
from rich.markup import escape
from soup_cli.config.schema import SoupConfig
from soup_cli.data.chat_templates import apply_chat_template_override
from soup_cli.trainer.loss_summary import summarize_training_loss
from soup_cli.utils.gpu import (
bf16_fp16_flags,
estimate_batch_size,
model_size_from_name,
resolve_device_map,
resolve_frozen_base_load_dtype,
)
from soup_cli.utils.mixed_precision import align_trainable_dtype_for_fp16
from soup_cli.utils.run_provenance import base_revision_kwargs
from soup_cli.utils.seeding import apply_training_seed, training_seed_kwargs
console = Console()
def _set_ppo_training_kwargs(
ppo_kwargs: dict[str, object],
ppo_config_cls: type,
tcfg: Any,
*,
total_steps: int | None = None,
device: str = "cpu",
) -> dict[str, str]:
"""Forward Soup's three PPO schedules across TRL parameter renames."""
from soup_cli.trainer._trl_compat import config_accepts, kl_penalty_kwargs
applied: dict[str, str] = {}
if config_accepts(ppo_config_cls, "num_train_epochs"):
ppo_kwargs["num_train_epochs"] = tcfg.epochs
applied["train_epochs"] = "num_train_epochs"
if config_accepts(ppo_config_cls, "num_ppo_epochs"):
ppo_kwargs["num_ppo_epochs"] = tcfg.ppo_epochs
applied["ppo_epochs"] = "num_ppo_epochs"
elif config_accepts(ppo_config_cls, "ppo_epochs"):
ppo_kwargs["ppo_epochs"] = tcfg.ppo_epochs
applied["ppo_epochs"] = "ppo_epochs"
kl_kwargs = kl_penalty_kwargs(ppo_config_cls, tcfg.ppo_kl_penalty)
ppo_kwargs.update(kl_kwargs)
for name in kl_kwargs:
applied["kl_coef"] = name
if total_steps is not None:
from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
from soup_cli.utils.warmup import resolve_trainer_warmup_steps
use_bf16, use_fp16 = bf16_fp16_flags(device)
warmup_steps = resolve_trainer_warmup_steps(tcfg.warmup_ratio)
training_kwargs = {
"warmup_steps": warmup_steps,
"weight_decay": tcfg.weight_decay,
"max_grad_norm": tcfg.max_grad_norm,
"optim": tcfg.optimizer,
"lr_scheduler_type": tcfg.scheduler,
"logging_steps": tcfg.logging_steps,
"save_steps": tcfg.save_steps,
"bf16": use_bf16,
"fp16": use_fp16,
"gradient_checkpointing": should_enable_hf_gradient_checkpointing(
tcfg.gradient_checkpointing, stream_layers=tcfg.stream_layers
),
}
for name, value in training_kwargs.items():
if config_accepts(ppo_config_cls, name):
ppo_kwargs[name] = value
return applied
def _effective_ppo_setting(
config: object,
kwargs: Mapping[str, object],
field: str | None,
fallback: object,
) -> object:
"""Read the constructed config value, with compatibility fallbacks."""
if field is None:
return f"{fallback} (not forwarded)"
return getattr(config, field, kwargs[field])
def _unsupported_rl_flags(tcfg) -> list[str]:
"""The RL-signal flags PPO cannot feed on the experimental trl API (#1441)."""
flags = []
if getattr(tcfg, "reward_hack_detector", None) is not None:
flags.append("reward_hack_detector")
if getattr(tcfg, "echo_trap_enabled", False):
flags.append("echo_trap_enabled")
if getattr(tcfg, "reward_hack_mitigation", "off") != "off":
flags.append("reward_hack_mitigation")
return flags
def refuse_unfed_rl_flags(tcfg, *, is_experimental: bool) -> None:
"""Refuse RL-signal flags PPO cannot feed, before anything is loaded.
#1441: the experimental ``PPOTrainer`` takes no ``reward_funcs`` — its
reward comes from the reward model it calls internally — so the wrappers
built at :282-299 were dropped on the floor. The buffer was still created,
and because the buffer object existed, the detector's ``on_log`` fallback
returned early, so nothing was ever written *and* nothing was ever read from
trl's logs. A run with a detector, a mitigation mode or the echo trap was
announced, loaded and inert.
Refusing beats a dead callback: the run stops before the model load, and the
message names every flag that was set.
"""
if not is_experimental:
return
flags = _unsupported_rl_flags(tcfg)
if not flags:
return
raise ValueError(
f"task: ppo cannot use {', '.join(flags)}: this trl's experimental "
"PPOTrainer takes no reward functions, so the reward/completion signal "
"these callbacks read is never captured. They are wired on GRPO only. "
"Remove the flag(s), or run task: grpo where the signal buffer is fed."
)
class PPOTrainerWrapper:
"""High-level wrapper for PPO training from SoupConfig.
PPO generates completions for each prompt, scores them with a reward model
or reward function, and optimizes using proximal policy optimization with
a KL penalty against a frozen reference model.
Supports two reward sources:
- reward_model: path/HF ID of a trained reward model (AutoModelForSequenceClassification)
- reward_fn: callable reward function (same as GRPO — 'accuracy', 'format', or custom .py)
At least one of reward_model or reward_fn must be specified.
"""
def __init__(
self,
config: SoupConfig,
device: str = "cuda",
report_to: str = "none",
deepspeed_config: Optional[str] = None,
fsdp_config: Optional[dict] = None,
trust_remote_code: bool = False,
):
self.config = config
self.device = device
self.report_to = report_to
self.deepspeed_config = deepspeed_config
self.fsdp_config = fsdp_config
self.trust_remote_code = trust_remote_code
from soup_cli.utils.trust_remote import (
model_requires_trust_remote_code,
resolve_trust_remote_code,
)
requires = model_requires_trust_remote_code(config.base) or False
self._trust_remote_code = resolve_trust_remote_code(
config.base,
requested=trust_remote_code,
console=console,
requires_remote_code=requires,
)
self.model = None
self.tokenizer = None
self.trainer = None
self.reward_model_instance = None
self.reward_fn = None
def setup(self, dataset: dict):
"""Load model, tokenizer, reward model/fn, apply LoRA, create PPO trainer."""
# Import PPOTrainer/PPOConfig — trl >=0.28 moved to trl.experimental
ppo_trainer_cls, ppo_config_cls, is_experimental = _import_ppo_classes()
from soup_cli.trainer._trl_compat import config_accepts
if self.deepspeed_config and not config_accepts(ppo_config_cls, "deepspeed"):
raise ValueError("task: ppo does not support --deepspeed on this trl version")
# Enable Rich progress bar for HuggingFace downloads
from soup_cli.trainer.sft import _enable_hf_transfer_progress
_enable_hf_transfer_progress()
cfg = self.config
tcfg = cfg.training
# #1441 — refuse before anything is loaded, including `datasets` and the
# model. These callbacks read a signal buffer the experimental
# PPOTrainer leaves empty, so a run with them is announced and inert.
refuse_unfed_rl_flags(tcfg, is_experimental=is_experimental)
# Deferred below the refusal: a config that cannot run at all should say
# which flag stopped it, not fail on an unrelated import first.
from datasets import Dataset
# #353: seed before the model and any adapter are built.
apply_training_seed(tcfg)
use_unsloth = cfg.backend == "unsloth"
# --- Batch size ---
batch_size = tcfg.batch_size
if batch_size == "auto":
from soup_cli.utils.gpu import get_gpu_info
gpu_info = get_gpu_info()
model_size = model_size_from_name(cfg.base)
batch_size = estimate_batch_size(
model_params_b=model_size,
seq_length=cfg.data.max_length,
gpu_memory_bytes=gpu_info["memory_total_bytes"],
quantization=tcfg.quantization,
lora_r=tcfg.lora.r,
)
# PPO needs memory for policy + ref model + reward model → conservative
batch_size = max(1, batch_size // 4)
console.print(f"[green]Auto batch size (PPO):[/] {batch_size}")
# #1391: a train set smaller than one rollout batch never reaches a
# step. _prepare_ppo_dataset turns every row into exactly one prompt,
# so the count is known before any model is loaded.
if is_experimental:
_check_one_rollout_batch(
len(dataset["train"]), batch_size, tcfg.gradient_accumulation_steps
)
# --- Load reward source ---
self._setup_reward(cfg, tcfg)
if use_unsloth:
self._setup_unsloth(cfg, tcfg)
else:
self._setup_transformers(cfg, tcfg)
apply_chat_template_override(
self.tokenizer, cfg.data.chat_template, console=console
)
trainable, total = self.model.get_nb_trainable_parameters()
pct = 100 * trainable / total
console.print(
f"[green]LoRA applied:[/] {trainable:,} trainable"
f" / {total:,} total ({pct:.2f}%)"
)
# --- Dataset ---
train_data = _prepare_ppo_dataset(dataset["train"], tokenizer=self.tokenizer)
train_ds = Dataset.from_list(train_data)
# Tokenize dataset: trl experimental PPOTrainer expects input_ids,
# not raw text. Add input_ids/attention_mask from prompt_text.
max_len = cfg.data.max_length
def _tokenize_ppo(examples):
return self.tokenizer(
examples["prompt_text"],
truncation=True,
max_length=max_len,
padding=False,
)
train_ds = train_ds.map(_tokenize_ppo, batched=True)
# #1391: trl's PPOTrainer pads every column with DataCollatorWithPadding and
# never drops unused ones, so the prompt_text strings crash the collator.
# The manual (trl<0.28) loop below still reads prompt_text from _train_ds.
trl_train_ds = train_ds.select_columns(["input_ids", "attention_mask"])
# --- Output dir ---
output_dir = Path(cfg.output)
if cfg.experiment_name:
output_dir = output_dir / cfg.experiment_name
output_dir.mkdir(parents=True, exist_ok=True)
# --- PPO config ---
# Build kwargs, handling trl version differences
import inspect
ppo_kwargs = {
"output_dir": str(output_dir),
"per_device_train_batch_size": batch_size,
"gradient_accumulation_steps": tcfg.gradient_accumulation_steps,
"learning_rate": tcfg.lr,
**training_seed_kwargs(tcfg),
}
# FSDP2 — alternative to DeepSpeed
if self.fsdp_config:
ppo_kwargs.update(self.fsdp_config)
ppo_params = inspect.signature(ppo_config_cls).parameters
total_steps = math.ceil(
len(train_ds) / batch_size / tcfg.gradient_accumulation_steps
) * tcfg.epochs
applied_ppo_fields = _set_ppo_training_kwargs(
ppo_kwargs, ppo_config_cls, tcfg,
total_steps=total_steps, device=self.device,
)
from soup_cli.trainer._trl_compat import config_accepts
if "cliprange" in ppo_params:
ppo_kwargs["cliprange"] = tcfg.ppo_clip_ratio
if config_accepts(ppo_config_cls, "deepspeed"):
ppo_kwargs["deepspeed"] = self.deepspeed_config
if config_accepts(ppo_config_cls, "response_length"):
ppo_kwargs["response_length"] = cfg.data.max_length
if config_accepts(ppo_config_cls, "save_total_limit"):
ppo_kwargs["save_total_limit"] = 3
# Optional params that may not exist in all trl versions
if "log_with" in ppo_params:
ppo_kwargs["log_with"] = (
self.report_to if self.report_to != "none" else None
)
elif "report_to" in ppo_params:
ppo_kwargs["report_to"] = self.report_to
if "optimize_cuda_cache" in ppo_params:
ppo_kwargs["optimize_cuda_cache"] = self.device == "cuda"
# CPU support: trl PPOConfig requires use_cpu=True when no CUDA
if self.device == "cpu" and "use_cpu" in ppo_params:
ppo_kwargs["use_cpu"] = True
ppo_config = ppo_config_cls(**ppo_kwargs)
effective_train_epochs = _effective_ppo_setting(
ppo_config,
ppo_kwargs,
applied_ppo_fields.get("train_epochs"),
tcfg.epochs,
)
effective_ppo_epochs = _effective_ppo_setting(
ppo_config,
ppo_kwargs,
applied_ppo_fields.get("ppo_epochs"),
tcfg.ppo_epochs,
)
effective_kl_coef = _effective_ppo_setting(
ppo_config,
ppo_kwargs,
applied_ppo_fields.get("kl_coef"),
tcfg.ppo_kl_penalty,
)
console.print(
"[green]PPO schedule:[/] "
f"train epochs={effective_train_epochs}, "
f"PPO epochs={effective_ppo_epochs}, "
f"KL coefficient={effective_kl_coef}"
)
# --- Build reward functions list for PPOTrainer ---
reward_funcs = []
if self.reward_model_instance is not None:
reward_funcs.append(self.reward_model_instance)
if self.reward_fn is not None:
reward_funcs.append(self.reward_fn)
# v0.71.26 — reward-hack mitigation buffer parity with GRPO. When a
# detector / mitigation mode is set, capture the callable reward fns'
# rewards + completions into a shared RLSignalBuffer so the mitigation
# callback can observe the step (BETA — the on-GPU proof is GRPO-only).
# nn.Module reward models are skipped (their call shape differs).
self._rl_buffer = None
from soup_cli.utils.peft_wiring import rl_callbacks_need_buffer
if rl_callbacks_need_buffer(tcfg) and reward_funcs:
# v0.71.26 Stage 3 — reward-shaping shim over callable reward fns,
# BEFORE the buffer capture (no-op when shaping is off).
from soup_cli.utils.reward_hack_control import apply_reward_shaping
from soup_cli.utils.rl_signal_buffer import (
RLSignalBuffer,
wrap_reward_funcs,
)
self._rl_buffer = RLSignalBuffer()
reward_funcs = [
wrap_reward_funcs(
apply_reward_shaping(fn, tcfg), self._rl_buffer
)
if callable(fn) and not hasattr(fn, "forward")
else fn
for fn in reward_funcs
]
# --- Trainer ---
ppo_trainer_params = inspect.signature(ppo_trainer_cls.__init__).parameters
# Store for use in train() — experimental API has different .train() signature
self._is_experimental = is_experimental
# LoRA+ optimizer (#724/#745), PPO's shape. The other LoRA
# trainers are plain transformers.Trainer subclasses whose
# create_optimizer runs lazily at train(), so a post-construction
# attach_loraplus_optimizer lands cleanly and the scheduler is built
# around it. trl.experimental's PPOTrainer is not that shape: its
# __init__ builds the optimizer AND scheduler eagerly
# (create_optimizer_and_scheduler) and then calls
# accelerator.prepare(model, optimizer, dataloader) without the
# scheduler. Assigning trainer.optimizer afterwards is still stepped
# (B does train at lr*ratio), but self.lr_scheduler stays bound to the
# discarded default optimizer, so warmup and decay never reach the B
# group and get_last_lr() reports the wrong optimizer. Build the LoRA+
# optimizer up front and hand it to the constructor via
# optimizers=(opt, None) so create_optimizer_and_scheduler binds the
# scheduler to it. If trl ever makes the PPO optimizer lazy, delete this
# branch and let PPO join the post-construction attach.
#
# Detect the shape, not the trl version (see trainer/_trl_compat.py):
# the eager shape is exactly the one whose __init__ accepts an
# `optimizers` tuple. If a future trl.experimental drops that keyword,
# fail loudly here rather than silently fall back to the post-attach and
# reintroduce the flat-LR bug.
loraplus_optimizer = None
if getattr(tcfg, "loraplus_lr_ratio", None) is not None:
if "optimizers" not in ppo_trainer_params:
raise RuntimeError(
"training.loraplus_lr_ratio is set, but this trl PPOTrainer "
f"({ppo_trainer_cls.__module__}.{ppo_trainer_cls.__qualname__}) "
"does not accept an `optimizers` argument, so the LoRA+ "
"optimizer cannot be bound to the LR scheduler at "
"construction. A post-construction attach would train the "
"LoRA B group at a flat lr*ratio with warmup and decay never "
"reaching it. Pin a trl whose PPO trainer takes "
"optimizers=(optimizer, scheduler), or remove "
"loraplus_lr_ratio for this run."
)
from soup_cli.utils.peft_wiring import build_loraplus_optimizer
loraplus_optimizer = build_loraplus_optimizer(
self.model, ppo_config, tcfg
)
if is_experimental:
# trl >=0.28 experimental API: PPOTrainer(args, processing_class,
# model, ref_model, reward_model, train_dataset, value_model, ...)
# ref_model=None is fine (auto-creates from policy model).
# reward_model and value_model are required nn.Modules.
reward_model_obj = self._get_or_create_reward_model(cfg, tcfg)
value_model_obj = self._create_value_model(cfg, tcfg)
trainer_kwargs = {
"args": ppo_config,
"processing_class": self.tokenizer,
"model": self.model,
"ref_model": None,
"reward_model": reward_model_obj,
"train_dataset": trl_train_ds,
"value_model": value_model_obj,
}
# LoRA+ (#724/#745): inject so the eagerly-built scheduler binds to it.
if loraplus_optimizer is not None:
self._complete_loraplus_optimizer(
loraplus_optimizer, value_model_obj, ppo_config
)
trainer_kwargs["optimizers"] = (loraplus_optimizer, None)
self._dataset_in_constructor = True
self.trainer = ppo_trainer_cls(**trainer_kwargs)
elif "args" in ppo_trainer_params:
# trl >=0.28 non-experimental (transitional API)
trainer_kwargs = {
"model": self.model,
"args": ppo_config,
"processing_class": self.tokenizer,
}
if "train_dataset" in ppo_trainer_params:
trainer_kwargs["train_dataset"] = trl_train_ds
elif "dataset" in ppo_trainer_params:
trainer_kwargs["dataset"] = trl_train_ds
if reward_funcs and "reward_funcs" in ppo_trainer_params:
trainer_kwargs["reward_funcs"] = reward_funcs
# Pass ref/reward/value models if required positionally
if "ref_model" in ppo_trainer_params:
trainer_kwargs["ref_model"] = None
if "reward_model" in ppo_trainer_params:
trainer_kwargs["reward_model"] = self._get_or_create_reward_model(
cfg, tcfg,
reward_funcs_supplied=bool(reward_funcs)
and "reward_funcs" in ppo_trainer_params,
)
if "value_model" in ppo_trainer_params:
trainer_kwargs["value_model"] = self._create_value_model(cfg, tcfg)
# LoRA+ (#724/#745): inject so the eagerly-built scheduler binds to it.
if loraplus_optimizer is not None:
self._complete_loraplus_optimizer(
loraplus_optimizer, trainer_kwargs.get("value_model"), ppo_config
)
trainer_kwargs["optimizers"] = (loraplus_optimizer, None)
self._dataset_in_constructor = (
"train_dataset" in trainer_kwargs or "dataset" in trainer_kwargs
)
self.trainer = ppo_trainer_cls(**trainer_kwargs)
else:
# trl <0.28: PPOTrainer(config=, model=, tokenizer=, dataset=)
trainer_kwargs = {
"model": self.model,
"config": ppo_config,
"tokenizer": self.tokenizer,
"dataset": train_ds,
}
self._dataset_in_constructor = True
self.trainer = ppo_trainer_cls(**trainer_kwargs)
# #1675 - trl.experimental's PPOTrainer subclasses transformers.Trainer
# but never runs Trainer.__init__, and its save_model / _save_checkpoint
# delegate to Trainer's. An attribute Trainer.__init__ assigns and those
# methods read is then missing: transformers 5.19.0 added one
# (is_distributed_loading_by_transformers) and the first checkpoint save
# raised AttributeError. Fill in what is absent, whichever branch built
# the trainer.
from soup_cli.trainer._trl_compat import ensure_trainer_init_attrs
ensure_trainer_init_attrs(self.trainer)
# #359 - the same exposure #336 fixed in sft.py: with LoRA the
# no-decay optimizer group is empty, DeepSpeed drops it, and the LR
# scheduler keeps two base_lrs until torch's strict zip raises at the
# first step. The guard prunes inside create_optimizer, i.e. before
# the scheduler is built. No-op for full fine-tuning, and only under
# DeepSpeed so the ordinary path keeps its own optimizer.
#
# On PPO's eager shape (trl.experimental and the transitional branch)
# this guard does nothing: the trainer built its optimizer inside
# __init__ and trl never calls create_optimizer again. That is safe for
# the default optimizer, whose no-decay group is filled by the value
# model's biases and norm weights. The LoRA+ optimizer can have empty
# groups, so _complete_loraplus_optimizer prunes it before injection.
if self.deepspeed_config:
from soup_cli.utils.deepspeed import attach_empty_param_group_guard
attach_empty_param_group_guard(self.trainer)
# v0.71.26 — reward-hack mitigation / echo-trap / RL-checkpoint callbacks
# (PPO parity with GRPO; kl_coef mutation for the controller).
from soup_cli.utils.peft_wiring import attach_rl_callbacks
attach_rl_callbacks(
self.trainer,
tcfg,
buffer=self._rl_buffer,
tokenizer=self.tokenizer,
output_dir=str(output_dir),
task="ppo",
)
# v0.40.6 #67 — ReLoRA merge-and-reinitialize restart callback.
# LoRA+ (#724/#745) is wired ABOVE via constructor injection, not here:
# PPO's trainer builds its scheduler eagerly, so the optimizer has to be
# passed to the constructor rather than attached after it.
from soup_cli.utils.peft_wiring import (
attach_curriculum_callback,
attach_plugin_callback,
attach_relora_callback,
)
attach_relora_callback(self.trainer, tcfg)
# v0.53.5 #114/#115 — dynamic curriculum live callback.
attach_curriculum_callback(self.trainer, tcfg, str(output_dir), console)
# v0.53.6 #101 — Soup plugin TrainerCallback.
attach_plugin_callback(self.trainer, console)
self._output_dir = str(output_dir)
self._train_ds = train_ds
self._trl_train_ds = trl_train_ds
self._batch_size = batch_size
self._num_epochs = tcfg.epochs
self._max_length = cfg.data.max_length
def _get_or_create_reward_model(self, cfg, tcfg, *, reward_funcs_supplied=False):
"""Get the reward model for trl's PPO API (an nn.Module, not a callable).
Uses a loaded instance, else loads the configured ``reward_model`` path.
If neither exists, PPO has NO real reward signal: a fresh
``AutoModelForSequenceClassification`` has a randomly-initialised
regression head, so optimising the policy against it trains toward
noise (silent, GPU-hours wasted). A ``reward_fn`` (callable) cannot fill
this nn.Module slot on trl's PPO API. Fail loudly instead — unless the
trainer is separately consuming ``reward_funcs`` (a newer trl API), in
which case the reward model is unused and a neutral head is fine.
"""
if self.reward_model_instance is not None:
return self.reward_model_instance
# If a reward_model path is configured, load it
if tcfg.reward_model:
return _load_reward_model(
tcfg.reward_model, self.device, self.trust_remote_code,
tcfg=tcfg,
)
if not reward_funcs_supplied:
raise RuntimeError(
"PPO requires a trained reward_model (set training.reward_model "
"to a path or HF id). trl's PPO reward-model slot needs an "
"nn.Module; a reward_fn cannot drive it, and creating one from "
"the base model would train the policy against a randomly-"
"initialised reward head. Use `task: grpo` for reward-function-"
"based RL instead."
)
# reward_funcs handles the reward signal on this trl API; the reward
# model is a required-but-unused formality (neutral head is fine).
from transformers import AutoModelForSequenceClassification
console.print(
"[dim]reward_funcs active; creating a neutral reward-model "
f"placeholder from base model: {cfg.base}[/]"
)
reward_model = AutoModelForSequenceClassification.from_pretrained(
cfg.base,
trust_remote_code=self._trust_remote_code,
num_labels=1,
device_map=resolve_device_map(self.device) if self.device != "cpu" else None,
**base_revision_kwargs(cfg.base),
)
reward_model.eval()
return reward_model
def _complete_loraplus_optimizer(self, optimizer, value_model, args):
"""Finish the LoRA+ optimizer before it is handed to PPOTrainer (#745).
trl's default PPO optimizer covers PolicyAndValueWrapper(policy,
value_model); the LoRA+ one is built over the policy alone. Give the
value model its decay / no-decay groups at the base learning rate, or
no optimizer owns the critic and it never trains.
Under DeepSpeed, drop empty groups (#359): LoRA+ emits groups such as
embedding and groupB_no_decay that hold nothing with LoRA, DeepSpeed
drops them, and the scheduler built from this optimizer would keep one
base_lr too many. Pruning in place here, before the constructor builds
the scheduler, is the same order the guard enforces elsewhere.
"""
from soup_cli.utils.peft_wiring import add_value_model_param_groups
if value_model is not None:
add_value_model_param_groups(optimizer, value_model, args)
if self.deepspeed_config:
from soup_cli.utils.deepspeed import prune_empty_param_groups
prune_empty_param_groups(optimizer)
return optimizer
def _create_value_model(self, cfg, tcfg):
"""Create a value model for trl experimental PPO API.
The value model estimates state values for GAE advantage estimation.
We create an AutoModelForSequenceClassification from the base model.
"""
from transformers import AutoModelForSequenceClassification
console.print(f"[dim]Creating value model from: {cfg.base}[/]")
value_model = AutoModelForSequenceClassification.from_pretrained(
cfg.base,
trust_remote_code=self._trust_remote_code,
num_labels=1,
device_map=resolve_device_map(self.device) if self.device != "cpu" else None,
**base_revision_kwargs(cfg.base),
)
return value_model
def _setup_reward(self, cfg, tcfg):
"""Load reward model and/or reward function."""
# Reward model (pre-trained classifier)
if tcfg.reward_model:
self.reward_model_instance = _load_reward_model(
tcfg.reward_model, self.device, self.trust_remote_code,
tcfg=tcfg,
)
console.print(f"[green]Reward model loaded:[/] {tcfg.reward_model}")
# Reward function (callable — reuse GRPO reward functions)
if tcfg.reward_fn:
from soup_cli.trainer.rewards import load_reward_fn
self.reward_fn = load_reward_fn(
tcfg.reward_fn, verifiable_domain=tcfg.verifiable_domain
)
if self.reward_model_instance is None and self.reward_fn is None:
console.print(
"[yellow]Warning: No reward_model or reward_fn specified. "
"Using default reward_fn='accuracy'.[/]"
)
from soup_cli.trainer.rewards import load_reward_fn
self.reward_fn = load_reward_fn("accuracy")
def _setup_transformers(self, cfg: SoupConfig, tcfg) -> None:
"""Load model via standard transformers + peft pipeline."""
from peft import TaskType, get_peft_model, prepare_model_for_kbit_training
from transformers import AutoModelForCausalLM, AutoTokenizer
console.print(f"[dim]Loading tokenizer: {cfg.base}[/]")
self.tokenizer = AutoTokenizer.from_pretrained(
cfg.base, trust_remote_code=self._trust_remote_code,
**base_revision_kwargs(cfg.base),
)
if self.tokenizer.pad_token is None:
self.tokenizer.pad_token = self.tokenizer.eos_token
# Quantization (v0.38.0 Quant Menu — see soup_cli.utils.quant_menu)
from soup_cli.utils.quant_menu import build_quantization_config_for_loader
quant_config_obj = build_quantization_config_for_loader(
tcfg=tcfg, base=cfg.base, console=console,
)
console.print(f"[dim]Loading model: {cfg.base}[/]")
# On CPU, use device_map="cpu" to avoid meta tensors from "auto"
dev_map = resolve_device_map(self.device)
model_kwargs = {
"trust_remote_code": self._trust_remote_code, "device_map": dev_map,
"torch_dtype": resolve_frozen_base_load_dtype(self.device),
}
if quant_config_obj is not None:
model_kwargs["quantization_config"] = quant_config_obj
self.model = AutoModelForCausalLM.from_pretrained(
cfg.base, **model_kwargs,
**base_revision_kwargs(cfg.base),
)
if tcfg.quantization in ("4bit", "8bit"):
from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
self.model = prepare_model_for_kbit_training(
self.model,
use_gradient_checkpointing=should_enable_hf_gradient_checkpointing(
tcfg.gradient_checkpointing, stream_layers=tcfg.stream_layers
),
)
from soup_cli.utils.peft_wiring import (
build_lora_config,
resolve_lora_target_modules,
)
target_modules = resolve_lora_target_modules(self.model, tcfg.lora.target_modules, console)
# #1099: moe_lora picks the expert-FFN targets. Without this the flag
# was accepted and ignored here, and on a fused-expert MoE the auto
# resolution leaves peft with nothing to attach.
from soup_cli.utils.moe import resolve_moe_lora_targets
target_modules = resolve_moe_lora_targets(
self.model, tcfg, target_modules, console
)
lora_config = build_lora_config(
tcfg.lora,
target_modules=target_modules,
task_type=TaskType.CAUSAL_LM,
)
# v0.40.6 #67 — surgical PEFT patches.
from soup_cli.utils.peft_wiring import (
apply_post_lora_patches,
apply_pre_lora_patches,
)
apply_pre_lora_patches(self.model, cfg.base)
self.model = get_peft_model(self.model, lora_config)
apply_post_lora_patches(self.model)
# QAT — insert fake quantization ops after LoRA. The "fp8" variant
# is FP8 training (handled by apply_v028_speed_memory), not int8 QAT.
# PPO has its own forward loop, so use_cut_ce will degrade gracefully.
if tcfg.quantization_aware and tcfg.quantization_aware != "fp8":
from soup_cli.utils.qat import prepare_model_for_qat
self.model = prepare_model_for_qat(self.model)
# v0.35.0 #60 — multi-trainer wiring of v0.28.0 speed/memory features.
from soup_cli.utils.v028_features import apply_v028_speed_memory
apply_v028_speed_memory(
model=self.model, tcfg=tcfg, base_model=cfg.base,
console=console, device=self.device, backend=cfg.backend,
)
def _setup_unsloth(self, cfg, tcfg):
"""Load model via unsloth FastLanguageModel (2-5x faster)."""
from soup_cli.utils import unsloth
from soup_cli.utils.unsloth import load_model_and_tokenizer
# #797: resolve training.gradient_checkpointing for the Unsloth path
# (streaming layers already checkpoint every layer, so it stays off).
use_gradient_checkpointing = unsloth.resolve_unsloth_gradient_checkpointing(
tcfg.gradient_checkpointing, stream_layers=getattr(tcfg, "stream_layers", False)
)
console.print(f"[dim]Loading model via [bold]unsloth[/]: {cfg.base}[/]")
self.model, self.tokenizer = load_model_and_tokenizer(
model_name=cfg.base,
max_seq_length=cfg.data.max_length,
quantization=tcfg.quantization,
lora_r=tcfg.lora.r,
lora_alpha=tcfg.lora.alpha,
lora_dropout=tcfg.lora.dropout,
target_modules=tcfg.lora.target_modules,
use_rslora=tcfg.lora.use_rslora,
use_gradient_checkpointing=use_gradient_checkpointing,
)
if self.tokenizer.pad_token is None:
self.tokenizer.pad_token = self.tokenizer.eos_token
def train(
self,
display: Optional[object] = None,
tracker: Optional[object] = None,
run_id: str = "",
resume_from_checkpoint: Optional[str] = None,
) -> dict:
"""Run PPO training loop and return results summary.
Supports two trl APIs:
- trl >=0.28: uses built-in trainer.train() (OnlineDPOTrainer style)
- trl <0.28: manual loop with generate() + step()
"""
# Detect API: trl >=0.28 PPOTrainer has a .train() from OnlineDPOTrainer
has_builtin_train = hasattr(self.trainer, "train") and not hasattr(
self.trainer, "step"
)
if has_builtin_train:
return self._train_builtin(display, tracker, run_id, resume_from_checkpoint)
return self._train_manual(display, tracker, run_id)
def _train_builtin(self, display, tracker, run_id, resume_from_checkpoint):
"""Train using trl >=0.28 built-in trainer.train() method."""
start = time.time()
# If dataset wasn't accepted by constructor, set it on the trainer
if not self._dataset_in_constructor:
if hasattr(self.trainer, "train_dataset"):
self.trainer.train_dataset = self._trl_train_ds
elif hasattr(self.trainer, "dataset"):
self.trainer.dataset = self._trl_train_ds
if display:
from soup_cli.monitoring.callback import build_soup_trainer_callback
self.trainer.add_callback(
build_soup_trainer_callback(
display,
config=self.config,
tracker=tracker,
run_id=run_id,
batch_size=self._batch_size,
output_dir=self._output_dir,
)
)
# trl experimental PPOTrainer.train() does not accept resume_from_checkpoint
import inspect
from soup_cli.utils.v028_features import activation_offloading_context
train_params = inspect.signature(self.trainer.train).parameters
with activation_offloading_context(
self.config.training, self._output_dir,
):
if resume_from_checkpoint and "resume_from_checkpoint" in train_params:
align_trainable_dtype_for_fp16(
self.trainer.model,
fp16=getattr(self.trainer.args, "fp16", False),
bf16=getattr(self.trainer.args, "bf16", False),
)
self.trainer.train(resume_from_checkpoint=resume_from_checkpoint)
else:
if resume_from_checkpoint:
console.print(
"[yellow]Warning: This PPOTrainer does not support "
"resume_from_checkpoint -- starting from scratch.[/]"
)
align_trainable_dtype_for_fp16(
self.trainer.model,
fp16=getattr(self.trainer.args, "fp16", False),
bf16=getattr(self.trainer.args, "bf16", False),
)
self.trainer.train()
duration = time.time() - start
if self.config.training.relora_steps is None:
self.trainer.save_model(self._output_dir)
else:
from soup_cli.utils.peft_wiring import save_model_with_relora
save_model_with_relora(
self.trainer, self._output_dir, self.config.training.relora_steps
)
self.tokenizer.save_pretrained(self._output_dir)
# Extract metrics. trl's PPO trainer (0.29) logs per-step losses under
# ``loss/policy_avg`` rather than ``loss``; the key is declared here
# instead of guessed (see #1413).
logs = self.trainer.state.log_history
loss_summary = summarize_training_loss(logs, loss_key="loss/policy_avg")
hours = int(duration // 3600)
minutes = int((duration % 3600) // 60)
duration_str = f"{hours}h {minutes}m" if hours > 0 else f"{minutes}m"
return {
**loss_summary,
"duration": duration_str,
"duration_secs": duration,
"output_dir": self._output_dir,
"total_steps": self.trainer.state.global_step,
}
def _train_manual(self, display, tracker, run_id):
"""Train using trl <0.28 manual loop: generate() + step()."""
import torch
start = time.time()
step = 0
all_rewards = []
log_history = []
for epoch in range(self._num_epochs):
for batch_idx in range(0, len(self._train_ds), self._batch_size):
batch_end = min(batch_idx + self._batch_size, len(self._train_ds))
batch = self._train_ds[batch_idx:batch_end]
# Tokenize prompts
prompt_texts = batch["prompt_text"]
query_tensors = [
self.tokenizer.encode(p, return_tensors="pt").squeeze()
for p in prompt_texts
]
# Generate completions
response_tensors = []
for query in query_tensors:
gen_kwargs = {
"max_new_tokens": min(256, self._max_length // 2),
"do_sample": True,
"top_p": 0.9,
"temperature": 0.7,
}
response = self.trainer.generate(query.unsqueeze(0), **gen_kwargs)
response_tensors.append(response.squeeze()[len(query):])
# Compute rewards
rewards = self._compute_rewards(
query_tensors, response_tensors, batch,
)
# PPO step
stats = self.trainer.step(query_tensors, response_tensors, rewards)
step += 1
mean_reward = torch.stack(rewards).mean().item()
all_rewards.append(mean_reward)
log_entry = {
"step": step,
"epoch": epoch + 1,
"loss": stats.get("ppo/loss/total", 0),
"reward": mean_reward,
"kl": stats.get("ppo/mean_kl", 0),
"lr": stats.get("ppo/learning_rate", self.config.training.lr),
}
log_history.append(log_entry)
# Update display
if display and hasattr(display, "update"):
display.update(
step=step,
epoch=epoch + 1,
loss=log_entry["loss"],
lr=log_entry["lr"],
)
# Update tracker
if tracker and run_id:
tracker.log_metrics(
run_id=run_id,
step=step,
epoch=epoch + 1,
loss=log_entry["loss"],
lr=log_entry["lr"],
)
duration = time.time() - start
if self.config.training.relora_steps is None:
self.trainer.save_model(self._output_dir)
else:
from soup_cli.utils.peft_wiring import save_model_with_relora
save_model_with_relora(
self.trainer, self._output_dir, self.config.training.relora_steps
)
self.tokenizer.save_pretrained(self._output_dir)
# Extract metrics
loss_summary = summarize_training_loss(log_history)
hours = int(duration // 3600)
minutes = int((duration % 3600) // 60)
duration_str = f"{hours}h {minutes}m" if hours > 0 else f"{minutes}m"
return {
**loss_summary,
"duration": duration_str,
"duration_secs": duration,
"output_dir": self._output_dir,
"total_steps": step,
}
def _compute_rewards(self, query_tensors, response_tensors, batch):
"""Compute rewards using reward model and/or reward function."""
import torch
rewards = []
num_samples = len(query_tensors)
if self.reward_model_instance is not None:
# Score with reward model
for query, response in zip(query_tensors, response_tensors):
full_ids = torch.cat([query, response]).unsqueeze(0)
full_ids = full_ids.to(self.reward_model_instance.device)
with torch.no_grad():
output = self.reward_model_instance(full_ids)
score = output.logits[:, -1].squeeze().float()
rewards.append(score.cpu())
elif self.reward_fn is not None:
# Score with reward function
completions = []
for response in response_tensors:
text = self.tokenizer.decode(response, skip_special_tokens=True)
completions.append([{"role": "assistant", "content": text}])
kwargs = {}
if "answer" in batch:
kwargs["answer"] = batch["answer"]
scores = self.reward_fn(completions, **kwargs)
rewards = [torch.tensor(score, dtype=torch.float32) for score in scores]
else:
# Fallback: zero reward
rewards = [torch.tensor(0.0) for _ in range(num_samples)]
return rewards
def _world_size() -> int:
try:
return max(1, int(os.environ.get("WORLD_SIZE", "1") or "1"))
except (TypeError, ValueError):
return 1
def _check_one_rollout_batch(rows: int, batch_size: int, grad_accum: int) -> None:
"""#1391: trl's PPO DataLoader uses ``drop_last=True`` with one rollout batch
(``batch_size * gradient_accumulation_steps`` rows per process) and repeats it
with ``while True``, so a smaller train set spins forever without a step."""
processes = _world_size()
needed = batch_size * grad_accum * processes
if rows < needed:
per_process = f" x {processes} processes" if processes > 1 else ""
raise ValueError(
f"PPO needs at least one rollout batch of training prompts: the train set "
f"has {rows} rows, but batch_size={batch_size} x "
f"gradient_accumulation_steps={grad_accum}{per_process} = {needed}. "
f"Add prompts, or lower batch_size / gradient_accumulation_steps."
)
def _import_ppo_classes():
"""Import PPOTrainer and PPOConfig, handling trl version differences.
trl >=0.28 moved PPOTrainer to trl.experimental with a new API that requires
ref_model, reward_model, train_dataset, and value_model as positional args.
The old trl import still works in 0.28 (deprecated) but is removed in 0.29.
Returns:
(PPOTrainer, PPOConfig, is_experimental): tuple with the classes and a flag
indicating whether the experimental API (with required positional args) is used.
"""
try:
from trl.experimental.ppo import PPOConfig, PPOTrainer
return PPOTrainer, PPOConfig, True
except (ImportError, ModuleNotFoundError):
pass
from trl import PPOConfig, PPOTrainer
return PPOTrainer, PPOConfig, False
def _load_reward_model(
model_path: str,
device: str = "cuda",
trust_remote_code: bool = False,
tcfg=None,
):
"""Load a pre-trained reward model for PPO scoring.
Reward models are typically AutoModelForSequenceClassification that output
a scalar reward score for a given input sequence.
v0.40.5 #66: when ``tcfg`` is provided and ``training.quantize_reward_model``
is set, the reward model is loaded with the same quantization config as
the policy model (Quant Menu), so it doesn't silently consume
full-precision VRAM and OOM when the policy is GPTQ/AWQ/HQQ/etc. v0.53.0
made this opt-in via that flag (default False) rather than unconditional,
matching ``double_quant_on``'s #321 precedent of honouring a configured
flag over a hardcoded default. Pass ``tcfg=None`` for unquantized
regardless of the flag.
"""
from transformers import AutoModelForSequenceClassification
from soup_cli.utils.trust_remote import (
model_requires_trust_remote_code,
resolve_trust_remote_code,
)
requires = model_requires_trust_remote_code(model_path) or False
resolved = resolve_trust_remote_code(
model_path,
requested=trust_remote_code,
console=console,
requires_remote_code=requires,
)
console.print(f"[dim]Loading reward model: {model_path}[/]")
dev_map = resolve_device_map(device)
model_kwargs: dict = {
"trust_remote_code": resolved,
"device_map": dev_map,
"num_labels": 1,
}
if tcfg is not None and tcfg.quantize_reward_model:
from soup_cli.utils.quant_menu import build_quantization_config_for_loader
quant_config_obj = build_quantization_config_for_loader(
tcfg=tcfg, base=model_path, console=console,
)
if quant_config_obj is not None:
model_kwargs["quantization_config"] = quant_config_obj
reward_model = AutoModelForSequenceClassification.from_pretrained(
model_path,
**model_kwargs,
)
reward_model.eval()
return reward_model
def _prepare_ppo_dataset(data: list[dict], tokenizer: Any | None = None) -> list[dict]:
"""Convert dataset rows to PPO format.
PPO expects each row to have a 'prompt_text' field (string for tokenization)
and optionally an 'answer' field (for reward function scoring).
Input can be:
- messages format: [{"role": "user", "content": "..."}, ...]
- DPO format: {"prompt": "...", "chosen": "...", "rejected": "..."}
- prompt field: {"prompt": "..."}
- alpaca format: {"instruction": "...", "output": "..."}
Returns list of dicts with 'prompt_text' (string) and optional 'answer'.
"""
prepared = []
for row in data:
if "prompt" in row and isinstance(row["prompt"], str):
entry = {"prompt_text": row["prompt"]}
if "answer" in row:
entry["answer"] = row["answer"]
prepared.append(entry)
elif "messages" in row:
messages = row["messages"]
prompt_text = None
if (
tokenizer is not None
and getattr(tokenizer, "chat_template", None)
and callable(getattr(tokenizer, "apply_chat_template", None))
):
prompt_messages = [
msg
for msg in messages
if isinstance(msg, dict) and msg.get("role") in ("system", "user")
]
if not prompt_messages:
prompt_messages = messages
try:
prompt_text = tokenizer.apply_chat_template(
prompt_messages, tokenize=False, add_generation_prompt=True
)
except Exception as exc:
console.print(
f"[yellow]Warning:[/] apply_chat_template failed: {escape(str(exc))}; "
"falling back to joined text"
)
prompt_text = None
if prompt_text is None:
# Use user messages as prompt text
prompt_parts = [
msg.get("content", "")
for msg in messages
if isinstance(msg, dict) and msg.get("role") in ("system", "user")
]
prompt_text = " ".join(prompt_parts)
entry = {"prompt_text": prompt_text}
if "answer" in row:
entry["answer"] = row["answer"]
prepared.append(entry)
elif "prompt" in row and isinstance(row["prompt"], list):
prompt_list = row["prompt"]
prompt_text = None
if (
tokenizer is not None
and getattr(tokenizer, "chat_template", None)
and callable(getattr(tokenizer, "apply_chat_template", None))
):
try:
prompt_text = tokenizer.apply_chat_template(
prompt_list, tokenize=False, add_generation_prompt=True
)
except Exception as exc:
console.print(
f"[yellow]Warning:[/] apply_chat_template failed: {escape(str(exc))}; "
"falling back to joined text"
)
prompt_text = None
if prompt_text is None:
# Message list → join content
prompt_parts = [
msg.get("content", "")
for msg in prompt_list
if isinstance(msg, dict)
]
prompt_text = " ".join(prompt_parts)
entry = {"prompt_text": prompt_text}
if "answer" in row:
entry["answer"] = row["answer"]
prepared.append(entry)
else:
# Alpaca fallback
instruction = row.get("instruction", row.get("input", ""))
entry = {"prompt_text": str(instruction)}
if "output" in row:
entry["answer"] = row["output"]
prepared.append(entry)
return prepared