Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Pretrain (Continued Pre-training) trainer — wraps HuggingFace SFTTrainer for CLM."""
import time
from pathlib import Path
from typing import Optional
from rich.console import Console
from soup_cli.config.schema import SoupConfig
from soup_cli.trainer.loss_summary import summarize_training_loss
from soup_cli.utils.eval_schedule import training_eval_kwargs
from soup_cli.utils.gpu import (
bf16_fp16_flags,
estimate_batch_size,
model_size_from_name,
resolve_base_load_dtype,
resolve_device_map,
)
from soup_cli.utils.mixed_precision import align_trainable_dtype_for_fp16
from soup_cli.utils.run_provenance import base_revision_kwargs
from soup_cli.utils.seeding import apply_training_seed, training_seed_kwargs
console = Console()
class PretrainTrainerWrapper:
"""High-level wrapper for continued pre-training from SoupConfig.
Pre-training uses raw text data (no instruction/response structure).
Each sample is a plain text document that the model learns via
causal language modelling (next-token prediction).
"""
def __init__(
self,
config: SoupConfig,
device: str = "cuda",
report_to: str = "none",
deepspeed_config: Optional[str] = None,
fsdp_config: Optional[dict] = None,
trust_remote_code: bool = False,
):
self.config = config
self.device = device
self.report_to = report_to
self.deepspeed_config = deepspeed_config
self.fsdp_config = fsdp_config
self.trust_remote_code = trust_remote_code
from soup_cli.utils.trust_remote import (
model_requires_trust_remote_code,
resolve_trust_remote_code,
)
requires = model_requires_trust_remote_code(config.base) or False
self._trust_remote_code = resolve_trust_remote_code(
config.base,
requested=trust_remote_code,
console=console,
requires_remote_code=requires,
)
self.model = None
self.tokenizer = None
self.trainer = None
self._output_dir = None
def setup(self, dataset: dict) -> None:
"""Load model, tokenizer, apply LoRA, create trainer for CLM."""
from datasets import Dataset
from transformers import TrainingArguments
from trl import SFTTrainer
# Enable Rich progress bar for HuggingFace downloads
from soup_cli.trainer.sft import _enable_hf_transfer_progress
_enable_hf_transfer_progress()
cfg = self.config
tcfg = cfg.training
# #353: seed before the model and any adapter are built.
apply_training_seed(tcfg)
use_unsloth = cfg.backend == "unsloth"
if use_unsloth:
self._setup_unsloth(cfg, tcfg)
else:
self._setup_transformers(cfg, tcfg)
# #307 — a LISA run has no PEFT wrapper, so `get_nb_trainable_parameters`
# (a PeftModel method) is absent and the count comes straight off the
# parameters. The label follows the same split: reporting "LoRA applied"
# for a run that applied no adapter is the docs-contradict-code defect.
if tcfg.lisa_enabled:
trainable = sum(
param.numel()
for param in self.model.parameters()
if param.requires_grad
)
total = sum(param.numel() for param in self.model.parameters())
label = "LISA"
else:
trainable, total = self.model.get_nb_trainable_parameters()
label = "LoRA applied"
pct = 100 * trainable / total
console.print(
f"[green]{label}:[/] {trainable:,} trainable"
f" / {total:,} total ({pct:.2f}%)"
)
# --- Batch size ---
batch_size = tcfg.batch_size
if batch_size == "auto":
from soup_cli.utils.gpu import get_gpu_info
gpu_info = get_gpu_info()
model_size = model_size_from_name(cfg.base)
batch_size = estimate_batch_size(
model_params_b=model_size,
seq_length=cfg.data.max_length,
gpu_memory_bytes=gpu_info["memory_total_bytes"],
quantization=tcfg.quantization,
lora_r=tcfg.lora.r,
)
if batch_size <= 0:
batch_size = 1
console.print(f"[green]Auto batch size:[/] {batch_size}")
# --- Dataset ---
# v0.53.7 #86 — short-circuit tokenization when caller pre-tokenized
# via `soup data preprocess`. Skips the raw-text load entirely.
from soup_cli.trainer.sft import _maybe_load_pretokenized
# #1054: pass ``tcfg`` so the cache key is derived the same way as in
# ``soup data preprocess``. The schema refuses ``training.train_on_eot``
# on ``task: pretrain`` (#806), so the mask mode here is the default.
pretok = _maybe_load_pretokenized(
cfg.data, cfg.base, console, getattr(cfg, "training", None),
task=cfg.task,
)
if pretok is not None:
train_ds, eval_ds = pretok
else:
# Plaintext: data already has {"text": "..."} from format conversion
train_ds = Dataset.from_list(dataset["train"])
eval_ds = None
if "val" in dataset and dataset["val"]:
eval_ds = Dataset.from_list(dataset["val"])
# --- Output dir ---
output_dir = Path(cfg.output)
if cfg.experiment_name:
output_dir = output_dir / cfg.experiment_name
output_dir.mkdir(parents=True, exist_ok=True)
# --- Calculate warmup steps from ratio (#1431) ---
from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
from soup_cli.utils.warmup import resolve_trainer_warmup_steps
warmup_steps = resolve_trainer_warmup_steps(tcfg.warmup_ratio)
# --- Training args ---
_bf16, _fp16 = bf16_fp16_flags(self.device)
training_kwargs = {
"output_dir": str(output_dir),
"num_train_epochs": tcfg.epochs,
"per_device_train_batch_size": batch_size,
"gradient_accumulation_steps": tcfg.gradient_accumulation_steps,
"learning_rate": tcfg.lr,
"warmup_steps": warmup_steps,
"gradient_checkpointing": should_enable_hf_gradient_checkpointing(
tcfg.gradient_checkpointing, stream_layers=tcfg.stream_layers
),
"weight_decay": tcfg.weight_decay,
"max_grad_norm": tcfg.max_grad_norm,
"optim": tcfg.optimizer,
"lr_scheduler_type": tcfg.scheduler,
"logging_steps": tcfg.logging_steps,
"save_steps": tcfg.save_steps,
"save_total_limit": 3,
"bf16": _bf16,
"fp16": _fp16,
"report_to": self.report_to,
"remove_unused_columns": False,
"deepspeed": self.deepspeed_config,
**training_seed_kwargs(tcfg),
**training_eval_kwargs(cfg, eval_ds, batch_size=batch_size),
}
# FSDP2 — alternative to DeepSpeed
if self.fsdp_config:
training_kwargs.update(self.fsdp_config)
# LoRA+ — different learning rates for A and B matrices. Not a
# TrainingArguments field: the optimizer is built and attached after the
# trainer exists (attach_loraplus_optimizer), so it must NOT be forwarded
# here (#724).
# LoRA-FA — freezes LoRA A matrices and trains B matrices. Not a
# TrainingArguments field: the optimizer is built and attached after the
# trainer exists (attach_lorafa_optimizer), so it must NOT be forwarded
# here (#725).
# GaLore — memory-efficient full-parameter training
if tcfg.use_galore:
from soup_cli.utils.galore import get_galore_optimizer_and_params
if tcfg.optimizer != "adamw_torch":
console.print(
f"[yellow]GaLore overrides optimizer '{tcfg.optimizer}' "
f"with 'galore_adamw'.[/]"
)
galore_kwargs = get_galore_optimizer_and_params(
galore_rank=tcfg.galore_rank,
galore_update_proj_gap=tcfg.galore_update_proj_gap,
galore_scale=tcfg.galore_scale,
)
training_kwargs.update(galore_kwargs)
console.print(
f"[green]GaLore enabled:[/] rank={tcfg.galore_rank}, "
f"update_gap={tcfg.galore_update_proj_gap}, scale={tcfg.galore_scale}"
)
training_args = TrainingArguments(**training_kwargs)
from soup_cli.trainer.sft import SFTTrainerWrapper
training_args = SFTTrainerWrapper._as_sft_config(
training_args, cfg.data.max_length, packing=tcfg.packing,
)
# --- Trainer ---
trainer_kwargs = {
"model": self.model,
"args": training_args,
"train_dataset": train_ds,
"eval_dataset": eval_ds,
"processing_class": self.tokenizer,
}
# Sample packing — pack multiple short samples into one sequence
if tcfg.packing:
console.print("[green]Sample packing enabled[/]")
# v0.40.4 #65 — multipack live wiring (mirrors sft.py).
use_multipack = bool(getattr(tcfg, "multipack", False))
if use_multipack:
from soup_cli.utils.multipack_sampler import (
validate_multipack_architecture,
)
from soup_cli.utils.multipack_trainer import (
attach_multipack_state,
detect_arch_name,
lengths_from_dataset,
make_multipack_trainer_class,
)
arch = detect_arch_name(self.model)
if arch:
validate_multipack_architecture(arch)
trainer_cls = make_multipack_trainer_class(SFTTrainer)
self.trainer = trainer_cls(**trainer_kwargs)
attach_multipack_state(
self.trainer,
lengths=lengths_from_dataset(train_ds),
max_seq_len=cfg.data.max_length,
batch_size=batch_size,
seed=getattr(tcfg, "seed", 0) or 0,
)
console.print("[green]Multipack FFD bin-packing sampler enabled[/]")
else:
self.trainer = SFTTrainer(**trainer_kwargs)
# #359 - the same exposure #336 fixed in sft.py: with LoRA the
# no-decay optimizer group is empty, DeepSpeed drops it, and the LR
# scheduler keeps two base_lrs until torch's strict zip raises at the
# first step. The guard prunes inside create_optimizer, i.e. before
# the scheduler is built. No-op for full fine-tuning, and only under
# DeepSpeed so the ordinary path keeps its own optimizer.
if self.deepspeed_config:
from soup_cli.utils.deepspeed import attach_empty_param_group_guard
attach_empty_param_group_guard(self.trainer)
# v0.40.6 #67 — ReLoRA merge-and-reinitialize restart callback.
from soup_cli.utils.peft_wiring import (
attach_curriculum_callback,
attach_lisa_callback,
attach_lorafa_optimizer,
attach_loraplus_optimizer,
attach_plugin_callback,
attach_relora_callback,
)
# LoRA+ optimizer (#724) — build and attach now that the trainer exists.
attach_loraplus_optimizer(self.trainer, tcfg)
# LoRA-FA optimizer (#725) — build and attach now that the trainer exists.
attach_lorafa_optimizer(self.trainer, tcfg)
attach_relora_callback(self.trainer, tcfg)
# #307 — LISA layerwise importance sampling (v0.71.34 #267 for sft).
attach_lisa_callback(self.trainer, tcfg)
# v0.53.5 #114/#115 — dynamic curriculum live callback.
attach_curriculum_callback(self.trainer, tcfg, str(output_dir), console)
# v0.53.6 #101 — Soup plugin TrainerCallback.
attach_plugin_callback(self.trainer, console)
self._output_dir = str(output_dir)
self._batch_size = batch_size
def _setup_transformers(self, cfg: SoupConfig, tcfg) -> None:
"""Load model via standard transformers + peft pipeline."""
from peft import TaskType, get_peft_model, prepare_model_for_kbit_training
from transformers import AutoModelForCausalLM, AutoTokenizer
from soup_cli.utils.moe import detect_moe_model
console.print(f"[dim]Loading tokenizer: {cfg.base}[/]")
self.tokenizer = AutoTokenizer.from_pretrained(
cfg.base, trust_remote_code=self._trust_remote_code,
**base_revision_kwargs(cfg.base),
)
if self.tokenizer.pad_token is None:
self.tokenizer.pad_token = self.tokenizer.eos_token
# Quantization (v0.38.0 Quant Menu — see soup_cli.utils.quant_menu)
from soup_cli.utils.quant_menu import build_quantization_config_for_loader
quant_config_obj = build_quantization_config_for_loader(
tcfg=tcfg, base=cfg.base, console=console,
)
console.print(f"[dim]Loading model: {cfg.base}[/]")
# On CPU, use device_map="cpu" to avoid meta tensors from "auto"
dev_map = resolve_device_map(self.device)
model_kwargs = {
"trust_remote_code": self._trust_remote_code, "device_map": dev_map,
"torch_dtype": resolve_base_load_dtype(
self.device, full_finetune=tcfg.lisa_enabled
),
}
if quant_config_obj is not None:
model_kwargs["quantization_config"] = quant_config_obj
rope_config = None
if tcfg.rope_scaling_type:
from transformers import AutoConfig
from soup_cli.utils.long_context import apply_long_context_config
model_config = AutoConfig.from_pretrained(
cfg.base, trust_remote_code=self._trust_remote_code,
**base_revision_kwargs(cfg.base),
)
rope_config = apply_long_context_config(
model_config,
target_length=cfg.data.max_length,
rope_scaling_type=tcfg.rope_scaling_type,
model_name=cfg.base,
yarn_factor=tcfg.yarn_factor,
yarn_attn_factor=tcfg.yarn_attn_factor,
yarn_beta_fast=tcfg.yarn_beta_fast,
yarn_beta_slow=tcfg.yarn_beta_slow,
)
if rope_config:
model_kwargs["config"] = model_config
self.model = AutoModelForCausalLM.from_pretrained(
cfg.base, **model_kwargs,
**base_revision_kwargs(cfg.base),
)
if rope_config:
console.print(
f"[green]Long-context enabled:[/] RoPE {tcfg.rope_scaling_type} "
f"scaling to {cfg.data.max_length} tokens"
)
# MoE aux loss for load balancing
is_moe = detect_moe_model(self.model)
if is_moe and tcfg.moe_aux_loss_coeff > 0:
if hasattr(self.model.config, "router_aux_loss_coef"):
self.model.config.router_aux_loss_coef = tcfg.moe_aux_loss_coeff
if hasattr(self.model.config, "output_router_logits"):
self.model.config.output_router_logits = True
console.print(
f"[green]MoE detected:[/] aux_loss_coeff={tcfg.moe_aux_loss_coeff}"
)
if tcfg.quantization in ("4bit", "8bit"):
from soup_cli.utils.layer_stream import should_enable_hf_gradient_checkpointing
self.model = prepare_model_for_kbit_training(
self.model,
use_gradient_checkpointing=should_enable_hf_gradient_checkpointing(
tcfg.gradient_checkpointing, stream_layers=tcfg.stream_layers
),
)
# v0.53.4 #83 — LLaMA Pro block expansion (centralised — see SFT).
from soup_cli.utils.block_expansion import (
apply_block_expansion_if_configured,
)
apply_block_expansion_if_configured(self.model, tcfg, console)
# #307 — LISA layerwise importance sampling. Full-FT of a rotating set
# of decoder layers, so it fully replaces the LoRA path below (the
# schema cross-validator guarantees no LoRA-feature / freeze flag is
# combined). Shared with the SFT trainer so the two cannot drift.
from soup_cli.utils.peft_wiring import (
apply_lisa_setup,
build_lora_config,
resolve_lora_target_modules,
resolve_lora_target_parameters,
)
if not apply_lisa_setup(self.model, tcfg, console):
# LoRA — with MoE-aware target modules if moe_lora is enabled
target_modules = resolve_lora_target_modules(
self.model, tcfg.lora.target_modules, console
)
target_parameters = resolve_lora_target_parameters(
self.model, tcfg.lora.target_parameters
)
# #798: the same helper every other trainer uses (see sft.py).
from soup_cli.utils.moe import resolve_moe_lora_targets
target_modules = resolve_moe_lora_targets(
self.model, tcfg, target_modules, console
)
lora_config = build_lora_config(
tcfg.lora,
target_modules=target_modules,
target_parameters=target_parameters,
task_type=TaskType.CAUSAL_LM,
)
# v0.40.6 #67 — surgical PEFT patches.
from soup_cli.utils.peft_wiring import (
apply_post_lora_patches,
apply_pre_lora_patches,
)
apply_pre_lora_patches(self.model, cfg.base)
self.model = get_peft_model(self.model, lora_config)
apply_post_lora_patches(self.model)
# v0.71.12 #84 — Mixture-of-Depths selective-token routing (applied
# after get_peft_model so the routers are trainable).
from soup_cli.utils.mod import apply_mod_if_configured
apply_mod_if_configured(self.model, tcfg, cfg.base, console)
# QAT — insert fake quantization ops after LoRA
if tcfg.quantization_aware and tcfg.quantization_aware != "fp8":
from soup_cli.utils.qat import prepare_model_for_qat
self.model = prepare_model_for_qat(self.model)
# v0.33.0 #43 — multi-trainer wiring of v0.28.0 speed/memory features.
from soup_cli.utils.v028_features import apply_v028_speed_memory
apply_v028_speed_memory(
model=self.model, tcfg=tcfg, base_model=cfg.base,
console=console, device=self.device, backend=cfg.backend,
)
def _setup_unsloth(self, cfg: SoupConfig, tcfg) -> None:
"""Load model via unsloth FastLanguageModel (2-5x faster)."""
from soup_cli.utils import unsloth
from soup_cli.utils.unsloth import load_model_and_tokenizer
# #797: resolve training.gradient_checkpointing for the Unsloth path
# (streaming layers already checkpoint every layer, so it stays off).
use_gradient_checkpointing = unsloth.resolve_unsloth_gradient_checkpointing(
tcfg.gradient_checkpointing, stream_layers=getattr(tcfg, "stream_layers", False)
)
console.print(f"[dim]Loading model via [bold]unsloth[/]: {cfg.base}[/]")
self.model, self.tokenizer = load_model_and_tokenizer(
model_name=cfg.base,
max_seq_length=cfg.data.max_length,
quantization=tcfg.quantization,
lora_r=tcfg.lora.r,
lora_alpha=tcfg.lora.alpha,
lora_dropout=tcfg.lora.dropout,
target_modules=tcfg.lora.target_modules,
use_rslora=tcfg.lora.use_rslora,
use_gradient_checkpointing=use_gradient_checkpointing,
)
if self.tokenizer.pad_token is None:
self.tokenizer.pad_token = self.tokenizer.eos_token
def train(
self,
display: Optional[object] = None,
tracker: Optional[object] = None,
run_id: str = "",
resume_from_checkpoint: Optional[str] = None,
) -> dict:
"""Run continued pre-training and return results summary."""
if self.trainer is None or self._output_dir is None:
raise RuntimeError(
"PretrainTrainerWrapper.train() called before setup(). "
"Call setup(dataset) first."
)
start = time.time()
# Add callback for live display and experiment tracking
if display:
from soup_cli.monitoring.callback import build_soup_trainer_callback
self.trainer.add_callback(
build_soup_trainer_callback(
display,
config=self.config,
tracker=tracker,
run_id=run_id,
batch_size=self._batch_size,
output_dir=self._output_dir,
)
)
from soup_cli.utils.v028_features import activation_offloading_context
with activation_offloading_context(
self.config.training, self._output_dir,
):
align_trainable_dtype_for_fp16(
self.trainer.model,
fp16=getattr(self.trainer.args, "fp16", False),
bf16=getattr(self.trainer.args, "bf16", False),
)
self.trainer.train(resume_from_checkpoint=resume_from_checkpoint)
duration = time.time() - start
# Save final model; ReLoRA output is dense.
if self.config.training.relora_steps is None:
self.trainer.save_model(self._output_dir)
else:
from soup_cli.utils.peft_wiring import save_model_with_relora
save_model_with_relora(
self.trainer, self._output_dir, self.config.training.relora_steps
)
self.tokenizer.save_pretrained(self._output_dir)
# Extract metrics
logs = self.trainer.state.log_history
loss_summary = summarize_training_loss(logs)
hours = int(duration // 3600)
minutes = int((duration % 3600) // 60)
duration_str = f"{hours}h {minutes}m" if hours > 0 else f"{minutes}m"
return {
**loss_summary,
"duration": duration_str,
"duration_secs": duration,
"output_dir": self._output_dir,
"total_steps": self.trainer.state.global_step,
}