Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Refuse Trainer(model_init=...) under UNSLOTH_TP_SIZE instead of silently skipping tensor parallel (#15424)
Daniel Hanf5d14f532633
diff --git a/tests/test_tp_env_gloo.py b/tests/test_tp_env_gloo.pyindex 0e2033f689d..b8c18d0beb8 100644--- a/tests/test_tp_env_gloo.py+++ b/tests/test_tp_env_gloo.py@@ -419,12 +419,102 @@ def _case_accelerate_fsdp(rank, world, tmp, ckpt):}+def _case_model_init(+ rank,+ world,+ tmp,+ ckpt,+ hooked,+ with_model = False,+ import_only = False,+):+ """Trainer(model_init = ...) under UNSLOTH_TP_SIZE: model_init runs after the tp setup saw model = None."""+ error = _import_checkout_unsloth()+ if error is not None:+ return {"skip": error}+ import transformers+ from peft import LoraConfig, get_peft_model+ from transformers import AutoModelForCausalLM++ from unsloth.models import tp_env, tp_native+ from unsloth.models._utils import patch_gradient_accumulation_fix++ if not import_only:+ patch_gradient_accumulation_fix(transformers.Trainer)++ def load(name, load_in_4bit = True):+ model = AutoModelForCausalLM.from_pretrained(name, torch_dtype = torch.float32)+ if (+ hooked and import_only+ ): # FastLanguageModel.from_pretrained installs the wrapper after loading+ patch_gradient_accumulation_fix(transformers.Trainer)+ return model++ def peft(model):+ return get_peft_model(model, LoraConfig(r = 4, target_modules = "all-linear"))++ if hooked: # FastLanguageModel.from_pretrained / get_peft_model inside model_init+ load = tp_env.tensor_parallel_env_load(load)+ peft = tp_env.tensor_parallel_env_peft(peft)+ built = []++ def model_init():+ model = peft(load(ckpt, load_in_4bit = False))+ built.append(tp_native.is_tensor_parallel(model))+ return model++ data = _Rows([torch.arange(9)] * 8)+ trainer = []+ if import_only:++ def run():+ trainer.append(+ transformers.Trainer(+ model_init = model_init, args = _trainer_args(tmp, 2), train_dataset = data+ )+ )+ trainer[0].train()++ refused = _raised(run)+ elif (+ with_model+ ): # transformers 4.x accepts both (FutureWarning) and train() rebuilds from model_init+ model = peft(load(ckpt, load_in_4bit = False))++ def run():+ trainer.append(+ transformers.Trainer(+ model = model,+ model_init = model_init,+ args = _trainer_args(tmp, 2),+ train_dataset = data,+ )+ )+ trainer[0].train()++ refused = _raised(run)+ else:+ refused = _raised(+ lambda: trainer.append(+ transformers.Trainer(+ model_init = model_init, args = _trainer_args(tmp, 2), train_dataset = data+ )+ )+ )+ return {+ "refused": refused,+ "built": built,+ "is_tp": bool(trainer) and tp_native.is_tensor_parallel(trainer[0].model),+ }++CASES = {"units": _case_units,"env": _case_env,"dp_average": _case_dp_average,"dp_average_dtypes": _case_dp_average_dtypes,"accelerate_fsdp": _case_accelerate_fsdp,+ "model_init": _case_model_init,}@@ -460,7 +550,7 @@ def _spawn(ctx = multiprocessing.get_context("spawn")port = _free_port()with tempfile.TemporaryDirectory() as tmp:- if case in ("env", "accelerate_fsdp"):+ if case in ("env", "accelerate_fsdp", "model_init"):kwargs["ckpt"] = _checkpoint(tmp)procs = [ctx.Process(target = _worker, args = (r, world, port, case, tmp, env or {}, kwargs))@@ -681,3 +771,27 @@ def test_env_tp_x_accelerate_fsdp_is_refused():env = {"UNSLOTH_TP_SIZE": "2", "UNSLOTH_FSDP2": "1", "ACCELERATE_USE_FSDP": "true"}for res in _spawn("accelerate_fsdp", 4, env):assert "needs Unsloth's native FSDP2" in (res["refused"] or ""), res++[email protected]("hooked", [False, True], ids = ["plain", "unsloth_loaders"])+def test_env_tp_model_init_is_refused(hooked):+ """Unrefused, a model_init model silently trained as plain DDP despite UNSLOTH_TP_SIZE=2."""+ for res in _spawn("model_init", 2, {"UNSLOTH_TP_SIZE": "2"}, hooked = hooked):+ assert "UNSLOTH_TP_SIZE=2" in (res["refused"] or ""), res+ assert "model_init" in res["refused"] and res["built"] == [], res+++def test_env_tp_model_and_model_init_is_refused():+ """Trainer(model = m, model_init = f): 4.x train() swapped tp m for a plain DDP model; 5.x raises ValueError."""+ for res in _spawn("model_init", 2, {"UNSLOTH_TP_SIZE": "2"}, hooked = False, with_model = True):+ assert res["refused"] is not None and res["built"] == [], res+ assert "model_init" in res["refused"], res++[email protected]("hooked", [False, True], ids = ["plain", "unsloth_loaders"])+def test_env_tp_model_init_is_refused_after_plain_import(hooked):+ """Only `import unsloth`: nothing installed the Trainer wrapper before model_init ran, so it trained."""+ env = {"UNSLOTH_TP_SIZE": "2"}+ for res in _spawn("model_init", 2, env, hooked = hooked, import_only = True):+ assert "UNSLOTH_TP_SIZE=2" in (res["refused"] or ""), res+ assert "model_init" in res["refused"] and res["built"] == [], resdiff --git a/unsloth/__init__.py b/unsloth/__init__.pyindex 5d749ca9099..3dcc26c8f15 100644--- a/unsloth/__init__.py+++ b/unsloth/__init__.py@@ -1691,6 +1691,13 @@ else:_install_native_trainer_init()del _install_native_trainer_init+ if os.environ.get("UNSLOTH_TP_SIZE", "").strip() not in ("", "1"):+ from .models.tp_env import (+ install_tensor_parallel_env_trainer_init as _install_tp_trainer_init,+ )+ _install_tp_trainer_init()+ del _install_tp_trainer_init+def get_gpu_memory_stats():"""Return CUDA/ROCm/XPU/NPU device stats, peak memory, and total memory in GiB."""try:diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 513577dfdd0..67c386c59f6 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -6030,6 +6030,10 @@ def patch_gradient_accumulation_fix(Trainer):_refuse_pipeline_parallel_for_trainer(model)# UNSLOTH_TP_SIZE (unmodified scripts): tensor parallel before native FSDP2 shards the dp dimension.tp_env = _tensor_parallel_env_trainer_pre(model)+ if (+ tp_env is not None+ ): # model_init alone, or beside model (4.x: train() rebuilds from it)+ _refuse_tensor_parallel_env_model_init(tp_env, _original_trainer_init, args, kwargs)from .. import fsdp2_launch as _fsdp2_launch# Env reads only; fsdp2_native (and torch.distributed.fsdp) is imported only under the flag.@@ -6199,6 +6203,46 @@ def _tensor_parallel_env_trainer_pre(model):return tensor_parallel_env_trainer_pre(model)+def _refuse_tensor_parallel_env_model_init(layout, init, args, kwargs):+ """UNSLOTH_TP_SIZE setup runs on the __init__ model, so a model_init model would silently train as plain DDP."""+ model_init = kwargs.get("model_init")+ if model_init is None:+ import inspect++ try:+ names = [+ p.name+ for p in inspect.signature(init).parameters.values()+ if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)+ ]+ except (TypeError, ValueError):+ names = []+ if names and names[0] == "self":+ names = names[1:]+ if (+ "model" not in names+ ): # Trainer.__init__ wrapped without functools.wraps: transformers' order+ names = [+ "model",+ "args",+ "data_collator",+ "train_dataset",+ "eval_dataset",+ "processing_class",+ "model_init",+ ]+ if "model_init" in names:+ index = names.index("model_init")+ model_init = args[index] if index < len(args) else None+ if model_init is not None:+ raise RuntimeError(+ f"Unsloth: UNSLOTH_TP_SIZE={layout.tp} (tensor parallel) does not support "+ "`Trainer(model_init = ...)` or `hyperparameter_search`: the model model_init builds would not "+ "be tensor parallel. Build the model first and pass it as `Trainer(model = model)`, or unset "+ "UNSLOTH_TP_SIZE."+ )++def _tensor_parallel_trainer_pre(model):if not _is_unsloth_tensor_parallel(model):return Falsediff --git a/unsloth/models/tp_env.py b/unsloth/models/tp_env.pyindex 2d0c7d02d46..dae8157207d 100644--- a/unsloth/models/tp_env.py+++ b/unsloth/models/tp_env.py@@ -31,6 +31,7 @@ __all__ = ["TPEnvLayout","env_layout","env_hint",+ "install_tensor_parallel_env_trainer_init","tensor_parallel_env_load","tensor_parallel_env_peft","tensor_parallel_env_trainer_pre",@@ -368,6 +369,19 @@ def _levels(model):return [model]+def install_tensor_parallel_env_trainer_init():+ """Install the Trainer.__init__ wrapper at import: loads that all run inside model_init would otherwise+ train in an unwrapped Trainer, without tensor parallel."""+ if str(os.environ.get("UNSLOTH_TP_SIZE", "")).strip() in ("", "1"):+ return False+ from transformers import Trainer++ from ._utils import patch_gradient_accumulation_fix++ patch_gradient_accumulation_fix(Trainer)+ return True++def tensor_parallel_env_trainer_pre(model):"""Trainer.__init__ before anything else touches the model: tensor parallel (if get_peft_model did notapply it) and the DDP-style dp dimension. Returns the layout (None without UNSLOTH_TP_SIZE)."""