Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Refuse Trainer(model_init=...) under UNSLOTH_TP_SIZE instead of silently skipping tensor parallel (#15424)

Daniel Hanf5d14f532633
diff --git a/tests/test_tp_env_gloo.py b/tests/test_tp_env_gloo.py
index 0e2033f689d..b8c18d0beb8 100644
--- a/tests/test_tp_env_gloo.py
+++ b/tests/test_tp_env_gloo.py
@@ -419,12 +419,102 @@ def _case_accelerate_fsdp(rank, world, tmp, ckpt):
}
+def _case_model_init(
+ rank,
+ world,
+ tmp,
+ ckpt,
+ hooked,
+ with_model = False,
+ import_only = False,
+):
+ """Trainer(model_init = ...) under UNSLOTH_TP_SIZE: model_init runs after the tp setup saw model = None."""
+ error = _import_checkout_unsloth()
+ if error is not None:
+ return {"skip": error}
+ import transformers
+ from peft import LoraConfig, get_peft_model
+ from transformers import AutoModelForCausalLM
+
+ from unsloth.models import tp_env, tp_native
+ from unsloth.models._utils import patch_gradient_accumulation_fix
+
+ if not import_only:
+ patch_gradient_accumulation_fix(transformers.Trainer)
+
+ def load(name, load_in_4bit = True):
+ model = AutoModelForCausalLM.from_pretrained(name, torch_dtype = torch.float32)
+ if (
+ hooked and import_only
+ ): # FastLanguageModel.from_pretrained installs the wrapper after loading
+ patch_gradient_accumulation_fix(transformers.Trainer)
+ return model
+
+ def peft(model):
+ return get_peft_model(model, LoraConfig(r = 4, target_modules = "all-linear"))
+
+ if hooked: # FastLanguageModel.from_pretrained / get_peft_model inside model_init
+ load = tp_env.tensor_parallel_env_load(load)
+ peft = tp_env.tensor_parallel_env_peft(peft)
+ built = []
+
+ def model_init():
+ model = peft(load(ckpt, load_in_4bit = False))
+ built.append(tp_native.is_tensor_parallel(model))
+ return model
+
+ data = _Rows([torch.arange(9)] * 8)
+ trainer = []
+ if import_only:
+
+ def run():
+ trainer.append(
+ transformers.Trainer(
+ model_init = model_init, args = _trainer_args(tmp, 2), train_dataset = data
+ )
+ )
+ trainer[0].train()
+
+ refused = _raised(run)
+ elif (
+ with_model
+ ): # transformers 4.x accepts both (FutureWarning) and train() rebuilds from model_init
+ model = peft(load(ckpt, load_in_4bit = False))
+
+ def run():
+ trainer.append(
+ transformers.Trainer(
+ model = model,
+ model_init = model_init,
+ args = _trainer_args(tmp, 2),
+ train_dataset = data,
+ )
+ )
+ trainer[0].train()
+
+ refused = _raised(run)
+ else:
+ refused = _raised(
+ lambda: trainer.append(
+ transformers.Trainer(
+ model_init = model_init, args = _trainer_args(tmp, 2), train_dataset = data
+ )
+ )
+ )
+ return {
+ "refused": refused,
+ "built": built,
+ "is_tp": bool(trainer) and tp_native.is_tensor_parallel(trainer[0].model),
+ }
+
+
CASES = {
"units": _case_units,
"env": _case_env,
"dp_average": _case_dp_average,
"dp_average_dtypes": _case_dp_average_dtypes,
"accelerate_fsdp": _case_accelerate_fsdp,
+ "model_init": _case_model_init,
}
@@ -460,7 +550,7 @@ def _spawn(
ctx = multiprocessing.get_context("spawn")
port = _free_port()
with tempfile.TemporaryDirectory() as tmp:
- if case in ("env", "accelerate_fsdp"):
+ if case in ("env", "accelerate_fsdp", "model_init"):
kwargs["ckpt"] = _checkpoint(tmp)
procs = [
ctx.Process(target = _worker, args = (r, world, port, case, tmp, env or {}, kwargs))
@@ -681,3 +771,27 @@ def test_env_tp_x_accelerate_fsdp_is_refused():
env = {"UNSLOTH_TP_SIZE": "2", "UNSLOTH_FSDP2": "1", "ACCELERATE_USE_FSDP": "true"}
for res in _spawn("accelerate_fsdp", 4, env):
assert "needs Unsloth's native FSDP2" in (res["refused"] or ""), res
+
+
[email protected]("hooked", [False, True], ids = ["plain", "unsloth_loaders"])
+def test_env_tp_model_init_is_refused(hooked):
+ """Unrefused, a model_init model silently trained as plain DDP despite UNSLOTH_TP_SIZE=2."""
+ for res in _spawn("model_init", 2, {"UNSLOTH_TP_SIZE": "2"}, hooked = hooked):
+ assert "UNSLOTH_TP_SIZE=2" in (res["refused"] or ""), res
+ assert "model_init" in res["refused"] and res["built"] == [], res
+
+
+def test_env_tp_model_and_model_init_is_refused():
+ """Trainer(model = m, model_init = f): 4.x train() swapped tp m for a plain DDP model; 5.x raises ValueError."""
+ for res in _spawn("model_init", 2, {"UNSLOTH_TP_SIZE": "2"}, hooked = False, with_model = True):
+ assert res["refused"] is not None and res["built"] == [], res
+ assert "model_init" in res["refused"], res
+
+
[email protected]("hooked", [False, True], ids = ["plain", "unsloth_loaders"])
+def test_env_tp_model_init_is_refused_after_plain_import(hooked):
+ """Only `import unsloth`: nothing installed the Trainer wrapper before model_init ran, so it trained."""
+ env = {"UNSLOTH_TP_SIZE": "2"}
+ for res in _spawn("model_init", 2, env, hooked = hooked, import_only = True):
+ assert "UNSLOTH_TP_SIZE=2" in (res["refused"] or ""), res
+ assert "model_init" in res["refused"] and res["built"] == [], res
diff --git a/unsloth/__init__.py b/unsloth/__init__.py
index 5d749ca9099..3dcc26c8f15 100644
--- a/unsloth/__init__.py
+++ b/unsloth/__init__.py
@@ -1691,6 +1691,13 @@ else:
_install_native_trainer_init()
del _install_native_trainer_init
+ if os.environ.get("UNSLOTH_TP_SIZE", "").strip() not in ("", "1"):
+ from .models.tp_env import (
+ install_tensor_parallel_env_trainer_init as _install_tp_trainer_init,
+ )
+ _install_tp_trainer_init()
+ del _install_tp_trainer_init
+
def get_gpu_memory_stats():
"""Return CUDA/ROCm/XPU/NPU device stats, peak memory, and total memory in GiB."""
try:
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 513577dfdd0..67c386c59f6 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -6030,6 +6030,10 @@ def patch_gradient_accumulation_fix(Trainer):
_refuse_pipeline_parallel_for_trainer(model)
# UNSLOTH_TP_SIZE (unmodified scripts): tensor parallel before native FSDP2 shards the dp dimension.
tp_env = _tensor_parallel_env_trainer_pre(model)
+ if (
+ tp_env is not None
+ ): # model_init alone, or beside model (4.x: train() rebuilds from it)
+ _refuse_tensor_parallel_env_model_init(tp_env, _original_trainer_init, args, kwargs)
from .. import fsdp2_launch as _fsdp2_launch
# Env reads only; fsdp2_native (and torch.distributed.fsdp) is imported only under the flag.
@@ -6199,6 +6203,46 @@ def _tensor_parallel_env_trainer_pre(model):
return tensor_parallel_env_trainer_pre(model)
+def _refuse_tensor_parallel_env_model_init(layout, init, args, kwargs):
+ """UNSLOTH_TP_SIZE setup runs on the __init__ model, so a model_init model would silently train as plain DDP."""
+ model_init = kwargs.get("model_init")
+ if model_init is None:
+ import inspect
+
+ try:
+ names = [
+ p.name
+ for p in inspect.signature(init).parameters.values()
+ if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)
+ ]
+ except (TypeError, ValueError):
+ names = []
+ if names and names[0] == "self":
+ names = names[1:]
+ if (
+ "model" not in names
+ ): # Trainer.__init__ wrapped without functools.wraps: transformers' order
+ names = [
+ "model",
+ "args",
+ "data_collator",
+ "train_dataset",
+ "eval_dataset",
+ "processing_class",
+ "model_init",
+ ]
+ if "model_init" in names:
+ index = names.index("model_init")
+ model_init = args[index] if index < len(args) else None
+ if model_init is not None:
+ raise RuntimeError(
+ f"Unsloth: UNSLOTH_TP_SIZE={layout.tp} (tensor parallel) does not support "
+ "`Trainer(model_init = ...)` or `hyperparameter_search`: the model model_init builds would not "
+ "be tensor parallel. Build the model first and pass it as `Trainer(model = model)`, or unset "
+ "UNSLOTH_TP_SIZE."
+ )
+
+
def _tensor_parallel_trainer_pre(model):
if not _is_unsloth_tensor_parallel(model):
return False
diff --git a/unsloth/models/tp_env.py b/unsloth/models/tp_env.py
index 2d0c7d02d46..dae8157207d 100644
--- a/unsloth/models/tp_env.py
+++ b/unsloth/models/tp_env.py
@@ -31,6 +31,7 @@ __all__ = [
"TPEnvLayout",
"env_layout",
"env_hint",
+ "install_tensor_parallel_env_trainer_init",
"tensor_parallel_env_load",
"tensor_parallel_env_peft",
"tensor_parallel_env_trainer_pre",
@@ -368,6 +369,19 @@ def _levels(model):
return [model]
+def install_tensor_parallel_env_trainer_init():
+ """Install the Trainer.__init__ wrapper at import: loads that all run inside model_init would otherwise
+ train in an unwrapped Trainer, without tensor parallel."""
+ if str(os.environ.get("UNSLOTH_TP_SIZE", "")).strip() in ("", "1"):
+ return False
+ from transformers import Trainer
+
+ from ._utils import patch_gradient_accumulation_fix
+
+ patch_gradient_accumulation_fix(Trainer)
+ return True
+
+
def tensor_parallel_env_trainer_pre(model):
"""Trainer.__init__ before anything else touches the model: tensor parallel (if get_peft_model did not
apply it) and the DDP-style dp dimension. Returns the layout (None without UNSLOTH_TP_SIZE)."""