Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""UNSLOTH_TP_SIZE (models/tp_env.py): tensor parallel from the environment, on a gloo CPU world, vs one process.
The workers set the torchrun environment and let tp_env create the process group and the mesh. The loader /
get_peft_model hooks wrap AutoModelForCausalLM / PEFT the way they wrap FastLanguageModel / FastModel."""
import multiprocessing
import os
if multiprocessing.parent_process() is not None or getattr(
multiprocessing.current_process(), "_inheriting", False
):
os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
import sys # noqa: E402
import tempfile # noqa: E402
import pytest # noqa: E402
import torch # noqa: E402
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from test_tp_native_gloo import ( # noqa: E402
_Rows,
_free_port,
_import_checkout_unsloth,
_raised,
_save_checkpoint,
_trainer_args,
)
# Only these arms use eps 1e-4: at the default 1e-8 AdamW turns tp=2 vs one-process fp32 reduction
# noise into 1e-4 weight drift by step 3; at 1e-4 drift stays below 1e-6 while a real missed
# all-reduce still gives 3.5e-4; B = 0 native Trainer tests pass at default eps on CI but fail at
# 1e-4.
_ADAM_EPS = 1e-4
_ENV_KEYS = (
"UNSLOTH_TP_SIZE",
"UNSLOTH_PP_SIZE",
"UNSLOTH_FSDP2",
"UNSLOTH_FSDP2_BACKEND",
"ACCELERATE_USE_FSDP",
)
def _checkpoint(tmp):
"""A tiny random Llama saved as sharded safetensors (index + shards), like a Hub checkpoint."""
return _save_checkpoint("llama", os.path.join(tmp, "ckpt"))
def _patch_bnb_8bit_everywhere():
# bitsandbytes keeps 32-bit state below min_8bit_size (4096) elements: every tiny LoRA factor would.
# transformers builds optim = "adamw_8bit" as bitsandbytes.optim.AdamW(optim_bits = 8).
import bitsandbytes as bnb
for cls in (bnb.optim.AdamW, bnb.optim.AdamW8bit):
init = cls.__init__
def patched(
self,
*args,
__init = init,
**kwargs,
):
kwargs["min_8bit_size"] = 0
__init(self, *args, **kwargs)
cls.__init__ = patched
def _case_units(rank, world, tmp):
"""Layout parsing and refusals (no process group)."""
error = _import_checkout_unsloth()
if error is not None:
return {"skip": error}
from unsloth.models import tp_env
out = {}
def layout(**env):
for key in _ENV_KEYS + ("WORLD_SIZE",):
os.environ.pop(key, None)
os.environ.update({k: str(v) for k, v in env.items()})
tp_env._reset_env_layout()
try:
got = tp_env.env_layout()
except Exception as e:
return f"{type(e).__name__}: {e}"
return None if got is None else (got.tp, got.dp, got.world, got.fsdp2)
out["unset"] = layout(WORLD_SIZE = 4)
out["one"] = layout(UNSLOTH_TP_SIZE = 1, WORLD_SIZE = 4)
out["zero"] = layout(UNSLOTH_TP_SIZE = 0, WORLD_SIZE = 4)
out["text"] = layout(UNSLOTH_TP_SIZE = "two", WORLD_SIZE = 4)
out["no_divide"] = layout(UNSLOTH_TP_SIZE = 3, WORLD_SIZE = 4)
out["no_torchrun"] = layout(UNSLOTH_TP_SIZE = 2)
out["pp"] = layout(UNSLOTH_PP_SIZE = 2, WORLD_SIZE = 4)
out["pp_with_tp"] = layout(UNSLOTH_PP_SIZE = 2, UNSLOTH_TP_SIZE = 2, WORLD_SIZE = 4)
out["pp_one"] = layout(UNSLOTH_PP_SIZE = 1, UNSLOTH_TP_SIZE = 2, WORLD_SIZE = 4)
out["tp_world"] = layout(UNSLOTH_TP_SIZE = 4, WORLD_SIZE = 4)
out["ddp"] = layout(UNSLOTH_TP_SIZE = 2, WORLD_SIZE = 4)
out["fsdp2"] = layout(UNSLOTH_TP_SIZE = 2, UNSLOTH_FSDP2 = 1, WORLD_SIZE = 4)
out["accelerate_fsdp"] = layout(
UNSLOTH_TP_SIZE = 2, UNSLOTH_FSDP2 = 1, UNSLOTH_FSDP2_BACKEND = "accelerate", WORLD_SIZE = 4
)
calls = []
def loader(
model_name,
load_in_4bit = True,
fast_inference = False,
full_finetuning = False,
**kw,
):
calls.append(model_name)
return "loaded"
layout(UNSLOTH_TP_SIZE = 2, WORLD_SIZE = 2)
wrapped = tp_env.tensor_parallel_env_load(loader)
out["refuse_vllm"] = _raised(lambda: wrapped("m", fast_inference = True))
out["refuse_8bit"] = _raised(lambda: wrapped("m", load_in_8bit = True))
out["refuse_float8"] = _raised(lambda: wrapped("m", full_finetuning = True, float8_training = True))
layout(UNSLOTH_TP_SIZE = 2, UNSLOTH_FSDP2 = 1, UNSLOTH_FSDP2_SHARDED_LOAD = 1, WORLD_SIZE = 2)
out["refuse_fsdp2_sharded_load"] = _raised(lambda: wrapped("m", False))
os.environ.pop("UNSLOTH_FSDP2_SHARDED_LOAD", None)
out["refused_before_load"] = calls == []
layout(WORLD_SIZE = 2)
out["no_env_passthrough"] = wrapped("m", fast_inference = True)
out["four_bit_default"] = tp_env._four_bit({}, loader)
out["four_bit_16"] = tp_env._four_bit({"load_in_16bit": True}, loader)
out["four_bit_qconfig"] = tp_env._four_bit(
{"load_in_4bit": False, "quantization_config": {}}, loader
)
# The hooks sit on the real loaders.
from unsloth import FastLanguageModel, FastModel, FastVisionModel
out["hooked"] = {}
for label, cls in (
("lm", FastLanguageModel),
("model", FastModel),
("vision", FastVisionModel),
):
for attr in ("from_pretrained", "get_peft_model"):
out["hooked"][f"{label}.{attr}"] = getattr(getattr(cls, attr), "_unsloth_tp_env", False)
return out
def _case_env(
rank,
world,
tmp,
ckpt,
bs,
optim = "adamw_torch",
four_bit = False,
resume = False,
bnb_8bit = False,
refusals = False,
):
error = _import_checkout_unsloth()
if error is not None:
return {"skip": error}
import torch.distributed as dist
import transformers
from peft import LoraConfig, get_peft_model
from transformers import AutoModelForCausalLM
from unsloth.models import tp_env, tp_native
from unsloth.models._utils import patch_gradient_accumulation_fix
patch_gradient_accumulation_fix(transformers.Trainer)
if bnb_8bit:
_patch_bnb_8bit_everywhere()
layout = tp_env.env_layout()
seen = []
class SequentialTrainer(transformers.Trainer):
def _get_train_sampler(self, *args, **kwargs):
return torch.utils.data.SequentialSampler(self.train_dataset)
def compute_loss(self, model, inputs, *args, **kwargs):
seen.append(inputs["input_ids"].sum().item())
return super().compute_loss(model, inputs, *args, **kwargs)
def load(
model_name,
load_in_4bit = True,
**kwargs,
): # FastLanguageModel.from_pretrained's flags
return AutoModelForCausalLM.from_pretrained(
model_name, torch_dtype = torch.float32, attn_implementation = "eager"
)
def peft(model):
torch.manual_seed(1)
config = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False)
return get_peft_model(model, config)
load = tp_env.tensor_parallel_env_load(load)
peft = tp_env.tensor_parallel_env_peft(peft)
def fresh():
model = load(ckpt, load_in_4bit = four_bit)
sharded = "_unsloth_tp_load_stats" in model.__dict__
return peft(model), sharded
dp = layout.dp if layout is not None else 1
steps = 4 if resume else 3
g = torch.Generator().manual_seed(1)
ids = torch.randint(0, 1000, (steps * 4 * max(dp, bs // 2), 9), generator = g)
data = _Rows(list(ids))
def run(out_dir, **overrides):
model, sharded = fresh()
args = _trainer_args(
out_dir, bs, optim = optim, max_steps = steps, adam_epsilon = _ADAM_EPS, **overrides
)
trainer = SequentialTrainer(model = model, args = args, train_dataset = data)
return trainer, sharded
save = dict(save_strategy = "steps", save_steps = 2) if resume else {}
out_a = os.path.join(tmp, "a")
trainer, sharded = run(out_a, **save)
trainer.train()
def summary(trainer):
logs = [h for h in trainer.state.log_history if "loss" in h]
model = trainer.model
is_tp = tp_native.is_tensor_parallel(model)
state = (
tp_native.tp_full_state_dict(model, adapters_only = True)
if is_tp
else {k: v.detach().clone() for k, v in model.state_dict().items() if "lora_" in k}
)
return {
"steps": [h["step"] for h in logs],
"losses": [h["loss"] for h in logs],
"grad_norms": [h.get("grad_norm") for h in logs],
"state": {k: v for k, v in state.items() if "lora_" in k},
}
from unsloth.models.fsdp2_bnb_optim import is_local_shard_bnb_optimizer
from unsloth.models.fsdp2_native import _inner_optimizer, is_native_fsdp2
res = {
"a": summary(trainer),
"sharded_load": sharded,
"is_tp": tp_native.is_tensor_parallel(trainer.model),
"native_fsdp2": is_native_fsdp2(trainer.model),
"ddp_mesh": tp_native._ddp_mesh(trainer.model) is not None,
"wrapped": type(trainer.model_wrapped).__name__,
"optimizer": type(_inner_optimizer(trainer.optimizer)).__module__
+ "."
+ type(_inner_optimizer(trainer.optimizer)).__name__,
"optim_bits": getattr(
getattr(_inner_optimizer(trainer.optimizer), "args", None), "optim_bits", None
),
"state_dtypes": sorted(
{
str(v.dtype)
for st in _inner_optimizer(trainer.optimizer).state.values()
for v in st.values()
if torch.is_tensor(v) and v.ndim > 0
}
),
"local_shard_bnb": is_local_shard_bnb_optimizer(_inner_optimizer(trainer.optimizer)),
"optim_arg": str(getattr(trainer.args.optim, "value", trainer.args.optim)),
"seen": list(seen),
"layout": None if layout is None else repr(layout),
}
if resume:
ckpt2 = os.path.join(out_a, "checkpoint-2")
res["ckpt_files"] = sorted(os.listdir(ckpt2)) + sorted(
os.listdir(os.path.join(ckpt2, "unsloth_tp")) if layout is not None else []
)
trainer_b, _ = run(os.path.join(tmp, "b"), **save)
trainer_b.train(resume_from_checkpoint = ckpt2)
res["b"] = summary(trainer_b)
if layout is not None:
# A checkpoint written on another layout is refused before anything is loaded.
bad = os.path.join(tmp, "bad")
if rank == 0:
import shutil
shutil.copytree(ckpt2, bad)
dist.barrier()
path = os.path.join(bad, "unsloth_tp", f"rank{rank:05d}.pt")
payload = torch.load(path, weights_only = True)
payload["layout"]["tp"] = 99
torch.save(payload, path)
trainer_c, _ = run(os.path.join(tmp, "c"), **save)
res["bad_layout"] = _raised(lambda: trainer_c.train(resume_from_checkpoint = bad))
dist.barrier()
if refusals:
model = trainer.model
class Stream(torch.utils.data.IterableDataset):
def __iter__(self):
return iter([data[i] for i in range(len(data))])
res["iterable"] = _raised(
lambda: SequentialTrainer(
model = model, args = _trainer_args(os.path.join(tmp, "it"), bs), train_dataset = Stream()
).train()
)
res["eval"] = _raised(
lambda: SequentialTrainer(
model = model,
args = _trainer_args(
os.path.join(tmp, "ev"), bs, eval_strategy = "steps", eval_steps = 1
),
train_dataset = data,
eval_dataset = data,
)
)
res["evaluate"] = _raised(
lambda: res.__setitem__("eval_loss", trainer.evaluate(data)["eval_loss"])
)
res["lion"] = _raised(
lambda: SequentialTrainer(
model = model,
args = _trainer_args(os.path.join(tmp, "li"), bs, optim = "lion_8bit"),
train_dataset = data,
)
)
res["generate"] = _raised(
lambda: model.generate(input_ids = ids[:1, :3], max_new_tokens = 2, pad_token_id = 0)
)
if dist.is_initialized():
dist.barrier()
return res
def _case_dp_average(rank, world, tmp):
"""DDP-style dp averaging: a gradient on one replica only is averaged with zeros (DDP's behaviour), one on
none stays None, and both replicas end equal."""
error = _import_checkout_unsloth()
if error is not None:
return {"skip": error}
import torch.distributed as dist
from torch.distributed.device_mesh import init_device_mesh
from unsloth.models import tp_native
dist.init_process_group("gloo")
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("dp",))
one, both, none, frozen = (torch.nn.Parameter(torch.zeros(3)) for _ in range(4))
frozen.requires_grad_(False)
one.grad = torch.full((3,), 4.0) if rank == 0 else None
both.grad = torch.full((3,), float(rank + 1))
tp_native._dp_average([one, both, none, frozen], mesh["dp"])
return {
"one": one.grad.tolist(),
"both": both.grad.tolist(),
"none": none.grad,
"frozen": frozen.grad,
}
def _case_dp_average_dtypes(rank, world, tmp):
"""fp16 grads whose sum overflows but mean does not; bf16 / fp32 match own-dtype all-reduce."""
error = _import_checkout_unsloth()
if error is not None:
return {"skip": error}
import torch.distributed as dist
from torch.distributed.device_mesh import init_device_mesh
from unsloth.models import tp_native
dist.init_process_group("gloo")
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("dp",))
torch.manual_seed(rank)
out, params = {}, {}
for dtype in (torch.float16, torch.bfloat16, torch.float32):
big = torch.nn.Parameter(torch.zeros(4, dtype = dtype))
big.grad = torch.tensor([40000.0, 40000.0 if rank else 30000.0, 1e-3, -6e4], dtype = dtype)
noise = torch.nn.Parameter(torch.zeros(64, dtype = dtype))
noise.grad = (torch.randn(64) * 10.0 ** torch.randint(-6, 4, (64,))).to(dtype)
params[dtype] = (big, noise)
reference = []
for p in (big, noise):
g = p.grad.clone()
dist.all_reduce(g)
reference.append(g.div_(world))
out[str(dtype)] = {"reference": reference}
tp_native._dp_average([p for pair in params.values() for p in pair], mesh["dp"])
for dtype, pair in params.items():
out[str(dtype)]["got"] = [p.grad.clone() for p in pair]
out[str(dtype)]["dtype"] = [p.grad.dtype for p in pair]
return out
def _case_accelerate_fsdp(rank, world, tmp, ckpt):
"""UNSLOTH_FSDP2=1 that resolves to accelerate's FSDP: the dp dimension would be left unsynced."""
error = _import_checkout_unsloth()
if error is not None:
return {"skip": error}
import transformers
from peft import LoraConfig, get_peft_model
from transformers import AutoModelForCausalLM
from unsloth.models import tp_env
from unsloth.models._utils import patch_gradient_accumulation_fix
patch_gradient_accumulation_fix(transformers.Trainer)
model = tp_env.tensor_parallel_env_load(
lambda name, load_in_4bit = True: AutoModelForCausalLM.from_pretrained(
name, torch_dtype = torch.float32
)
)(ckpt, load_in_4bit = False)
model = get_peft_model(model, LoraConfig(r = 4, target_modules = "all-linear"))
data = _Rows([torch.arange(9)] * 8)
return {
"refused": _raised(
lambda: transformers.Trainer(
model = model, args = _trainer_args(tmp, 2), train_dataset = data
)
)
}
def _case_model_init(
rank,
world,
tmp,
ckpt,
hooked,
with_model = False,
import_only = False,
):
"""Trainer(model_init = ...) under UNSLOTH_TP_SIZE: model_init runs after the tp setup saw model = None."""
error = _import_checkout_unsloth()
if error is not None:
return {"skip": error}
import transformers
from peft import LoraConfig, get_peft_model
from transformers import AutoModelForCausalLM
from unsloth.models import tp_env, tp_native
from unsloth.models._utils import patch_gradient_accumulation_fix
if not import_only:
patch_gradient_accumulation_fix(transformers.Trainer)
def load(name, load_in_4bit = True):
model = AutoModelForCausalLM.from_pretrained(name, torch_dtype = torch.float32)
if (
hooked and import_only
): # FastLanguageModel.from_pretrained installs the wrapper after loading
patch_gradient_accumulation_fix(transformers.Trainer)
return model
def peft(model):
return get_peft_model(model, LoraConfig(r = 4, target_modules = "all-linear"))
if hooked: # FastLanguageModel.from_pretrained / get_peft_model inside model_init
load = tp_env.tensor_parallel_env_load(load)
peft = tp_env.tensor_parallel_env_peft(peft)
built = []
def model_init():
model = peft(load(ckpt, load_in_4bit = False))
built.append(tp_native.is_tensor_parallel(model))
return model
data = _Rows([torch.arange(9)] * 8)
trainer = []
if import_only:
def run():
trainer.append(
transformers.Trainer(
model_init = model_init, args = _trainer_args(tmp, 2), train_dataset = data
)
)
trainer[0].train()
refused = _raised(run)
elif (
with_model
): # transformers 4.x accepts both (FutureWarning) and train() rebuilds from model_init
model = peft(load(ckpt, load_in_4bit = False))
def run():
trainer.append(
transformers.Trainer(
model = model,
model_init = model_init,
args = _trainer_args(tmp, 2),
train_dataset = data,
)
)
trainer[0].train()
refused = _raised(run)
else:
refused = _raised(
lambda: trainer.append(
transformers.Trainer(
model_init = model_init, args = _trainer_args(tmp, 2), train_dataset = data
)
)
)
return {
"refused": refused,
"built": built,
"is_tp": bool(trainer) and tp_native.is_tensor_parallel(trainer[0].model),
}
CASES = {
"units": _case_units,
"env": _case_env,
"dp_average": _case_dp_average,
"dp_average_dtypes": _case_dp_average_dtypes,
"accelerate_fsdp": _case_accelerate_fsdp,
"model_init": _case_model_init,
}
def _worker(rank, world, port, case, tmp, env, kwargs):
os.environ.update(
MASTER_ADDR = "127.0.0.1",
MASTER_PORT = str(port),
RANK = str(rank),
WORLD_SIZE = str(world),
LOCAL_RANK = str(rank),
LOCAL_WORLD_SIZE = str(world),
)
for key in _ENV_KEYS:
os.environ.pop(key, None)
os.environ.update(env)
torch.set_num_threads(2)
import torch.distributed as dist
try:
res = CASES[case](rank, world, tmp, **kwargs)
torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
finally:
if dist.is_initialized():
dist.destroy_process_group()
def _spawn(
case,
world = 1,
env = None,
**kwargs,
):
ctx = multiprocessing.get_context("spawn")
port = _free_port()
with tempfile.TemporaryDirectory() as tmp:
if case in ("env", "accelerate_fsdp", "model_init"):
kwargs["ckpt"] = _checkpoint(tmp)
procs = [
ctx.Process(target = _worker, args = (r, world, port, case, tmp, env or {}, kwargs))
for r in range(world)
]
for p in procs:
p.start()
for p in procs:
p.join(600)
codes = [p.exitcode for p in procs]
for p in procs:
if p.is_alive():
p.kill()
assert codes == [0] * world, f"rank exit codes {codes}"
results = [
torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
for r in range(world)
]
if "skip" in results[0]:
pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")
return results
def _close(
got,
want,
rel = 1e-5,
what = "",
):
assert got["steps"] == want["steps"], (what, got["steps"], want["steps"])
assert got["losses"] == pytest.approx(want["losses"], rel = rel), (
what,
got["losses"],
want["losses"],
)
assert got["grad_norms"] == pytest.approx(want["grad_norms"], rel = rel), (
what,
got["grad_norms"],
want["grad_norms"],
)
assert set(got["state"]) == set(want["state"]) and want["state"]
for key, value in want["state"].items():
torch.testing.assert_close(got["state"][key], value, rtol = rel * 10, atol = 1e-6, msg = key)
def test_env_layout_parsing_and_refusals():
(res,) = _spawn("units")
assert res["unset"] is None and res["one"] is None
assert "must be >= 1" in res["zero"] and "must be an integer" in res["text"]
assert "must divide the number of ranks (WORLD_SIZE=4)" in res["no_divide"]
assert (
"WORLD_SIZE=1" in res["no_torchrun"] and "torchrun --nproc-per-node 2" in res["no_torchrun"]
)
# UNSLOTH_PP_SIZE is env-driven too now (models/pp_env.py, tests/test_pp_env_gloo.py).
assert res["pp"] == (1, 2, 4, False) and res["pp_with_tp"] == (2, 1, 4, False), (
res["pp"],
res["pp_with_tp"],
)
assert res["pp_one"] == (2, 2, 4, False)
assert res["tp_world"] == (4, 1, 4, False)
assert res["ddp"] == (2, 2, 4, False)
assert res["fsdp2"] == (2, 2, 4, True)
assert "native FSDP2 only" in res["accelerate_fsdp"]
assert "vLLM" in res["refuse_vllm"] and "UNSLOTH_TP_SIZE=2" in res["refuse_vllm"]
assert "float8_training" in res["refuse_float8"] and "UNSLOTH_FSDP2=1" in res["refuse_float8"]
assert "8-bit weights" in res["refuse_8bit"]
assert "UNSLOTH_FSDP2_SHARDED_LOAD=1" in (res["refuse_fsdp2_sharded_load"] or ""), res[
"refuse_fsdp2_sharded_load"
]
assert res["refused_before_load"] and res["no_env_passthrough"] == "loaded"
assert res["four_bit_default"] and not res["four_bit_16"] and res["four_bit_qconfig"]
assert all(res["hooked"].values()), res["hooked"]
def _ref(bs, **kwargs):
return _spawn("env", 1, bs = bs, **kwargs)[0]
@pytest.mark.parametrize("four_bit", [False, True], ids = ["sharded_16bit", "full_load"])
def test_env_tp2_trainer_matches_one_process(four_bit):
ref = _ref(2, four_bit = four_bit)
assert ref["layout"] is None and not ref["is_tp"]
results = _spawn("env", 2, {"UNSLOTH_TP_SIZE": "2"}, bs = 2, four_bit = four_bit, refusals = True)
for rank, res in enumerate(results):
assert res["layout"] == "tp 2" and res["is_tp"], res
assert res["sharded_load"] is (not four_bit), res["sharded_load"]
assert not res["native_fsdp2"] and not res["ddp_mesh"]
assert "DistributedDataParallel" not in res["wrapped"]
_close(res["a"], ref["a"], what = rank)
hint = "UNSLOTH_TP_SIZE=2: unset it"
assert "map-style dataset" in res["iterable"] and hint in res["iterable"], res["iterable"]
assert res["eval"] is None and res["evaluate"] is None, (res["eval"], res["evaluate"])
assert "lion_8bit" in res["lion"], res["lion"]
assert res["generate"] is None, res["generate"] # a whole replica per rank (tp_replica.py)
assert results[0]["seen"] == results[1]["seen"] # both tp ranks read the same batches
assert results[0]["eval_loss"] == results[1]["eval_loss"]
def test_env_tp2_with_fsdp2_flag_and_no_dp_dimension():
ref = _ref(2)
for res in _spawn("env", 2, {"UNSLOTH_TP_SIZE": "2", "UNSLOTH_FSDP2": "1"}, bs = 2):
assert res["layout"] == "tp 2" and res["is_tp"] and not res["native_fsdp2"], res
_close(res["a"], ref["a"])
@pytest.mark.parametrize("fsdp2", [True, False], ids = ["fsdp2_dp2", "ddp_dp2"])
def test_env_tp2_x_dp2_trainer_matches_one_process(fsdp2):
ref = _ref(4)
env = {"UNSLOTH_TP_SIZE": "2"}
if fsdp2:
env["UNSLOTH_FSDP2"] = "1"
results = _spawn("env", 4, env, bs = 2)
for rank, res in enumerate(results):
assert res["layout"] == (
"dp 2 (native FSDP2) x tp 2" if fsdp2 else "dp 2 (DDP) x tp 2"
), res
assert res["is_tp"] and res["native_fsdp2"] is fsdp2 and res["ddp_mesh"] is (not fsdp2)
assert "DistributedDataParallel" not in res["wrapped"]
_close(res["a"], ref["a"], what = rank)
for rank, res in enumerate(results):
assert res["seen"] == results[rank ^ 1]["seen"] # tp peers: same rows
assert res["seen"] != results[(rank + 2) % 4]["seen"] # dp replicas: different rows
def _bnb_cpu():
try:
import bitsandbytes # noqa: F401
except Exception:
return False
return True
@pytest.mark.skipif(not _bnb_cpu(), reason = "bitsandbytes not installed")
@pytest.mark.parametrize(
"world,env", [(2, {}), (4, {"UNSLOTH_FSDP2": "1"})], ids = ["tp2", "tp2_fsdp2_dp2"]
)
def test_env_tp_adamw_8bit_steps_local_shards(world, env):
ref = _ref(2 * (world // 2), optim = "adamw_8bit", bnb_8bit = True)
if not ref["optimizer"].startswith("bitsandbytes"):
pytest.skip(f"bitsandbytes cannot step CPU tensors here ({ref['optimizer']})")
assert ref["optim_bits"] == 8 and "torch.uint8" in ref["state_dtypes"], ref
results = _spawn(
"env", world, {"UNSLOTH_TP_SIZE": "2", **env}, bs = 2, optim = "adamw_8bit", bnb_8bit = True
)
for rank, res in enumerate(results):
assert res["optim_arg"] == "adamw_8bit" and res["local_shard_bnb"], res["optimizer"]
assert res["optim_bits"] == 8 and "torch.uint8" in res["state_dtypes"], res["state_dtypes"]
# 8-bit state is quantized per 256-element block of each local shard (one GPU: of the whole
# tensor), so the run matches one process within the 8-bit band, not bitwise.
assert res["a"]["losses"] == pytest.approx(ref["a"]["losses"], rel = 2e-3), (
rank,
res["a"]["losses"],
ref["a"]["losses"],
)
for key, value in ref["a"]["state"].items():
torch.testing.assert_close(res["a"]["state"][key], value, rtol = 0.05, atol = 2e-3, msg = key)
@pytest.mark.parametrize(
"world,env,optim",
[
(2, {}, "adamw_torch"),
(4, {"UNSLOTH_FSDP2": "1"}, "adamw_torch"),
(4, {}, "adamw_torch"),
(2, {}, "adamw_8bit"),
],
ids = ["tp2", "tp2_fsdp2_dp2", "tp2_ddp_dp2", "tp2_adamw_8bit"],
)
def test_env_tp_checkpoint_resume_equals_uninterrupted(world, env, optim):
if optim == "adamw_8bit" and not _bnb_cpu():
pytest.skip("bitsandbytes not installed")
results = _spawn(
"env",
world,
{"UNSLOTH_TP_SIZE": "2", **env},
bs = 2,
optim = optim,
resume = True,
bnb_8bit = optim == "adamw_8bit",
)
for rank, res in enumerate(results):
a, b = res["a"], dict(res["b"])
# The resumed log history starts with steps 1-2 restored from trainer_state.json.
assert a["steps"] == [1, 2, 3, 4] and b["steps"] == [1, 2, 3, 4], (a["steps"], b["steps"])
b["losses"], b["grad_norms"] = b["losses"][2:], b["grad_norms"][2:]
assert "adapter_model.safetensors" in res["ckpt_files"], res["ckpt_files"]
assert f"rank{rank:05d}.pt" in res["ckpt_files"], res["ckpt_files"]
# Bitwise: the same local shards, optimizer state, RNG and data position as the uninterrupted run.
assert b["losses"] == a["losses"][2:], (rank, b["losses"], a["losses"])
assert b["grad_norms"] == a["grad_norms"][2:], (rank, b["grad_norms"], a["grad_norms"])
for key, value in a["state"].items():
assert torch.equal(b["state"][key], value), key
assert "written on another layout" in (res["bad_layout"] or ""), res["bad_layout"]
def test_dp_average_matches_ddp_for_gradients_on_some_replicas():
for res in _spawn("dp_average", 2):
assert res["one"] == [2.0] * 3 and res["both"] == [1.5] * 3, res
assert res["none"] is None and res["frozen"] is None, res
def test_dp_average_keeps_fp16_finite_and_bf16_fp32_unchanged():
results = _spawn("dp_average_dtypes", 2)
for res in results:
fp16 = res[str(torch.float16)]
big, noise = fp16["got"]
assert fp16["dtype"] == [torch.float16] * 2
assert torch.isfinite(big).all() and torch.isfinite(noise).all(), big
assert big.tolist() == [40000.0, 35008.0, 0.0010004043579101562, -6e4], big.tolist()
assert not torch.isfinite(fp16["reference"][0]).all()
for dtype in (torch.bfloat16, torch.float32):
entry = res[str(dtype)]
assert entry["dtype"] == [dtype] * 2
for got, want in zip(entry["got"], entry["reference"]):
assert torch.equal(got, want), (dtype, got, want)
for a, b in zip(results[0][str(torch.float16)]["got"], results[1][str(torch.float16)]["got"]):
assert torch.equal(a, b)
def test_env_tp_x_accelerate_fsdp_is_refused():
env = {"UNSLOTH_TP_SIZE": "2", "UNSLOTH_FSDP2": "1", "ACCELERATE_USE_FSDP": "true"}
for res in _spawn("accelerate_fsdp", 4, env):
assert "needs Unsloth's native FSDP2" in (res["refused"] or ""), res
@pytest.mark.parametrize("hooked", [False, True], ids = ["plain", "unsloth_loaders"])
def test_env_tp_model_init_is_refused(hooked):
"""Unrefused, a model_init model silently trained as plain DDP despite UNSLOTH_TP_SIZE=2."""
for res in _spawn("model_init", 2, {"UNSLOTH_TP_SIZE": "2"}, hooked = hooked):
assert "UNSLOTH_TP_SIZE=2" in (res["refused"] or ""), res
assert "model_init" in res["refused"] and res["built"] == [], res
def test_env_tp_model_and_model_init_is_refused():
"""Trainer(model = m, model_init = f): 4.x train() swapped tp m for a plain DDP model; 5.x raises ValueError."""
for res in _spawn("model_init", 2, {"UNSLOTH_TP_SIZE": "2"}, hooked = False, with_model = True):
assert res["refused"] is not None and res["built"] == [], res
assert "model_init" in res["refused"], res
def _pipelining_step_outputs():
try:
import inspect
from torch.distributed.pipelining import ScheduleGPipe
except Exception:
return False
return "return_outputs" in inspect.signature(ScheduleGPipe.step).parameters
@pytest.mark.skipif(not _pipelining_step_outputs(), reason = "UNSLOTH_PP_SIZE needs torch >= 2.10")
@pytest.mark.parametrize("import_only", [False, True], ids = ["patched", "plain_import"])
@pytest.mark.parametrize("hooked", [False, True], ids = ["plain", "unsloth_loaders"])
def test_env_pp_model_init_is_refused(hooked, import_only):
"""The same silent DDP under UNSLOTH_PP_SIZE: the pipeline split runs on the __init__ model only."""
for res in _spawn(
"model_init", 2, {"UNSLOTH_PP_SIZE": "2"}, hooked = hooked, import_only = import_only
):
assert "UNSLOTH_PP_SIZE=2 (pipeline parallel)" in (res["refused"] or ""), res
assert "unset UNSLOTH_PP_SIZE." in res["refused"], res
assert "model_init" in res["refused"] and res["built"] == [], res
@pytest.mark.parametrize("hooked", [False, True], ids = ["plain", "unsloth_loaders"])
def test_env_tp_model_init_is_refused_after_plain_import(hooked):
"""Only `import unsloth`: nothing installed the Trainer wrapper before model_init ran, so it trained."""
env = {"UNSLOTH_TP_SIZE": "2"}
for res in _spawn("model_init", 2, env, hooked = hooked, import_only = True):
assert "UNSLOTH_TP_SIZE=2" in (res["refused"] or ""), res
assert "model_init" in res["refused"] and res["built"] == [], res