Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Native FSDP2: memory-scalable 16-bit loading (each rank reads only its shard), and the fp16 start scale on sub-meshes (#15225)
Daniel Han20bbee4c90fc
diff --git a/tests/test_fsdp2_native_fp16_scale_mesh_gloo.py b/tests/test_fsdp2_native_fp16_scale_mesh_gloo.pynew file mode 100644index 00000000000..9b4376dd834--- /dev/null+++ b/tests/test_fsdp2_native_fp16_scale_mesh_gloo.py@@ -0,0 +1,163 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Native FSDP2 float16 start scale on context parallel and sub-mesh layouts (gloo CPU worlds).++The Trainer multiplies every rank's loss by the number of processes (token mean over the world), so each rank's+backward carries world x one GPU's per-token gradient whatever mesh FSDP2 shards over. The start scale must be+divided by that factor:++* CP = 2, DP = 1 (world 2, both ranks share one batch, each takes half of the sequence, CP divides the token+ count by 2): factor 2 = the FSDP2 mesh (the world), the step one GPU takes is taken;+* two disjoint 2-rank sub-meshes of a world of 4 (each rank its own quarter of the batch): factor 4, not the+ 2 of the mesh; dividing by the mesh left the scale 2x too large and the first step overflowed."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = ""++import sys # noqa: E402+import types # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402++SCALE = 2.0**16+TOKENS = 8+FP16_GRAD = 40000.0 # one GPU's scaled per-token output gradient: below the float16 max (65504)+++def _data():+ torch.manual_seed(0)+ x = torch.rand(TOKENS, 8) * 1e-3+ weight = torch.randn(8, 8) * 0.1+ return x, weight, FP16_GRAD * TOKENS / SCALE+++def _one_gpu():+ x, weight, coef = _data()+ w = torch.nn.Parameter(weight.clone())+ scaler = torch.amp.GradScaler("cpu", init_scale = SCALE)+ opt = torch.optim.SGD([w], lr = 1e-3)+ out = torch.nn.functional.linear(x.half(), w.half())+ scaler.scale((out.float() * coef).sum() / TOKENS).backward()+ scaler.step(opt)+ scaler.update()+ return float(scaler.get_scale()), w.detach().clone()+++def _native(rank, world, layout):+ native = _load_native()+ from torch.distributed.device_mesh import DeviceMesh+ from torch.distributed.fsdp import MixedPrecisionPolicy++ x, weight, coef = _data()+ model = torch.nn.Linear(8, 8, bias = False)+ with torch.no_grad():+ model.weight.copy_(weight)+ if layout == "cp":+ # CP = 2, DP = 1: native FSDP2 shards over the world; the CP peers share the batch, each its half.+ mesh = None+ share = TOKENS // world+ tokens = x[rank * share : (rank + 1) * share]+ num_items = (+ TOKENS * world+ ) // world # HF sums the count over the world, CP divides it by cp+ else:+ # Two disjoint 2-rank meshes [0, 1] and [2, 3]; every rank reads its own quarter of the batch.+ mesh = DeviceMesh("cpu", [[0, 1], [2, 3]], mesh_dim_names = ("pair", "shard"))["shard"]+ share = TOKENS // world+ tokens = x[rank * share : (rank + 1) * share]+ num_items = TOKENS+ native.shard_model(+ model,+ native.FSDP2Config(param_dtype = torch.float16, mesh = mesh, device = "cpu", units = []),+ )+ assert isinstance(model.weight, torch.distributed.tensor.DTensor)+ old = torch.amp.GradScaler("cpu", init_scale = SCALE)+ trainer = types.SimpleNamespace(+ model = model,+ accelerator = types.SimpleNamespace(scaler = old, scaler_handler = None, num_processes = world),+ )+ assert native._scaler_for_native(trainer) is True+ scaler = trainer.accelerator.scaler+ opt = torch.optim.SGD(model.parameters(), lr = 1e-3)+ out = model(tokens.half())+ # transformers Trainer: sum / num_items_in_batch, times the number of processes.+ scaler.scale((out.float() * coef).sum() / num_items * world).backward()+ scaler.unscale_(opt)+ found_inf = float(+ sum(+ v.item() for v in scaler._per_optimizer_states[id(opt)]["found_inf_per_device"].values()+ )+ )+ scaler.step(opt)+ scaler.update()+ return (+ found_inf,+ float(scaler._init_scale),+ float(scaler.get_scale()),+ model.weight.full_tensor().detach().clone(),+ )+++def _worker(rank, world, port, out, layout):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ )+ torch.distributed.init_process_group("gloo", rank = rank, world_size = world)+ try:+ res = {"one_gpu": _one_gpu()}+ try:+ res["native"] = _native(rank, world, layout)+ except Exception as e: # noqa: BLE001+ import traceback+ res["native"] = repr(e) + traceback.format_exc()+ torch.save(res, os.path.join(out, f"rank{rank}.pt"))+ finally:+ torch.distributed.destroy_process_group()+++def _run(tmp_path_factory, layout, world):+ out = str(tmp_path_factory.mktemp(f"fp16_{layout}"))+ torch.multiprocessing.spawn(+ _worker, args = (world, _free_port(), out, layout), nprocs = world, join = True+ )+ return [torch.load(os.path.join(out, f"rank{r}.pt"), weights_only = False) for r in range(world)]++[email protected]("layout,world", [("cp", 2), ("submesh", 4)])+def test_fp16_start_scale_matches_the_trainer_loss_multiplier(tmp_path_factory, layout, world):+ for rank, res in enumerate(_run(tmp_path_factory, layout, world)):+ assert not isinstance(res["native"], str), res["native"]+ found_inf, init_scale, scale, weight = res["native"]+ assert init_scale == SCALE / world, (layout, rank, init_scale)+ assert found_inf == 0.0, f"{layout}: rank {rank} overflowed where one GPU did not"+ assert scale == SCALE / world, (layout, rank, scale)+ if layout == "cp":+ _, one_gpu_weight = res["one_gpu"]+ torch.testing.assert_close(weight, one_gpu_weight, rtol = 1e-3, atol = 1e-6)+ else:+ assert torch.isfinite(weight).all()+++def test_loss_scale_ranks_counts_processes_not_the_mesh():+ native = _load_native()+ model = torch.nn.Linear(2, 2)+ model.__dict__["_unsloth_fsdp2_mesh"] = types.SimpleNamespace(size = lambda: 2)+ trainer = types.SimpleNamespace(model = model, accelerator = types.SimpleNamespace(num_processes = 4))+ assert native._loss_scale_ranks(trainer) == 4+ trainer.accelerator.parallelism_config = types.SimpleNamespace(tp_size = 2)+ assert native._loss_scale_ranks(trainer) == 2diff --git a/tests/test_fsdp2_native_sharded_load_gloo.py b/tests/test_fsdp2_native_sharded_load_gloo.pynew file mode 100644index 00000000000..58fa11d87f9--- /dev/null+++ b/tests/test_fsdp2_native_sharded_load_gloo.py@@ -0,0 +1,461 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Native FSDP2 sharded loading (fsdp2_sharded_loading) on a gloo CPU world of 2 vs the full load + shard_model:+every FSDP2 local shard and 3 training steps are bitwise equal, and each rank reads only its rows."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import dataclasses # noqa: E402+import sys # noqa: E402+import tempfile # noqa: E402+import types # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402+from test_tp_native_oracle import ARCHS, build # noqa: E402+from test_tp_native_gloo import _free_port # noqa: E402++WORLD = 2+++def _load_module():+ native = _load_native()+ import importlib+ return native, importlib.import_module("unsloth.models.fsdp2_load")+++VLM = "trl-internal-testing/tiny-Qwen2_5_VLForConditionalGeneration"+++def _auto(arch):+ import transformers+ return (+ transformers.AutoModelForImageTextToText+ if arch == "qwen2_5_vl"+ else transformers.AutoModelForCausalLM+ )+++def _save(+ arch,+ directory,+ overrides = None,+ remap = False,+):+ if arch == "qwen2_5_vl":+ import transformers++ config = transformers.AutoConfig.from_pretrained(VLM)+ torch.manual_seed(0)+ model = transformers.AutoModelForImageTextToText.from_config(+ config, torch_dtype = torch.float32+ )+ else:+ repo, base = ARCHS[arch]+ model = build(repo, {**base, **(overrides or {})}, lora = False)+ model.generation_config.pad_token_id = None+ model.save_pretrained(directory, max_shard_size = "1MB", safe_serialization = True)+ if remap:+ _prefix_checkpoint_keys(directory)+ return directory+++# A VLM-style checkpoint of a text decoder, loaded through from_pretrained(key_mapping = ...) as Unsloth's+# text-only loads do (_apply_text_only_key_mapping).+REMAP = {r"^language_model\.model\.": "model.", r"^language_model\.lm_head\.": "lm_head."}+++def _prefix_checkpoint_keys(directory):+ import glob+ import json++ from safetensors.torch import load_file, save_file++ rename = lambda k: "language_model." + k+ for path in glob.glob(os.path.join(directory, "*.safetensors")):+ save_file(+ {rename(k): v for k, v in load_file(path).items()}, path, metadata = {"format": "pt"}+ )+ index = os.path.join(directory, "model.safetensors.index.json")+ if os.path.exists(index):+ with open(index, encoding = "utf-8") as f:+ data = json.load(f)+ data["weight_map"] = {rename(k): v for k, v in data["weight_map"].items()}+ with open(index, "w", encoding = "utf-8") as f:+ json.dump(data, f)+++def _raised(fn):+ try:+ fn()+ except Exception as e: # noqa: BLE001+ return str(e)+ return None+++def _prep(model, mode):+ if mode in ("lora", "lora_mts"):+ from peft import LoraConfig, get_peft_model++ torch.manual_seed(1)+ if mode == "lora":+ config = LoraConfig(+ r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False+ )+ else:+ # modules_to_save deep-copies a sharded-loaded linear (its chunk and its forward guard).+ config = LoraConfig(+ r = 4,+ lora_alpha = 8,+ target_modules = ["q_proj"],+ modules_to_save = ["down_proj"],+ init_lora_weights = False,+ )+ return get_peft_model(model, config)+ for p in model.parameters():+ p.requires_grad_(True)+ return model+++def _local(t):+ return t.to_local() if hasattr(t, "to_local") else t+++def _shards(model):+ """name -> (type, local tensor, padded FSDP2 storage or None)."""+ padded = {}+ for module in model.modules():+ get = getattr(module, "_get_fsdp_state", None)+ if get is None:+ continue+ group = getattr(get(), "_fsdp_param_group", None)+ for fp in getattr(group, "fsdp_params", None) or ():+ padded[id(fp.sharded_param)] = fp._sharded_param_data.detach().clone()+ return {+ n: (type(p).__name__, _local(p).detach().clone(), padded.get(id(p)))+ for n, p in model.named_parameters()+ }+++def _train(+ native,+ model,+ ids,+ steps = 3,+):+ opt = torch.optim.AdamW([p for p in model.parameters() if p.requires_grad], lr = 1e-2)+ losses, norms = [], []+ for _ in range(steps):+ loss = model(input_ids = ids, labels = ids).loss+ loss.backward()+ norms.append(float(native.native_clip_grad_norm_(model.parameters(), 1.0)))+ opt.step()+ opt.zero_grad(set_to_none = True)+ losses.append(float(loss))+ return losses, norms+++IGNORED = ("layers.0.mlp.down_proj.weight", "layers.0.mlp.down_proj.base_layer.weight")+++def _ignoring(config, model, ignore):+ """`config` with FSDP2Config.ignored_params = {the weight of `model` named `ignore`} (None: as is)."""+ if not ignore:+ return config+ picked = {p for n, p in model.named_parameters() if n.endswith(ignore)}+ assert len(picked) == 1, ignore+ return dataclasses.replace(config, ignored_params = picked)+++def _ignored_state(model, config):+ """Each FSDP2Config.ignored_params entry: (is a live parameter of `model`, kept a plain whole tensor)."""+ live = {id(p): n for n, p in model.named_parameters()}+ return [+ (id(p) in live, type(p).__name__ != "DTensor" and p.device.type != "meta")+ for p in config.ignored_params or ()+ ]+++def _case(+ rank,+ world,+ tmp,+ ckpt,+ mode,+ fit,+ arch,+ ignore = None,+ remap = False,+):+ native, loading = _load_module()+ AutoModelForCausalLM = _auto(arch)++ kw = dict(torch_dtype = torch.float32, attn_implementation = "eager")+ if remap:+ kw["key_mapping"] = REMAP+ base_config = native.FSDP2Config(param_dtype = torch.float32, fit_layout = fit, device = "cpu")++ full = _prep(AutoModelForCausalLM.from_pretrained(ckpt, **kw), mode)+ config = _ignoring(base_config, full, ignore)+ native.shard_model(full, config)+ ignored_full = _ignored_state(full, config)++ with loading.fsdp2_sharded_loading() as state:+ model = AutoModelForCausalLM.from_pretrained(ckpt, **kw)+ stats = dict(model._unsloth_fsdp2_load_stats)+ loaded_numel = sum(p.numel() for p in model.parameters())+ full_numel = sum(+ p.numel() for p in AutoModelForCausalLM.from_pretrained(ckpt, **kw).parameters()+ )+ expect_sliced = 0+ for p in model.parameters():+ info = getattr(p, loading._PRELOAD, None)+ if info is not None:+ start, stop = info["rows"]+ assert tuple(p.shape) == (stop - start,) + info["shape"][1:]+ expect_sliced += p.numel() * 4+ ids = torch.randint(0, 100, (2, 7), generator = torch.Generator().manual_seed(1))+ unsharded = _raised(lambda: model(input_ids = ids))+ model = _prep(model, mode)+ config = _ignoring(base_config, model, ignore)+ native.shard_model(model, config)+ ignored_sharded = _ignored_state(model, config)++ a, b = _shards(full), _shards(model)+ diffs = sorted(set(a) ^ set(b))+ for name in a:+ if name not in b:+ continue+ (ta, la, pa), (tb, lb, pb) = a[name], b[name]+ if ta != tb or la.shape != lb.shape or not torch.equal(la, lb):+ diffs.append(name)+ elif (pa is None) != (pb is None) or (pa is not None and not torch.equal(pa, pb)):+ diffs.append(name + " (padded storage)")+ ref = _train(native, full, ids)+ out = _train(native, model, ids)+ a, b = _shards(full), _shards(model)+ trained_diffs = [n for n in a if n in b and not torch.equal(a[n][1], b[n][1])]+ return {+ "diffs": diffs,+ "trained_diffs": trained_diffs,+ "ref": ref,+ "out": out,+ "stats": stats,+ "expect_sliced": expect_sliced,+ "numel": (loaded_numel, full_numel),+ "unsharded": unsharded,+ "models": len(state["models"]),+ "dtensors": sum(1 for v in b.values() if v[0] == "DTensor"),+ "ignored": (ignored_full, ignored_sharded),+ }+++def _case_refusals(rank, world, tmp, ckpt):+ native, loading = _load_module()+ from torch.distributed.device_mesh import init_device_mesh+ from transformers import AutoModelForCausalLM++ kw = dict(torch_dtype = torch.float32, attn_implementation = "eager")+ out = {}+ with loading.fsdp2_sharded_loading():+ out["4bit"] = _raised(+ lambda: AutoModelForCausalLM.from_pretrained(ckpt, load_in_4bit = True, **kw)+ )+ with loading.fsdp2_sharded_loading():+ model = AutoModelForCausalLM.from_pretrained(ckpt, **kw)+ mesh = init_device_mesh("cpu", (1, world), mesh_dim_names = ("r", "s"))+ out["2d"] = _raised(+ lambda: native.shard_model(+ model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh, device = "cpu")+ )+ )+ out["offload"] = _raised(+ lambda: native.shard_model(+ model, native.FSDP2Config(param_dtype = torch.float32, cpu_offload = True, device = "cpu")+ )+ )+ return out+++def _case_dora(rank, world, tmp, ckpt):+ """DoRA's magnitude init reads the whole base weight: refused with sharded loading, before PEFT fails on+ this rank's rows."""+ from peft import LoraConfig, get_peft_model+ from transformers import AutoModelForCausalLM++ native, loading = _load_module()+ kw = dict(torch_dtype = torch.float32, attn_implementation = "eager")+ with loading.fsdp2_sharded_loading():+ model = AutoModelForCausalLM.from_pretrained(ckpt, **kw)+ dora = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", use_dora = True)+ out = {"dora": _raised(lambda: get_peft_model(model, dora))}+ # The full load still takes DoRA (the refusal is scoped to sharded-loaded weights).+ full = AutoModelForCausalLM.from_pretrained(ckpt, **kw)+ out["full"] = _raised(lambda: get_peft_model(full, dora))+ return out+++CASES = {"parity": _case, "refusals": _case_refusals, "dora": _case_dora}+++def _worker(rank, world, port, case, tmp, kwargs):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ LOCAL_WORLD_SIZE = str(world),+ )+ import datetime++ import torch.distributed as dist++ torch.set_num_threads(2)+ dist.init_process_group(+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)+ )+ try:+ res = CASES[case](rank, world, tmp, **kwargs)+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))+ finally:+ dist.destroy_process_group()+++def _spawn(+ case,+ world = WORLD,+ **kwargs,+):+ ctx = multiprocessing.get_context("spawn")+ port = _free_port()+ with tempfile.TemporaryDirectory() as tmp:+ procs = [+ ctx.Process(target = _worker, args = (r, world, port, case, tmp, kwargs))+ for r in range(world)+ ]+ for p in procs:+ p.start()+ for p in procs:+ p.join(600)+ codes = [p.exitcode for p in procs]+ assert codes == [0] * world, f"rank exit codes {codes}"+ return [+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)+ for r in range(world)+ ]+++ "arch,overrides,mode,fit",+ [+ (+ "llama",+ {"intermediate_size": 37},+ "lora",+ False,+ ), # odd rows: uneven chunks, FSDP2 padding+ (+ "llama",+ {"intermediate_size": 37},+ "lora",+ True,+ ), # fit layout: frozen chunks gathered back+ ("llama", {"intermediate_size": 37}, "full", False),+ ("qwen3", None, "lora", False),+ ("qwen3", None, "full", False),+ ("qwen2_5_vl", None, "lora", False), # vision tower + projector linears sliced too+ # FSDP2Config.ignored_params naming a sharded-loaded weight: gathered whole and kept replicated.+ ("llama", {"intermediate_size": 37}, "full", "ignore"),+ ("llama", {"intermediate_size": 37}, "lora", "ignore"),+ ("llama", {"intermediate_size": 37}, "lora_mts", False),+ # A load-specific key_mapping (VLM-prefixed checkpoint of a text decoder).+ ("llama", {"intermediate_size": 37}, "lora", "remap"),+ ],+)+def test_sharded_loading_matches_full_load_shard_model(arch, overrides, mode, fit):+ ignore, remap = None, fit == "remap"+ if fit == "ignore":+ fit, ignore = False, IGNORED[mode == "lora"]+ if remap:+ fit = False+ with tempfile.TemporaryDirectory() as ckpt:+ _save(arch, ckpt, overrides, remap = remap)+ results = _spawn(+ "parity", ckpt = ckpt, mode = mode, fit = fit, arch = arch, ignore = ignore, remap = remap+ )+ sliced = [r["stats"]["read_bytes_sliced"] for r in results]+ for rank, res in enumerate(results):+ assert res["models"] == 1+ ignored_full, ignored_sharded = res["ignored"]+ assert ignored_sharded == ignored_full == ([(True, True)] if ignore else []), res["ignored"]+ assert res["diffs"] == [], (rank, res["diffs"][:6])+ assert res["dtensors"] > 0+ # 3 steps bitwise equal to the full load (losses, clipped grad norms, final local shards).+ assert res["out"] == res["ref"], (rank, res["out"], res["ref"])+ assert res["trained_diffs"] == [], (rank, res["trained_diffs"][:6])+ assert res["unsharded"] and "shard it first" in res["unsharded"], res["unsharded"]+ stats = res["stats"]+ # This rank read only its own rows of the sliced weights, everything else whole, nothing twice.+ assert stats["read_bytes_sliced"] == res["expect_sliced"] > 0, stats+ assert stats["read_bytes_full"] == stats["replicated_numel"] * 4, stats+ loaded, full = res["numel"]+ assert loaded == full - stats["sharded_numel"] + res["expect_sliced"] // 4 < full+ assert sum(sliced) == results[0]["stats"]["sharded_numel"] * 4+++def test_sharded_loading_refusals():+ with tempfile.TemporaryDirectory() as ckpt:+ _save("llama", ckpt, None)+ results = _spawn("refusals", ckpt = ckpt)+ for res in results:+ assert "16-bit safetensors checkpoints only" in (res["4bit"] or ""), res["4bit"]+ assert "QLoRA" in res["4bit"]+ assert "1-D shard mesh" in (res["2d"] or ""), res["2d"]+ assert "cpu_offload" in (res["offload"] or ""), res["offload"]+++def test_sharded_loading_refuses_dora():+ with tempfile.TemporaryDirectory() as ckpt:+ _save("llama", ckpt, {"intermediate_size": 37})+ results = _spawn("dora", ckpt = ckpt)+ for res in results:+ assert res["full"] is None, res["full"]+ assert "use_dora" in (res["dora"] or "") and "full" in res["dora"], res["dora"]+++def test_chunk_rows_match_fully_shard():+ _, loading = _load_module()+ for rows in (1, 2, 5, 7, 37, 64):+ for world in (1, 2, 3, 4, 8):+ chunks = list(torch.arange(rows).chunk(world))+ chunks += [torch.empty(0, dtype = torch.long)] * (world - len(chunks))+ for rank in range(world):+ start, stop = loading._chunk_rows(rows, world, rank)+ assert torch.equal(torch.arange(rows)[start:stop], chunks[rank]), (+ rows,+ world,+ rank,+ )+++def test_env_wrapper_is_inert_without_the_flag(monkeypatch):+ _, loading = _load_module()+ monkeypatch.delenv("UNSLOTH_FSDP2_SHARDED_LOAD", raising = False)+ calls = []+ wrapped = loading.sharded_load_env(lambda *a, **k: calls.append((a, k)) or "ok")+ assert wrapped(1, x = 2) == "ok" and calls == [((1,), {"x": 2})]+ assert loading.fsdp2_sharded_load_requested() is False+ monkeypatch.setenv("UNSLOTH_FSDP2_SHARDED_LOAD", "1")+ monkeypatch.setenv("UNSLOTH_FSDP2", "0")+ assert loading.fsdp2_sharded_load_requested() is Falsediff --git a/unsloth/models/fsdp2_load.py b/unsloth/models/fsdp2_load.pynew file mode 100644index 00000000000..8e19133db36--- /dev/null+++ b/unsloth/models/fsdp2_load.py@@ -0,0 +1,504 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Memory-scalable loading for native FSDP2: each rank reads only its FSDP2 shard of the weights.++Without it every rank loads the whole model before `shard_model`, so a model larger than one GPU cannot load.+Inside `fsdp2_sharded_loading()` every transformers `from_pretrained` (FastLanguageModel, FastModel,+FastVisionModel and plain transformers loaders all end there) builds the model on meta, then each rank reads+from the safetensors checkpoint:++* every nn.Linear weight except the output head as this rank's dim-0 chunk (torch.chunk rows, the slice+ `fully_shard` keeps), via safetensors `get_slice` (only those bytes are read);+* everything else whole (embeddings, norms, biases, buffers), as the full load does.++A chunk stays a plain tensor tagged with its full shape, so nn.Linear keeps its full features and+get_peft_model builds full-size LoRA with the 1-GPU init. `shard_model` then swaps each chunk for a meta+placeholder of the full shape (the layout planner sees the whole model), fully_shards, and installs the chunk+as the FSDP2 local shard (padded as fully_shard pads it), so the full weight never exists on any rank. A chunk+the layout keeps replicated (fit layout frozen weights, minority-dtype parameters) is all-gathered back,+exactly what the full load would have held. 16-bit checkpoints only.++ with fsdp2_sharded_loading():+ model, tokenizer = FastLanguageModel.from_pretrained(name, load_in_4bit = False, ...)+ model = FastLanguageModel.get_peft_model(model, ...)+ trainer = SFTTrainer(model = model, ...) # UNSLOTH_FSDP2=1: shard_model installs the shards++or `UNSLOTH_FSDP2_SHARDED_LOAD=1` next to `UNSLOTH_FSDP2=1` (Unsloth's loaders then run inside the context).+"""++from __future__ import annotations++import contextlib+import dataclasses+import functools+import math+import os++import torch+import torch.nn as nn++__all__ = [+ "fsdp2_sharded_loading",+ "load_fsdp2_sharded",+ "fsdp2_sharded_load_requested",+]++_PRELOAD = "_unsloth_fsdp2_preload"+_GUARD = "_unsloth_fsdp2_load_guard"+_STATS = "_unsloth_fsdp2_load_stats"+_ACTIVE = [0]++_QUANTIZED_HINT = (+ "4-bit / 8-bit QLoRA under native FSDP2 still loads the full quantized model on every rank: drop "+ "fsdp2_sharded_loading / UNSLOTH_FSDP2_SHARDED_LOAD for it, or pass load_in_4bit = False."+)+++def _chunk_rows(rows, world, rank):+ """Rows [start, stop) of dim-0 chunk `rank` as fully_shard cuts it (torch.chunk: ceil(rows / world) per+ chunk, so the last ranks may hold fewer rows or none)."""+ size = math.ceil(rows / world) if rows else 0+ start = min(rank * size, rows)+ return start, min(start + size, rows)+++def _output_head_weight(model):+ try:+ head = model.get_output_embeddings()+ except Exception:+ head = None+ weight = getattr(head, "weight", None)+ return weight if isinstance(weight, torch.Tensor) else None+++_DORA_REFUSAL = (+ "Unsloth: LoraConfig(use_dora = True) initializes DoRA's magnitude from the whole base weight, which "+ "native FSDP2 sharded loading holds only as this rank's rows; load the full model (drop "+ "fsdp2_sharded_loading / UNSLOTH_FSDP2_SHARDED_LOAD) to train DoRA under native FSDP2."+)+++def _sharded_chunk(weight):+ info = _preload(weight)+ return info is not None and info["world"] > 1+++def _guard_dora_init():+ """DoRA's magnitude init (peft DoraLinearLayer.update_layer, every PEFT version with DoRA) adds a full+ lora_B @ lora_A to the base weight: refuse it on a sharded-loaded chunk with a clear message instead of+ PEFT's shape mismatch. Installed once, inert for any other base layer."""+ try:+ from peft.tuners.lora import dora+ except Exception:+ return+ cls = getattr(dora, "DoraLinearLayer", None)+ update = getattr(cls, "update_layer", None)+ if update is None or getattr(update, "_unsloth_fsdp2_load_guard", False):+ return++ @functools.wraps(update)+ def update_layer(self, *args, **kwargs):+ base = kwargs.get("base_layer", args[0] if args else None)+ if _sharded_chunk(getattr(base, "weight", None)):+ raise ValueError(_DORA_REFUSAL)+ return update(self, *args, **kwargs)++ update_layer._unsloth_fsdp2_load_guard = True+ cls.update_layer = update_layer+++def _refuse_unsharded_forward(+ module,+ args,+ kwargs = None,+):+ raise RuntimeError(+ "Unsloth: this model was loaded with native FSDP2 sharded loading (each rank holds only its shard "+ "of the linear weights); shard it first (unsloth.models.fsdp2_native.shard_model(model), or let the "+ "Trainer do it under UNSLOTH_FSDP2=1) before running it."+ )+++def _materialize(model, name, kwargs, mesh, plan, device):+ """Fill a meta model: nn.Linear weights (not the output head) as this rank's dim-0 chunk, the rest whole."""+ from .tp_native import _fill_from_checkpoint++ world, rank = mesh.size(), mesh.get_local_rank()+ ranks = mesh.mesh.tolist()+ head = _output_head_weight(model)+ linear = {+ id(m.weight)+ for m in model.modules()+ if isinstance(m, nn.Linear) and isinstance(getattr(m, "weight", None), torch.Tensor)+ }+ owners = {}++ def read_local(old, reader, key):+ if id(old) not in linear or old is head or old.ndim < 2:+ return None+ rows = old.shape[0]+ start, stop = _chunk_rows(rows, world, rank)+ local = reader.rows(key, start, stop).to(device = device, dtype = old.dtype)+ new = nn.Parameter(local.contiguous(), requires_grad = old.requires_grad)+ setattr(+ new,+ _PRELOAD,+ {+ "shape": tuple(old.shape),+ "ranks": ranks,+ "world": world,+ "rank": rank,+ "rows": (start, stop),+ },+ )+ owners[id(new)] = new+ return new++ reader, sharded_numel, full_numel = _fill_from_checkpoint(+ model,+ name,+ kwargs,+ device,+ read_local,+ label = "native FSDP2",+ fallback = "a full load, then shard_model",+ )+ guards = [model.register_forward_pre_hook(_refuse_unsharded_forward)]+ for module in model.modules():+ if isinstance(module, nn.Linear) and id(module._parameters.get("weight")) in owners:+ guards.append(module.register_forward_pre_hook(_refuse_unsharded_forward))+ model.__dict__[_GUARD] = guards+ stats = {+ "world": world,+ "rank": rank,+ "sharded_tensors": len(owners),+ "sharded_numel": sharded_numel,+ "replicated_numel": full_numel,+ "read_bytes_full": reader.full_bytes,+ "read_bytes_sliced": reader.slice_bytes,+ }+ model.__dict__[_STATS] = stats+ if owners and world > 1:+ _guard_dora_init()+ return stats+++def fsdp2_sharded_loading(mesh = None):+ """Every transformers `from_pretrained` inside reads only this rank's native FSDP2 shard of the linear+ weights (see the module docstring). `mesh`: the 1-D DeviceMesh shard_model will use (default: the+ world, which is shard_model's default too)."""+ from .tp_native import _sharded_from_pretrained++ if mesh is None:+ mesh = _default_load_mesh()+ state = {+ "mesh": mesh,+ "plan": None,+ "materialize": _materialize,+ "label": "native FSDP2",+ "quantized_hint": _QUANTIZED_HINT,+ "prequantized_hint": _QUANTIZED_HINT,+ "mesh_hint": "Unsloth: native FSDP2 sharded loading takes a 1-D mesh (HSDP / 2-D meshes load in full).",+ }+ _ACTIVE[0] += 1+ try:+ with _sharded_from_pretrained(state) as live:+ yield live+ finally:+ _ACTIVE[0] -= 1+++def _default_load_mesh():+ import torch.distributed as dist++ if not (dist.is_available() and dist.is_initialized()):+ from .fsdp2_native import _ensure_process_group, _local_cuda_index++ index = _local_cuda_index() if torch.cuda.is_available() else None+ if torch.cuda.is_available() and index is None:+ index = torch.cuda.current_device()+ device = torch.device("cuda", index) if index is not None else torch.device("cpu")+ _ensure_process_group(device, None, None) # sets this rank's CUDA device first+ from torch.distributed.device_mesh import init_device_mesh++ device_type = "cuda" if torch.cuda.is_available() else "cpu"+ return init_device_mesh(device_type, (dist.get_world_size(),))+++def load_fsdp2_sharded(+ model_name,+ mesh = None,+ loader = None,+ **kwargs,+):+ """`loader(model_name, **kwargs)` under fsdp2_sharded_loading (default FastLanguageModel.from_pretrained,+ 16-bit). Raises if no model went through the sharded path."""+ if loader is None:+ from unsloth import FastLanguageModel+ loader = FastLanguageModel.from_pretrained+ kwargs.setdefault("load_in_4bit", False)+ with fsdp2_sharded_loading(mesh) as state:+ out = loader(model_name, **kwargs)+ if not state["models"]:+ raise RuntimeError(+ "Unsloth: load_fsdp2_sharded's loader did not call transformers from_pretrained; nothing was "+ "loaded sharded."+ )+ return out+++def fsdp2_sharded_load_requested():+ """UNSLOTH_FSDP2_SHARDED_LOAD=1 with native FSDP2 requested (UNSLOTH_FSDP2=1 under WORLD_SIZE > 1)."""+ if os.environ.get("UNSLOTH_FSDP2_SHARDED_LOAD", "0").strip().lower() not in (+ "1",+ "true",+ "yes",+ "on",+ ):+ return False+ try:+ from .fsdp2_native import native_fsdp2_requested+ return bool(native_fsdp2_requested())+ except Exception:+ return False+++def sharded_load_env(loader):+ """Decorator for Unsloth's top-level from_pretrained: under UNSLOTH_FSDP2_SHARDED_LOAD=1 (+ native FSDP2)+ the outermost call runs inside fsdp2_sharded_loading. 4-bit / 8-bit loads are refused there."""++ @functools.wraps(loader)+ def wrapper(*args, **kwargs):+ if _ACTIVE[0] or not fsdp2_sharded_load_requested():+ return loader(*args, **kwargs)+ with fsdp2_sharded_loading():+ return loader(*args, **kwargs)++ return wrapper+++# --------------------------------------------------------------------------------------------------+# shard_model side+# --------------------------------------------------------------------------------------------------+++def _preload(p):+ return getattr(p, _PRELOAD, None) if isinstance(p, torch.Tensor) else None+++def has_preloaded_shards(model):+ return any(_preload(p) is not None for p in model.parameters())+++def _refuse_weight_dependent_lora_init(model):+ configs = {}+ m, seen = model, 0+ while isinstance(m, nn.Module) and seen < 8:+ configs.update(getattr(m, "peft_config", None) or {})+ modules = m.__dict__.get("_modules") or {}+ m = modules.get("base_model") or modules.get("model")+ seen += 1+ bad = {+ name: cfg.init_lora_weights+ for name, cfg in configs.items()+ if getattr(cfg, "init_lora_weights", True) not in (True, False, "gaussian")+ }+ if any(getattr(cfg, "use_dora", False) for cfg in configs.values()) and any(+ _sharded_chunk(p) for p in model.parameters()+ ):+ raise ValueError(_DORA_REFUSAL)+ if bad:+ raise ValueError(+ f"Unsloth: init_lora_weights {bad} reads the base weight, which native FSDP2 sharded loading "+ "holds only as this rank's rows; use init_lora_weights = True / 'gaussian', or load the full "+ "model (no sharded loading)."+ )+++def _tag_modules_to_save_copies(model):+ """PEFT modules_to_save deep-copies a sharded-loaded linear, but Parameter.__deepcopy__ drops the chunk's+ tag: give each still-identical copy its original's, so it is sharded like the full load's copy."""+ for module in model.modules():+ original = getattr(module, "original_module", None)+ copies = getattr(module, "modules_to_save", None)+ if not isinstance(original, nn.Module) or not isinstance(copies, nn.ModuleDict):+ continue+ params = dict(original.named_parameters())+ for copy in copies.values():+ for name, q in copy.named_parameters():+ p = params.get(name)+ info = _preload(p)+ if info is not None and _preload(q) is None and q.shape == p.shape:+ q.__dict__[_PRELOAD] = info+++def begin_shard(model, mesh, config):+ """Before shard_model's layout: check the preloaded chunks match `mesh`, drop the forward guards and+ replace every chunk by a meta Parameter of the full shape. Returns ({(id(module), name): (local, info)}+ (empty without preloaded chunks), config): FSDP2Config.ignored_params entries naming a chunk now name its+ placeholder (the caller's set is updated in place, a non-set copied), so gather_replicated keeps them whole."""+ _tag_modules_to_save_copies(model)+ slots = {}+ for module in model.modules():+ for name, p in module._parameters.items():+ info = _preload(p)+ if info is not None:+ slots.setdefault(id(p), (p, info, []))[2].append((module, name))+ if not slots:+ return {}, config+ if mesh.ndim != 1:+ raise RuntimeError(+ "Unsloth: native FSDP2 sharded loading needs a 1-D shard mesh (the one it loaded over); this "+ f"FSDP2Config.mesh has {mesh.ndim} dims. Load in full for HSDP."+ )+ ranks = mesh.mesh.tolist()+ if config.shard_placement_fn is not None:+ raise RuntimeError(+ "Unsloth: native FSDP2 sharded loading holds dim-0 chunks; FSDP2Config.shard_placement_fn is "+ "not supported with it."+ )+ if config.cpu_offload:+ raise RuntimeError(+ "Unsloth: native FSDP2 sharded loading does not support cpu_offload yet; load in full for it."+ )+ for p, info, _ in slots.values():+ if info["ranks"] != ranks:+ raise RuntimeError(+ f"Unsloth: this model was sharded-loaded over ranks {info['ranks']}, but shard_model shards "+ f"over {ranks}; pass the same mesh to fsdp2_sharded_loading and FSDP2Config."+ )+ _refuse_weight_dependent_lora_init(model)+ for level in list(model.modules()):+ for handle in level.__dict__.pop(_GUARD, None) or ():+ handle.remove()+ # PEFT modules_to_save deep-copies a guarded linear, hook included, with no handle on the root.+ hooks = level.__dict__.get("_forward_pre_hooks") or {}+ for key in [k for k, fn in hooks.items() if fn is _refuse_unsharded_forward]:+ del hooks[key]+ pending, swap = {}, {}+ for p, info, where in slots.values():+ meta = nn.Parameter(+ torch.empty(info["shape"], dtype = p.dtype, device = "meta"),+ requires_grad = p.requires_grad,+ )+ for attr, value in p.__dict__.items():+ if attr != _PRELOAD:+ meta.__dict__[attr] = value+ meta.__dict__["_unsloth_fsdp2_pending"] = True+ local = p.detach()+ swap[id(p)] = meta+ for module, name in where:+ module._parameters[name] = meta+ pending[(id(module), name)] = (local, info)+ ignored = config.ignored_params+ if ignored and any(id(q) in swap for q in ignored):+ if isinstance(ignored, set):+ for q in [q for q in ignored if id(q) in swap]:+ ignored.discard(q)+ ignored.add(swap[id(q)])+ else:+ config = dataclasses.replace(+ config, ignored_params = {swap.get(id(q), q) for q in ignored}+ )+ return pending, config+++def _full_from_chunks(local, info, mesh, dtype):+ """All-gather the dim-0 chunks back into the full tensor (exactly the checkpoint's)."""+ import torch.distributed as dist++ rows, world = info["shape"][0], info["world"]+ size = math.ceil(rows / world)+ padded = local.new_zeros((size,) + tuple(info["shape"][1:]))+ padded[: local.shape[0]].copy_(local)+ pieces = [torch.empty_like(padded) for _ in range(world)]+ dist.all_gather(pieces, padded.contiguous(), group = mesh.get_group())+ return torch.cat(pieces, 0)[:rows].to(dtype).contiguous()+++def gather_replicated(+ model,+ pending,+ replicated,+ mesh,+ also = (),+):+ """Chunks whose parameter stays replicated (outside FSDP2) are gathered whole, and swapped in+ `replicated` and the sets in `also`. Collective over `mesh`; every rank reaches it with the same set+ (the layout and dtype decisions are agreed)."""+ if not pending:+ return 0+ keep = {id(p) for p in replicated}+ done = {}+ for module in model.modules():+ for name, p in list(module._parameters.items()):+ key = (id(module), name)+ if key not in pending or id(p) not in keep:+ continue+ local, info = pending.pop(key)+ if id(p) not in done:+ with torch.no_grad():+ full = _full_from_chunks(local, info, mesh, p.dtype)+ new = nn.Parameter(full, requires_grad = p.requires_grad)+ for attr, value in p.__dict__.items():+ if attr != "_unsloth_fsdp2_pending":+ new.__dict__[attr] = value+ done[id(p)] = (p, new)+ module._parameters[name] = done[id(p)][1]+ if done:+ swap = {id(old): new for old, new in done.values()}+ for params in (replicated, *also):+ for p in list(params):+ if id(p) in swap:+ params.discard(p)+ params.add(swap[id(p)])+ return len(done)+++def finish_shard(model, pending):+ """After fully_shard: each pending chunk becomes its FSDPParam's local shard (padded like fully_shard)."""+ if not pending:+ return 0+ from .fsdp2_native import _fsdp_module_class++ FSDPModule = _fsdp_module_class()+ installed = 0+ for module in model.modules():+ if not isinstance(module, FSDPModule):+ continue+ state = module._get_fsdp_state()+ group = getattr(state, "_fsdp_param_group", None)+ for fsdp_param in getattr(group, "fsdp_params", None) or ():+ info = fsdp_param._module_info+ entry = pending.pop((id(info.module), info.param_name), None)+ if entry is None:+ continue+ for shared, shared_name in zip(info.shared_modules, info.shared_param_names):+ pending.pop((id(shared), shared_name), None)+ local, _ = entry+ old = fsdp_param.sharded_param+ if tuple(local.shape) != tuple(fsdp_param.sharded_size):+ raise RuntimeError(+ f"Unsloth: native FSDP2 sharded loading read {tuple(local.shape)} rows for "+ f"'{info.param_name}', but fully_shard keeps {tuple(fsdp_param.sharded_size)} on this rank."+ )+ with torch.no_grad():+ local = local.to(dtype = old.dtype)+ new = nn.Parameter(+ fsdp_param.to_sharded_dtensor(local), requires_grad = old.requires_grad+ )+ for attr, value in old.__dict__.items():+ if attr != "_unsloth_fsdp2_pending":+ new.__dict__[attr] = value+ fsdp_param._setattr_on_modules(new)+ fsdp_param.reset_sharded_param()+ installed += 1+ if pending:+ names = sorted({name for (_, name) in pending})+ raise RuntimeError(+ f"Unsloth: native FSDP2 sharded loading left {len(pending)} chunks ({names[:4]}) outside every "+ "FSDP2 unit; load this model in full."+ )+ return installeddiff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex f47ea04f998..8ecbd9099e2 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -1547,6 +1547,10 @@ def shard_model(model, config: Optional[FSDP2Config] = None):torch.cuda.set_device(device)_warn_nccl2_memory(dist, device)mesh = config.mesh if config.mesh is not None else _default_mesh(device, dist)+ from .fsdp2_load import begin_shard, finish_shard, gather_replicated++ # fsdp2_sharded_loading chunks -> full-shape meta placeholders until fully_shard (empty otherwise).+ pending, config = begin_shard(model, mesh, config)_sync_trainable_from_first_rank(model, dist, device, config.mesh)param_dtype = _resolve_param_dtype(model, config)mp_policy = MixedPrecisionPolicy(param_dtype = param_dtype, reduce_dtype = config.reduce_dtype)@@ -1655,6 +1659,10 @@ def shard_model(model, config: Optional[FSDP2Config] = None):if layout.replicate_frozen:replicated.update(p for p in model.parameters() if not p.requires_grad)_refuse_units_sharing_params(model, units + list(extra_units), replicated)+ # The caller's ignored_params set follows a gathered chunk to its whole parameter, as the full+ # load's set names the live one.+ user_ignored = config.ignored_params if isinstance(config.ignored_params, set) else set()+ gather_replicated(model, pending, replicated, mesh, also = (ignored, user_ignored))if replicated:if not _supports_ignored_params(fully_shard):raise RuntimeError(@@ -1682,6 +1690,7 @@ def shard_model(model, config: Optional[FSDP2Config] = None):else None),)+ finish_shard(model, pending)model._unsloth_reshard_after_forward = Falsesetattr(model, _NATIVE_FLAG, True)for param in model.parameters():@@ -3342,15 +3351,47 @@ def _scaler_for_native(trainer):if handler is not None and hasattr(handler, "to_kwargs"):kwargs = handler.to_kwargs()kwargs.setdefault("device", getattr(scaler, "_device", "cuda"))- # The Trainer scales each rank's loss by world size, so float16 activation grads are world times- # one GPU's and overflow at its scale. Divide init_scale by the DP size unless the user set it.- world = _data_parallel_size(getattr(trainer, "model", None))+ # The Trainer scales each rank's loss by the number of processes, so float16 activation grads are+ # that many times one GPU's and overflow at its scale. Divide init_scale by it unless the user set it.+ world = _loss_scale_ranks(trainer)if world > 1 and "init_scale" not in kwargs:kwargs["init_scale"] = 2.0**16 / worldaccelerator.scaler = _ShardedGradScaler(**kwargs)return True+def _loss_scale_ranks(trainer):+ """How many times one GPU's per-token loss gradient a rank's backward carries: the Trainer multiplies+ each rank's loss by accelerator.num_processes (its token mean over the world), divided by the tensor+ parallel size (tp peers share a batch whose tokens the gathered count repeats). That is the world, not+ the FSDP2 mesh: on two disjoint 2-rank sub-meshes of 4 ranks it is 4. Context parallel peers share a+ batch but CP divides the gathered count by cp, so CP = 2, DP = 1 is the world (2) too."""+ accelerator = getattr(trainer, "accelerator", None)+ model = getattr(trainer, "model", None)+ ranks = getattr(accelerator, "num_processes", None)+ if not isinstance(ranks, int) or ranks < 1:+ ranks = None+ try:+ import torch.distributed as dist+ if dist.is_available() and dist.is_initialized():+ ranks = int(dist.get_world_size())+ except Exception:+ pass+ if ranks is None:+ return _data_parallel_size(model)+ pc = getattr(accelerator, "parallelism_config", None)+ tp = int(getattr(pc, "tp_size", 1) or 1) if pc is not None else 1+ tp_native = sys.modules.get(f"{__package__}.tp_native") if __package__ else None+ if tp == 1 and tp_native is not None and model is not None:+ try:+ if tp_native.is_tensor_parallel(model):+ tp_mesh = tp_native._tp_meshes(model)[0]+ tp = int(tp_mesh.size()) if tp_mesh is not None else 1+ except Exception:+ tp = 1+ return max(1, ranks // max(1, tp))++def _data_parallel_size(model):mesh = getattr(model, "__dict__", {}).get("_unsloth_fsdp2_mesh") if model is not None else Nonetry:diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 9b2d3bb8707..5c6d273056e 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -35,6 +35,7 @@ from transformers import __version__ as transformers_versionfrom peft import PeftConfig, PeftModelfrom .grouped_linear_lora import register_grouped_linear_lora_for_adapterfrom .fp8_to_nf4 import track_explicit_4bit_request+from .fsdp2_load import sharded_load_env as fsdp2_sharded_load_envfrom .float8_training import convert_model_to_float8_training, resolve_float8_trainingfrom .mistral_format import (MistralFormatRedirect,@@ -1229,6 +1230,7 @@ class FastLanguageModel(FastLlamaModel):return get_mtp_head(model, *args, **kwargs)@staticmethod+ @fsdp2_sharded_load_env@tokenizers_v1_on_return@_offline_aware_load@mistral_format_redirect@@ -2186,6 +2188,7 @@ class FastModel(FastBaseModel):return FastBaseModel.for_training(model, use_gradient_checkpointing)@staticmethod+ @fsdp2_sharded_load_env@tokenizers_v1_on_return@_restore_load_scoped_env@_offline_aware_loaddiff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.pyindex 619d22690e3..26ef5d01240 100644--- a/unsloth/models/tp_native.py+++ b/unsloth/models/tp_native.py@@ -1717,6 +1717,7 @@ def save_tp_adapter(model, save_directory, **kwargs):# load-time peak is the replicated parts plus 1 / tp of the attention / MLP weights, never the whole model._PRELOADED = "_unsloth_tp_preloaded"+_TP_QUANTIZED_HINT = "Load 4-bit / 8-bit / quantized models in full and call apply_tensor_parallel, or pass load_in_4bit = False."_UNSUPPORTED_LOAD_KWARGS = ("load_in_4bit","load_in_8bit",@@ -1900,6 +1901,12 @@ class _SliceReader:self.slice_bytes += t.numel() * t.element_size()return t+ def rows(self, key, start, stop):+ """Rows [start, stop) of dim 0 (an FSDP2 chunk); only those bytes are read."""+ t = self._open(key).get_slice(key)[start:stop]+ self.slice_bytes += t.numel() * t.element_size()+ return t+def _checkpoint_key(names, model, index):prefix = getattr(_unwrap(model), "base_model_prefix", "") or ""@@ -1915,6 +1922,59 @@ def _checkpoint_key(names, model, index):return None+def _checkpoint_aliases(+ model,+ index,+ key_mapping = None,+):+ """{model key: checkpoint key} for a checkpoint saved under another naming: transformers 5's conversion+ mapping renames (VLMs save `model.layers.*` for `model.language_model.layers.*`) and 4.x+ `_checkpoint_conversion_mapping`. A model key a WeightConverter builds (merged experts, fused or permuted+ weights) maps to None: it is not one checkpoint tensor, so it cannot be read sliced. `key_mapping` is the+ load's own (from_pretrained(key_mapping = ...), e.g. Unsloth's text-only VLM loads)."""+ legacy = getattr(model, "_checkpoint_conversion_mapping", None) or {}+ renamings = converters = None+ try:+ from transformers.conversion_mapping import get_model_conversion_mapping+ from transformers.core_model_loading import (+ WeightConverter,+ WeightRenaming,+ rename_source_key,+ )++ try:+ transforms = get_model_conversion_mapping(model, key_mapping = key_mapping)+ except TypeError:+ transforms = get_model_conversion_mapping(model)+ renamings = [t for t in transforms if isinstance(t, WeightRenaming)]+ converters = [t for t in transforms if isinstance(t, WeightConverter)]+ except Exception:+ # transformers 4.x: from_pretrained's key_mapping replaces _checkpoint_conversion_mapping.+ legacy = key_mapping or legacy+ if not legacy and not renamings and not converters:+ return {}+ meta = dict.fromkeys(model.state_dict().keys(), True)+ prefix = getattr(model, "base_model_prefix", None) or None+ out = {}+ for key in index:+ new = key+ for pattern, replacement in legacy.items():+ new, n = re.subn(pattern, replacement, new)+ if n:+ break+ converted = None+ if renamings is not None:+ try:+ new, converted = rename_source_key(new, renamings, converters, prefix, meta)+ except Exception:+ converted = None+ if converted is not None:+ out[new] = None+ elif new != key:+ out.setdefault(new, key)+ return out++def _buffer_skeleton(model):"""The model rebuilt with meta parameters and real buffers (RoPE tables), as from_pretrained builds them."""from accelerate import init_empty_weights@@ -1929,20 +1989,8 @@ def _buffer_skeleton(model):torch.set_default_dtype(previous)-def _materialize_sharded(model, name, kwargs, mesh, plan, device):- """Fill a meta model from the checkpoint: replicated tensors whole, sharded projections as local slices."""- tp, rank = mesh.size(), mesh.get_local_rank()- tp_plan = derive_tp_plan(model, tp, plan)- shards = {}- for region in tp_plan.regions:- for proj in region.projs:- weight, base = _weight(proj.module)- col = proj.style == COLWISE- shards[id(weight)] = (proj, 0 if col else 1, proj.parts if col else (weight.shape[1],))- if base.bias is not None and col:- shards[id(base.bias)] = (proj, 0, proj.parts)- index = _checkpoint_index(name, kwargs)- reader = _SliceReader(index)+def _parameter_slots(model):+ """{id(parameter): (parameter, [(module, name)], [qualified names])}: tied parameters are one slot."""slots = {}for module_name, module in model.named_modules(remove_duplicate = False):for pname, p in module._parameters.items():@@ -1950,36 +1998,63 @@ def _materialize_sharded(model, name, kwargs, mesh, plan, device):entry = slots.setdefault(id(p), (p, [], []))entry[1].append((module, pname))entry[2].append(f"{module_name}.{pname}" if module_name else pname)+ return slots+++def _fill_from_checkpoint(+ model,+ name,+ kwargs,+ device,+ read_local,+ label = "tensor parallel",+ fallback = "full load, then apply_tensor_parallel",+):+ """Fill a meta model from its safetensors checkpoint. `read_local(old, reader, key)` returns this rank's+ Parameter for a sharded tensor (reading only its bytes) or None, and then the tensor is read whole.+ Meta buffers come from the checkpoint or a meta-parameter skeleton of the same class (RoPE tables).+ Returns (reader, sharded numel, whole numel)."""+ index = _checkpoint_index(name, kwargs)+ reader = _SliceReader(index)+ aliases = Nonesharded_numel = full_numel = 0- for pid, (old, where, names) in slots.items():++ def find(names):+ nonlocal aliaseskey = _checkpoint_key(names, model, index)+ if key is not None:+ return key+ if aliases is None:+ aliases = _checkpoint_aliases(model, index, (kwargs or {}).get("key_mapping"))+ for n in names:+ if n in aliases:+ if aliases[n] is None:+ raise ValueError(+ f"Unsloth: {label} sharded loading cannot read '{n}': transformers builds it from several "+ f"checkpoint tensors (a weight conversion); load this model without sharded loading "+ f"({fallback})."+ )+ return aliases[n]+ return None++ for old, where, names in _parameter_slots(model).values():+ key = find(names)if key is None:raise ValueError(- f"Unsloth: tensor parallel sharded loading found no checkpoint tensor for '{names[0]}'; load "- "this model without sharded loading (full load, then apply_tensor_parallel)."+ f"Unsloth: {label} sharded loading found no checkpoint tensor for '{names[0]}'; load "+ f"this model without sharded loading ({fallback}).")if reader.shape(key) != tuple(old.shape):raise ValueError(f"Unsloth: checkpoint tensor '{key}' has shape {reader.shape(key)}, the model expects "f"{tuple(old.shape)}.")- shard = shards.get(pid)- if shard is None:+ new = read_local(old, reader, key)+ if new is None:value = reader.full(key).to(device = device, dtype = old.dtype)new = nn.Parameter(value, requires_grad = old.requires_grad)full_numel += old.numel()else:- proj, dim, parts = shard- local = reader.slice(key, dim, parts, rank, tp).to(device = device, dtype = old.dtype)- # A plain local tensor tagged with its full shape: nn.Linear keeps the full in / out features, so- # get_peft_model builds full-size LoRA factors with the 1-GPU init (a DTensor would make PEFT size- # them from the shard). apply_tensor_parallel wraps it into the DTensor.- new = nn.Parameter(local.contiguous(), requires_grad = old.requires_grad)- if len(parts) > 1:- new._unsloth_tp_parts = (dim, tuple(parts))- new._unsloth_tp_full_shape = tuple(old.shape)- new._unsloth_tp_preload_ranks = mesh.mesh.tolist()- setattr(new, _PRELOADED, (proj.style, tuple(proj.parts), tp))sharded_numel += old.numel()for module, pname in where:module._parameters[pname] = new@@ -1989,22 +2064,59 @@ def _materialize_sharded(model, name, kwargs, mesh, plan, device):if buf is None or not buf.is_meta:continuekey = f"{module_name}.{bname}" if module_name else bname- if bname not in module._non_persistent_buffers_set and key in index:- value = reader.full(key)+ source = find([key]) if bname not in module._non_persistent_buffers_set else None+ if source is not None:+ value = reader.full(source)else:skeleton = skeleton if skeleton is not None else _buffer_skeleton(model)value = skeleton.get_buffer(key)if value.is_meta:raise ValueError(f"Unsloth: sharded loading cannot rebuild the buffer '{key}'.")module._buffers[bname] = value.to(device = device, dtype = buf.dtype)+ if isinstance(getattr(model, "hf_device_map", None), dict):+ model.hf_device_map = {"": device.index if device.type == "cuda" else device.type}+ return reader, sharded_numel, full_numel+++def _materialize_sharded(model, name, kwargs, mesh, plan, device):+ """Fill a meta model from the checkpoint: replicated tensors whole, sharded projections as local slices."""+ tp, rank = mesh.size(), mesh.get_local_rank()+ tp_plan = derive_tp_plan(model, tp, plan)+ shards = {}+ for region in tp_plan.regions:+ for proj in region.projs:+ weight, base = _weight(proj.module)+ col = proj.style == COLWISE+ shards[id(weight)] = (proj, 0 if col else 1, proj.parts if col else (weight.shape[1],))+ if base.bias is not None and col:+ shards[id(base.bias)] = (proj, 0, proj.parts)++ def read_local(old, reader, key):+ shard = shards.get(id(old))+ if shard is None:+ return None+ proj, dim, parts = shard+ local = reader.slice(key, dim, parts, rank, tp).to(device = device, dtype = old.dtype)+ # A plain local tensor tagged with its full shape: nn.Linear keeps the full in / out features, so+ # get_peft_model builds full-size LoRA factors with the 1-GPU init (a DTensor would make PEFT size+ # them from the shard). apply_tensor_parallel wraps it into the DTensor.+ new = nn.Parameter(local.contiguous(), requires_grad = old.requires_grad)+ if len(parts) > 1:+ new._unsloth_tp_parts = (dim, tuple(parts))+ new._unsloth_tp_full_shape = tuple(old.shape)+ new._unsloth_tp_preload_ranks = mesh.mesh.tolist()+ setattr(new, _PRELOADED, (proj.style, tuple(proj.parts), tp))+ return new++ reader, sharded_numel, full_numel = _fill_from_checkpoint(+ model, name, kwargs, device, read_local+ )for region in tp_plan.regions:for proj in region.projs:base = _weight(proj.module)[1]base.__dict__["_unsloth_tp_load_guard"] = base.register_forward_pre_hook(_refuse_unapplied_forward)- if isinstance(getattr(model, "hf_device_map", None), dict):- model.hf_device_map = {"": device.index if device.type == "cuda" else device.type}stats = {"tp": tp,"rank": rank,@@ -2037,16 +2149,20 @@ def _from_pretrained_sharded(original, cls, name, args, kwargs, state):if state["active"]:return original(cls, name, *args, **kwargs) # a nested load inside from_pretrainedconfig = kwargs.get("config")+ label = state.get("label", "tensor parallel")for key in _UNSUPPORTED_LOAD_KWARGS:if kwargs.get(key) or (key == "quantization_config" and getattr(config, key, None)):raise ValueError(- f"Unsloth: tensor parallel sharded loading supports 16-bit safetensors checkpoints only "- f"(got {key}). Load 4-bit / 8-bit / quantized models in full and call apply_tensor_parallel, "- "or pass load_in_4bit = False."+ f"Unsloth: {label} sharded loading supports 16-bit safetensors checkpoints only "+ f"(got {key}). {state.get('quantized_hint', _TP_QUANTIZED_HINT)}")mesh = state["mesh"] if state["mesh"] is not None else _world_mesh()if mesh.ndim != 1:- raise ValueError("Unsloth: sharded_loading takes a 1-D tp mesh (pass mesh['tp']).")+ raise ValueError(+ state.get(+ "mesh_hint", "Unsloth: sharded_loading takes a 1-D tp mesh (pass mesh['tp'])."+ )+ )device = _load_device(kwargs.get("device_map"), mesh)kwargs = dict(kwargs, device_map = "meta")state["active"] = True@@ -2060,10 +2176,11 @@ def _from_pretrained_sharded(original, cls, name, args, kwargs, state):model.config, "quantization_config", None):raise ValueError(- "Unsloth: tensor parallel sharded loading supports 16-bit checkpoints only; this checkpoint is "- "pre-quantized. Load it in full and call apply_tensor_parallel."+ f"Unsloth: {label} sharded loading supports 16-bit checkpoints only; this checkpoint is "+ f"pre-quantized. {state.get('prequantized_hint', 'Load it in full and call apply_tensor_parallel.')}")- _materialize_sharded(model, name, kwargs, mesh, state["plan"], device)+ materialize = state.get("materialize", _materialize_sharded)+ materialize(model, name, kwargs, mesh, state["plan"], device)state["models"].append(model)return model@@ -2074,7 +2191,7 @@ def _world_mesh():if not dist.is_initialized():raise RuntimeError(- "Unsloth: tensor parallel sharded loading needs an initialized process group (torchrun) or a tp_mesh."+ "Unsloth: sharded loading needs an initialized process group (torchrun) or a mesh.")device_type = "cuda" if torch.cuda.is_available() else "cpu"return init_device_mesh(device_type, (dist.get_world_size(),), mesh_dim_names = ("tp",))@@ -2092,11 +2209,20 @@ def sharded_loading(tp_mesh = None, plan = None):model = FastLanguageModel.get_peft_model(model, ...)apply_tensor_parallel(model, mesh["tp"]) # required before the first forward"""+ with _sharded_from_pretrained({"mesh": tp_mesh, "plan": plan}) as state:+ yield state+++def _sharded_from_pretrained(state):+ """Swap transformers' PreTrainedModel.from_pretrained for the meta build + checkpoint fill of `state`+ (mesh, plan, materialize(model, name, kwargs, mesh, plan, device), message parts). Shared by tensor+ parallel and native FSDP2 sharded loading."""from transformers import PreTrainedModeldescriptor = PreTrainedModel.__dict__["from_pretrained"]original = descriptor.__func__- state = {"mesh": tp_mesh, "plan": plan, "models": [], "active": False}+ state = dict(state, models = [], active = False)def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):return _from_pretrained_sharded(