Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2: memory-scalable 16-bit loading (each rank reads only its shard), and the fp16 start scale on sub-meshes (#15225)

Daniel Han20bbee4c90fc
diff --git a/tests/test_fsdp2_native_fp16_scale_mesh_gloo.py b/tests/test_fsdp2_native_fp16_scale_mesh_gloo.py
new file mode 100644
index 00000000000..9b4376dd834
--- /dev/null
+++ b/tests/test_fsdp2_native_fp16_scale_mesh_gloo.py
@@ -0,0 +1,163 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Native FSDP2 float16 start scale on context parallel and sub-mesh layouts (gloo CPU worlds).
+
+The Trainer multiplies every rank's loss by the number of processes (token mean over the world), so each rank's
+backward carries world x one GPU's per-token gradient whatever mesh FSDP2 shards over. The start scale must be
+divided by that factor:
+
+* CP = 2, DP = 1 (world 2, both ranks share one batch, each takes half of the sequence, CP divides the token
+ count by 2): factor 2 = the FSDP2 mesh (the world), the step one GPU takes is taken;
+* two disjoint 2-rank sub-meshes of a world of 4 (each rank its own quarter of the batch): factor 4, not the
+ 2 of the mesh; dividing by the mesh left the scale 2x too large and the first step overflowed."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = ""
+
+import sys # noqa: E402
+import types # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402
+
+SCALE = 2.0**16
+TOKENS = 8
+FP16_GRAD = 40000.0 # one GPU's scaled per-token output gradient: below the float16 max (65504)
+
+
+def _data():
+ torch.manual_seed(0)
+ x = torch.rand(TOKENS, 8) * 1e-3
+ weight = torch.randn(8, 8) * 0.1
+ return x, weight, FP16_GRAD * TOKENS / SCALE
+
+
+def _one_gpu():
+ x, weight, coef = _data()
+ w = torch.nn.Parameter(weight.clone())
+ scaler = torch.amp.GradScaler("cpu", init_scale = SCALE)
+ opt = torch.optim.SGD([w], lr = 1e-3)
+ out = torch.nn.functional.linear(x.half(), w.half())
+ scaler.scale((out.float() * coef).sum() / TOKENS).backward()
+ scaler.step(opt)
+ scaler.update()
+ return float(scaler.get_scale()), w.detach().clone()
+
+
+def _native(rank, world, layout):
+ native = _load_native()
+ from torch.distributed.device_mesh import DeviceMesh
+ from torch.distributed.fsdp import MixedPrecisionPolicy
+
+ x, weight, coef = _data()
+ model = torch.nn.Linear(8, 8, bias = False)
+ with torch.no_grad():
+ model.weight.copy_(weight)
+ if layout == "cp":
+ # CP = 2, DP = 1: native FSDP2 shards over the world; the CP peers share the batch, each its half.
+ mesh = None
+ share = TOKENS // world
+ tokens = x[rank * share : (rank + 1) * share]
+ num_items = (
+ TOKENS * world
+ ) // world # HF sums the count over the world, CP divides it by cp
+ else:
+ # Two disjoint 2-rank meshes [0, 1] and [2, 3]; every rank reads its own quarter of the batch.
+ mesh = DeviceMesh("cpu", [[0, 1], [2, 3]], mesh_dim_names = ("pair", "shard"))["shard"]
+ share = TOKENS // world
+ tokens = x[rank * share : (rank + 1) * share]
+ num_items = TOKENS
+ native.shard_model(
+ model,
+ native.FSDP2Config(param_dtype = torch.float16, mesh = mesh, device = "cpu", units = []),
+ )
+ assert isinstance(model.weight, torch.distributed.tensor.DTensor)
+ old = torch.amp.GradScaler("cpu", init_scale = SCALE)
+ trainer = types.SimpleNamespace(
+ model = model,
+ accelerator = types.SimpleNamespace(scaler = old, scaler_handler = None, num_processes = world),
+ )
+ assert native._scaler_for_native(trainer) is True
+ scaler = trainer.accelerator.scaler
+ opt = torch.optim.SGD(model.parameters(), lr = 1e-3)
+ out = model(tokens.half())
+ # transformers Trainer: sum / num_items_in_batch, times the number of processes.
+ scaler.scale((out.float() * coef).sum() / num_items * world).backward()
+ scaler.unscale_(opt)
+ found_inf = float(
+ sum(
+ v.item() for v in scaler._per_optimizer_states[id(opt)]["found_inf_per_device"].values()
+ )
+ )
+ scaler.step(opt)
+ scaler.update()
+ return (
+ found_inf,
+ float(scaler._init_scale),
+ float(scaler.get_scale()),
+ model.weight.full_tensor().detach().clone(),
+ )
+
+
+def _worker(rank, world, port, out, layout):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ )
+ torch.distributed.init_process_group("gloo", rank = rank, world_size = world)
+ try:
+ res = {"one_gpu": _one_gpu()}
+ try:
+ res["native"] = _native(rank, world, layout)
+ except Exception as e: # noqa: BLE001
+ import traceback
+ res["native"] = repr(e) + traceback.format_exc()
+ torch.save(res, os.path.join(out, f"rank{rank}.pt"))
+ finally:
+ torch.distributed.destroy_process_group()
+
+
+def _run(tmp_path_factory, layout, world):
+ out = str(tmp_path_factory.mktemp(f"fp16_{layout}"))
+ torch.multiprocessing.spawn(
+ _worker, args = (world, _free_port(), out, layout), nprocs = world, join = True
+ )
+ return [torch.load(os.path.join(out, f"rank{r}.pt"), weights_only = False) for r in range(world)]
+
+
[email protected]("layout,world", [("cp", 2), ("submesh", 4)])
+def test_fp16_start_scale_matches_the_trainer_loss_multiplier(tmp_path_factory, layout, world):
+ for rank, res in enumerate(_run(tmp_path_factory, layout, world)):
+ assert not isinstance(res["native"], str), res["native"]
+ found_inf, init_scale, scale, weight = res["native"]
+ assert init_scale == SCALE / world, (layout, rank, init_scale)
+ assert found_inf == 0.0, f"{layout}: rank {rank} overflowed where one GPU did not"
+ assert scale == SCALE / world, (layout, rank, scale)
+ if layout == "cp":
+ _, one_gpu_weight = res["one_gpu"]
+ torch.testing.assert_close(weight, one_gpu_weight, rtol = 1e-3, atol = 1e-6)
+ else:
+ assert torch.isfinite(weight).all()
+
+
+def test_loss_scale_ranks_counts_processes_not_the_mesh():
+ native = _load_native()
+ model = torch.nn.Linear(2, 2)
+ model.__dict__["_unsloth_fsdp2_mesh"] = types.SimpleNamespace(size = lambda: 2)
+ trainer = types.SimpleNamespace(model = model, accelerator = types.SimpleNamespace(num_processes = 4))
+ assert native._loss_scale_ranks(trainer) == 4
+ trainer.accelerator.parallelism_config = types.SimpleNamespace(tp_size = 2)
+ assert native._loss_scale_ranks(trainer) == 2
diff --git a/tests/test_fsdp2_native_sharded_load_gloo.py b/tests/test_fsdp2_native_sharded_load_gloo.py
new file mode 100644
index 00000000000..58fa11d87f9
--- /dev/null
+++ b/tests/test_fsdp2_native_sharded_load_gloo.py
@@ -0,0 +1,461 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Native FSDP2 sharded loading (fsdp2_sharded_loading) on a gloo CPU world of 2 vs the full load + shard_model:
+every FSDP2 local shard and 3 training steps are bitwise equal, and each rank reads only its rows."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import dataclasses # noqa: E402
+import sys # noqa: E402
+import tempfile # noqa: E402
+import types # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+from test_tp_native_oracle import ARCHS, build # noqa: E402
+from test_tp_native_gloo import _free_port # noqa: E402
+
+WORLD = 2
+
+
+def _load_module():
+ native = _load_native()
+ import importlib
+ return native, importlib.import_module("unsloth.models.fsdp2_load")
+
+
+VLM = "trl-internal-testing/tiny-Qwen2_5_VLForConditionalGeneration"
+
+
+def _auto(arch):
+ import transformers
+ return (
+ transformers.AutoModelForImageTextToText
+ if arch == "qwen2_5_vl"
+ else transformers.AutoModelForCausalLM
+ )
+
+
+def _save(
+ arch,
+ directory,
+ overrides = None,
+ remap = False,
+):
+ if arch == "qwen2_5_vl":
+ import transformers
+
+ config = transformers.AutoConfig.from_pretrained(VLM)
+ torch.manual_seed(0)
+ model = transformers.AutoModelForImageTextToText.from_config(
+ config, torch_dtype = torch.float32
+ )
+ else:
+ repo, base = ARCHS[arch]
+ model = build(repo, {**base, **(overrides or {})}, lora = False)
+ model.generation_config.pad_token_id = None
+ model.save_pretrained(directory, max_shard_size = "1MB", safe_serialization = True)
+ if remap:
+ _prefix_checkpoint_keys(directory)
+ return directory
+
+
+# A VLM-style checkpoint of a text decoder, loaded through from_pretrained(key_mapping = ...) as Unsloth's
+# text-only loads do (_apply_text_only_key_mapping).
+REMAP = {r"^language_model\.model\.": "model.", r"^language_model\.lm_head\.": "lm_head."}
+
+
+def _prefix_checkpoint_keys(directory):
+ import glob
+ import json
+
+ from safetensors.torch import load_file, save_file
+
+ rename = lambda k: "language_model." + k
+ for path in glob.glob(os.path.join(directory, "*.safetensors")):
+ save_file(
+ {rename(k): v for k, v in load_file(path).items()}, path, metadata = {"format": "pt"}
+ )
+ index = os.path.join(directory, "model.safetensors.index.json")
+ if os.path.exists(index):
+ with open(index, encoding = "utf-8") as f:
+ data = json.load(f)
+ data["weight_map"] = {rename(k): v for k, v in data["weight_map"].items()}
+ with open(index, "w", encoding = "utf-8") as f:
+ json.dump(data, f)
+
+
+def _raised(fn):
+ try:
+ fn()
+ except Exception as e: # noqa: BLE001
+ return str(e)
+ return None
+
+
+def _prep(model, mode):
+ if mode in ("lora", "lora_mts"):
+ from peft import LoraConfig, get_peft_model
+
+ torch.manual_seed(1)
+ if mode == "lora":
+ config = LoraConfig(
+ r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False
+ )
+ else:
+ # modules_to_save deep-copies a sharded-loaded linear (its chunk and its forward guard).
+ config = LoraConfig(
+ r = 4,
+ lora_alpha = 8,
+ target_modules = ["q_proj"],
+ modules_to_save = ["down_proj"],
+ init_lora_weights = False,
+ )
+ return get_peft_model(model, config)
+ for p in model.parameters():
+ p.requires_grad_(True)
+ return model
+
+
+def _local(t):
+ return t.to_local() if hasattr(t, "to_local") else t
+
+
+def _shards(model):
+ """name -> (type, local tensor, padded FSDP2 storage or None)."""
+ padded = {}
+ for module in model.modules():
+ get = getattr(module, "_get_fsdp_state", None)
+ if get is None:
+ continue
+ group = getattr(get(), "_fsdp_param_group", None)
+ for fp in getattr(group, "fsdp_params", None) or ():
+ padded[id(fp.sharded_param)] = fp._sharded_param_data.detach().clone()
+ return {
+ n: (type(p).__name__, _local(p).detach().clone(), padded.get(id(p)))
+ for n, p in model.named_parameters()
+ }
+
+
+def _train(
+ native,
+ model,
+ ids,
+ steps = 3,
+):
+ opt = torch.optim.AdamW([p for p in model.parameters() if p.requires_grad], lr = 1e-2)
+ losses, norms = [], []
+ for _ in range(steps):
+ loss = model(input_ids = ids, labels = ids).loss
+ loss.backward()
+ norms.append(float(native.native_clip_grad_norm_(model.parameters(), 1.0)))
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ losses.append(float(loss))
+ return losses, norms
+
+
+IGNORED = ("layers.0.mlp.down_proj.weight", "layers.0.mlp.down_proj.base_layer.weight")
+
+
+def _ignoring(config, model, ignore):
+ """`config` with FSDP2Config.ignored_params = {the weight of `model` named `ignore`} (None: as is)."""
+ if not ignore:
+ return config
+ picked = {p for n, p in model.named_parameters() if n.endswith(ignore)}
+ assert len(picked) == 1, ignore
+ return dataclasses.replace(config, ignored_params = picked)
+
+
+def _ignored_state(model, config):
+ """Each FSDP2Config.ignored_params entry: (is a live parameter of `model`, kept a plain whole tensor)."""
+ live = {id(p): n for n, p in model.named_parameters()}
+ return [
+ (id(p) in live, type(p).__name__ != "DTensor" and p.device.type != "meta")
+ for p in config.ignored_params or ()
+ ]
+
+
+def _case(
+ rank,
+ world,
+ tmp,
+ ckpt,
+ mode,
+ fit,
+ arch,
+ ignore = None,
+ remap = False,
+):
+ native, loading = _load_module()
+ AutoModelForCausalLM = _auto(arch)
+
+ kw = dict(torch_dtype = torch.float32, attn_implementation = "eager")
+ if remap:
+ kw["key_mapping"] = REMAP
+ base_config = native.FSDP2Config(param_dtype = torch.float32, fit_layout = fit, device = "cpu")
+
+ full = _prep(AutoModelForCausalLM.from_pretrained(ckpt, **kw), mode)
+ config = _ignoring(base_config, full, ignore)
+ native.shard_model(full, config)
+ ignored_full = _ignored_state(full, config)
+
+ with loading.fsdp2_sharded_loading() as state:
+ model = AutoModelForCausalLM.from_pretrained(ckpt, **kw)
+ stats = dict(model._unsloth_fsdp2_load_stats)
+ loaded_numel = sum(p.numel() for p in model.parameters())
+ full_numel = sum(
+ p.numel() for p in AutoModelForCausalLM.from_pretrained(ckpt, **kw).parameters()
+ )
+ expect_sliced = 0
+ for p in model.parameters():
+ info = getattr(p, loading._PRELOAD, None)
+ if info is not None:
+ start, stop = info["rows"]
+ assert tuple(p.shape) == (stop - start,) + info["shape"][1:]
+ expect_sliced += p.numel() * 4
+ ids = torch.randint(0, 100, (2, 7), generator = torch.Generator().manual_seed(1))
+ unsharded = _raised(lambda: model(input_ids = ids))
+ model = _prep(model, mode)
+ config = _ignoring(base_config, model, ignore)
+ native.shard_model(model, config)
+ ignored_sharded = _ignored_state(model, config)
+
+ a, b = _shards(full), _shards(model)
+ diffs = sorted(set(a) ^ set(b))
+ for name in a:
+ if name not in b:
+ continue
+ (ta, la, pa), (tb, lb, pb) = a[name], b[name]
+ if ta != tb or la.shape != lb.shape or not torch.equal(la, lb):
+ diffs.append(name)
+ elif (pa is None) != (pb is None) or (pa is not None and not torch.equal(pa, pb)):
+ diffs.append(name + " (padded storage)")
+ ref = _train(native, full, ids)
+ out = _train(native, model, ids)
+ a, b = _shards(full), _shards(model)
+ trained_diffs = [n for n in a if n in b and not torch.equal(a[n][1], b[n][1])]
+ return {
+ "diffs": diffs,
+ "trained_diffs": trained_diffs,
+ "ref": ref,
+ "out": out,
+ "stats": stats,
+ "expect_sliced": expect_sliced,
+ "numel": (loaded_numel, full_numel),
+ "unsharded": unsharded,
+ "models": len(state["models"]),
+ "dtensors": sum(1 for v in b.values() if v[0] == "DTensor"),
+ "ignored": (ignored_full, ignored_sharded),
+ }
+
+
+def _case_refusals(rank, world, tmp, ckpt):
+ native, loading = _load_module()
+ from torch.distributed.device_mesh import init_device_mesh
+ from transformers import AutoModelForCausalLM
+
+ kw = dict(torch_dtype = torch.float32, attn_implementation = "eager")
+ out = {}
+ with loading.fsdp2_sharded_loading():
+ out["4bit"] = _raised(
+ lambda: AutoModelForCausalLM.from_pretrained(ckpt, load_in_4bit = True, **kw)
+ )
+ with loading.fsdp2_sharded_loading():
+ model = AutoModelForCausalLM.from_pretrained(ckpt, **kw)
+ mesh = init_device_mesh("cpu", (1, world), mesh_dim_names = ("r", "s"))
+ out["2d"] = _raised(
+ lambda: native.shard_model(
+ model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh, device = "cpu")
+ )
+ )
+ out["offload"] = _raised(
+ lambda: native.shard_model(
+ model, native.FSDP2Config(param_dtype = torch.float32, cpu_offload = True, device = "cpu")
+ )
+ )
+ return out
+
+
+def _case_dora(rank, world, tmp, ckpt):
+ """DoRA's magnitude init reads the whole base weight: refused with sharded loading, before PEFT fails on
+ this rank's rows."""
+ from peft import LoraConfig, get_peft_model
+ from transformers import AutoModelForCausalLM
+
+ native, loading = _load_module()
+ kw = dict(torch_dtype = torch.float32, attn_implementation = "eager")
+ with loading.fsdp2_sharded_loading():
+ model = AutoModelForCausalLM.from_pretrained(ckpt, **kw)
+ dora = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", use_dora = True)
+ out = {"dora": _raised(lambda: get_peft_model(model, dora))}
+ # The full load still takes DoRA (the refusal is scoped to sharded-loaded weights).
+ full = AutoModelForCausalLM.from_pretrained(ckpt, **kw)
+ out["full"] = _raised(lambda: get_peft_model(full, dora))
+ return out
+
+
+CASES = {"parity": _case, "refusals": _case_refusals, "dora": _case_dora}
+
+
+def _worker(rank, world, port, case, tmp, kwargs):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ import datetime
+
+ import torch.distributed as dist
+
+ torch.set_num_threads(2)
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)
+ )
+ try:
+ res = CASES[case](rank, world, tmp, **kwargs)
+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
+ finally:
+ dist.destroy_process_group()
+
+
+def _spawn(
+ case,
+ world = WORLD,
+ **kwargs,
+):
+ ctx = multiprocessing.get_context("spawn")
+ port = _free_port()
+ with tempfile.TemporaryDirectory() as tmp:
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, case, tmp, kwargs))
+ for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(600)
+ codes = [p.exitcode for p in procs]
+ assert codes == [0] * world, f"rank exit codes {codes}"
+ return [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
+ for r in range(world)
+ ]
+
+
+ "arch,overrides,mode,fit",
+ [
+ (
+ "llama",
+ {"intermediate_size": 37},
+ "lora",
+ False,
+ ), # odd rows: uneven chunks, FSDP2 padding
+ (
+ "llama",
+ {"intermediate_size": 37},
+ "lora",
+ True,
+ ), # fit layout: frozen chunks gathered back
+ ("llama", {"intermediate_size": 37}, "full", False),
+ ("qwen3", None, "lora", False),
+ ("qwen3", None, "full", False),
+ ("qwen2_5_vl", None, "lora", False), # vision tower + projector linears sliced too
+ # FSDP2Config.ignored_params naming a sharded-loaded weight: gathered whole and kept replicated.
+ ("llama", {"intermediate_size": 37}, "full", "ignore"),
+ ("llama", {"intermediate_size": 37}, "lora", "ignore"),
+ ("llama", {"intermediate_size": 37}, "lora_mts", False),
+ # A load-specific key_mapping (VLM-prefixed checkpoint of a text decoder).
+ ("llama", {"intermediate_size": 37}, "lora", "remap"),
+ ],
+)
+def test_sharded_loading_matches_full_load_shard_model(arch, overrides, mode, fit):
+ ignore, remap = None, fit == "remap"
+ if fit == "ignore":
+ fit, ignore = False, IGNORED[mode == "lora"]
+ if remap:
+ fit = False
+ with tempfile.TemporaryDirectory() as ckpt:
+ _save(arch, ckpt, overrides, remap = remap)
+ results = _spawn(
+ "parity", ckpt = ckpt, mode = mode, fit = fit, arch = arch, ignore = ignore, remap = remap
+ )
+ sliced = [r["stats"]["read_bytes_sliced"] for r in results]
+ for rank, res in enumerate(results):
+ assert res["models"] == 1
+ ignored_full, ignored_sharded = res["ignored"]
+ assert ignored_sharded == ignored_full == ([(True, True)] if ignore else []), res["ignored"]
+ assert res["diffs"] == [], (rank, res["diffs"][:6])
+ assert res["dtensors"] > 0
+ # 3 steps bitwise equal to the full load (losses, clipped grad norms, final local shards).
+ assert res["out"] == res["ref"], (rank, res["out"], res["ref"])
+ assert res["trained_diffs"] == [], (rank, res["trained_diffs"][:6])
+ assert res["unsharded"] and "shard it first" in res["unsharded"], res["unsharded"]
+ stats = res["stats"]
+ # This rank read only its own rows of the sliced weights, everything else whole, nothing twice.
+ assert stats["read_bytes_sliced"] == res["expect_sliced"] > 0, stats
+ assert stats["read_bytes_full"] == stats["replicated_numel"] * 4, stats
+ loaded, full = res["numel"]
+ assert loaded == full - stats["sharded_numel"] + res["expect_sliced"] // 4 < full
+ assert sum(sliced) == results[0]["stats"]["sharded_numel"] * 4
+
+
+def test_sharded_loading_refusals():
+ with tempfile.TemporaryDirectory() as ckpt:
+ _save("llama", ckpt, None)
+ results = _spawn("refusals", ckpt = ckpt)
+ for res in results:
+ assert "16-bit safetensors checkpoints only" in (res["4bit"] or ""), res["4bit"]
+ assert "QLoRA" in res["4bit"]
+ assert "1-D shard mesh" in (res["2d"] or ""), res["2d"]
+ assert "cpu_offload" in (res["offload"] or ""), res["offload"]
+
+
+def test_sharded_loading_refuses_dora():
+ with tempfile.TemporaryDirectory() as ckpt:
+ _save("llama", ckpt, {"intermediate_size": 37})
+ results = _spawn("dora", ckpt = ckpt)
+ for res in results:
+ assert res["full"] is None, res["full"]
+ assert "use_dora" in (res["dora"] or "") and "full" in res["dora"], res["dora"]
+
+
+def test_chunk_rows_match_fully_shard():
+ _, loading = _load_module()
+ for rows in (1, 2, 5, 7, 37, 64):
+ for world in (1, 2, 3, 4, 8):
+ chunks = list(torch.arange(rows).chunk(world))
+ chunks += [torch.empty(0, dtype = torch.long)] * (world - len(chunks))
+ for rank in range(world):
+ start, stop = loading._chunk_rows(rows, world, rank)
+ assert torch.equal(torch.arange(rows)[start:stop], chunks[rank]), (
+ rows,
+ world,
+ rank,
+ )
+
+
+def test_env_wrapper_is_inert_without_the_flag(monkeypatch):
+ _, loading = _load_module()
+ monkeypatch.delenv("UNSLOTH_FSDP2_SHARDED_LOAD", raising = False)
+ calls = []
+ wrapped = loading.sharded_load_env(lambda *a, **k: calls.append((a, k)) or "ok")
+ assert wrapped(1, x = 2) == "ok" and calls == [((1,), {"x": 2})]
+ assert loading.fsdp2_sharded_load_requested() is False
+ monkeypatch.setenv("UNSLOTH_FSDP2_SHARDED_LOAD", "1")
+ monkeypatch.setenv("UNSLOTH_FSDP2", "0")
+ assert loading.fsdp2_sharded_load_requested() is False
diff --git a/unsloth/models/fsdp2_load.py b/unsloth/models/fsdp2_load.py
new file mode 100644
index 00000000000..8e19133db36
--- /dev/null
+++ b/unsloth/models/fsdp2_load.py
@@ -0,0 +1,504 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Memory-scalable loading for native FSDP2: each rank reads only its FSDP2 shard of the weights.
+
+Without it every rank loads the whole model before `shard_model`, so a model larger than one GPU cannot load.
+Inside `fsdp2_sharded_loading()` every transformers `from_pretrained` (FastLanguageModel, FastModel,
+FastVisionModel and plain transformers loaders all end there) builds the model on meta, then each rank reads
+from the safetensors checkpoint:
+
+* every nn.Linear weight except the output head as this rank's dim-0 chunk (torch.chunk rows, the slice
+ `fully_shard` keeps), via safetensors `get_slice` (only those bytes are read);
+* everything else whole (embeddings, norms, biases, buffers), as the full load does.
+
+A chunk stays a plain tensor tagged with its full shape, so nn.Linear keeps its full features and
+get_peft_model builds full-size LoRA with the 1-GPU init. `shard_model` then swaps each chunk for a meta
+placeholder of the full shape (the layout planner sees the whole model), fully_shards, and installs the chunk
+as the FSDP2 local shard (padded as fully_shard pads it), so the full weight never exists on any rank. A chunk
+the layout keeps replicated (fit layout frozen weights, minority-dtype parameters) is all-gathered back,
+exactly what the full load would have held. 16-bit checkpoints only.
+
+ with fsdp2_sharded_loading():
+ model, tokenizer = FastLanguageModel.from_pretrained(name, load_in_4bit = False, ...)
+ model = FastLanguageModel.get_peft_model(model, ...)
+ trainer = SFTTrainer(model = model, ...) # UNSLOTH_FSDP2=1: shard_model installs the shards
+
+or `UNSLOTH_FSDP2_SHARDED_LOAD=1` next to `UNSLOTH_FSDP2=1` (Unsloth's loaders then run inside the context).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import dataclasses
+import functools
+import math
+import os
+
+import torch
+import torch.nn as nn
+
+__all__ = [
+ "fsdp2_sharded_loading",
+ "load_fsdp2_sharded",
+ "fsdp2_sharded_load_requested",
+]
+
+_PRELOAD = "_unsloth_fsdp2_preload"
+_GUARD = "_unsloth_fsdp2_load_guard"
+_STATS = "_unsloth_fsdp2_load_stats"
+_ACTIVE = [0]
+
+_QUANTIZED_HINT = (
+ "4-bit / 8-bit QLoRA under native FSDP2 still loads the full quantized model on every rank: drop "
+ "fsdp2_sharded_loading / UNSLOTH_FSDP2_SHARDED_LOAD for it, or pass load_in_4bit = False."
+)
+
+
+def _chunk_rows(rows, world, rank):
+ """Rows [start, stop) of dim-0 chunk `rank` as fully_shard cuts it (torch.chunk: ceil(rows / world) per
+ chunk, so the last ranks may hold fewer rows or none)."""
+ size = math.ceil(rows / world) if rows else 0
+ start = min(rank * size, rows)
+ return start, min(start + size, rows)
+
+
+def _output_head_weight(model):
+ try:
+ head = model.get_output_embeddings()
+ except Exception:
+ head = None
+ weight = getattr(head, "weight", None)
+ return weight if isinstance(weight, torch.Tensor) else None
+
+
+_DORA_REFUSAL = (
+ "Unsloth: LoraConfig(use_dora = True) initializes DoRA's magnitude from the whole base weight, which "
+ "native FSDP2 sharded loading holds only as this rank's rows; load the full model (drop "
+ "fsdp2_sharded_loading / UNSLOTH_FSDP2_SHARDED_LOAD) to train DoRA under native FSDP2."
+)
+
+
+def _sharded_chunk(weight):
+ info = _preload(weight)
+ return info is not None and info["world"] > 1
+
+
+def _guard_dora_init():
+ """DoRA's magnitude init (peft DoraLinearLayer.update_layer, every PEFT version with DoRA) adds a full
+ lora_B @ lora_A to the base weight: refuse it on a sharded-loaded chunk with a clear message instead of
+ PEFT's shape mismatch. Installed once, inert for any other base layer."""
+ try:
+ from peft.tuners.lora import dora
+ except Exception:
+ return
+ cls = getattr(dora, "DoraLinearLayer", None)
+ update = getattr(cls, "update_layer", None)
+ if update is None or getattr(update, "_unsloth_fsdp2_load_guard", False):
+ return
+
+ @functools.wraps(update)
+ def update_layer(self, *args, **kwargs):
+ base = kwargs.get("base_layer", args[0] if args else None)
+ if _sharded_chunk(getattr(base, "weight", None)):
+ raise ValueError(_DORA_REFUSAL)
+ return update(self, *args, **kwargs)
+
+ update_layer._unsloth_fsdp2_load_guard = True
+ cls.update_layer = update_layer
+
+
+def _refuse_unsharded_forward(
+ module,
+ args,
+ kwargs = None,
+):
+ raise RuntimeError(
+ "Unsloth: this model was loaded with native FSDP2 sharded loading (each rank holds only its shard "
+ "of the linear weights); shard it first (unsloth.models.fsdp2_native.shard_model(model), or let the "
+ "Trainer do it under UNSLOTH_FSDP2=1) before running it."
+ )
+
+
+def _materialize(model, name, kwargs, mesh, plan, device):
+ """Fill a meta model: nn.Linear weights (not the output head) as this rank's dim-0 chunk, the rest whole."""
+ from .tp_native import _fill_from_checkpoint
+
+ world, rank = mesh.size(), mesh.get_local_rank()
+ ranks = mesh.mesh.tolist()
+ head = _output_head_weight(model)
+ linear = {
+ id(m.weight)
+ for m in model.modules()
+ if isinstance(m, nn.Linear) and isinstance(getattr(m, "weight", None), torch.Tensor)
+ }
+ owners = {}
+
+ def read_local(old, reader, key):
+ if id(old) not in linear or old is head or old.ndim < 2:
+ return None
+ rows = old.shape[0]
+ start, stop = _chunk_rows(rows, world, rank)
+ local = reader.rows(key, start, stop).to(device = device, dtype = old.dtype)
+ new = nn.Parameter(local.contiguous(), requires_grad = old.requires_grad)
+ setattr(
+ new,
+ _PRELOAD,
+ {
+ "shape": tuple(old.shape),
+ "ranks": ranks,
+ "world": world,
+ "rank": rank,
+ "rows": (start, stop),
+ },
+ )
+ owners[id(new)] = new
+ return new
+
+ reader, sharded_numel, full_numel = _fill_from_checkpoint(
+ model,
+ name,
+ kwargs,
+ device,
+ read_local,
+ label = "native FSDP2",
+ fallback = "a full load, then shard_model",
+ )
+ guards = [model.register_forward_pre_hook(_refuse_unsharded_forward)]
+ for module in model.modules():
+ if isinstance(module, nn.Linear) and id(module._parameters.get("weight")) in owners:
+ guards.append(module.register_forward_pre_hook(_refuse_unsharded_forward))
+ model.__dict__[_GUARD] = guards
+ stats = {
+ "world": world,
+ "rank": rank,
+ "sharded_tensors": len(owners),
+ "sharded_numel": sharded_numel,
+ "replicated_numel": full_numel,
+ "read_bytes_full": reader.full_bytes,
+ "read_bytes_sliced": reader.slice_bytes,
+ }
+ model.__dict__[_STATS] = stats
+ if owners and world > 1:
+ _guard_dora_init()
+ return stats
+
+
+def fsdp2_sharded_loading(mesh = None):
+ """Every transformers `from_pretrained` inside reads only this rank's native FSDP2 shard of the linear
+ weights (see the module docstring). `mesh`: the 1-D DeviceMesh shard_model will use (default: the
+ world, which is shard_model's default too)."""
+ from .tp_native import _sharded_from_pretrained
+
+ if mesh is None:
+ mesh = _default_load_mesh()
+ state = {
+ "mesh": mesh,
+ "plan": None,
+ "materialize": _materialize,
+ "label": "native FSDP2",
+ "quantized_hint": _QUANTIZED_HINT,
+ "prequantized_hint": _QUANTIZED_HINT,
+ "mesh_hint": "Unsloth: native FSDP2 sharded loading takes a 1-D mesh (HSDP / 2-D meshes load in full).",
+ }
+ _ACTIVE[0] += 1
+ try:
+ with _sharded_from_pretrained(state) as live:
+ yield live
+ finally:
+ _ACTIVE[0] -= 1
+
+
+def _default_load_mesh():
+ import torch.distributed as dist
+
+ if not (dist.is_available() and dist.is_initialized()):
+ from .fsdp2_native import _ensure_process_group, _local_cuda_index
+
+ index = _local_cuda_index() if torch.cuda.is_available() else None
+ if torch.cuda.is_available() and index is None:
+ index = torch.cuda.current_device()
+ device = torch.device("cuda", index) if index is not None else torch.device("cpu")
+ _ensure_process_group(device, None, None) # sets this rank's CUDA device first
+ from torch.distributed.device_mesh import init_device_mesh
+
+ device_type = "cuda" if torch.cuda.is_available() else "cpu"
+ return init_device_mesh(device_type, (dist.get_world_size(),))
+
+
+def load_fsdp2_sharded(
+ model_name,
+ mesh = None,
+ loader = None,
+ **kwargs,
+):
+ """`loader(model_name, **kwargs)` under fsdp2_sharded_loading (default FastLanguageModel.from_pretrained,
+ 16-bit). Raises if no model went through the sharded path."""
+ if loader is None:
+ from unsloth import FastLanguageModel
+ loader = FastLanguageModel.from_pretrained
+ kwargs.setdefault("load_in_4bit", False)
+ with fsdp2_sharded_loading(mesh) as state:
+ out = loader(model_name, **kwargs)
+ if not state["models"]:
+ raise RuntimeError(
+ "Unsloth: load_fsdp2_sharded's loader did not call transformers from_pretrained; nothing was "
+ "loaded sharded."
+ )
+ return out
+
+
+def fsdp2_sharded_load_requested():
+ """UNSLOTH_FSDP2_SHARDED_LOAD=1 with native FSDP2 requested (UNSLOTH_FSDP2=1 under WORLD_SIZE > 1)."""
+ if os.environ.get("UNSLOTH_FSDP2_SHARDED_LOAD", "0").strip().lower() not in (
+ "1",
+ "true",
+ "yes",
+ "on",
+ ):
+ return False
+ try:
+ from .fsdp2_native import native_fsdp2_requested
+ return bool(native_fsdp2_requested())
+ except Exception:
+ return False
+
+
+def sharded_load_env(loader):
+ """Decorator for Unsloth's top-level from_pretrained: under UNSLOTH_FSDP2_SHARDED_LOAD=1 (+ native FSDP2)
+ the outermost call runs inside fsdp2_sharded_loading. 4-bit / 8-bit loads are refused there."""
+
+ @functools.wraps(loader)
+ def wrapper(*args, **kwargs):
+ if _ACTIVE[0] or not fsdp2_sharded_load_requested():
+ return loader(*args, **kwargs)
+ with fsdp2_sharded_loading():
+ return loader(*args, **kwargs)
+
+ return wrapper
+
+
+# --------------------------------------------------------------------------------------------------
+# shard_model side
+# --------------------------------------------------------------------------------------------------
+
+
+def _preload(p):
+ return getattr(p, _PRELOAD, None) if isinstance(p, torch.Tensor) else None
+
+
+def has_preloaded_shards(model):
+ return any(_preload(p) is not None for p in model.parameters())
+
+
+def _refuse_weight_dependent_lora_init(model):
+ configs = {}
+ m, seen = model, 0
+ while isinstance(m, nn.Module) and seen < 8:
+ configs.update(getattr(m, "peft_config", None) or {})
+ modules = m.__dict__.get("_modules") or {}
+ m = modules.get("base_model") or modules.get("model")
+ seen += 1
+ bad = {
+ name: cfg.init_lora_weights
+ for name, cfg in configs.items()
+ if getattr(cfg, "init_lora_weights", True) not in (True, False, "gaussian")
+ }
+ if any(getattr(cfg, "use_dora", False) for cfg in configs.values()) and any(
+ _sharded_chunk(p) for p in model.parameters()
+ ):
+ raise ValueError(_DORA_REFUSAL)
+ if bad:
+ raise ValueError(
+ f"Unsloth: init_lora_weights {bad} reads the base weight, which native FSDP2 sharded loading "
+ "holds only as this rank's rows; use init_lora_weights = True / 'gaussian', or load the full "
+ "model (no sharded loading)."
+ )
+
+
+def _tag_modules_to_save_copies(model):
+ """PEFT modules_to_save deep-copies a sharded-loaded linear, but Parameter.__deepcopy__ drops the chunk's
+ tag: give each still-identical copy its original's, so it is sharded like the full load's copy."""
+ for module in model.modules():
+ original = getattr(module, "original_module", None)
+ copies = getattr(module, "modules_to_save", None)
+ if not isinstance(original, nn.Module) or not isinstance(copies, nn.ModuleDict):
+ continue
+ params = dict(original.named_parameters())
+ for copy in copies.values():
+ for name, q in copy.named_parameters():
+ p = params.get(name)
+ info = _preload(p)
+ if info is not None and _preload(q) is None and q.shape == p.shape:
+ q.__dict__[_PRELOAD] = info
+
+
+def begin_shard(model, mesh, config):
+ """Before shard_model's layout: check the preloaded chunks match `mesh`, drop the forward guards and
+ replace every chunk by a meta Parameter of the full shape. Returns ({(id(module), name): (local, info)}
+ (empty without preloaded chunks), config): FSDP2Config.ignored_params entries naming a chunk now name its
+ placeholder (the caller's set is updated in place, a non-set copied), so gather_replicated keeps them whole."""
+ _tag_modules_to_save_copies(model)
+ slots = {}
+ for module in model.modules():
+ for name, p in module._parameters.items():
+ info = _preload(p)
+ if info is not None:
+ slots.setdefault(id(p), (p, info, []))[2].append((module, name))
+ if not slots:
+ return {}, config
+ if mesh.ndim != 1:
+ raise RuntimeError(
+ "Unsloth: native FSDP2 sharded loading needs a 1-D shard mesh (the one it loaded over); this "
+ f"FSDP2Config.mesh has {mesh.ndim} dims. Load in full for HSDP."
+ )
+ ranks = mesh.mesh.tolist()
+ if config.shard_placement_fn is not None:
+ raise RuntimeError(
+ "Unsloth: native FSDP2 sharded loading holds dim-0 chunks; FSDP2Config.shard_placement_fn is "
+ "not supported with it."
+ )
+ if config.cpu_offload:
+ raise RuntimeError(
+ "Unsloth: native FSDP2 sharded loading does not support cpu_offload yet; load in full for it."
+ )
+ for p, info, _ in slots.values():
+ if info["ranks"] != ranks:
+ raise RuntimeError(
+ f"Unsloth: this model was sharded-loaded over ranks {info['ranks']}, but shard_model shards "
+ f"over {ranks}; pass the same mesh to fsdp2_sharded_loading and FSDP2Config."
+ )
+ _refuse_weight_dependent_lora_init(model)
+ for level in list(model.modules()):
+ for handle in level.__dict__.pop(_GUARD, None) or ():
+ handle.remove()
+ # PEFT modules_to_save deep-copies a guarded linear, hook included, with no handle on the root.
+ hooks = level.__dict__.get("_forward_pre_hooks") or {}
+ for key in [k for k, fn in hooks.items() if fn is _refuse_unsharded_forward]:
+ del hooks[key]
+ pending, swap = {}, {}
+ for p, info, where in slots.values():
+ meta = nn.Parameter(
+ torch.empty(info["shape"], dtype = p.dtype, device = "meta"),
+ requires_grad = p.requires_grad,
+ )
+ for attr, value in p.__dict__.items():
+ if attr != _PRELOAD:
+ meta.__dict__[attr] = value
+ meta.__dict__["_unsloth_fsdp2_pending"] = True
+ local = p.detach()
+ swap[id(p)] = meta
+ for module, name in where:
+ module._parameters[name] = meta
+ pending[(id(module), name)] = (local, info)
+ ignored = config.ignored_params
+ if ignored and any(id(q) in swap for q in ignored):
+ if isinstance(ignored, set):
+ for q in [q for q in ignored if id(q) in swap]:
+ ignored.discard(q)
+ ignored.add(swap[id(q)])
+ else:
+ config = dataclasses.replace(
+ config, ignored_params = {swap.get(id(q), q) for q in ignored}
+ )
+ return pending, config
+
+
+def _full_from_chunks(local, info, mesh, dtype):
+ """All-gather the dim-0 chunks back into the full tensor (exactly the checkpoint's)."""
+ import torch.distributed as dist
+
+ rows, world = info["shape"][0], info["world"]
+ size = math.ceil(rows / world)
+ padded = local.new_zeros((size,) + tuple(info["shape"][1:]))
+ padded[: local.shape[0]].copy_(local)
+ pieces = [torch.empty_like(padded) for _ in range(world)]
+ dist.all_gather(pieces, padded.contiguous(), group = mesh.get_group())
+ return torch.cat(pieces, 0)[:rows].to(dtype).contiguous()
+
+
+def gather_replicated(
+ model,
+ pending,
+ replicated,
+ mesh,
+ also = (),
+):
+ """Chunks whose parameter stays replicated (outside FSDP2) are gathered whole, and swapped in
+ `replicated` and the sets in `also`. Collective over `mesh`; every rank reaches it with the same set
+ (the layout and dtype decisions are agreed)."""
+ if not pending:
+ return 0
+ keep = {id(p) for p in replicated}
+ done = {}
+ for module in model.modules():
+ for name, p in list(module._parameters.items()):
+ key = (id(module), name)
+ if key not in pending or id(p) not in keep:
+ continue
+ local, info = pending.pop(key)
+ if id(p) not in done:
+ with torch.no_grad():
+ full = _full_from_chunks(local, info, mesh, p.dtype)
+ new = nn.Parameter(full, requires_grad = p.requires_grad)
+ for attr, value in p.__dict__.items():
+ if attr != "_unsloth_fsdp2_pending":
+ new.__dict__[attr] = value
+ done[id(p)] = (p, new)
+ module._parameters[name] = done[id(p)][1]
+ if done:
+ swap = {id(old): new for old, new in done.values()}
+ for params in (replicated, *also):
+ for p in list(params):
+ if id(p) in swap:
+ params.discard(p)
+ params.add(swap[id(p)])
+ return len(done)
+
+
+def finish_shard(model, pending):
+ """After fully_shard: each pending chunk becomes its FSDPParam's local shard (padded like fully_shard)."""
+ if not pending:
+ return 0
+ from .fsdp2_native import _fsdp_module_class
+
+ FSDPModule = _fsdp_module_class()
+ installed = 0
+ for module in model.modules():
+ if not isinstance(module, FSDPModule):
+ continue
+ state = module._get_fsdp_state()
+ group = getattr(state, "_fsdp_param_group", None)
+ for fsdp_param in getattr(group, "fsdp_params", None) or ():
+ info = fsdp_param._module_info
+ entry = pending.pop((id(info.module), info.param_name), None)
+ if entry is None:
+ continue
+ for shared, shared_name in zip(info.shared_modules, info.shared_param_names):
+ pending.pop((id(shared), shared_name), None)
+ local, _ = entry
+ old = fsdp_param.sharded_param
+ if tuple(local.shape) != tuple(fsdp_param.sharded_size):
+ raise RuntimeError(
+ f"Unsloth: native FSDP2 sharded loading read {tuple(local.shape)} rows for "
+ f"'{info.param_name}', but fully_shard keeps {tuple(fsdp_param.sharded_size)} on this rank."
+ )
+ with torch.no_grad():
+ local = local.to(dtype = old.dtype)
+ new = nn.Parameter(
+ fsdp_param.to_sharded_dtensor(local), requires_grad = old.requires_grad
+ )
+ for attr, value in old.__dict__.items():
+ if attr != "_unsloth_fsdp2_pending":
+ new.__dict__[attr] = value
+ fsdp_param._setattr_on_modules(new)
+ fsdp_param.reset_sharded_param()
+ installed += 1
+ if pending:
+ names = sorted({name for (_, name) in pending})
+ raise RuntimeError(
+ f"Unsloth: native FSDP2 sharded loading left {len(pending)} chunks ({names[:4]}) outside every "
+ "FSDP2 unit; load this model in full."
+ )
+ return installed
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index f47ea04f998..8ecbd9099e2 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -1547,6 +1547,10 @@ def shard_model(model, config: Optional[FSDP2Config] = None):
torch.cuda.set_device(device)
_warn_nccl2_memory(dist, device)
mesh = config.mesh if config.mesh is not None else _default_mesh(device, dist)
+ from .fsdp2_load import begin_shard, finish_shard, gather_replicated
+
+ # fsdp2_sharded_loading chunks -> full-shape meta placeholders until fully_shard (empty otherwise).
+ pending, config = begin_shard(model, mesh, config)
_sync_trainable_from_first_rank(model, dist, device, config.mesh)
param_dtype = _resolve_param_dtype(model, config)
mp_policy = MixedPrecisionPolicy(param_dtype = param_dtype, reduce_dtype = config.reduce_dtype)
@@ -1655,6 +1659,10 @@ def shard_model(model, config: Optional[FSDP2Config] = None):
if layout.replicate_frozen:
replicated.update(p for p in model.parameters() if not p.requires_grad)
_refuse_units_sharing_params(model, units + list(extra_units), replicated)
+ # The caller's ignored_params set follows a gathered chunk to its whole parameter, as the full
+ # load's set names the live one.
+ user_ignored = config.ignored_params if isinstance(config.ignored_params, set) else set()
+ gather_replicated(model, pending, replicated, mesh, also = (ignored, user_ignored))
if replicated:
if not _supports_ignored_params(fully_shard):
raise RuntimeError(
@@ -1682,6 +1690,7 @@ def shard_model(model, config: Optional[FSDP2Config] = None):
else None
),
)
+ finish_shard(model, pending)
model._unsloth_reshard_after_forward = False
setattr(model, _NATIVE_FLAG, True)
for param in model.parameters():
@@ -3342,15 +3351,47 @@ def _scaler_for_native(trainer):
if handler is not None and hasattr(handler, "to_kwargs"):
kwargs = handler.to_kwargs()
kwargs.setdefault("device", getattr(scaler, "_device", "cuda"))
- # The Trainer scales each rank's loss by world size, so float16 activation grads are world times
- # one GPU's and overflow at its scale. Divide init_scale by the DP size unless the user set it.
- world = _data_parallel_size(getattr(trainer, "model", None))
+ # The Trainer scales each rank's loss by the number of processes, so float16 activation grads are
+ # that many times one GPU's and overflow at its scale. Divide init_scale by it unless the user set it.
+ world = _loss_scale_ranks(trainer)
if world > 1 and "init_scale" not in kwargs:
kwargs["init_scale"] = 2.0**16 / world
accelerator.scaler = _ShardedGradScaler(**kwargs)
return True
+def _loss_scale_ranks(trainer):
+ """How many times one GPU's per-token loss gradient a rank's backward carries: the Trainer multiplies
+ each rank's loss by accelerator.num_processes (its token mean over the world), divided by the tensor
+ parallel size (tp peers share a batch whose tokens the gathered count repeats). That is the world, not
+ the FSDP2 mesh: on two disjoint 2-rank sub-meshes of 4 ranks it is 4. Context parallel peers share a
+ batch but CP divides the gathered count by cp, so CP = 2, DP = 1 is the world (2) too."""
+ accelerator = getattr(trainer, "accelerator", None)
+ model = getattr(trainer, "model", None)
+ ranks = getattr(accelerator, "num_processes", None)
+ if not isinstance(ranks, int) or ranks < 1:
+ ranks = None
+ try:
+ import torch.distributed as dist
+ if dist.is_available() and dist.is_initialized():
+ ranks = int(dist.get_world_size())
+ except Exception:
+ pass
+ if ranks is None:
+ return _data_parallel_size(model)
+ pc = getattr(accelerator, "parallelism_config", None)
+ tp = int(getattr(pc, "tp_size", 1) or 1) if pc is not None else 1
+ tp_native = sys.modules.get(f"{__package__}.tp_native") if __package__ else None
+ if tp == 1 and tp_native is not None and model is not None:
+ try:
+ if tp_native.is_tensor_parallel(model):
+ tp_mesh = tp_native._tp_meshes(model)[0]
+ tp = int(tp_mesh.size()) if tp_mesh is not None else 1
+ except Exception:
+ tp = 1
+ return max(1, ranks // max(1, tp))
+
+
def _data_parallel_size(model):
mesh = getattr(model, "__dict__", {}).get("_unsloth_fsdp2_mesh") if model is not None else None
try:
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 9b2d3bb8707..5c6d273056e 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -35,6 +35,7 @@ from transformers import __version__ as transformers_version
from peft import PeftConfig, PeftModel
from .grouped_linear_lora import register_grouped_linear_lora_for_adapter
from .fp8_to_nf4 import track_explicit_4bit_request
+from .fsdp2_load import sharded_load_env as fsdp2_sharded_load_env
from .float8_training import convert_model_to_float8_training, resolve_float8_training
from .mistral_format import (
MistralFormatRedirect,
@@ -1229,6 +1230,7 @@ class FastLanguageModel(FastLlamaModel):
return get_mtp_head(model, *args, **kwargs)
@staticmethod
+ @fsdp2_sharded_load_env
@tokenizers_v1_on_return
@_offline_aware_load
@mistral_format_redirect
@@ -2186,6 +2188,7 @@ class FastModel(FastBaseModel):
return FastBaseModel.for_training(model, use_gradient_checkpointing)
@staticmethod
+ @fsdp2_sharded_load_env
@tokenizers_v1_on_return
@_restore_load_scoped_env
@_offline_aware_load
diff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.py
index 619d22690e3..26ef5d01240 100644
--- a/unsloth/models/tp_native.py
+++ b/unsloth/models/tp_native.py
@@ -1717,6 +1717,7 @@ def save_tp_adapter(model, save_directory, **kwargs):
# load-time peak is the replicated parts plus 1 / tp of the attention / MLP weights, never the whole model.
_PRELOADED = "_unsloth_tp_preloaded"
+_TP_QUANTIZED_HINT = "Load 4-bit / 8-bit / quantized models in full and call apply_tensor_parallel, or pass load_in_4bit = False."
_UNSUPPORTED_LOAD_KWARGS = (
"load_in_4bit",
"load_in_8bit",
@@ -1900,6 +1901,12 @@ class _SliceReader:
self.slice_bytes += t.numel() * t.element_size()
return t
+ def rows(self, key, start, stop):
+ """Rows [start, stop) of dim 0 (an FSDP2 chunk); only those bytes are read."""
+ t = self._open(key).get_slice(key)[start:stop]
+ self.slice_bytes += t.numel() * t.element_size()
+ return t
+
def _checkpoint_key(names, model, index):
prefix = getattr(_unwrap(model), "base_model_prefix", "") or ""
@@ -1915,6 +1922,59 @@ def _checkpoint_key(names, model, index):
return None
+def _checkpoint_aliases(
+ model,
+ index,
+ key_mapping = None,
+):
+ """{model key: checkpoint key} for a checkpoint saved under another naming: transformers 5's conversion
+ mapping renames (VLMs save `model.layers.*` for `model.language_model.layers.*`) and 4.x
+ `_checkpoint_conversion_mapping`. A model key a WeightConverter builds (merged experts, fused or permuted
+ weights) maps to None: it is not one checkpoint tensor, so it cannot be read sliced. `key_mapping` is the
+ load's own (from_pretrained(key_mapping = ...), e.g. Unsloth's text-only VLM loads)."""
+ legacy = getattr(model, "_checkpoint_conversion_mapping", None) or {}
+ renamings = converters = None
+ try:
+ from transformers.conversion_mapping import get_model_conversion_mapping
+ from transformers.core_model_loading import (
+ WeightConverter,
+ WeightRenaming,
+ rename_source_key,
+ )
+
+ try:
+ transforms = get_model_conversion_mapping(model, key_mapping = key_mapping)
+ except TypeError:
+ transforms = get_model_conversion_mapping(model)
+ renamings = [t for t in transforms if isinstance(t, WeightRenaming)]
+ converters = [t for t in transforms if isinstance(t, WeightConverter)]
+ except Exception:
+ # transformers 4.x: from_pretrained's key_mapping replaces _checkpoint_conversion_mapping.
+ legacy = key_mapping or legacy
+ if not legacy and not renamings and not converters:
+ return {}
+ meta = dict.fromkeys(model.state_dict().keys(), True)
+ prefix = getattr(model, "base_model_prefix", None) or None
+ out = {}
+ for key in index:
+ new = key
+ for pattern, replacement in legacy.items():
+ new, n = re.subn(pattern, replacement, new)
+ if n:
+ break
+ converted = None
+ if renamings is not None:
+ try:
+ new, converted = rename_source_key(new, renamings, converters, prefix, meta)
+ except Exception:
+ converted = None
+ if converted is not None:
+ out[new] = None
+ elif new != key:
+ out.setdefault(new, key)
+ return out
+
+
def _buffer_skeleton(model):
"""The model rebuilt with meta parameters and real buffers (RoPE tables), as from_pretrained builds them."""
from accelerate import init_empty_weights
@@ -1929,20 +1989,8 @@ def _buffer_skeleton(model):
torch.set_default_dtype(previous)
-def _materialize_sharded(model, name, kwargs, mesh, plan, device):
- """Fill a meta model from the checkpoint: replicated tensors whole, sharded projections as local slices."""
- tp, rank = mesh.size(), mesh.get_local_rank()
- tp_plan = derive_tp_plan(model, tp, plan)
- shards = {}
- for region in tp_plan.regions:
- for proj in region.projs:
- weight, base = _weight(proj.module)
- col = proj.style == COLWISE
- shards[id(weight)] = (proj, 0 if col else 1, proj.parts if col else (weight.shape[1],))
- if base.bias is not None and col:
- shards[id(base.bias)] = (proj, 0, proj.parts)
- index = _checkpoint_index(name, kwargs)
- reader = _SliceReader(index)
+def _parameter_slots(model):
+ """{id(parameter): (parameter, [(module, name)], [qualified names])}: tied parameters are one slot."""
slots = {}
for module_name, module in model.named_modules(remove_duplicate = False):
for pname, p in module._parameters.items():
@@ -1950,36 +1998,63 @@ def _materialize_sharded(model, name, kwargs, mesh, plan, device):
entry = slots.setdefault(id(p), (p, [], []))
entry[1].append((module, pname))
entry[2].append(f"{module_name}.{pname}" if module_name else pname)
+ return slots
+
+
+def _fill_from_checkpoint(
+ model,
+ name,
+ kwargs,
+ device,
+ read_local,
+ label = "tensor parallel",
+ fallback = "full load, then apply_tensor_parallel",
+):
+ """Fill a meta model from its safetensors checkpoint. `read_local(old, reader, key)` returns this rank's
+ Parameter for a sharded tensor (reading only its bytes) or None, and then the tensor is read whole.
+ Meta buffers come from the checkpoint or a meta-parameter skeleton of the same class (RoPE tables).
+ Returns (reader, sharded numel, whole numel)."""
+ index = _checkpoint_index(name, kwargs)
+ reader = _SliceReader(index)
+ aliases = None
sharded_numel = full_numel = 0
- for pid, (old, where, names) in slots.items():
+
+ def find(names):
+ nonlocal aliases
key = _checkpoint_key(names, model, index)
+ if key is not None:
+ return key
+ if aliases is None:
+ aliases = _checkpoint_aliases(model, index, (kwargs or {}).get("key_mapping"))
+ for n in names:
+ if n in aliases:
+ if aliases[n] is None:
+ raise ValueError(
+ f"Unsloth: {label} sharded loading cannot read '{n}': transformers builds it from several "
+ f"checkpoint tensors (a weight conversion); load this model without sharded loading "
+ f"({fallback})."
+ )
+ return aliases[n]
+ return None
+
+ for old, where, names in _parameter_slots(model).values():
+ key = find(names)
if key is None:
raise ValueError(
- f"Unsloth: tensor parallel sharded loading found no checkpoint tensor for '{names[0]}'; load "
- "this model without sharded loading (full load, then apply_tensor_parallel)."
+ f"Unsloth: {label} sharded loading found no checkpoint tensor for '{names[0]}'; load "
+ f"this model without sharded loading ({fallback})."
)
if reader.shape(key) != tuple(old.shape):
raise ValueError(
f"Unsloth: checkpoint tensor '{key}' has shape {reader.shape(key)}, the model expects "
f"{tuple(old.shape)}."
)
- shard = shards.get(pid)
- if shard is None:
+ new = read_local(old, reader, key)
+ if new is None:
value = reader.full(key).to(device = device, dtype = old.dtype)
new = nn.Parameter(value, requires_grad = old.requires_grad)
full_numel += old.numel()
else:
- proj, dim, parts = shard
- local = reader.slice(key, dim, parts, rank, tp).to(device = device, dtype = old.dtype)
- # A plain local tensor tagged with its full shape: nn.Linear keeps the full in / out features, so
- # get_peft_model builds full-size LoRA factors with the 1-GPU init (a DTensor would make PEFT size
- # them from the shard). apply_tensor_parallel wraps it into the DTensor.
- new = nn.Parameter(local.contiguous(), requires_grad = old.requires_grad)
- if len(parts) > 1:
- new._unsloth_tp_parts = (dim, tuple(parts))
- new._unsloth_tp_full_shape = tuple(old.shape)
- new._unsloth_tp_preload_ranks = mesh.mesh.tolist()
- setattr(new, _PRELOADED, (proj.style, tuple(proj.parts), tp))
sharded_numel += old.numel()
for module, pname in where:
module._parameters[pname] = new
@@ -1989,22 +2064,59 @@ def _materialize_sharded(model, name, kwargs, mesh, plan, device):
if buf is None or not buf.is_meta:
continue
key = f"{module_name}.{bname}" if module_name else bname
- if bname not in module._non_persistent_buffers_set and key in index:
- value = reader.full(key)
+ source = find([key]) if bname not in module._non_persistent_buffers_set else None
+ if source is not None:
+ value = reader.full(source)
else:
skeleton = skeleton if skeleton is not None else _buffer_skeleton(model)
value = skeleton.get_buffer(key)
if value.is_meta:
raise ValueError(f"Unsloth: sharded loading cannot rebuild the buffer '{key}'.")
module._buffers[bname] = value.to(device = device, dtype = buf.dtype)
+ if isinstance(getattr(model, "hf_device_map", None), dict):
+ model.hf_device_map = {"": device.index if device.type == "cuda" else device.type}
+ return reader, sharded_numel, full_numel
+
+
+def _materialize_sharded(model, name, kwargs, mesh, plan, device):
+ """Fill a meta model from the checkpoint: replicated tensors whole, sharded projections as local slices."""
+ tp, rank = mesh.size(), mesh.get_local_rank()
+ tp_plan = derive_tp_plan(model, tp, plan)
+ shards = {}
+ for region in tp_plan.regions:
+ for proj in region.projs:
+ weight, base = _weight(proj.module)
+ col = proj.style == COLWISE
+ shards[id(weight)] = (proj, 0 if col else 1, proj.parts if col else (weight.shape[1],))
+ if base.bias is not None and col:
+ shards[id(base.bias)] = (proj, 0, proj.parts)
+
+ def read_local(old, reader, key):
+ shard = shards.get(id(old))
+ if shard is None:
+ return None
+ proj, dim, parts = shard
+ local = reader.slice(key, dim, parts, rank, tp).to(device = device, dtype = old.dtype)
+ # A plain local tensor tagged with its full shape: nn.Linear keeps the full in / out features, so
+ # get_peft_model builds full-size LoRA factors with the 1-GPU init (a DTensor would make PEFT size
+ # them from the shard). apply_tensor_parallel wraps it into the DTensor.
+ new = nn.Parameter(local.contiguous(), requires_grad = old.requires_grad)
+ if len(parts) > 1:
+ new._unsloth_tp_parts = (dim, tuple(parts))
+ new._unsloth_tp_full_shape = tuple(old.shape)
+ new._unsloth_tp_preload_ranks = mesh.mesh.tolist()
+ setattr(new, _PRELOADED, (proj.style, tuple(proj.parts), tp))
+ return new
+
+ reader, sharded_numel, full_numel = _fill_from_checkpoint(
+ model, name, kwargs, device, read_local
+ )
for region in tp_plan.regions:
for proj in region.projs:
base = _weight(proj.module)[1]
base.__dict__["_unsloth_tp_load_guard"] = base.register_forward_pre_hook(
_refuse_unapplied_forward
)
- if isinstance(getattr(model, "hf_device_map", None), dict):
- model.hf_device_map = {"": device.index if device.type == "cuda" else device.type}
stats = {
"tp": tp,
"rank": rank,
@@ -2037,16 +2149,20 @@ def _from_pretrained_sharded(original, cls, name, args, kwargs, state):
if state["active"]:
return original(cls, name, *args, **kwargs) # a nested load inside from_pretrained
config = kwargs.get("config")
+ label = state.get("label", "tensor parallel")
for key in _UNSUPPORTED_LOAD_KWARGS:
if kwargs.get(key) or (key == "quantization_config" and getattr(config, key, None)):
raise ValueError(
- f"Unsloth: tensor parallel sharded loading supports 16-bit safetensors checkpoints only "
- f"(got {key}). Load 4-bit / 8-bit / quantized models in full and call apply_tensor_parallel, "
- "or pass load_in_4bit = False."
+ f"Unsloth: {label} sharded loading supports 16-bit safetensors checkpoints only "
+ f"(got {key}). {state.get('quantized_hint', _TP_QUANTIZED_HINT)}"
)
mesh = state["mesh"] if state["mesh"] is not None else _world_mesh()
if mesh.ndim != 1:
- raise ValueError("Unsloth: sharded_loading takes a 1-D tp mesh (pass mesh['tp']).")
+ raise ValueError(
+ state.get(
+ "mesh_hint", "Unsloth: sharded_loading takes a 1-D tp mesh (pass mesh['tp'])."
+ )
+ )
device = _load_device(kwargs.get("device_map"), mesh)
kwargs = dict(kwargs, device_map = "meta")
state["active"] = True
@@ -2060,10 +2176,11 @@ def _from_pretrained_sharded(original, cls, name, args, kwargs, state):
model.config, "quantization_config", None
):
raise ValueError(
- "Unsloth: tensor parallel sharded loading supports 16-bit checkpoints only; this checkpoint is "
- "pre-quantized. Load it in full and call apply_tensor_parallel."
+ f"Unsloth: {label} sharded loading supports 16-bit checkpoints only; this checkpoint is "
+ f"pre-quantized. {state.get('prequantized_hint', 'Load it in full and call apply_tensor_parallel.')}"
)
- _materialize_sharded(model, name, kwargs, mesh, state["plan"], device)
+ materialize = state.get("materialize", _materialize_sharded)
+ materialize(model, name, kwargs, mesh, state["plan"], device)
state["models"].append(model)
return model
@@ -2074,7 +2191,7 @@ def _world_mesh():
if not dist.is_initialized():
raise RuntimeError(
- "Unsloth: tensor parallel sharded loading needs an initialized process group (torchrun) or a tp_mesh."
+ "Unsloth: sharded loading needs an initialized process group (torchrun) or a mesh."
)
device_type = "cuda" if torch.cuda.is_available() else "cpu"
return init_device_mesh(device_type, (dist.get_world_size(),), mesh_dim_names = ("tp",))
@@ -2092,11 +2209,20 @@ def sharded_loading(tp_mesh = None, plan = None):
model = FastLanguageModel.get_peft_model(model, ...)
apply_tensor_parallel(model, mesh["tp"]) # required before the first forward
"""
+ with _sharded_from_pretrained({"mesh": tp_mesh, "plan": plan}) as state:
+ yield state
+
+
+def _sharded_from_pretrained(state):
+ """Swap transformers' PreTrainedModel.from_pretrained for the meta build + checkpoint fill of `state`
+ (mesh, plan, materialize(model, name, kwargs, mesh, plan, device), message parts). Shared by tensor
+ parallel and native FSDP2 sharded loading."""
from transformers import PreTrainedModel
descriptor = PreTrainedModel.__dict__["from_pretrained"]
original = descriptor.__func__
- state = {"mesh": tp_mesh, "plan": plan, "models": [], "active": False}
+ state = dict(state, models = [], active = False)
def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
return _from_pretrained_sharded(