Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 sharded loading: allow orthogonal LoRA init (#17038)
Daniel Han1f4e34e4ff2a
diff --git a/tests/test_fsdp2_native_sharded_load_gloo.py b/tests/test_fsdp2_native_sharded_load_gloo.pyindex a2d66441e7f..3bff975b869 100644--- a/tests/test_fsdp2_native_sharded_load_gloo.py+++ b/tests/test_fsdp2_native_sharded_load_gloo.py@@ -305,7 +305,91 @@ def _case_dora(rank, world, tmp, ckpt):return out-CASES = {"parity": _case, "refusals": _case_refusals, "dora": _case_dora}+def _lora_factors(model):+ return {+ n: _full_tensor(p)+ for n, p in model.named_parameters()+ if ".lora_A." in n or ".lora_B." in n+ }+++def _full_tensor(t):+ t = t.full_tensor() if hasattr(t, "full_tensor") else t+ return t.detach().clone()+++def _one_step(native, model, ids):+ """One forward / backward / AdamW step: (loss, clipped grad norm, full grads, full trained factors)."""+ opt = torch.optim.AdamW([p for p in model.parameters() if p.requires_grad], lr = 1e-2)+ loss = model(input_ids = ids, labels = ids).loss+ loss.backward()+ norm = float(native.native_clip_grad_norm_(model.parameters(), 1.0))+ grads = {+ n: _full_tensor(p.grad)+ for n, p in model.named_parameters()+ if p.requires_grad and p.grad is not None+ }+ opt.step()+ return float(loss), norm, grads, _lora_factors(model)+++def _case_random_init(rank, world, tmp, ckpt, init):+ """A random LoRA init under sharded loading vs one process (a full load, no FSDP2) from the same seed.+ Rank 1 draws from another seed: shard_model's first-rank sync must make every rank agree."""+ from peft import LoraConfig, get_peft_model+ from transformers import AutoModelForCausalLM++ native, loading = _load_module()+ kw = dict(torch_dtype = torch.float32, attn_implementation = "eager")+ lora = lambda r = 4, i = init: LoraConfig(+ r = r, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = i+ )+ ids = torch.randint(0, 100, (2, 7), generator = torch.Generator().manual_seed(1))+ out = {}++ torch.manual_seed(1)+ ref = get_peft_model(AutoModelForCausalLM.from_pretrained(ckpt, **kw), lora())+ out["init_ref"] = _lora_factors(ref)+ out["step_ref"] = _one_step(native, ref, ids)++ with loading.fsdp2_sharded_loading():+ model = AutoModelForCausalLM.from_pretrained(ckpt, **kw)+ torch.manual_seed(1 + rank)+ model = get_peft_model(model, lora())+ out["init_local"] = _lora_factors(model) # before shard_model: this rank's own draw, full size+ out["sliced"] = sum(+ 1 for p in model.parameters() if getattr(p, loading._PRELOAD, None) is not None+ )+ config = native.FSDP2Config(param_dtype = torch.float32, device = "cpu")+ out["error"] = _raised(lambda: native.shard_model(model, config))+ if out["error"] is None:+ out["init_sharded"] = _lora_factors(model)+ out["step"] = _one_step(native, model, ids)++ for name, kwargs in (+ ("odd_r", dict(r = 3, i = "orthogonal")),+ ("pissa", dict(i = "pissa")),+ ("olora", dict(i = "olora")),+ ):+ with loading.fsdp2_sharded_loading():+ other = AutoModelForCausalLM.from_pretrained(ckpt, **kw)++ def attempt():+ peft_model = get_peft_model(other, lora(**kwargs))+ native.shard_model(+ peft_model, native.FSDP2Config(param_dtype = torch.float32, device = "cpu")+ )++ out[name] = _raised(attempt)+ return out+++CASES = {+ "parity": _case,+ "refusals": _case_refusals,+ "dora": _case_dora,+ "random_init": _case_random_init,+}def _worker(rank, world, port, case, tmp, kwargs):@@ -437,6 +521,44 @@ def test_sharded_loading_refuses_dora():assert "use_dora" in (res["dora"] or "") and "full" in res["dora"], res["dora"][email protected]("init", ["orthogonal", "gaussian", True])+def test_sharded_loading_random_lora_init_matches_one_process(init):+ """Random LoRA inits read only the full in / out features and the weight dtype, never the sharded-loaded+ rows: the factors are full size, and after shard_model's first-rank sync every rank holds exactly the+ one-process init from rank 0's seed; one AdamW step then matches the one-process run."""+ with tempfile.TemporaryDirectory() as ckpt:+ _save("llama", ckpt, {"intermediate_size": 37})+ results = _spawn("random_init", ckpt = ckpt, init = init)+ for rank, res in enumerate(results):+ assert res["sliced"] > 0+ assert res["error"] is None, res["error"]+ ref = res["init_ref"]+ assert set(res["init_local"]) == set(res["init_sharded"]) == set(ref) and ref+ for name, full in ref.items():+ assert res["init_local"][name].shape == full.shape, name+ if rank == 0: # same seed as the one-process run: the sharded rows change nothing+ assert torch.equal(res["init_local"][name], full), name+ assert torch.equal(res["init_sharded"][name], full), name+ if init == "orthogonal":+ for name, A in ref.items():+ if ".lora_A." in name:+ B = ref[name.replace(".lora_A.", ".lora_B.")]+ assert A.abs().sum() > 0 and B.abs().sum() > 0, name+ torch.testing.assert_close(B @ A, torch.zeros_like(B @ A), rtol = 0, atol = 1e-6)+ loss_ref, norm_ref, grads_ref, step_ref = res["step_ref"]+ loss, norm, grads, step = res["step"]+ assert loss == pytest.approx(loss_ref, rel = 1e-6)+ assert norm == pytest.approx(norm_ref, rel = 1e-5)+ assert set(grads) == set(grads_ref) and grads+ for name in grads_ref:+ torch.testing.assert_close(grads[name], grads_ref[name], rtol = 1e-4, atol = 1e-6, msg = name)+ for name in step_ref:+ torch.testing.assert_close(step[name], step_ref[name], rtol = 1e-4, atol = 1e-6, msg = name)+ assert "rank to be even" in (res["odd_r"] or ""), res["odd_r"]+ for name in ("pissa", "olora"):+ assert "reads the base weight" in (res[name] or ""), (name, res[name])++def test_chunk_rows_match_fully_shard():_, loading = _load_module()for rows in (1, 2, 5, 7, 37, 64):diff --git a/unsloth/models/fsdp2_load.py b/unsloth/models/fsdp2_load.pyindex 67f99b2e383..fc1c2f2b9b2 100644--- a/unsloth/models/fsdp2_load.py+++ b/unsloth/models/fsdp2_load.py@@ -277,10 +277,12 @@ def _refuse_weight_dependent_lora_init(model):modules = m.__dict__.get("_modules") or {}m = modules.get("base_model") or modules.get("model")seen += 1+ # Random inits (True / "gaussian" / "orthogonal") read only the module's full in / out features and the+ # weight dtype, so a rank's rows give the one-process factors; the others read the weight itself.bad = {name: cfg.init_lora_weightsfor name, cfg in configs.items()- if getattr(cfg, "init_lora_weights", True) not in (True, False, "gaussian")+ if getattr(cfg, "init_lora_weights", True) not in (True, False, "gaussian", "orthogonal")}if any(getattr(cfg, "use_dora", False) for cfg in configs.values()) and any(_sharded_chunk(p) for p in model.parameters()@@ -289,8 +291,8 @@ def _refuse_weight_dependent_lora_init(model):if bad:raise ValueError(f"Unsloth: init_lora_weights {bad} reads the base weight, which native FSDP2 sharded loading "- "holds only as this rank's rows; use init_lora_weights = True / 'gaussian', or load the full "- "model (no sharded loading)."+ "holds only as this rank's rows; use init_lora_weights = True / 'gaussian' / 'orthogonal', or load "+ "the full model (no sharded loading).")