Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Tensor parallel sharded loading: allow init_lora_weights = "orthogonal" (#15218)
Shubhankar Tripathyc92009bbafc1
diff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.pyindex 2c85c2967ce..428e787e71f 100644--- a/tests/test_tp_native_gloo.py+++ b/tests/test_tp_native_gloo.py@@ -868,14 +868,15 @@ def _case_trainer(def _case_sharded_lora_init(rank, world, tmp, ckpt):- """Weight-dependent LoRA init (PiSSA) on sharded_loading slices is refused before any mutation."""+ """Weight-dependent LoRA init (PiSSA) on sharded_loading slices is refused before any mutation;+ orthogonal init only reads the full in / out features and the dtype, so it is allowed like gaussian."""from peft import LoraConfig, get_peft_modelfrom torch.distributed.device_mesh import init_device_meshfrom transformers import AutoModelForCausalLMmesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))out = {}- for init in ("pissa", "gaussian"):+ for init in ("pissa", "gaussian", "orthogonal"):model = tpn.load_tensor_parallel(ckpt, mesh["tp"], loader = AutoModelForCausalLM.from_pretrained, torch_dtype = torch.float32)@@ -886,6 +887,60 @@ def _case_sharded_lora_init(rank, world, tmp, ckpt):return out+def _case_sharded_random_lora_init(rank, world, tmp, ckpt, init, sync_trainable, seed_per_rank):+ """A random LoRA init on sharded_loading slices vs the same init on a full load in one process. Each rank+ seeds get_peft_model differently when seed_per_rank, so only the tp rank 0 broadcast can make them agree."""+ from peft import LoraConfig, get_peft_model+ from torch.distributed.device_mesh import init_device_mesh+ from transformers import AutoModelForCausalLM++ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))+ kw = dict(torch_dtype = torch.float32, attn_implementation = "eager")++ def peft(model, seed):+ torch.manual_seed(seed)+ config = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = init)+ return get_peft_model(model, config)++ def lora(model):+ return {+ n: tpn._full(p.detach(), p).clone() for n, p in model.named_parameters() if "lora_" in n+ }++ ref = peft(AutoModelForCausalLM.from_pretrained(ckpt, **kw), 7)+ model = tpn.load_tensor_parallel(+ ckpt, mesh["tp"], loader = AutoModelForCausalLM.from_pretrained, **kw+ )+ model = peft(model, 7 + rank if seed_per_rank else 7)+ error = _raised(+ lambda: tpn.apply_tensor_parallel(+ model, mesh["tp"], config = tpn.TPConfig(sync_trainable = sync_trainable)+ )+ )+ if error is not None:+ return {"error": error}+ # Replicated factors hold this rank's own copy: the raw local tensors show any per-rank divergence.+ local = {+ n: (p.to_local() if tpn._is_dtensor(p) else p).detach().clone()+ for n, p in model.named_parameters()+ if "lora_" in n+ }+ init_ref, init_tp = lora(ref), lora(model)+ ids = _batch(ref)+ ref_out = _train(ref, ids, 1, torch.nn.utils.clip_grad_norm_)+ tp_out = _train(model, ids, 1, tpn.tp_clip_grad_norm_)+ return {+ "error": None,+ "local": local,+ "init_ref": init_ref,+ "init_tp": init_tp,+ "ref": ref_out,+ "tp": tp_out,+ "step_ref": lora(ref),+ "step_tp": lora(model),+ }++def _case_sampler(rank, world, tmp, dp, strategy):"""Rows each rank's dataloader yields under a length-aware sampler, after a rank-local RNG draw."""import transformers@@ -1044,6 +1099,7 @@ CASES = {"tp_fsdp2_one_writer": _case_tp_fsdp2_writer,"tp_fsdp2_writer_pipeline_slice": _case_tp_fsdp2_writer_pipeline_slice,"trainer": _case_trainer,+ "sharded_random_lora_init": _case_sharded_random_lora_init,"sampler": _case_sampler,"sampler_resume": _case_sampler_resume,}@@ -1342,6 +1398,57 @@ def test_sharded_loading_refuses_weight_dependent_lora_init():assert "depends on the base weight" in (res["pissa"] or ""), resassert not res["pissa_tp"]assert res["gaussian"] is None and res["gaussian_tp"], res+ assert res["orthogonal"] is None and res["orthogonal_tp"], res++[email protected]("init", ["orthogonal", "gaussian", True])+def test_sharded_loading_random_lora_init_matches_one_process(init):+ """Random LoRA inits read only the full in / out features, so the sharded factors are full size; tp rank 0's+ broadcast makes per-rank seeds agree, and one AdamW step matches the unsharded run from the same seed."""+ with tempfile.TemporaryDirectory() as ckpt:+ _save_checkpoint("llama", ckpt, None)+ results = _spawn(+ "sharded_random_lora_init",+ 2,+ ckpt = ckpt,+ init = init,+ sync_trainable = True,+ seed_per_rank = True,+ )+ r0, r1 = results+ assert r0["error"] is None and r1["error"] is None, (r0["error"], r1["error"])+ assert set(r0["local"]) == set(r1["local"]) and r0["local"]+ for res in results:+ assert set(res["init_tp"]) == set(res["init_ref"])+ for name, ref in res["init_ref"].items():+ # Bit-identical to the one-process init from tp rank 0's seed: full-size factors, no shard-sized draw.+ assert torch.equal(res["init_tp"][name], ref), name+ if init == "orthogonal":+ for name, A in res["init_ref"].items():+ if ".lora_A." not in name:+ continue+ B = res["init_ref"][name.replace(".lora_A.", ".lora_B.")]+ # Orthogonal init: both factors non-zero, B @ A == 0 (rows from complementary halves of one Q).+ assert A.abs().sum() > 0 and B.abs().sum() > 0, name+ torch.testing.assert_close(+ B @ A, torch.zeros_like(B @ A), rtol = 0, atol = 1e-6, msg = name+ )+ ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]+ logits, grads, losses, norms = res["tp"]+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)+ assert set(grads) == set(ref_grads) and grads+ for name in ref_grads:+ torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)+ assert losses == pytest.approx(ref_losses, rel = 1e-5)+ assert norms == pytest.approx(ref_norms, rel = 1e-5)+ for name, ref in res["step_ref"].items():+ torch.testing.assert_close(res["step_tp"][name], ref, rtol = 1e-4, atol = 1e-6, msg = name)+ # Every rank's local copy of a replicated factor is the same tensor after the broadcast.+ for name in r0["local"]:+ if r0["local"][name].shape == r1["local"][name].shape and name in r0["init_ref"]:+ full = r0["init_ref"][name]+ if r0["local"][name].shape == full.shape:+ assert torch.equal(r0["local"][name], r1["local"][name]), namedef test_sharded_loading_refuses_quantized_loads():diff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.pyindex 1081498f8cd..9f7d7061e0f 100644--- a/unsloth/models/tp_native.py+++ b/unsloth/models/tp_native.py@@ -160,6 +160,8 @@ class TPConfig:# {"layers.*.self_attn.q_proj": "colwise", ...}; None = the model's TP plan, else structural discovery.plan: Optional[dict] = None# Broadcast trainable parameters from tp rank 0 first (every rank must start from the same adapter).+ # False: the caller guarantees it, e.g. a loaded adapter or the same seed before get_peft_model on every+ # rank (random LoRA inits draw full-size factors, so per-rank seeds give per-rank factors).sync_trainable: bool = True@@ -2328,20 +2330,21 @@ def _preloaded(t):def _refuse_weight_dependent_lora_init(model):- """PiSSA / OLoRA / CorDA / EVA / LoftQ build the factors (and residual) from the weight; on a sharded_loading- slice that is this rank's slice only, so the factors come out shard-sized and differ per rank."""+ """PiSSA / OLoRA / CorDA / EVA / LoftQ / MiCA build the factors (and residual) from the weight; on a sharded_loading+ slice that is this rank's slice only, so the factors come out shard-sized and differ per rank. Orthogonal init+ reads only the full in / out features and the dtype, so like gaussian it gives full-size factors."""configs = {}for level in _tp_chain(model):configs.update(getattr(level, "peft_config", None) or {})bad = {name: cfg.init_lora_weightsfor name, cfg in configs.items()- if getattr(cfg, "init_lora_weights", True) not in (True, False, "gaussian")+ if getattr(cfg, "init_lora_weights", True) not in (True, False, "gaussian", "orthogonal")}if bad:raise ValueError(f"Unsloth: init_lora_weights {bad} depends on the base weight, which sharded_loading holds only as "- "this rank's slice; use init_lora_weights = True / 'gaussian', or load the full model (no "+ "this rank's slice; use init_lora_weights = True / 'gaussian' / 'orthogonal', or load the full model (no ""sharded_loading) before get_peft_model and apply_tensor_parallel.")