Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Tensor parallel sharded loading: allow init_lora_weights = "orthogonal" (#15218)

Shubhankar Tripathyc92009bbafc1
diff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.py
index 2c85c2967ce..428e787e71f 100644
--- a/tests/test_tp_native_gloo.py
+++ b/tests/test_tp_native_gloo.py
@@ -868,14 +868,15 @@ def _case_trainer(
def _case_sharded_lora_init(rank, world, tmp, ckpt):
- """Weight-dependent LoRA init (PiSSA) on sharded_loading slices is refused before any mutation."""
+ """Weight-dependent LoRA init (PiSSA) on sharded_loading slices is refused before any mutation;
+ orthogonal init only reads the full in / out features and the dtype, so it is allowed like gaussian."""
from peft import LoraConfig, get_peft_model
from torch.distributed.device_mesh import init_device_mesh
from transformers import AutoModelForCausalLM
mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
out = {}
- for init in ("pissa", "gaussian"):
+ for init in ("pissa", "gaussian", "orthogonal"):
model = tpn.load_tensor_parallel(
ckpt, mesh["tp"], loader = AutoModelForCausalLM.from_pretrained, torch_dtype = torch.float32
)
@@ -886,6 +887,60 @@ def _case_sharded_lora_init(rank, world, tmp, ckpt):
return out
+def _case_sharded_random_lora_init(rank, world, tmp, ckpt, init, sync_trainable, seed_per_rank):
+ """A random LoRA init on sharded_loading slices vs the same init on a full load in one process. Each rank
+ seeds get_peft_model differently when seed_per_rank, so only the tp rank 0 broadcast can make them agree."""
+ from peft import LoraConfig, get_peft_model
+ from torch.distributed.device_mesh import init_device_mesh
+ from transformers import AutoModelForCausalLM
+
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))
+ kw = dict(torch_dtype = torch.float32, attn_implementation = "eager")
+
+ def peft(model, seed):
+ torch.manual_seed(seed)
+ config = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = init)
+ return get_peft_model(model, config)
+
+ def lora(model):
+ return {
+ n: tpn._full(p.detach(), p).clone() for n, p in model.named_parameters() if "lora_" in n
+ }
+
+ ref = peft(AutoModelForCausalLM.from_pretrained(ckpt, **kw), 7)
+ model = tpn.load_tensor_parallel(
+ ckpt, mesh["tp"], loader = AutoModelForCausalLM.from_pretrained, **kw
+ )
+ model = peft(model, 7 + rank if seed_per_rank else 7)
+ error = _raised(
+ lambda: tpn.apply_tensor_parallel(
+ model, mesh["tp"], config = tpn.TPConfig(sync_trainable = sync_trainable)
+ )
+ )
+ if error is not None:
+ return {"error": error}
+ # Replicated factors hold this rank's own copy: the raw local tensors show any per-rank divergence.
+ local = {
+ n: (p.to_local() if tpn._is_dtensor(p) else p).detach().clone()
+ for n, p in model.named_parameters()
+ if "lora_" in n
+ }
+ init_ref, init_tp = lora(ref), lora(model)
+ ids = _batch(ref)
+ ref_out = _train(ref, ids, 1, torch.nn.utils.clip_grad_norm_)
+ tp_out = _train(model, ids, 1, tpn.tp_clip_grad_norm_)
+ return {
+ "error": None,
+ "local": local,
+ "init_ref": init_ref,
+ "init_tp": init_tp,
+ "ref": ref_out,
+ "tp": tp_out,
+ "step_ref": lora(ref),
+ "step_tp": lora(model),
+ }
+
+
def _case_sampler(rank, world, tmp, dp, strategy):
"""Rows each rank's dataloader yields under a length-aware sampler, after a rank-local RNG draw."""
import transformers
@@ -1044,6 +1099,7 @@ CASES = {
"tp_fsdp2_one_writer": _case_tp_fsdp2_writer,
"tp_fsdp2_writer_pipeline_slice": _case_tp_fsdp2_writer_pipeline_slice,
"trainer": _case_trainer,
+ "sharded_random_lora_init": _case_sharded_random_lora_init,
"sampler": _case_sampler,
"sampler_resume": _case_sampler_resume,
}
@@ -1342,6 +1398,57 @@ def test_sharded_loading_refuses_weight_dependent_lora_init():
assert "depends on the base weight" in (res["pissa"] or ""), res
assert not res["pissa_tp"]
assert res["gaussian"] is None and res["gaussian_tp"], res
+ assert res["orthogonal"] is None and res["orthogonal_tp"], res
+
+
[email protected]("init", ["orthogonal", "gaussian", True])
+def test_sharded_loading_random_lora_init_matches_one_process(init):
+ """Random LoRA inits read only the full in / out features, so the sharded factors are full size; tp rank 0's
+ broadcast makes per-rank seeds agree, and one AdamW step matches the unsharded run from the same seed."""
+ with tempfile.TemporaryDirectory() as ckpt:
+ _save_checkpoint("llama", ckpt, None)
+ results = _spawn(
+ "sharded_random_lora_init",
+ 2,
+ ckpt = ckpt,
+ init = init,
+ sync_trainable = True,
+ seed_per_rank = True,
+ )
+ r0, r1 = results
+ assert r0["error"] is None and r1["error"] is None, (r0["error"], r1["error"])
+ assert set(r0["local"]) == set(r1["local"]) and r0["local"]
+ for res in results:
+ assert set(res["init_tp"]) == set(res["init_ref"])
+ for name, ref in res["init_ref"].items():
+ # Bit-identical to the one-process init from tp rank 0's seed: full-size factors, no shard-sized draw.
+ assert torch.equal(res["init_tp"][name], ref), name
+ if init == "orthogonal":
+ for name, A in res["init_ref"].items():
+ if ".lora_A." not in name:
+ continue
+ B = res["init_ref"][name.replace(".lora_A.", ".lora_B.")]
+ # Orthogonal init: both factors non-zero, B @ A == 0 (rows from complementary halves of one Q).
+ assert A.abs().sum() > 0 and B.abs().sum() > 0, name
+ torch.testing.assert_close(
+ B @ A, torch.zeros_like(B @ A), rtol = 0, atol = 1e-6, msg = name
+ )
+ ref_logits, ref_grads, ref_losses, ref_norms = res["ref"]
+ logits, grads, losses, norms = res["tp"]
+ torch.testing.assert_close(logits, ref_logits, rtol = 1e-5, atol = 1e-5)
+ assert set(grads) == set(ref_grads) and grads
+ for name in ref_grads:
+ torch.testing.assert_close(grads[name], ref_grads[name], rtol = 1e-4, atol = 1e-6, msg = name)
+ assert losses == pytest.approx(ref_losses, rel = 1e-5)
+ assert norms == pytest.approx(ref_norms, rel = 1e-5)
+ for name, ref in res["step_ref"].items():
+ torch.testing.assert_close(res["step_tp"][name], ref, rtol = 1e-4, atol = 1e-6, msg = name)
+ # Every rank's local copy of a replicated factor is the same tensor after the broadcast.
+ for name in r0["local"]:
+ if r0["local"][name].shape == r1["local"][name].shape and name in r0["init_ref"]:
+ full = r0["init_ref"][name]
+ if r0["local"][name].shape == full.shape:
+ assert torch.equal(r0["local"][name], r1["local"][name]), name
def test_sharded_loading_refuses_quantized_loads():
diff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.py
index 1081498f8cd..9f7d7061e0f 100644
--- a/unsloth/models/tp_native.py
+++ b/unsloth/models/tp_native.py
@@ -160,6 +160,8 @@ class TPConfig:
# {"layers.*.self_attn.q_proj": "colwise", ...}; None = the model's TP plan, else structural discovery.
plan: Optional[dict] = None
# Broadcast trainable parameters from tp rank 0 first (every rank must start from the same adapter).
+ # False: the caller guarantees it, e.g. a loaded adapter or the same seed before get_peft_model on every
+ # rank (random LoRA inits draw full-size factors, so per-rank seeds give per-rank factors).
sync_trainable: bool = True
@@ -2328,20 +2330,21 @@ def _preloaded(t):
def _refuse_weight_dependent_lora_init(model):
- """PiSSA / OLoRA / CorDA / EVA / LoftQ build the factors (and residual) from the weight; on a sharded_loading
- slice that is this rank's slice only, so the factors come out shard-sized and differ per rank."""
+ """PiSSA / OLoRA / CorDA / EVA / LoftQ / MiCA build the factors (and residual) from the weight; on a sharded_loading
+ slice that is this rank's slice only, so the factors come out shard-sized and differ per rank. Orthogonal init
+ reads only the full in / out features and the dtype, so like gaussian it gives full-size factors."""
configs = {}
for level in _tp_chain(model):
configs.update(getattr(level, "peft_config", None) or {})
bad = {
name: cfg.init_lora_weights
for name, cfg in configs.items()
- if getattr(cfg, "init_lora_weights", True) not in (True, False, "gaussian")
+ if getattr(cfg, "init_lora_weights", True) not in (True, False, "gaussian", "orthogonal")
}
if bad:
raise ValueError(
f"Unsloth: init_lora_weights {bad} depends on the base weight, which sharded_loading holds only as "
- "this rank's slice; use init_lora_weights = True / 'gaussian', or load the full model (no "
+ "this rank's slice; use init_lora_weights = True / 'gaussian' / 'orthogonal', or load the full model (no "
"sharded_loading) before get_peft_model and apply_tensor_parallel."
)