Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Pipeline parallel: weight the MoE router aux loss per accumulation step as the model forward does (#16708)
Daniel Han590119de67cd
diff --git a/tests/test_parallel_cp_compose_gloo.py b/tests/test_parallel_cp_compose_gloo.pyindex 73ead931c99..da00662a39b 100644--- a/tests/test_parallel_cp_compose_gloo.py+++ b/tests/test_parallel_cp_compose_gloo.py@@ -180,6 +180,8 @@ def _case_train(ungate = False,mode = "lora",trl_aux = False,+ aux_coef = None,+ zoo_compile = False,device = "cpu",bypass_pre = False,bypass_post = False,@@ -269,6 +271,10 @@ def _case_train()return get_peft_model(model, config)+ if zoo_compile:+ # FastModel's path: Zoo's compiled CausalLM forward delegates to a module-level rewrite.+ from unsloth_zoo.compiler import unsloth_compile_transformers+ unsloth_compile_transformers(model_type = arch)load = tp_env.tensor_parallel_env_load(load)peft = tp_env.tensor_parallel_env_peft(peft)model = load(ckpt, load_in_4bit = False)@@ -280,6 +286,14 @@ def _case_train(moe_calls = _count_moe_backend_calls() if count else Noneif router_aux:model.config.output_router_logits = True+ if aux_coef is not None:+ # TRL < 1.6 has no SFTConfig.router_aux_loss_coef: set the model's own+ for m in [model, *model.modules()]:+ config = getattr(m, "config", None)+ if config is not None and hasattr(config, "router_aux_loss_coef"):+ config.router_aux_loss_coef = aux_coef+ if "router_aux_loss_coef" in vars(m):+ m.router_aux_loss_coef = aux_coefcompiled = _block_compile_on_cpu(model) if compile else []# Data parallel replicas read different rows; cp / tp / pp peers share theirs.@@ -351,7 +365,7 @@ def _case_train(if hasattr(args, "router_aux_loss_coef"):# TRL turns the MoE load balancing loss on from the architecture's coefficient unless this is 0.0# (it then sets output_router_logits on the config itself): aux off unless the case asks for it.- args.router_aux_loss_coef = None if (router_aux or trl_aux) else 0.0+ args.router_aux_loss_coef = aux_coef if (router_aux or trl_aux) else 0.0if sft_cp:args.context_parallel_size = cpargs.context_parallel_backend = "ulysses"@@ -538,6 +552,7 @@ def _ref(def _rows_per_step(res):"""The dataset rows each optimizer step trained on (one rank's view)."""+ assert "error" not in res, ("a worker failed", res.get("error"), res.get("trace"))ga, read = res["ga"], res["rows_read"]return [sorted(r for mb in read[i : i + ga] for r in mb)@@ -545,6 +560,21 @@ def _rows_per_step(res):]+# torch < 2.13 refuses fewer micro-batches than stages in every single-stage schedule (PipelineScheduleSingle);+# 2.13 keeps the check in Schedule1F1B only.+_ONE_MICROBATCH_REFUSED = "pipeline schedule cannot run 1 micro-batches over 2 stages"+++def _skip_if_one_microbatch_refused(results):+ errors = [res.get("error") or "" for res in results]+ if errors and all(_ONE_MICROBATCH_REFUSED in e for e in errors):+ import torch as _torch+ pytest.skip(+ f"torch {_torch.__version__}'s GPipe schedule refuses 1 micro-batch over 2 stages (torch < 2.13), and "+ "the pipeline takes the router aux loss over one micro-batch only: " + errors[0][:200]+ )++def _same_rows(results,want,@@ -552,6 +582,13 @@ def _same_rows():"""Every step of the composed run trains on exactly the rows one process trains on (data parallel replicassplit them, cp / tp / pp peers share theirs), and no side read a row twice (no second epoch)."""+ for rank, res in enumerate([want, *results]):+ assert "error" not in res, (+ what,+ "reference" if rank == 0 else f"rank {rank - 1}",+ res.get("error"),+ res.get("trace"),+ )expected = _rows_per_step(want)flat = [r for step in expected for r in step]assert len(flat) == len(set(flat)), (what, "one process read a row twice", want["rows_read"])@@ -813,6 +850,39 @@ def test_moe_with_router_aux_loss_under_ddp_matches_per_micro_batch(env):_close(res, ref, what = (env, rank))[email protected]("zoo_compile", [False, True], ids = ["forward_install", "zoo_compiled"])[email protected]("aux", [True, False], ids = ["aux_on", "aux_off"])[email protected]("ga", [1, 2, 4])+def test_pp_moe_router_aux_loss_with_accumulation_matches_one_process(+ ga, aux, zoo_compile, tmp_path+):+ """PP=2 Qwen3-MoE GA 1/2/4 matches one process: the aux term is token-share weighted (full = ~ga x larger)."""+ kw = dict(bs = 2, ga = ga, steps = 3, arch = "qwen3_moe", router_aux = aux, aux_coef = 0.5 if aux else None)+ base = {}+ if zoo_compile:+ kw["zoo_compile"] = True+ base = {+ "UNSLOTH_FUSED_FORWARD": "0",+ "UNSLOTH_COMPILE_DISABLE": "1",+ "UNSLOTH_COMPILE_LOCATION": str(tmp_path / "unsloth_compiled_cache"),+ }+ (ref,) = _spawn("train", 1, base, **kw)+ assert ref["output_router_logits"] is aux, ref+ env = {+ "UNSLOTH_PP_SIZE": "2",+ "UNSLOTH_PP_MICROBATCHES": "1",+ "UNSLOTH_PP_SCHEDULE": "gpipe",+ **base,+ }+ results = _spawn("train", 2, env, **kw)+ _skip_if_one_microbatch_refused(results)+ _same_rows(results, ref, what = (ga, aux))+ assert sorted(res.get("stage") for res in results) == [0, 1], results+ for rank, res in enumerate(results):+ # transformers 4.x drifts ~1.4e-4 / ~5e-6 at GA 4 even aux off; the guarded error is 13-42% of the loss+ _close(res, ref, rel = 5e-4, state_atol = 1e-5, what = (ga, aux, rank))++# LoRA on the experts is eager only under cp (the registry's qwen3_moe x lora+experts x cp row): its per instance# ParametrizedQwen3MoeTopKRouter type guards recompile more than one process once the router capture hooks are# installed before the first block compiles.diff --git a/unsloth/models/pp_native.py b/unsloth/models/pp_native.pyindex 4400ef4cde4..19a0ba52a7b 100644--- a/unsloth/models/pp_native.py+++ b/unsloth/models/pp_native.py@@ -635,6 +635,59 @@ def _moe_aux_plan(parts):return _MoEAux(inner, router_class, index, num_experts, routers)+def _scaler_called_by(fn, depth = 1):+ """`unsloth_ga_scale_aux_loss` if `fn`, a function it wraps, or (depth) a module-level *_forward it calls+ (Unsloth Zoo's compiled `<Class>_forward`) calls it, else None."""+ seen = 0+ while fn is not None and seen < 8:+ seen += 1+ fn = getattr(fn, "__func__", fn)+ code = getattr(fn, "__code__", None)+ names = getattr(code, "co_names", ())+ scope = getattr(fn, "__globals__", None) or {}+ if "unsloth_ga_scale_aux_loss" in names:+ scaler = scope.get("unsloth_ga_scale_aux_loss")+ if callable(scaler):+ return scaler+ elif depth > 0:+ for name in names:+ if name.endswith("_forward") and callable(scope.get(name)):+ scaler = _scaler_called_by(scope[name], depth - 1)+ if scaler is not None:+ return scaler+ fn = getattr(fn, "__wrapped__", None)+ return None+++def _forward_aux_scaler(owner):+ """The scaler the installed forward calls, or None; read from the forward, not a version check."""+ scaler = _scaler_called_by(owner.__dict__.get("forward"))+ return (+ scaler if scaler is not None else _scaler_called_by(getattr(type(owner), "forward", None))+ )+++def _forward_aux_weighting(owner, term, kwargs, num_items_in_batch):+ """Weight `term` by this micro-batch's token share of the window, as the rewritten forward does on one GPU."""+ scaler = _forward_aux_scaler(owner)+ labels = kwargs.get("labels")+ if scaler is None or labels is None:+ return term+ loss_kwargs = {+ k: kwargs[k] for k in ("shift_labels", "packed_seq_lengths") if kwargs.get(k) is not None+ }+ if num_items_in_batch is not None:+ loss_kwargs["num_items_in_batch"] = num_items_in_batch+ return scaler(+ term,+ labels,+ kwargs.get("attention_mask"),+ loss_kwargs,+ num_items_in_batch = num_items_in_batch,+ model = owner,+ )++def _record_router(ctx, module, args, output):if ctx.router_logits is not None: # inside this stage's forward (not a checkpoint recompute)index = ctx.moe.index@@ -1167,7 +1220,10 @@ class _PPStageModule(nn.Module):# transformers' CausalLM forward: loss += router_aux_loss_coef * load_balancing_loss.aux = _load_balancing_from_statistics(stats, moe.num_experts)ctx.aux_losses.append(aux.detach())- loss = loss + moe.owner.router_aux_loss_coef * aux.to(loss.device)+ term = moe.owner.router_aux_loss_coef * aux.to(loss.device)+ loss = loss + _forward_aux_weighting(+ moe.owner, term, kwargs, ctx.num_items_in_batch+ )elif carry:# Auxiliary loss off: a zero term keeps the received carry in the graph (see the stage return).loss = loss + 0 * carry[0].sum().to(loss.device)