Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Tests: run the DDP no_sync accumulation steps' forward inside no_sync (#14711)
Daniel Han4647d6108e0f
diff --git a/tests/test_moe_dw_accumulate.py b/tests/test_moe_dw_accumulate.pyindex 6d2a2e2e448..e9096fbe872 100644--- a/tests/test_moe_dw_accumulate.py+++ b/tests/test_moe_dw_accumulate.py@@ -468,13 +468,26 @@ def _ddp_worker(rank, world, port, inplace, bucket_view, out_path):cases = [_Case(E, N, K, T, topk, "permute_x", torch.bfloat16, seed = 100 * rank + i) for i in range(4)]+ allreduces = []++ def counting_allreduce(state, bucket):+ allreduces.append(1)+ return (+ dist.all_reduce(bucket.buffer(), async_op = True)+ .get_future()+ .then(lambda fut: fut.value()[0].div_(world))+ )++ ddp.register_comm_hook(None, counting_allreduce)for i, case in enumerate(cases):- loss = ddp(case.X, case.m_sizes, case.gather).float().pow(2).mean()+ # The forward runs inside no_sync too: DDP decides at forward time whether to reduce.if i < 3:with ddp.no_sync():- loss.backward()+ ddp(case.X, case.m_sizes, case.gather).float().pow(2).mean().backward()else:- loss.backward()+ assert not allreduces, "a no_sync step reduced gradients"+ ddp(case.X, case.m_sizes, case.gather).float().pow(2).mean().backward()+ assert allreducestorch.save({"grad": model.w.grad.float().cpu(), "inplace": gg.DW_ACCUMULATE_STATS["inplace"]},f"{out_path}.{rank}",