Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
PP gloo parity test: size gemma3's grad bounds to its noise on AVX2 kernels (#15286)
Daniel Han4bdf6c46f513
diff --git a/tests/test_pp_fsdp2_gloo.py b/tests/test_pp_fsdp2_gloo.pyindex 9005f0eafa5..029c823ab49 100644--- a/tests/test_pp_fsdp2_gloo.py+++ b/tests/test_pp_fsdp2_gloo.py@@ -37,10 +37,14 @@ STEPS = 3# Step-0 grad norm tolerance vs the one-process reference. Gemma3's own fp32 grad norm sits 1.0e-4 from fp64 on one# process (Llama 7.8e-7), so any change of summation order (TP splits, CPU ISA) moves it by that much.STEP0_NORM_REL = {"llama": 1e-5, "qwen3": 1e-5, "gemma3": 3e-4}-# Per-element step-0 grad tolerance, relative to the tensor's largest entry. Gemma3's fp32 reordering-# noise is about 1e-5 of the tensor (relative L2) on every LoRA weight, up to 2.1e-5 per element, so-# the llama / qwen3 bound sat on it. The 5e-5 relative L2 bound below still catches systematic errors.-GRAD_ATOL_REL = {"llama": 2e-5, "qwen3": 2e-5, "gemma3": 1e-4}+# Step-0 grad tolerances: per element relative to the tensor's largest entry, and relative L2 over all+# grads. Gemma3's fp32 reordering noise under tensor parallel depends on the CPU's kernels: worst+# element / relative L2 2.8e-5 / 2.4e-5 with this host's AVX-512 MKL, 1.6e-4 / 1.4e-4 with MKL and+# oneDNN held to AVX2, and 2e-4 per element on a CI runner, against 2.6e-6 for llama and 5.8e-6 for+# qwen3 on the same AVX2 run (pp x dp gemma3 stays near 4e-7). llama and qwen3 keep the strict bounds+# on the same tensor parallel code; a gemma3-specific fault (a wrong shard, a missed reduction) is O(1).+GRAD_ATOL_REL = {"llama": 2e-5, "qwen3": 2e-5, "gemma3": 1e-3}+GRAD_L2_REL = {"llama": 5e-5, "qwen3": 5e-5, "gemma3": 5e-4}def _native():@@ -434,6 +438,7 @@ def _check(n_mb = 2,norm_rel = 1e-5,grad_rel = 2e-5,+ l2_rel = 5e-5,):ref = results[0]["ref"]union = set()@@ -466,7 +471,7 @@ def _check()num += float((grad.double() - want.double()).square().sum())den += float(want.double().square().sum())- assert (num / den) ** 0.5 < 5e-5, (rank, (num / den) ** 0.5)+ assert (num / den) ** 0.5 < l2_rel, (rank, (num / den) ** 0.5)union |= set(res["grads"])refusals = res["refusals"]if len(results) != dims["pp"]:@@ -518,6 +523,7 @@ def test_pp2_x_fsdp2_dp2_matches_one_process(arch):dims,norm_rel = STEP0_NORM_REL[arch],grad_rel = GRAD_ATOL_REL[arch],+ l2_rel = GRAD_L2_REL[arch],)@@ -538,6 +544,7 @@ def test_pp2_x_tp2_matches_one_process(arch):dims,norm_rel = STEP0_NORM_REL[arch],grad_rel = GRAD_ATOL_REL[arch],+ l2_rel = GRAD_L2_REL[arch],)