Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

TP Trainer parity tests: run AdamW with eps 1e-4 so reduction-order noise is not amplified (#15381)

Daniel Han7b144172b543
diff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.py
index 60197b9202c..fcd4cea183f 100644
--- a/tests/test_tp_native_gloo.py
+++ b/tests/test_tp_native_gloo.py
@@ -33,6 +33,11 @@ from test_tp_native_oracle import ( # noqa: E402
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Small next to a typical trained-weight gradient (~6e-6 on the tiny llama lm_head), so Adam still moves it.
_ADAM_EPS = 1e-6
+# The Trainer arms compare whole runs over 3 steps. At the default 1e-8, m / (sqrt(v) + eps) turns the fp32
+# reduction-order noise between tp = 2 and one process (~3e-7) into 1e-5 grad-norm and 1e-4 weight drift
+# by the third step on near-zero LoRA gradients, while the losses still match exactly. At 1e-4 that drift
+# stays below 1e-6, and a real tp error (a missed all-reduce, a wrong shard) is still orders larger.
+_TRAINER_ADAM_EPS = 1e-4
def _free_port():
@@ -638,6 +643,7 @@ def _trainer_args(tmp, bs, **overrides):
max_steps = 3,
learning_rate = 1e-2,
optim = "adamw_torch",
+ adam_epsilon = _TRAINER_ADAM_EPS,
lr_scheduler_type = "constant",
warmup_steps = 0,
weight_decay = 0.0,