Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

TP Trainer parity tests: keep eps 1e-4 to the UNSLOTH_TP_SIZE runs, native Trainer runs back on the default (#15397)

Daniel Hanf63ceddc6e66
diff --git a/tests/test_tp_env_gloo.py b/tests/test_tp_env_gloo.py
index a517186ea88..05c0fc03909 100644
--- a/tests/test_tp_env_gloo.py
+++ b/tests/test_tp_env_gloo.py
@@ -30,6 +30,13 @@ from test_tp_native_gloo import ( # noqa: E402
_trainer_args,
)
+# These arms start the adapters at init_lora_weights = False and compare 3 whole Trainer steps. At AdamW's
+# default eps 1e-8, m / (sqrt(v) + eps) turns the fp32 reduction-order noise between tp = 2 and one process
+# into 1e-5 grad-norm and 1e-4 weight drift by step 3 while the losses still match exactly; at 1e-4 it stays
+# below 1e-6, and a real tp error (a missed all-reduce) still lands at 3.5e-4. Only these runs take it: the
+# native Trainer tests (B = 0 adapters) pass at the default eps on the CI runners and fail at 1e-4 there.
+_ADAM_EPS = 1e-4
+
_ENV_KEYS = (
"UNSLOTH_TP_SIZE",
"UNSLOTH_PP_SIZE",
@@ -208,7 +215,9 @@ def _case_env(
def run(out_dir, **overrides):
model, sharded = fresh()
- args = _trainer_args(out_dir, bs, optim = optim, max_steps = steps, **overrides)
+ args = _trainer_args(
+ out_dir, bs, optim = optim, max_steps = steps, adam_epsilon = _ADAM_EPS, **overrides
+ )
trainer = SequentialTrainer(model = model, args = args, train_dataset = data)
return trainer, sharded
diff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.py
index fcd4cea183f..60197b9202c 100644
--- a/tests/test_tp_native_gloo.py
+++ b/tests/test_tp_native_gloo.py
@@ -33,11 +33,6 @@ from test_tp_native_oracle import ( # noqa: E402
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Small next to a typical trained-weight gradient (~6e-6 on the tiny llama lm_head), so Adam still moves it.
_ADAM_EPS = 1e-6
-# The Trainer arms compare whole runs over 3 steps. At the default 1e-8, m / (sqrt(v) + eps) turns the fp32
-# reduction-order noise between tp = 2 and one process (~3e-7) into 1e-5 grad-norm and 1e-4 weight drift
-# by the third step on near-zero LoRA gradients, while the losses still match exactly. At 1e-4 that drift
-# stays below 1e-6, and a real tp error (a missed all-reduce, a wrong shard) is still orders larger.
-_TRAINER_ADAM_EPS = 1e-4
def _free_port():
@@ -643,7 +638,6 @@ def _trainer_args(tmp, bs, **overrides):
max_steps = 3,
learning_rate = 1e-2,
optim = "adamw_torch",
- adam_epsilon = _TRAINER_ADAM_EPS,
lr_scheduler_type = "constant",
warmup_steps = 0,
weight_decay = 0.0,