Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
TP Trainer parity tests: keep eps 1e-4 to the UNSLOTH_TP_SIZE runs, native Trainer runs back on the default (#15397)
Daniel Hanf63ceddc6e66
diff --git a/tests/test_tp_env_gloo.py b/tests/test_tp_env_gloo.pyindex a517186ea88..05c0fc03909 100644--- a/tests/test_tp_env_gloo.py+++ b/tests/test_tp_env_gloo.py@@ -30,6 +30,13 @@ from test_tp_native_gloo import ( # noqa: E402_trainer_args,)+# These arms start the adapters at init_lora_weights = False and compare 3 whole Trainer steps. At AdamW's+# default eps 1e-8, m / (sqrt(v) + eps) turns the fp32 reduction-order noise between tp = 2 and one process+# into 1e-5 grad-norm and 1e-4 weight drift by step 3 while the losses still match exactly; at 1e-4 it stays+# below 1e-6, and a real tp error (a missed all-reduce) still lands at 3.5e-4. Only these runs take it: the+# native Trainer tests (B = 0 adapters) pass at the default eps on the CI runners and fail at 1e-4 there.+_ADAM_EPS = 1e-4+_ENV_KEYS = ("UNSLOTH_TP_SIZE","UNSLOTH_PP_SIZE",@@ -208,7 +215,9 @@ def _case_env(def run(out_dir, **overrides):model, sharded = fresh()- args = _trainer_args(out_dir, bs, optim = optim, max_steps = steps, **overrides)+ args = _trainer_args(+ out_dir, bs, optim = optim, max_steps = steps, adam_epsilon = _ADAM_EPS, **overrides+ )trainer = SequentialTrainer(model = model, args = args, train_dataset = data)return trainer, shardeddiff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.pyindex fcd4cea183f..60197b9202c 100644--- a/tests/test_tp_native_gloo.py+++ b/tests/test_tp_native_gloo.py@@ -33,11 +33,6 @@ from test_tp_native_oracle import ( # noqa: E402REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))# Small next to a typical trained-weight gradient (~6e-6 on the tiny llama lm_head), so Adam still moves it._ADAM_EPS = 1e-6-# The Trainer arms compare whole runs over 3 steps. At the default 1e-8, m / (sqrt(v) + eps) turns the fp32-# reduction-order noise between tp = 2 and one process (~3e-7) into 1e-5 grad-norm and 1e-4 weight drift-# by the third step on near-zero LoRA gradients, while the losses still match exactly. At 1e-4 that drift-# stays below 1e-6, and a real tp error (a missed all-reduce, a wrong shard) is still orders larger.-_TRAINER_ADAM_EPS = 1e-4def _free_port():@@ -643,7 +638,6 @@ def _trainer_args(tmp, bs, **overrides):max_steps = 3,learning_rate = 1e-2,optim = "adamw_torch",- adam_epsilon = _TRAINER_ADAM_EPS,lr_scheduler_type = "constant",warmup_steps = 0,weight_decay = 0.0,