Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Stop dividing the GRPO eval loss by the accumulation steps (#11337)
Vineeth Sai Varikuntla06c1bb400cc6
diff --git a/tests/test_grpo_eval_accumulation.py b/tests/test_grpo_eval_accumulation.pynew file mode 100644index 00000000000..e1f465df351--- /dev/null+++ b/tests/test_grpo_eval_accumulation.py@@ -0,0 +1,106 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The GRPO loss must not divide an eval pass by the training accumulation window.++Every GRPO loss type except dapo/cispo/vespo ends with++ loss = loss / current_gradient_accumulation_steps++and TRL sets that divisor to `self.current_gradient_accumulation_steps` in train mode and to+1.0 in eval, because an eval pass accumulates nothing. `Trainer` assigns the attribute inside+the training loop and never clears it, so during an in-training evaluation it still holds the+training window's size: reading it unconditionally makes the reported `eval_loss` smaller than+the train loss by exactly `gradient_accumulation_steps`, and moves it when that setting changes.++The helper is lifted from `unsloth/models/rl_replacements.py` with `ast` so the test tracks the+shipped source without importing unsloth, the same trick as `tests/_grpo_dispatch_source.py`.+"""++from __future__ import annotations++import ast+from pathlib import Path++import pytest+++SOURCE_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"+HELPER_NAME = "_unsloth_grpo_accumulation_steps"+++def _load_helper():+ tree = ast.parse(SOURCE_PATH.read_text(encoding = "utf-8"), filename = str(SOURCE_PATH))+ found = [+ node+ for node in tree.body+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == HELPER_NAME+ ]+ assert len(found) == 1, f"expected one module-level def {HELPER_NAME}, found {len(found)}"+ namespace: dict = {}+ exec(compile(ast.Module(body = found, type_ignores = []), str(SOURCE_PATH), "exec"), namespace)+ return namespace[HELPER_NAME]+++class _Model:+ def __init__(self, training):+ self.training = training+++class _Trainer:+ def __init__(+ self,+ training = None,+ steps = None,+ ):+ if training is not None:+ self.model = _Model(training)+ if steps is not None:+ self.current_gradient_accumulation_steps = steps+++ ("training", "steps", "expected"),+ [+ # Training: the accumulation window is the divisor, as before.+ (True, 4, 4),+ (True, 1, 1),+ # Evaluating mid-run: the stale training window must not reach the loss.+ (False, 4, 1),+ (False, 16, 1),+ # Standalone evaluate(): the attribute never existed (#2464).+ (False, None, 1),+ (True, None, 1),+ # No model to ask: keep reading the attribute rather than guessing.+ (None, 4, 4),+ ],+)+def test_grpo_accumulation_divisor_is_one_outside_training(training, steps, expected):+ assert _load_helper()(_Trainer(training, steps)) == expected+++def test_compute_loss_uses_the_helper():+ """The generated trainer's `compute_loss` must go through the helper, not the raw attribute."""+ tree = ast.parse(SOURCE_PATH.read_text(encoding = "utf-8"), filename = str(SOURCE_PATH))+ outer = next(+ node+ for node in tree.body+ if isinstance(node, ast.FunctionDef) and node.name == "grpo_trainer_compute_loss"+ )+ compute_loss = next(+ node+ for node in ast.walk(outer)+ if isinstance(node, ast.FunctionDef) and node.name == "compute_loss"+ )+ calls = [+ node.func.id+ for node in ast.walk(compute_loss)+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)+ ]+ assert HELPER_NAME in calls+ reads = [+ node+ for node in ast.walk(compute_loss)+ if isinstance(node, ast.Attribute) and node.attr == "current_gradient_accumulation_steps"+ ]+ assert reads == [], "compute_loss reads the attribute directly, bypassing the eval guard"diff --git a/tests/test_grpo_eval_mode_restored_for_loss.py b/tests/test_grpo_eval_mode_restored_for_loss.pynew file mode 100644index 00000000000..97b6c74c272--- /dev/null+++ b/tests/test_grpo_eval_mode_restored_for_loss.py@@ -0,0 +1,152 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The divisor helper is only correct while eval mode survives GRPO generation.++`_unsloth_grpo_accumulation_steps` decides between 1 and the accumulation window by reading+`model.training`, the way TRL does. In unsloth that flag is not left alone across a GRPO step:+`grpo_trainer__generate_and_score_completions` injects an unconditional++ self.model.for_training(use_gradient_checkpointing = _use_gc)++into the generated `_generate_and_score_completions`, so an eval batch is flipped into training+mode partway through, and the only thing that puts it back is the `finally` in+`_wrap_grpo_generate_and_score`, which restores inference mode solely when it entered in it.++`test_grpo_eval_accumulation.py` covers the helper's own truth table. This file covers the+assumption underneath it: that by the time `compute_loss` runs, an eval pass still reports+`model.training == False`, so the helper returns 1 rather than the stale training window. If that+restore ever regresses, the helper silently starts dividing eval_loss again and the truth-table+tests stay green, so the property is asserted here directly.++Both pieces are lifted with `ast` from the shipped source, so the test tracks the real code+without importing unsloth and stays CPU-only.+"""++from __future__ import annotations++import ast+import sys+from pathlib import Path++import pytest+++sys.path.insert(0, str(Path(__file__).resolve().parent))++from _rl_source import load_rl_wrapper # noqa: E402+++RL_REPLACEMENTS_PATH = (+ Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"+)+HELPER_NAME = "_unsloth_grpo_accumulation_steps"+WRAPPER_NAME = "_wrap_grpo_generate_and_score"+++def _load_divisor_helper():+ tree = ast.parse(RL_REPLACEMENTS_PATH.read_text(encoding = "utf-8"))+ found = [+ node+ for node in tree.body+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == HELPER_NAME+ ]+ assert len(found) == 1, f"expected one module-level def {HELPER_NAME}, found {len(found)}"+ namespace: dict = {}+ exec(+ compile(ast.Module(body = found, type_ignores = []), str(RL_REPLACEMENTS_PATH), "exec"),+ namespace,+ )+ return namespace[HELPER_NAME]+++def _load_generate_wrapper():+ return load_rl_wrapper((WRAPPER_NAME,))[WRAPPER_NAME]+++class _Model:+ """Only the surface `_wrap_grpo_generate_and_score` touches."""++ def __init__(self, training):+ self.training = training+ self.calls = []++ def for_training(self, use_gradient_checkpointing = True):+ self.calls.append("for_training")+ self.training = True++ def for_inference(self):+ self.calls.append("for_inference")+ self.training = False+++class _Trainer:+ current_gradient_accumulation_steps = 4++ def __init__(+ self,+ training,+ raises = False,+ ):+ self.model = _Model(training)+ self.raises = raises++ def _generate_and_score_completions(self, *args, **kwargs):+ # Mirrors the injection at rl_replacements.py: unconditional, eval batches included.+ self.model.for_training(use_gradient_checkpointing = True)+ if self.raises:+ raise RuntimeError("generation blew up")+ return "completions"+++def _wrapped_trainer_cls(**kwargs):+ cls = type("Wrapped", (_Trainer,), {})+ _load_generate_wrapper()(cls)+ return cls(**kwargs)+++def test_eval_mode_survives_generation_so_the_divisor_is_one():+ """The property the fix rests on, asserted end to end."""+ trainer = _wrapped_trainer_cls(training = False)+ trainer._generate_and_score_completions()+ assert trainer.model.training is False, "generation left the model in training mode"+ assert _load_divisor_helper()(trainer) == 1+++def test_training_mode_survives_generation_so_the_window_is_kept():+ trainer = _wrapped_trainer_cls(training = True)+ trainer._generate_and_score_completions()+ assert trainer.model.training is True+ assert _load_divisor_helper()(trainer) == 4+++def test_generation_really_does_flip_the_flag_midway():+ """Guard the guard: if the injection ever stops firing, this file proves nothing."""+ trainer = _wrapped_trainer_cls(training = False)+ trainer._generate_and_score_completions()+ assert "for_training" in trainer.model.calls+ assert trainer.model.calls[-1] == "for_inference"+++def test_eval_mode_is_restored_even_when_generation_raises():+ trainer = _wrapped_trainer_cls(training = False, raises = True)+ with pytest.raises(RuntimeError):+ trainer._generate_and_score_completions()+ assert trainer.model.training is False+ assert _load_divisor_helper()(trainer) == 1+++def test_wrapping_twice_does_not_stack():+ cls = type("Twice", (_Trainer,), {})+ wrap = _load_generate_wrapper()+ wrap(cls)+ first = cls._generate_and_score_completions+ wrap(cls)+ assert cls._generate_and_score_completions is first+ assert getattr(first, "_unsloth_restore_training_wrapped", False) is True+++def test_a_trainer_without_the_method_is_left_alone():+ cls = type("Bare", (), {})+ _load_generate_wrapper()(cls)+ assert not hasattr(cls, "_generate_and_score_completions")diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 3893796d581..e3c52618e67 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -920,6 +920,22 @@ def _unsloth_grpo_autocast_kwargs(self, device_type = "cuda"):return {"enabled": enabled, "dtype": dtype}+def _unsloth_grpo_accumulation_steps(trainer):+ """Gradient accumulation divisor for the GRPO loss. 1 whenever the model is not training.++ TRL divides by `current_gradient_accumulation_steps` in train mode and by 1.0 in eval, since+ an eval pass accumulates nothing. Trainer sets that attribute inside the training loop and+ never clears it, so an in-training evaluation still sees the training window's size and+ eval_loss comes back that many times too small. Read off `model.training` like TRL does+ rather than a trainer flag, because that is what the eval loop actually switches.+ The attribute is also missing when evaluate() runs standalone (#2464), hence the default.+ """+ model = getattr(trainer, "model", None)+ if model is not None and not getattr(model, "training", True):+ return 1+ return getattr(trainer, "current_gradient_accumulation_steps", 1)++def _unsloth_grpo_vision_inputs(source):"""unsloth_zoo owns this key tuple; the copy below is the fallback for a zoo predatingGRPO_VISION_KEYS, and test_grpo_vision_kwargs_forwarded.py fails if the two diverge."""@@ -2746,6 +2762,7 @@ RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_returns_hiddRL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_hidden_states_signal))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_get_mm_token_id))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_fix_mm_token_type_ids))+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_accumulation_steps))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_vision_inputs))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_split_vision_by_sample))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_unsplit_vision))@@ -2825,10 +2842,7 @@ def grpo_trainer_compute_loss(function_name, function):num_items_in_batch = inputs.get("num_items_in_batch", None)sampling_per_token_logps = inputs.get("sampling_per_token_logps", None)tool_mask = inputs.get("tool_mask", None)- # Missing when evaluate() runs standalone; eval does not accumulate, so fall back to 1 rather than underreport eval_loss (#2464).- current_gradient_accumulation_steps = getattr(- self, "current_gradient_accumulation_steps", 1- )+ current_gradient_accumulation_steps = _unsloth_grpo_accumulation_steps(self)num_processes = self.accelerator.num_processesinput_ids = torch.cat([prompt_ids, completion_ids], dim = 1)