Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stop dividing the GRPO eval loss by the accumulation steps (#11337)

Vineeth Sai Varikuntla06c1bb400cc6
diff --git a/tests/test_grpo_eval_accumulation.py b/tests/test_grpo_eval_accumulation.py
new file mode 100644
index 00000000000..e1f465df351
--- /dev/null
+++ b/tests/test_grpo_eval_accumulation.py
@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The GRPO loss must not divide an eval pass by the training accumulation window.
+
+Every GRPO loss type except dapo/cispo/vespo ends with
+
+ loss = loss / current_gradient_accumulation_steps
+
+and TRL sets that divisor to `self.current_gradient_accumulation_steps` in train mode and to
+1.0 in eval, because an eval pass accumulates nothing. `Trainer` assigns the attribute inside
+the training loop and never clears it, so during an in-training evaluation it still holds the
+training window's size: reading it unconditionally makes the reported `eval_loss` smaller than
+the train loss by exactly `gradient_accumulation_steps`, and moves it when that setting changes.
+
+The helper is lifted from `unsloth/models/rl_replacements.py` with `ast` so the test tracks the
+shipped source without importing unsloth, the same trick as `tests/_grpo_dispatch_source.py`.
+"""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+import pytest
+
+
+SOURCE_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"
+HELPER_NAME = "_unsloth_grpo_accumulation_steps"
+
+
+def _load_helper():
+ tree = ast.parse(SOURCE_PATH.read_text(encoding = "utf-8"), filename = str(SOURCE_PATH))
+ found = [
+ node
+ for node in tree.body
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == HELPER_NAME
+ ]
+ assert len(found) == 1, f"expected one module-level def {HELPER_NAME}, found {len(found)}"
+ namespace: dict = {}
+ exec(compile(ast.Module(body = found, type_ignores = []), str(SOURCE_PATH), "exec"), namespace)
+ return namespace[HELPER_NAME]
+
+
+class _Model:
+ def __init__(self, training):
+ self.training = training
+
+
+class _Trainer:
+ def __init__(
+ self,
+ training = None,
+ steps = None,
+ ):
+ if training is not None:
+ self.model = _Model(training)
+ if steps is not None:
+ self.current_gradient_accumulation_steps = steps
+
+
+ ("training", "steps", "expected"),
+ [
+ # Training: the accumulation window is the divisor, as before.
+ (True, 4, 4),
+ (True, 1, 1),
+ # Evaluating mid-run: the stale training window must not reach the loss.
+ (False, 4, 1),
+ (False, 16, 1),
+ # Standalone evaluate(): the attribute never existed (#2464).
+ (False, None, 1),
+ (True, None, 1),
+ # No model to ask: keep reading the attribute rather than guessing.
+ (None, 4, 4),
+ ],
+)
+def test_grpo_accumulation_divisor_is_one_outside_training(training, steps, expected):
+ assert _load_helper()(_Trainer(training, steps)) == expected
+
+
+def test_compute_loss_uses_the_helper():
+ """The generated trainer's `compute_loss` must go through the helper, not the raw attribute."""
+ tree = ast.parse(SOURCE_PATH.read_text(encoding = "utf-8"), filename = str(SOURCE_PATH))
+ outer = next(
+ node
+ for node in tree.body
+ if isinstance(node, ast.FunctionDef) and node.name == "grpo_trainer_compute_loss"
+ )
+ compute_loss = next(
+ node
+ for node in ast.walk(outer)
+ if isinstance(node, ast.FunctionDef) and node.name == "compute_loss"
+ )
+ calls = [
+ node.func.id
+ for node in ast.walk(compute_loss)
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
+ ]
+ assert HELPER_NAME in calls
+ reads = [
+ node
+ for node in ast.walk(compute_loss)
+ if isinstance(node, ast.Attribute) and node.attr == "current_gradient_accumulation_steps"
+ ]
+ assert reads == [], "compute_loss reads the attribute directly, bypassing the eval guard"
diff --git a/tests/test_grpo_eval_mode_restored_for_loss.py b/tests/test_grpo_eval_mode_restored_for_loss.py
new file mode 100644
index 00000000000..97b6c74c272
--- /dev/null
+++ b/tests/test_grpo_eval_mode_restored_for_loss.py
@@ -0,0 +1,152 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The divisor helper is only correct while eval mode survives GRPO generation.
+
+`_unsloth_grpo_accumulation_steps` decides between 1 and the accumulation window by reading
+`model.training`, the way TRL does. In unsloth that flag is not left alone across a GRPO step:
+`grpo_trainer__generate_and_score_completions` injects an unconditional
+
+ self.model.for_training(use_gradient_checkpointing = _use_gc)
+
+into the generated `_generate_and_score_completions`, so an eval batch is flipped into training
+mode partway through, and the only thing that puts it back is the `finally` in
+`_wrap_grpo_generate_and_score`, which restores inference mode solely when it entered in it.
+
+`test_grpo_eval_accumulation.py` covers the helper's own truth table. This file covers the
+assumption underneath it: that by the time `compute_loss` runs, an eval pass still reports
+`model.training == False`, so the helper returns 1 rather than the stale training window. If that
+restore ever regresses, the helper silently starts dividing eval_loss again and the truth-table
+tests stay green, so the property is asserted here directly.
+
+Both pieces are lifted with `ast` from the shipped source, so the test tracks the real code
+without importing unsloth and stays CPU-only.
+"""
+
+from __future__ import annotations
+
+import ast
+import sys
+from pathlib import Path
+
+import pytest
+
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from _rl_source import load_rl_wrapper # noqa: E402
+
+
+RL_REPLACEMENTS_PATH = (
+ Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"
+)
+HELPER_NAME = "_unsloth_grpo_accumulation_steps"
+WRAPPER_NAME = "_wrap_grpo_generate_and_score"
+
+
+def _load_divisor_helper():
+ tree = ast.parse(RL_REPLACEMENTS_PATH.read_text(encoding = "utf-8"))
+ found = [
+ node
+ for node in tree.body
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == HELPER_NAME
+ ]
+ assert len(found) == 1, f"expected one module-level def {HELPER_NAME}, found {len(found)}"
+ namespace: dict = {}
+ exec(
+ compile(ast.Module(body = found, type_ignores = []), str(RL_REPLACEMENTS_PATH), "exec"),
+ namespace,
+ )
+ return namespace[HELPER_NAME]
+
+
+def _load_generate_wrapper():
+ return load_rl_wrapper((WRAPPER_NAME,))[WRAPPER_NAME]
+
+
+class _Model:
+ """Only the surface `_wrap_grpo_generate_and_score` touches."""
+
+ def __init__(self, training):
+ self.training = training
+ self.calls = []
+
+ def for_training(self, use_gradient_checkpointing = True):
+ self.calls.append("for_training")
+ self.training = True
+
+ def for_inference(self):
+ self.calls.append("for_inference")
+ self.training = False
+
+
+class _Trainer:
+ current_gradient_accumulation_steps = 4
+
+ def __init__(
+ self,
+ training,
+ raises = False,
+ ):
+ self.model = _Model(training)
+ self.raises = raises
+
+ def _generate_and_score_completions(self, *args, **kwargs):
+ # Mirrors the injection at rl_replacements.py: unconditional, eval batches included.
+ self.model.for_training(use_gradient_checkpointing = True)
+ if self.raises:
+ raise RuntimeError("generation blew up")
+ return "completions"
+
+
+def _wrapped_trainer_cls(**kwargs):
+ cls = type("Wrapped", (_Trainer,), {})
+ _load_generate_wrapper()(cls)
+ return cls(**kwargs)
+
+
+def test_eval_mode_survives_generation_so_the_divisor_is_one():
+ """The property the fix rests on, asserted end to end."""
+ trainer = _wrapped_trainer_cls(training = False)
+ trainer._generate_and_score_completions()
+ assert trainer.model.training is False, "generation left the model in training mode"
+ assert _load_divisor_helper()(trainer) == 1
+
+
+def test_training_mode_survives_generation_so_the_window_is_kept():
+ trainer = _wrapped_trainer_cls(training = True)
+ trainer._generate_and_score_completions()
+ assert trainer.model.training is True
+ assert _load_divisor_helper()(trainer) == 4
+
+
+def test_generation_really_does_flip_the_flag_midway():
+ """Guard the guard: if the injection ever stops firing, this file proves nothing."""
+ trainer = _wrapped_trainer_cls(training = False)
+ trainer._generate_and_score_completions()
+ assert "for_training" in trainer.model.calls
+ assert trainer.model.calls[-1] == "for_inference"
+
+
+def test_eval_mode_is_restored_even_when_generation_raises():
+ trainer = _wrapped_trainer_cls(training = False, raises = True)
+ with pytest.raises(RuntimeError):
+ trainer._generate_and_score_completions()
+ assert trainer.model.training is False
+ assert _load_divisor_helper()(trainer) == 1
+
+
+def test_wrapping_twice_does_not_stack():
+ cls = type("Twice", (_Trainer,), {})
+ wrap = _load_generate_wrapper()
+ wrap(cls)
+ first = cls._generate_and_score_completions
+ wrap(cls)
+ assert cls._generate_and_score_completions is first
+ assert getattr(first, "_unsloth_restore_training_wrapped", False) is True
+
+
+def test_a_trainer_without_the_method_is_left_alone():
+ cls = type("Bare", (), {})
+ _load_generate_wrapper()(cls)
+ assert not hasattr(cls, "_generate_and_score_completions")
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 3893796d581..e3c52618e67 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -920,6 +920,22 @@ def _unsloth_grpo_autocast_kwargs(self, device_type = "cuda"):
return {"enabled": enabled, "dtype": dtype}
+def _unsloth_grpo_accumulation_steps(trainer):
+ """Gradient accumulation divisor for the GRPO loss. 1 whenever the model is not training.
+
+ TRL divides by `current_gradient_accumulation_steps` in train mode and by 1.0 in eval, since
+ an eval pass accumulates nothing. Trainer sets that attribute inside the training loop and
+ never clears it, so an in-training evaluation still sees the training window's size and
+ eval_loss comes back that many times too small. Read off `model.training` like TRL does
+ rather than a trainer flag, because that is what the eval loop actually switches.
+ The attribute is also missing when evaluate() runs standalone (#2464), hence the default.
+ """
+ model = getattr(trainer, "model", None)
+ if model is not None and not getattr(model, "training", True):
+ return 1
+ return getattr(trainer, "current_gradient_accumulation_steps", 1)
+
+
def _unsloth_grpo_vision_inputs(source):
"""unsloth_zoo owns this key tuple; the copy below is the fallback for a zoo predating
GRPO_VISION_KEYS, and test_grpo_vision_kwargs_forwarded.py fails if the two diverge."""
@@ -2746,6 +2762,7 @@ RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_returns_hidd
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_hidden_states_signal))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_get_mm_token_id))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_fix_mm_token_type_ids))
+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_accumulation_steps))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_vision_inputs))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_split_vision_by_sample))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_unsplit_vision))
@@ -2825,10 +2842,7 @@ def grpo_trainer_compute_loss(function_name, function):
num_items_in_batch = inputs.get("num_items_in_batch", None)
sampling_per_token_logps = inputs.get("sampling_per_token_logps", None)
tool_mask = inputs.get("tool_mask", None)
- # Missing when evaluate() runs standalone; eval does not accumulate, so fall back to 1 rather than underreport eval_loss (#2464).
- current_gradient_accumulation_steps = getattr(
- self, "current_gradient_accumulation_steps", 1
- )
+ current_gradient_accumulation_steps = _unsloth_grpo_accumulation_steps(self)
num_processes = self.accelerator.num_processes
input_ids = torch.cat([prompt_ids, completion_ids], dim = 1)