Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO PrefixGrouper: engage only when the model's forward applies prefix_seg_info (#17058)
Daniel Han7ad3923d7fd0
diff --git a/tests/test_grpo_accumulated_loss_hidden_states_signal.py b/tests/test_grpo_accumulated_loss_hidden_states_signal.pyindex 90f51c631eb..dde56670e31 100644--- a/tests/test_grpo_accumulated_loss_hidden_states_signal.py+++ b/tests/test_grpo_accumulated_loss_hidden_states_signal.py@@ -228,7 +228,7 @@ def test_generated_trainer_embeds_the_patched_gradient_function():and ast.unparse(node.func.value) == "RL_PRE_ITEMS['grpo_trainer']"and node.args]- # The dispatch patch may be nested inside the hooked-head patch.+ # The dispatch patch may be nested inside further source patches (hooked head, PrefixGrouper gate).calls = [innerfor item in appendeddiff --git a/tests/test_grpo_prefix_grouper_capability.py b/tests/test_grpo_prefix_grouper_capability.pynew file mode 100644index 00000000000..3a9dd627c64--- /dev/null+++ b/tests/test_grpo_prefix_grouper_capability.py@@ -0,0 +1,272 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""GRPO PrefixGrouper engages only when the forward that runs applies prefix_seg_info.++A transformers / generic / compiled forward takes prefix_seg_info through **kwargs and drops it, so+completion 2 of a group attends to completion 1. On a tiny Qwen3 that is a 0.036-nat old / ref+log-prob error, inside the 0.7-nat self-verify, giving step-1 KL 5e-5 instead of 1e-11.+"""++import ast+import functools+import importlib.util+import inspect+import re+import sys+import types+from pathlib import Path++import pytest+import torch++ROOT = Path(__file__).resolve().parents[1]+UTILS = ROOT / "unsloth" / "utils"+MODELS = ROOT / "unsloth" / "models"+RL_REPLACEMENTS = MODELS / "rl_replacements.py"+MARK = "_unsloth_applies_prefix_seg_info"+FAST_ATTENTION = {+ "llama": "LlamaAttention_fast_forward",+ "qwen3": "Qwen3Attention_fast_forward",+ "mistral": "MistralAttention_fast_forward",+ "gemma2": "Gemma2Attention_fast_forward",+ "cohere": "CohereAttention_fast_forward",+ "granite": "GraniteAttention_fast_forward",+}+++def _load_prefix_grouper():+ pkg_name = "_pg_capability_test_pkg"+ if pkg_name not in sys.modules:+ pkg = types.ModuleType(pkg_name)+ pkg.__path__ = [str(UTILS)]+ sys.modules[pkg_name] = pkg+ for name in ("prefix_grouper_kernel", "prefix_grouper"):+ full = f"{pkg_name}.{name}"+ if full not in sys.modules:+ spec = importlib.util.spec_from_file_location(full, UTILS / f"{name}.py")+ module = importlib.util.module_from_spec(spec)+ sys.modules[full] = module+ spec.loader.exec_module(module)+ return sys.modules[f"{pkg_name}.prefix_grouper"]+++pg = _load_prefix_grouper()+++def _top_level(path, names):+ tree = ast.parse(path.read_text(encoding = "utf-8"), filename = str(path))+ body = [+ node+ for node in tree.body+ if (isinstance(node, ast.FunctionDef) and node.name in names)+ or (+ isinstance(node, ast.Assign)+ and any(isinstance(t, ast.Name) and t.id in names for t in node.targets)+ )+ ]+ namespace = {"re": re, "inspect": inspect, "functools": functools}+ exec(compile(ast.Module(body = body, type_ignores = []), str(path), "exec"), namespace)+ return namespace+++def _model(+ attention_forwards,+ layers = None,+ config_layers = "auto",+):+ """A decoder whose layer i runs attention_forwards[i] as its attention class forward."""++ class Config:+ pass++ config = Config()+ if config_layers == "auto":+ config.num_hidden_layers = len(attention_forwards) if layers is None else layers+ elif config_layers is not None:+ config.num_hidden_layers = config_layers++ class Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.config = config+ self.layers = torch.nn.ModuleList()+ for forward in attention_forwards:+ attn_cls = type("Attention", (torch.nn.Module,), {"forward": forward})+ layer = torch.nn.Module()+ layer.self_attn = attn_cls()+ self.layers.append(layer)++ return Model()+++def _plain_forward(self, hidden_states, **kwargs):+ return hidden_states+++def _marked_forward(self, hidden_states, **kwargs):+ return hidden_states+++setattr(_marked_forward, MARK, True)+++def test_unmarked_forwards_do_not_apply_prefix_seg_info():+ assert pg.prefix_seg_info_applied(_model([_plain_forward, _plain_forward])) is False+++def test_every_layer_marked_applies_prefix_seg_info():+ assert pg.prefix_seg_info_applied(_model([_marked_forward, _marked_forward])) is True+++def test_one_unmarked_layer_is_enough_to_refuse():+ assert pg.prefix_seg_info_applied(_model([_marked_forward, _plain_forward])) is False+++def test_unknown_layer_count_refuses():+ assert pg.prefix_seg_info_applied(_model([_marked_forward], config_layers = None)) is False+++def test_model_declaration_overrides_detection():+ leak = _model([_plain_forward])+ leak._unsloth_applies_prefix_seg_info = True+ assert pg.prefix_seg_info_applied(leak) is True+ capable = _model([_marked_forward])+ capable._unsloth_applies_prefix_seg_info = False+ assert pg.prefix_seg_info_applied(capable) is False+++def test_pre_patch_gate_applies_only_on_marked_modules():+ """pre_patch's gate copies the fast forward's attributes (functools.wraps), so the class method+ carries the mark either way; only modules marked by mark_unsloth_fast_path run it."""+ llama = _top_level(MODELS / "llama.py", {"_gated_method", "running_forward", "_MISSING"})+ gated = llama["_gated_method"](torch.nn.Module, "forward", _marked_forward, _plain_forward)+ assert getattr(gated, MARK, False) is True+ unmarked = _model([gated, gated])+ assert pg.prefix_seg_info_applied(unmarked) is False+ marked = _model([gated, gated])+ for module in marked.modules():+ module._unsloth_fast_path = True+ assert pg.prefix_seg_info_applied(marked) is True+ for module in marked.modules():+ attn_forward = pg._running_forward(module)+ assert attn_forward is llama["running_forward"](module)+++def test_prefix_grouper_enabled_checks_the_model(monkeypatch):+ monkeypatch.setattr(pg, "_ENABLED", True)+ monkeypatch.setattr(pg, "compile_check_failed", lambda: None)+ assert pg.prefix_grouper_enabled() is True # no model: the previous behaviour+ assert pg.prefix_grouper_enabled(_model([_marked_forward, _marked_forward])) is True+ assert pg.prefix_grouper_enabled(_model([_plain_forward, _plain_forward])) is False+ monkeypatch.setattr(pg, "_ENABLED", False)+ assert pg.prefix_grouper_enabled(_model([_marked_forward])) is False+++def test_self_verify_tolerances_unchanged():+ source = (UTILS / "prefix_grouper.py").read_text(encoding = "utf-8")+ assert 'os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER_TOL", "0.7")' in source+ assert pg.TOL_KILL == 1.5++[email protected]("module, function", sorted(FAST_ATTENTION.items()))+def test_fast_attention_forwards_are_marked(module, function):+ path = MODELS / f"{module}.py"+ tree = ast.parse(path.read_text(encoding = "utf-8"), filename = str(path))+ node = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == function)+ decorators = [d.id for d in node.decorator_list if isinstance(d, ast.Name)]+ assert "applies_prefix_seg_info" in decorators+ calls = {+ n.func.id+ for n in ast.walk(node)+ if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)+ }+ assert "resolve_prefix_seg_info" in calls+++def test_marker_names_agree():+ dispatch = _top_level(+ UTILS / "attention_dispatch.py", {"APPLIES_PREFIX_SEG_INFO", "applies_prefix_seg_info"}+ )+ assert dispatch["APPLIES_PREFIX_SEG_INFO"] == pg.APPLIES_PREFIX_SEG_INFO == MARK++ def forward(self):+ return None++ assert dispatch["applies_prefix_seg_info"](forward) is forward+ assert getattr(forward, MARK) is True+++def test_nograd_gate_passes_the_model():+ text = RL_REPLACEMENTS.read_text(encoding = "utf-8")+ node = next(+ n+ for n in ast.parse(text).body+ if isinstance(n, ast.FunctionDef)+ and n.name == "grpo_trainer__get_per_token_logps_and_entropies"+ )+ source = ast.get_source_segment(text, node)+ assert "_pg_enabled_fn(unwrapped_model)" in source+ assert re.search(r"\b_pg_enabled_fn\(\s*\)", source) is None+++_PATCH = _top_level(+ RL_REPLACEMENTS,+ {"_GRPO_PREFIX_GROUPER_GATE", "_patch_grpo_accumulated_loss_prefix_grouper_gate"},+)+patch_gate = _PATCH["_patch_grpo_accumulated_loss_prefix_grouper_gate"]++_ZOO_LIKE = """+def grpo_accumulated_loss(trainer, unwrapped_model, _pg_funcs):+ _pg_build_layout, _pg_enabled_fn = _pg_funcs+ _pg_engage = (+ _pg_enabled_fn()+ and True+ )+ if _pg_engage:+ return unwrapped_model(prefix_seg_info = _pg_build_layout())+ return None+"""+++def test_zoo_gate_patch_passes_the_model():+ patched = patch_gate(_ZOO_LIKE)+ assert patched.count("_pg_enabled_fn(unwrapped_model)") == 1+ assert "_pg_enabled_fn()" not in patched+ assert patch_gate(patched) == patched # already patched: unchanged+ namespace = {}+ exec(patched, namespace)+ seen = []++ def enabled(model = None):+ seen.append(model)+ return False++ model = object()+ assert namespace["grpo_accumulated_loss"](None, model, (lambda: None, enabled)) is None+ assert seen == [model]+++def test_zoo_gate_patch_leaves_zoo_without_prefix_grouper():+ source = "def grpo_accumulated_loss(unwrapped_model):\n return unwrapped_model()\n"+ assert patch_gate(source) == source+++def test_zoo_gate_patch_refuses_an_unrecognised_gate():+ source = _ZOO_LIKE.replace("_pg_enabled_fn()", "_pg_on")+ with pytest.raises(RuntimeError, match = "PrefixGrouper gate"):+ patch_gate(source)+++def test_installed_zoo_gate_is_patched():+ zoo = pytest.importorskip("unsloth_zoo.rl_replacements")+ function = getattr(zoo, "grpo_accumulated_loss", None)+ if function is None:+ function = zoo.RL_REPLACEMENTS.get("grpo_accumulated_loss")+ source = inspect.getsource(function)+ if "prefix_seg_info" not in source:+ pytest.skip("this unsloth_zoo has no gradient PrefixGrouper")+ patched = patch_gate(source)+ assert "_pg_enabled_fn(unwrapped_model)" in patched+ assert re.search(r"\b_pg_enabled_fn\(\s*\)", patched) is None+ compile(patched, "<patched grpo_accumulated_loss>", "exec")diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.pyindex eb76bab868a..f5a6ea0a1c4 100644--- a/unsloth/models/cohere.py+++ b/unsloth/models/cohere.py@@ -31,6 +31,7 @@ from ..utils.attention_dispatch import (run_attention,select_attention_backend,resolve_prefix_seg_info,+ applies_prefix_seg_info,)try:@@ -84,6 +85,7 @@ def fast_layernorm_inference(# QK norm in Cohere.+@applies_prefix_seg_info@context_parallel_attentiondef CohereAttention_fast_forward(self,diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex 3b435404094..39d9a86aec8 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -31,6 +31,7 @@ from ..utils.attention_dispatch import (run_attention,select_attention_backend,resolve_prefix_seg_info,+ applies_prefix_seg_info,SDPA,)from .gemma import (@@ -134,6 +135,7 @@ def _flash_decode_usable(config, hidden_states, attention_mask):# Logit softcapping.+@applies_prefix_seg_infodef Gemma2Attention_fast_forward(self,hidden_states: torch.Tensor,diff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex 57ee0af0efe..5755b841b50 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -25,6 +25,7 @@ from ..utils.attention_dispatch import (run_attention,select_attention_backend,resolve_prefix_seg_info,+ applies_prefix_seg_info,SDPA,)from .llama import (@@ -83,6 +84,7 @@ except:GraniteFlashAttention2 = GraniteAttention+@applies_prefix_seg_info@context_parallel_attentiondef GraniteAttention_fast_forward(self,diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 51071714d89..c943a8d79ca 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -104,6 +104,7 @@ from ..utils.attention_dispatch import (SDPA,select_attention_backend,resolve_prefix_seg_info,+ applies_prefix_seg_info,)from torch.nn.functional import scaled_dot_product_attention as _torch_sdpafrom transformers import __version__ as transformers_version@@ -1095,6 +1096,7 @@ def fast_layernorm_compiled(layernorm, X):# Ported from transformers models/llama/modeling_llama.py#L320+@applies_prefix_seg_info@context_parallel_attention(rl = True)def LlamaAttention_fast_forward(self,diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.pyindex 0df2dd10bd0..01e23cb4595 100644--- a/unsloth/models/mistral.py+++ b/unsloth/models/mistral.py@@ -29,6 +29,7 @@ from ..utils.attention_dispatch import (SDPA,select_attention_backend,resolve_prefix_seg_info,+ applies_prefix_seg_info,)from .llama import (LlamaRotaryEmbedding,@@ -64,6 +65,7 @@ except:from unsloth_zoo.utils import Version+@applies_prefix_seg_info@context_parallel_attentiondef MistralAttention_fast_forward(self,diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex c9d263f3fed..0c59ca80c6d 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -28,6 +28,7 @@ from ..utils.attention_dispatch import (SDPA,select_attention_backend,resolve_prefix_seg_info,+ applies_prefix_seg_info,)from .llama import (LlamaRotaryEmbedding,@@ -70,6 +71,7 @@ except:Qwen3FlashAttention2 = Qwen3Attention+@applies_prefix_seg_info@context_parallel_attention(rl = True)def Qwen3Attention_fast_forward(self,diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 8b05ba2c85a..485e35c0875 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -4011,7 +4011,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):# Skip softcap models (the flex kernel never applies attn_logit_softcapping) and hybrid SSM / MoE models: only the threaded attention forwards get shared-prefix isolation, so a decoder that does not forward prefix_seg_info leaks suffixes across completions. PG also rides on sequence packing, so it needs the same zoo masked-column guard._pg_cfg = getattr(unwrapped_model, "config", None)_pg_engage = (- _pg_enabled_fn()+ # Given the model, also requires prefix_seg_info_applied(model).+ _pg_enabled_fn(unwrapped_model)and UNSLOTH_ZOO_HAS_MASKED_COL_GUARDand pixel_values is Noneand token_type_ids is None@@ -4799,6 +4800,22 @@ def _patch_grpo_accumulated_loss_hidden_states_dispatch(function):return source+_GRPO_PREFIX_GROUPER_GATE = re.compile(r"\b_pg_enabled_fn\(\s*\)")+++def _patch_grpo_accumulated_loss_prefix_grouper_gate(source):+ """Pass the model to zoo's ``_pg_enabled_fn()`` so the gradient PrefixGrouper checks ``prefix_seg_info_applied``: a forward ignoring ``prefix_seg_info`` leaks completions across a group yet passes the 0.7-nat self-verify. Fails at generation if zoo has the feature but not this gate."""+ if "prefix_seg_info" not in source:+ return source+ source, replacements = _GRPO_PREFIX_GROUPER_GATE.subn("_pg_enabled_fn(unwrapped_model)", source)+ if replacements == 0 and "_pg_enabled_fn(unwrapped_model)" not in source:+ raise RuntimeError(+ "Unsloth: could not find the GRPO gradient PrefixGrouper gate in this "+ "unsloth_zoo version. Please upgrade unsloth_zoo."+ )+ return source++_GRPO_HIDDEN_STATES_REQUEST = re.compile(r"""os\.environ\[\s*(["'])UNSLOTH_RETURN_HIDDEN_STATES\1\s*\]\s*=\s*(["'])1\2""")@@ -5157,8 +5174,10 @@ RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_clear_stateful_mrRL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(grpo_compute_loss))RL_PRE_ITEMS["grpo_trainer"].append(_class_source(UnslothEfficientGRPO))RL_PRE_ITEMS["grpo_trainer"].append(- _patch_grpo_accumulated_loss_head_hooks(- _patch_grpo_accumulated_loss_hidden_states_dispatch(grpo_accumulated_loss)+ _patch_grpo_accumulated_loss_prefix_grouper_gate(+ _patch_grpo_accumulated_loss_head_hooks(+ _patch_grpo_accumulated_loss_hidden_states_dispatch(grpo_accumulated_loss)+ )))RL_PRE_ITEMS["grpo_trainer"].append(grpo_compute_loss_slow)diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 8b1150dbe31..8b855ce5759 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -1306,6 +1306,16 @@ def resolve_prefix_seg_info(kwargs, past_key_value, attention_mask):return seg+# Marks forwards that apply prefix_seg_info; a bool on a model overrides detection (prefix_seg_info_applied).+APPLIES_PREFIX_SEG_INFO = "_unsloth_applies_prefix_seg_info"+++def applies_prefix_seg_info(forward):+ """Mark an attention forward that passes ``resolve_prefix_seg_info``'s table to ``run_attention``."""+ setattr(forward, APPLIES_PREFIX_SEG_INFO, True)+ return forward++# One dense window mask per (device, shape, window) shared by all layers: at 32K it is 1 GiB._WINDOW_MASK_CACHE: dict = {}@@ -1810,6 +1820,8 @@ __all__ = ["AttentionContext","select_attention_backend","resolve_prefix_seg_info",+ "APPLIES_PREFIX_SEG_INFO",+ "applies_prefix_seg_info","run_attention","context_parallel_attention","is_context_parallel_attention",diff --git a/unsloth/utils/prefix_grouper.py b/unsloth/utils/prefix_grouper.pyindex 3829337b270..c7d8a203abd 100644--- a/unsloth/utils/prefix_grouper.py+++ b/unsloth/utils/prefix_grouper.py@@ -51,8 +51,9 @@ _TOKR_THRESHOLD = float(os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER_TOKR", "1.3"_TOL_OK = float(os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER_TOL", "0.7"))-def prefix_grouper_enabled() -> bool:- """Needs seq-packing, torch >= 2.7 and a flex kernel that passed its compile check."""+def prefix_grouper_enabled(model = None) -> bool:+ """Needs seq-packing, torch >= 2.7 and a flex kernel that passed its compile check; given the+ model that will run the shared-prefix forward, also that its forward applies prefix_seg_info."""global _OFF_LOGGEDif _ENV_ENABLED and not _ENABLED and not _OFF_LOGGED:_OFF_LOGGED = True@@ -62,7 +63,76 @@ def prefix_grouper_enabled() -> bool:"does not compile there); using the packed path.",flush = True,)- return _ENABLED and compile_check_failed() is None+ if not (_ENABLED and compile_check_failed() is None):+ return False+ return model is None or prefix_seg_info_applied(model)+++# attention_dispatch.APPLIES_PREFIX_SEG_INFO; spelled out so this module loads without attention_dispatch.+APPLIES_PREFIX_SEG_INFO = "_unsloth_applies_prefix_seg_info"+_APPLIED_CACHE = "_unsloth_prefix_seg_info_applied"+++def _running_forward(module):+ """llama.running_forward without importing the model code: the class forward, and for a method+ pre_patch gated, the wrapped fast forward only on modules marked by mark_unsloth_fast_path."""+ method = getattr(type(module), "forward", None)+ method = getattr(method, "__func__", method)+ if getattr(method, "_unsloth_gated", False):+ if not getattr(module, "_unsloth_fast_path", False):+ return None+ return getattr(method, "__wrapped__", None)+ return method+++def _decoder_layer_count(model):+ config = getattr(model, "config", None)+ get_text_config = getattr(config, "get_text_config", None)+ if callable(get_text_config):+ try:+ config = get_text_config()+ except Exception:+ pass+ layers = getattr(config, "num_hidden_layers", None)+ return layers if isinstance(layers, int) and not isinstance(layers, bool) else None+++def prefix_seg_info_applied(model) -> bool:+ """True only when the forward that runs on ``model`` applies ``prefix_seg_info``.++ Other forwards drop it via ``**kwargs`` and leak completions across a group (0.02-0.04 nats, under+ the 0.7-nat self-verify), so every decoder layer must run a forward marked by ``applies_prefix_seg_info``.+ """+ if model is None:+ return False+ declared = getattr(model, APPLIES_PREFIX_SEG_INFO, None)+ if isinstance(declared, bool):+ return declared+ cached = getattr(model, _APPLIED_CACHE, None)+ if isinstance(cached, bool):+ return cached+ applied = False+ try:+ layers = _decoder_layer_count(model)+ if layers is not None and layers > 0:+ marked = sum(+ bool(getattr(_running_forward(module), APPLIES_PREFIX_SEG_INFO, False))+ for module in model.modules()+ )+ applied = marked >= layers+ except Exception:+ applied = False+ try:+ setattr(model, _APPLIED_CACHE, applied)+ except Exception:+ pass+ if not applied and os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") in ("1", "True", "true"):+ print(+ f"Unsloth: GRPO PrefixGrouper is off for {type(model).__name__}: its forward does not "+ "apply prefix_seg_info; using the packed path.",+ flush = True,+ )+ return applieddef verify_on() -> bool:@@ -502,6 +572,7 @@ __all__ = ["GroupLayout","build_group_layout","prefix_grouper_enabled",+ "prefix_seg_info_applied","verify_on","tokr_threshold","tol_ok",