Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO PrefixGrouper: import cleanly on torch 2.6 and keep it off below 2.7 with a log line (#15836)

Daniel Hanedfcb3c9c283
diff --git a/tests/test_prefix_grouper_dynamo_limits.py b/tests/test_prefix_grouper_dynamo_limits.py
new file mode 100644
index 00000000000..d0e4d8af9f5
--- /dev/null
+++ b/tests/test_prefix_grouper_dynamo_limits.py
@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""PrefixGrouper on torch < 2.7: kernel imports cleanly, stays off with one log, dynamo untouched."""
+
+import importlib.util
+import pathlib
+import sys
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("torch.nn.attention.flex_attention")
+
+UTILS = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "utils"
+KERNEL = UTILS / "prefix_grouper_kernel.py"
+PKG = "_pg_pkg_under_test"
+NEW_NAMES = ("recompile_limit", "accumulated_recompile_limit")
+
+
+class _OldDynamoConfig:
+ """torch 2.6 view of torch._dynamo.config: no recompile_limit names, unknown writes raise."""
+
+ def __init__(self, real):
+ object.__setattr__(self, "_real", real)
+ object.__setattr__(
+ self, "_local", {"cache_size_limit": 8, "accumulated_cache_size_limit": 256}
+ )
+
+ def __getattr__(self, name):
+ if name in NEW_NAMES:
+ raise AttributeError(name)
+ local = object.__getattribute__(self, "_local")
+ if name in local:
+ return local[name]
+ return getattr(object.__getattribute__(self, "_real"), name)
+
+ def __setattr__(self, name, value):
+ local = object.__getattribute__(self, "_local")
+ if name not in local:
+ raise AttributeError(f"torch._dynamo.config.{name} does not exist")
+ local[name] = value
+
+
+def _load_kernel():
+ name = "_pg_kernel_under_test"
+ spec = importlib.util.spec_from_file_location(name, KERNEL)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module # dataclasses resolve their module through sys.modules
+ try:
+ spec.loader.exec_module(module)
+ finally:
+ sys.modules.pop(name, None)
+ return module
+
+
+def test_import_raises_the_old_names_when_only_they_exist(monkeypatch):
+ fake = _OldDynamoConfig(torch._dynamo.config)
+ monkeypatch.setattr(torch._dynamo, "config", fake)
+ monkeypatch.setattr(torch, "__version__", "2.7.0")
+ module = _load_kernel()
+ assert callable(module.flex_shared_prefix_attention)
+ assert fake.cache_size_limit == 256
+ assert fake.accumulated_cache_size_limit == 2048
+
+
+ not hasattr(torch._dynamo.config, "recompile_limit"),
+ reason = "torch < 2.7 has no recompile_limit; the old names are covered by the test above",
+)
+def test_import_raises_the_new_names_when_present(monkeypatch):
+ config = torch._dynamo.config
+ monkeypatch.setattr(config, "recompile_limit", 8)
+ monkeypatch.setattr(config, "accumulated_recompile_limit", 256)
+ _load_kernel()
+ assert config.recompile_limit == 256
+ assert config.accumulated_recompile_limit == 2048
+ monkeypatch.setattr(config, "recompile_limit", 4096)
+ _load_kernel()
+ assert config.recompile_limit == 4096
+
+
+def _load_prefix_grouper():
+ """Load prefix_grouper.py + its kernel under a throwaway package (no full unsloth import)."""
+ pkg = types.ModuleType(PKG)
+ pkg.__path__ = [str(UTILS)]
+ names = (PKG, f"{PKG}.prefix_grouper_kernel", f"{PKG}.prefix_grouper")
+ sys.modules[PKG] = pkg
+ try:
+ name = f"{PKG}.prefix_grouper"
+ spec = importlib.util.spec_from_file_location(name, UTILS / "prefix_grouper.py")
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ finally:
+ for n in names:
+ sys.modules.pop(n, None)
+ return module
+
+
+ "version, ok",
+ [
+ ("2.6.0+cu124", False),
+ ("2.5.1", False),
+ ("1.13.1", False),
+ ("2.7.0a0+git1234", True),
+ ("2.7.1", True),
+ ("2.10.0", True),
+ ("3.0.0", True),
+ ("not-a-version", True),
+ ],
+)
+def test_torch_version_gate(version, ok):
+ assert _load_kernel().torch_supports_prefix_grouper(version) is ok
+
+
+def test_torch_26_import_is_clean_and_leaves_dynamo_alone(monkeypatch):
+ fake = _OldDynamoConfig(torch._dynamo.config)
+ monkeypatch.setattr(torch._dynamo, "config", fake)
+ monkeypatch.setattr(torch, "__version__", "2.6.0+cu124")
+ module = _load_kernel()
+ assert module.TORCH_SUPPORTS_PREFIX_GROUPER is False
+ assert fake.cache_size_limit == 8
+ assert fake.accumulated_cache_size_limit == 256
+
+
+def test_torch_26_prefix_grouper_off_with_one_log(monkeypatch, capsys):
+ monkeypatch.setenv("UNSLOTH_GRPO_PREFIX_GROUPER", "1")
+ monkeypatch.setenv("UNSLOTH_GRPO_SEQ_PACKING", "1")
+ monkeypatch.setattr(torch._dynamo, "config", _OldDynamoConfig(torch._dynamo.config))
+ monkeypatch.setattr(torch, "__version__", "2.6.0+cu124")
+ pg = _load_prefix_grouper()
+ assert capsys.readouterr().out == ""
+ assert pg.prefix_grouper_enabled() is False
+ out = capsys.readouterr().out
+ assert out.count("\n") == 1 and "PrefixGrouper is off on torch 2.6.0+cu124" in out
+ assert pg.prefix_grouper_enabled() is False
+ assert capsys.readouterr().out == ""
+
+
+def test_torch_26_env_off_stays_quiet(monkeypatch, capsys):
+ monkeypatch.setenv("UNSLOTH_GRPO_PREFIX_GROUPER", "0")
+ monkeypatch.setattr(torch, "__version__", "2.6.0")
+ pg = _load_prefix_grouper()
+ assert pg.prefix_grouper_enabled() is False
+ assert capsys.readouterr().out == ""
+
+
[email protected]("version", ["2.7.1", "2.8.0", "2.14.1"])
+def test_torch_27_plus_unchanged(monkeypatch, capsys, version):
+ monkeypatch.setenv("UNSLOTH_GRPO_PREFIX_GROUPER", "1")
+ monkeypatch.setenv("UNSLOTH_GRPO_SEQ_PACKING", "1")
+ monkeypatch.setattr(torch, "__version__", version)
+ config = torch._dynamo.config
+ for name in ("recompile_limit", "accumulated_recompile_limit"):
+ if hasattr(config, name):
+ monkeypatch.setattr(config, name, getattr(config, name))
+ pg = _load_prefix_grouper()
+ assert pg.prefix_grouper_enabled() is True
+ monkeypatch.setenv("UNSLOTH_GRPO_SEQ_PACKING", "0")
+ assert _load_prefix_grouper().prefix_grouper_enabled() is False
+ assert capsys.readouterr().out == ""
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 5b0ac7f67b6..ba35399f6b7 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -1045,8 +1045,11 @@ _flex_shared_prefix_attention = None
if os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER", "1").lower() not in ("0", "false", "no", "off"):
try:
from .prefix_grouper_kernel import (
+ TORCH_SUPPORTS_PREFIX_GROUPER as _pg_torch_ok,
flex_shared_prefix_attention as _flex_shared_prefix_attention,
)
+ if not _pg_torch_ok:
+ _flex_shared_prefix_attention = None
except Exception:
_flex_shared_prefix_attention = None
diff --git a/unsloth/utils/prefix_grouper.py b/unsloth/utils/prefix_grouper.py
index 9a67dbeef83..744dc652d3e 100644
--- a/unsloth/utils/prefix_grouper.py
+++ b/unsloth/utils/prefix_grouper.py
@@ -51,7 +51,12 @@ from typing import List, Optional, Tuple
import torch
-from .prefix_grouper_kernel import build_seg_info_multigroup, PrefixSegInfo
+from .prefix_grouper_kernel import (
+ build_seg_info_multigroup,
+ PrefixSegInfo,
+ PREFIX_GROUPER_MIN_TORCH,
+ TORCH_SUPPORTS_PREFIX_GROUPER,
+)
def env_on(name: str, default: str = "0") -> bool:
@@ -59,14 +64,27 @@ def env_on(name: str, default: str = "0") -> bool:
# One-time env reads; the helpers stay callable, since unsloth_zoo imports and calls them.
-_ENABLED = env_on("UNSLOTH_GRPO_SEQ_PACKING", "1") and env_on("UNSLOTH_GRPO_PREFIX_GROUPER", "1")
+_ENV_ENABLED = env_on("UNSLOTH_GRPO_SEQ_PACKING", "1") and env_on(
+ "UNSLOTH_GRPO_PREFIX_GROUPER", "1"
+)
+_ENABLED = _ENV_ENABLED and TORCH_SUPPORTS_PREFIX_GROUPER
+_OFF_LOGGED = False
_VERIFY_ON = env_on("UNSLOTH_GRPO_PREFIX_GROUPER_VERIFY", "1")
_TOKR_THRESHOLD = float(os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER_TOKR", "1.3"))
_TOL_OK = float(os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER_TOL", "0.7"))
def prefix_grouper_enabled() -> bool:
- """PrefixGrouper requires seq-packing on (it reuses its de-pad + scatter machinery)."""
+ """Needs seq-packing (reuses its de-pad + scatter) and torch >= 2.7; unsloth_zoo gates on it too."""
+ global _OFF_LOGGED
+ if _ENV_ENABLED and not _ENABLED and not _OFF_LOGGED:
+ _OFF_LOGGED = True
+ print(
+ f"Unsloth: GRPO PrefixGrouper is off on torch {torch.__version__} (needs torch >= "
+ f"{PREFIX_GROUPER_MIN_TORCH[0]}.{PREFIX_GROUPER_MIN_TORCH[1]}: its flex attention kernel "
+ "does not compile there); using the packed path.",
+ flush = True,
+ )
return _ENABLED
diff --git a/unsloth/utils/prefix_grouper_kernel.py b/unsloth/utils/prefix_grouper_kernel.py
index 7e3b514754b..ae5a873d4d8 100644
--- a/unsloth/utils/prefix_grouper_kernel.py
+++ b/unsloth/utils/prefix_grouper_kernel.py
@@ -16,6 +16,7 @@ dependency); imported lazily only when prefix_seg_info is set, so default-off ne
from __future__ import annotations
import os
+import re
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple
@@ -26,12 +27,33 @@ from torch.nn.attention.flex_attention import (
flex_attention,
)
-# Many distinct segment lengths: at the default recompile_limit (8) the kernel silently reuses a
-# mismatched specialisation.
-torch._dynamo.config.recompile_limit = max(getattr(torch._dynamo.config, "recompile_limit", 8), 256)
-torch._dynamo.config.accumulated_recompile_limit = max(
- getattr(torch._dynamo.config, "accumulated_recompile_limit", 256), 2048
-)
+
+# torch 2.6 inductor fails on the dynamic mask + flex kernel; suppress_errors then runs eager T x T flex.
+PREFIX_GROUPER_MIN_TORCH = (2, 7)
+
+
+def torch_supports_prefix_grouper(version = None) -> bool:
+ """True unless the torch major.minor is known to be below PREFIX_GROUPER_MIN_TORCH."""
+ match = re.match(r"(\d+)\.(\d+)", str(torch.__version__ if version is None else version))
+ if match is None:
+ return True
+ return (int(match.group(1)), int(match.group(2))) >= PREFIX_GROUPER_MIN_TORCH
+
+
+TORCH_SUPPORTS_PREFIX_GROUPER = torch_supports_prefix_grouper()
+
+
+# Default recompile_limit (8) silently reuses a mismatched specialisation; old name cache_size_limit.
+def _raise_dynamo_limit(names, floor):
+ config = torch._dynamo.config
+ name = next((n for n in names if hasattr(config, n)), None)
+ if name is not None:
+ setattr(config, name, max(getattr(config, name), floor))
+
+
+if TORCH_SUPPORTS_PREFIX_GROUPER:
+ _raise_dynamo_limit(("recompile_limit", "cache_size_limit"), 256)
+ _raise_dynamo_limit(("accumulated_recompile_limit", "accumulated_cache_size_limit"), 2048)
# dynamic=True is required: T changes almost every GRPO batch, and dynamic=False recompiles per T (~14s each).
@@ -371,6 +393,9 @@ def flex_shared_prefix_attention(
__all__ = [
+ "PREFIX_GROUPER_MIN_TORCH",
+ "TORCH_SUPPORTS_PREFIX_GROUPER",
+ "torch_supports_prefix_grouper",
"PrefixSegInfo",
"build_seg_info_multigroup",
"build_seg_info_from_layout",