Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO PrefixGrouper: import cleanly on torch 2.6 and keep it off below 2.7 with a log line (#15836)
Daniel Hanedfcb3c9c283
diff --git a/tests/test_prefix_grouper_dynamo_limits.py b/tests/test_prefix_grouper_dynamo_limits.pynew file mode 100644index 00000000000..d0e4d8af9f5--- /dev/null+++ b/tests/test_prefix_grouper_dynamo_limits.py@@ -0,0 +1,164 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""PrefixGrouper on torch < 2.7: kernel imports cleanly, stays off with one log, dynamo untouched."""++import importlib.util+import pathlib+import sys+import types++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("torch.nn.attention.flex_attention")++UTILS = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "utils"+KERNEL = UTILS / "prefix_grouper_kernel.py"+PKG = "_pg_pkg_under_test"+NEW_NAMES = ("recompile_limit", "accumulated_recompile_limit")+++class _OldDynamoConfig:+ """torch 2.6 view of torch._dynamo.config: no recompile_limit names, unknown writes raise."""++ def __init__(self, real):+ object.__setattr__(self, "_real", real)+ object.__setattr__(+ self, "_local", {"cache_size_limit": 8, "accumulated_cache_size_limit": 256}+ )++ def __getattr__(self, name):+ if name in NEW_NAMES:+ raise AttributeError(name)+ local = object.__getattribute__(self, "_local")+ if name in local:+ return local[name]+ return getattr(object.__getattribute__(self, "_real"), name)++ def __setattr__(self, name, value):+ local = object.__getattribute__(self, "_local")+ if name not in local:+ raise AttributeError(f"torch._dynamo.config.{name} does not exist")+ local[name] = value+++def _load_kernel():+ name = "_pg_kernel_under_test"+ spec = importlib.util.spec_from_file_location(name, KERNEL)+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module # dataclasses resolve their module through sys.modules+ try:+ spec.loader.exec_module(module)+ finally:+ sys.modules.pop(name, None)+ return module+++def test_import_raises_the_old_names_when_only_they_exist(monkeypatch):+ fake = _OldDynamoConfig(torch._dynamo.config)+ monkeypatch.setattr(torch._dynamo, "config", fake)+ monkeypatch.setattr(torch, "__version__", "2.7.0")+ module = _load_kernel()+ assert callable(module.flex_shared_prefix_attention)+ assert fake.cache_size_limit == 256+ assert fake.accumulated_cache_size_limit == 2048+++ not hasattr(torch._dynamo.config, "recompile_limit"),+ reason = "torch < 2.7 has no recompile_limit; the old names are covered by the test above",+)+def test_import_raises_the_new_names_when_present(monkeypatch):+ config = torch._dynamo.config+ monkeypatch.setattr(config, "recompile_limit", 8)+ monkeypatch.setattr(config, "accumulated_recompile_limit", 256)+ _load_kernel()+ assert config.recompile_limit == 256+ assert config.accumulated_recompile_limit == 2048+ monkeypatch.setattr(config, "recompile_limit", 4096)+ _load_kernel()+ assert config.recompile_limit == 4096+++def _load_prefix_grouper():+ """Load prefix_grouper.py + its kernel under a throwaway package (no full unsloth import)."""+ pkg = types.ModuleType(PKG)+ pkg.__path__ = [str(UTILS)]+ names = (PKG, f"{PKG}.prefix_grouper_kernel", f"{PKG}.prefix_grouper")+ sys.modules[PKG] = pkg+ try:+ name = f"{PKG}.prefix_grouper"+ spec = importlib.util.spec_from_file_location(name, UTILS / "prefix_grouper.py")+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ finally:+ for n in names:+ sys.modules.pop(n, None)+ return module+++ "version, ok",+ [+ ("2.6.0+cu124", False),+ ("2.5.1", False),+ ("1.13.1", False),+ ("2.7.0a0+git1234", True),+ ("2.7.1", True),+ ("2.10.0", True),+ ("3.0.0", True),+ ("not-a-version", True),+ ],+)+def test_torch_version_gate(version, ok):+ assert _load_kernel().torch_supports_prefix_grouper(version) is ok+++def test_torch_26_import_is_clean_and_leaves_dynamo_alone(monkeypatch):+ fake = _OldDynamoConfig(torch._dynamo.config)+ monkeypatch.setattr(torch._dynamo, "config", fake)+ monkeypatch.setattr(torch, "__version__", "2.6.0+cu124")+ module = _load_kernel()+ assert module.TORCH_SUPPORTS_PREFIX_GROUPER is False+ assert fake.cache_size_limit == 8+ assert fake.accumulated_cache_size_limit == 256+++def test_torch_26_prefix_grouper_off_with_one_log(monkeypatch, capsys):+ monkeypatch.setenv("UNSLOTH_GRPO_PREFIX_GROUPER", "1")+ monkeypatch.setenv("UNSLOTH_GRPO_SEQ_PACKING", "1")+ monkeypatch.setattr(torch._dynamo, "config", _OldDynamoConfig(torch._dynamo.config))+ monkeypatch.setattr(torch, "__version__", "2.6.0+cu124")+ pg = _load_prefix_grouper()+ assert capsys.readouterr().out == ""+ assert pg.prefix_grouper_enabled() is False+ out = capsys.readouterr().out+ assert out.count("\n") == 1 and "PrefixGrouper is off on torch 2.6.0+cu124" in out+ assert pg.prefix_grouper_enabled() is False+ assert capsys.readouterr().out == ""+++def test_torch_26_env_off_stays_quiet(monkeypatch, capsys):+ monkeypatch.setenv("UNSLOTH_GRPO_PREFIX_GROUPER", "0")+ monkeypatch.setattr(torch, "__version__", "2.6.0")+ pg = _load_prefix_grouper()+ assert pg.prefix_grouper_enabled() is False+ assert capsys.readouterr().out == ""++[email protected]("version", ["2.7.1", "2.8.0", "2.14.1"])+def test_torch_27_plus_unchanged(monkeypatch, capsys, version):+ monkeypatch.setenv("UNSLOTH_GRPO_PREFIX_GROUPER", "1")+ monkeypatch.setenv("UNSLOTH_GRPO_SEQ_PACKING", "1")+ monkeypatch.setattr(torch, "__version__", version)+ config = torch._dynamo.config+ for name in ("recompile_limit", "accumulated_recompile_limit"):+ if hasattr(config, name):+ monkeypatch.setattr(config, name, getattr(config, name))+ pg = _load_prefix_grouper()+ assert pg.prefix_grouper_enabled() is True+ monkeypatch.setenv("UNSLOTH_GRPO_SEQ_PACKING", "0")+ assert _load_prefix_grouper().prefix_grouper_enabled() is False+ assert capsys.readouterr().out == ""diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 5b0ac7f67b6..ba35399f6b7 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -1045,8 +1045,11 @@ _flex_shared_prefix_attention = Noneif os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER", "1").lower() not in ("0", "false", "no", "off"):try:from .prefix_grouper_kernel import (+ TORCH_SUPPORTS_PREFIX_GROUPER as _pg_torch_ok,flex_shared_prefix_attention as _flex_shared_prefix_attention,)+ if not _pg_torch_ok:+ _flex_shared_prefix_attention = Noneexcept Exception:_flex_shared_prefix_attention = Nonediff --git a/unsloth/utils/prefix_grouper.py b/unsloth/utils/prefix_grouper.pyindex 9a67dbeef83..744dc652d3e 100644--- a/unsloth/utils/prefix_grouper.py+++ b/unsloth/utils/prefix_grouper.py@@ -51,7 +51,12 @@ from typing import List, Optional, Tupleimport torch-from .prefix_grouper_kernel import build_seg_info_multigroup, PrefixSegInfo+from .prefix_grouper_kernel import (+ build_seg_info_multigroup,+ PrefixSegInfo,+ PREFIX_GROUPER_MIN_TORCH,+ TORCH_SUPPORTS_PREFIX_GROUPER,+)def env_on(name: str, default: str = "0") -> bool:@@ -59,14 +64,27 @@ def env_on(name: str, default: str = "0") -> bool:# One-time env reads; the helpers stay callable, since unsloth_zoo imports and calls them.-_ENABLED = env_on("UNSLOTH_GRPO_SEQ_PACKING", "1") and env_on("UNSLOTH_GRPO_PREFIX_GROUPER", "1")+_ENV_ENABLED = env_on("UNSLOTH_GRPO_SEQ_PACKING", "1") and env_on(+ "UNSLOTH_GRPO_PREFIX_GROUPER", "1"+)+_ENABLED = _ENV_ENABLED and TORCH_SUPPORTS_PREFIX_GROUPER+_OFF_LOGGED = False_VERIFY_ON = env_on("UNSLOTH_GRPO_PREFIX_GROUPER_VERIFY", "1")_TOKR_THRESHOLD = float(os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER_TOKR", "1.3"))_TOL_OK = float(os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER_TOL", "0.7"))def prefix_grouper_enabled() -> bool:- """PrefixGrouper requires seq-packing on (it reuses its de-pad + scatter machinery)."""+ """Needs seq-packing (reuses its de-pad + scatter) and torch >= 2.7; unsloth_zoo gates on it too."""+ global _OFF_LOGGED+ if _ENV_ENABLED and not _ENABLED and not _OFF_LOGGED:+ _OFF_LOGGED = True+ print(+ f"Unsloth: GRPO PrefixGrouper is off on torch {torch.__version__} (needs torch >= "+ f"{PREFIX_GROUPER_MIN_TORCH[0]}.{PREFIX_GROUPER_MIN_TORCH[1]}: its flex attention kernel "+ "does not compile there); using the packed path.",+ flush = True,+ )return _ENABLEDdiff --git a/unsloth/utils/prefix_grouper_kernel.py b/unsloth/utils/prefix_grouper_kernel.pyindex 7e3b514754b..ae5a873d4d8 100644--- a/unsloth/utils/prefix_grouper_kernel.py+++ b/unsloth/utils/prefix_grouper_kernel.py@@ -16,6 +16,7 @@ dependency); imported lazily only when prefix_seg_info is set, so default-off nefrom __future__ import annotationsimport os+import refrom dataclasses import dataclassfrom typing import Dict, List, Optional, Tuple@@ -26,12 +27,33 @@ from torch.nn.attention.flex_attention import (flex_attention,)-# Many distinct segment lengths: at the default recompile_limit (8) the kernel silently reuses a-# mismatched specialisation.-torch._dynamo.config.recompile_limit = max(getattr(torch._dynamo.config, "recompile_limit", 8), 256)-torch._dynamo.config.accumulated_recompile_limit = max(- getattr(torch._dynamo.config, "accumulated_recompile_limit", 256), 2048-)++# torch 2.6 inductor fails on the dynamic mask + flex kernel; suppress_errors then runs eager T x T flex.+PREFIX_GROUPER_MIN_TORCH = (2, 7)+++def torch_supports_prefix_grouper(version = None) -> bool:+ """True unless the torch major.minor is known to be below PREFIX_GROUPER_MIN_TORCH."""+ match = re.match(r"(\d+)\.(\d+)", str(torch.__version__ if version is None else version))+ if match is None:+ return True+ return (int(match.group(1)), int(match.group(2))) >= PREFIX_GROUPER_MIN_TORCH+++TORCH_SUPPORTS_PREFIX_GROUPER = torch_supports_prefix_grouper()+++# Default recompile_limit (8) silently reuses a mismatched specialisation; old name cache_size_limit.+def _raise_dynamo_limit(names, floor):+ config = torch._dynamo.config+ name = next((n for n in names if hasattr(config, n)), None)+ if name is not None:+ setattr(config, name, max(getattr(config, name), floor))+++if TORCH_SUPPORTS_PREFIX_GROUPER:+ _raise_dynamo_limit(("recompile_limit", "cache_size_limit"), 256)+ _raise_dynamo_limit(("accumulated_recompile_limit", "accumulated_cache_size_limit"), 2048)# dynamic=True is required: T changes almost every GRPO batch, and dynamic=False recompiles per T (~14s each).@@ -371,6 +393,9 @@ def flex_shared_prefix_attention(__all__ = [+ "PREFIX_GROUPER_MIN_TORCH",+ "TORCH_SUPPORTS_PREFIX_GROUPER",+ "torch_supports_prefix_grouper","PrefixSegInfo","build_seg_info_multigroup","build_seg_info_from_layout",