Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block compile: per-region recompile limit with isolate_recompiles on torch 2.13+ (#15157)

Daniel Han5b7953bb3dee
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
index f522b12ce49..6526ab63d59 100644
--- a/tests/test_block_compile.py
+++ b/tests/test_block_compile.py
@@ -1551,3 +1551,74 @@ def test_training_flex_options_keep_a_callers_backend(backend):
return
q = torch.randn(1, 1, 16, 16)
flex._apply_kernel_options(q, q, q, False, M._training_flex_options(options))
+
+
+ "isolate_recompiles" not in __import__("inspect").signature(torch.compile).parameters,
+ reason = "torch < 2.13: no isolate_recompiles",
+)
+def test_recompile_storm_stops_at_the_region_limit(monkeypatch):
+ """A recompile storm stops at _region_recompile_limit (not the global 1024) and runs eager."""
+ M = _load()
+ from torch._dynamo.testing import CompileCounter
+
+ config = torch._dynamo.config
+ monkeypatch.setattr(config, "recompile_limit", 1024)
+ monkeypatch.setattr(config, "accumulated_recompile_limit", 1024)
+ counter, seen = CompileCounter(), []
+ real_compile = torch.compile
+
+ def counting_compile(fn, **kwargs):
+ seen.append(kwargs)
+ return real_compile(
+ fn,
+ backend = counter,
+ fullgraph = False,
+ dynamic = None,
+ isolate_recompiles = kwargs["isolate_recompiles"],
+ recompile_limit = kwargs["recompile_limit"],
+ )
+
+ monkeypatch.setattr(torch, "compile", counting_compile)
+ torch._dynamo.reset()
+
+ class StormDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+ self.tag = "a"
+
+ def forward(self, hidden_states, **kwargs):
+ # A str attribute is guarded by value: a new value every call recompiles.
+ return self.proj(hidden_states) + len(self.tag)
+
+ M._wrap(StormDecoderLayer, {}, 1)
+ layer = StormDecoderLayer().train()
+ for k in range(60):
+ layer.tag = "a" * (k + 1)
+ x = torch.randn(2, 5, 8, requires_grad = True)
+ out = layer(x)
+ torch.testing.assert_close(out, layer.proj(x) + (k + 1))
+ out.sum().backward()
+ torch._dynamo.reset()
+ assert seen[0]["isolate_recompiles"] is True
+ assert counter.frame_count == seen[0]["recompile_limit"] == M._region_recompile_limit(1) == 32
+
+
+def test_graphs_off_compile_gets_the_same_region_limit(monkeypatch):
+ # The graphs-off compile from UNSLOTH_BLOCK_CUDAGRAPHS needs the same isolation options.
+ M = _load()
+ if not M._ISOLATE_RECOMPILES:
+ pytest.skip("torch < 2.13: no per-region recompile limit")
+ seen = []
+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: seen.append(kwargs) or fn)
+
+ class GraphsDecoderLayer(torch.nn.Module):
+ def forward(self, hidden_states, **kwargs):
+ return hidden_states
+
+ M._wrap(GraphsDecoderLayer, {"triton.cudagraphs": True}, 2)
+ assert len(seen) == 2
+ for kwargs in seen:
+ assert kwargs["isolate_recompiles"] is True
+ assert kwargs["recompile_limit"] == M._region_recompile_limit(2) == 48
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index e31622e3cbc..b69350b14f1 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -6,6 +6,7 @@ norms, LoRA, NF4). UNSLOTH_BLOCK_COMPILE=0 or UNSLOTH_COMPILE_DISABLE turns it o
import contextlib
import functools
+import inspect
import os
import sys
import types
@@ -49,6 +50,29 @@ def _clone_outputs(out):
return pytree.tree_map(lambda x: x.clone() if torch.is_tensor(x) else x, out)
+def _isolate_supported():
+ # torch 2.13+: per-call recompile_limit and isolate_recompiles (own cache bucket per region).
+ try:
+ params = inspect.signature(torch.compile).parameters
+ except (TypeError, ValueError):
+ return False
+ return "isolate_recompiles" in params and "recompile_limit" in params
+
+
+_ISOLATE_RECOMPILES = _isolate_supported()
+
+
+def _region_recompile_limit(kinds):
+ """Graphs per block region before eager. Measured at most 4 per kind (Gemma3 GRPO: 20 for 2)."""
+ env = os.environ.get("UNSLOTH_BLOCK_COMPILE_RECOMPILE_LIMIT", "")
+ try:
+ if int(env) > 0:
+ return int(env)
+ except ValueError:
+ pass
+ return max(16 * kinds + 16, 32)
+
+
# gpt-oss builds its flex mask inside attention: the block broke and recompiled per length.
_SKIP = {"GptOssDecoderLayer"}
@@ -728,11 +752,20 @@ def _layer_kind(layer):
)
-def _wrap(cls, compile_options):
+def _wrap(
+ cls,
+ compile_options,
+ kinds = 1,
+):
original = cls.forward
# Host-read packed lengths / PrefixGrouper segments stay stock.
unsloth_forward = getattr(original, "__module__", "").startswith("unsloth.models.")
- compiled = torch.compile(original, fullgraph = False, dynamic = None, options = compile_options)
+ isolate = {}
+ if _ISOLATE_RECOMPILES:
+ isolate = dict(isolate_recompiles = True, recompile_limit = _region_recompile_limit(kinds))
+ compiled = torch.compile(
+ original, fullgraph = False, dynamic = None, options = compile_options, **isolate
+ )
try:
from unsloth_zoo.temporary_patches.utils import UNSLOTH_DECODE_COMPILE as inline_flag
except Exception:
@@ -754,6 +787,7 @@ def _wrap(cls, compile_options):
fullgraph = False,
dynamic = None,
options = {**compile_options, "triton.cudagraphs": False},
+ **isolate,
)
graph_shapes = set()
graphs_off = [False]
@@ -1132,7 +1166,9 @@ def patch_block_compile(model):
kinds = {}
for layer in compiled_layers:
kinds.setdefault(type(layer), set()).add(_layer_kind(layer))
- _raise_cache_limit(max(len(k) for k in kinds.values()))
+ if not _ISOLATE_RECOMPILES:
+ # Older torch: one global per-code-object limit. 2.13+ sets a limit per region in _wrap.
+ _raise_cache_limit(max(len(k) for k in kinds.values()))
# The wrap is per class: instances outside the decoder lists (T5 encoder) stay stock.
found = {id(layer) for layer, _ in layers}
for module in model.modules():
@@ -1142,7 +1178,7 @@ def patch_block_compile(model):
for cls in classes:
if getattr(cls.forward, _MARK, False):
continue
- _wrap(cls, _COMPILE_OPTIONS)
+ _wrap(cls, _COMPILE_OPTIONS, len(kinds.get(cls, ())) or 1)
wrapped.append(cls.__name__)
if wrapped:
print(f"Unsloth: compiling each transformer block as one region ({', '.join(wrapped)}).")