Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block compile: per-region recompile limit with isolate_recompiles on torch 2.13+ (#15157)
Daniel Han5b7953bb3dee
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pyindex f522b12ce49..6526ab63d59 100644--- a/tests/test_block_compile.py+++ b/tests/test_block_compile.py@@ -1551,3 +1551,74 @@ def test_training_flex_options_keep_a_callers_backend(backend):returnq = torch.randn(1, 1, 16, 16)flex._apply_kernel_options(q, q, q, False, M._training_flex_options(options))+++ "isolate_recompiles" not in __import__("inspect").signature(torch.compile).parameters,+ reason = "torch < 2.13: no isolate_recompiles",+)+def test_recompile_storm_stops_at_the_region_limit(monkeypatch):+ """A recompile storm stops at _region_recompile_limit (not the global 1024) and runs eager."""+ M = _load()+ from torch._dynamo.testing import CompileCounter++ config = torch._dynamo.config+ monkeypatch.setattr(config, "recompile_limit", 1024)+ monkeypatch.setattr(config, "accumulated_recompile_limit", 1024)+ counter, seen = CompileCounter(), []+ real_compile = torch.compile++ def counting_compile(fn, **kwargs):+ seen.append(kwargs)+ return real_compile(+ fn,+ backend = counter,+ fullgraph = False,+ dynamic = None,+ isolate_recompiles = kwargs["isolate_recompiles"],+ recompile_limit = kwargs["recompile_limit"],+ )++ monkeypatch.setattr(torch, "compile", counting_compile)+ torch._dynamo.reset()++ class StormDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)+ self.tag = "a"++ def forward(self, hidden_states, **kwargs):+ # A str attribute is guarded by value: a new value every call recompiles.+ return self.proj(hidden_states) + len(self.tag)++ M._wrap(StormDecoderLayer, {}, 1)+ layer = StormDecoderLayer().train()+ for k in range(60):+ layer.tag = "a" * (k + 1)+ x = torch.randn(2, 5, 8, requires_grad = True)+ out = layer(x)+ torch.testing.assert_close(out, layer.proj(x) + (k + 1))+ out.sum().backward()+ torch._dynamo.reset()+ assert seen[0]["isolate_recompiles"] is True+ assert counter.frame_count == seen[0]["recompile_limit"] == M._region_recompile_limit(1) == 32+++def test_graphs_off_compile_gets_the_same_region_limit(monkeypatch):+ # The graphs-off compile from UNSLOTH_BLOCK_CUDAGRAPHS needs the same isolation options.+ M = _load()+ if not M._ISOLATE_RECOMPILES:+ pytest.skip("torch < 2.13: no per-region recompile limit")+ seen = []+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: seen.append(kwargs) or fn)++ class GraphsDecoderLayer(torch.nn.Module):+ def forward(self, hidden_states, **kwargs):+ return hidden_states++ M._wrap(GraphsDecoderLayer, {"triton.cudagraphs": True}, 2)+ assert len(seen) == 2+ for kwargs in seen:+ assert kwargs["isolate_recompiles"] is True+ assert kwargs["recompile_limit"] == M._region_recompile_limit(2) == 48diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex e31622e3cbc..b69350b14f1 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -6,6 +6,7 @@ norms, LoRA, NF4). UNSLOTH_BLOCK_COMPILE=0 or UNSLOTH_COMPILE_DISABLE turns it oimport contextlibimport functools+import inspectimport osimport sysimport types@@ -49,6 +50,29 @@ def _clone_outputs(out):return pytree.tree_map(lambda x: x.clone() if torch.is_tensor(x) else x, out)+def _isolate_supported():+ # torch 2.13+: per-call recompile_limit and isolate_recompiles (own cache bucket per region).+ try:+ params = inspect.signature(torch.compile).parameters+ except (TypeError, ValueError):+ return False+ return "isolate_recompiles" in params and "recompile_limit" in params+++_ISOLATE_RECOMPILES = _isolate_supported()+++def _region_recompile_limit(kinds):+ """Graphs per block region before eager. Measured at most 4 per kind (Gemma3 GRPO: 20 for 2)."""+ env = os.environ.get("UNSLOTH_BLOCK_COMPILE_RECOMPILE_LIMIT", "")+ try:+ if int(env) > 0:+ return int(env)+ except ValueError:+ pass+ return max(16 * kinds + 16, 32)++# gpt-oss builds its flex mask inside attention: the block broke and recompiled per length._SKIP = {"GptOssDecoderLayer"}@@ -728,11 +752,20 @@ def _layer_kind(layer):)-def _wrap(cls, compile_options):+def _wrap(+ cls,+ compile_options,+ kinds = 1,+):original = cls.forward# Host-read packed lengths / PrefixGrouper segments stay stock.unsloth_forward = getattr(original, "__module__", "").startswith("unsloth.models.")- compiled = torch.compile(original, fullgraph = False, dynamic = None, options = compile_options)+ isolate = {}+ if _ISOLATE_RECOMPILES:+ isolate = dict(isolate_recompiles = True, recompile_limit = _region_recompile_limit(kinds))+ compiled = torch.compile(+ original, fullgraph = False, dynamic = None, options = compile_options, **isolate+ )try:from unsloth_zoo.temporary_patches.utils import UNSLOTH_DECODE_COMPILE as inline_flagexcept Exception:@@ -754,6 +787,7 @@ def _wrap(cls, compile_options):fullgraph = False,dynamic = None,options = {**compile_options, "triton.cudagraphs": False},+ **isolate,)graph_shapes = set()graphs_off = [False]@@ -1132,7 +1166,9 @@ def patch_block_compile(model):kinds = {}for layer in compiled_layers:kinds.setdefault(type(layer), set()).add(_layer_kind(layer))- _raise_cache_limit(max(len(k) for k in kinds.values()))+ if not _ISOLATE_RECOMPILES:+ # Older torch: one global per-code-object limit. 2.13+ sets a limit per region in _wrap.+ _raise_cache_limit(max(len(k) for k in kinds.values()))# The wrap is per class: instances outside the decoder lists (T5 encoder) stay stock.found = {id(layer) for layer, _ in layers}for module in model.modules():@@ -1142,7 +1178,7 @@ def patch_block_compile(model):for cls in classes:if getattr(cls.forward, _MARK, False):continue- _wrap(cls, _COMPILE_OPTIONS)+ _wrap(cls, _COMPILE_OPTIONS, len(kinds.get(cls, ())) or 1)wrapped.append(cls.__name__)if wrapped:print(f"Unsloth: compiling each transformer block as one region ({', '.join(wrapped)}).")