Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Decode CUDA graphs on by default for every FastModel model (2-8x on T4, tokens equal to eager) (#16587)
Daniel Han00d10566e67d
diff --git a/tests/test_generic_decode_graphs.py b/tests/test_generic_decode_graphs.pynew file mode 100644index 00000000000..f6ab77456a3--- /dev/null+++ b/tests/test_generic_decode_graphs.py@@ -0,0 +1,901 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FastModel decode CUDA graphs on the host, through a stand-in backend whose "graph" re-runs the+captured step into the same static outputs. The real capture is the gpu-marked test."""++import contextlib+import functools+import inspect+import os+import sys+import types++import pytest+import torch++import unsloth # noqa: F401 (must precede transformers)++from real_accelerator import has_real_cuda+import unsloth.models._decode_graphs as G+import unsloth.models._generic_decode_graphs as GG+from unsloth.models import vision++from torch._dynamo.utils import counters+from transformers import AutoModelForCausalLM, DynamicCache, StaticCache+from transformers import cache_utils++LLAMA = "hf-internal-testing/tiny-random-LlamaForCausalLM"+GEMMA3 = "hf-internal-testing/tiny-random-Gemma3ForCausalLM"+QWEN3_VL = "trl-internal-testing/tiny-Qwen3VLForConditionalGeneration"+NO_ROLL = "no static sliding layer with the host-built roll index in this transformers"+++def test_imports_this_tree():+ here = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+ assert os.path.abspath(GG.__file__).startswith(here)+++class _FakeStream:+ class device:+ index = None++ wait_stream = lambda self, other: None+ synchronize = lambda self: None+++def _copy_into(dst, src):+ xs, ys = GG._tensors(dst, []), GG._tensors(src, [])+ assert len(xs) == len(ys)+ for x, y in zip(xs, ys):+ x.copy_(y)+++class _FakeGraph:+ """Replay = the captured step run again into the captured outputs; host counters are left+ as a real replay leaves them (unchanged). perturb: True = every replay, an int n = from the+ n-th replay on (drift after validation)."""++ def __init__(self, fn, outputs, cache, perturb):+ self.fn, self.outputs, self.cache, self.perturb = fn, outputs, cache, perturb+ self.replays = 0++ def replay(self):+ self.replays += 1+ host = GG._snapshot(self.cache)[0]+ new = self.fn()+ GG._restore((host, []))+ _copy_into(self.outputs, new)+ if self.perturb is True or (type(self.perturb) is int and self.replays >= self.perturb):+ GG._tensors(self.outputs, [])[0][..., 0].add_(1)+++class _Host(GG._Cuda):+ device_type = "cpu"+ perturb = False+ fail = False+ graphs = []+ # As a real capture: no host-to-device copies (torch.tensor(..., device=...)) while recording.+ recording = False+ current_device = staticmethod(lambda: None)+ device = staticmethod(lambda index: contextlib.nullcontext())+ pool = staticmethod(lambda: object())+ new_stream = staticmethod(lambda: _FakeStream())+ stream = staticmethod(lambda stream: contextlib.nullcontext())+ current_stream = staticmethod(lambda: _FakeStream())++ @classmethod+ def capturing(cls):+ return cls.recording++ @classmethod+ def record(cls, pool, fn, cache):+ if cls.fail:+ error = cls.fail if isinstance(cls.fail, BaseException) else None+ raise error or RuntimeError("operation not permitted when stream is capturing")+ # A capture launches nothing: put back the length counters and every cache row the step wrote.+ snapshot = GG._snapshot(cache)+ rows = [+ (t, t.clone())+ for layer in cache.layers+ for t in (getattr(layer, "keys", None), getattr(layer, "values", None))+ if isinstance(t, torch.Tensor)+ ]+ real_tensor = torch.tensor++ def tensor(*args, **kwargs):+ if kwargs.get("device") is not None:+ raise RuntimeError(+ "Cannot copy between CPU and CUDA tensors during CUDA graph capture unless the CPU tensor is pinned."+ )+ return real_tensor(*args, **kwargs)++ cls.recording = True+ torch.tensor = tensor+ try:+ outputs = fn()+ finally:+ torch.tensor = real_tensor+ cls.recording = False+ GG._restore(snapshot)+ for live, saved in rows:+ live.copy_(saved)+ graph = _FakeGraph(fn, outputs, cache, cls.perturb)+ cls.graphs.append(graph)+ return graph, outputs+++def host(monkeypatch):+ backend = type("Host", (_Host,), {"graphs": []})+ monkeypatch.setattr(GG, "_BACKEND", backend)+ return backend++[email protected](scope = "module")+def llama():+ torch.manual_seed(0)+ return AutoModelForCausalLM.from_pretrained(LLAMA, dtype = torch.float32).eval()++[email protected](scope = "module")+def gemma3():+ torch.manual_seed(0)+ # A short window, so a short generation runs past it.+ return AutoModelForCausalLM.from_pretrained(+ GEMMA3, dtype = torch.float32, sliding_window = 8+ ).eval()+++def _fresh(model):+ model.__dict__.pop(GG._ATTR, None)+++def _ids(+ model,+ bsz = 1,+ length = 6,+ pad = 0,+):+ g = torch.Generator().manual_seed(1)+ vocab = model.config.get_text_config().vocab_size+ # Below the multimodal special tokens (image / video pads sit at the top of the vocabulary).+ ids = torch.randint(3, min(vocab, 1000), (bsz, length), generator = g)+ mask = torch.ones_like(ids)+ if pad:+ ids[0, :pad] = 0+ mask[0, :pad] = 0+ return ids, mask+++def _generate(+ model,+ ids,+ mask,+ graphs,+ new = 10,+):+ scope = vision._GraphDecodeSteps(model) if graphs else contextlib.nullcontext()+ with torch.inference_mode(), scope:+ return model.generate(+ input_ids = ids,+ attention_mask = mask,+ max_new_tokens = new,+ min_new_tokens = new,+ do_sample = False,+ cache_implementation = "static",+ pad_token_id = 0,+ # transformers 4.57 compiles static-cache generation on CUDA, which the graph path steps aside for.+ disable_compile = True,+ )+++def _both(+ model,+ ids,+ mask,+ new = 10,+):+ """Eager then graphed tokens for a fresh model state; asserts they match, returns stats."""+ _fresh(model)+ eager = _generate(model, ids, mask, False, new)+ assert torch.equal(eager, _generate(model, ids, mask, True, new))+ return GG.stats(model)+++def _scope(model, forward, eager):+ scope = vision._GraphDecodeSteps.__new__(vision._GraphDecodeSteps)+ scope.model, scope.forward = model, forward+ scope.eager_decodes = scope.eager_regions = eager+ return scope+++def _sliding_layer(cls):+ try:+ return cls(max_cache_len = 32, sliding_window = 4)+ except TypeError:+ return cls(sliding_window = 4)++[email protected]("bsz,pad,length", [(1, 0, 6), (3, 2, 6), (1, 0, 1)])+def test_greedy_tokens_match_eager_llama(host, llama, bsz, pad, length):+ s = _both(llama, *_ids(llama, bsz, length, pad))+ # Step 1 = prefill (eager, also for a one-token prompt), step 2 eager warm-up + capture, step 3+ # validated replay, then replays: 9 decode steps take the graph path.+ assert GG._state(llama).step == 9+ assert s["captures"] == 1 and s["validations"] == 1 and s["replays"] == 8, s+ assert s["refusals"] == {}, s+ assert s["graphs"] == 0 # freed when generate() returns+ assert "forward" not in llama.__dict__+++def test_batches_over_the_knob_decode_eagerly(host, llama, monkeypatch):+ monkeypatch.setattr(GG, "MAX_BSZ", 2)+ ids, mask = _ids(llama, 3, pad = 1)+ s = _both(llama, ids, mask)+ assert s["captures"] == 0 and s["refusals"] == {"batch over max": 9}, s+ monkeypatch.setattr(GG, "MAX_BSZ", 3)+ _generate(llama, ids, mask, True)+ assert GG.stats(llama)["replays"] == 8+++def test_counts_captures_refusals_and_recompiles_separately(host, llama):+ def compiles():+ return counters["stats"]["unique_graphs"], sum(counters["recompiles"].values())++ layer = torch.nn.Linear(4, 4)+ compiled = torch.compile(lambda x: layer(x).relu(), backend = "eager")+ compiled(torch.ones(2, 4))+ ids, mask = _ids(llama, 2, pad = 1)+ # Warm the compiled mask builder for these shapes eagerly first, so the count does not depend on test order.+ _generate(llama, ids, mask, False)+ before = compiles()+ _fresh(llama)+ _generate(llama, ids, mask, True)+ compiled(torch.ones(2, 4))+ # Decode graphs add no Dynamo graph and no recompile of an existing compiled region.+ assert compiles() == before+ s = GG.stats(llama)+ assert s["captures"] == 1 and s["max_captures_per_key"] == 1 and s["refusals"] == {}, s+ # A second call captures again but does not revalidate the key.+ _generate(llama, ids, mask, True)+ s = GG.stats(llama)+ assert s["captures"] == 2 and s["max_captures_per_key"] == 1 and s["validations"] == 1, s+ assert sum(g.replays for g in host.graphs) == 2 * 8+++def test_graph_path_runs_the_compiled_variant_eager_runs(host, llama):+ """A compiled region on the step's arguments (zoo compiles create_causal_mask) guards on their+ inference-ness and strides: the graph path must hit the variant the eager step compiled, never+ a new one, and give the eager logits. (Dynamo's global-state guard also covers the autocast cache+ flag, so the step keeps the ambient autocast; CUDA autocast is not reachable on the host.)"""+ seen, calls = [], []++ @torch.compile(backend = "eager", dynamic = False)+ def region(mask, position):+ # A different variant would add to the logits: the tokens would move.+ flip = 0.0 if mask.is_inference() else 1000.0+ return mask.float().mean() * 0 + position.float().mean() * 0 + flip++ def pre_hook(module, args, kwargs):+ seen.append(kwargs.get("attention_mask"))+ # transformers 5.x passes no cache_position: fall back to the positions.+ position = kwargs.get("cache_position")+ seen.append(position if position is not None else kwargs.get("position_ids"))++ def head_hook(module, args, out):+ mask, position = seen[-2], seen[-1]+ if mask is None or position is None:+ return out+ if isinstance(mask, dict):+ mask = next(iter(mask.values()))+ calls.append(1)+ return out + region(mask, position)++ handles = [+ llama.model.register_forward_pre_hook(pre_hook, with_kwargs = True),+ llama.lm_head.register_forward_hook(head_hook),+ ]+ try:+ _fresh(llama)+ ids, mask = _ids(llama, 3, pad = 2)+ eager = _generate(llama, ids, mask, False)+ assert calls, "the compiled region never ran: the check would be vacuous"+ before = counters["stats"]["unique_graphs"]+ graphed = _generate(llama, ids, mask, True)+ assert counters["stats"]["unique_graphs"] == before+ assert torch.equal(eager, graphed)+ s = GG.stats(llama)+ assert s["replays"] == 8 and not s["refusals"], s+ finally:+ for h in handles:+ h.remove()++[email protected]("make", ["dense", "sliced", "broadcast", "sliced_broadcast"])+def test_static_buffers_keep_eager_strides(host, make):+ base = torch.arange(64, dtype = torch.float32).view(2, 32)+ leaf = {+ "dense": base,+ "sliced": base[:, ::2], # shape (2, 16), strides (32, 2)+ "broadcast": base[:1].expand(4, 32), # as Qwen3-VL positions+ "sliced_broadcast": base[:1, ::2].expand(3, 16),+ }[make]+ compact = GG._compact(leaf)+ if "broadcast" in make:+ assert compact.shape[0] == 1+ buffer = GG._buffer_like(compact)+ buffer.copy_(compact)+ replayed = buffer.expand(leaf.shape)+ assert replayed.stride() == leaf.stride() and torch.equal(replayed, leaf)+ with torch.inference_mode():+ x = torch.arange(6.0).view(2, 3)+ assert GG._buffer_like(x).is_inference() and not GG._buffer_like(base).is_inference()+ t = torch.zeros(4, 6).t() # non-contiguous+ assert GG._buffer_like(t).stride() == t.stride()+ spec = GG._spec(t, [], None)+ assert t.stride() in spec and False in spec++[email protected]("past", [False, True])+def test_sliding_window_roll_is_captured_without_host_copies(host, gemma3, monkeypatch, past):+ # Inside the window every sliding layer updates by index: one key, every step after the first two replays.+ # Past the window the sliding layers roll by one slot per step: a second key (window full) is captured and+ # validated (its rolled rows restored between the replay and the eager step), then replays. transformers+ # rolls a full window with an index made by torch.tensor(..., device=...), a host-to-device copy a real+ # capture refuses; the warm-up and recording run that branch on an index made outside the capture. The+ # recording runs no device work, so the capture holds no copy of the full windows' rows while it records.+ taken = []+ snapshot = GG._snapshot++ def recorder(cache, *args, **kwargs):+ out = snapshot(cache, *args, **kwargs)+ if sys._getframe(1).f_code.co_name == "_capture":+ taken.append(max((e[1].numel() for e in out[1]), default = 0))+ return out++ monkeypatch.setattr(GG, "_snapshot", recorder)+ assert "sliding_attention" in gemma3.config.layer_types+ new = gemma3.config.sliding_window + (12 if past else -3)+ keys = 1 + past+ s = _both(gemma3, *_ids(gemma3, 2, length = 3, pad = 1), new)+ assert not s["refusals"], s+ assert s["captures"] == keys and s["validations"] == keys, s+ assert s["replays"] == new - 1 - keys, s+ assert len(taken) == keys and max(taken) <= 1, taken+ for cls, _ in GG._rolling_update_classes():+ assert not getattr(cls.__dict__["update"], "_unsloth_graphable_roll", False)+++def test_beam_search_reordering_the_cache_stays_eager(host, llama):+ _fresh(llama)+ ids, mask = _ids(llama)+ kw = dict(max_new_tokens = 8, min_new_tokens = 8, num_beams = 2, do_sample = False)+ kw.update(cache_implementation = "static", pad_token_id = 0, attention_mask = mask)+ with torch.inference_mode():+ eager = llama.generate(ids, **kw)+ with vision._GraphDecodeSteps(llama):+ graphed = llama.generate(ids, **kw)+ assert torch.equal(eager, graphed)+ s = GG.stats(llama)+ moved = s["refusals"].get("cache moved", 0)+ # Either the reorder kept the cache in place (all replays) or every step after it ran eagerly.+ assert (moved == 0 and s["replays"] > 0) or s["replays"] + moved + s["captures"] + s[+ "validations"+ ] == 7, s+++def test_qwen3_vl_text_decode_positions_match_the_model(host, monkeypatch):+ import transformers+ from transformers import AutoModelForImageTextToText++ torch.manual_seed(0)+ model = AutoModelForImageTextToText.from_pretrained(QWEN3_VL, dtype = torch.float32).eval()+ seen = []+ real = GG._mrope_positions++ def spy(m, kwargs):+ out = real(m, kwargs)+ seen.append(out is not kwargs)+ return out++ monkeypatch.setattr(GG, "_mrope_positions", spy)+ s = _both(model, *_ids(model, 2, length = 5, pad = 1))+ assert s["replays"] == 8 and not s["refusals"], s+ if transformers.__version__.startswith("4."):+ # 4.x leaves decode positions to the forward, which reads cache_position[0] on the host.+ assert seen and all(seen)+++def test_validation_mismatch_refuses_the_key(host, llama):+ host.perturb = True+ ids, mask = _ids(llama)+ s = _both(llama, ids, mask)+ assert s["refusals"].get("validation mismatch", 0) >= 1 and s["replays"] == 0, s+ # Refused for the process: the next call does not capture again.+ _generate(llama, ids, mask, True)+ assert GG.stats(llama)["captures"] == 1+++def test_validate_all_pinpoints_a_later_drift(host, llama, monkeypatch):+ _fresh(llama)+ host.perturb = 4 # bitwise at the validated step, then the replay drifts+ ids, mask = _ids(llama)+ eager = _generate(llama, ids, mask, False)+ monkeypatch.setenv("UNSLOTH_DECODE_GENERIC_GRAPHS_VALIDATE_ALL", "1")+ # Every replay is compared and the eager result returned: tokens stay eager's.+ assert torch.equal(eager, _generate(llama, ids, mask, True))+ check = GG.stats(llama)["check"]+ # Decode step 1 captures, step 2 validates (replay 1), drift from replay 4 = decode step 5.+ assert check["steps"] == 8 and check["mismatches"] == 5, check+ assert check["first_mismatch_step"] == 5 and check["max_abs_diff"] == 1.0, check+ assert check["max_kl"] >= 0.0 and check["top1_disagree"] >= 0+ monkeypatch.delenv("UNSLOTH_DECODE_GENERIC_GRAPHS_VALIDATE_ALL")+ # Without it, the drift goes unseen once the key is trusted: why the debug mode exists.+ _fresh(llama)+ _generate(llama, ids, mask, True)+ assert GG.stats(llama)["check"]["mismatches"] == 0+++def test_graph_path_runs_the_eager_path_regions(monkeypatch):+ seen = []+ monkeypatch.setattr(+ vision._generic_decode_graphs,+ "decode_step",+ lambda model, forward, kwargs, regions: seen.append(regions),+ )+ model = torch.nn.Linear(1, 1)+ model.forward = lambda **kw: 0+ for compiled_decode in (False, True):+ _scope(model, model.forward, compiled_decode)._forward(input_ids = torch.zeros(1, 1))+ # Compiled regions stay compiled unless this model's eager decode already leaves them.+ assert seen == [None, vision.unsloth_eager_decode]+++def test_capture_failure_falls_back_for_the_key(host, llama):+ host.fail = True+ s = _both(llama, *_ids(llama))+ assert s["captures"] == 0 and s["replays"] == 0, s+ # Refused once for the process; every later step of the key counts as a refused eager step.+ reason = "capture failed: RuntimeError: operation not permitted when stream is capturing"+ assert s["refused_keys"] == 1 and s["refusals"] == {reason: 9}, s+ # What broke it and where (innermost frame outside torch).+ assert s["capture_errors"] == [(reason, s["capture_errors"][0][1])], s+ assert "test_generic_decode_graphs.py" in s["capture_errors"][0][1], s+ assert s["eager_steps"] == 8, s+++def test_out_of_memory_capture_stops_the_call_not_the_key(host, llama):+ host.fail = torch.cuda.OutOfMemoryError("CUDA out of memory. Tried to allocate 2.00 MiB")+ ids, mask = _ids(llama)+ s = _both(llama, ids, mask)+ assert s["captures"] == 0 and s["refused_keys"] == 0, s+ assert s["refusals"] == {"capture failed: out of memory": 9}, s+ host.fail = False+ assert torch.equal(_generate(llama, ids, mask, False), _generate(llama, ids, mask, True))+ s = GG.stats(llama)+ assert s["captures"] == 1 and s["replays"] == 8, s+++def test_capture_setup_failure_runs_eagerly(host, llama, monkeypatch):+ def no_pool():+ raise torch.OutOfMemoryError("pool")++ monkeypatch.setattr(host, "pool", staticmethod(no_pool))+ s = _both(llama, *_ids(llama))+ assert s["replays"] == 0 and any("capture setup failed" in r for r in s["refusals"]), s+++def test_fingerprint_invalidation(host, llama, monkeypatch):+ from unsloth.kernels import utils as K++ _fresh(llama)+ first = GG._fingerprint(llama)+ assert first is not None and first == GG._fingerprint(llama)+ with torch.no_grad():+ next(llama.parameters()).add_(0) # in-place edit (merge, optimizer step): bumps the version+ assert GG._fingerprint(llama) != first+ before = GG._fingerprint(llama)+ monkeypatch.setattr(K, "_OPTIMIZER_STEPS", K._OPTIMIZER_STEPS + 1)+ assert GG._fingerprint(llama) != before+ # A step after the optimizer moved re-checks, and a changed fingerprint drops the graphs.+ state = GG._state(llama)+ state.fingerprint = ("stale",)+ state.graphs["k"] = object()+ state.recheck = False+ _generate(llama, *_ids(llama), True)+ assert state.fingerprint != ("stale",)+++def test_lora_scale_changes_the_fingerprint(host, llama):+ # PEFT's set_scale / scale_layer move a LoRA layer's Python scale without touching a parameter; the captured+ # kernels baked it in, so a kept graph must not replay across it.+ _fresh(llama)+ proj = llama.model.layers[0].self_attn.o_proj+ proj.scaling = {"default": 1.0}+ try:+ first = GG._fingerprint(llama)+ assert first == GG._fingerprint(llama)+ proj.scaling["default"] = 0.0+ assert GG._fingerprint(llama) != first+ proj.scaling["default"] = 1.0+ assert GG._fingerprint(llama) == first+ finally:+ del proj.scaling+ peft = pytest.importorskip("peft")+ model = peft.get_peft_model(+ AutoModelForCausalLM.from_pretrained(LLAMA, dtype = torch.float32).eval(),+ peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["o_proj"]),+ )+ first = GG._fingerprint(model)+ model.base_model.model.model.layers[0].self_attn.o_proj.set_scale("default", 0.0)+ assert GG._fingerprint(model) != first+++def test_model_on_two_devices_stays_eager_for_the_call(host, llama, monkeypatch):+ monkeypatch.setattr(GG, "_fingerprint", lambda model, state = None: None)+ s = _both(llama, *_ids(llama))+ assert s["captures"] == 0 and s["refusals"] == {"not one CUDA device / adapters": 9}, s++[email protected]("drop", [GG.release, G.invalidate, G.release])+def test_weight_changes_revoke_graphs_and_key_decisions(host, llama, drop):+ _fresh(llama)+ state = GG._state(llama)+ state.graphs["k"] = object()+ state.validated.add("key")+ state.refused["other"] = "validation mismatch"+ state.fingerprint = ("old",)+ drop(llama)+ assert not state.graphs and not state.validated and not state.refused+ assert state.fingerprint is None+++def test_cache_layer_checks(host, llama):+ assert GG._cache_reason(None) == "no cache"+ assert GG._cache_reason(DynamicCache()) == "cache DynamicCache"+ cache = StaticCache(config = llama.config, max_cache_len = 16)+ assert GG._cache_reason(cache) == "uninitialized layer"+ with torch.inference_mode():+ llama(input_ids = torch.tensor([[5, 6, 7]]), past_key_values = cache, use_cache = True)+ assert GG._cache_reason(cache) is None++ class Recurrent(cache_utils.StaticLayer): # 5.x linear-attention layers subclass StaticLayer+ pass++ layer = cache.layers[0]+ layer.__class__ = Recurrent+ assert GG._cache_reason(cache) == "layer Recurrent"+ layer.__class__ = cache_utils.StaticLayer+ cache.offloading = True+ assert GG._cache_reason(cache) == "offloaded cache"+ cache.offloading = False+ # Real backend: a host cache is not replayable.+ host.device_type = "cuda"+ assert GG._cache_reason(cache) == "cache not on CUDA"+++def test_window_and_host_state_checks():+ layer = cache_utils.StaticSlidingWindowLayer(max_cache_len = 32, sliding_window = 4)+ layer.keys = torch.zeros(1)+ cache = types.SimpleNamespace(layers = [layer])+ name = (+ "cumulative_length_int"+ if "cumulative_length_int" in layer.__dict__+ else "cumulative_length"+ )+ layer.__dict__[name] = 3+ assert GG._window_reason(cache, 1) is None+ # Full (rolls every step): replayable, keyed apart from the filling state.+ layer.__dict__[name] = 4+ assert GG._window_reason(cache, 1) is None+ assert GG._window_state(cache) == (True,)+ # A multi-token step that fills the window switches branches mid-step.+ layer.__dict__[name] = 3+ assert GG._window_reason(cache, 2) == "sliding window filling"+ assert GG._window_state(cache) == (False,)+ layer.__dict__[name] = 4+ before = GG._host_state(cache)+ layer.__dict__[name] = 5+ assert GG._host_state_reason(before, GG._host_state(cache), 1) is None+ layer.__dict__[name] = 7+ assert GG._host_state_reason(before, GG._host_state(cache), 1) == f"cache host state {name}"+ layer.__dict__[name] = 5+ layer.other = "changed"+ assert GG._host_state_reason(before, GG._host_state(cache), 1) == "cache host state other"+ snap = GG._snapshot(cache)+ GG._advance(cache, 1)+ assert layer.__dict__[name] == 6+ GG._restore(snap)+ assert layer.__dict__[name] == 5+++def test_argument_keys(host):+ cache = object()+ a, b = [], []+ x = torch.zeros(2, 1, dtype = torch.long)+ k1 = GG._spec({"ids": x, "mask": {"full": torch.zeros(2, 1, 1, 8)}, "n": 1}, a, cache)+ k2 = GG._spec({"ids": x, "mask": {"full": torch.zeros(2, 1, 1, 9)}, "n": 1}, b, cache)+ assert k1 != k2 and len(a) == len(b) == 2+ assert GG._spec({"ids": x.float()}, [], cache) != GG._spec({"ids": x}, [], cache)+ assert GG._spec({"n": 2}, [], cache) != GG._spec({"n": 1}, [], cache)+ assert GG._spec(cache, [], cache) == ("C",)+ for bad, why in (+ (object(), "argument object"),+ (torch.zeros(3, requires_grad = True), "tensor argument requires grad"),+ (torch.zeros(3, device = "meta"), "tensor argument on meta"),+ (torch.zeros(5 * 1024 * 1024), "large tensor argument"),+ ):+ with pytest.raises(GG._Refuse, match = why):+ GG._spec(bad, [], cache)+ leaves = []+ tree = {"ids": x, "mask": {"full": torch.ones(2)}, "n": 1}+ GG._spec(tree, leaves, cache)+ rebuilt = GG._build(tree, iter([t + 1 for t in leaves]), cache)+ assert torch.equal(rebuilt["ids"], x + 1) and rebuilt["n"] == 1+++def test_wanted_gating(monkeypatch):+ model = types.SimpleNamespace(device = torch.device("cuda", 0))+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ monkeypatch.delenv("UNSLOTH_DECODE_GENERIC_GRAPHS", raising = False)+ monkeypatch.setattr(G, "ENABLED", True)+ assert GG.wanted(model)+ monkeypatch.setenv("UNSLOTH_DECODE_GENERIC_GRAPHS", "0")+ assert not GG.wanted(model)+ monkeypatch.delenv("UNSLOTH_DECODE_GENERIC_GRAPHS")+ monkeypatch.setattr(G, "ENABLED", False)+ assert not GG.wanted(model)+ monkeypatch.setattr(G, "ENABLED", None)+ monkeypatch.setenv("UNSLOTH_DECODE_CUDA_GRAPHS", "0")+ with G.rl_generation():+ assert not GG.wanted(model)+ monkeypatch.setenv("UNSLOTH_DECODE_CUDA_GRAPHS", "1")+ assert GG.wanted(model)+ monkeypatch.setattr(torch.version, "hip", "6.4", raising = False)+ assert not GG.wanted(model) # =1 does not force it on ROCm+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ model.forward = lambda **kw: None # an instance forward (accelerate hook) is left alone+ assert not GG.wanted(model)+ del model.forward+ model.device = torch.device("cpu")+ assert not GG.wanted(model)+++def test_inert_when_compiling(host):+ def forward(**kwargs):+ return kwargs["input_ids"] * 2++ model = torch.nn.Linear(1, 1)+ model.forward = forward+ scope = _scope(model, forward, False)++ def step(x):+ assert GG.decode_step(model, forward, {"input_ids": x}) is None+ assert not GG.wanted(model)+ return scope._forward(input_ids = x) + 1++ counters.clear()+ x = torch.ones(2, 1)+ assert torch.equal(torch.compile(step, backend = "eager", fullgraph = True)(x), x * 2 + 1)+ assert not counters["graph_break"]+ assert GG._ATTR not in model.__dict__+++def test_generate_routes_only_static_cache_calls():+ source = inspect.getsource(vision.unsloth_base_fast_generate)+ # plan() checks wanted() itself; only static-cache, non-compiled-decode calls reach it.+ call = "_generic_decode_graphs.plan(self, args, kwargs, input_ids)"+ assert call in source+ i = source.index(call)+ guard = source[source.rindex("\n", 0, source.rindex(" if ", 0, i)) : i]+ assert guard.count(" if ") == 1, guard+ for clause in (+ 'cache_implementation == "static"',+ "not force_dynamic_cache",+ "not compile_decode",+ ):+ assert clause in guard++[email protected](not has_real_cuda() or torch.version.hip is not None, reason = "NVIDIA CUDA")[email protected]("name", [LLAMA, GEMMA3])+def test_real_capture_matches_eager(name, monkeypatch):+ monkeypatch.setattr(G, "ENABLED", True)+ torch.manual_seed(0)+ # bf16 only where the GPU computes it natively: below sm80 Inductor skips a bf16 frame, which torch 2.14 turns+ # into a hard error inside zoo's fullgraph-compiled Gemma3 RMSNorm (and the skip sticks to that code object for+ # the rest of the process). Unsloth itself never runs Gemma3 in bf16 there (it forces float32).+ bf16 = torch.cuda.is_bf16_supported(including_emulation = False)+ model = (+ AutoModelForCausalLM.from_pretrained(name, dtype = torch.bfloat16 if bf16 else torch.float16)+ .cuda()+ .eval()+ )+ ids, mask = _ids(model, 2, pad = 1)+ ids, mask = ids.cuda(), mask.cuda()+ eager = _generate(model, ids, mask, False, 24)+ graphed = _generate(model, ids, mask, True, 24)+ s = GG.stats(model)+ assert torch.equal(eager, graphed), s+ assert s["captures"] == 1 and s["replays"] == 22 and not s["refusals"], s+++def test_grpo_hidden_states_wrapper_keeps_graphs_eligible():+ class M(torch.nn.Module):+ def forward(self, x):+ return x++ m = M()+ assert GG._plain_forward(m)+ m.forward = types.MethodType(M.forward, m) # the class forward re-bound on the instance+ assert GG._plain_forward(m)+ m.forward = types.MethodType(M.forward, M()) # bound to another module: not plain+ assert not GG._plain_forward(m)++ @functools.wraps(M.forward)+ def autocast_forward(self, *args, **kwargs): # accelerate's mixed-precision wrapper shape+ return M.forward(self, *args, **kwargs)++ m.forward = types.MethodType(autocast_forward, m)+ assert GG._plain_forward(m)+ m._hf_hook = object() # accelerate device hook on the module: stays eager+ assert not GG._plain_forward(m)+ del m._hf_hook++ def wrapped_forward(*args, **kwargs):+ return M.forward(m, *args, **kwargs)++ wrapped_forward._unsloth_grpo_hidden_states_forward_wrapped = True+ m.forward = wrapped_forward+ assert GG._plain_forward(m)+ m.forward = types.MethodType(wrapped_forward, m) # accelerate rebinds it as a bound method+ assert GG._plain_forward(m)+ m.forward = lambda *a, **k: None # e.g. an accelerate device hook: stays eager+ assert not GG._plain_forward(m)++ # The decode scope puts back whatever instance forward was there before (none: none).+ m = M()+ with vision._EagerDecodeSteps(m):+ assert "forward" in m.__dict__+ assert "forward" not in m.__dict__+ m.forward = types.MethodType(autocast_forward, m)+ previous = m.__dict__["forward"]+ with vision._EagerDecodeSteps(m):+ assert m.__dict__["forward"] is not previous+ assert m.__dict__["forward"] is previous+++def test_rolling_update_matches_transformers_rows(host):+ classes = GG._rolling_update_classes()+ assert classes, NO_ROLL+ for cls, counter in classes:+ torch.manual_seed(0)+ layers = [_sliding_layer(cls), _sliding_layer(cls)]+ prefill = torch.randn(2, 3, 6, 8), torch.randn(2, 3, 6, 8)+ steps = [(torch.randn(2, 3, 1, 8), torch.randn(2, 3, 1, 8)) for _ in range(5)]+ for layer in layers:+ layer.update(*prefill, {"cache_position": torch.arange(6)})+ for i, (k, v) in enumerate(steps):+ step_kwargs = {"cache_position": torch.tensor([6 + i])}+ a = layers[0].update(k, v, step_kwargs)+ with GG._graphable_rolls():+ b = layers[1].update(k, v, step_kwargs)+ assert torch.equal(a[0], b[0]) and torch.equal(a[1], b[1])+ assert layers[0].__dict__[counter] == layers[1].__dict__[counter]+++def test_full_window_snapshot_keeps_slot_zero_and_undoes_the_roll(host):+ # Validation replays a step that rolls every full window, then puts the rows back for the eager step: keeping+ # the slot the roll drops is enough, not a clone of every window (bsz x 116 MiB on Gemma-3-4B).+ classes = GG._rolling_update_classes()+ assert classes, NO_ROLL+ for cls, counter in classes:+ for graphable in (False, True):+ torch.manual_seed(0)+ layer = _sliding_layer(cls)+ layer.update(+ torch.randn(2, 3, 6, 8),+ torch.randn(2, 3, 6, 8),+ {"cache_position": torch.arange(6)},+ )+ cache = types.SimpleNamespace(layers = [layer])+ for i in range(3):+ keys, values = layer.keys.clone(), layer.values.clone()+ count = layer.__dict__[counter]+ snapshot = GG._snapshot(cache)+ rows = [e[1] for e in snapshot[1] if e[0] is layer.keys or e[0] is layer.values]+ assert len(rows) == 2 and all(r.shape[-2] == 1 for r in rows), [+ r.shape for r in rows+ ]+ step = (+ torch.randn(2, 3, 1, 8),+ torch.randn(2, 3, 1, 8),+ {"cache_position": torch.tensor([6 + i])},+ )+ with GG._graphable_rolls() if graphable else contextlib.nullcontext():+ layer.update(*step)+ assert not torch.equal(layer.keys, keys)+ GG._restore(snapshot)+ assert torch.equal(layer.keys, keys) and torch.equal(layer.values, values)+ assert layer.__dict__[counter] == count+ layer.update(*step)+ # Counters only: what a recording, which runs no device work, needs back.+ assert all(e[1].numel() == 1 for e in GG._snapshot(cache, rows = False)[1])+++def test_keys_changing_every_step_stop_capturing_for_the_call(host, llama, monkeypatch):+ # An argument whose shape or value moves every step gives every step a new key: each capture is an eager+ # warm-up plus a recording that is never replayed. After a few in a row the call decodes eagerly.+ ids, mask = _ids(llama)+ step = iter(range(10**6))+ real = GG._window_state+ monkeypatch.setattr(GG, "_window_state", lambda cache: real(cache) + (next(step),))+ s = _both(llama, ids, mask, 16)+ assert s["captures"] == GG._MAX_UNREPLAYED_CAPTURES and s["replays"] == 0, s+ assert s["graphs"] == 0, s+ assert s["refusals"] == {"graph key changes every step": 15 - GG._MAX_UNREPLAYED_CAPTURES}, s+ _generate(llama, ids, mask, True, 16)+ assert GG.stats(llama)["captures"] == 2 * GG._MAX_UNREPLAYED_CAPTURES+++def test_full_length_type_ids_from_a_trainer_still_replay(host, gemma3, monkeypatch):+ # TRL's GRPO passes the processor's mm_token_type_ids to generate(); it reached every decode step at full+ # length (one column longer per step), so every step had a new key and nothing replayed. With the step's+ # masks prebuilt (a dict) the forward never reads them: the graphed step gets the step's columns.+ real = type(gemma3).prepare_inputs_for_generation++ def prepare(self, input_ids, *args, **kwargs):+ out = real(self, input_ids, *args, **kwargs)+ out["mm_token_type_ids"] = torch.zeros(input_ids.shape, dtype = torch.long)+ return out++ monkeypatch.setattr(type(gemma3), "prepare_inputs_for_generation", prepare)+ seen = []+ real_spec = GG._spec++ def spec(value, leaves, cache):+ out = real_spec(value, leaves, cache)+ if isinstance(value, torch.Tensor) and value.dtype == torch.long and value.dim() == 2:+ seen.append(tuple(value.shape))+ return out++ monkeypatch.setattr(GG, "_spec", spec)+ new = 6+ s = _both(gemma3, *_ids(gemma3, 2, length = 3), new)+ assert s["captures"] == 1 and s["replays"] == new - 2 and not s["refusals"], s+ assert seen and all(shape[-1] == 1 for shape in seen), seen+++def test_step_type_ids_only_with_prebuilt_masks():+ t = torch.zeros(2, 9, dtype = torch.long)+ kwargs = {"mm_token_type_ids": t, "attention_mask": torch.ones(2, 9)}+ assert GG._step_type_ids(kwargs, 1) is kwargs+ kwargs = {+ "mm_token_type_ids": t,+ "token_type_ids": t,+ "attention_mask": {"full_attention": None},+ }+ out = GG._step_type_ids(kwargs, 1)+ assert out is not kwargs and kwargs["mm_token_type_ids"] is t+ for name in ("mm_token_type_ids", "token_type_ids"):+ assert out[name].shape == (2, 1) and out[name].stride() == (1, 1)+++def test_step_that_fills_the_window_is_never_captured(host, gemma3):+ # A first decode step at count == window - 1 fills the window. Its key is the filling one, but the capture+ # records after the warm-up advanced the counters (the rolling branch): a kept graph of that key would then+ # roll a window that is still filling. That step runs eagerly; the window-full key is captured after it.+ window = gemma3.config.sliding_window+ # A call past the window first: the roll index exists, so a capture of the rolling branch would succeed.+ _fresh(gemma3)+ _generate(gemma3, *_ids(gemma3, 2, length = 3), True, window + 4)+ assert GG._ROLL_INDEX+ new = 8+ s = _both(gemma3, *_ids(gemma3, 2, length = window - 1), new)+ assert s["refusals"] == {"sliding window fills in this step": 1}, s+ assert s["captures"] == 1 and s["validations"] == 1 and s["replays"] == new - 3, s+ state = gemma3.__dict__[GG._ATTR]+ assert all(all(key[2]) for key in state.validated), state.validateddiff --git a/tests/test_generic_decode_graphs_default.py b/tests/test_generic_decode_graphs_default.pynew file mode 100644index 00000000000..00693e6c815--- /dev/null+++ b/tests/test_generic_decode_graphs_default.py@@ -0,0 +1,517 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FastModel generate() decodes through CUDA graphs by default (generate_scope + plan()), on the host+through the stand-in backend of test_generic_decode_graphs: kill switches, the short-call rule, the+out-of-memory eager redo, the per-call plan the LoRA fold reads, the graph + StaticCache kept between+inference calls (released by training), and the BlockMask graph-argument contract."""++import functools+import os+import sys+import types++import pytest+import torch++import unsloth # noqa: F401 (must precede transformers)++import unsloth.models._decode_graphs as G+import unsloth.models._generic_decode_graphs as GG+from unsloth.models import vision++from transformers import AutoModelForCausalLM++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_generic_decode_graphs import LLAMA, _Host, _ids # noqa: E402++ENV = "UNSLOTH_DECODE_CUDA_GRAPHS"+++class _DefaultHost(_Host):+ available = staticmethod(lambda: True)+ free_bytes = staticmethod(lambda device: 1 << 40)+++def host(monkeypatch):+ backend = type("Host", (_DefaultHost,), {"graphs": []})+ monkeypatch.setattr(GG, "_BACKEND", backend)+ # The runtime default (NVIDIA CUDA, not Windows), not a programmatic override.+ monkeypatch.setattr(G, "ENABLED", None)+ monkeypatch.setattr(G, "_DEFAULT_OK", True)+ monkeypatch.setattr(G, "_STUDIO_DEFAULT_OK", True)+ for name in (ENV, "UNSLOTH_DECODE_GENERIC_GRAPHS", "UNSLOTH_DECODE_GENERIC_GRAPHS_KEEP"):+ monkeypatch.delenv(name, raising = False)+ return backend+++def llama():+ torch.manual_seed(0)+ model = AutoModelForCausalLM.from_pretrained(LLAMA, dtype = torch.float32).eval()+ model._old_generate = model.generate+ model.generate = types.MethodType(vision.unsloth_base_fast_generate, model)+ return model+++def _gen(+ model,+ new = 10,+ bsz = 1,+ **kwargs,+):+ ids, mask = _ids(model, bsz)+ return model.generate(+ input_ids = ids,+ attention_mask = mask,+ max_new_tokens = new,+ min_new_tokens = new,+ do_sample = False,+ pad_token_id = 0,+ **kwargs,+ )+++def _eager(+ model,+ monkeypatch,+ new = 10,+):+ monkeypatch.setenv(ENV, "0")+ out = _gen(model, new)+ monkeypatch.delenv(ENV)+ return out+++def _stats(model):+ return GG.stats(model) or {"captures": 0, "replays": 0, "graphs": 0}++[email protected]("name", [ENV, "UNSLOTH_DECODE_GENERIC_GRAPHS"])+def test_kill_switches_keep_generate_eager(host, llama, monkeypatch, name):+ monkeypatch.setenv(name, "0")+ _gen(llama)+ assert _stats(llama)["replays"] == 0 and _stats(llama)["captures"] == 0+++def test_short_calls_decode_eagerly(host, llama):+ before = GG.PLANS["short call"]+ _gen(llama, new = 3)+ assert GG.PLANS["short call"] == before + 1+ assert _stats(llama)["captures"] == 0+++def test_call_plan_follows_the_call(host, llama, monkeypatch):+ seen = []+ real = llama._old_generate++ def recording(*args, **kwargs):+ seen.append(G.call_plan())+ return real(*args, **kwargs)++ llama._old_generate = recording+ assert G.call_plan() is None+ _gen(llama)+ _gen(llama, new = 2) # one decode step: never a graph, kept or not+ monkeypatch.setenv(ENV, "0")+ _gen(llama)+ assert seen == ["graph", "eager", "eager"]+ assert G.call_plan() is None+++def test_out_of_memory_redoes_the_call_eagerly_without_the_wrapper(host, llama, monkeypatch):+ eager = _eager(llama, monkeypatch)+ calls = []+ real = GG.decode_step++ def oom_once(*args, **kwargs):+ calls.append(1)+ if len(calls) == 2:+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (test)")+ return real(*args, **kwargs)++ monkeypatch.setattr(GG, "decode_step", oom_once)+ # The redo runs after the handler: its traceback would keep the failed call's cache alive.+ handling = []+ real_redo = GG.redo_eagerly++ def redo(model):+ handling.append(sys.exc_info()[0])+ return real_redo(model)++ monkeypatch.setattr(GG, "redo_eagerly", redo)+ redos = GG.EAGER_REDOS+ with pytest.warns(UserWarning, match = "out of GPU memory"):+ out = _gen(llama)+ assert torch.equal(out, eager)+ assert GG.EAGER_REDOS == redos + 1+ assert handling == [None]+ # The redo ran without _GraphDecodeSteps: decode_step saw only the failed attempt's steps.+ assert len(calls) == 2+ assert "forward" not in llama.__dict__+ assert GG.kept_cache(llama) is None+++def test_out_of_memory_with_a_streamer_reraises(host, llama, monkeypatch):+ def oom(*args, **kwargs):+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (test)")++ monkeypatch.setattr(GG, "decode_step", oom)+ streamer = types.SimpleNamespace(put = lambda value: None, end = lambda: None)+ with pytest.raises(torch.cuda.OutOfMemoryError):+ _gen(llama, streamer = streamer)+ assert "forward" not in llama.__dict__+++def test_kept_graph_replays_the_next_call_and_training_releases_it(host, llama, monkeypatch):+ eager = _eager(llama, monkeypatch)+ assert _stats(llama)["replays"] == 0+ # Plain generate() graphs by default.+ first = _gen(llama)+ s1 = _stats(llama)+ assert s1["captures"] == 1 and s1["graphs"] == 1 and s1["replays"] >= 8, s1+ assert "forward" not in llama.__dict__+ kept = GG.kept_cache(llama)+ assert kept is not None+ second = _gen(llama)+ s2 = _stats(llama)+ # Same cache tensors, same graph: no capture, every decode step replays (from the first one).+ assert s2["captures"] == 1, s2+ assert s2["replays"] - s1["replays"] == 9, (s1, s2)+ assert GG.kept_cache(llama) is kept+ assert torch.equal(first, eager) and torch.equal(second, eager)+ _gen(llama, new = 4)+ assert _stats(llama)["captures"] == 1+ # Training forward without for_training() frees the graph and the cache.+ llama.train()+ llama(input_ids = _ids(llama)[0])+ assert GG.kept_cache(llama) is None and _stats(llama)["graphs"] == 0+ assert "_cache" not in llama.__dict__+ llama.eval()+++def test_kept_graph_hook_is_inert_in_a_compiled_training_forward(host, llama):+ # A kept graph leaves a forward pre-hook that frees it when training resumes. Dynamo tracing a compiled+ # training forward must not run it (removing a hook while tracing is a graph break, an error under fullgraph);+ # the next eager training forward still frees the graph.+ _gen(llama)+ state = llama.__dict__[GG._ATTR]+ assert GG.kept_cache(llama) is not None and state.hook is not None+ torch._dynamo.reset()+ llama.train()+ try:+ ids, _ = _ids(llama)+ compiled = torch.compile(+ lambda x: llama(input_ids = x).logits.float().sum(), fullgraph = True, backend = "eager"+ )+ compiled(ids).backward()+ assert GG.kept_cache(llama) is not None+ llama.zero_grad(set_to_none = True)+ llama(input_ids = ids)+ assert GG.kept_cache(llama) is None and state.hook is None+ finally:+ llama.eval()+ llama.zero_grad(set_to_none = True)+ torch._dynamo.reset()+++def test_a_short_call_after_a_long_one_replays_the_kept_graph(host, llama, monkeypatch):+ # transformers 5.x sizes every later cache to the longest call so far (_previous_max_cache_length), so a short+ # call after a long one gets a cache as long as the kept one: it must reuse it, not re-capture every call.+ _gen(llama, new = 40)+ s1 = _stats(llama)+ assert s1["captures"] == 1 and GG.kept_cache(llama) is not None, s1+ monkeypatch.setattr(+ llama, "_previous_max_cache_length", GG._kept_len(llama.__dict__[GG._ATTR]), raising = False+ )+ eager = _eager(llama, monkeypatch)+ # The eager call dropped the kept graph; capture again, then short calls reuse it.+ _gen(llama, new = 40)+ s2 = _stats(llama)+ for _ in range(2):+ assert torch.equal(_gen(llama, new = 10), eager)+ s3 = _stats(llama)+ assert s3["captures"] == s2["captures"], (s2, s3)+ assert s3["replays"] - s2["replays"] == 2 * 9, (s2, s3)+++def test_other_batch_or_returned_cache_drops_the_kept_graph(host, llama):+ _gen(llama)+ assert GG.kept_cache(llama) is not None+ _gen(llama, bsz = 2)+ assert _stats(llama)["captures"] == 2, _stats(llama)+ # A returned cache belongs to the caller: never kept, never reset by a later call.+ out = _gen(llama, return_dict_in_generate = True)+ assert out.past_key_values is not None+ assert GG.kept_cache(llama) is None+++def test_weights_changed_and_release_for_training_drop_the_kept_graph(host, llama):+ _gen(llama)+ assert _stats(llama)["captures"] == 1+ G.weights_changed()+ _gen(llama)+ assert _stats(llama)["captures"] == 2+ assert GG.kept_cache(llama) is not None+ G.release_for_training(llama)+ assert GG.kept_cache(llama) is None and "_cache" not in llama.__dict__+++def _block_mask(offset, contract = True):+ from torch.nn.attention.flex_attention import BlockMask, _DEFAULT_SPARSE_BLOCK_SIZE as BLOCK++ def rebuild(tensors):+ (off,) = tensors+ return lambda b, h, q_idx, kv_idx: q_idx + off >= kv_idx++ n = 2+ mask = BlockMask.from_kv_blocks(+ torch.ones(1, 1, 1, dtype = torch.int32),+ torch.arange(n, dtype = torch.int32).view(1, 1, 1, n),+ BLOCK_SIZE = BLOCK,+ mask_mod = rebuild((offset,)),+ seq_lengths = (1, n * BLOCK),+ )+ if contract:+ mask._unsloth_graph_closure = ((offset,), rebuild)+ return mask+++def test_block_mask_contract_round_trip(host):+ offset = torch.tensor(3)+ mask = _block_mask(offset)+ leaves = []+ spec = GG._spec({"full_attention": mask}, leaves, None)+ assert GG._spec({"full_attention": _block_mask(torch.tensor(7))}, [], None) == spec+ assert any(leaf is offset for leaf in leaves)++ def at(t):+ return next(i for i, leaf in enumerate(leaves) if leaf is t)++ buffers = [leaf.clone() for leaf in leaves]+ built = GG._build({"full_attention": mask}, iter(buffers), None)["full_attention"]+ assert built is not mask and built.kv_num_blocks is buffers[at(mask.kv_num_blocks)]+ static = built._unsloth_graph_closure[0][0]+ assert static is buffers[at(offset)]+ q, k = torch.zeros((), dtype = torch.long), torch.tensor(5)+ assert not bool(built.mask_mod(0, 0, q, k))+ static.fill_(5) # a refill reaches the rebuilt mask_mod+ assert bool(built.mask_mod(0, 0, q, k))+ with pytest.raises(GG._Refuse, match = "argument BlockMask"):+ GG._spec(_block_mask(torch.tensor(3), contract = False), [], None)+++class _Muse(torch.nn.Module):+ """A Muse-Glimmer-like decoder: the embedding is trainable and may answer on another device."""++ def __init__(+ self,+ answer_on = None,+ model_type = "muse_glimmer",+ ):+ super().__init__()+ self.config = types.SimpleNamespace(+ model_type = model_type, image_token_id = 5, video_token_id = 6+ )+ self.embed = torch.nn.Embedding(10, 4)+ self.answer_on = answer_on++ def get_input_embeddings(self):+ if self.answer_on is None:+ return self.embed+ return lambda ids: torch.zeros(*ids.shape, 4, device = self.answer_on)+++def test_placeholder_masking_models_embed_outside_the_graph(host):+ """Muse-Glimmer / Muse-Spark / Kimi-K2.5 zero placeholder ids with ids[mask] = 0 (a boolean index_put_, a+ host sync): their decode step gets the same embeddings computed before the graph, from the same ops, never+ tracking grad (the embedding is trainable) so they pass the argument check."""+ ids = torch.tensor([[5], [3], [6]])+ model = _Muse()+ assert model.embed.weight.requires_grad+ with torch.enable_grad():+ out = GG._placeholder_embeds(+ model, {"input_ids": ids, "inputs_embeds": None, "use_cache": True}+ )+ llm = ids.clone()+ llm[(ids == 5) | (ids == 6)] = 0 # what the model's forward does+ assert out["input_ids"] is None and out["use_cache"] is True+ embeds = out["inputs_embeds"]+ assert torch.equal(embeds, model.embed(llm)) and not embeds.requires_grad+ leaves = []+ GG._spec(out, leaves, None) # not "tensor argument requires grad"+ assert any(leaf is embeds for leaf in leaves)+ # Other models, media steps and an explicit inputs_embeds are left alone.+ kwargs = {"input_ids": ids}+ assert GG._placeholder_embeds(_Muse(model_type = "gemma4"), kwargs) is kwargs+ media = {"input_ids": ids, "pixel_values": torch.ones(1)}+ assert GG._placeholder_embeds(model, media) is media+ # The lookup answers on the host while the ids live elsewhere (meta stands in for the card).+ ids = torch.tensor([[5], [3]], device = "meta")+ out = GG._placeholder_embeds(_Muse(answer_on = "cpu"), {"input_ids": ids})+ assert out["inputs_embeds"].device == ids.device+++def _hooked(model, hook):+ """What accelerate's add_hook_to_module leaves on a module (without moving its parameters)."""++ def new_forward(module, *args, **kwargs):+ args, kwargs = module._hf_hook.pre_forward(module, *args, **kwargs)+ return module._hf_hook.post_forward(module, module._old_forward(*args, **kwargs))++ old = model.forward+ model._old_forward = old+ model._hf_hook = hook+ model.forward = functools.update_wrapper(functools.partial(new_forward, model), old)+ return model+++def test_a_single_device_accelerate_hook_keeps_graphs_eligible(monkeypatch):+ """A bitsandbytes model placed on cuda:1 gets accelerate's AlignDevicesHook (force_hooks, vision.py+ _attach_bnb_multidevice_hooks): an instance forward plus _hf_hook, which wanted() used to refuse, so+ no decode step of a model on a second card was ever graphed."""+ from accelerate.hooks import AlignDevicesHook, ModelHook++ class OnCard1(torch.nn.Linear):+ @property+ def device(self):+ return torch.device("cuda", 1)++ def wanted(hook):+ return GG.wanted(_hooked(OnCard1(2, 2), hook))++ monkeypatch.setattr(G, "ENABLED", True)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ monkeypatch.delenv("UNSLOTH_DECODE_GENERIC_GRAPHS", raising = False)+ assert wanted(AlignDevicesHook(execution_device = 1, io_same_device = True))+ assert wanted(AlignDevicesHook(execution_device = "cuda:1"))+ # Hooks that move weights or run elsewhere stay eager.+ assert not wanted(AlignDevicesHook(execution_device = 0))+ assert not wanted(AlignDevicesHook(execution_device = 1, offload = True))+ assert not wanted(ModelHook())+++def test_generate_through_a_single_device_hook_uses_graphs(host, llama, monkeypatch):+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module++ eager = _eager(llama, monkeypatch)+ add_hook_to_module(llama, AlignDevicesHook(execution_device = "cpu", io_same_device = True))+ hooked = llama.__dict__["forward"]+ graphed = _gen(llama)+ s = _stats(llama)+ assert s["captures"] == 1 and s["replays"] >= 8, s+ assert torch.equal(eager, graphed)+ assert llama.__dict__["forward"] is hooked+++def test_refusal_reasons_name_the_argument(host, llama, monkeypatch):+ real = GG._mrope_positions++ def with_grad_tensor(model, kwargs):+ return {**real(model, kwargs), "extra": torch.zeros(1, requires_grad = True)}++ monkeypatch.setattr(GG, "_mrope_positions", with_grad_tensor)+ cls_forward = type(llama).forward++ @functools.wraps(cls_forward)+ def forward(+ self,+ *a,+ extra = None,+ **k,+ ):+ return cls_forward(self, *a, **k)++ monkeypatch.setattr(type(llama), "forward", forward)+ _gen(llama)+ assert _stats(llama)["refusals"] == {"tensor argument requires grad (extra)": 9}, _stats(llama)+++class _HostEmbedding(torch.nn.Module):+ """Unsloth's offloaded input embedding as the decode step sees it: the weight lives off the compute device+ (meta stands in for host RAM) behind hooks that return the rows on the decoder's device. Records whether+ each lookup ran inside a graph capture or replay."""++ def __init__(self, real, inside):+ super().__init__()+ self.weight = torch.nn.Parameter(+ torch.empty_like(real.weight, device = "meta"), requires_grad = False+ )+ self.table = real.weight.detach().clone() # a plain attribute: the lookup's data+ self.inside = inside+ self.calls = []+ self._unsloth_offload_hooks_installed = True++ def forward(self, ids):+ self.calls.append(self.inside[0])+ return torch.nn.functional.embedding(ids, self.table)+++def _offload(+ host,+ llama,+ monkeypatch,+ extra = False,+):+ inside = [False]+ record = host.record.__func__++ def tracked(fn):+ def run():+ inside[0] = True+ try:+ return fn()+ finally:+ inside[0] = False++ return run++ monkeypatch.setattr(+ host,+ "record",+ classmethod(lambda cls, pool, fn, cache: record(cls, pool, tracked(fn), cache)),+ )+ embedding = _HostEmbedding(llama.get_input_embeddings(), inside)+ llama.set_input_embeddings(embedding)+ # As `_pin_device_to_decoder`: model.device is the decoder's, not the offloaded table's.+ monkeypatch.setattr(type(llama), "device", property(lambda self: torch.device("cpu")))+ if extra:+ llama.model.norm.register_parameter(+ "stray", torch.nn.Parameter(torch.zeros(1, device = "meta"), requires_grad = False)+ )+ return embedding+++def test_offloaded_input_embedding_is_looked_up_outside_the_graph(host, llama, monkeypatch):+ eager = _eager(llama, monkeypatch)+ embedding = _offload(host, llama, monkeypatch)+ graphed = _gen(llama)+ s = _stats(llama)+ assert s["captures"] == 1 and s["replays"] >= 8, s+ assert torch.equal(eager, graphed)+ # Every decode step looked its rows up eagerly; none inside a capture or replay.+ assert embedding.calls and not any(embedding.calls)+++def test_offloaded_input_embedding_swap_recaptures_and_unhooked_is_refused(+ host, llama, monkeypatch+):+ embedding = _offload(host, llama, monkeypatch)+ first = GG._fingerprint(llama)+ assert first is not None+ embedding.weight = torch.nn.Parameter(torch.empty_like(embedding.weight), requires_grad = False)+ assert GG._fingerprint(llama) not in (None, first)+ # An off-device embedding without Unsloth's offload hooks stays refused.+ embedding._unsloth_offload_hooks_installed = False+ assert GG._offloaded_input_embedding(llama) is None+ assert GG._fingerprint(llama) is None+++def test_another_offloaded_parameter_is_still_refused(host, llama, monkeypatch):+ _offload(host, llama, monkeypatch, extra = True)+ assert GG._fingerprint(llama) is None+ _gen(llama)+ s = _stats(llama)+ assert s["replays"] == 0 and "not one CUDA device / adapters" in s["refusals"], sdiff --git a/tests/test_generic_decode_graphs_multidevice.py b/tests/test_generic_decode_graphs_multidevice.pynew file mode 100644index 00000000000..9f8ade49cf7--- /dev/null+++ b/tests/test_generic_decode_graphs_multidevice.py@@ -0,0 +1,460 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FastModel decode CUDA graphs for a model split over cards (device_map / Unsloth's layer split):+one graph holds every card's kernels. On the host through the stand-in backend of+test_generic_decode_graphs with the parameter devices faked as two cards; the real two-card capture+is the gpu-marked test."""++import contextlib+import os+import sys++import pytest+import torch++import unsloth # noqa: F401 (must precede transformers)++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+import test_generic_decode_graphs as T # noqa: E402+import unsloth.models._generic_decode_graphs as GG+from real_accelerator import has_real_cuda++from transformers import AutoModelForCausalLM+++class _Stream(T._FakeStream):+ def __init__(self, log, name):+ self.log, self.name = log, name++ def wait_stream(self, other):+ self.log.append((self.name, "waits", getattr(other, "name", "?")))+++class _TwoCards(T._Host):+ """Host stand-in for a model split over cards 0 and 1 (this thread on card 0)."""++ others_recorded = []+ supported = True+ log = []+ current_device = staticmethod(lambda: 0)+ mempool = staticmethod(lambda: object())++ @classmethod+ def multidevice_supported(cls):+ return cls.supported++ @classmethod+ def current_stream(cls):+ return _Stream(cls.log, "card0")++ @classmethod+ def device_stream(cls, index):+ return _Stream(cls.log, f"card{index}")++ @classmethod+ def side_stream(cls, index):+ return _Stream(cls.log, f"side{index}")++ @classmethod+ @contextlib.contextmanager+ def on_streams(cls, streams):+ cls.log.append(("on_streams", tuple(sorted(streams))))+ yield++ @classmethod+ def record(cls, pool, fn, cache, *side):+ others, mempools = side or ({}, {})+ cls.others_recorded.append((tuple(sorted(others)), sorted(mempools)))+ return super().record(pool, fn, cache)+++def two_cards(monkeypatch):+ backend = type(+ "TwoCards",+ (_TwoCards,),+ {"graphs": [], "others_recorded": [], "log": [], "supported": True},+ )+ monkeypatch.setattr(GG, "_BACKEND", backend)+ # Parameters on "card 0" and "card 1" (host tensors: the stand-in backend runs on the CPU).+ monkeypatch.setattr(+ GG, "_param_devices", lambda model: {torch.device("cpu", 0), torch.device("cpu", 1)}+ )+ monkeypatch.delenv("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", raising = False)+ return backend+++def llama():+ torch.manual_seed(0)+ return AutoModelForCausalLM.from_pretrained(T.LLAMA, dtype = torch.float32).eval()+++def test_split_model_is_graphed_as_one_multi_card_graph(two_cards, llama):+ hooks = pytest.importorskip("accelerate.hooks")+ # As dispatch_model leaves a split model: an AlignDevicesHook on every decoder layer.+ for layer in llama.model.layers:+ hooks.add_hook_to_module(layer, hooks.AlignDevicesHook(execution_device = "cpu"))+ s = T._both(llama, *T._ids(llama, 2, pad = 1))+ assert s["captures"] == 1 and s["replays"] == 8 and s["refusals"] == {}, s+ # Card 1 joined with its own MemPool, and on its side stream for the warm-up.+ assert two_cards.others_recorded == [((1,), [1])]+ assert ("on_streams", (1,)) in two_cards.log+ # Every replay waits on card 1's stream first, and card 1 waits on the replay after.+ assert ("card0", "waits", "card1") in two_cards.log+ assert ("card1", "waits", "card0") in two_cards.log+++ "setup, reason",+ [+ ("kill", "multi-device (UNSLOTH_DECODE_MULTIDEVICE_GRAPHS=0)"),+ ("old_torch", "multi-device capture needs a newer torch"),+ ("offload", "multi-device with offload or non-align hooks"),+ ("other_hook", "multi-device with offload or non-align hooks"),+ ],+)+def test_split_model_refusals_decode_eagerly(two_cards, llama, monkeypatch, setup, reason):+ accelerate_hooks = pytest.importorskip("accelerate.hooks")+ if setup == "kill":+ monkeypatch.setenv("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", "0")+ elif setup == "old_torch":+ two_cards.supported = False+ elif setup == "offload":+ hook = accelerate_hooks.AlignDevicesHook(execution_device = "cpu")+ hook.offload = True+ llama.model.layers[0].__dict__["_hf_hook"] = hook+ else:+ llama.model.layers[0].__dict__["_hf_hook"] = accelerate_hooks.ModelHook()+ s = T._both(llama, *T._ids(llama, 1))+ assert s["captures"] == 0 and s["replays"] == 0, s+ assert set(s["refusals"]) == {reason}, s+ assert two_cards.others_recorded == []+++def test_single_card_model_records_without_other_cards(two_cards, llama, monkeypatch):+ monkeypatch.setattr(GG, "_param_devices", lambda model: {torch.device("cpu", 0)})+ s = T._both(llama, *T._ids(llama, 1))+ assert s["captures"] == 1 and s["refusals"] == {}, s+ # The single-card record call is unchanged (no other cards, no MemPools).+ assert two_cards.others_recorded == [((), [])]+ assert not any("card1" in entry for entry in two_cards.log), two_cards.log+++ not has_real_cuda() or torch.cuda.device_count() < 2, reason = "needs two CUDA GPUs"+)+def test_real_two_card_capture_matches_eager():+ """A tiny Llama split over cuda:0 / cuda:1 with accelerate's dispatch: one multi-card graph,+ tokens and logits equal to the eager step."""+ from accelerate import dispatch_model++ torch.manual_seed(0)+ model = AutoModelForCausalLM.from_pretrained(T.LLAMA, dtype = torch.float16).eval()+ n = len(model.model.layers)+ device_map = {"model.embed_tokens": 0, "model.rotary_emb": 0, "model.norm": 1, "lm_head": 1}+ for i in range(n):+ device_map[f"model.layers.{i}"] = 0 if i < n // 2 else 1+ dispatch_model(model, device_map = device_map, main_device = 0, force_hooks = True)+ ids, mask = T._ids(model, 2, pad = 1)+ ids, mask = ids.cuda(0), mask.cuda(0)+ eager = T._generate(model, ids, mask, False, new = 24)+ graphed = T._generate(model, ids, mask, True, new = 24)+ assert torch.equal(eager, graphed)+ s = GG.stats(model)+ assert s["captures"] >= 1 and s["replays"] > 0, s+++class _Seg:+ def __init__(self, log, n):+ self.log, self.n = log, n++ def replay(self):+ self.log.append(("replay", self.n))+++class _HostCards:+ """Two host "cards" for the segment recorder: card 0 = cpu, card 1 = meta (copies cpu -> meta work)."""++ device_type = "cpu"++ def __init__(self):+ self.log = []+ self.n = 0++ @staticmethod+ def card_of(value):+ return {"cpu": 0, "meta": 1}.get(value.device.type)++ @staticmethod+ def index_of(device):+ if isinstance(device, int):+ return device+ return {"cpu": 0, "meta": 1}.get(torch.device(device).type)++ @staticmethod+ def card_device(index):+ return torch.device(("cpu", "meta")[index])++ current_device = staticmethod(lambda: 0)+ device = staticmethod(lambda index: contextlib.nullcontext())++ def new_graph(self):+ self.n += 1+ return _Seg(self.log, self.n)++ def begin_on(self, graph, pool, card):+ self.log.append(("begin", graph.n, card, pool))++ def end_on(self, graph):+ self.log.append(("end", graph.n))+++def _send_to_device(value, device, *args, **kwargs):+ return value.to(device)+++class _Block(torch.nn.Module):+ def __init__(self, device):+ super().__init__()+ self.lin = torch.nn.Linear(4, 4, device = device)++ def forward(self, x, extra):+ return self.lin(x) + extra+++def test_segment_recorder_cuts_at_the_card_boundary_and_moves_shared_inputs_once(monkeypatch):+ hooks = pytest.importorskip("accelerate.hooks")+ cards = _HostCards()+ monkeypatch.setattr(GG, "_BACKEND", cards)+ torch.manual_seed(0)+ first = _Block("cpu")+ second, third = _Block("meta"), _Block("meta")+ for block in (second, third):+ hooks.add_hook_to_module(block, hooks.AlignDevicesHook(execution_device = "meta"))+ x = torch.randn(2, 4)+ extra = torch.randn(2, 4) # a layer-invariant input on card 0 (rotary cos / sin, a mask)++ def step():+ h = first(x, extra = extra)+ h = second(h, extra = extra)+ return third(h, extra = extra)++ originals = (+ hooks.send_to_device,+ hooks.AlignDevicesHook.pre_forward,+ hooks.AlignDevicesHook.post_forward,+ )+ segments, out = GG._SegmentRecorder(cards, {0: "pool0", 1: "pool1"}, 0).run(step)+ assert out.device.type == "meta"+ kinds = [+ (p[0], p[2] if p[0] == "graph" else (p[2].device.type, p[1].device.type))+ for p in segments.parts+ ]+ # Card 0's segment, the hidden state and the shared input cross once, then one card-1 segment+ # for both card-1 blocks (the second block's inputs are already there: no new cut).+ assert kinds == [+ ("graph", 0),+ ("copy", ("cpu", "meta")),+ ("copy", ("cpu", "meta")),+ ("graph", 1),+ ], kinds+ assert cards.log == [+ ("begin", 1, 0, "pool0"),+ ("end", 1),+ ("begin", 2, 1, "pool1"),+ ("end", 2),+ ], cards.log+ assert (+ hooks.send_to_device,+ hooks.AlignDevicesHook.pre_forward,+ hooks.AlignDevicesHook.post_forward,+ ) == originals+ assert second(x.to("meta"), extra = extra).device.type == "meta"+++def test_segments_replay_graphs_and_hand_offs_in_capture_order(monkeypatch):+ monkeypatch.setattr(GG, "_BACKEND", _HostCards())+ src, dst = torch.zeros(3), torch.zeros(3)+ log = []++ class Writes:+ def replay(self):+ log.append("card0")+ src.fill_(3)++ class Reads:+ def replay(self):+ log.append(("card1 sees", dst.tolist()))++ segments = GG._Segments()+ segments.parts = [("graph", Writes(), 0), ("copy", dst, src), ("graph", Reads(), 1)]+ segments.replay()+ assert log == ["card0", ("card1 sees", [3.0, 3.0, 3.0])]+++def test_segment_recorder_leaves_movable_objects_to_accelerate(monkeypatch):+ # A non-tensor with .to (BatchEncoding, ...) goes through accelerate's own send_to_device, as eager.+ cards = _HostCards()+ monkeypatch.setattr(GG, "_BACKEND", cards)+ seen = []++ class Movable:+ def to(self, device, *args, **kwargs):+ seen.append(device)+ return self++ recorder = GG._SegmentRecorder(cards, {0: "pool0", 1: "pool1"}, 0)+ recorder.original = _send_to_device+ item = Movable()+ assert recorder.send(_send_to_device, {"a": item}, "meta")["a"] is item+ assert (+ seen == [torch.device("meta")]+ and recorder.segments.parts == []+ and recorder.pending is None+ )+++def test_split_model_root_hook_on_the_first_blocks_card_is_plain(monkeypatch):+ # 16-bit models with tied embeddings: Unsloth's split puts the tied embedding / lm_head (the first+ # parameter, so model.device) on cuda:1 while the root hook moves inputs to cuda:0.+ from accelerate.hooks import AlignDevicesHook++ class Root:+ device = torch.device("cuda", 1)++ root = Root()+ root.__dict__["_hf_hook"] = AlignDevicesHook(+ execution_device = torch.device("cuda", 0), io_same_device = True+ )+ cards = {torch.device("cuda", 0), torch.device("cuda", 1)}+ monkeypatch.setattr(GG, "_param_devices", lambda model: cards)+ monkeypatch.setenv("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", "1")+ assert GG._single_device_hook(root)+ monkeypatch.setenv("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", "0")+ assert not GG._single_device_hook(root)+ monkeypatch.setenv("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", "1")+ # A card the model is not on stays refused; a model on one card keeps the exact-device rule.+ root.__dict__["_hf_hook"] = AlignDevicesHook(execution_device = torch.device("cuda", 2))+ assert not GG._single_device_hook(root)+ monkeypatch.setattr(GG, "_param_devices", lambda model: {torch.device("cuda", 1)})+ root.__dict__["_hf_hook"] = AlignDevicesHook(execution_device = torch.device("cuda", 0))+ assert not GG._single_device_hook(root)+++def _graph_block_mask(device = "cpu"):+ # A BlockMask publishing the graph contract as zoo's gpt-oss flex mask does: mask_mod reads an offset+ # and padding rows through `rebuild`, and the builder's own per-device rebuild sits on the mask.+ from torch.nn.attention.flex_attention import BlockMask++ offset = torch.full((), 3, dtype = torch.int64, device = device)+ pad = torch.ones(1, 128, dtype = torch.bool, device = device)++ def rebuild(ts):+ off, rows = ts++ def mask_mod(b, h, q_idx, kv_idx):+ return (kv_idx + off <= q_idx + off) & rows[b, kv_idx]++ mask_mod.reads = ts+ return mask_mod++ bm = BlockMask.from_kv_blocks(+ torch.ones(1, 1, 1, dtype = torch.int32, device = device),+ torch.zeros(1, 1, 1, 1, dtype = torch.int32, device = device),+ BLOCK_SIZE = 128,+ mask_mod = rebuild((offset, pad)),+ seq_lengths = (1, 128),+ )+ bm._unsloth_graph_closure = ((offset, pad), rebuild)+ bm._unsloth_on_device = lambda target: pytest.fail(+ "the builder's rebuild reads the unrefilled tensors"+ )+ return bm+++def test_graph_block_mask_moves_its_closure_tensors_to_the_layer_card(monkeypatch):+ # Split model + flex decode: the static BlockMask is built on card 0 and accelerate moves it to card 1's+ # layers. BlockMask.to leaves mask_mod's tensors on card 0 (the gpt-oss "Tensor device mismatch ...+ # cuda:1 and cuda:0" at kv_idx + kv_o), so both the warm-up and the recording move them as well.+ pytest.importorskip("torch.nn.attention.flex_attention")+ cards = _HostCards()+ monkeypatch.setattr(GG, "_BACKEND", cards)+ value = _graph_block_mask()+ leaves, cache = [], object()+ GG._spec(value, leaves, cache)+ static = [leaf.clone() for leaf in leaves]+ built = GG._build(value, iter(static), cache)+ closure = built._unsloth_graph_closure[0]+ assert built.mask_mod.reads == closure and all(any(t is s for s in static) for t in closure)++ # Eager warm-up: accelerate's .to (zoo's patched BlockMask.to calls _unsloth_on_device).+ on_device = built.__dict__.get("_unsloth_on_device")+ assert on_device is not None+ moved = on_device("meta")+ assert all(t.device.type == "meta" for t in moved.mask_mod.reads)+ assert moved.kv_num_blocks.device.type == "meta"+ # Same card (the root hook, the first card's layers): the mask itself, which keeps its per-device move,+ # so the next card's hook still moves mask_mod's tensors (Kaggle T4x2: the root hook's same-card move+ # dropped it and the cuda:1 layers read the cuda:0 offsets).+ here = on_device("cpu")+ assert here is built+ later = here.__dict__["_unsloth_on_device"]("meta")+ assert all(t.device.type == "meta" for t in later.mask_mod.reads)+ assert moved.__dict__["_unsloth_on_device"]("meta") is moved++ # Recording: the fields and closure tensors cross as hand-offs (refilled every replay), once per card.+ recorder = GG._SegmentRecorder(cards, {0: "pool0", 1: "pool1"}, 0)+ recorder.original = _send_to_device+ out = recorder.send(recorder.original, {"attention_mask": built}, "meta")["attention_mask"]+ again = recorder.send(recorder.original, {"attention_mask": built}, "meta")["attention_mask"]+ assert again is out+ assert all(t.device.type == "meta" for t in out.mask_mod.reads)+ assert out._unsloth_graph_closure[0] == out.mask_mod.reads+ copies = [p for p in recorder.segments.parts if p[0] == "copy"]+ sources = [p[2] for p in copies]+ assert all(any(t is s for s in sources) for t in closure), "closure tensors not handed off"+ assert any(s is built.kv_num_blocks for s in sources)+ assert recorder.pending == 1+++def test_overlapping_recordings_from_two_threads_restore_accelerates_hooks():+ # accelerate's hooks are process-global: overlapping recordings used to restore each other's replacements and+ # leave a finished recorder's send_to_device installed.+ import threading+ import time++ import accelerate.hooks as hooks++ originals = (+ hooks.send_to_device,+ hooks.AlignDevicesHook.pre_forward,+ hooks.AlignDevicesHook.post_forward,+ )+ active, peak, errors = [0], [0], []++ def body():+ active[0] += 1+ peak[0] = max(peak[0], active[0])+ time.sleep(0.05)+ active[0] -= 1++ def record():+ try:+ GG._SegmentRecorder(_HostCards(), {0: "pool0"}, 0).run(body)+ except BaseException as error:+ errors.append(repr(error))++ threads = [threading.Thread(target = record) for _ in range(2)]+ for thread in threads:+ thread.start()+ for thread in threads:+ thread.join(10)+ assert not errors and peak[0] == 1, (errors, peak)+ assert (+ hooks.send_to_device,+ hooks.AlignDevicesHook.pre_forward,+ hooks.AlignDevicesHook.post_forward,+ ) == originalsdiff --git a/tests/test_return_logits_survives_for_training.py b/tests/test_return_logits_survives_for_training.pyindex 31f32d76a7c..3512fe5cf8a 100644--- a/tests/test_return_logits_survives_for_training.py+++ b/tests/test_return_logits_survives_for_training.py@@ -81,6 +81,8 @@ def _fast_base_model():"_pipeline_keeps_replica": lambda model: False,"enable_fp8_lora": lambda model: [],"restore_fp8_lora_for_inference": lambda model: None,+ # for_training frees kept decode graphs and their cache; none exist for these fakes.+ "_decode_graphs": types.SimpleNamespace(release_for_training = lambda model: None),}lifted = {n.name for n in wanted} | set(stubbed)todo = list(wanted)diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.pyindex e5294f5a722..39c0f4665fe 100644--- a/unsloth/models/_decode_graphs.py+++ b/unsloth/models/_decode_graphs.py@@ -163,6 +163,45 @@ class eager_scope:return False+class plan_scope:+ """Per-thread: the decode plan ("graph" / "eager") Unsloth's generate() chose for the call it runs."""++ __slots__ = ("plan", "_saved")++ def __init__(self, plan):+ self.plan = plan++ def __enter__(self):+ self._saved = getattr(_TLS, "plan", None)+ _TLS.plan = self.plan+ return self++ def __exit__(self, *exc):+ _TLS.plan = self._saved+ return False+++def call_plan():+ """ "graph" / "eager" for the generate() call running on this thread, None outside Unsloth's+ generate(). Per-call policies (the 16-bit LoRA fold threshold) read this, not enabled(), which only+ says graphs are allowed."""+ if torch.compiler.is_compiling():+ return None+ if getattr(_TLS, "eager", 0):+ return "eager"+ return getattr(_TLS, "plan", None)+++# Bumped by code that rewrites weights in place without a version bump (the 16-bit LoRA fold writes+# W.data): every captured decode graph is fingerprinted with it, so none outlives such a rewrite.+WEIGHTS_EPOCH = 0+++def weights_changed():+ global WEIGHTS_EPOCH+ WEIGHTS_EPOCH += 1++# Need = CALIB_WORK x (KV + prefill work) + CALIB_GRAPH_KV x KV. Calibrated on Qwen3-4B QLoRA bs16 x 2048,# T4: eager peaked 1.14x the estimate; graph capture OOMed at ~12 GiB free (~0.26 x KV extra).CALIB_WORK = float(os.environ.get("UNSLOTH_DECODE_GRAPH_CALIB_WORK", "1.15"))@@ -694,6 +733,10 @@ def invalidate(model):global BUFFER_EPOCHfor state in _states(model):state.clear()+ for module in model.modules():+ generic = module.__dict__.get("_unsloth_generic_decode_graphs")+ if generic is not None:+ generic.invalidate()BUFFER_EPOCH += 1@@ -709,6 +752,9 @@ def release(model):found = Truestate.clear()state.statics.clear()+ generic = d.get("_unsloth_generic_decode_graphs")+ if generic is not None:+ generic.invalidate()if "_unsloth_graph_buffers" in d:found = Truedel d["_unsloth_graph_buffers"]@@ -787,6 +833,18 @@ def _fingerprint(model, layers, rope_len):items.append((cos.data_ptr(), sin.data_ptr(), cos.shape[0]))# Every adapter-aware module outside the projections too (embeddings, lm_head, modules_to_save):# their enabled / active / merged state picks the branch a captured step baked in.+ adapters = _adapter_state(model)+ if adapters is None:+ return None+ items.extend(adapters)+ items.append(K._OPTIMIZER_STEPS)+ items.append(WEIGHTS_EPOCH)+ return tuple(items)+++def _adapter_state(model):+ """(module, disabled, active adapters, merged) of every adapter-aware module; None if unreadable."""+ items = []for module in model.modules():if hasattr(module, "disable_adapters") or hasattr(module, "active_adapter"):try:@@ -805,8 +863,7 @@ def _fingerprint(model, layers, rope_len):)except Exception:return None- items.append(K._OPTIMIZER_STEPS)- return tuple(items)+ return itemsdef _eligible(model, layers):diff --git a/unsloth/models/_generic_decode_graphs.py b/unsloth/models/_generic_decode_graphs.pynew file mode 100644index 00000000000..86a29271cb7--- /dev/null+++ b/unsloth/models/_generic_decode_graphs.py@@ -0,0 +1,1936 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""CUDA-graph decode for any FastModel model generating over a transformers StaticCache.++Each one-token decode step of `unsloth_base_fast_generate` replays a CUDAGraph of the model's own step,+with tensor arguments copied into static buffers. Only where `_decode_graphs.enabled()`, on NVIDIA CUDA,+over static attention layers that index the cache on the device. Per graph key: the first step runs+eagerly, then is captured; the first replay must match the eager step bitwise; a host sync, host-state+change or mismatch refuses the key. The step runs in the eager step's exact state (argument inference-ness+and strides, ambient autocast + cache flag, compiled regions as eager runs them): Dynamo guards on all of+these and another compiled variant rounds differently. Plain generate() runs inside generate_scope(), so+graphs are on by default there (plan() decides per call: short calls, low free memory and unsupported+calls decode eagerly; an out-of-memory graphed call is redone once eagerly). After an inference-mode call+the graph and its StaticCache are kept for the next call of the same batch (keep()); training, a returned+or caller-owned cache, and parameter / adapter / quant changes drop them. UNSLOTH_DECODE_GENERIC_GRAPHS=0+disables; _VALIDATE_ALL=1 checks every replay (debug); batches over UNSLOTH_DECODE_GENERIC_GRAPH_MAX_BSZ+(16) decode eagerly; UNSLOTH_DECODE_GENERIC_GRAPHS_KEEP=0 drops graphs at the end of every call."""++import collections+import collections.abc+import contextlib+import warnings+import copy+import functools+import threading+import os+import traceback+import weakref++import torch++from . import _decode_graphs as G++__all__ = [+ "wanted",+ "plan",+ "ineligible",+ "begin",+ "end",+ "decode_step",+ "release",+ "stats",+ "keepable",+ "kept_cache",+ "redo_eagerly",+]++_ATTR = "_unsloth_generic_decode_graphs"+# Host length counters of static cache layers (4.57 sliding: cumulative_length; 5.x: cumulative_length_int).+_COUNTERS = ("cumulative_length", "cumulative_length_int")+_MAX_ARG_BYTES = 16 * 1024 * 1024+_CONSTANTS = (bool, int, float, str, torch.dtype, torch.device)+_MAX_KEYS = 4096+# Graphs only remove launch overhead (T4 1B 4-bit: 4-7x at bsz 1, 2.7-4.7x at 16).+MAX_BSZ = int(os.environ.get("UNSLOTH_DECODE_GENERIC_GRAPH_MAX_BSZ", "16"))+# Free memory a graphed call needs beyond the eager one: the graph pool (8-16 MiB measured on 1B models),+# static input buffers and the side stream's workspace. Below it the call decodes eagerly.+_FREE_MARGIN_BYTES = 256 * 1024 * 1024+# A kept cache longer than this many times the call's own length is not reused (attention reads every slot).+_KEEP_MAX_RATIO = 2+# Captures in a row without a replay: the key changes every step, so the rest of the call decodes eagerly.+# A healthy call captures at most twice in a row (window filling, then full).+_MAX_UNREPLAYED_CAPTURES = 4+PLANS = collections.Counter()+EAGER_REDOS = 0+_OOM_WARNED = False++try:+ from torch.nn.attention.flex_attention import BlockMask as _BlockMask+except Exception:+ _BlockMask = None+++class _Refuse(Exception):+ pass+++class _Cuda:+ """The CUDA calls this path makes (tests swap in a host stand-in)."""++ device_type = "cuda"++ @staticmethod+ def available():+ # NVIDIA CUDA only: HIP replay is not verified against the eager step.+ return torch.version.hip is None and torch.cuda.is_available()++ @staticmethod+ def current_device():+ return torch.cuda.current_device()++ @staticmethod+ def free_bytes(device):+ return G._free_bytes(device)++ @staticmethod+ def device(index):+ return torch.cuda.device(index)++ @staticmethod+ def capturing():+ return torch.cuda.is_current_stream_capturing()++ @staticmethod+ def pool():+ return torch.cuda.graph_pool_handle()++ @staticmethod+ def new_stream():+ return torch.cuda.Stream()++ @staticmethod+ def stream(stream):+ return torch.cuda.stream(stream)++ @staticmethod+ def current_stream():+ return torch.cuda.current_stream()++ @staticmethod+ def device_stream(index):+ return torch.cuda.current_stream(index)++ @staticmethod+ def multidevice_supported():+ return True++ @staticmethod+ def mempool():+ # One graph pool per other card, shared by that card's segments (captured and replayed in order).+ return torch.cuda.graph_pool_handle()++ @staticmethod+ def side_stream(index):+ return torch.cuda.Stream(device = index)++ @staticmethod+ @contextlib.contextmanager+ def on_streams(streams):+ """The other cards' current streams set to `streams` ({index: stream}), restored after."""+ before = {d: torch.cuda.current_stream(d) for d in streams}+ here = torch.cuda.current_device()+ try:+ for d, stream in streams.items():+ stream.wait_stream(before[d])+ with torch.cuda.device(d):+ torch.cuda.set_stream(stream)+ yield+ finally:+ for d, stream in before.items():+ stream.wait_stream(streams[d])+ with torch.cuda.device(d):+ torch.cuda.set_stream(stream)+ torch.cuda.set_device(here)++ @staticmethod+ def card_of(value):+ """The card index a tensor lives on, or None (host tensors)."""+ if not value.is_cuda:+ return None+ return value.device.index if value.device.index is not None else torch.cuda.current_device()++ @staticmethod+ def index_of(device):+ """The card index a send_to_device target names (int, "cuda:1", torch.device), or None."""+ if isinstance(device, int):+ return device+ try:+ device = torch.device(device)+ except Exception:+ return None+ if device.type != "cuda":+ return None+ return device.index if device.index is not None else torch.cuda.current_device()++ @staticmethod+ def card_device(index):+ return torch.device("cuda", index)++ @staticmethod+ def new_graph():+ return torch.cuda.CUDAGraph()++ @staticmethod+ def begin_on(graph, pool, index):+ """Capture on card `index`'s current stream (its side stream, made current by on_streams)."""+ torch.cuda.set_device(index)+ G._begin(graph, pool)++ @staticmethod+ def end_on(graph):+ with warnings.catch_warnings():+ # A segment with no kernels (the tail after the last hand-off) is an empty, valid graph.+ warnings.simplefilter("ignore")+ G._end(graph)++ @staticmethod+ def record(+ pool,+ fn,+ cache,+ others = None,+ mempools = None,+ ):+ """(graph, fn()) with fn's launches captured into graph (`cache`: for host stand-ins).+ `others` ({index: stream}): the other cards of a model split over cards, their side streams+ current there (on_streams). The step is then captured as one graph per run of work on one+ card, cut where accelerate's AlignDevicesHook moves tensors to another card; those hand-offs+ run eagerly between the segments' replays (_Segments). Capturing a cross-card copy is not+ possible without peer access (cudaErrorStreamCaptureUnsupported on PCIe T4s)."""+ if others:+ here = torch.cuda.current_device()+ pools = dict(mempools)+ pools[here] = pool+ try:+ return _SegmentRecorder(_BACKEND, pools, here).run(fn)+ finally:+ torch.cuda.set_device(here)+ graph = torch.cuda.CUDAGraph()+ G._begin(graph, pool)+ try:+ out = fn()+ except BaseException:+ # End the invalidated capture, but report what broke it, not the capture_end error.+ if torch.cuda.is_current_stream_capturing():+ try:+ G._end(graph)+ except Exception:+ pass+ raise+ G._end(graph)+ return graph, out+++class _Segments:+ """One decode step of a model split over cards: ("graph", graph, card) parts replayed on their card's+ current stream, and ("copy", dst, src) hand-offs (accelerate's moves at a block boundary) run eagerly+ between them into the static buffers the next segment reads. Same order as the capture."""++ def __init__(self):+ self.parts = []++ def replay(self):+ for kind, a, b in self.parts:+ if kind == "graph":+ with _BACKEND.device(b):+ a.replay()+ else:+ # Eager cross-card copy: PyTorch orders it after src's stream and before dst's.+ a.copy_(b)+++_RECORDING = threading.Lock()+++class _SegmentRecorder:+ """Captures fn() as per-card segments. accelerate's AlignDevicesHook pre_forward / post_forward move+ tensors through accelerate.hooks.send_to_device; while recording, that name points at self.send:+ the first move of a tensor to another card ends the open segment, copies eagerly into a buffer kept+ for replay (recorded as a hand-off), and the next segment starts on the target card when the hook+ returns. A tensor already moved to that card in this step reuses its buffer (no new cut), so a+ layer-invariant input (rotary cos / sin, masks) crosses once."""++ def __init__(self, backend, pools, first):+ self.backend = backend+ self.pools = pools+ self.segments = _Segments()+ self.graph = None+ self.card = first+ self.pending = None+ self.moved = {}+ self.original = None++ def begin(self, card):+ self.graph = self.backend.new_graph()+ self.card = card+ self.backend.begin_on(self.graph, self.pools[card], card)++ def end(self):+ if self.graph is not None:+ graph, self.graph = self.graph, None+ self.backend.end_on(graph)+ self.segments.parts.append(("graph", graph, self.card))++ def resume(self):+ if self.pending is not None and self.graph is None:+ card, self.pending = self.pending, None+ self.begin(card)++ def _move(self, value, card, target, skip_keys):+ if isinstance(value, torch.Tensor):+ if self.backend.card_of(value) in (None, card):+ return value+ key = (id(value), card)+ hit = self.moved.get(key)+ if hit is not None:+ return hit[1]+ # First crossing of this tensor to `card`: the segment ends here, the copy runs eagerly.+ self.end()+ dst = value.to(target)+ self.moved[key] = (value, dst)+ self.segments.parts.append(("copy", dst, value))+ self.pending = card+ return dst+ if (+ _BlockMask is not None+ and isinstance(value, _BlockMask)+ and "_unsloth_graph_closure" in value.__dict__+ ):+ # A graph BlockMask (_build): its fields and the tensors its mask_mod reads cross like tensors+ # (eager hand-offs refilled every replay); BlockMask.to would leave mask_mod's tensors on this+ # card, and a cross-card copy inside the capture is refused.+ key = (id(value), card)+ hit = self.moved.get(key)+ if hit is None:+ hit = self.moved[key] = (+ value,+ _block_mask_moved(value, lambda t: self._move(t, card, target, skip_keys)),+ )+ return hit[1]+ if hasattr(value, "to"):+ # Not a tensor but movable (BatchEncoding, ...): accelerate's own move, as eager. A real+ # cross-card copy in it fails the capture, which refuses the key.+ return self.original(value, target, skip_keys = skip_keys)+ if isinstance(value, (tuple, list)):+ from accelerate.utils.operations import honor_type+ return honor_type(value, (self._move(v, card, target, skip_keys) for v in value))+ if isinstance(value, collections.abc.Mapping):+ skip = [skip_keys] if isinstance(skip_keys, str) else (skip_keys or [])+ return type(value)(+ {+ k: v if k in skip else self._move(v, card, target, skip_keys)+ for k, v in value.items()+ }+ )+ return value++ def send(+ self,+ original,+ tensor,+ device,+ non_blocking = False,+ skip_keys = None,+ ):+ card = self.backend.index_of(device)+ if card is None:+ return original(tensor, device, non_blocking = non_blocking, skip_keys = skip_keys)+ return self._move(tensor, card, self.backend.card_device(card), skip_keys)++ def run(self, fn):+ # accelerate's hooks are process-global: two overlapping recordings (threads) would restore each+ # other's replacements, leaving a finished recorder installed.+ with _RECORDING:+ return self._run(fn)++ def _run(self, fn):+ import accelerate.hooks as hooks++ original_send = hooks.send_to_device+ cls = hooks.AlignDevicesHook+ original_pre, original_post = cls.pre_forward, cls.post_forward+ recorder = self++ def pre_forward(hook, module, *args, **kwargs):+ out = original_pre(hook, module, *args, **kwargs)+ recorder.resume()+ return out++ def post_forward(hook, module, output):+ out = original_post(hook, module, output)+ recorder.resume()+ return out++ self.original = original_send+ hooks.send_to_device = functools.partial(self.send, original_send)+ cls.pre_forward, cls.post_forward = pre_forward, post_forward+ try:+ self.begin(self.card)+ out = fn()+ self.resume()+ self.end()+ except BaseException:+ if self.graph is not None:+ try:+ self.backend.end_on(self.graph)+ except Exception:+ pass+ self.graph = None+ raise+ finally:+ hooks.send_to_device = original_send+ cls.pre_forward, cls.post_forward = original_pre, original_post+ return self.segments, out+++_BACKEND = _Cuda+++def _static_layer_types():+ try:+ from transformers import cache_utils+ except Exception:+ return (), ()+ full = tuple(t for t in (getattr(cache_utils, "StaticLayer", None),) if isinstance(t, type))+ sliding = tuple(+ t+ for t in (+ getattr(cache_utils, name, None)+ for name in ("StaticSlidingWindowLayer", "SlidingWindowLayer", "ChunkedSlidingLayer")+ )+ if isinstance(t, type)+ )+ return full, sliding+++_FULL_LAYERS, _SLIDING_LAYERS = _static_layer_types()+++def wanted(model):+ """Whether this generate() call routes its decode steps through decode_step()."""+ if torch.compiler.is_compiling():+ return False+ if os.environ.get("UNSLOTH_DECODE_GENERIC_GRAPHS", "1") == "0" or not G.enabled():+ return False+ if not _BACKEND.available(): # NVIDIA CUDA only (the backend refuses HIP)+ return False+ if not _FULL_LAYERS:+ return False+ device = getattr(model, "device", None)+ return getattr(device, "type", None) == _BACKEND.device_type and _plain_forward(model)+++def _plan_len(model, args, kwargs):+ """prompt + new tokens this call can reach (0 if unknown), as the llama path sizes its plan."""+ try:+ from .llama import _generation_plan_len+ return int(_generation_plan_len(model, args, kwargs) or 0)+ except Exception:+ return 0+++def plan(model, args, kwargs, input_ids):+ """ "graph" or "eager" for this generate() call, decided once before it decodes (inside+ generate_scope(), so graphs are on by default). Eager: graphs not wanted (wanted()), batch over+ MAX_BSZ, too few decode steps to win back a capture (unless a kept graph of this batch replays from+ the first step), or too little free memory for the graph pool. A kept graph / cache that this call+ cannot reuse is dropped here."""+ state = model.__dict__.get(_ATTR)+ if not wanted(model):+ if state is not None:+ state.clear()+ PLANS["not wanted"] += 1+ return "eager"+ try:+ prompt = int(input_ids.shape[1]) if input_ids.dim() >= 2 else 0+ bsz = G.decode_batch(+ int(input_ids.shape[0]), kwargs, getattr(model, "generation_config", None)+ )+ except Exception:+ PLANS["unknown shape"] += 1+ return "eager"+ max_len = _plan_len(model, args, kwargs)+ kept = (+ state is not None+ and state.kept is not None+ and state.kept_bsz == bsz+ and bool(state.graphs)+ )+ if kept and max_len:+ # transformers sizes the cache prompt + new tokens - 1 (the last token is never written back).+ length = _kept_len(state)+ # transformers 5.x grows every later call's cache to the longest one so far (_previous_max_cache_length):+ # this call gets a cache of that size whether or not the kept one is reused, so compare against it.+ need = max_len - 1+ previous = getattr(model, "_previous_max_cache_length", -1)+ if type(previous) is int:+ need = max(need, previous)+ kept = length is not None and need <= length <= _KEEP_MAX_RATIO * max(need, 1)+ if state is not None and state.kept is not None and not kept:+ state.clear()+ if bsz > MAX_BSZ:+ PLANS["batch over max"] += 1+ return "eager"+ if max_len:+ left = max_len - prompt - 1+ # Step 1 is eager + capture, step 2 a validated replay: fewer steps cannot win back a capture.+ if left < 2 or (left < G.CAPTURE_MIN_STEPS and not kept):+ PLANS["short call"] += 1+ return "eager"+ if not kept:+ try:+ # A model split over cards needs room for its graph pool on every card it runs on.+ devices = [input_ids.device] + [+ torch.device(_BACKEND.device_type, d)+ for d in sorted(set((getattr(model, "hf_device_map", None) or {}).values()))+ if type(d) is int and d != input_ids.device.index+ ]+ free = min(_BACKEND.free_bytes(d) for d in devices)+ except Exception:+ free = None+ if free is not None and free < _FREE_MARGIN_BYTES:+ PLANS["low free memory"] += 1+ return "eager"+ PLANS["graph kept" if kept else "graph"] += 1+ return "graph"+++def ineligible(model):+ """A call that cannot decode through graphs (dynamic cache, compiled decode): eager, and a kept+ graph / cache goes (it would only hold memory)."""+ state = model.__dict__.get(_ATTR)+ if state is not None and state.kept is not None:+ state.clear()+ PLANS["ineligible"] += 1+ return "eager"+++def replayable(kwargs):+ """A failed graphed call can be redone eagerly only when no caller object holds its partial state."""+ return not (+ kwargs.get("streamer", None) is not None+ or kwargs.get("past_key_values", None) is not None+ or kwargs.get("stopping_criteria", None)+ or kwargs.get("logits_processor", None)+ )+++def rng_state(device):+ """Host + this CUDA device's RNG state, so an eager redo samples as the failed attempt started."""+ try:+ cuda = torch.cuda.get_rng_state(device) if getattr(device, "type", None) == "cuda" else None+ except Exception:+ cuda = None+ return torch.get_rng_state(), cuda, device+++def set_rng_state(saved):+ host, cuda, device = saved+ torch.set_rng_state(host)+ if cuda is not None:+ torch.cuda.set_rng_state(cuda, device)+++def redo_eagerly(model):+ """After an out-of-memory graphed call: free graphs, kept cache and allocator blocks before the redo."""+ global EAGER_REDOS, _OOM_WARNED+ release(model)+ model.__dict__.pop("_unsloth_generic_call_cache", None)+ try:+ G._empty_cache()+ except Exception:+ pass+ EAGER_REDOS += 1+ if not _OOM_WARNED:+ _OOM_WARNED = True+ import warnings+ warnings.warn(+ "Unsloth: generate() ran out of GPU memory with CUDA-graph decode; freed the decode graphs "+ "and decoded this call eagerly (UNSLOTH_DECODE_CUDA_GRAPHS=0 always decodes eagerly).",+ stacklevel = 3,+ )+++def _plain_forward(model):+ """No instance forward; the class forward bound on the instance, bare or under functools.wraps+ wrappers (accelerate's mixed-precision autocast / fp32-output wrappers, which eager decode runs too);+ Unsloth's GRPO hidden-states wrapper, which passes straight through while generating; or accelerate's+ device hook of a model on one device (a bitsandbytes model placed on a card other than cuda:0 gets+ one): its input / output moves are no-ops there, and the eager step runs the same hook. Offloading+ hooks and device maps over several devices stay eager."""+ forward = model.__dict__.get("forward")+ if forward is None:+ return True+ if isinstance(forward, functools.partial) and "_hf_hook" in model.__dict__:+ # accelerate's add_hook_to_module: forward = partial(new_forward, model), original in _old_forward.+ if not (forward.args and forward.args[0] is model and not forward.keywords):+ return False+ if not _single_device_hook(model):+ return False+ old = model.__dict__.get("_old_forward")+ return old is not None and _plain_bound(model, old)+ if hasattr(model, "_hf_hook"):+ return False+ return _plain_bound(model, forward)+++def _plain_bound(model, forward):+ func = getattr(forward, "__func__", forward)+ if getattr(func, "_unsloth_grpo_hidden_states_forward_wrapped", False):+ return True+ if getattr(forward, "__self__", None) is not model:+ return False+ target = getattr(type(model), "forward", None)+ for _ in range(4):+ if func is target:+ return True+ func = getattr(func, "__wrapped__", None)+ if func is None:+ return False+ return False+++def _single_device_hook(model):+ """accelerate AlignDevicesHook(s) that only move tensors to the model's own (single) device: no+ offload, no weights map. Another hook type, or an execution device elsewhere, is not ours to graph."""+ try:+ from accelerate.hooks import AlignDevicesHook, SequentialHook+ except Exception:+ return False+ hook = model.__dict__.get("_hf_hook")+ hooks = list(hook.hooks) if isinstance(hook, SequentialHook) else [hook]+ device = getattr(model, "device", None)+ for h in hooks:+ if type(h) is not AlignDevicesHook:+ return False+ if h.offload or getattr(h, "offload_buffers", False) or h.weights_map is not None:+ return False+ target = h.execution_device+ if target is None:+ continue+ try:+ # accelerate names a GPU by its index; torch.device(int) asks for the current accelerator, which+ # a CPU-only torch build does not have (it raises).+ if type(target) is int:+ target = torch.device(_BACKEND.device_type, target)+ target = torch.device(target)+ except Exception:+ return False+ if not isinstance(device, torch.device) or target.type != device.type:+ return False+ same = (target.index if target.index is not None else 0) == (+ device.index if device.index is not None else 0+ )+ if not same and not _split_entry_card(model, target):+ return False+ return True+++def _split_entry_card(model, target):+ """A model split over cards (multi-device graphs on): its root hook moves the inputs to the card of the+ first block, which need not be `model.device` (the first parameter: a tied embedding / lm_head can sit+ on the last card). Any card the model's parameters are on is its own."""+ if not _multidevice_enabled():+ return False+ devices = _param_devices(model)+ return len(devices) > 1 and _device_index(target) in {+ _device_index(d) for d in devices if d.type == target.type+ }+++class _Entry:+ __slots__ = ("graph", "leaves", "outputs", "validated")++ def __init__(self, graph, leaves, outputs, validated):+ self.graph = graph+ self.leaves = leaves+ self.outputs = outputs+ self.validated = validated+++class _State:+ def __init__(self):+ self.graphs = collections.OrderedDict()+ self.pool = None+ self.others = ()+ self.mempools = {}+ self.side = {}+ self.fp_reason = None+ self.stream = None+ self.fingerprint = None+ self.optimizer_steps = -1+ self.recheck = True+ self.cache_sig = None+ # Keys refused for the process (host sync in capture, host state, validation mismatch).+ self.refused = {}+ self.validated = set()+ self.moved = None+ self.key_captures = collections.Counter()+ self.step = 0+ self.forwards = 0+ self.capture_errors = []+ self.check = _new_check()+ self.captures = 0+ self.replays = 0+ self.validations = 0+ self.eager_steps = 0+ self.refusals = collections.Counter()+ # keep(): the StaticCache the kept graphs address, its batch, and the training-forward hook.+ self.kept = None+ self.kept_bsz = None+ # The max_cache_len the kept cache was built for (5.x), else None (its full layers' length is used).+ self.kept_len = None+ self.hook = None+ self.owner = None+ self.kept_calls = 0+ self.active = False+ self.unreplayed = 0+ self.call_stop = None++ def clear(self):+ self.graphs.clear()+ self.cache_sig = None+ # A pool whose graphs are all freed is released by the allocator: never reuse its handle.+ self.pool = None+ self.mempools = {}+ self.drop_kept()++ def invalidate(self):+ """Weights / adapters / quant state changed: graphs and every per-key decision go."""+ self.clear()+ self.fingerprint = None+ self.validated.clear()+ self.refused.clear()++ def drop_kept(self):+ """Forget the kept cache (and free it when the model still holds it for the next call)."""+ kept, self.kept, self.kept_bsz, self.kept_len = self.kept, None, None, None+ if self.hook is not None:+ self.hook.remove()+ self.hook = None+ owner = self.owner() if self.owner is not None else None+ # Inside a call the cache is that call's (transformers holds it): only forget it.+ if (+ not self.active+ and kept is not None+ and owner is not None+ and owner.__dict__.get("_cache") is kept+ ):+ del owner.__dict__["_cache"]+++def _state(model):+ state = model.__dict__.get(_ATTR)+ if state is None:+ state = model.__dict__[_ATTR] = _State()+ state.owner = weakref.ref(model)+ return state+++def begin(model):+ """Start of a generate() call: the first graph step re-checks the fingerprint."""+ state = _state(model)+ state.recheck = True+ state.forwards = 0+ state.moved = None+ state.key_captures.clear()+ state.step = 0+ state.check = _new_check()+ state.active = True+ state.unreplayed = 0+ state.call_stop = None+ if state.hook is not None:+ # Only needed between calls: no per-step hook while this call decodes.+ state.hook.remove()+ state.hook = None+ if len(state.refused) + len(state.validated) > _MAX_KEYS:+ state.refused.clear()+ state.validated.clear()+++def end(model, keep = False):+ """End of a generate() call. Its graphs read this call's cache: freed, unless `keep` (keepable()), when+ the graphs and that cache stay for the next call of the same batch (released by training)."""+ state = model.__dict__.get(_ATTR)+ if state is None:+ return+ state.active = False+ cache = model.__dict__.pop("_unsloth_generic_call_cache", None)+ length = model.__dict__.pop("_unsloth_generic_call_len", None)+ if (+ not keep+ or not state.graphs+ or cache is None+ or state.cache_sig is None+ or state.cache_sig[0] != id(cache)+ ):+ state.clear()+ return+ state.kept, state.kept_bsz, state.kept_len = cache, _cache_batch(cache), length+ state.kept_calls += 1+ if state.hook is None:+ state.hook = model.register_forward_pre_hook(_release_if_training)+++def _release_if_training(module, args):+ # Training can resume without for_training() (sampling in a trainer callback): free the kept graph and cache.+ # Not while Dynamo traces a compiled training forward (removing this hook there is a graph break, an error+ # under fullgraph): the next generate() re-checks the fingerprint, and for_training() releases as before.+ if torch.compiler.is_compiling():+ return+ if module.training and torch.is_grad_enabled():+ release(module)+++def _cache_batch(cache):+ """Rows of a StaticCache (4.x: max_batch_size; 5.x layers only carry their tensors)."""+ try:+ keys = cache.layers[0].keys+ if isinstance(keys, torch.Tensor) and keys.dim() >= 1:+ return int(keys.shape[0])+ except Exception:+ pass+ try:+ return int(cache.max_batch_size)+ except Exception:+ return None+++def _kept_len(state):+ # Sliding layers are capped at their window, so only the length the cache was built for says what it holds.+ return state.kept_len if state.kept_len is not None else _cache_len(state.kept)+++def _cache_len(cache):+ """Slots of a StaticCache's full layers (sliding layers are shorter; 5.x has no cache-level length)."""+ try:+ return max(int(layer.max_cache_len) for layer in cache.layers)+ except Exception:+ return None+++def kept_cache(model):+ state = model.__dict__.get(_ATTR)+ return state.kept if state is not None else None+++def keepable(model, kwargs, output):+ """Whether this inference-mode call's graphs and StaticCache may stay for the next call: not when the+ caller passed or got back the cache (a later call would overwrite it), nor in training mode."""+ if os.environ.get("UNSLOTH_DECODE_GENERIC_GRAPHS_KEEP", "1") == "0":+ return False+ if getattr(model, "training", False) or kwargs.get("past_key_values", None) is not None:+ return False+ if getattr(output, "past_key_values", None) is not None:+ return False+ return True+++def release(model):+ for module in model.modules():+ state = module.__dict__.get(_ATTR)+ if state is not None:+ state.invalidate()+++def _kept_compatible(state, batch_size, max_cache_len, offloading):+ kept = state.kept+ if kept is None:+ return False+ length = _kept_len(state)+ if length is None:+ return False+ return (+ state.kept_bsz == batch_size+ and bool(getattr(kept, "offloading", False)) == bool(offloading)+ and max_cache_len <= length <= _KEEP_MAX_RATIO * max(int(max_cache_len), 1)+ )+++class reuse_kept_cache:+ """One graphed generate() call: transformers builds this call's StaticCache, unless a compatible kept+ one exists, which is reset and handed back (same tensors, so the kept graphs replay). 5.x builds a new+ cache per call in _prepare_static_cache; 4.x reuses model._cache, which _clear_generation_caches keeps+ while it is the kept cache. The cache the call ends up with is recorded for end(keep=True)."""++ def __init__(self, model):+ self.model = model+ self.prepare = None++ def _prepare(self, *args, **kwargs):+ model = self.model+ state = _state(model)+ names = ("cache_implementation", "batch_size", "max_cache_len")+ values = dict(zip(names, args))+ values.update({k: v for k, v in kwargs.items() if k in names})+ cache_implementation = values.get("cache_implementation", "static")+ batch_size, max_cache_len = values.get("batch_size"), values.get("max_cache_len")+ prefill_chunk = kwargs.get("prefill_chunk_size", args[3] if len(args) > 3 else None)+ effective = max_cache_len+ previous = getattr(model, "_previous_max_cache_length", -1)+ if type(effective) is int and type(previous) is int:+ effective = max(effective, previous)+ if (+ type(batch_size) is int+ and type(effective) is int+ and prefill_chunk is None+ and not getattr(model.config, "is_encoder_decoder", False)+ and _kept_compatible(+ state, batch_size, effective, "offloaded" in str(cache_implementation)+ )+ ):+ cache = state.kept+ cache.reset()+ PLANS["reused kept cache"] += 1+ else:+ if state.kept is not None:+ state.clear()+ cache = self.prepare(*args, **kwargs)+ model.__dict__["_unsloth_generic_call_cache"] = cache+ if type(effective) is int:+ model.__dict__["_unsloth_generic_call_len"] = (+ effective if cache is not state.kept else _kept_len(state)+ )+ return cache++ def __enter__(self):+ model = self.model+ if "_prepare_static_cache" in model.__dict__ or not hasattr(model, "_prepare_static_cache"):+ # 4.x (or a wrapper owns the hook): the call's cache is model._cache, read in end().+ self.prepare = None+ return self+ self.prepare = model._prepare_static_cache+ model._prepare_static_cache = self._prepare+ return self++ def __exit__(self, *exc):+ model = self.model+ if self.prepare is not None:+ del model._prepare_static_cache+ elif "_unsloth_generic_call_cache" not in model.__dict__:+ cache = model.__dict__.get("_cache")+ if cache is not None:+ model.__dict__["_unsloth_generic_call_cache"] = cache+ return False+++def stats(model):+ state = getattr(model, "__dict__", {}).get(_ATTR)+ if state is None:+ return {}+ return {+ "captures": state.captures,+ "replays": state.replays,+ "validations": state.validations,+ "eager_steps": state.eager_steps,+ "graphs": len(state.graphs),+ "refusals": dict(state.refusals),+ "refused_keys": len(state.refused),+ "max_captures_per_key": max(state.key_captures.values(), default = 0),+ "capture_errors": list(state.capture_errors),+ "check": dict(state.check),+ "other_devices": list(state.others),+ }+++def _new_check():+ return {+ "steps": 0,+ "mismatches": 0,+ "first_mismatch_step": None,+ "max_abs_diff": 0.0,+ "max_kl": 0.0,+ "top1_disagree": 0,+ }+++def _validate_all():+ return os.environ.get("UNSLOTH_DECODE_GENERIC_GRAPHS_VALIDATE_ALL", "0") == "1"+++def _record_check(state, eager, replay, equal):+ """Accumulates how far the replay's first output (logits) is from the eager step's."""+ check = state.check+ check["steps"] += 1+ if not equal:+ check["mismatches"] += 1+ if check["first_mismatch_step"] is None:+ check["first_mismatch_step"] = state.step+ xs, ys = _tensors(eager, []), _tensors(replay, [])+ if not xs or not ys or not xs[0].is_floating_point() or xs[0].shape != ys[0].shape:+ return+ x, y = xs[0].float(), ys[0].float()+ check["max_abs_diff"] = max(check["max_abs_diff"], float((x - y).abs().max()))+ if x.dim() >= 2:+ p, q = torch.log_softmax(x, -1), torch.log_softmax(y, -1)+ kl = (p.exp() * (p - q)).sum(-1).max()+ check["max_kl"] = max(check["max_kl"], float(kl))+ check["top1_disagree"] += int((x.argmax(-1) != y.argmax(-1)).sum())+++def _where(error):+ """file:line of the innermost frame outside torch that raised `error`."""+ for frame in reversed(traceback.extract_tb(error.__traceback__)):+ if f"{os.sep}torch{os.sep}" not in frame.filename:+ return f"{os.path.basename(frame.filename)}:{frame.lineno} {frame.name}"+ return None+++def _refusal(state, reason):+ state.refusals[reason] += 1+ state.eager_steps += 1+ return None+++def _cache_reason(cache):+ """None when every layer of `cache` is a static attention layer this path can replay."""+ if cache is None:+ return "no cache"+ try:+ from transformers import StaticCache+ except Exception:+ return "no StaticCache"+ if not isinstance(cache, StaticCache):+ return f"cache {type(cache).__name__}"+ if getattr(cache, "offloading", False):+ return "offloaded cache"+ layers = getattr(cache, "layers", None)+ if not layers:+ return "legacy cache"+ for layer in layers:+ kind = type(layer)+ # Exact types: 5.x linear-attention / indexed layers subclass StaticLayer.+ if kind not in _FULL_LAYERS and kind not in _SLIDING_LAYERS:+ return f"layer {kind.__name__}"+ keys = getattr(layer, "keys", None)+ if not getattr(layer, "is_initialized", False) or not isinstance(keys, torch.Tensor):+ return "uninitialized layer"+ if keys.device.type != _BACKEND.device_type:+ return "cache not on CUDA"+ return None+++def _host_counter(layer):+ for name in ("cumulative_length_int", "cumulative_length"):+ value = layer.__dict__.get(name)+ if type(value) is int:+ return value+ return None+++def _window_reason(cache, q_len):+ # A step that fills a sliding window switches the update's branch mid-call: eager. Before and after it, every+ # one-token step runs the same device work (write at the device counter, or roll by one slot), and the key+ # carries which of the two each layer is in (_window_state), so a replay repeats the right one.+ for layer in cache.layers:+ if type(layer) in _SLIDING_LAYERS:+ count = _host_counter(layer)+ if count is None:+ return "sliding layer without host counter"+ if count < layer.max_cache_len < count + q_len:+ return "sliding window filling"+ return None+++def _window_fills(cache, q_len):+ """Whether this step fills a sliding window (count + q_len reaches it). Its key is the filling one, but a+ capture records after the warm-up advanced the counters, i.e. the rolling branch: never capture it."""+ for layer in cache.layers:+ if type(layer) in _SLIDING_LAYERS:+ count = _host_counter(layer)+ if count is not None and count < layer.max_cache_len <= count + q_len:+ return True+ return False+++def _window_state(cache):+ """Per sliding layer: whether its window is full (it rolls) or still filling (writes at the counter)."""+ return tuple(+ _host_counter(layer) >= layer.max_cache_len+ for layer in cache.layers+ if type(layer) in _SLIDING_LAYERS+ )+++# transformers 5.x rolls a full static sliding window by one slot per one-token step and indexes the last slot+# with `torch.tensor([-1], device=...)`: a host-to-device copy a CUDA graph capture refuses. While a step is+# warmed up and recorded, that branch runs the same ops on an index tensor made outside the capture.+_ROLL_INDEX = {}+++def _roll_index(device):+ index = _ROLL_INDEX.get(device)+ if index is None:+ if _BACKEND.capturing():+ return None+ with G._no_inference_tensors():+ index = _ROLL_INDEX[device] = torch.tensor([-1], dtype = int, device = device)+ return index+++def _rolling_update_classes():+ """(class, host counter) of the static sliding layers (5.x StaticSlidingWindowLayer, 4.57 SlidingWindowLayer)+ whose one-token full-window update builds its roll index on the host."""+ try:+ import inspect+ from transformers import cache_utils+ except Exception:+ return ()+ out = []+ for cls in _SLIDING_LAYERS:+ try:+ source = inspect.getsource(cls.__dict__["update"])+ except Exception:+ continue+ counter = next(+ (c for c in ("cumulative_length_int", "cumulative_length") if f"self.{c} +=" in source),+ None,+ )+ needed = ("roll(-1, dims=-2)", "torch.tensor([-1], dtype=int", "self.keys.copy_(new_keys)")+ if counter is not None and all(n in source for n in needed):+ out.append((cls, counter))+ return tuple(out)+++_ROLLING_CLASSES = []+++def _graphable_update(original, counter):+ def update(self, key_states, value_states, *args, **kwargs):+ count = self.__dict__.get(counter)+ if (+ getattr(self, "is_initialized", False)+ and key_states.shape[-2] == 1+ and type(count) is int+ and count >= self.max_cache_len+ ):+ index = _roll_index(self.keys.device)+ if index is not None:+ # transformers' one-token full-window branch, op for op.+ self.__dict__[counter] = count + 1+ new_keys = self.keys.roll(-1, dims = -2)+ new_values = self.values.roll(-1, dims = -2)+ new_keys[:, :, index] = key_states+ new_values[:, :, index] = value_states+ self.keys.copy_(new_keys)+ self.values.copy_(new_values)+ return self.keys, self.values+ return original(self, key_states, value_states, *args, **kwargs)++ update._unsloth_graphable_roll = True+ return update+++def _rolling_classes():+ if not _ROLLING_CLASSES:+ _ROLLING_CLASSES.append(_rolling_update_classes())+ return _ROLLING_CLASSES[0]+++def _graphable_rolls():+ patched = []+ try:+ for cls, counter in _rolling_classes():+ original = cls.__dict__.get("update")+ if original is None or getattr(original, "_unsloth_graphable_roll", False):+ continue+ cls.update = _graphable_update(original, counter)+ patched.append((cls, original))+ yield+ finally:+ for cls, original in reversed(patched):+ cls.update = original+++def _host_state(cache):+ """Non-tensor scalar attributes of the cache and its layers."""+ out = []+ for owner in (cache, *cache.layers):+ out.append({k: v for k, v in owner.__dict__.items() if v is None or type(v) in _CONSTANTS})+ return out+++def _host_state_reason(before, after, q_len):+ for b, a in zip(before, after):+ for name in set(b) | set(a):+ if b.get(name) == a.get(name):+ continue+ if name in _COUNTERS and type(b.get(name)) is int and a.get(name) == b[name] + q_len:+ continue+ return f"cache host state {name}"+ return None+++def _snapshot(cache, rows = True):+ """Length counters (host ints, device scalars) a step advances, plus what undoes the roll of every full sliding+ window (a step rolls it, so unlike a slot write it is not idempotent); restored by _restore. rows=False: the+ counters only (a recording runs no device work)."""+ host, device = [], []+ rolling = None+ for layer in cache.layers:+ for name in _COUNTERS:+ value = layer.__dict__.get(name)+ if type(value) is int:+ host.append((layer, name, value))+ elif isinstance(value, torch.Tensor) and value.numel() == 1:+ device.append((value, value.clone(), False))+ if rows and type(layer) in _SLIDING_LAYERS:+ count = _host_counter(layer)+ if count is not None and count >= layer.max_cache_len:+ if rolling is None:+ rolling = {cls for cls, _ in _rolling_classes()}+ # A transformers rolling layer's one-token step drops slot 0 and shifts the rest down by one:+ # keeping slot 0 undoes it with one layer's temporary, not a copy of every window held at once.+ rolls = type(layer) in rolling+ for t in (getattr(layer, "keys", None), getattr(layer, "values", None)):+ if isinstance(t, torch.Tensor):+ one = rolls and t.dim() == 4+ device.append((t, t[..., :1, :].clone() if one else t.clone(), one))+ return host, device+++def _restore(snapshot):+ host, device = snapshot+ for layer, name, value in host:+ layer.__dict__[name] = value+ for live, saved, rolled in device:+ if rolled:+ live.copy_(torch.cat((saved, live[..., :-1, :]), dim = -2))+ else:+ live.copy_(saved)+++def _advance(cache, q_len):+ # The replay advanced the device counters; the host ones are advanced here, as the eager update.+ for layer in cache.layers:+ for name in _COUNTERS:+ value = layer.__dict__.get(name)+ if type(value) is int:+ layer.__dict__[name] = value + q_len+++def _spec(value, leaves, cache):+ """Hashable structure of a forward argument; its tensors appended to `leaves` in order."""+ if isinstance(value, torch.Tensor):+ if value.requires_grad:+ raise _Refuse("tensor argument requires grad")+ if value.device.type != _BACKEND.device_type:+ raise _Refuse(f"tensor argument on {value.device}")+ if value.numel() * value.element_size() > _MAX_ARG_BYTES:+ raise _Refuse("large tensor argument")+ leaves.append(value)+ # Inference-ness: compiled regions (create_causal_mask) guard on the dispatch key set.+ return (+ "T",+ tuple(value.shape),+ value.stride(),+ value.dtype,+ value.device,+ value.is_inference(),+ )+ if value is cache:+ return ("C",)+ if value is None or type(value) in _CONSTANTS:+ return ("V", value)+ if type(value) in (tuple, list):+ return (type(value).__name__, tuple(_spec(v, leaves, cache) for v in value))+ if type(value) is dict and all(type(k) is str for k in value):+ return ("D", tuple((k, _spec(v, leaves, cache)) for k, v in value.items()))+ if _BlockMask is not None and isinstance(value, _BlockMask):+ return _block_mask_spec(value, leaves, cache)+ raise _Refuse(f"argument {type(value).__name__}")+++# BlockMask graph contract. A flex BlockMask argument replays only when its builder publishes+# block_mask._unsloth_graph_closure = (tensors, rebuild)+# `tensors`: a tuple of every tensor its mask_mod reads (offsets, padding rows, ...), fixed shapes from+# step to step; `rebuild(tensors)`: a mask_mod of the same code reading the given tensors instead. The+# graph keeps static copies of the BlockMask's tensor fields and of `tensors`, refills them each step and+# captures flex over a BlockMask built from them (mask_mod = rebuild(static copies)). Key: BLOCK_SIZE,+# seq_lengths, the field layout, mask_mod's code and the closure tensors' layout. Any other BlockMask+# (a mask_mod closing over tensors the graph cannot see) is refused: "argument BlockMask".+_BLOCK_MASK_SKIP = ("mask_mod", "_unsloth_graph_closure", "_unsloth_on_device")+++def _block_mask_fields(value):+ return sorted((k, v) for k, v in vars(value).items() if k not in _BLOCK_MASK_SKIP)+++def _block_mask_spec(value, leaves, cache):+ contract = value.__dict__.get("_unsloth_graph_closure")+ if (+ type(contract) is not tuple+ or len(contract) != 2+ or type(contract[0]) is not tuple+ or not all(isinstance(t, torch.Tensor) for t in contract[0])+ or not callable(contract[1])+ ):+ raise _Refuse("argument BlockMask")+ fields = []+ for name, field in _block_mask_fields(value):+ if field is None or isinstance(field, torch.Tensor) or type(field) in _CONSTANTS:+ fields.append((name, _spec(field, leaves, cache)))+ elif type(field) is tuple and all(type(x) in _CONSTANTS for x in field):+ fields.append((name, ("V", field)))+ else:+ raise _Refuse("argument BlockMask")+ code = getattr(value.mask_mod, "__code__", None)+ if code is None:+ raise _Refuse("argument BlockMask")+ closure = tuple(_spec(t, leaves, cache) for t in contract[0])+ return ("BM", tuple(fields), code, closure)+++def _build(value, buffers, cache):+ """`value` with its tensors replaced by the next static buffers (same order as _spec)."""+ if isinstance(value, torch.Tensor):+ return next(buffers)+ if type(value) in (tuple, list):+ return type(value)(_build(v, buffers, cache) for v in value)+ if type(value) is dict:+ return {k: _build(v, buffers, cache) for k, v in value.items()}+ if _BlockMask is not None and isinstance(value, _BlockMask):+ built = copy.copy(value)+ for name, field in _block_mask_fields(value):+ if isinstance(field, torch.Tensor):+ setattr(built, name, next(buffers))+ tensors, rebuild = value.__dict__["_unsloth_graph_closure"]+ static = tuple(next(buffers) for _ in tensors)+ built.mask_mod = rebuild(static)+ built._unsloth_graph_closure = (static, rebuild)+ # The builder's per-device rebuild reads its own tensors, not these static copies: replaced by one+ # over the static copies, so an accelerate hook moving the mask to another card of a split model+ # (BlockMask.to leaves mask_mod's tensors behind) hands that card's layers a mask_mod on their card.+ built._unsloth_on_device = functools.partial(_block_mask_on_device, built)+ return built+ return value+++def _block_mask_moved(value, move):+ """A graph BlockMask with every tensor field and closure tensor passed through `move` (tensor -> tensor+ on the target card) and mask_mod rebuilt over the moved closure tensors; `value` itself when nothing+ moved (accelerate also "moves" the mask to the card it is on: the root hook, the first card's layers),+ so it keeps the per-device move for the next card's hook."""+ fields = [+ (name, field, move(field))+ for name, field in _block_mask_fields(value)+ if isinstance(field, torch.Tensor)+ ]+ tensors, rebuild = value.__dict__["_unsloth_graph_closure"]+ moved_tensors = tuple(move(t) for t in tensors)+ if all(a is b for _, a, b in fields) and all(a is b for a, b in zip(tensors, moved_tensors)):+ return value+ moved = copy.copy(value)+ for name, _, field in fields:+ setattr(moved, name, field)+ moved.mask_mod = rebuild(moved_tensors)+ moved._unsloth_graph_closure = (moved_tensors, rebuild)+ moved._unsloth_on_device = functools.partial(_block_mask_on_device, moved)+ return moved+++def _block_mask_on_device(value, device):+ """Eager (warm-up) move of a graph BlockMask: fresh copies of the current static values, never cached."""+ device = torch.device(device)+ return _block_mask_moved(value, lambda t: t.to(device))+++def _map_outputs(value, fn):+ if isinstance(value, torch.Tensor):+ return fn(value)+ if isinstance(value, collections.OrderedDict) and hasattr(value, "to_tuple"):+ return type(value)(**{k: _map_outputs(v, fn) for k, v in value.items()})+ if type(value) in (tuple, list):+ return type(value)(_map_outputs(v, fn) for v in value)+ if type(value) is dict:+ return {k: _map_outputs(v, fn) for k, v in value.items()}+ return value+++def _tensors(value, out):+ _map_outputs(value, lambda t: out.append(t) or t)+ return out+++def _outputs_equal(a, b):+ xs, ys = _tensors(a, []), _tensors(b, [])+ if len(xs) != len(ys):+ return False+ for x, y in zip(xs, ys):+ if x.shape != y.shape or x.dtype != y.dtype:+ return False+ if not torch.equal(x, y):+ same_nan = torch.isnan(x) & torch.isnan(y) if x.is_floating_point() else None+ if same_nan is None or not torch.equal(+ torch.where(same_nan, 0, x), torch.where(same_nan, 0, y)+ ):+ return False+ return True+++def _multidevice_enabled():+ return os.environ.get("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", "1") != "0"+++def _hooks_align_only(model):+ """Every accelerate hook in `model` only moves tensors to its block's card (AlignDevicesHook, no+ offload, no weights map): a model split over cards by device_map / Unsloth's layer split. Offload,+ other hook types or an unreadable hook keep the model eager."""+ hooked = [m.__dict__["_hf_hook"] for m in model.modules() if "_hf_hook" in m.__dict__]+ if not hooked:+ return True+ try:+ from accelerate.hooks import AlignDevicesHook, SequentialHook+ except Exception:+ return False+ for hook in hooked:+ hooks = list(hook.hooks) if isinstance(hook, SequentialHook) else [hook]+ for h in hooks:+ if type(h) is not AlignDevicesHook:+ return False+ if h.offload or getattr(h, "offload_buffers", False) or h.weights_map is not None:+ return False+ target = h.execution_device+ if target is not None:+ try:+ target = torch.device(target)+ except Exception:+ return False+ if target.type != _BACKEND.device_type:+ return False+ return True+++def _param_devices(model):+ # An input embedding offloaded to the host is looked up outside the graph: its device does not count.+ host = _offloaded_input_embedding(model)+ exempt = {id(p) for p in host.parameters()} if host is not None else ()+ return {p.device for p in model.parameters() if id(p) not in exempt}+++def _device_index(device):+ return device.index if device.index is not None else _BACKEND.current_device()+++def _multidevice_reason(model, devices):+ """None when a model split over several cards may be graphed as one multi-card graph."""+ if not _multidevice_enabled():+ return "multi-device (UNSLOTH_DECODE_MULTIDEVICE_GRAPHS=0)"+ if not _BACKEND.multidevice_supported():+ return "multi-device capture needs a newer torch"+ if not _hooks_align_only(model):+ return "multi-device with offload or non-align hooks"+ return None+++def _fingerprint(model, state = None):+ """Parameters (address, dtype, shape, in-place version), quant state and adapter state; None+ when the model is not on CUDA cards only, or is split over cards in a way this path does not+ graph (offload, other hooks). `state` gets the other cards of a split model and the refusal."""+ from ..kernels import utils as K++ items = []+ host = _offloaded_input_embedding(model)+ # Its weights stay fingerprinted, by identity too (a swapped table re-captures); only their device is exempt.+ exempt = {id(p) for p in host.parameters()} if host is not None else ()+ for p in model.parameters():+ if id(p) in exempt:+ items.append(id(p))+ items.append((p.data_ptr(), p.dtype, tuple(p.shape), G._version(p)))+ devices = _param_devices(model)+ if not devices or any(d.type != _BACKEND.device_type for d in devices):+ if state is not None:+ state.fp_reason = "not one CUDA device / adapters"+ return None+ others = ()+ if len(devices) > 1:+ reason = _multidevice_reason(model, devices)+ if reason is not None:+ if state is not None:+ state.fp_reason = reason+ return None+ here = _BACKEND.current_device()+ others = tuple(sorted({_device_index(d) for d in devices} - {here}))+ if state is not None:+ state.others = others+ state.fp_reason = None+ items.append(("devices", tuple(sorted(str(d) for d in devices))))+ for b in model.buffers():+ items.append((b.data_ptr(), b.dtype, tuple(b.shape)))+ for module in model.modules():+ # A LoRA layer's scale is a Python float the captured kernels baked in; PEFT's set_scale / scale_layer+ # change it without touching a parameter.+ scaling = module.__dict__.get("scaling")+ if type(scaling) is dict and scaling:+ items.append((id(module), tuple(sorted((str(k), v) for k, v in scaling.items()))))+ weight = getattr(module, "weight", None)+ quant_state = getattr(weight, "quant_state", None)+ if quant_state is not None:+ absmax = getattr(quant_state, "absmax", None)+ items.append(+ (id(quant_state), absmax.data_ptr() if isinstance(absmax, torch.Tensor) else None)+ )+ adapters = G._adapter_state(model)+ if adapters is None:+ if state is not None:+ state.fp_reason = "not one CUDA device / adapters"+ return None+ items.extend(adapters)+ items.append(K._OPTIMIZER_STEPS)+ items.append(G.WEIGHTS_EPOCH)+ return tuple(items)+++# Forward derives positions from `cache_position[0] == 0` (host sync) without position_ids (4.x Qwen3-VL).+_MROPE_MODEL_TYPES = ("qwen3_vl", "qwen3_vl_moe")++# Type ids a trainer passes to generate() (TRL's GRPO: mm_token_type_ids / token_type_ids) reach every decode+# step at full length, a new graph key per step. Gemma 3 / 4 read them only to build masks, skipped when the+# masks come prebuilt (a dict per layer type), so with prebuilt masks the step gets the last q_len columns.+_TYPE_ID_ARGS = ("token_type_ids", "mm_token_type_ids")+++def _step_type_ids(kwargs, q_len):+ if not isinstance(kwargs.get("attention_mask"), dict):+ return kwargs+ out = None+ for name in _TYPE_ID_ARGS:+ value = kwargs.get(name)+ if isinstance(value, torch.Tensor) and value.dim() >= 2 and value.shape[-1] > q_len:+ if out is None:+ out = dict(kwargs)+ # As transformers slices them: a contiguous copy (a view's row stride would still grow every step).+ out[name] = value[..., -q_len:].clone(memory_format = torch.contiguous_format)+ return kwargs if out is None else out+++def _mrope_positions(model, kwargs):+ """kwargs with the decode position_ids the model's forward would derive, computed outside the+ graph (same ops, so the same values); unchanged when they do not apply."""+ if kwargs.get("position_ids") is not None:+ return kwargs+ if getattr(getattr(model, "config", None), "model_type", None) not in _MROPE_MODEL_TYPES:+ return kwargs+ inner = getattr(model, "model", None)+ deltas = getattr(inner, "rope_deltas", None)+ cache_position = kwargs.get("cache_position")+ ids = kwargs.get("input_ids")+ if ids is None:+ ids = kwargs.get("inputs_embeds")+ if not isinstance(deltas, torch.Tensor) or not isinstance(cache_position, torch.Tensor):+ return kwargs+ batch_size, seq_length = ids.shape[0], ids.shape[1]+ if deltas.shape[0] == 0 or batch_size % deltas.shape[0] != 0:+ return kwargs+ delta = (cache_position[0] + deltas).to(ids.device)+ position_ids = torch.arange(seq_length, device = ids.device)+ position_ids = position_ids.view(1, -1).expand(batch_size, -1)+ delta = delta.repeat_interleave(batch_size // delta.shape[0], dim = 0)+ position_ids = position_ids.add(delta)+ position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)+ return {**kwargs, "position_ids": position_ids}+++# Multimodal decoders whose forward zeroes placeholder ids with a boolean index (`ids[mask] = 0`, a host sync a+# capture cannot hold): the step's embeddings are looked up outside the graph with the same ops. Model type ->+# placeholder token ids.+_PLACEHOLDER_EMBED_MODEL_TYPES = {+ "muse_glimmer": ("image_token_id", "video_token_id"),+ "muse_spark": ("image_token_id", "video_token_id", "audio_token_id"),+ "kimi_k25": ("image_token_id", "video_token_id"),+}+++def _placeholder_embeds(model, kwargs):+ """kwargs with input_ids replaced by the inputs_embeds the model's forward would compute from them (a+ one-token text decode step: no media); unchanged when this does not apply."""+ config = getattr(model, "config", None)+ names = _PLACEHOLDER_EMBED_MODEL_TYPES.get(getattr(config, "model_type", None))+ ids = kwargs.get("input_ids")+ if (+ names is None+ or not isinstance(ids, torch.Tensor)+ or kwargs.get("inputs_embeds") is not None+ ):+ return kwargs+ if any(+ kwargs.get(k) is not None+ for k in kwargs+ if k.startswith(("pixel_values", "mm_encoder", "input_features"))+ ):+ return kwargs+ tokens = [getattr(config, name, None) for name in names]+ embed = getattr(model, "get_input_embeddings", None)+ if any(type(t) is not int for t in tokens) or embed is None:+ return kwargs+ mask = ids == tokens[0]+ for token in tokens[1:]:+ mask = mask | (ids == token)+ # As the forward's own lookup runs (no autograd under generate), on the decoder's card: an offloaded or+ # hooked embedding may hand back a host tensor or one that tracks its trainable weight.+ with torch.no_grad():+ embeds = embed()(ids.masked_fill(mask, 0))+ embeds = embeds.detach()+ if embeds.device != ids.device:+ embeds = embeds.to(ids.device)+ return {**kwargs, "input_ids": None, "inputs_embeds": embeds}+++def _offloaded_input_embedding(model):+ """The input embedding Unsloth's `offload_embedding` keeps in host RAM (its hooks move the ids there and the+ looked-up rows back to the decoder's card), or None. Not when the output head shares a weight with it."""+ get = getattr(model, "get_input_embeddings", None)+ try:+ module = get() if get is not None else None+ except Exception:+ return None+ if not isinstance(module, torch.nn.Module) or not getattr(+ module, "_unsloth_offload_hooks_installed", False+ ):+ return None+ params = list(module.parameters())+ if not params or all(p.device.type == _BACKEND.device_type for p in params):+ return None+ get_out = getattr(model, "get_output_embeddings", None)+ try:+ out = get_out() if get_out is not None else None+ except Exception:+ return None+ if isinstance(out, torch.nn.Module):+ shared = {id(p) for p in params}+ if any(id(p) in shared for p in out.parameters()):+ return None+ return module+++def _host_embeds(model, kwargs):+ """kwargs with input_ids replaced by the offloaded input embedding's rows, looked up outside the graph (a host+ lookup and copy cannot be captured) exactly as the forward would look them up; unchanged otherwise."""+ ids = kwargs.get("input_ids")+ if not isinstance(ids, torch.Tensor) or kwargs.get("inputs_embeds") is not None:+ return kwargs+ module = _offloaded_input_embedding(model)+ if module is None:+ return kwargs+ with torch.no_grad():+ embeds = module(ids)+ embeds = embeds.detach()+ if embeds.device != ids.device:+ embeds = embeds.to(ids.device)+ return {**kwargs, "input_ids": None, "inputs_embeds": embeds}+++def decode_step(+ model,+ forward,+ kwargs,+ eager_regions = None,+):+ """The outputs of one decode step run through a CUDA graph (captured on first use), or None for+ the caller's eager step. `forward` is the model's real forward; `eager_regions` an optional+ context manager (zoo `unsloth_eager_decode`) every graph-path step runs inside."""+ if torch.compiler.is_compiling() or torch.is_grad_enabled():+ return None+ if _BACKEND.capturing():+ return None+ state = _state(model)+ state.forwards += 1+ if state.forwards == 1:+ # The first forward is the prefill, even for a one-token prompt (Qwen3-VL sets rope_deltas there).+ return None+ ids = kwargs.get("input_ids")+ if ids is None:+ ids = kwargs.get("inputs_embeds")+ if (+ not isinstance(ids, torch.Tensor)+ or ids.dim() < 2+ or ids.shape[1] != 1+ or ids.device.type != _BACKEND.device_type+ ):+ return None+ state.step += 1+ if ids.shape[0] > MAX_BSZ:+ return _refusal(state, "batch over max")+ cache = kwargs.get("past_key_values")+ reason = _cache_reason(cache)+ if reason is None:+ reason = _window_reason(cache, 1)+ if reason is not None:+ return _refusal(state, reason)+ device = getattr(model, "device", None)+ if (+ isinstance(device, torch.device)+ and device.type == ids.device.type+ and device.index is not None+ and ids.device.index is not None+ and device.index != ids.device.index+ ):+ return _refusal(state, "inputs on another device")+ index = ids.device.index+ if index is not None and index != _BACKEND.current_device():+ with _BACKEND.device(index):+ return decode_step(model, forward, kwargs, eager_regions)++ from ..kernels import utils as K++ if state.recheck or state.optimizer_steps != K._OPTIMIZER_STEPS:+ fingerprint = _fingerprint(model, state)+ if fingerprint is None:+ state.invalidate()+ state.recheck = False+ state.optimizer_steps = K._OPTIMIZER_STEPS+ return _refusal(state, state.fp_reason or "not one CUDA device / adapters")+ if fingerprint != state.fingerprint:+ state.invalidate()+ state.fingerprint = fingerprint+ state.recheck = False+ state.optimizer_steps = K._OPTIMIZER_STEPS+ elif state.fingerprint is None:+ return _refusal(state, state.fp_reason or "not one CUDA device / adapters")++ kwargs = _host_embeds(+ model,+ _placeholder_embeds(model, _mrope_positions(model, _step_type_ids(kwargs, ids.shape[1]))),+ )+ leaves = []+ spec = []+ for name, value in kwargs.items():+ try:+ spec.append((name, _spec(value, leaves, cache)))+ except _Refuse as error:+ return _refusal(state, f"{error} ({name})")+ spec = tuple(spec)+ if not torch.is_inference_mode_enabled() and any(leaf.is_inference() for leaf in leaves):+ # Their static buffers could not be refilled outside inference mode.+ return _refusal(state, "inference tensor outside inference mode")+ layout = tuple(+ (type(layer).__name__, layer.max_cache_len, tuple(layer.keys.shape), layer.keys.dtype)+ for layer in cache.layers+ )+ # Ambient autocast incl. cache flag (Dynamo guards on it); cached casts outlive end() in generate().+ autocast = (G._autocast_state(), torch.is_autocast_cache_enabled())+ key = (+ spec,+ layout,+ _window_state(cache),+ autocast,+ torch.is_inference_mode_enabled(),+ eager_regions is not None,+ )+ refused = state.refused.get(key)+ if refused is not None:+ return _refusal(state, refused)+ # Graphs address this cache's tensors: a new cache re-captures; one reordered in place stays eager.+ cache_sig = (id(cache),) + tuple(+ (layer.keys.data_ptr(), layer.values.data_ptr()) for layer in cache.layers+ )+ if cache_sig != state.cache_sig:+ moved = state.cache_sig is not None and state.cache_sig[0] == id(cache)+ state.clear()+ if moved or state.moved == id(cache):+ state.moved = id(cache)+ return _refusal(state, "cache moved")+ state.cache_sig = cache_sig+ regions = eager_regions() if eager_regions is not None else contextlib.nullcontext()+ with regions:+ entry = state.graphs.get(key)+ if entry is None:+ if state.call_stop is None and state.unreplayed >= _MAX_UNREPLAYED_CAPTURES:+ state.call_stop = "graph key changes every step"+ sig = state.cache_sig+ state.clear()+ state.cache_sig = sig+ if state.call_stop is not None:+ return _refusal(state, state.call_stop)+ if _window_fills(cache, 1):+ # A filling-key graph already captured replays this step correctly; a new one would not.+ return _refusal(state, "sliding window fills in this step")+ return _capture(state, key, forward, kwargs, leaves, cache)+ for buffer, leaf in zip(entry.leaves, leaves):+ buffer.copy_(_compact(leaf))+ if not entry.validated or _validate_all():+ return _validate(state, key, entry, forward, kwargs, cache)+ _replay(state, entry)+ _advance(cache, 1)+ state.replays += 1+ state.unreplayed = 0+ return _map_outputs(entry.outputs, torch.Tensor.clone)+++def _replay(state, entry):+ """Replays from this card's current stream. A split model's segments run on their own cards'+ current streams: every card's stream waits on the others before (the eager work of the step+ before) and after (outputs and cache rows are read on any of them)."""+ if not state.others:+ entry.graph.replay()+ return+ streams = [_BACKEND.current_stream()] + [_BACKEND.device_stream(d) for d in state.others]+ for a in streams:+ for b in streams:+ if a is not b:+ a.wait_stream(b)+ entry.graph.replay()+ for a in streams:+ for b in streams:+ if a is not b:+ a.wait_stream(b)+++def _compact(leaf):+ """`leaf` without its broadcast (stride 0) dimensions: one row of each."""+ if 0 not in leaf.stride():+ return leaf+ return leaf[tuple(slice(0, 1) if st == 0 else slice(None) for st in leaf.stride())]+++def _empty_strided_like(leaf):+ # empty_like makes a non-dense view (a sliced mask) dense: keep eager's strides exactly.+ return torch.empty_strided(leaf.shape, leaf.stride(), dtype = leaf.dtype, device = leaf.device)+++def _buffer_like(leaf):+ if leaf.is_inference():+ with torch.inference_mode():+ return _empty_strided_like(leaf)+ with G._no_inference_tensors():+ return _empty_strided_like(leaf)+++def _capture(state, key, forward, kwargs, leaves, cache):+ """Runs this step eagerly (warm-up, its outputs are returned), then captures it."""+ if len(state.graphs) >= G.MAX_GRAPHS:+ sig = state.cache_sig+ state.clear()+ state.cache_sig = sig+ try:+ if state.stream is not None and state.stream.device.index != _BACKEND.current_device():+ state.stream = None+ state.pool = None+ if state.pool is None:+ state.pool = _BACKEND.pool()+ state.mempools = {}+ for d in state.others:+ if d not in state.mempools:+ state.mempools[d] = _BACKEND.mempool()+ if d not in state.side:+ state.side[d] = _BACKEND.side_stream(d)+ side = {d: state.side[d] for d in state.others}+ if state.stream is None:+ state.stream = _BACKEND.new_stream()+ stream = state.stream+ # Static buffers match their argument (dtype, strides, inference-ness): same compiled variants as eager.+ buffers = [_buffer_like(_compact(leaf)) for leaf in leaves]+ for buffer, leaf in zip(buffers, leaves):+ buffer.copy_(_compact(leaf))+ it = iter([buffer.expand(leaf.shape) for buffer, leaf in zip(buffers, leaves)])+ static_kwargs = {k: _build(v, it, cache) for k, v in kwargs.items()}+ except Exception as error:+ reason = f"capture setup failed: {type(error).__name__}"+ state.pool = None+ state.refused[key] = reason+ state.refusals[reason] += 1+ return forward(**kwargs)+ before = _host_state(cache)+ stream.wait_stream(_BACKEND.current_stream())+ graph = None+ try:+ # A split model's other cards run on their side streams in the warm-up too (their cuBLAS+ # workspaces exist before the capture).+ with (+ _BACKEND.stream(stream),+ _BACKEND.on_streams(side) if side else contextlib.nullcontext(),+ _graphable_rolls(),+ ):+ outputs = forward(**static_kwargs)+ stream.synchronize()+ reason = _host_state_reason(before, _host_state(cache), 1)+ if reason is not None:+ state.refused[key] = reason+ state.refusals[reason] += 1+ return outputs+ after = _snapshot(cache, rows = False)+ graph = False+ try:+ if side:+ graph, static_outputs = _BACKEND.record(+ state.pool, lambda: forward(**static_kwargs), cache, side, state.mempools+ )+ else:+ graph, static_outputs = _BACKEND.record(+ state.pool, lambda: forward(**static_kwargs), cache+ )+ finally:+ # Capturing ran the Python update again: put the host counters back.+ _restore((after[0], []))+ except Exception as error:+ if graph is None:+ raise+ if isinstance(error, torch.cuda.OutOfMemoryError):+ # Out of memory while recording: free every graph of this call and decode the rest of it eagerly.+ # Not refused for the process (the next call may have the room).+ reason = "capture failed: out of memory"+ if len(state.capture_errors) < 4:+ state.capture_errors.append((reason, _where(error)))+ sig = state.cache_sig+ state.clear()+ state.cache_sig = sig+ state.call_stop = reason+ state.refusals[reason] += 1+ return outputs+ message = " ".join(str(error).split())[:200]+ reason = f"capture failed: {type(error).__name__}: {message}"+ if len(state.capture_errors) < 4:+ state.capture_errors.append((reason, _where(error)))+ state.pool = None+ state.refused[key] = reason+ state.refusals[reason] += 1+ if os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_STRICT", "0") == "1":+ raise+ return outputs+ finally:+ # Also after a failed capture: the eager steps must not race the side stream.+ _BACKEND.current_stream().wait_stream(stream)+ state.graphs[key] = _Entry(graph, buffers, static_outputs, key in state.validated)+ state.captures += 1+ state.unreplayed += 1+ state.key_captures[hash(key)] += 1+ return outputs+++def _validate(state, key, entry, forward, kwargs, cache):+ """First replay of a key must match the eager step bitwise, else the key is refused. Eager runs+ last, so generation continues from its result and cache rows."""+ state.validations += 1+ start = _snapshot(cache)+ _replay(state, entry)+ _restore(start)+ eager = forward(**kwargs)+ equal = _outputs_equal(eager, entry.outputs)+ _record_check(state, eager, entry.outputs, equal)+ if equal or entry.validated:+ # VALIDATE_ALL only records: generation stays on the eager tokens, so steps share a prefix.+ entry.validated = True+ state.validated.add(key)+ state.replays += 1+ state.unreplayed = 0+ return eager+ reason = "validation mismatch"+ state.refused[key] = reason+ state.refusals[reason] += 1+ state.graphs.pop(key, None)+ if os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_STRICT", "0") == "1":+ raise RuntimeError("Unsloth: decode CUDA graph replay differs from the eager step")+ return eagerdiff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 104efb17707..a592da20e6d 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -3123,6 +3123,7 @@ def unsloth_fast_generate(self, *args, **kwargs):try:with (_decode_graphs.eager_scope() if plan == "eager" else contextlib.nullcontext(),+ _decode_graphs.plan_scope("graph" if graphs else "eager"),_get_inference_mode_context_manager(self),torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),):diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 9630ae83ea9..098f3822808 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -292,6 +292,7 @@ from ._custom_dtype import cast_mixed_linear_inputs, resolve_dtype, trusted_custfrom ._mamba_lora import patch_mamba_fused_lorafrom .remote_code_shims import apply_remote_code_shimsfrom .grouped_linear_lora import register_grouped_linear_lora+from . import _decode_graphs, _generic_decode_graphsfrom .loader_utils import (_bnb_bits_requested,vlm_gradient_checkpointing_reentrant,@@ -1504,6 +1505,9 @@ def _eager_decodes(model):return _is_decode_compile_model(model) and "forward" not in model.__dict__+_NO_FORWARD = object()++class _EagerDecodeSteps:"""One generate() call. Each eager decode step (one new token per row) runs inside zoo's`unsloth_eager_decode()`, so compiled regions call their eager originals; prefill and@@ -1531,14 +1535,99 @@ class _EagerDecodeSteps:def forward(*args, **kwargs):return step(*args, **kwargs)+ self._previous = self.model.__dict__.get("forward", _NO_FORWARD)self.model.forward = forwardreturn selfdef __exit__(self, *exc):- del self.model.forward+ if self._previous is _NO_FORWARD:+ del self.model.forward+ else:+ self.model.forward = self._previousreturn False+try:+ from unsloth_zoo.decode_graph_state import (+ graph_generate as _zoo_graph_generate,+ graph_decode_step as _zoo_graph_decode_step,+ )+except ImportError:+ _zoo_graph_generate = _zoo_graph_decode_step = None+++def _one_token_step(kwargs):+ ids = kwargs.get("input_ids")+ if ids is None:+ ids = kwargs.get("inputs_embeds")+ return isinstance(ids, torch.Tensor) and ids.dim() >= 2 and ids.shape[1] == 1+++class _GraphDecodeSteps(_EagerDecodeSteps):+ """One generate() call over a static cache with decode CUDA graphs on: each one-token decode+ step replays a captured graph of the step the eager path runs (_generic_decode_graphs), else+ runs as before (inside `unsloth_eager_decode()` for the models that compile their decode)."""++ def __init__(self, model):+ super().__init__(model)+ self.eager_decodes = _eager_decodes(model)+ # Diagnostic: =1 graphs the eager originals of compiled regions (not what eager decode runs).+ self.eager_regions = self.eager_decodes or (+ os.environ.get("UNSLOTH_DECODE_GENERIC_GRAPHS_EAGER_REGIONS", "0") == "1"+ )+ # Set by generate() after the call: keep the graphs and their cache for the next call.+ self.keep = False+ self.caches = _generic_decode_graphs.reuse_kept_cache(model)++ def _forward(self, *args, **kwargs):+ # zoo picks its graph-safe decode kernels (gpt-oss dense sink attention, routed MoE below sm80) for+ # one-token steps in this scope: the eager warm-up, the capture and every eager step agree.+ if _zoo_graph_decode_step is not None and _one_token_step(kwargs):+ with _zoo_graph_decode_step():+ return self._graph_forward(*args, **kwargs)+ return self._graph_forward(*args, **kwargs)++ def _graph_forward(self, *args, **kwargs):+ if not args and not torch.compiler.is_compiling():+ # Compiled regions stay compiled unless this model's eager decode leaves them: a+ # different kernel mix rounds differently and greedy tokens drift after a few dozen steps.+ out = _generic_decode_graphs.decode_step(+ self.model,+ self.forward,+ kwargs,+ unsloth_eager_decode if self.eager_regions else None,+ )+ if out is not None:+ return out+ if self.eager_decodes:+ return super()._forward(*args, **kwargs)+ return self.forward(*args, **kwargs)++ def __enter__(self):+ _generic_decode_graphs.begin(self.model)+ self.caches.__enter__()+ # Masks are built in prepare_inputs_for_generation, outside the forward: the call-wide scope covers them.+ self._zoo_scope = _zoo_graph_generate() if _zoo_graph_generate is not None else None+ if self._zoo_scope is not None:+ self._zoo_scope.__enter__()+ try:+ return super().__enter__()+ except BaseException:+ if self._zoo_scope is not None:+ self._zoo_scope.__exit__(None, None, None)+ self.caches.__exit__(None, None, None)+ raise++ def __exit__(self, *exc):+ try:+ return super().__exit__(*exc)+ finally:+ if self._zoo_scope is not None:+ self._zoo_scope.__exit__(None, None, None)+ self.caches.__exit__(*exc)+ _generic_decode_graphs.end(self.model, keep = self.keep and exc[0] is None)++# One-token decode is slower on flex than SDPA, but prefill must stay on flex (its block mask skips tiles a# dense mask computes). The path is picked in prepare_inputs_for_generation, where the step's mask is built._FLEX_GENERATE_ENV_VAR = "UNSLOTH_FLEX_ATTENTION_GENERATE"@@ -2155,6 +2244,9 @@ def _uses_flash_attention_for_generation(config):def _clear_generation_caches(model):+ # The StaticCache decode graphs keep for the next call stays. Guarded: tests exec this function alone.+ graphs = globals().get("_generic_decode_graphs")+ kept = graphs.kept_cache(model) if graphs is not None else Nonefor name, module in model.named_modules():if hasattr(module, "_flex_attention_cache"):try:@@ -2162,6 +2254,8 @@ def _clear_generation_caches(model):except:pass# Solves AttributeError: 'SlidingWindowLayer' object has no attribute 'max_batch_size'.+ if kept is not None and module.__dict__.get("_cache") is kept:+ continueif hasattr(module, "_cache") and "cache_utils" in str(module._cache.__class__):try:del module._cache@@ -2387,16 +2481,49 @@ def unsloth_base_fast_generate(self, *args, **kwargs):kwargs["compile_config"] = compile_configdecode_scope = _CompileDecodeOnRepeat(self) if compile_decode else contextlib.nullcontext()- eager_scope = _EagerDecodeSteps(self) if _eager_decodes(self) else contextlib.nullcontext()- try:++ def _eager_scope():+ return _EagerDecodeSteps(self) if _eager_decodes(self) else contextlib.nullcontext()++ def _run(scope, plan):with (_NonFlexGenerate(self),decode_scope,- eager_scope,+ scope,+ _decode_graphs.plan_scope(plan),_get_inference_mode_context_manager(self),autocaster,):- output = self._old_generate(*args, **kwargs)+ out = self._old_generate(*args, **kwargs)+ if isinstance(scope, _GraphDecodeSteps):+ scope.keep = _generic_decode_graphs.keepable(self, kwargs, out)+ return out++ try:+ with _decode_graphs.generate_scope():+ if cache_implementation == "static" and not force_dynamic_cache and not compile_decode:+ plan = _generic_decode_graphs.plan(self, args, kwargs, input_ids)+ else:+ plan = _generic_decode_graphs.ineligible(self)+ if plan != "graph":+ output = _run(_eager_scope(), "eager")+ else:+ rng = _generic_decode_graphs.rng_state(input_ids.device)+ out_of_memory = False+ try:+ output = _run(_GraphDecodeSteps(self), "graph")+ except torch.cuda.OutOfMemoryError:+ if not _generic_decode_graphs.replayable(kwargs):+ raise+ out_of_memory = True+ if out_of_memory:+ # Outside the handler: its traceback holds the failed call's frames (cache, graph+ # buffers), which would stay allocated through the redo.+ _generic_decode_graphs.redo_eagerly(self)+ _clear_generation_caches(self)+ _generic_decode_graphs.set_rng_state(rng)+ with _decode_graphs.eager_scope():+ output = _run(_eager_scope(), "eager")finally:_clear_generation_caches(self)@@ -6075,6 +6202,7 @@ class FastBaseModel:"Unsloth: I think you're passing a tokenizer, not the model to for_training!")_tensor_parallel_for_training(model)+ _decode_graphs.release_for_training(model)# Mode last, as model.train() was: a custom train() keeps the final say over the chain below.checkpointing, apply_training_mode = switch_training_mode(model, True, defer = True)use_gradient_checkpointing = resolve_training_gradient_checkpointing(