Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Decode CUDA graphs on by default for every FastModel model (2-8x on T4, tokens equal to eager) (#16587)

Daniel Han00d10566e67d
diff --git a/tests/test_generic_decode_graphs.py b/tests/test_generic_decode_graphs.py
new file mode 100644
index 00000000000..f6ab77456a3
--- /dev/null
+++ b/tests/test_generic_decode_graphs.py
@@ -0,0 +1,901 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FastModel decode CUDA graphs on the host, through a stand-in backend whose "graph" re-runs the
+captured step into the same static outputs. The real capture is the gpu-marked test."""
+
+import contextlib
+import functools
+import inspect
+import os
+import sys
+import types
+
+import pytest
+import torch
+
+import unsloth # noqa: F401 (must precede transformers)
+
+from real_accelerator import has_real_cuda
+import unsloth.models._decode_graphs as G
+import unsloth.models._generic_decode_graphs as GG
+from unsloth.models import vision
+
+from torch._dynamo.utils import counters
+from transformers import AutoModelForCausalLM, DynamicCache, StaticCache
+from transformers import cache_utils
+
+LLAMA = "hf-internal-testing/tiny-random-LlamaForCausalLM"
+GEMMA3 = "hf-internal-testing/tiny-random-Gemma3ForCausalLM"
+QWEN3_VL = "trl-internal-testing/tiny-Qwen3VLForConditionalGeneration"
+NO_ROLL = "no static sliding layer with the host-built roll index in this transformers"
+
+
+def test_imports_this_tree():
+ here = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ assert os.path.abspath(GG.__file__).startswith(here)
+
+
+class _FakeStream:
+ class device:
+ index = None
+
+ wait_stream = lambda self, other: None
+ synchronize = lambda self: None
+
+
+def _copy_into(dst, src):
+ xs, ys = GG._tensors(dst, []), GG._tensors(src, [])
+ assert len(xs) == len(ys)
+ for x, y in zip(xs, ys):
+ x.copy_(y)
+
+
+class _FakeGraph:
+ """Replay = the captured step run again into the captured outputs; host counters are left
+ as a real replay leaves them (unchanged). perturb: True = every replay, an int n = from the
+ n-th replay on (drift after validation)."""
+
+ def __init__(self, fn, outputs, cache, perturb):
+ self.fn, self.outputs, self.cache, self.perturb = fn, outputs, cache, perturb
+ self.replays = 0
+
+ def replay(self):
+ self.replays += 1
+ host = GG._snapshot(self.cache)[0]
+ new = self.fn()
+ GG._restore((host, []))
+ _copy_into(self.outputs, new)
+ if self.perturb is True or (type(self.perturb) is int and self.replays >= self.perturb):
+ GG._tensors(self.outputs, [])[0][..., 0].add_(1)
+
+
+class _Host(GG._Cuda):
+ device_type = "cpu"
+ perturb = False
+ fail = False
+ graphs = []
+ # As a real capture: no host-to-device copies (torch.tensor(..., device=...)) while recording.
+ recording = False
+ current_device = staticmethod(lambda: None)
+ device = staticmethod(lambda index: contextlib.nullcontext())
+ pool = staticmethod(lambda: object())
+ new_stream = staticmethod(lambda: _FakeStream())
+ stream = staticmethod(lambda stream: contextlib.nullcontext())
+ current_stream = staticmethod(lambda: _FakeStream())
+
+ @classmethod
+ def capturing(cls):
+ return cls.recording
+
+ @classmethod
+ def record(cls, pool, fn, cache):
+ if cls.fail:
+ error = cls.fail if isinstance(cls.fail, BaseException) else None
+ raise error or RuntimeError("operation not permitted when stream is capturing")
+ # A capture launches nothing: put back the length counters and every cache row the step wrote.
+ snapshot = GG._snapshot(cache)
+ rows = [
+ (t, t.clone())
+ for layer in cache.layers
+ for t in (getattr(layer, "keys", None), getattr(layer, "values", None))
+ if isinstance(t, torch.Tensor)
+ ]
+ real_tensor = torch.tensor
+
+ def tensor(*args, **kwargs):
+ if kwargs.get("device") is not None:
+ raise RuntimeError(
+ "Cannot copy between CPU and CUDA tensors during CUDA graph capture unless the CPU tensor is pinned."
+ )
+ return real_tensor(*args, **kwargs)
+
+ cls.recording = True
+ torch.tensor = tensor
+ try:
+ outputs = fn()
+ finally:
+ torch.tensor = real_tensor
+ cls.recording = False
+ GG._restore(snapshot)
+ for live, saved in rows:
+ live.copy_(saved)
+ graph = _FakeGraph(fn, outputs, cache, cls.perturb)
+ cls.graphs.append(graph)
+ return graph, outputs
+
+
+def host(monkeypatch):
+ backend = type("Host", (_Host,), {"graphs": []})
+ monkeypatch.setattr(GG, "_BACKEND", backend)
+ return backend
+
+
[email protected](scope = "module")
+def llama():
+ torch.manual_seed(0)
+ return AutoModelForCausalLM.from_pretrained(LLAMA, dtype = torch.float32).eval()
+
+
[email protected](scope = "module")
+def gemma3():
+ torch.manual_seed(0)
+ # A short window, so a short generation runs past it.
+ return AutoModelForCausalLM.from_pretrained(
+ GEMMA3, dtype = torch.float32, sliding_window = 8
+ ).eval()
+
+
+def _fresh(model):
+ model.__dict__.pop(GG._ATTR, None)
+
+
+def _ids(
+ model,
+ bsz = 1,
+ length = 6,
+ pad = 0,
+):
+ g = torch.Generator().manual_seed(1)
+ vocab = model.config.get_text_config().vocab_size
+ # Below the multimodal special tokens (image / video pads sit at the top of the vocabulary).
+ ids = torch.randint(3, min(vocab, 1000), (bsz, length), generator = g)
+ mask = torch.ones_like(ids)
+ if pad:
+ ids[0, :pad] = 0
+ mask[0, :pad] = 0
+ return ids, mask
+
+
+def _generate(
+ model,
+ ids,
+ mask,
+ graphs,
+ new = 10,
+):
+ scope = vision._GraphDecodeSteps(model) if graphs else contextlib.nullcontext()
+ with torch.inference_mode(), scope:
+ return model.generate(
+ input_ids = ids,
+ attention_mask = mask,
+ max_new_tokens = new,
+ min_new_tokens = new,
+ do_sample = False,
+ cache_implementation = "static",
+ pad_token_id = 0,
+ # transformers 4.57 compiles static-cache generation on CUDA, which the graph path steps aside for.
+ disable_compile = True,
+ )
+
+
+def _both(
+ model,
+ ids,
+ mask,
+ new = 10,
+):
+ """Eager then graphed tokens for a fresh model state; asserts they match, returns stats."""
+ _fresh(model)
+ eager = _generate(model, ids, mask, False, new)
+ assert torch.equal(eager, _generate(model, ids, mask, True, new))
+ return GG.stats(model)
+
+
+def _scope(model, forward, eager):
+ scope = vision._GraphDecodeSteps.__new__(vision._GraphDecodeSteps)
+ scope.model, scope.forward = model, forward
+ scope.eager_decodes = scope.eager_regions = eager
+ return scope
+
+
+def _sliding_layer(cls):
+ try:
+ return cls(max_cache_len = 32, sliding_window = 4)
+ except TypeError:
+ return cls(sliding_window = 4)
+
+
[email protected]("bsz,pad,length", [(1, 0, 6), (3, 2, 6), (1, 0, 1)])
+def test_greedy_tokens_match_eager_llama(host, llama, bsz, pad, length):
+ s = _both(llama, *_ids(llama, bsz, length, pad))
+ # Step 1 = prefill (eager, also for a one-token prompt), step 2 eager warm-up + capture, step 3
+ # validated replay, then replays: 9 decode steps take the graph path.
+ assert GG._state(llama).step == 9
+ assert s["captures"] == 1 and s["validations"] == 1 and s["replays"] == 8, s
+ assert s["refusals"] == {}, s
+ assert s["graphs"] == 0 # freed when generate() returns
+ assert "forward" not in llama.__dict__
+
+
+def test_batches_over_the_knob_decode_eagerly(host, llama, monkeypatch):
+ monkeypatch.setattr(GG, "MAX_BSZ", 2)
+ ids, mask = _ids(llama, 3, pad = 1)
+ s = _both(llama, ids, mask)
+ assert s["captures"] == 0 and s["refusals"] == {"batch over max": 9}, s
+ monkeypatch.setattr(GG, "MAX_BSZ", 3)
+ _generate(llama, ids, mask, True)
+ assert GG.stats(llama)["replays"] == 8
+
+
+def test_counts_captures_refusals_and_recompiles_separately(host, llama):
+ def compiles():
+ return counters["stats"]["unique_graphs"], sum(counters["recompiles"].values())
+
+ layer = torch.nn.Linear(4, 4)
+ compiled = torch.compile(lambda x: layer(x).relu(), backend = "eager")
+ compiled(torch.ones(2, 4))
+ ids, mask = _ids(llama, 2, pad = 1)
+ # Warm the compiled mask builder for these shapes eagerly first, so the count does not depend on test order.
+ _generate(llama, ids, mask, False)
+ before = compiles()
+ _fresh(llama)
+ _generate(llama, ids, mask, True)
+ compiled(torch.ones(2, 4))
+ # Decode graphs add no Dynamo graph and no recompile of an existing compiled region.
+ assert compiles() == before
+ s = GG.stats(llama)
+ assert s["captures"] == 1 and s["max_captures_per_key"] == 1 and s["refusals"] == {}, s
+ # A second call captures again but does not revalidate the key.
+ _generate(llama, ids, mask, True)
+ s = GG.stats(llama)
+ assert s["captures"] == 2 and s["max_captures_per_key"] == 1 and s["validations"] == 1, s
+ assert sum(g.replays for g in host.graphs) == 2 * 8
+
+
+def test_graph_path_runs_the_compiled_variant_eager_runs(host, llama):
+ """A compiled region on the step's arguments (zoo compiles create_causal_mask) guards on their
+ inference-ness and strides: the graph path must hit the variant the eager step compiled, never
+ a new one, and give the eager logits. (Dynamo's global-state guard also covers the autocast cache
+ flag, so the step keeps the ambient autocast; CUDA autocast is not reachable on the host.)"""
+ seen, calls = [], []
+
+ @torch.compile(backend = "eager", dynamic = False)
+ def region(mask, position):
+ # A different variant would add to the logits: the tokens would move.
+ flip = 0.0 if mask.is_inference() else 1000.0
+ return mask.float().mean() * 0 + position.float().mean() * 0 + flip
+
+ def pre_hook(module, args, kwargs):
+ seen.append(kwargs.get("attention_mask"))
+ # transformers 5.x passes no cache_position: fall back to the positions.
+ position = kwargs.get("cache_position")
+ seen.append(position if position is not None else kwargs.get("position_ids"))
+
+ def head_hook(module, args, out):
+ mask, position = seen[-2], seen[-1]
+ if mask is None or position is None:
+ return out
+ if isinstance(mask, dict):
+ mask = next(iter(mask.values()))
+ calls.append(1)
+ return out + region(mask, position)
+
+ handles = [
+ llama.model.register_forward_pre_hook(pre_hook, with_kwargs = True),
+ llama.lm_head.register_forward_hook(head_hook),
+ ]
+ try:
+ _fresh(llama)
+ ids, mask = _ids(llama, 3, pad = 2)
+ eager = _generate(llama, ids, mask, False)
+ assert calls, "the compiled region never ran: the check would be vacuous"
+ before = counters["stats"]["unique_graphs"]
+ graphed = _generate(llama, ids, mask, True)
+ assert counters["stats"]["unique_graphs"] == before
+ assert torch.equal(eager, graphed)
+ s = GG.stats(llama)
+ assert s["replays"] == 8 and not s["refusals"], s
+ finally:
+ for h in handles:
+ h.remove()
+
+
[email protected]("make", ["dense", "sliced", "broadcast", "sliced_broadcast"])
+def test_static_buffers_keep_eager_strides(host, make):
+ base = torch.arange(64, dtype = torch.float32).view(2, 32)
+ leaf = {
+ "dense": base,
+ "sliced": base[:, ::2], # shape (2, 16), strides (32, 2)
+ "broadcast": base[:1].expand(4, 32), # as Qwen3-VL positions
+ "sliced_broadcast": base[:1, ::2].expand(3, 16),
+ }[make]
+ compact = GG._compact(leaf)
+ if "broadcast" in make:
+ assert compact.shape[0] == 1
+ buffer = GG._buffer_like(compact)
+ buffer.copy_(compact)
+ replayed = buffer.expand(leaf.shape)
+ assert replayed.stride() == leaf.stride() and torch.equal(replayed, leaf)
+ with torch.inference_mode():
+ x = torch.arange(6.0).view(2, 3)
+ assert GG._buffer_like(x).is_inference() and not GG._buffer_like(base).is_inference()
+ t = torch.zeros(4, 6).t() # non-contiguous
+ assert GG._buffer_like(t).stride() == t.stride()
+ spec = GG._spec(t, [], None)
+ assert t.stride() in spec and False in spec
+
+
[email protected]("past", [False, True])
+def test_sliding_window_roll_is_captured_without_host_copies(host, gemma3, monkeypatch, past):
+ # Inside the window every sliding layer updates by index: one key, every step after the first two replays.
+ # Past the window the sliding layers roll by one slot per step: a second key (window full) is captured and
+ # validated (its rolled rows restored between the replay and the eager step), then replays. transformers
+ # rolls a full window with an index made by torch.tensor(..., device=...), a host-to-device copy a real
+ # capture refuses; the warm-up and recording run that branch on an index made outside the capture. The
+ # recording runs no device work, so the capture holds no copy of the full windows' rows while it records.
+ taken = []
+ snapshot = GG._snapshot
+
+ def recorder(cache, *args, **kwargs):
+ out = snapshot(cache, *args, **kwargs)
+ if sys._getframe(1).f_code.co_name == "_capture":
+ taken.append(max((e[1].numel() for e in out[1]), default = 0))
+ return out
+
+ monkeypatch.setattr(GG, "_snapshot", recorder)
+ assert "sliding_attention" in gemma3.config.layer_types
+ new = gemma3.config.sliding_window + (12 if past else -3)
+ keys = 1 + past
+ s = _both(gemma3, *_ids(gemma3, 2, length = 3, pad = 1), new)
+ assert not s["refusals"], s
+ assert s["captures"] == keys and s["validations"] == keys, s
+ assert s["replays"] == new - 1 - keys, s
+ assert len(taken) == keys and max(taken) <= 1, taken
+ for cls, _ in GG._rolling_update_classes():
+ assert not getattr(cls.__dict__["update"], "_unsloth_graphable_roll", False)
+
+
+def test_beam_search_reordering_the_cache_stays_eager(host, llama):
+ _fresh(llama)
+ ids, mask = _ids(llama)
+ kw = dict(max_new_tokens = 8, min_new_tokens = 8, num_beams = 2, do_sample = False)
+ kw.update(cache_implementation = "static", pad_token_id = 0, attention_mask = mask)
+ with torch.inference_mode():
+ eager = llama.generate(ids, **kw)
+ with vision._GraphDecodeSteps(llama):
+ graphed = llama.generate(ids, **kw)
+ assert torch.equal(eager, graphed)
+ s = GG.stats(llama)
+ moved = s["refusals"].get("cache moved", 0)
+ # Either the reorder kept the cache in place (all replays) or every step after it ran eagerly.
+ assert (moved == 0 and s["replays"] > 0) or s["replays"] + moved + s["captures"] + s[
+ "validations"
+ ] == 7, s
+
+
+def test_qwen3_vl_text_decode_positions_match_the_model(host, monkeypatch):
+ import transformers
+ from transformers import AutoModelForImageTextToText
+
+ torch.manual_seed(0)
+ model = AutoModelForImageTextToText.from_pretrained(QWEN3_VL, dtype = torch.float32).eval()
+ seen = []
+ real = GG._mrope_positions
+
+ def spy(m, kwargs):
+ out = real(m, kwargs)
+ seen.append(out is not kwargs)
+ return out
+
+ monkeypatch.setattr(GG, "_mrope_positions", spy)
+ s = _both(model, *_ids(model, 2, length = 5, pad = 1))
+ assert s["replays"] == 8 and not s["refusals"], s
+ if transformers.__version__.startswith("4."):
+ # 4.x leaves decode positions to the forward, which reads cache_position[0] on the host.
+ assert seen and all(seen)
+
+
+def test_validation_mismatch_refuses_the_key(host, llama):
+ host.perturb = True
+ ids, mask = _ids(llama)
+ s = _both(llama, ids, mask)
+ assert s["refusals"].get("validation mismatch", 0) >= 1 and s["replays"] == 0, s
+ # Refused for the process: the next call does not capture again.
+ _generate(llama, ids, mask, True)
+ assert GG.stats(llama)["captures"] == 1
+
+
+def test_validate_all_pinpoints_a_later_drift(host, llama, monkeypatch):
+ _fresh(llama)
+ host.perturb = 4 # bitwise at the validated step, then the replay drifts
+ ids, mask = _ids(llama)
+ eager = _generate(llama, ids, mask, False)
+ monkeypatch.setenv("UNSLOTH_DECODE_GENERIC_GRAPHS_VALIDATE_ALL", "1")
+ # Every replay is compared and the eager result returned: tokens stay eager's.
+ assert torch.equal(eager, _generate(llama, ids, mask, True))
+ check = GG.stats(llama)["check"]
+ # Decode step 1 captures, step 2 validates (replay 1), drift from replay 4 = decode step 5.
+ assert check["steps"] == 8 and check["mismatches"] == 5, check
+ assert check["first_mismatch_step"] == 5 and check["max_abs_diff"] == 1.0, check
+ assert check["max_kl"] >= 0.0 and check["top1_disagree"] >= 0
+ monkeypatch.delenv("UNSLOTH_DECODE_GENERIC_GRAPHS_VALIDATE_ALL")
+ # Without it, the drift goes unseen once the key is trusted: why the debug mode exists.
+ _fresh(llama)
+ _generate(llama, ids, mask, True)
+ assert GG.stats(llama)["check"]["mismatches"] == 0
+
+
+def test_graph_path_runs_the_eager_path_regions(monkeypatch):
+ seen = []
+ monkeypatch.setattr(
+ vision._generic_decode_graphs,
+ "decode_step",
+ lambda model, forward, kwargs, regions: seen.append(regions),
+ )
+ model = torch.nn.Linear(1, 1)
+ model.forward = lambda **kw: 0
+ for compiled_decode in (False, True):
+ _scope(model, model.forward, compiled_decode)._forward(input_ids = torch.zeros(1, 1))
+ # Compiled regions stay compiled unless this model's eager decode already leaves them.
+ assert seen == [None, vision.unsloth_eager_decode]
+
+
+def test_capture_failure_falls_back_for_the_key(host, llama):
+ host.fail = True
+ s = _both(llama, *_ids(llama))
+ assert s["captures"] == 0 and s["replays"] == 0, s
+ # Refused once for the process; every later step of the key counts as a refused eager step.
+ reason = "capture failed: RuntimeError: operation not permitted when stream is capturing"
+ assert s["refused_keys"] == 1 and s["refusals"] == {reason: 9}, s
+ # What broke it and where (innermost frame outside torch).
+ assert s["capture_errors"] == [(reason, s["capture_errors"][0][1])], s
+ assert "test_generic_decode_graphs.py" in s["capture_errors"][0][1], s
+ assert s["eager_steps"] == 8, s
+
+
+def test_out_of_memory_capture_stops_the_call_not_the_key(host, llama):
+ host.fail = torch.cuda.OutOfMemoryError("CUDA out of memory. Tried to allocate 2.00 MiB")
+ ids, mask = _ids(llama)
+ s = _both(llama, ids, mask)
+ assert s["captures"] == 0 and s["refused_keys"] == 0, s
+ assert s["refusals"] == {"capture failed: out of memory": 9}, s
+ host.fail = False
+ assert torch.equal(_generate(llama, ids, mask, False), _generate(llama, ids, mask, True))
+ s = GG.stats(llama)
+ assert s["captures"] == 1 and s["replays"] == 8, s
+
+
+def test_capture_setup_failure_runs_eagerly(host, llama, monkeypatch):
+ def no_pool():
+ raise torch.OutOfMemoryError("pool")
+
+ monkeypatch.setattr(host, "pool", staticmethod(no_pool))
+ s = _both(llama, *_ids(llama))
+ assert s["replays"] == 0 and any("capture setup failed" in r for r in s["refusals"]), s
+
+
+def test_fingerprint_invalidation(host, llama, monkeypatch):
+ from unsloth.kernels import utils as K
+
+ _fresh(llama)
+ first = GG._fingerprint(llama)
+ assert first is not None and first == GG._fingerprint(llama)
+ with torch.no_grad():
+ next(llama.parameters()).add_(0) # in-place edit (merge, optimizer step): bumps the version
+ assert GG._fingerprint(llama) != first
+ before = GG._fingerprint(llama)
+ monkeypatch.setattr(K, "_OPTIMIZER_STEPS", K._OPTIMIZER_STEPS + 1)
+ assert GG._fingerprint(llama) != before
+ # A step after the optimizer moved re-checks, and a changed fingerprint drops the graphs.
+ state = GG._state(llama)
+ state.fingerprint = ("stale",)
+ state.graphs["k"] = object()
+ state.recheck = False
+ _generate(llama, *_ids(llama), True)
+ assert state.fingerprint != ("stale",)
+
+
+def test_lora_scale_changes_the_fingerprint(host, llama):
+ # PEFT's set_scale / scale_layer move a LoRA layer's Python scale without touching a parameter; the captured
+ # kernels baked it in, so a kept graph must not replay across it.
+ _fresh(llama)
+ proj = llama.model.layers[0].self_attn.o_proj
+ proj.scaling = {"default": 1.0}
+ try:
+ first = GG._fingerprint(llama)
+ assert first == GG._fingerprint(llama)
+ proj.scaling["default"] = 0.0
+ assert GG._fingerprint(llama) != first
+ proj.scaling["default"] = 1.0
+ assert GG._fingerprint(llama) == first
+ finally:
+ del proj.scaling
+ peft = pytest.importorskip("peft")
+ model = peft.get_peft_model(
+ AutoModelForCausalLM.from_pretrained(LLAMA, dtype = torch.float32).eval(),
+ peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["o_proj"]),
+ )
+ first = GG._fingerprint(model)
+ model.base_model.model.model.layers[0].self_attn.o_proj.set_scale("default", 0.0)
+ assert GG._fingerprint(model) != first
+
+
+def test_model_on_two_devices_stays_eager_for_the_call(host, llama, monkeypatch):
+ monkeypatch.setattr(GG, "_fingerprint", lambda model, state = None: None)
+ s = _both(llama, *_ids(llama))
+ assert s["captures"] == 0 and s["refusals"] == {"not one CUDA device / adapters": 9}, s
+
+
[email protected]("drop", [GG.release, G.invalidate, G.release])
+def test_weight_changes_revoke_graphs_and_key_decisions(host, llama, drop):
+ _fresh(llama)
+ state = GG._state(llama)
+ state.graphs["k"] = object()
+ state.validated.add("key")
+ state.refused["other"] = "validation mismatch"
+ state.fingerprint = ("old",)
+ drop(llama)
+ assert not state.graphs and not state.validated and not state.refused
+ assert state.fingerprint is None
+
+
+def test_cache_layer_checks(host, llama):
+ assert GG._cache_reason(None) == "no cache"
+ assert GG._cache_reason(DynamicCache()) == "cache DynamicCache"
+ cache = StaticCache(config = llama.config, max_cache_len = 16)
+ assert GG._cache_reason(cache) == "uninitialized layer"
+ with torch.inference_mode():
+ llama(input_ids = torch.tensor([[5, 6, 7]]), past_key_values = cache, use_cache = True)
+ assert GG._cache_reason(cache) is None
+
+ class Recurrent(cache_utils.StaticLayer): # 5.x linear-attention layers subclass StaticLayer
+ pass
+
+ layer = cache.layers[0]
+ layer.__class__ = Recurrent
+ assert GG._cache_reason(cache) == "layer Recurrent"
+ layer.__class__ = cache_utils.StaticLayer
+ cache.offloading = True
+ assert GG._cache_reason(cache) == "offloaded cache"
+ cache.offloading = False
+ # Real backend: a host cache is not replayable.
+ host.device_type = "cuda"
+ assert GG._cache_reason(cache) == "cache not on CUDA"
+
+
+def test_window_and_host_state_checks():
+ layer = cache_utils.StaticSlidingWindowLayer(max_cache_len = 32, sliding_window = 4)
+ layer.keys = torch.zeros(1)
+ cache = types.SimpleNamespace(layers = [layer])
+ name = (
+ "cumulative_length_int"
+ if "cumulative_length_int" in layer.__dict__
+ else "cumulative_length"
+ )
+ layer.__dict__[name] = 3
+ assert GG._window_reason(cache, 1) is None
+ # Full (rolls every step): replayable, keyed apart from the filling state.
+ layer.__dict__[name] = 4
+ assert GG._window_reason(cache, 1) is None
+ assert GG._window_state(cache) == (True,)
+ # A multi-token step that fills the window switches branches mid-step.
+ layer.__dict__[name] = 3
+ assert GG._window_reason(cache, 2) == "sliding window filling"
+ assert GG._window_state(cache) == (False,)
+ layer.__dict__[name] = 4
+ before = GG._host_state(cache)
+ layer.__dict__[name] = 5
+ assert GG._host_state_reason(before, GG._host_state(cache), 1) is None
+ layer.__dict__[name] = 7
+ assert GG._host_state_reason(before, GG._host_state(cache), 1) == f"cache host state {name}"
+ layer.__dict__[name] = 5
+ layer.other = "changed"
+ assert GG._host_state_reason(before, GG._host_state(cache), 1) == "cache host state other"
+ snap = GG._snapshot(cache)
+ GG._advance(cache, 1)
+ assert layer.__dict__[name] == 6
+ GG._restore(snap)
+ assert layer.__dict__[name] == 5
+
+
+def test_argument_keys(host):
+ cache = object()
+ a, b = [], []
+ x = torch.zeros(2, 1, dtype = torch.long)
+ k1 = GG._spec({"ids": x, "mask": {"full": torch.zeros(2, 1, 1, 8)}, "n": 1}, a, cache)
+ k2 = GG._spec({"ids": x, "mask": {"full": torch.zeros(2, 1, 1, 9)}, "n": 1}, b, cache)
+ assert k1 != k2 and len(a) == len(b) == 2
+ assert GG._spec({"ids": x.float()}, [], cache) != GG._spec({"ids": x}, [], cache)
+ assert GG._spec({"n": 2}, [], cache) != GG._spec({"n": 1}, [], cache)
+ assert GG._spec(cache, [], cache) == ("C",)
+ for bad, why in (
+ (object(), "argument object"),
+ (torch.zeros(3, requires_grad = True), "tensor argument requires grad"),
+ (torch.zeros(3, device = "meta"), "tensor argument on meta"),
+ (torch.zeros(5 * 1024 * 1024), "large tensor argument"),
+ ):
+ with pytest.raises(GG._Refuse, match = why):
+ GG._spec(bad, [], cache)
+ leaves = []
+ tree = {"ids": x, "mask": {"full": torch.ones(2)}, "n": 1}
+ GG._spec(tree, leaves, cache)
+ rebuilt = GG._build(tree, iter([t + 1 for t in leaves]), cache)
+ assert torch.equal(rebuilt["ids"], x + 1) and rebuilt["n"] == 1
+
+
+def test_wanted_gating(monkeypatch):
+ model = types.SimpleNamespace(device = torch.device("cuda", 0))
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.delenv("UNSLOTH_DECODE_GENERIC_GRAPHS", raising = False)
+ monkeypatch.setattr(G, "ENABLED", True)
+ assert GG.wanted(model)
+ monkeypatch.setenv("UNSLOTH_DECODE_GENERIC_GRAPHS", "0")
+ assert not GG.wanted(model)
+ monkeypatch.delenv("UNSLOTH_DECODE_GENERIC_GRAPHS")
+ monkeypatch.setattr(G, "ENABLED", False)
+ assert not GG.wanted(model)
+ monkeypatch.setattr(G, "ENABLED", None)
+ monkeypatch.setenv("UNSLOTH_DECODE_CUDA_GRAPHS", "0")
+ with G.rl_generation():
+ assert not GG.wanted(model)
+ monkeypatch.setenv("UNSLOTH_DECODE_CUDA_GRAPHS", "1")
+ assert GG.wanted(model)
+ monkeypatch.setattr(torch.version, "hip", "6.4", raising = False)
+ assert not GG.wanted(model) # =1 does not force it on ROCm
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ model.forward = lambda **kw: None # an instance forward (accelerate hook) is left alone
+ assert not GG.wanted(model)
+ del model.forward
+ model.device = torch.device("cpu")
+ assert not GG.wanted(model)
+
+
+def test_inert_when_compiling(host):
+ def forward(**kwargs):
+ return kwargs["input_ids"] * 2
+
+ model = torch.nn.Linear(1, 1)
+ model.forward = forward
+ scope = _scope(model, forward, False)
+
+ def step(x):
+ assert GG.decode_step(model, forward, {"input_ids": x}) is None
+ assert not GG.wanted(model)
+ return scope._forward(input_ids = x) + 1
+
+ counters.clear()
+ x = torch.ones(2, 1)
+ assert torch.equal(torch.compile(step, backend = "eager", fullgraph = True)(x), x * 2 + 1)
+ assert not counters["graph_break"]
+ assert GG._ATTR not in model.__dict__
+
+
+def test_generate_routes_only_static_cache_calls():
+ source = inspect.getsource(vision.unsloth_base_fast_generate)
+ # plan() checks wanted() itself; only static-cache, non-compiled-decode calls reach it.
+ call = "_generic_decode_graphs.plan(self, args, kwargs, input_ids)"
+ assert call in source
+ i = source.index(call)
+ guard = source[source.rindex("\n", 0, source.rindex(" if ", 0, i)) : i]
+ assert guard.count(" if ") == 1, guard
+ for clause in (
+ 'cache_implementation == "static"',
+ "not force_dynamic_cache",
+ "not compile_decode",
+ ):
+ assert clause in guard
+
+
[email protected](not has_real_cuda() or torch.version.hip is not None, reason = "NVIDIA CUDA")
[email protected]("name", [LLAMA, GEMMA3])
+def test_real_capture_matches_eager(name, monkeypatch):
+ monkeypatch.setattr(G, "ENABLED", True)
+ torch.manual_seed(0)
+ # bf16 only where the GPU computes it natively: below sm80 Inductor skips a bf16 frame, which torch 2.14 turns
+ # into a hard error inside zoo's fullgraph-compiled Gemma3 RMSNorm (and the skip sticks to that code object for
+ # the rest of the process). Unsloth itself never runs Gemma3 in bf16 there (it forces float32).
+ bf16 = torch.cuda.is_bf16_supported(including_emulation = False)
+ model = (
+ AutoModelForCausalLM.from_pretrained(name, dtype = torch.bfloat16 if bf16 else torch.float16)
+ .cuda()
+ .eval()
+ )
+ ids, mask = _ids(model, 2, pad = 1)
+ ids, mask = ids.cuda(), mask.cuda()
+ eager = _generate(model, ids, mask, False, 24)
+ graphed = _generate(model, ids, mask, True, 24)
+ s = GG.stats(model)
+ assert torch.equal(eager, graphed), s
+ assert s["captures"] == 1 and s["replays"] == 22 and not s["refusals"], s
+
+
+def test_grpo_hidden_states_wrapper_keeps_graphs_eligible():
+ class M(torch.nn.Module):
+ def forward(self, x):
+ return x
+
+ m = M()
+ assert GG._plain_forward(m)
+ m.forward = types.MethodType(M.forward, m) # the class forward re-bound on the instance
+ assert GG._plain_forward(m)
+ m.forward = types.MethodType(M.forward, M()) # bound to another module: not plain
+ assert not GG._plain_forward(m)
+
+ @functools.wraps(M.forward)
+ def autocast_forward(self, *args, **kwargs): # accelerate's mixed-precision wrapper shape
+ return M.forward(self, *args, **kwargs)
+
+ m.forward = types.MethodType(autocast_forward, m)
+ assert GG._plain_forward(m)
+ m._hf_hook = object() # accelerate device hook on the module: stays eager
+ assert not GG._plain_forward(m)
+ del m._hf_hook
+
+ def wrapped_forward(*args, **kwargs):
+ return M.forward(m, *args, **kwargs)
+
+ wrapped_forward._unsloth_grpo_hidden_states_forward_wrapped = True
+ m.forward = wrapped_forward
+ assert GG._plain_forward(m)
+ m.forward = types.MethodType(wrapped_forward, m) # accelerate rebinds it as a bound method
+ assert GG._plain_forward(m)
+ m.forward = lambda *a, **k: None # e.g. an accelerate device hook: stays eager
+ assert not GG._plain_forward(m)
+
+ # The decode scope puts back whatever instance forward was there before (none: none).
+ m = M()
+ with vision._EagerDecodeSteps(m):
+ assert "forward" in m.__dict__
+ assert "forward" not in m.__dict__
+ m.forward = types.MethodType(autocast_forward, m)
+ previous = m.__dict__["forward"]
+ with vision._EagerDecodeSteps(m):
+ assert m.__dict__["forward"] is not previous
+ assert m.__dict__["forward"] is previous
+
+
+def test_rolling_update_matches_transformers_rows(host):
+ classes = GG._rolling_update_classes()
+ assert classes, NO_ROLL
+ for cls, counter in classes:
+ torch.manual_seed(0)
+ layers = [_sliding_layer(cls), _sliding_layer(cls)]
+ prefill = torch.randn(2, 3, 6, 8), torch.randn(2, 3, 6, 8)
+ steps = [(torch.randn(2, 3, 1, 8), torch.randn(2, 3, 1, 8)) for _ in range(5)]
+ for layer in layers:
+ layer.update(*prefill, {"cache_position": torch.arange(6)})
+ for i, (k, v) in enumerate(steps):
+ step_kwargs = {"cache_position": torch.tensor([6 + i])}
+ a = layers[0].update(k, v, step_kwargs)
+ with GG._graphable_rolls():
+ b = layers[1].update(k, v, step_kwargs)
+ assert torch.equal(a[0], b[0]) and torch.equal(a[1], b[1])
+ assert layers[0].__dict__[counter] == layers[1].__dict__[counter]
+
+
+def test_full_window_snapshot_keeps_slot_zero_and_undoes_the_roll(host):
+ # Validation replays a step that rolls every full window, then puts the rows back for the eager step: keeping
+ # the slot the roll drops is enough, not a clone of every window (bsz x 116 MiB on Gemma-3-4B).
+ classes = GG._rolling_update_classes()
+ assert classes, NO_ROLL
+ for cls, counter in classes:
+ for graphable in (False, True):
+ torch.manual_seed(0)
+ layer = _sliding_layer(cls)
+ layer.update(
+ torch.randn(2, 3, 6, 8),
+ torch.randn(2, 3, 6, 8),
+ {"cache_position": torch.arange(6)},
+ )
+ cache = types.SimpleNamespace(layers = [layer])
+ for i in range(3):
+ keys, values = layer.keys.clone(), layer.values.clone()
+ count = layer.__dict__[counter]
+ snapshot = GG._snapshot(cache)
+ rows = [e[1] for e in snapshot[1] if e[0] is layer.keys or e[0] is layer.values]
+ assert len(rows) == 2 and all(r.shape[-2] == 1 for r in rows), [
+ r.shape for r in rows
+ ]
+ step = (
+ torch.randn(2, 3, 1, 8),
+ torch.randn(2, 3, 1, 8),
+ {"cache_position": torch.tensor([6 + i])},
+ )
+ with GG._graphable_rolls() if graphable else contextlib.nullcontext():
+ layer.update(*step)
+ assert not torch.equal(layer.keys, keys)
+ GG._restore(snapshot)
+ assert torch.equal(layer.keys, keys) and torch.equal(layer.values, values)
+ assert layer.__dict__[counter] == count
+ layer.update(*step)
+ # Counters only: what a recording, which runs no device work, needs back.
+ assert all(e[1].numel() == 1 for e in GG._snapshot(cache, rows = False)[1])
+
+
+def test_keys_changing_every_step_stop_capturing_for_the_call(host, llama, monkeypatch):
+ # An argument whose shape or value moves every step gives every step a new key: each capture is an eager
+ # warm-up plus a recording that is never replayed. After a few in a row the call decodes eagerly.
+ ids, mask = _ids(llama)
+ step = iter(range(10**6))
+ real = GG._window_state
+ monkeypatch.setattr(GG, "_window_state", lambda cache: real(cache) + (next(step),))
+ s = _both(llama, ids, mask, 16)
+ assert s["captures"] == GG._MAX_UNREPLAYED_CAPTURES and s["replays"] == 0, s
+ assert s["graphs"] == 0, s
+ assert s["refusals"] == {"graph key changes every step": 15 - GG._MAX_UNREPLAYED_CAPTURES}, s
+ _generate(llama, ids, mask, True, 16)
+ assert GG.stats(llama)["captures"] == 2 * GG._MAX_UNREPLAYED_CAPTURES
+
+
+def test_full_length_type_ids_from_a_trainer_still_replay(host, gemma3, monkeypatch):
+ # TRL's GRPO passes the processor's mm_token_type_ids to generate(); it reached every decode step at full
+ # length (one column longer per step), so every step had a new key and nothing replayed. With the step's
+ # masks prebuilt (a dict) the forward never reads them: the graphed step gets the step's columns.
+ real = type(gemma3).prepare_inputs_for_generation
+
+ def prepare(self, input_ids, *args, **kwargs):
+ out = real(self, input_ids, *args, **kwargs)
+ out["mm_token_type_ids"] = torch.zeros(input_ids.shape, dtype = torch.long)
+ return out
+
+ monkeypatch.setattr(type(gemma3), "prepare_inputs_for_generation", prepare)
+ seen = []
+ real_spec = GG._spec
+
+ def spec(value, leaves, cache):
+ out = real_spec(value, leaves, cache)
+ if isinstance(value, torch.Tensor) and value.dtype == torch.long and value.dim() == 2:
+ seen.append(tuple(value.shape))
+ return out
+
+ monkeypatch.setattr(GG, "_spec", spec)
+ new = 6
+ s = _both(gemma3, *_ids(gemma3, 2, length = 3), new)
+ assert s["captures"] == 1 and s["replays"] == new - 2 and not s["refusals"], s
+ assert seen and all(shape[-1] == 1 for shape in seen), seen
+
+
+def test_step_type_ids_only_with_prebuilt_masks():
+ t = torch.zeros(2, 9, dtype = torch.long)
+ kwargs = {"mm_token_type_ids": t, "attention_mask": torch.ones(2, 9)}
+ assert GG._step_type_ids(kwargs, 1) is kwargs
+ kwargs = {
+ "mm_token_type_ids": t,
+ "token_type_ids": t,
+ "attention_mask": {"full_attention": None},
+ }
+ out = GG._step_type_ids(kwargs, 1)
+ assert out is not kwargs and kwargs["mm_token_type_ids"] is t
+ for name in ("mm_token_type_ids", "token_type_ids"):
+ assert out[name].shape == (2, 1) and out[name].stride() == (1, 1)
+
+
+def test_step_that_fills_the_window_is_never_captured(host, gemma3):
+ # A first decode step at count == window - 1 fills the window. Its key is the filling one, but the capture
+ # records after the warm-up advanced the counters (the rolling branch): a kept graph of that key would then
+ # roll a window that is still filling. That step runs eagerly; the window-full key is captured after it.
+ window = gemma3.config.sliding_window
+ # A call past the window first: the roll index exists, so a capture of the rolling branch would succeed.
+ _fresh(gemma3)
+ _generate(gemma3, *_ids(gemma3, 2, length = 3), True, window + 4)
+ assert GG._ROLL_INDEX
+ new = 8
+ s = _both(gemma3, *_ids(gemma3, 2, length = window - 1), new)
+ assert s["refusals"] == {"sliding window fills in this step": 1}, s
+ assert s["captures"] == 1 and s["validations"] == 1 and s["replays"] == new - 3, s
+ state = gemma3.__dict__[GG._ATTR]
+ assert all(all(key[2]) for key in state.validated), state.validated
diff --git a/tests/test_generic_decode_graphs_default.py b/tests/test_generic_decode_graphs_default.py
new file mode 100644
index 00000000000..00693e6c815
--- /dev/null
+++ b/tests/test_generic_decode_graphs_default.py
@@ -0,0 +1,517 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FastModel generate() decodes through CUDA graphs by default (generate_scope + plan()), on the host
+through the stand-in backend of test_generic_decode_graphs: kill switches, the short-call rule, the
+out-of-memory eager redo, the per-call plan the LoRA fold reads, the graph + StaticCache kept between
+inference calls (released by training), and the BlockMask graph-argument contract."""
+
+import functools
+import os
+import sys
+import types
+
+import pytest
+import torch
+
+import unsloth # noqa: F401 (must precede transformers)
+
+import unsloth.models._decode_graphs as G
+import unsloth.models._generic_decode_graphs as GG
+from unsloth.models import vision
+
+from transformers import AutoModelForCausalLM
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_generic_decode_graphs import LLAMA, _Host, _ids # noqa: E402
+
+ENV = "UNSLOTH_DECODE_CUDA_GRAPHS"
+
+
+class _DefaultHost(_Host):
+ available = staticmethod(lambda: True)
+ free_bytes = staticmethod(lambda device: 1 << 40)
+
+
+def host(monkeypatch):
+ backend = type("Host", (_DefaultHost,), {"graphs": []})
+ monkeypatch.setattr(GG, "_BACKEND", backend)
+ # The runtime default (NVIDIA CUDA, not Windows), not a programmatic override.
+ monkeypatch.setattr(G, "ENABLED", None)
+ monkeypatch.setattr(G, "_DEFAULT_OK", True)
+ monkeypatch.setattr(G, "_STUDIO_DEFAULT_OK", True)
+ for name in (ENV, "UNSLOTH_DECODE_GENERIC_GRAPHS", "UNSLOTH_DECODE_GENERIC_GRAPHS_KEEP"):
+ monkeypatch.delenv(name, raising = False)
+ return backend
+
+
+def llama():
+ torch.manual_seed(0)
+ model = AutoModelForCausalLM.from_pretrained(LLAMA, dtype = torch.float32).eval()
+ model._old_generate = model.generate
+ model.generate = types.MethodType(vision.unsloth_base_fast_generate, model)
+ return model
+
+
+def _gen(
+ model,
+ new = 10,
+ bsz = 1,
+ **kwargs,
+):
+ ids, mask = _ids(model, bsz)
+ return model.generate(
+ input_ids = ids,
+ attention_mask = mask,
+ max_new_tokens = new,
+ min_new_tokens = new,
+ do_sample = False,
+ pad_token_id = 0,
+ **kwargs,
+ )
+
+
+def _eager(
+ model,
+ monkeypatch,
+ new = 10,
+):
+ monkeypatch.setenv(ENV, "0")
+ out = _gen(model, new)
+ monkeypatch.delenv(ENV)
+ return out
+
+
+def _stats(model):
+ return GG.stats(model) or {"captures": 0, "replays": 0, "graphs": 0}
+
+
[email protected]("name", [ENV, "UNSLOTH_DECODE_GENERIC_GRAPHS"])
+def test_kill_switches_keep_generate_eager(host, llama, monkeypatch, name):
+ monkeypatch.setenv(name, "0")
+ _gen(llama)
+ assert _stats(llama)["replays"] == 0 and _stats(llama)["captures"] == 0
+
+
+def test_short_calls_decode_eagerly(host, llama):
+ before = GG.PLANS["short call"]
+ _gen(llama, new = 3)
+ assert GG.PLANS["short call"] == before + 1
+ assert _stats(llama)["captures"] == 0
+
+
+def test_call_plan_follows_the_call(host, llama, monkeypatch):
+ seen = []
+ real = llama._old_generate
+
+ def recording(*args, **kwargs):
+ seen.append(G.call_plan())
+ return real(*args, **kwargs)
+
+ llama._old_generate = recording
+ assert G.call_plan() is None
+ _gen(llama)
+ _gen(llama, new = 2) # one decode step: never a graph, kept or not
+ monkeypatch.setenv(ENV, "0")
+ _gen(llama)
+ assert seen == ["graph", "eager", "eager"]
+ assert G.call_plan() is None
+
+
+def test_out_of_memory_redoes_the_call_eagerly_without_the_wrapper(host, llama, monkeypatch):
+ eager = _eager(llama, monkeypatch)
+ calls = []
+ real = GG.decode_step
+
+ def oom_once(*args, **kwargs):
+ calls.append(1)
+ if len(calls) == 2:
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (test)")
+ return real(*args, **kwargs)
+
+ monkeypatch.setattr(GG, "decode_step", oom_once)
+ # The redo runs after the handler: its traceback would keep the failed call's cache alive.
+ handling = []
+ real_redo = GG.redo_eagerly
+
+ def redo(model):
+ handling.append(sys.exc_info()[0])
+ return real_redo(model)
+
+ monkeypatch.setattr(GG, "redo_eagerly", redo)
+ redos = GG.EAGER_REDOS
+ with pytest.warns(UserWarning, match = "out of GPU memory"):
+ out = _gen(llama)
+ assert torch.equal(out, eager)
+ assert GG.EAGER_REDOS == redos + 1
+ assert handling == [None]
+ # The redo ran without _GraphDecodeSteps: decode_step saw only the failed attempt's steps.
+ assert len(calls) == 2
+ assert "forward" not in llama.__dict__
+ assert GG.kept_cache(llama) is None
+
+
+def test_out_of_memory_with_a_streamer_reraises(host, llama, monkeypatch):
+ def oom(*args, **kwargs):
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (test)")
+
+ monkeypatch.setattr(GG, "decode_step", oom)
+ streamer = types.SimpleNamespace(put = lambda value: None, end = lambda: None)
+ with pytest.raises(torch.cuda.OutOfMemoryError):
+ _gen(llama, streamer = streamer)
+ assert "forward" not in llama.__dict__
+
+
+def test_kept_graph_replays_the_next_call_and_training_releases_it(host, llama, monkeypatch):
+ eager = _eager(llama, monkeypatch)
+ assert _stats(llama)["replays"] == 0
+ # Plain generate() graphs by default.
+ first = _gen(llama)
+ s1 = _stats(llama)
+ assert s1["captures"] == 1 and s1["graphs"] == 1 and s1["replays"] >= 8, s1
+ assert "forward" not in llama.__dict__
+ kept = GG.kept_cache(llama)
+ assert kept is not None
+ second = _gen(llama)
+ s2 = _stats(llama)
+ # Same cache tensors, same graph: no capture, every decode step replays (from the first one).
+ assert s2["captures"] == 1, s2
+ assert s2["replays"] - s1["replays"] == 9, (s1, s2)
+ assert GG.kept_cache(llama) is kept
+ assert torch.equal(first, eager) and torch.equal(second, eager)
+ _gen(llama, new = 4)
+ assert _stats(llama)["captures"] == 1
+ # Training forward without for_training() frees the graph and the cache.
+ llama.train()
+ llama(input_ids = _ids(llama)[0])
+ assert GG.kept_cache(llama) is None and _stats(llama)["graphs"] == 0
+ assert "_cache" not in llama.__dict__
+ llama.eval()
+
+
+def test_kept_graph_hook_is_inert_in_a_compiled_training_forward(host, llama):
+ # A kept graph leaves a forward pre-hook that frees it when training resumes. Dynamo tracing a compiled
+ # training forward must not run it (removing a hook while tracing is a graph break, an error under fullgraph);
+ # the next eager training forward still frees the graph.
+ _gen(llama)
+ state = llama.__dict__[GG._ATTR]
+ assert GG.kept_cache(llama) is not None and state.hook is not None
+ torch._dynamo.reset()
+ llama.train()
+ try:
+ ids, _ = _ids(llama)
+ compiled = torch.compile(
+ lambda x: llama(input_ids = x).logits.float().sum(), fullgraph = True, backend = "eager"
+ )
+ compiled(ids).backward()
+ assert GG.kept_cache(llama) is not None
+ llama.zero_grad(set_to_none = True)
+ llama(input_ids = ids)
+ assert GG.kept_cache(llama) is None and state.hook is None
+ finally:
+ llama.eval()
+ llama.zero_grad(set_to_none = True)
+ torch._dynamo.reset()
+
+
+def test_a_short_call_after_a_long_one_replays_the_kept_graph(host, llama, monkeypatch):
+ # transformers 5.x sizes every later cache to the longest call so far (_previous_max_cache_length), so a short
+ # call after a long one gets a cache as long as the kept one: it must reuse it, not re-capture every call.
+ _gen(llama, new = 40)
+ s1 = _stats(llama)
+ assert s1["captures"] == 1 and GG.kept_cache(llama) is not None, s1
+ monkeypatch.setattr(
+ llama, "_previous_max_cache_length", GG._kept_len(llama.__dict__[GG._ATTR]), raising = False
+ )
+ eager = _eager(llama, monkeypatch)
+ # The eager call dropped the kept graph; capture again, then short calls reuse it.
+ _gen(llama, new = 40)
+ s2 = _stats(llama)
+ for _ in range(2):
+ assert torch.equal(_gen(llama, new = 10), eager)
+ s3 = _stats(llama)
+ assert s3["captures"] == s2["captures"], (s2, s3)
+ assert s3["replays"] - s2["replays"] == 2 * 9, (s2, s3)
+
+
+def test_other_batch_or_returned_cache_drops_the_kept_graph(host, llama):
+ _gen(llama)
+ assert GG.kept_cache(llama) is not None
+ _gen(llama, bsz = 2)
+ assert _stats(llama)["captures"] == 2, _stats(llama)
+ # A returned cache belongs to the caller: never kept, never reset by a later call.
+ out = _gen(llama, return_dict_in_generate = True)
+ assert out.past_key_values is not None
+ assert GG.kept_cache(llama) is None
+
+
+def test_weights_changed_and_release_for_training_drop_the_kept_graph(host, llama):
+ _gen(llama)
+ assert _stats(llama)["captures"] == 1
+ G.weights_changed()
+ _gen(llama)
+ assert _stats(llama)["captures"] == 2
+ assert GG.kept_cache(llama) is not None
+ G.release_for_training(llama)
+ assert GG.kept_cache(llama) is None and "_cache" not in llama.__dict__
+
+
+def _block_mask(offset, contract = True):
+ from torch.nn.attention.flex_attention import BlockMask, _DEFAULT_SPARSE_BLOCK_SIZE as BLOCK
+
+ def rebuild(tensors):
+ (off,) = tensors
+ return lambda b, h, q_idx, kv_idx: q_idx + off >= kv_idx
+
+ n = 2
+ mask = BlockMask.from_kv_blocks(
+ torch.ones(1, 1, 1, dtype = torch.int32),
+ torch.arange(n, dtype = torch.int32).view(1, 1, 1, n),
+ BLOCK_SIZE = BLOCK,
+ mask_mod = rebuild((offset,)),
+ seq_lengths = (1, n * BLOCK),
+ )
+ if contract:
+ mask._unsloth_graph_closure = ((offset,), rebuild)
+ return mask
+
+
+def test_block_mask_contract_round_trip(host):
+ offset = torch.tensor(3)
+ mask = _block_mask(offset)
+ leaves = []
+ spec = GG._spec({"full_attention": mask}, leaves, None)
+ assert GG._spec({"full_attention": _block_mask(torch.tensor(7))}, [], None) == spec
+ assert any(leaf is offset for leaf in leaves)
+
+ def at(t):
+ return next(i for i, leaf in enumerate(leaves) if leaf is t)
+
+ buffers = [leaf.clone() for leaf in leaves]
+ built = GG._build({"full_attention": mask}, iter(buffers), None)["full_attention"]
+ assert built is not mask and built.kv_num_blocks is buffers[at(mask.kv_num_blocks)]
+ static = built._unsloth_graph_closure[0][0]
+ assert static is buffers[at(offset)]
+ q, k = torch.zeros((), dtype = torch.long), torch.tensor(5)
+ assert not bool(built.mask_mod(0, 0, q, k))
+ static.fill_(5) # a refill reaches the rebuilt mask_mod
+ assert bool(built.mask_mod(0, 0, q, k))
+ with pytest.raises(GG._Refuse, match = "argument BlockMask"):
+ GG._spec(_block_mask(torch.tensor(3), contract = False), [], None)
+
+
+class _Muse(torch.nn.Module):
+ """A Muse-Glimmer-like decoder: the embedding is trainable and may answer on another device."""
+
+ def __init__(
+ self,
+ answer_on = None,
+ model_type = "muse_glimmer",
+ ):
+ super().__init__()
+ self.config = types.SimpleNamespace(
+ model_type = model_type, image_token_id = 5, video_token_id = 6
+ )
+ self.embed = torch.nn.Embedding(10, 4)
+ self.answer_on = answer_on
+
+ def get_input_embeddings(self):
+ if self.answer_on is None:
+ return self.embed
+ return lambda ids: torch.zeros(*ids.shape, 4, device = self.answer_on)
+
+
+def test_placeholder_masking_models_embed_outside_the_graph(host):
+ """Muse-Glimmer / Muse-Spark / Kimi-K2.5 zero placeholder ids with ids[mask] = 0 (a boolean index_put_, a
+ host sync): their decode step gets the same embeddings computed before the graph, from the same ops, never
+ tracking grad (the embedding is trainable) so they pass the argument check."""
+ ids = torch.tensor([[5], [3], [6]])
+ model = _Muse()
+ assert model.embed.weight.requires_grad
+ with torch.enable_grad():
+ out = GG._placeholder_embeds(
+ model, {"input_ids": ids, "inputs_embeds": None, "use_cache": True}
+ )
+ llm = ids.clone()
+ llm[(ids == 5) | (ids == 6)] = 0 # what the model's forward does
+ assert out["input_ids"] is None and out["use_cache"] is True
+ embeds = out["inputs_embeds"]
+ assert torch.equal(embeds, model.embed(llm)) and not embeds.requires_grad
+ leaves = []
+ GG._spec(out, leaves, None) # not "tensor argument requires grad"
+ assert any(leaf is embeds for leaf in leaves)
+ # Other models, media steps and an explicit inputs_embeds are left alone.
+ kwargs = {"input_ids": ids}
+ assert GG._placeholder_embeds(_Muse(model_type = "gemma4"), kwargs) is kwargs
+ media = {"input_ids": ids, "pixel_values": torch.ones(1)}
+ assert GG._placeholder_embeds(model, media) is media
+ # The lookup answers on the host while the ids live elsewhere (meta stands in for the card).
+ ids = torch.tensor([[5], [3]], device = "meta")
+ out = GG._placeholder_embeds(_Muse(answer_on = "cpu"), {"input_ids": ids})
+ assert out["inputs_embeds"].device == ids.device
+
+
+def _hooked(model, hook):
+ """What accelerate's add_hook_to_module leaves on a module (without moving its parameters)."""
+
+ def new_forward(module, *args, **kwargs):
+ args, kwargs = module._hf_hook.pre_forward(module, *args, **kwargs)
+ return module._hf_hook.post_forward(module, module._old_forward(*args, **kwargs))
+
+ old = model.forward
+ model._old_forward = old
+ model._hf_hook = hook
+ model.forward = functools.update_wrapper(functools.partial(new_forward, model), old)
+ return model
+
+
+def test_a_single_device_accelerate_hook_keeps_graphs_eligible(monkeypatch):
+ """A bitsandbytes model placed on cuda:1 gets accelerate's AlignDevicesHook (force_hooks, vision.py
+ _attach_bnb_multidevice_hooks): an instance forward plus _hf_hook, which wanted() used to refuse, so
+ no decode step of a model on a second card was ever graphed."""
+ from accelerate.hooks import AlignDevicesHook, ModelHook
+
+ class OnCard1(torch.nn.Linear):
+ @property
+ def device(self):
+ return torch.device("cuda", 1)
+
+ def wanted(hook):
+ return GG.wanted(_hooked(OnCard1(2, 2), hook))
+
+ monkeypatch.setattr(G, "ENABLED", True)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.delenv("UNSLOTH_DECODE_GENERIC_GRAPHS", raising = False)
+ assert wanted(AlignDevicesHook(execution_device = 1, io_same_device = True))
+ assert wanted(AlignDevicesHook(execution_device = "cuda:1"))
+ # Hooks that move weights or run elsewhere stay eager.
+ assert not wanted(AlignDevicesHook(execution_device = 0))
+ assert not wanted(AlignDevicesHook(execution_device = 1, offload = True))
+ assert not wanted(ModelHook())
+
+
+def test_generate_through_a_single_device_hook_uses_graphs(host, llama, monkeypatch):
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+
+ eager = _eager(llama, monkeypatch)
+ add_hook_to_module(llama, AlignDevicesHook(execution_device = "cpu", io_same_device = True))
+ hooked = llama.__dict__["forward"]
+ graphed = _gen(llama)
+ s = _stats(llama)
+ assert s["captures"] == 1 and s["replays"] >= 8, s
+ assert torch.equal(eager, graphed)
+ assert llama.__dict__["forward"] is hooked
+
+
+def test_refusal_reasons_name_the_argument(host, llama, monkeypatch):
+ real = GG._mrope_positions
+
+ def with_grad_tensor(model, kwargs):
+ return {**real(model, kwargs), "extra": torch.zeros(1, requires_grad = True)}
+
+ monkeypatch.setattr(GG, "_mrope_positions", with_grad_tensor)
+ cls_forward = type(llama).forward
+
+ @functools.wraps(cls_forward)
+ def forward(
+ self,
+ *a,
+ extra = None,
+ **k,
+ ):
+ return cls_forward(self, *a, **k)
+
+ monkeypatch.setattr(type(llama), "forward", forward)
+ _gen(llama)
+ assert _stats(llama)["refusals"] == {"tensor argument requires grad (extra)": 9}, _stats(llama)
+
+
+class _HostEmbedding(torch.nn.Module):
+ """Unsloth's offloaded input embedding as the decode step sees it: the weight lives off the compute device
+ (meta stands in for host RAM) behind hooks that return the rows on the decoder's device. Records whether
+ each lookup ran inside a graph capture or replay."""
+
+ def __init__(self, real, inside):
+ super().__init__()
+ self.weight = torch.nn.Parameter(
+ torch.empty_like(real.weight, device = "meta"), requires_grad = False
+ )
+ self.table = real.weight.detach().clone() # a plain attribute: the lookup's data
+ self.inside = inside
+ self.calls = []
+ self._unsloth_offload_hooks_installed = True
+
+ def forward(self, ids):
+ self.calls.append(self.inside[0])
+ return torch.nn.functional.embedding(ids, self.table)
+
+
+def _offload(
+ host,
+ llama,
+ monkeypatch,
+ extra = False,
+):
+ inside = [False]
+ record = host.record.__func__
+
+ def tracked(fn):
+ def run():
+ inside[0] = True
+ try:
+ return fn()
+ finally:
+ inside[0] = False
+
+ return run
+
+ monkeypatch.setattr(
+ host,
+ "record",
+ classmethod(lambda cls, pool, fn, cache: record(cls, pool, tracked(fn), cache)),
+ )
+ embedding = _HostEmbedding(llama.get_input_embeddings(), inside)
+ llama.set_input_embeddings(embedding)
+ # As `_pin_device_to_decoder`: model.device is the decoder's, not the offloaded table's.
+ monkeypatch.setattr(type(llama), "device", property(lambda self: torch.device("cpu")))
+ if extra:
+ llama.model.norm.register_parameter(
+ "stray", torch.nn.Parameter(torch.zeros(1, device = "meta"), requires_grad = False)
+ )
+ return embedding
+
+
+def test_offloaded_input_embedding_is_looked_up_outside_the_graph(host, llama, monkeypatch):
+ eager = _eager(llama, monkeypatch)
+ embedding = _offload(host, llama, monkeypatch)
+ graphed = _gen(llama)
+ s = _stats(llama)
+ assert s["captures"] == 1 and s["replays"] >= 8, s
+ assert torch.equal(eager, graphed)
+ # Every decode step looked its rows up eagerly; none inside a capture or replay.
+ assert embedding.calls and not any(embedding.calls)
+
+
+def test_offloaded_input_embedding_swap_recaptures_and_unhooked_is_refused(
+ host, llama, monkeypatch
+):
+ embedding = _offload(host, llama, monkeypatch)
+ first = GG._fingerprint(llama)
+ assert first is not None
+ embedding.weight = torch.nn.Parameter(torch.empty_like(embedding.weight), requires_grad = False)
+ assert GG._fingerprint(llama) not in (None, first)
+ # An off-device embedding without Unsloth's offload hooks stays refused.
+ embedding._unsloth_offload_hooks_installed = False
+ assert GG._offloaded_input_embedding(llama) is None
+ assert GG._fingerprint(llama) is None
+
+
+def test_another_offloaded_parameter_is_still_refused(host, llama, monkeypatch):
+ _offload(host, llama, monkeypatch, extra = True)
+ assert GG._fingerprint(llama) is None
+ _gen(llama)
+ s = _stats(llama)
+ assert s["replays"] == 0 and "not one CUDA device / adapters" in s["refusals"], s
diff --git a/tests/test_generic_decode_graphs_multidevice.py b/tests/test_generic_decode_graphs_multidevice.py
new file mode 100644
index 00000000000..9f8ade49cf7
--- /dev/null
+++ b/tests/test_generic_decode_graphs_multidevice.py
@@ -0,0 +1,460 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FastModel decode CUDA graphs for a model split over cards (device_map / Unsloth's layer split):
+one graph holds every card's kernels. On the host through the stand-in backend of
+test_generic_decode_graphs with the parameter devices faked as two cards; the real two-card capture
+is the gpu-marked test."""
+
+import contextlib
+import os
+import sys
+
+import pytest
+import torch
+
+import unsloth # noqa: F401 (must precede transformers)
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import test_generic_decode_graphs as T # noqa: E402
+import unsloth.models._generic_decode_graphs as GG
+from real_accelerator import has_real_cuda
+
+from transformers import AutoModelForCausalLM
+
+
+class _Stream(T._FakeStream):
+ def __init__(self, log, name):
+ self.log, self.name = log, name
+
+ def wait_stream(self, other):
+ self.log.append((self.name, "waits", getattr(other, "name", "?")))
+
+
+class _TwoCards(T._Host):
+ """Host stand-in for a model split over cards 0 and 1 (this thread on card 0)."""
+
+ others_recorded = []
+ supported = True
+ log = []
+ current_device = staticmethod(lambda: 0)
+ mempool = staticmethod(lambda: object())
+
+ @classmethod
+ def multidevice_supported(cls):
+ return cls.supported
+
+ @classmethod
+ def current_stream(cls):
+ return _Stream(cls.log, "card0")
+
+ @classmethod
+ def device_stream(cls, index):
+ return _Stream(cls.log, f"card{index}")
+
+ @classmethod
+ def side_stream(cls, index):
+ return _Stream(cls.log, f"side{index}")
+
+ @classmethod
+ @contextlib.contextmanager
+ def on_streams(cls, streams):
+ cls.log.append(("on_streams", tuple(sorted(streams))))
+ yield
+
+ @classmethod
+ def record(cls, pool, fn, cache, *side):
+ others, mempools = side or ({}, {})
+ cls.others_recorded.append((tuple(sorted(others)), sorted(mempools)))
+ return super().record(pool, fn, cache)
+
+
+def two_cards(monkeypatch):
+ backend = type(
+ "TwoCards",
+ (_TwoCards,),
+ {"graphs": [], "others_recorded": [], "log": [], "supported": True},
+ )
+ monkeypatch.setattr(GG, "_BACKEND", backend)
+ # Parameters on "card 0" and "card 1" (host tensors: the stand-in backend runs on the CPU).
+ monkeypatch.setattr(
+ GG, "_param_devices", lambda model: {torch.device("cpu", 0), torch.device("cpu", 1)}
+ )
+ monkeypatch.delenv("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", raising = False)
+ return backend
+
+
+def llama():
+ torch.manual_seed(0)
+ return AutoModelForCausalLM.from_pretrained(T.LLAMA, dtype = torch.float32).eval()
+
+
+def test_split_model_is_graphed_as_one_multi_card_graph(two_cards, llama):
+ hooks = pytest.importorskip("accelerate.hooks")
+ # As dispatch_model leaves a split model: an AlignDevicesHook on every decoder layer.
+ for layer in llama.model.layers:
+ hooks.add_hook_to_module(layer, hooks.AlignDevicesHook(execution_device = "cpu"))
+ s = T._both(llama, *T._ids(llama, 2, pad = 1))
+ assert s["captures"] == 1 and s["replays"] == 8 and s["refusals"] == {}, s
+ # Card 1 joined with its own MemPool, and on its side stream for the warm-up.
+ assert two_cards.others_recorded == [((1,), [1])]
+ assert ("on_streams", (1,)) in two_cards.log
+ # Every replay waits on card 1's stream first, and card 1 waits on the replay after.
+ assert ("card0", "waits", "card1") in two_cards.log
+ assert ("card1", "waits", "card0") in two_cards.log
+
+
+ "setup, reason",
+ [
+ ("kill", "multi-device (UNSLOTH_DECODE_MULTIDEVICE_GRAPHS=0)"),
+ ("old_torch", "multi-device capture needs a newer torch"),
+ ("offload", "multi-device with offload or non-align hooks"),
+ ("other_hook", "multi-device with offload or non-align hooks"),
+ ],
+)
+def test_split_model_refusals_decode_eagerly(two_cards, llama, monkeypatch, setup, reason):
+ accelerate_hooks = pytest.importorskip("accelerate.hooks")
+ if setup == "kill":
+ monkeypatch.setenv("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", "0")
+ elif setup == "old_torch":
+ two_cards.supported = False
+ elif setup == "offload":
+ hook = accelerate_hooks.AlignDevicesHook(execution_device = "cpu")
+ hook.offload = True
+ llama.model.layers[0].__dict__["_hf_hook"] = hook
+ else:
+ llama.model.layers[0].__dict__["_hf_hook"] = accelerate_hooks.ModelHook()
+ s = T._both(llama, *T._ids(llama, 1))
+ assert s["captures"] == 0 and s["replays"] == 0, s
+ assert set(s["refusals"]) == {reason}, s
+ assert two_cards.others_recorded == []
+
+
+def test_single_card_model_records_without_other_cards(two_cards, llama, monkeypatch):
+ monkeypatch.setattr(GG, "_param_devices", lambda model: {torch.device("cpu", 0)})
+ s = T._both(llama, *T._ids(llama, 1))
+ assert s["captures"] == 1 and s["refusals"] == {}, s
+ # The single-card record call is unchanged (no other cards, no MemPools).
+ assert two_cards.others_recorded == [((), [])]
+ assert not any("card1" in entry for entry in two_cards.log), two_cards.log
+
+
+ not has_real_cuda() or torch.cuda.device_count() < 2, reason = "needs two CUDA GPUs"
+)
+def test_real_two_card_capture_matches_eager():
+ """A tiny Llama split over cuda:0 / cuda:1 with accelerate's dispatch: one multi-card graph,
+ tokens and logits equal to the eager step."""
+ from accelerate import dispatch_model
+
+ torch.manual_seed(0)
+ model = AutoModelForCausalLM.from_pretrained(T.LLAMA, dtype = torch.float16).eval()
+ n = len(model.model.layers)
+ device_map = {"model.embed_tokens": 0, "model.rotary_emb": 0, "model.norm": 1, "lm_head": 1}
+ for i in range(n):
+ device_map[f"model.layers.{i}"] = 0 if i < n // 2 else 1
+ dispatch_model(model, device_map = device_map, main_device = 0, force_hooks = True)
+ ids, mask = T._ids(model, 2, pad = 1)
+ ids, mask = ids.cuda(0), mask.cuda(0)
+ eager = T._generate(model, ids, mask, False, new = 24)
+ graphed = T._generate(model, ids, mask, True, new = 24)
+ assert torch.equal(eager, graphed)
+ s = GG.stats(model)
+ assert s["captures"] >= 1 and s["replays"] > 0, s
+
+
+class _Seg:
+ def __init__(self, log, n):
+ self.log, self.n = log, n
+
+ def replay(self):
+ self.log.append(("replay", self.n))
+
+
+class _HostCards:
+ """Two host "cards" for the segment recorder: card 0 = cpu, card 1 = meta (copies cpu -> meta work)."""
+
+ device_type = "cpu"
+
+ def __init__(self):
+ self.log = []
+ self.n = 0
+
+ @staticmethod
+ def card_of(value):
+ return {"cpu": 0, "meta": 1}.get(value.device.type)
+
+ @staticmethod
+ def index_of(device):
+ if isinstance(device, int):
+ return device
+ return {"cpu": 0, "meta": 1}.get(torch.device(device).type)
+
+ @staticmethod
+ def card_device(index):
+ return torch.device(("cpu", "meta")[index])
+
+ current_device = staticmethod(lambda: 0)
+ device = staticmethod(lambda index: contextlib.nullcontext())
+
+ def new_graph(self):
+ self.n += 1
+ return _Seg(self.log, self.n)
+
+ def begin_on(self, graph, pool, card):
+ self.log.append(("begin", graph.n, card, pool))
+
+ def end_on(self, graph):
+ self.log.append(("end", graph.n))
+
+
+def _send_to_device(value, device, *args, **kwargs):
+ return value.to(device)
+
+
+class _Block(torch.nn.Module):
+ def __init__(self, device):
+ super().__init__()
+ self.lin = torch.nn.Linear(4, 4, device = device)
+
+ def forward(self, x, extra):
+ return self.lin(x) + extra
+
+
+def test_segment_recorder_cuts_at_the_card_boundary_and_moves_shared_inputs_once(monkeypatch):
+ hooks = pytest.importorskip("accelerate.hooks")
+ cards = _HostCards()
+ monkeypatch.setattr(GG, "_BACKEND", cards)
+ torch.manual_seed(0)
+ first = _Block("cpu")
+ second, third = _Block("meta"), _Block("meta")
+ for block in (second, third):
+ hooks.add_hook_to_module(block, hooks.AlignDevicesHook(execution_device = "meta"))
+ x = torch.randn(2, 4)
+ extra = torch.randn(2, 4) # a layer-invariant input on card 0 (rotary cos / sin, a mask)
+
+ def step():
+ h = first(x, extra = extra)
+ h = second(h, extra = extra)
+ return third(h, extra = extra)
+
+ originals = (
+ hooks.send_to_device,
+ hooks.AlignDevicesHook.pre_forward,
+ hooks.AlignDevicesHook.post_forward,
+ )
+ segments, out = GG._SegmentRecorder(cards, {0: "pool0", 1: "pool1"}, 0).run(step)
+ assert out.device.type == "meta"
+ kinds = [
+ (p[0], p[2] if p[0] == "graph" else (p[2].device.type, p[1].device.type))
+ for p in segments.parts
+ ]
+ # Card 0's segment, the hidden state and the shared input cross once, then one card-1 segment
+ # for both card-1 blocks (the second block's inputs are already there: no new cut).
+ assert kinds == [
+ ("graph", 0),
+ ("copy", ("cpu", "meta")),
+ ("copy", ("cpu", "meta")),
+ ("graph", 1),
+ ], kinds
+ assert cards.log == [
+ ("begin", 1, 0, "pool0"),
+ ("end", 1),
+ ("begin", 2, 1, "pool1"),
+ ("end", 2),
+ ], cards.log
+ assert (
+ hooks.send_to_device,
+ hooks.AlignDevicesHook.pre_forward,
+ hooks.AlignDevicesHook.post_forward,
+ ) == originals
+ assert second(x.to("meta"), extra = extra).device.type == "meta"
+
+
+def test_segments_replay_graphs_and_hand_offs_in_capture_order(monkeypatch):
+ monkeypatch.setattr(GG, "_BACKEND", _HostCards())
+ src, dst = torch.zeros(3), torch.zeros(3)
+ log = []
+
+ class Writes:
+ def replay(self):
+ log.append("card0")
+ src.fill_(3)
+
+ class Reads:
+ def replay(self):
+ log.append(("card1 sees", dst.tolist()))
+
+ segments = GG._Segments()
+ segments.parts = [("graph", Writes(), 0), ("copy", dst, src), ("graph", Reads(), 1)]
+ segments.replay()
+ assert log == ["card0", ("card1 sees", [3.0, 3.0, 3.0])]
+
+
+def test_segment_recorder_leaves_movable_objects_to_accelerate(monkeypatch):
+ # A non-tensor with .to (BatchEncoding, ...) goes through accelerate's own send_to_device, as eager.
+ cards = _HostCards()
+ monkeypatch.setattr(GG, "_BACKEND", cards)
+ seen = []
+
+ class Movable:
+ def to(self, device, *args, **kwargs):
+ seen.append(device)
+ return self
+
+ recorder = GG._SegmentRecorder(cards, {0: "pool0", 1: "pool1"}, 0)
+ recorder.original = _send_to_device
+ item = Movable()
+ assert recorder.send(_send_to_device, {"a": item}, "meta")["a"] is item
+ assert (
+ seen == [torch.device("meta")]
+ and recorder.segments.parts == []
+ and recorder.pending is None
+ )
+
+
+def test_split_model_root_hook_on_the_first_blocks_card_is_plain(monkeypatch):
+ # 16-bit models with tied embeddings: Unsloth's split puts the tied embedding / lm_head (the first
+ # parameter, so model.device) on cuda:1 while the root hook moves inputs to cuda:0.
+ from accelerate.hooks import AlignDevicesHook
+
+ class Root:
+ device = torch.device("cuda", 1)
+
+ root = Root()
+ root.__dict__["_hf_hook"] = AlignDevicesHook(
+ execution_device = torch.device("cuda", 0), io_same_device = True
+ )
+ cards = {torch.device("cuda", 0), torch.device("cuda", 1)}
+ monkeypatch.setattr(GG, "_param_devices", lambda model: cards)
+ monkeypatch.setenv("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", "1")
+ assert GG._single_device_hook(root)
+ monkeypatch.setenv("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", "0")
+ assert not GG._single_device_hook(root)
+ monkeypatch.setenv("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", "1")
+ # A card the model is not on stays refused; a model on one card keeps the exact-device rule.
+ root.__dict__["_hf_hook"] = AlignDevicesHook(execution_device = torch.device("cuda", 2))
+ assert not GG._single_device_hook(root)
+ monkeypatch.setattr(GG, "_param_devices", lambda model: {torch.device("cuda", 1)})
+ root.__dict__["_hf_hook"] = AlignDevicesHook(execution_device = torch.device("cuda", 0))
+ assert not GG._single_device_hook(root)
+
+
+def _graph_block_mask(device = "cpu"):
+ # A BlockMask publishing the graph contract as zoo's gpt-oss flex mask does: mask_mod reads an offset
+ # and padding rows through `rebuild`, and the builder's own per-device rebuild sits on the mask.
+ from torch.nn.attention.flex_attention import BlockMask
+
+ offset = torch.full((), 3, dtype = torch.int64, device = device)
+ pad = torch.ones(1, 128, dtype = torch.bool, device = device)
+
+ def rebuild(ts):
+ off, rows = ts
+
+ def mask_mod(b, h, q_idx, kv_idx):
+ return (kv_idx + off <= q_idx + off) & rows[b, kv_idx]
+
+ mask_mod.reads = ts
+ return mask_mod
+
+ bm = BlockMask.from_kv_blocks(
+ torch.ones(1, 1, 1, dtype = torch.int32, device = device),
+ torch.zeros(1, 1, 1, 1, dtype = torch.int32, device = device),
+ BLOCK_SIZE = 128,
+ mask_mod = rebuild((offset, pad)),
+ seq_lengths = (1, 128),
+ )
+ bm._unsloth_graph_closure = ((offset, pad), rebuild)
+ bm._unsloth_on_device = lambda target: pytest.fail(
+ "the builder's rebuild reads the unrefilled tensors"
+ )
+ return bm
+
+
+def test_graph_block_mask_moves_its_closure_tensors_to_the_layer_card(monkeypatch):
+ # Split model + flex decode: the static BlockMask is built on card 0 and accelerate moves it to card 1's
+ # layers. BlockMask.to leaves mask_mod's tensors on card 0 (the gpt-oss "Tensor device mismatch ...
+ # cuda:1 and cuda:0" at kv_idx + kv_o), so both the warm-up and the recording move them as well.
+ pytest.importorskip("torch.nn.attention.flex_attention")
+ cards = _HostCards()
+ monkeypatch.setattr(GG, "_BACKEND", cards)
+ value = _graph_block_mask()
+ leaves, cache = [], object()
+ GG._spec(value, leaves, cache)
+ static = [leaf.clone() for leaf in leaves]
+ built = GG._build(value, iter(static), cache)
+ closure = built._unsloth_graph_closure[0]
+ assert built.mask_mod.reads == closure and all(any(t is s for s in static) for t in closure)
+
+ # Eager warm-up: accelerate's .to (zoo's patched BlockMask.to calls _unsloth_on_device).
+ on_device = built.__dict__.get("_unsloth_on_device")
+ assert on_device is not None
+ moved = on_device("meta")
+ assert all(t.device.type == "meta" for t in moved.mask_mod.reads)
+ assert moved.kv_num_blocks.device.type == "meta"
+ # Same card (the root hook, the first card's layers): the mask itself, which keeps its per-device move,
+ # so the next card's hook still moves mask_mod's tensors (Kaggle T4x2: the root hook's same-card move
+ # dropped it and the cuda:1 layers read the cuda:0 offsets).
+ here = on_device("cpu")
+ assert here is built
+ later = here.__dict__["_unsloth_on_device"]("meta")
+ assert all(t.device.type == "meta" for t in later.mask_mod.reads)
+ assert moved.__dict__["_unsloth_on_device"]("meta") is moved
+
+ # Recording: the fields and closure tensors cross as hand-offs (refilled every replay), once per card.
+ recorder = GG._SegmentRecorder(cards, {0: "pool0", 1: "pool1"}, 0)
+ recorder.original = _send_to_device
+ out = recorder.send(recorder.original, {"attention_mask": built}, "meta")["attention_mask"]
+ again = recorder.send(recorder.original, {"attention_mask": built}, "meta")["attention_mask"]
+ assert again is out
+ assert all(t.device.type == "meta" for t in out.mask_mod.reads)
+ assert out._unsloth_graph_closure[0] == out.mask_mod.reads
+ copies = [p for p in recorder.segments.parts if p[0] == "copy"]
+ sources = [p[2] for p in copies]
+ assert all(any(t is s for s in sources) for t in closure), "closure tensors not handed off"
+ assert any(s is built.kv_num_blocks for s in sources)
+ assert recorder.pending == 1
+
+
+def test_overlapping_recordings_from_two_threads_restore_accelerates_hooks():
+ # accelerate's hooks are process-global: overlapping recordings used to restore each other's replacements and
+ # leave a finished recorder's send_to_device installed.
+ import threading
+ import time
+
+ import accelerate.hooks as hooks
+
+ originals = (
+ hooks.send_to_device,
+ hooks.AlignDevicesHook.pre_forward,
+ hooks.AlignDevicesHook.post_forward,
+ )
+ active, peak, errors = [0], [0], []
+
+ def body():
+ active[0] += 1
+ peak[0] = max(peak[0], active[0])
+ time.sleep(0.05)
+ active[0] -= 1
+
+ def record():
+ try:
+ GG._SegmentRecorder(_HostCards(), {0: "pool0"}, 0).run(body)
+ except BaseException as error:
+ errors.append(repr(error))
+
+ threads = [threading.Thread(target = record) for _ in range(2)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(10)
+ assert not errors and peak[0] == 1, (errors, peak)
+ assert (
+ hooks.send_to_device,
+ hooks.AlignDevicesHook.pre_forward,
+ hooks.AlignDevicesHook.post_forward,
+ ) == originals
diff --git a/tests/test_return_logits_survives_for_training.py b/tests/test_return_logits_survives_for_training.py
index 31f32d76a7c..3512fe5cf8a 100644
--- a/tests/test_return_logits_survives_for_training.py
+++ b/tests/test_return_logits_survives_for_training.py
@@ -81,6 +81,8 @@ def _fast_base_model():
"_pipeline_keeps_replica": lambda model: False,
"enable_fp8_lora": lambda model: [],
"restore_fp8_lora_for_inference": lambda model: None,
+ # for_training frees kept decode graphs and their cache; none exist for these fakes.
+ "_decode_graphs": types.SimpleNamespace(release_for_training = lambda model: None),
}
lifted = {n.name for n in wanted} | set(stubbed)
todo = list(wanted)
diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.py
index e5294f5a722..39c0f4665fe 100644
--- a/unsloth/models/_decode_graphs.py
+++ b/unsloth/models/_decode_graphs.py
@@ -163,6 +163,45 @@ class eager_scope:
return False
+class plan_scope:
+ """Per-thread: the decode plan ("graph" / "eager") Unsloth's generate() chose for the call it runs."""
+
+ __slots__ = ("plan", "_saved")
+
+ def __init__(self, plan):
+ self.plan = plan
+
+ def __enter__(self):
+ self._saved = getattr(_TLS, "plan", None)
+ _TLS.plan = self.plan
+ return self
+
+ def __exit__(self, *exc):
+ _TLS.plan = self._saved
+ return False
+
+
+def call_plan():
+ """ "graph" / "eager" for the generate() call running on this thread, None outside Unsloth's
+ generate(). Per-call policies (the 16-bit LoRA fold threshold) read this, not enabled(), which only
+ says graphs are allowed."""
+ if torch.compiler.is_compiling():
+ return None
+ if getattr(_TLS, "eager", 0):
+ return "eager"
+ return getattr(_TLS, "plan", None)
+
+
+# Bumped by code that rewrites weights in place without a version bump (the 16-bit LoRA fold writes
+# W.data): every captured decode graph is fingerprinted with it, so none outlives such a rewrite.
+WEIGHTS_EPOCH = 0
+
+
+def weights_changed():
+ global WEIGHTS_EPOCH
+ WEIGHTS_EPOCH += 1
+
+
# Need = CALIB_WORK x (KV + prefill work) + CALIB_GRAPH_KV x KV. Calibrated on Qwen3-4B QLoRA bs16 x 2048,
# T4: eager peaked 1.14x the estimate; graph capture OOMed at ~12 GiB free (~0.26 x KV extra).
CALIB_WORK = float(os.environ.get("UNSLOTH_DECODE_GRAPH_CALIB_WORK", "1.15"))
@@ -694,6 +733,10 @@ def invalidate(model):
global BUFFER_EPOCH
for state in _states(model):
state.clear()
+ for module in model.modules():
+ generic = module.__dict__.get("_unsloth_generic_decode_graphs")
+ if generic is not None:
+ generic.invalidate()
BUFFER_EPOCH += 1
@@ -709,6 +752,9 @@ def release(model):
found = True
state.clear()
state.statics.clear()
+ generic = d.get("_unsloth_generic_decode_graphs")
+ if generic is not None:
+ generic.invalidate()
if "_unsloth_graph_buffers" in d:
found = True
del d["_unsloth_graph_buffers"]
@@ -787,6 +833,18 @@ def _fingerprint(model, layers, rope_len):
items.append((cos.data_ptr(), sin.data_ptr(), cos.shape[0]))
# Every adapter-aware module outside the projections too (embeddings, lm_head, modules_to_save):
# their enabled / active / merged state picks the branch a captured step baked in.
+ adapters = _adapter_state(model)
+ if adapters is None:
+ return None
+ items.extend(adapters)
+ items.append(K._OPTIMIZER_STEPS)
+ items.append(WEIGHTS_EPOCH)
+ return tuple(items)
+
+
+def _adapter_state(model):
+ """(module, disabled, active adapters, merged) of every adapter-aware module; None if unreadable."""
+ items = []
for module in model.modules():
if hasattr(module, "disable_adapters") or hasattr(module, "active_adapter"):
try:
@@ -805,8 +863,7 @@ def _fingerprint(model, layers, rope_len):
)
except Exception:
return None
- items.append(K._OPTIMIZER_STEPS)
- return tuple(items)
+ return items
def _eligible(model, layers):
diff --git a/unsloth/models/_generic_decode_graphs.py b/unsloth/models/_generic_decode_graphs.py
new file mode 100644
index 00000000000..86a29271cb7
--- /dev/null
+++ b/unsloth/models/_generic_decode_graphs.py
@@ -0,0 +1,1936 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""CUDA-graph decode for any FastModel model generating over a transformers StaticCache.
+
+Each one-token decode step of `unsloth_base_fast_generate` replays a CUDAGraph of the model's own step,
+with tensor arguments copied into static buffers. Only where `_decode_graphs.enabled()`, on NVIDIA CUDA,
+over static attention layers that index the cache on the device. Per graph key: the first step runs
+eagerly, then is captured; the first replay must match the eager step bitwise; a host sync, host-state
+change or mismatch refuses the key. The step runs in the eager step's exact state (argument inference-ness
+and strides, ambient autocast + cache flag, compiled regions as eager runs them): Dynamo guards on all of
+these and another compiled variant rounds differently. Plain generate() runs inside generate_scope(), so
+graphs are on by default there (plan() decides per call: short calls, low free memory and unsupported
+calls decode eagerly; an out-of-memory graphed call is redone once eagerly). After an inference-mode call
+the graph and its StaticCache are kept for the next call of the same batch (keep()); training, a returned
+or caller-owned cache, and parameter / adapter / quant changes drop them. UNSLOTH_DECODE_GENERIC_GRAPHS=0
+disables; _VALIDATE_ALL=1 checks every replay (debug); batches over UNSLOTH_DECODE_GENERIC_GRAPH_MAX_BSZ
+(16) decode eagerly; UNSLOTH_DECODE_GENERIC_GRAPHS_KEEP=0 drops graphs at the end of every call."""
+
+import collections
+import collections.abc
+import contextlib
+import warnings
+import copy
+import functools
+import threading
+import os
+import traceback
+import weakref
+
+import torch
+
+from . import _decode_graphs as G
+
+__all__ = [
+ "wanted",
+ "plan",
+ "ineligible",
+ "begin",
+ "end",
+ "decode_step",
+ "release",
+ "stats",
+ "keepable",
+ "kept_cache",
+ "redo_eagerly",
+]
+
+_ATTR = "_unsloth_generic_decode_graphs"
+# Host length counters of static cache layers (4.57 sliding: cumulative_length; 5.x: cumulative_length_int).
+_COUNTERS = ("cumulative_length", "cumulative_length_int")
+_MAX_ARG_BYTES = 16 * 1024 * 1024
+_CONSTANTS = (bool, int, float, str, torch.dtype, torch.device)
+_MAX_KEYS = 4096
+# Graphs only remove launch overhead (T4 1B 4-bit: 4-7x at bsz 1, 2.7-4.7x at 16).
+MAX_BSZ = int(os.environ.get("UNSLOTH_DECODE_GENERIC_GRAPH_MAX_BSZ", "16"))
+# Free memory a graphed call needs beyond the eager one: the graph pool (8-16 MiB measured on 1B models),
+# static input buffers and the side stream's workspace. Below it the call decodes eagerly.
+_FREE_MARGIN_BYTES = 256 * 1024 * 1024
+# A kept cache longer than this many times the call's own length is not reused (attention reads every slot).
+_KEEP_MAX_RATIO = 2
+# Captures in a row without a replay: the key changes every step, so the rest of the call decodes eagerly.
+# A healthy call captures at most twice in a row (window filling, then full).
+_MAX_UNREPLAYED_CAPTURES = 4
+PLANS = collections.Counter()
+EAGER_REDOS = 0
+_OOM_WARNED = False
+
+try:
+ from torch.nn.attention.flex_attention import BlockMask as _BlockMask
+except Exception:
+ _BlockMask = None
+
+
+class _Refuse(Exception):
+ pass
+
+
+class _Cuda:
+ """The CUDA calls this path makes (tests swap in a host stand-in)."""
+
+ device_type = "cuda"
+
+ @staticmethod
+ def available():
+ # NVIDIA CUDA only: HIP replay is not verified against the eager step.
+ return torch.version.hip is None and torch.cuda.is_available()
+
+ @staticmethod
+ def current_device():
+ return torch.cuda.current_device()
+
+ @staticmethod
+ def free_bytes(device):
+ return G._free_bytes(device)
+
+ @staticmethod
+ def device(index):
+ return torch.cuda.device(index)
+
+ @staticmethod
+ def capturing():
+ return torch.cuda.is_current_stream_capturing()
+
+ @staticmethod
+ def pool():
+ return torch.cuda.graph_pool_handle()
+
+ @staticmethod
+ def new_stream():
+ return torch.cuda.Stream()
+
+ @staticmethod
+ def stream(stream):
+ return torch.cuda.stream(stream)
+
+ @staticmethod
+ def current_stream():
+ return torch.cuda.current_stream()
+
+ @staticmethod
+ def device_stream(index):
+ return torch.cuda.current_stream(index)
+
+ @staticmethod
+ def multidevice_supported():
+ return True
+
+ @staticmethod
+ def mempool():
+ # One graph pool per other card, shared by that card's segments (captured and replayed in order).
+ return torch.cuda.graph_pool_handle()
+
+ @staticmethod
+ def side_stream(index):
+ return torch.cuda.Stream(device = index)
+
+ @staticmethod
+ @contextlib.contextmanager
+ def on_streams(streams):
+ """The other cards' current streams set to `streams` ({index: stream}), restored after."""
+ before = {d: torch.cuda.current_stream(d) for d in streams}
+ here = torch.cuda.current_device()
+ try:
+ for d, stream in streams.items():
+ stream.wait_stream(before[d])
+ with torch.cuda.device(d):
+ torch.cuda.set_stream(stream)
+ yield
+ finally:
+ for d, stream in before.items():
+ stream.wait_stream(streams[d])
+ with torch.cuda.device(d):
+ torch.cuda.set_stream(stream)
+ torch.cuda.set_device(here)
+
+ @staticmethod
+ def card_of(value):
+ """The card index a tensor lives on, or None (host tensors)."""
+ if not value.is_cuda:
+ return None
+ return value.device.index if value.device.index is not None else torch.cuda.current_device()
+
+ @staticmethod
+ def index_of(device):
+ """The card index a send_to_device target names (int, "cuda:1", torch.device), or None."""
+ if isinstance(device, int):
+ return device
+ try:
+ device = torch.device(device)
+ except Exception:
+ return None
+ if device.type != "cuda":
+ return None
+ return device.index if device.index is not None else torch.cuda.current_device()
+
+ @staticmethod
+ def card_device(index):
+ return torch.device("cuda", index)
+
+ @staticmethod
+ def new_graph():
+ return torch.cuda.CUDAGraph()
+
+ @staticmethod
+ def begin_on(graph, pool, index):
+ """Capture on card `index`'s current stream (its side stream, made current by on_streams)."""
+ torch.cuda.set_device(index)
+ G._begin(graph, pool)
+
+ @staticmethod
+ def end_on(graph):
+ with warnings.catch_warnings():
+ # A segment with no kernels (the tail after the last hand-off) is an empty, valid graph.
+ warnings.simplefilter("ignore")
+ G._end(graph)
+
+ @staticmethod
+ def record(
+ pool,
+ fn,
+ cache,
+ others = None,
+ mempools = None,
+ ):
+ """(graph, fn()) with fn's launches captured into graph (`cache`: for host stand-ins).
+ `others` ({index: stream}): the other cards of a model split over cards, their side streams
+ current there (on_streams). The step is then captured as one graph per run of work on one
+ card, cut where accelerate's AlignDevicesHook moves tensors to another card; those hand-offs
+ run eagerly between the segments' replays (_Segments). Capturing a cross-card copy is not
+ possible without peer access (cudaErrorStreamCaptureUnsupported on PCIe T4s)."""
+ if others:
+ here = torch.cuda.current_device()
+ pools = dict(mempools)
+ pools[here] = pool
+ try:
+ return _SegmentRecorder(_BACKEND, pools, here).run(fn)
+ finally:
+ torch.cuda.set_device(here)
+ graph = torch.cuda.CUDAGraph()
+ G._begin(graph, pool)
+ try:
+ out = fn()
+ except BaseException:
+ # End the invalidated capture, but report what broke it, not the capture_end error.
+ if torch.cuda.is_current_stream_capturing():
+ try:
+ G._end(graph)
+ except Exception:
+ pass
+ raise
+ G._end(graph)
+ return graph, out
+
+
+class _Segments:
+ """One decode step of a model split over cards: ("graph", graph, card) parts replayed on their card's
+ current stream, and ("copy", dst, src) hand-offs (accelerate's moves at a block boundary) run eagerly
+ between them into the static buffers the next segment reads. Same order as the capture."""
+
+ def __init__(self):
+ self.parts = []
+
+ def replay(self):
+ for kind, a, b in self.parts:
+ if kind == "graph":
+ with _BACKEND.device(b):
+ a.replay()
+ else:
+ # Eager cross-card copy: PyTorch orders it after src's stream and before dst's.
+ a.copy_(b)
+
+
+_RECORDING = threading.Lock()
+
+
+class _SegmentRecorder:
+ """Captures fn() as per-card segments. accelerate's AlignDevicesHook pre_forward / post_forward move
+ tensors through accelerate.hooks.send_to_device; while recording, that name points at self.send:
+ the first move of a tensor to another card ends the open segment, copies eagerly into a buffer kept
+ for replay (recorded as a hand-off), and the next segment starts on the target card when the hook
+ returns. A tensor already moved to that card in this step reuses its buffer (no new cut), so a
+ layer-invariant input (rotary cos / sin, masks) crosses once."""
+
+ def __init__(self, backend, pools, first):
+ self.backend = backend
+ self.pools = pools
+ self.segments = _Segments()
+ self.graph = None
+ self.card = first
+ self.pending = None
+ self.moved = {}
+ self.original = None
+
+ def begin(self, card):
+ self.graph = self.backend.new_graph()
+ self.card = card
+ self.backend.begin_on(self.graph, self.pools[card], card)
+
+ def end(self):
+ if self.graph is not None:
+ graph, self.graph = self.graph, None
+ self.backend.end_on(graph)
+ self.segments.parts.append(("graph", graph, self.card))
+
+ def resume(self):
+ if self.pending is not None and self.graph is None:
+ card, self.pending = self.pending, None
+ self.begin(card)
+
+ def _move(self, value, card, target, skip_keys):
+ if isinstance(value, torch.Tensor):
+ if self.backend.card_of(value) in (None, card):
+ return value
+ key = (id(value), card)
+ hit = self.moved.get(key)
+ if hit is not None:
+ return hit[1]
+ # First crossing of this tensor to `card`: the segment ends here, the copy runs eagerly.
+ self.end()
+ dst = value.to(target)
+ self.moved[key] = (value, dst)
+ self.segments.parts.append(("copy", dst, value))
+ self.pending = card
+ return dst
+ if (
+ _BlockMask is not None
+ and isinstance(value, _BlockMask)
+ and "_unsloth_graph_closure" in value.__dict__
+ ):
+ # A graph BlockMask (_build): its fields and the tensors its mask_mod reads cross like tensors
+ # (eager hand-offs refilled every replay); BlockMask.to would leave mask_mod's tensors on this
+ # card, and a cross-card copy inside the capture is refused.
+ key = (id(value), card)
+ hit = self.moved.get(key)
+ if hit is None:
+ hit = self.moved[key] = (
+ value,
+ _block_mask_moved(value, lambda t: self._move(t, card, target, skip_keys)),
+ )
+ return hit[1]
+ if hasattr(value, "to"):
+ # Not a tensor but movable (BatchEncoding, ...): accelerate's own move, as eager. A real
+ # cross-card copy in it fails the capture, which refuses the key.
+ return self.original(value, target, skip_keys = skip_keys)
+ if isinstance(value, (tuple, list)):
+ from accelerate.utils.operations import honor_type
+ return honor_type(value, (self._move(v, card, target, skip_keys) for v in value))
+ if isinstance(value, collections.abc.Mapping):
+ skip = [skip_keys] if isinstance(skip_keys, str) else (skip_keys or [])
+ return type(value)(
+ {
+ k: v if k in skip else self._move(v, card, target, skip_keys)
+ for k, v in value.items()
+ }
+ )
+ return value
+
+ def send(
+ self,
+ original,
+ tensor,
+ device,
+ non_blocking = False,
+ skip_keys = None,
+ ):
+ card = self.backend.index_of(device)
+ if card is None:
+ return original(tensor, device, non_blocking = non_blocking, skip_keys = skip_keys)
+ return self._move(tensor, card, self.backend.card_device(card), skip_keys)
+
+ def run(self, fn):
+ # accelerate's hooks are process-global: two overlapping recordings (threads) would restore each
+ # other's replacements, leaving a finished recorder installed.
+ with _RECORDING:
+ return self._run(fn)
+
+ def _run(self, fn):
+ import accelerate.hooks as hooks
+
+ original_send = hooks.send_to_device
+ cls = hooks.AlignDevicesHook
+ original_pre, original_post = cls.pre_forward, cls.post_forward
+ recorder = self
+
+ def pre_forward(hook, module, *args, **kwargs):
+ out = original_pre(hook, module, *args, **kwargs)
+ recorder.resume()
+ return out
+
+ def post_forward(hook, module, output):
+ out = original_post(hook, module, output)
+ recorder.resume()
+ return out
+
+ self.original = original_send
+ hooks.send_to_device = functools.partial(self.send, original_send)
+ cls.pre_forward, cls.post_forward = pre_forward, post_forward
+ try:
+ self.begin(self.card)
+ out = fn()
+ self.resume()
+ self.end()
+ except BaseException:
+ if self.graph is not None:
+ try:
+ self.backend.end_on(self.graph)
+ except Exception:
+ pass
+ self.graph = None
+ raise
+ finally:
+ hooks.send_to_device = original_send
+ cls.pre_forward, cls.post_forward = original_pre, original_post
+ return self.segments, out
+
+
+_BACKEND = _Cuda
+
+
+def _static_layer_types():
+ try:
+ from transformers import cache_utils
+ except Exception:
+ return (), ()
+ full = tuple(t for t in (getattr(cache_utils, "StaticLayer", None),) if isinstance(t, type))
+ sliding = tuple(
+ t
+ for t in (
+ getattr(cache_utils, name, None)
+ for name in ("StaticSlidingWindowLayer", "SlidingWindowLayer", "ChunkedSlidingLayer")
+ )
+ if isinstance(t, type)
+ )
+ return full, sliding
+
+
+_FULL_LAYERS, _SLIDING_LAYERS = _static_layer_types()
+
+
+def wanted(model):
+ """Whether this generate() call routes its decode steps through decode_step()."""
+ if torch.compiler.is_compiling():
+ return False
+ if os.environ.get("UNSLOTH_DECODE_GENERIC_GRAPHS", "1") == "0" or not G.enabled():
+ return False
+ if not _BACKEND.available(): # NVIDIA CUDA only (the backend refuses HIP)
+ return False
+ if not _FULL_LAYERS:
+ return False
+ device = getattr(model, "device", None)
+ return getattr(device, "type", None) == _BACKEND.device_type and _plain_forward(model)
+
+
+def _plan_len(model, args, kwargs):
+ """prompt + new tokens this call can reach (0 if unknown), as the llama path sizes its plan."""
+ try:
+ from .llama import _generation_plan_len
+ return int(_generation_plan_len(model, args, kwargs) or 0)
+ except Exception:
+ return 0
+
+
+def plan(model, args, kwargs, input_ids):
+ """ "graph" or "eager" for this generate() call, decided once before it decodes (inside
+ generate_scope(), so graphs are on by default). Eager: graphs not wanted (wanted()), batch over
+ MAX_BSZ, too few decode steps to win back a capture (unless a kept graph of this batch replays from
+ the first step), or too little free memory for the graph pool. A kept graph / cache that this call
+ cannot reuse is dropped here."""
+ state = model.__dict__.get(_ATTR)
+ if not wanted(model):
+ if state is not None:
+ state.clear()
+ PLANS["not wanted"] += 1
+ return "eager"
+ try:
+ prompt = int(input_ids.shape[1]) if input_ids.dim() >= 2 else 0
+ bsz = G.decode_batch(
+ int(input_ids.shape[0]), kwargs, getattr(model, "generation_config", None)
+ )
+ except Exception:
+ PLANS["unknown shape"] += 1
+ return "eager"
+ max_len = _plan_len(model, args, kwargs)
+ kept = (
+ state is not None
+ and state.kept is not None
+ and state.kept_bsz == bsz
+ and bool(state.graphs)
+ )
+ if kept and max_len:
+ # transformers sizes the cache prompt + new tokens - 1 (the last token is never written back).
+ length = _kept_len(state)
+ # transformers 5.x grows every later call's cache to the longest one so far (_previous_max_cache_length):
+ # this call gets a cache of that size whether or not the kept one is reused, so compare against it.
+ need = max_len - 1
+ previous = getattr(model, "_previous_max_cache_length", -1)
+ if type(previous) is int:
+ need = max(need, previous)
+ kept = length is not None and need <= length <= _KEEP_MAX_RATIO * max(need, 1)
+ if state is not None and state.kept is not None and not kept:
+ state.clear()
+ if bsz > MAX_BSZ:
+ PLANS["batch over max"] += 1
+ return "eager"
+ if max_len:
+ left = max_len - prompt - 1
+ # Step 1 is eager + capture, step 2 a validated replay: fewer steps cannot win back a capture.
+ if left < 2 or (left < G.CAPTURE_MIN_STEPS and not kept):
+ PLANS["short call"] += 1
+ return "eager"
+ if not kept:
+ try:
+ # A model split over cards needs room for its graph pool on every card it runs on.
+ devices = [input_ids.device] + [
+ torch.device(_BACKEND.device_type, d)
+ for d in sorted(set((getattr(model, "hf_device_map", None) or {}).values()))
+ if type(d) is int and d != input_ids.device.index
+ ]
+ free = min(_BACKEND.free_bytes(d) for d in devices)
+ except Exception:
+ free = None
+ if free is not None and free < _FREE_MARGIN_BYTES:
+ PLANS["low free memory"] += 1
+ return "eager"
+ PLANS["graph kept" if kept else "graph"] += 1
+ return "graph"
+
+
+def ineligible(model):
+ """A call that cannot decode through graphs (dynamic cache, compiled decode): eager, and a kept
+ graph / cache goes (it would only hold memory)."""
+ state = model.__dict__.get(_ATTR)
+ if state is not None and state.kept is not None:
+ state.clear()
+ PLANS["ineligible"] += 1
+ return "eager"
+
+
+def replayable(kwargs):
+ """A failed graphed call can be redone eagerly only when no caller object holds its partial state."""
+ return not (
+ kwargs.get("streamer", None) is not None
+ or kwargs.get("past_key_values", None) is not None
+ or kwargs.get("stopping_criteria", None)
+ or kwargs.get("logits_processor", None)
+ )
+
+
+def rng_state(device):
+ """Host + this CUDA device's RNG state, so an eager redo samples as the failed attempt started."""
+ try:
+ cuda = torch.cuda.get_rng_state(device) if getattr(device, "type", None) == "cuda" else None
+ except Exception:
+ cuda = None
+ return torch.get_rng_state(), cuda, device
+
+
+def set_rng_state(saved):
+ host, cuda, device = saved
+ torch.set_rng_state(host)
+ if cuda is not None:
+ torch.cuda.set_rng_state(cuda, device)
+
+
+def redo_eagerly(model):
+ """After an out-of-memory graphed call: free graphs, kept cache and allocator blocks before the redo."""
+ global EAGER_REDOS, _OOM_WARNED
+ release(model)
+ model.__dict__.pop("_unsloth_generic_call_cache", None)
+ try:
+ G._empty_cache()
+ except Exception:
+ pass
+ EAGER_REDOS += 1
+ if not _OOM_WARNED:
+ _OOM_WARNED = True
+ import warnings
+ warnings.warn(
+ "Unsloth: generate() ran out of GPU memory with CUDA-graph decode; freed the decode graphs "
+ "and decoded this call eagerly (UNSLOTH_DECODE_CUDA_GRAPHS=0 always decodes eagerly).",
+ stacklevel = 3,
+ )
+
+
+def _plain_forward(model):
+ """No instance forward; the class forward bound on the instance, bare or under functools.wraps
+ wrappers (accelerate's mixed-precision autocast / fp32-output wrappers, which eager decode runs too);
+ Unsloth's GRPO hidden-states wrapper, which passes straight through while generating; or accelerate's
+ device hook of a model on one device (a bitsandbytes model placed on a card other than cuda:0 gets
+ one): its input / output moves are no-ops there, and the eager step runs the same hook. Offloading
+ hooks and device maps over several devices stay eager."""
+ forward = model.__dict__.get("forward")
+ if forward is None:
+ return True
+ if isinstance(forward, functools.partial) and "_hf_hook" in model.__dict__:
+ # accelerate's add_hook_to_module: forward = partial(new_forward, model), original in _old_forward.
+ if not (forward.args and forward.args[0] is model and not forward.keywords):
+ return False
+ if not _single_device_hook(model):
+ return False
+ old = model.__dict__.get("_old_forward")
+ return old is not None and _plain_bound(model, old)
+ if hasattr(model, "_hf_hook"):
+ return False
+ return _plain_bound(model, forward)
+
+
+def _plain_bound(model, forward):
+ func = getattr(forward, "__func__", forward)
+ if getattr(func, "_unsloth_grpo_hidden_states_forward_wrapped", False):
+ return True
+ if getattr(forward, "__self__", None) is not model:
+ return False
+ target = getattr(type(model), "forward", None)
+ for _ in range(4):
+ if func is target:
+ return True
+ func = getattr(func, "__wrapped__", None)
+ if func is None:
+ return False
+ return False
+
+
+def _single_device_hook(model):
+ """accelerate AlignDevicesHook(s) that only move tensors to the model's own (single) device: no
+ offload, no weights map. Another hook type, or an execution device elsewhere, is not ours to graph."""
+ try:
+ from accelerate.hooks import AlignDevicesHook, SequentialHook
+ except Exception:
+ return False
+ hook = model.__dict__.get("_hf_hook")
+ hooks = list(hook.hooks) if isinstance(hook, SequentialHook) else [hook]
+ device = getattr(model, "device", None)
+ for h in hooks:
+ if type(h) is not AlignDevicesHook:
+ return False
+ if h.offload or getattr(h, "offload_buffers", False) or h.weights_map is not None:
+ return False
+ target = h.execution_device
+ if target is None:
+ continue
+ try:
+ # accelerate names a GPU by its index; torch.device(int) asks for the current accelerator, which
+ # a CPU-only torch build does not have (it raises).
+ if type(target) is int:
+ target = torch.device(_BACKEND.device_type, target)
+ target = torch.device(target)
+ except Exception:
+ return False
+ if not isinstance(device, torch.device) or target.type != device.type:
+ return False
+ same = (target.index if target.index is not None else 0) == (
+ device.index if device.index is not None else 0
+ )
+ if not same and not _split_entry_card(model, target):
+ return False
+ return True
+
+
+def _split_entry_card(model, target):
+ """A model split over cards (multi-device graphs on): its root hook moves the inputs to the card of the
+ first block, which need not be `model.device` (the first parameter: a tied embedding / lm_head can sit
+ on the last card). Any card the model's parameters are on is its own."""
+ if not _multidevice_enabled():
+ return False
+ devices = _param_devices(model)
+ return len(devices) > 1 and _device_index(target) in {
+ _device_index(d) for d in devices if d.type == target.type
+ }
+
+
+class _Entry:
+ __slots__ = ("graph", "leaves", "outputs", "validated")
+
+ def __init__(self, graph, leaves, outputs, validated):
+ self.graph = graph
+ self.leaves = leaves
+ self.outputs = outputs
+ self.validated = validated
+
+
+class _State:
+ def __init__(self):
+ self.graphs = collections.OrderedDict()
+ self.pool = None
+ self.others = ()
+ self.mempools = {}
+ self.side = {}
+ self.fp_reason = None
+ self.stream = None
+ self.fingerprint = None
+ self.optimizer_steps = -1
+ self.recheck = True
+ self.cache_sig = None
+ # Keys refused for the process (host sync in capture, host state, validation mismatch).
+ self.refused = {}
+ self.validated = set()
+ self.moved = None
+ self.key_captures = collections.Counter()
+ self.step = 0
+ self.forwards = 0
+ self.capture_errors = []
+ self.check = _new_check()
+ self.captures = 0
+ self.replays = 0
+ self.validations = 0
+ self.eager_steps = 0
+ self.refusals = collections.Counter()
+ # keep(): the StaticCache the kept graphs address, its batch, and the training-forward hook.
+ self.kept = None
+ self.kept_bsz = None
+ # The max_cache_len the kept cache was built for (5.x), else None (its full layers' length is used).
+ self.kept_len = None
+ self.hook = None
+ self.owner = None
+ self.kept_calls = 0
+ self.active = False
+ self.unreplayed = 0
+ self.call_stop = None
+
+ def clear(self):
+ self.graphs.clear()
+ self.cache_sig = None
+ # A pool whose graphs are all freed is released by the allocator: never reuse its handle.
+ self.pool = None
+ self.mempools = {}
+ self.drop_kept()
+
+ def invalidate(self):
+ """Weights / adapters / quant state changed: graphs and every per-key decision go."""
+ self.clear()
+ self.fingerprint = None
+ self.validated.clear()
+ self.refused.clear()
+
+ def drop_kept(self):
+ """Forget the kept cache (and free it when the model still holds it for the next call)."""
+ kept, self.kept, self.kept_bsz, self.kept_len = self.kept, None, None, None
+ if self.hook is not None:
+ self.hook.remove()
+ self.hook = None
+ owner = self.owner() if self.owner is not None else None
+ # Inside a call the cache is that call's (transformers holds it): only forget it.
+ if (
+ not self.active
+ and kept is not None
+ and owner is not None
+ and owner.__dict__.get("_cache") is kept
+ ):
+ del owner.__dict__["_cache"]
+
+
+def _state(model):
+ state = model.__dict__.get(_ATTR)
+ if state is None:
+ state = model.__dict__[_ATTR] = _State()
+ state.owner = weakref.ref(model)
+ return state
+
+
+def begin(model):
+ """Start of a generate() call: the first graph step re-checks the fingerprint."""
+ state = _state(model)
+ state.recheck = True
+ state.forwards = 0
+ state.moved = None
+ state.key_captures.clear()
+ state.step = 0
+ state.check = _new_check()
+ state.active = True
+ state.unreplayed = 0
+ state.call_stop = None
+ if state.hook is not None:
+ # Only needed between calls: no per-step hook while this call decodes.
+ state.hook.remove()
+ state.hook = None
+ if len(state.refused) + len(state.validated) > _MAX_KEYS:
+ state.refused.clear()
+ state.validated.clear()
+
+
+def end(model, keep = False):
+ """End of a generate() call. Its graphs read this call's cache: freed, unless `keep` (keepable()), when
+ the graphs and that cache stay for the next call of the same batch (released by training)."""
+ state = model.__dict__.get(_ATTR)
+ if state is None:
+ return
+ state.active = False
+ cache = model.__dict__.pop("_unsloth_generic_call_cache", None)
+ length = model.__dict__.pop("_unsloth_generic_call_len", None)
+ if (
+ not keep
+ or not state.graphs
+ or cache is None
+ or state.cache_sig is None
+ or state.cache_sig[0] != id(cache)
+ ):
+ state.clear()
+ return
+ state.kept, state.kept_bsz, state.kept_len = cache, _cache_batch(cache), length
+ state.kept_calls += 1
+ if state.hook is None:
+ state.hook = model.register_forward_pre_hook(_release_if_training)
+
+
+def _release_if_training(module, args):
+ # Training can resume without for_training() (sampling in a trainer callback): free the kept graph and cache.
+ # Not while Dynamo traces a compiled training forward (removing this hook there is a graph break, an error
+ # under fullgraph): the next generate() re-checks the fingerprint, and for_training() releases as before.
+ if torch.compiler.is_compiling():
+ return
+ if module.training and torch.is_grad_enabled():
+ release(module)
+
+
+def _cache_batch(cache):
+ """Rows of a StaticCache (4.x: max_batch_size; 5.x layers only carry their tensors)."""
+ try:
+ keys = cache.layers[0].keys
+ if isinstance(keys, torch.Tensor) and keys.dim() >= 1:
+ return int(keys.shape[0])
+ except Exception:
+ pass
+ try:
+ return int(cache.max_batch_size)
+ except Exception:
+ return None
+
+
+def _kept_len(state):
+ # Sliding layers are capped at their window, so only the length the cache was built for says what it holds.
+ return state.kept_len if state.kept_len is not None else _cache_len(state.kept)
+
+
+def _cache_len(cache):
+ """Slots of a StaticCache's full layers (sliding layers are shorter; 5.x has no cache-level length)."""
+ try:
+ return max(int(layer.max_cache_len) for layer in cache.layers)
+ except Exception:
+ return None
+
+
+def kept_cache(model):
+ state = model.__dict__.get(_ATTR)
+ return state.kept if state is not None else None
+
+
+def keepable(model, kwargs, output):
+ """Whether this inference-mode call's graphs and StaticCache may stay for the next call: not when the
+ caller passed or got back the cache (a later call would overwrite it), nor in training mode."""
+ if os.environ.get("UNSLOTH_DECODE_GENERIC_GRAPHS_KEEP", "1") == "0":
+ return False
+ if getattr(model, "training", False) or kwargs.get("past_key_values", None) is not None:
+ return False
+ if getattr(output, "past_key_values", None) is not None:
+ return False
+ return True
+
+
+def release(model):
+ for module in model.modules():
+ state = module.__dict__.get(_ATTR)
+ if state is not None:
+ state.invalidate()
+
+
+def _kept_compatible(state, batch_size, max_cache_len, offloading):
+ kept = state.kept
+ if kept is None:
+ return False
+ length = _kept_len(state)
+ if length is None:
+ return False
+ return (
+ state.kept_bsz == batch_size
+ and bool(getattr(kept, "offloading", False)) == bool(offloading)
+ and max_cache_len <= length <= _KEEP_MAX_RATIO * max(int(max_cache_len), 1)
+ )
+
+
+class reuse_kept_cache:
+ """One graphed generate() call: transformers builds this call's StaticCache, unless a compatible kept
+ one exists, which is reset and handed back (same tensors, so the kept graphs replay). 5.x builds a new
+ cache per call in _prepare_static_cache; 4.x reuses model._cache, which _clear_generation_caches keeps
+ while it is the kept cache. The cache the call ends up with is recorded for end(keep=True)."""
+
+ def __init__(self, model):
+ self.model = model
+ self.prepare = None
+
+ def _prepare(self, *args, **kwargs):
+ model = self.model
+ state = _state(model)
+ names = ("cache_implementation", "batch_size", "max_cache_len")
+ values = dict(zip(names, args))
+ values.update({k: v for k, v in kwargs.items() if k in names})
+ cache_implementation = values.get("cache_implementation", "static")
+ batch_size, max_cache_len = values.get("batch_size"), values.get("max_cache_len")
+ prefill_chunk = kwargs.get("prefill_chunk_size", args[3] if len(args) > 3 else None)
+ effective = max_cache_len
+ previous = getattr(model, "_previous_max_cache_length", -1)
+ if type(effective) is int and type(previous) is int:
+ effective = max(effective, previous)
+ if (
+ type(batch_size) is int
+ and type(effective) is int
+ and prefill_chunk is None
+ and not getattr(model.config, "is_encoder_decoder", False)
+ and _kept_compatible(
+ state, batch_size, effective, "offloaded" in str(cache_implementation)
+ )
+ ):
+ cache = state.kept
+ cache.reset()
+ PLANS["reused kept cache"] += 1
+ else:
+ if state.kept is not None:
+ state.clear()
+ cache = self.prepare(*args, **kwargs)
+ model.__dict__["_unsloth_generic_call_cache"] = cache
+ if type(effective) is int:
+ model.__dict__["_unsloth_generic_call_len"] = (
+ effective if cache is not state.kept else _kept_len(state)
+ )
+ return cache
+
+ def __enter__(self):
+ model = self.model
+ if "_prepare_static_cache" in model.__dict__ or not hasattr(model, "_prepare_static_cache"):
+ # 4.x (or a wrapper owns the hook): the call's cache is model._cache, read in end().
+ self.prepare = None
+ return self
+ self.prepare = model._prepare_static_cache
+ model._prepare_static_cache = self._prepare
+ return self
+
+ def __exit__(self, *exc):
+ model = self.model
+ if self.prepare is not None:
+ del model._prepare_static_cache
+ elif "_unsloth_generic_call_cache" not in model.__dict__:
+ cache = model.__dict__.get("_cache")
+ if cache is not None:
+ model.__dict__["_unsloth_generic_call_cache"] = cache
+ return False
+
+
+def stats(model):
+ state = getattr(model, "__dict__", {}).get(_ATTR)
+ if state is None:
+ return {}
+ return {
+ "captures": state.captures,
+ "replays": state.replays,
+ "validations": state.validations,
+ "eager_steps": state.eager_steps,
+ "graphs": len(state.graphs),
+ "refusals": dict(state.refusals),
+ "refused_keys": len(state.refused),
+ "max_captures_per_key": max(state.key_captures.values(), default = 0),
+ "capture_errors": list(state.capture_errors),
+ "check": dict(state.check),
+ "other_devices": list(state.others),
+ }
+
+
+def _new_check():
+ return {
+ "steps": 0,
+ "mismatches": 0,
+ "first_mismatch_step": None,
+ "max_abs_diff": 0.0,
+ "max_kl": 0.0,
+ "top1_disagree": 0,
+ }
+
+
+def _validate_all():
+ return os.environ.get("UNSLOTH_DECODE_GENERIC_GRAPHS_VALIDATE_ALL", "0") == "1"
+
+
+def _record_check(state, eager, replay, equal):
+ """Accumulates how far the replay's first output (logits) is from the eager step's."""
+ check = state.check
+ check["steps"] += 1
+ if not equal:
+ check["mismatches"] += 1
+ if check["first_mismatch_step"] is None:
+ check["first_mismatch_step"] = state.step
+ xs, ys = _tensors(eager, []), _tensors(replay, [])
+ if not xs or not ys or not xs[0].is_floating_point() or xs[0].shape != ys[0].shape:
+ return
+ x, y = xs[0].float(), ys[0].float()
+ check["max_abs_diff"] = max(check["max_abs_diff"], float((x - y).abs().max()))
+ if x.dim() >= 2:
+ p, q = torch.log_softmax(x, -1), torch.log_softmax(y, -1)
+ kl = (p.exp() * (p - q)).sum(-1).max()
+ check["max_kl"] = max(check["max_kl"], float(kl))
+ check["top1_disagree"] += int((x.argmax(-1) != y.argmax(-1)).sum())
+
+
+def _where(error):
+ """file:line of the innermost frame outside torch that raised `error`."""
+ for frame in reversed(traceback.extract_tb(error.__traceback__)):
+ if f"{os.sep}torch{os.sep}" not in frame.filename:
+ return f"{os.path.basename(frame.filename)}:{frame.lineno} {frame.name}"
+ return None
+
+
+def _refusal(state, reason):
+ state.refusals[reason] += 1
+ state.eager_steps += 1
+ return None
+
+
+def _cache_reason(cache):
+ """None when every layer of `cache` is a static attention layer this path can replay."""
+ if cache is None:
+ return "no cache"
+ try:
+ from transformers import StaticCache
+ except Exception:
+ return "no StaticCache"
+ if not isinstance(cache, StaticCache):
+ return f"cache {type(cache).__name__}"
+ if getattr(cache, "offloading", False):
+ return "offloaded cache"
+ layers = getattr(cache, "layers", None)
+ if not layers:
+ return "legacy cache"
+ for layer in layers:
+ kind = type(layer)
+ # Exact types: 5.x linear-attention / indexed layers subclass StaticLayer.
+ if kind not in _FULL_LAYERS and kind not in _SLIDING_LAYERS:
+ return f"layer {kind.__name__}"
+ keys = getattr(layer, "keys", None)
+ if not getattr(layer, "is_initialized", False) or not isinstance(keys, torch.Tensor):
+ return "uninitialized layer"
+ if keys.device.type != _BACKEND.device_type:
+ return "cache not on CUDA"
+ return None
+
+
+def _host_counter(layer):
+ for name in ("cumulative_length_int", "cumulative_length"):
+ value = layer.__dict__.get(name)
+ if type(value) is int:
+ return value
+ return None
+
+
+def _window_reason(cache, q_len):
+ # A step that fills a sliding window switches the update's branch mid-call: eager. Before and after it, every
+ # one-token step runs the same device work (write at the device counter, or roll by one slot), and the key
+ # carries which of the two each layer is in (_window_state), so a replay repeats the right one.
+ for layer in cache.layers:
+ if type(layer) in _SLIDING_LAYERS:
+ count = _host_counter(layer)
+ if count is None:
+ return "sliding layer without host counter"
+ if count < layer.max_cache_len < count + q_len:
+ return "sliding window filling"
+ return None
+
+
+def _window_fills(cache, q_len):
+ """Whether this step fills a sliding window (count + q_len reaches it). Its key is the filling one, but a
+ capture records after the warm-up advanced the counters, i.e. the rolling branch: never capture it."""
+ for layer in cache.layers:
+ if type(layer) in _SLIDING_LAYERS:
+ count = _host_counter(layer)
+ if count is not None and count < layer.max_cache_len <= count + q_len:
+ return True
+ return False
+
+
+def _window_state(cache):
+ """Per sliding layer: whether its window is full (it rolls) or still filling (writes at the counter)."""
+ return tuple(
+ _host_counter(layer) >= layer.max_cache_len
+ for layer in cache.layers
+ if type(layer) in _SLIDING_LAYERS
+ )
+
+
+# transformers 5.x rolls a full static sliding window by one slot per one-token step and indexes the last slot
+# with `torch.tensor([-1], device=...)`: a host-to-device copy a CUDA graph capture refuses. While a step is
+# warmed up and recorded, that branch runs the same ops on an index tensor made outside the capture.
+_ROLL_INDEX = {}
+
+
+def _roll_index(device):
+ index = _ROLL_INDEX.get(device)
+ if index is None:
+ if _BACKEND.capturing():
+ return None
+ with G._no_inference_tensors():
+ index = _ROLL_INDEX[device] = torch.tensor([-1], dtype = int, device = device)
+ return index
+
+
+def _rolling_update_classes():
+ """(class, host counter) of the static sliding layers (5.x StaticSlidingWindowLayer, 4.57 SlidingWindowLayer)
+ whose one-token full-window update builds its roll index on the host."""
+ try:
+ import inspect
+ from transformers import cache_utils
+ except Exception:
+ return ()
+ out = []
+ for cls in _SLIDING_LAYERS:
+ try:
+ source = inspect.getsource(cls.__dict__["update"])
+ except Exception:
+ continue
+ counter = next(
+ (c for c in ("cumulative_length_int", "cumulative_length") if f"self.{c} +=" in source),
+ None,
+ )
+ needed = ("roll(-1, dims=-2)", "torch.tensor([-1], dtype=int", "self.keys.copy_(new_keys)")
+ if counter is not None and all(n in source for n in needed):
+ out.append((cls, counter))
+ return tuple(out)
+
+
+_ROLLING_CLASSES = []
+
+
+def _graphable_update(original, counter):
+ def update(self, key_states, value_states, *args, **kwargs):
+ count = self.__dict__.get(counter)
+ if (
+ getattr(self, "is_initialized", False)
+ and key_states.shape[-2] == 1
+ and type(count) is int
+ and count >= self.max_cache_len
+ ):
+ index = _roll_index(self.keys.device)
+ if index is not None:
+ # transformers' one-token full-window branch, op for op.
+ self.__dict__[counter] = count + 1
+ new_keys = self.keys.roll(-1, dims = -2)
+ new_values = self.values.roll(-1, dims = -2)
+ new_keys[:, :, index] = key_states
+ new_values[:, :, index] = value_states
+ self.keys.copy_(new_keys)
+ self.values.copy_(new_values)
+ return self.keys, self.values
+ return original(self, key_states, value_states, *args, **kwargs)
+
+ update._unsloth_graphable_roll = True
+ return update
+
+
+def _rolling_classes():
+ if not _ROLLING_CLASSES:
+ _ROLLING_CLASSES.append(_rolling_update_classes())
+ return _ROLLING_CLASSES[0]
+
+
+def _graphable_rolls():
+ patched = []
+ try:
+ for cls, counter in _rolling_classes():
+ original = cls.__dict__.get("update")
+ if original is None or getattr(original, "_unsloth_graphable_roll", False):
+ continue
+ cls.update = _graphable_update(original, counter)
+ patched.append((cls, original))
+ yield
+ finally:
+ for cls, original in reversed(patched):
+ cls.update = original
+
+
+def _host_state(cache):
+ """Non-tensor scalar attributes of the cache and its layers."""
+ out = []
+ for owner in (cache, *cache.layers):
+ out.append({k: v for k, v in owner.__dict__.items() if v is None or type(v) in _CONSTANTS})
+ return out
+
+
+def _host_state_reason(before, after, q_len):
+ for b, a in zip(before, after):
+ for name in set(b) | set(a):
+ if b.get(name) == a.get(name):
+ continue
+ if name in _COUNTERS and type(b.get(name)) is int and a.get(name) == b[name] + q_len:
+ continue
+ return f"cache host state {name}"
+ return None
+
+
+def _snapshot(cache, rows = True):
+ """Length counters (host ints, device scalars) a step advances, plus what undoes the roll of every full sliding
+ window (a step rolls it, so unlike a slot write it is not idempotent); restored by _restore. rows=False: the
+ counters only (a recording runs no device work)."""
+ host, device = [], []
+ rolling = None
+ for layer in cache.layers:
+ for name in _COUNTERS:
+ value = layer.__dict__.get(name)
+ if type(value) is int:
+ host.append((layer, name, value))
+ elif isinstance(value, torch.Tensor) and value.numel() == 1:
+ device.append((value, value.clone(), False))
+ if rows and type(layer) in _SLIDING_LAYERS:
+ count = _host_counter(layer)
+ if count is not None and count >= layer.max_cache_len:
+ if rolling is None:
+ rolling = {cls for cls, _ in _rolling_classes()}
+ # A transformers rolling layer's one-token step drops slot 0 and shifts the rest down by one:
+ # keeping slot 0 undoes it with one layer's temporary, not a copy of every window held at once.
+ rolls = type(layer) in rolling
+ for t in (getattr(layer, "keys", None), getattr(layer, "values", None)):
+ if isinstance(t, torch.Tensor):
+ one = rolls and t.dim() == 4
+ device.append((t, t[..., :1, :].clone() if one else t.clone(), one))
+ return host, device
+
+
+def _restore(snapshot):
+ host, device = snapshot
+ for layer, name, value in host:
+ layer.__dict__[name] = value
+ for live, saved, rolled in device:
+ if rolled:
+ live.copy_(torch.cat((saved, live[..., :-1, :]), dim = -2))
+ else:
+ live.copy_(saved)
+
+
+def _advance(cache, q_len):
+ # The replay advanced the device counters; the host ones are advanced here, as the eager update.
+ for layer in cache.layers:
+ for name in _COUNTERS:
+ value = layer.__dict__.get(name)
+ if type(value) is int:
+ layer.__dict__[name] = value + q_len
+
+
+def _spec(value, leaves, cache):
+ """Hashable structure of a forward argument; its tensors appended to `leaves` in order."""
+ if isinstance(value, torch.Tensor):
+ if value.requires_grad:
+ raise _Refuse("tensor argument requires grad")
+ if value.device.type != _BACKEND.device_type:
+ raise _Refuse(f"tensor argument on {value.device}")
+ if value.numel() * value.element_size() > _MAX_ARG_BYTES:
+ raise _Refuse("large tensor argument")
+ leaves.append(value)
+ # Inference-ness: compiled regions (create_causal_mask) guard on the dispatch key set.
+ return (
+ "T",
+ tuple(value.shape),
+ value.stride(),
+ value.dtype,
+ value.device,
+ value.is_inference(),
+ )
+ if value is cache:
+ return ("C",)
+ if value is None or type(value) in _CONSTANTS:
+ return ("V", value)
+ if type(value) in (tuple, list):
+ return (type(value).__name__, tuple(_spec(v, leaves, cache) for v in value))
+ if type(value) is dict and all(type(k) is str for k in value):
+ return ("D", tuple((k, _spec(v, leaves, cache)) for k, v in value.items()))
+ if _BlockMask is not None and isinstance(value, _BlockMask):
+ return _block_mask_spec(value, leaves, cache)
+ raise _Refuse(f"argument {type(value).__name__}")
+
+
+# BlockMask graph contract. A flex BlockMask argument replays only when its builder publishes
+# block_mask._unsloth_graph_closure = (tensors, rebuild)
+# `tensors`: a tuple of every tensor its mask_mod reads (offsets, padding rows, ...), fixed shapes from
+# step to step; `rebuild(tensors)`: a mask_mod of the same code reading the given tensors instead. The
+# graph keeps static copies of the BlockMask's tensor fields and of `tensors`, refills them each step and
+# captures flex over a BlockMask built from them (mask_mod = rebuild(static copies)). Key: BLOCK_SIZE,
+# seq_lengths, the field layout, mask_mod's code and the closure tensors' layout. Any other BlockMask
+# (a mask_mod closing over tensors the graph cannot see) is refused: "argument BlockMask".
+_BLOCK_MASK_SKIP = ("mask_mod", "_unsloth_graph_closure", "_unsloth_on_device")
+
+
+def _block_mask_fields(value):
+ return sorted((k, v) for k, v in vars(value).items() if k not in _BLOCK_MASK_SKIP)
+
+
+def _block_mask_spec(value, leaves, cache):
+ contract = value.__dict__.get("_unsloth_graph_closure")
+ if (
+ type(contract) is not tuple
+ or len(contract) != 2
+ or type(contract[0]) is not tuple
+ or not all(isinstance(t, torch.Tensor) for t in contract[0])
+ or not callable(contract[1])
+ ):
+ raise _Refuse("argument BlockMask")
+ fields = []
+ for name, field in _block_mask_fields(value):
+ if field is None or isinstance(field, torch.Tensor) or type(field) in _CONSTANTS:
+ fields.append((name, _spec(field, leaves, cache)))
+ elif type(field) is tuple and all(type(x) in _CONSTANTS for x in field):
+ fields.append((name, ("V", field)))
+ else:
+ raise _Refuse("argument BlockMask")
+ code = getattr(value.mask_mod, "__code__", None)
+ if code is None:
+ raise _Refuse("argument BlockMask")
+ closure = tuple(_spec(t, leaves, cache) for t in contract[0])
+ return ("BM", tuple(fields), code, closure)
+
+
+def _build(value, buffers, cache):
+ """`value` with its tensors replaced by the next static buffers (same order as _spec)."""
+ if isinstance(value, torch.Tensor):
+ return next(buffers)
+ if type(value) in (tuple, list):
+ return type(value)(_build(v, buffers, cache) for v in value)
+ if type(value) is dict:
+ return {k: _build(v, buffers, cache) for k, v in value.items()}
+ if _BlockMask is not None and isinstance(value, _BlockMask):
+ built = copy.copy(value)
+ for name, field in _block_mask_fields(value):
+ if isinstance(field, torch.Tensor):
+ setattr(built, name, next(buffers))
+ tensors, rebuild = value.__dict__["_unsloth_graph_closure"]
+ static = tuple(next(buffers) for _ in tensors)
+ built.mask_mod = rebuild(static)
+ built._unsloth_graph_closure = (static, rebuild)
+ # The builder's per-device rebuild reads its own tensors, not these static copies: replaced by one
+ # over the static copies, so an accelerate hook moving the mask to another card of a split model
+ # (BlockMask.to leaves mask_mod's tensors behind) hands that card's layers a mask_mod on their card.
+ built._unsloth_on_device = functools.partial(_block_mask_on_device, built)
+ return built
+ return value
+
+
+def _block_mask_moved(value, move):
+ """A graph BlockMask with every tensor field and closure tensor passed through `move` (tensor -> tensor
+ on the target card) and mask_mod rebuilt over the moved closure tensors; `value` itself when nothing
+ moved (accelerate also "moves" the mask to the card it is on: the root hook, the first card's layers),
+ so it keeps the per-device move for the next card's hook."""
+ fields = [
+ (name, field, move(field))
+ for name, field in _block_mask_fields(value)
+ if isinstance(field, torch.Tensor)
+ ]
+ tensors, rebuild = value.__dict__["_unsloth_graph_closure"]
+ moved_tensors = tuple(move(t) for t in tensors)
+ if all(a is b for _, a, b in fields) and all(a is b for a, b in zip(tensors, moved_tensors)):
+ return value
+ moved = copy.copy(value)
+ for name, _, field in fields:
+ setattr(moved, name, field)
+ moved.mask_mod = rebuild(moved_tensors)
+ moved._unsloth_graph_closure = (moved_tensors, rebuild)
+ moved._unsloth_on_device = functools.partial(_block_mask_on_device, moved)
+ return moved
+
+
+def _block_mask_on_device(value, device):
+ """Eager (warm-up) move of a graph BlockMask: fresh copies of the current static values, never cached."""
+ device = torch.device(device)
+ return _block_mask_moved(value, lambda t: t.to(device))
+
+
+def _map_outputs(value, fn):
+ if isinstance(value, torch.Tensor):
+ return fn(value)
+ if isinstance(value, collections.OrderedDict) and hasattr(value, "to_tuple"):
+ return type(value)(**{k: _map_outputs(v, fn) for k, v in value.items()})
+ if type(value) in (tuple, list):
+ return type(value)(_map_outputs(v, fn) for v in value)
+ if type(value) is dict:
+ return {k: _map_outputs(v, fn) for k, v in value.items()}
+ return value
+
+
+def _tensors(value, out):
+ _map_outputs(value, lambda t: out.append(t) or t)
+ return out
+
+
+def _outputs_equal(a, b):
+ xs, ys = _tensors(a, []), _tensors(b, [])
+ if len(xs) != len(ys):
+ return False
+ for x, y in zip(xs, ys):
+ if x.shape != y.shape or x.dtype != y.dtype:
+ return False
+ if not torch.equal(x, y):
+ same_nan = torch.isnan(x) & torch.isnan(y) if x.is_floating_point() else None
+ if same_nan is None or not torch.equal(
+ torch.where(same_nan, 0, x), torch.where(same_nan, 0, y)
+ ):
+ return False
+ return True
+
+
+def _multidevice_enabled():
+ return os.environ.get("UNSLOTH_DECODE_MULTIDEVICE_GRAPHS", "1") != "0"
+
+
+def _hooks_align_only(model):
+ """Every accelerate hook in `model` only moves tensors to its block's card (AlignDevicesHook, no
+ offload, no weights map): a model split over cards by device_map / Unsloth's layer split. Offload,
+ other hook types or an unreadable hook keep the model eager."""
+ hooked = [m.__dict__["_hf_hook"] for m in model.modules() if "_hf_hook" in m.__dict__]
+ if not hooked:
+ return True
+ try:
+ from accelerate.hooks import AlignDevicesHook, SequentialHook
+ except Exception:
+ return False
+ for hook in hooked:
+ hooks = list(hook.hooks) if isinstance(hook, SequentialHook) else [hook]
+ for h in hooks:
+ if type(h) is not AlignDevicesHook:
+ return False
+ if h.offload or getattr(h, "offload_buffers", False) or h.weights_map is not None:
+ return False
+ target = h.execution_device
+ if target is not None:
+ try:
+ target = torch.device(target)
+ except Exception:
+ return False
+ if target.type != _BACKEND.device_type:
+ return False
+ return True
+
+
+def _param_devices(model):
+ # An input embedding offloaded to the host is looked up outside the graph: its device does not count.
+ host = _offloaded_input_embedding(model)
+ exempt = {id(p) for p in host.parameters()} if host is not None else ()
+ return {p.device for p in model.parameters() if id(p) not in exempt}
+
+
+def _device_index(device):
+ return device.index if device.index is not None else _BACKEND.current_device()
+
+
+def _multidevice_reason(model, devices):
+ """None when a model split over several cards may be graphed as one multi-card graph."""
+ if not _multidevice_enabled():
+ return "multi-device (UNSLOTH_DECODE_MULTIDEVICE_GRAPHS=0)"
+ if not _BACKEND.multidevice_supported():
+ return "multi-device capture needs a newer torch"
+ if not _hooks_align_only(model):
+ return "multi-device with offload or non-align hooks"
+ return None
+
+
+def _fingerprint(model, state = None):
+ """Parameters (address, dtype, shape, in-place version), quant state and adapter state; None
+ when the model is not on CUDA cards only, or is split over cards in a way this path does not
+ graph (offload, other hooks). `state` gets the other cards of a split model and the refusal."""
+ from ..kernels import utils as K
+
+ items = []
+ host = _offloaded_input_embedding(model)
+ # Its weights stay fingerprinted, by identity too (a swapped table re-captures); only their device is exempt.
+ exempt = {id(p) for p in host.parameters()} if host is not None else ()
+ for p in model.parameters():
+ if id(p) in exempt:
+ items.append(id(p))
+ items.append((p.data_ptr(), p.dtype, tuple(p.shape), G._version(p)))
+ devices = _param_devices(model)
+ if not devices or any(d.type != _BACKEND.device_type for d in devices):
+ if state is not None:
+ state.fp_reason = "not one CUDA device / adapters"
+ return None
+ others = ()
+ if len(devices) > 1:
+ reason = _multidevice_reason(model, devices)
+ if reason is not None:
+ if state is not None:
+ state.fp_reason = reason
+ return None
+ here = _BACKEND.current_device()
+ others = tuple(sorted({_device_index(d) for d in devices} - {here}))
+ if state is not None:
+ state.others = others
+ state.fp_reason = None
+ items.append(("devices", tuple(sorted(str(d) for d in devices))))
+ for b in model.buffers():
+ items.append((b.data_ptr(), b.dtype, tuple(b.shape)))
+ for module in model.modules():
+ # A LoRA layer's scale is a Python float the captured kernels baked in; PEFT's set_scale / scale_layer
+ # change it without touching a parameter.
+ scaling = module.__dict__.get("scaling")
+ if type(scaling) is dict and scaling:
+ items.append((id(module), tuple(sorted((str(k), v) for k, v in scaling.items()))))
+ weight = getattr(module, "weight", None)
+ quant_state = getattr(weight, "quant_state", None)
+ if quant_state is not None:
+ absmax = getattr(quant_state, "absmax", None)
+ items.append(
+ (id(quant_state), absmax.data_ptr() if isinstance(absmax, torch.Tensor) else None)
+ )
+ adapters = G._adapter_state(model)
+ if adapters is None:
+ if state is not None:
+ state.fp_reason = "not one CUDA device / adapters"
+ return None
+ items.extend(adapters)
+ items.append(K._OPTIMIZER_STEPS)
+ items.append(G.WEIGHTS_EPOCH)
+ return tuple(items)
+
+
+# Forward derives positions from `cache_position[0] == 0` (host sync) without position_ids (4.x Qwen3-VL).
+_MROPE_MODEL_TYPES = ("qwen3_vl", "qwen3_vl_moe")
+
+# Type ids a trainer passes to generate() (TRL's GRPO: mm_token_type_ids / token_type_ids) reach every decode
+# step at full length, a new graph key per step. Gemma 3 / 4 read them only to build masks, skipped when the
+# masks come prebuilt (a dict per layer type), so with prebuilt masks the step gets the last q_len columns.
+_TYPE_ID_ARGS = ("token_type_ids", "mm_token_type_ids")
+
+
+def _step_type_ids(kwargs, q_len):
+ if not isinstance(kwargs.get("attention_mask"), dict):
+ return kwargs
+ out = None
+ for name in _TYPE_ID_ARGS:
+ value = kwargs.get(name)
+ if isinstance(value, torch.Tensor) and value.dim() >= 2 and value.shape[-1] > q_len:
+ if out is None:
+ out = dict(kwargs)
+ # As transformers slices them: a contiguous copy (a view's row stride would still grow every step).
+ out[name] = value[..., -q_len:].clone(memory_format = torch.contiguous_format)
+ return kwargs if out is None else out
+
+
+def _mrope_positions(model, kwargs):
+ """kwargs with the decode position_ids the model's forward would derive, computed outside the
+ graph (same ops, so the same values); unchanged when they do not apply."""
+ if kwargs.get("position_ids") is not None:
+ return kwargs
+ if getattr(getattr(model, "config", None), "model_type", None) not in _MROPE_MODEL_TYPES:
+ return kwargs
+ inner = getattr(model, "model", None)
+ deltas = getattr(inner, "rope_deltas", None)
+ cache_position = kwargs.get("cache_position")
+ ids = kwargs.get("input_ids")
+ if ids is None:
+ ids = kwargs.get("inputs_embeds")
+ if not isinstance(deltas, torch.Tensor) or not isinstance(cache_position, torch.Tensor):
+ return kwargs
+ batch_size, seq_length = ids.shape[0], ids.shape[1]
+ if deltas.shape[0] == 0 or batch_size % deltas.shape[0] != 0:
+ return kwargs
+ delta = (cache_position[0] + deltas).to(ids.device)
+ position_ids = torch.arange(seq_length, device = ids.device)
+ position_ids = position_ids.view(1, -1).expand(batch_size, -1)
+ delta = delta.repeat_interleave(batch_size // delta.shape[0], dim = 0)
+ position_ids = position_ids.add(delta)
+ position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
+ return {**kwargs, "position_ids": position_ids}
+
+
+# Multimodal decoders whose forward zeroes placeholder ids with a boolean index (`ids[mask] = 0`, a host sync a
+# capture cannot hold): the step's embeddings are looked up outside the graph with the same ops. Model type ->
+# placeholder token ids.
+_PLACEHOLDER_EMBED_MODEL_TYPES = {
+ "muse_glimmer": ("image_token_id", "video_token_id"),
+ "muse_spark": ("image_token_id", "video_token_id", "audio_token_id"),
+ "kimi_k25": ("image_token_id", "video_token_id"),
+}
+
+
+def _placeholder_embeds(model, kwargs):
+ """kwargs with input_ids replaced by the inputs_embeds the model's forward would compute from them (a
+ one-token text decode step: no media); unchanged when this does not apply."""
+ config = getattr(model, "config", None)
+ names = _PLACEHOLDER_EMBED_MODEL_TYPES.get(getattr(config, "model_type", None))
+ ids = kwargs.get("input_ids")
+ if (
+ names is None
+ or not isinstance(ids, torch.Tensor)
+ or kwargs.get("inputs_embeds") is not None
+ ):
+ return kwargs
+ if any(
+ kwargs.get(k) is not None
+ for k in kwargs
+ if k.startswith(("pixel_values", "mm_encoder", "input_features"))
+ ):
+ return kwargs
+ tokens = [getattr(config, name, None) for name in names]
+ embed = getattr(model, "get_input_embeddings", None)
+ if any(type(t) is not int for t in tokens) or embed is None:
+ return kwargs
+ mask = ids == tokens[0]
+ for token in tokens[1:]:
+ mask = mask | (ids == token)
+ # As the forward's own lookup runs (no autograd under generate), on the decoder's card: an offloaded or
+ # hooked embedding may hand back a host tensor or one that tracks its trainable weight.
+ with torch.no_grad():
+ embeds = embed()(ids.masked_fill(mask, 0))
+ embeds = embeds.detach()
+ if embeds.device != ids.device:
+ embeds = embeds.to(ids.device)
+ return {**kwargs, "input_ids": None, "inputs_embeds": embeds}
+
+
+def _offloaded_input_embedding(model):
+ """The input embedding Unsloth's `offload_embedding` keeps in host RAM (its hooks move the ids there and the
+ looked-up rows back to the decoder's card), or None. Not when the output head shares a weight with it."""
+ get = getattr(model, "get_input_embeddings", None)
+ try:
+ module = get() if get is not None else None
+ except Exception:
+ return None
+ if not isinstance(module, torch.nn.Module) or not getattr(
+ module, "_unsloth_offload_hooks_installed", False
+ ):
+ return None
+ params = list(module.parameters())
+ if not params or all(p.device.type == _BACKEND.device_type for p in params):
+ return None
+ get_out = getattr(model, "get_output_embeddings", None)
+ try:
+ out = get_out() if get_out is not None else None
+ except Exception:
+ return None
+ if isinstance(out, torch.nn.Module):
+ shared = {id(p) for p in params}
+ if any(id(p) in shared for p in out.parameters()):
+ return None
+ return module
+
+
+def _host_embeds(model, kwargs):
+ """kwargs with input_ids replaced by the offloaded input embedding's rows, looked up outside the graph (a host
+ lookup and copy cannot be captured) exactly as the forward would look them up; unchanged otherwise."""
+ ids = kwargs.get("input_ids")
+ if not isinstance(ids, torch.Tensor) or kwargs.get("inputs_embeds") is not None:
+ return kwargs
+ module = _offloaded_input_embedding(model)
+ if module is None:
+ return kwargs
+ with torch.no_grad():
+ embeds = module(ids)
+ embeds = embeds.detach()
+ if embeds.device != ids.device:
+ embeds = embeds.to(ids.device)
+ return {**kwargs, "input_ids": None, "inputs_embeds": embeds}
+
+
+def decode_step(
+ model,
+ forward,
+ kwargs,
+ eager_regions = None,
+):
+ """The outputs of one decode step run through a CUDA graph (captured on first use), or None for
+ the caller's eager step. `forward` is the model's real forward; `eager_regions` an optional
+ context manager (zoo `unsloth_eager_decode`) every graph-path step runs inside."""
+ if torch.compiler.is_compiling() or torch.is_grad_enabled():
+ return None
+ if _BACKEND.capturing():
+ return None
+ state = _state(model)
+ state.forwards += 1
+ if state.forwards == 1:
+ # The first forward is the prefill, even for a one-token prompt (Qwen3-VL sets rope_deltas there).
+ return None
+ ids = kwargs.get("input_ids")
+ if ids is None:
+ ids = kwargs.get("inputs_embeds")
+ if (
+ not isinstance(ids, torch.Tensor)
+ or ids.dim() < 2
+ or ids.shape[1] != 1
+ or ids.device.type != _BACKEND.device_type
+ ):
+ return None
+ state.step += 1
+ if ids.shape[0] > MAX_BSZ:
+ return _refusal(state, "batch over max")
+ cache = kwargs.get("past_key_values")
+ reason = _cache_reason(cache)
+ if reason is None:
+ reason = _window_reason(cache, 1)
+ if reason is not None:
+ return _refusal(state, reason)
+ device = getattr(model, "device", None)
+ if (
+ isinstance(device, torch.device)
+ and device.type == ids.device.type
+ and device.index is not None
+ and ids.device.index is not None
+ and device.index != ids.device.index
+ ):
+ return _refusal(state, "inputs on another device")
+ index = ids.device.index
+ if index is not None and index != _BACKEND.current_device():
+ with _BACKEND.device(index):
+ return decode_step(model, forward, kwargs, eager_regions)
+
+ from ..kernels import utils as K
+
+ if state.recheck or state.optimizer_steps != K._OPTIMIZER_STEPS:
+ fingerprint = _fingerprint(model, state)
+ if fingerprint is None:
+ state.invalidate()
+ state.recheck = False
+ state.optimizer_steps = K._OPTIMIZER_STEPS
+ return _refusal(state, state.fp_reason or "not one CUDA device / adapters")
+ if fingerprint != state.fingerprint:
+ state.invalidate()
+ state.fingerprint = fingerprint
+ state.recheck = False
+ state.optimizer_steps = K._OPTIMIZER_STEPS
+ elif state.fingerprint is None:
+ return _refusal(state, state.fp_reason or "not one CUDA device / adapters")
+
+ kwargs = _host_embeds(
+ model,
+ _placeholder_embeds(model, _mrope_positions(model, _step_type_ids(kwargs, ids.shape[1]))),
+ )
+ leaves = []
+ spec = []
+ for name, value in kwargs.items():
+ try:
+ spec.append((name, _spec(value, leaves, cache)))
+ except _Refuse as error:
+ return _refusal(state, f"{error} ({name})")
+ spec = tuple(spec)
+ if not torch.is_inference_mode_enabled() and any(leaf.is_inference() for leaf in leaves):
+ # Their static buffers could not be refilled outside inference mode.
+ return _refusal(state, "inference tensor outside inference mode")
+ layout = tuple(
+ (type(layer).__name__, layer.max_cache_len, tuple(layer.keys.shape), layer.keys.dtype)
+ for layer in cache.layers
+ )
+ # Ambient autocast incl. cache flag (Dynamo guards on it); cached casts outlive end() in generate().
+ autocast = (G._autocast_state(), torch.is_autocast_cache_enabled())
+ key = (
+ spec,
+ layout,
+ _window_state(cache),
+ autocast,
+ torch.is_inference_mode_enabled(),
+ eager_regions is not None,
+ )
+ refused = state.refused.get(key)
+ if refused is not None:
+ return _refusal(state, refused)
+ # Graphs address this cache's tensors: a new cache re-captures; one reordered in place stays eager.
+ cache_sig = (id(cache),) + tuple(
+ (layer.keys.data_ptr(), layer.values.data_ptr()) for layer in cache.layers
+ )
+ if cache_sig != state.cache_sig:
+ moved = state.cache_sig is not None and state.cache_sig[0] == id(cache)
+ state.clear()
+ if moved or state.moved == id(cache):
+ state.moved = id(cache)
+ return _refusal(state, "cache moved")
+ state.cache_sig = cache_sig
+ regions = eager_regions() if eager_regions is not None else contextlib.nullcontext()
+ with regions:
+ entry = state.graphs.get(key)
+ if entry is None:
+ if state.call_stop is None and state.unreplayed >= _MAX_UNREPLAYED_CAPTURES:
+ state.call_stop = "graph key changes every step"
+ sig = state.cache_sig
+ state.clear()
+ state.cache_sig = sig
+ if state.call_stop is not None:
+ return _refusal(state, state.call_stop)
+ if _window_fills(cache, 1):
+ # A filling-key graph already captured replays this step correctly; a new one would not.
+ return _refusal(state, "sliding window fills in this step")
+ return _capture(state, key, forward, kwargs, leaves, cache)
+ for buffer, leaf in zip(entry.leaves, leaves):
+ buffer.copy_(_compact(leaf))
+ if not entry.validated or _validate_all():
+ return _validate(state, key, entry, forward, kwargs, cache)
+ _replay(state, entry)
+ _advance(cache, 1)
+ state.replays += 1
+ state.unreplayed = 0
+ return _map_outputs(entry.outputs, torch.Tensor.clone)
+
+
+def _replay(state, entry):
+ """Replays from this card's current stream. A split model's segments run on their own cards'
+ current streams: every card's stream waits on the others before (the eager work of the step
+ before) and after (outputs and cache rows are read on any of them)."""
+ if not state.others:
+ entry.graph.replay()
+ return
+ streams = [_BACKEND.current_stream()] + [_BACKEND.device_stream(d) for d in state.others]
+ for a in streams:
+ for b in streams:
+ if a is not b:
+ a.wait_stream(b)
+ entry.graph.replay()
+ for a in streams:
+ for b in streams:
+ if a is not b:
+ a.wait_stream(b)
+
+
+def _compact(leaf):
+ """`leaf` without its broadcast (stride 0) dimensions: one row of each."""
+ if 0 not in leaf.stride():
+ return leaf
+ return leaf[tuple(slice(0, 1) if st == 0 else slice(None) for st in leaf.stride())]
+
+
+def _empty_strided_like(leaf):
+ # empty_like makes a non-dense view (a sliced mask) dense: keep eager's strides exactly.
+ return torch.empty_strided(leaf.shape, leaf.stride(), dtype = leaf.dtype, device = leaf.device)
+
+
+def _buffer_like(leaf):
+ if leaf.is_inference():
+ with torch.inference_mode():
+ return _empty_strided_like(leaf)
+ with G._no_inference_tensors():
+ return _empty_strided_like(leaf)
+
+
+def _capture(state, key, forward, kwargs, leaves, cache):
+ """Runs this step eagerly (warm-up, its outputs are returned), then captures it."""
+ if len(state.graphs) >= G.MAX_GRAPHS:
+ sig = state.cache_sig
+ state.clear()
+ state.cache_sig = sig
+ try:
+ if state.stream is not None and state.stream.device.index != _BACKEND.current_device():
+ state.stream = None
+ state.pool = None
+ if state.pool is None:
+ state.pool = _BACKEND.pool()
+ state.mempools = {}
+ for d in state.others:
+ if d not in state.mempools:
+ state.mempools[d] = _BACKEND.mempool()
+ if d not in state.side:
+ state.side[d] = _BACKEND.side_stream(d)
+ side = {d: state.side[d] for d in state.others}
+ if state.stream is None:
+ state.stream = _BACKEND.new_stream()
+ stream = state.stream
+ # Static buffers match their argument (dtype, strides, inference-ness): same compiled variants as eager.
+ buffers = [_buffer_like(_compact(leaf)) for leaf in leaves]
+ for buffer, leaf in zip(buffers, leaves):
+ buffer.copy_(_compact(leaf))
+ it = iter([buffer.expand(leaf.shape) for buffer, leaf in zip(buffers, leaves)])
+ static_kwargs = {k: _build(v, it, cache) for k, v in kwargs.items()}
+ except Exception as error:
+ reason = f"capture setup failed: {type(error).__name__}"
+ state.pool = None
+ state.refused[key] = reason
+ state.refusals[reason] += 1
+ return forward(**kwargs)
+ before = _host_state(cache)
+ stream.wait_stream(_BACKEND.current_stream())
+ graph = None
+ try:
+ # A split model's other cards run on their side streams in the warm-up too (their cuBLAS
+ # workspaces exist before the capture).
+ with (
+ _BACKEND.stream(stream),
+ _BACKEND.on_streams(side) if side else contextlib.nullcontext(),
+ _graphable_rolls(),
+ ):
+ outputs = forward(**static_kwargs)
+ stream.synchronize()
+ reason = _host_state_reason(before, _host_state(cache), 1)
+ if reason is not None:
+ state.refused[key] = reason
+ state.refusals[reason] += 1
+ return outputs
+ after = _snapshot(cache, rows = False)
+ graph = False
+ try:
+ if side:
+ graph, static_outputs = _BACKEND.record(
+ state.pool, lambda: forward(**static_kwargs), cache, side, state.mempools
+ )
+ else:
+ graph, static_outputs = _BACKEND.record(
+ state.pool, lambda: forward(**static_kwargs), cache
+ )
+ finally:
+ # Capturing ran the Python update again: put the host counters back.
+ _restore((after[0], []))
+ except Exception as error:
+ if graph is None:
+ raise
+ if isinstance(error, torch.cuda.OutOfMemoryError):
+ # Out of memory while recording: free every graph of this call and decode the rest of it eagerly.
+ # Not refused for the process (the next call may have the room).
+ reason = "capture failed: out of memory"
+ if len(state.capture_errors) < 4:
+ state.capture_errors.append((reason, _where(error)))
+ sig = state.cache_sig
+ state.clear()
+ state.cache_sig = sig
+ state.call_stop = reason
+ state.refusals[reason] += 1
+ return outputs
+ message = " ".join(str(error).split())[:200]
+ reason = f"capture failed: {type(error).__name__}: {message}"
+ if len(state.capture_errors) < 4:
+ state.capture_errors.append((reason, _where(error)))
+ state.pool = None
+ state.refused[key] = reason
+ state.refusals[reason] += 1
+ if os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_STRICT", "0") == "1":
+ raise
+ return outputs
+ finally:
+ # Also after a failed capture: the eager steps must not race the side stream.
+ _BACKEND.current_stream().wait_stream(stream)
+ state.graphs[key] = _Entry(graph, buffers, static_outputs, key in state.validated)
+ state.captures += 1
+ state.unreplayed += 1
+ state.key_captures[hash(key)] += 1
+ return outputs
+
+
+def _validate(state, key, entry, forward, kwargs, cache):
+ """First replay of a key must match the eager step bitwise, else the key is refused. Eager runs
+ last, so generation continues from its result and cache rows."""
+ state.validations += 1
+ start = _snapshot(cache)
+ _replay(state, entry)
+ _restore(start)
+ eager = forward(**kwargs)
+ equal = _outputs_equal(eager, entry.outputs)
+ _record_check(state, eager, entry.outputs, equal)
+ if equal or entry.validated:
+ # VALIDATE_ALL only records: generation stays on the eager tokens, so steps share a prefix.
+ entry.validated = True
+ state.validated.add(key)
+ state.replays += 1
+ state.unreplayed = 0
+ return eager
+ reason = "validation mismatch"
+ state.refused[key] = reason
+ state.refusals[reason] += 1
+ state.graphs.pop(key, None)
+ if os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_STRICT", "0") == "1":
+ raise RuntimeError("Unsloth: decode CUDA graph replay differs from the eager step")
+ return eager
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 104efb17707..a592da20e6d 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -3123,6 +3123,7 @@ def unsloth_fast_generate(self, *args, **kwargs):
try:
with (
_decode_graphs.eager_scope() if plan == "eager" else contextlib.nullcontext(),
+ _decode_graphs.plan_scope("graph" if graphs else "eager"),
_get_inference_mode_context_manager(self),
torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),
):
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 9630ae83ea9..098f3822808 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -292,6 +292,7 @@ from ._custom_dtype import cast_mixed_linear_inputs, resolve_dtype, trusted_cust
from ._mamba_lora import patch_mamba_fused_lora
from .remote_code_shims import apply_remote_code_shims
from .grouped_linear_lora import register_grouped_linear_lora
+from . import _decode_graphs, _generic_decode_graphs
from .loader_utils import (
_bnb_bits_requested,
vlm_gradient_checkpointing_reentrant,
@@ -1504,6 +1505,9 @@ def _eager_decodes(model):
return _is_decode_compile_model(model) and "forward" not in model.__dict__
+_NO_FORWARD = object()
+
+
class _EagerDecodeSteps:
"""One generate() call. Each eager decode step (one new token per row) runs inside zoo's
`unsloth_eager_decode()`, so compiled regions call their eager originals; prefill and
@@ -1531,14 +1535,99 @@ class _EagerDecodeSteps:
def forward(*args, **kwargs):
return step(*args, **kwargs)
+ self._previous = self.model.__dict__.get("forward", _NO_FORWARD)
self.model.forward = forward
return self
def __exit__(self, *exc):
- del self.model.forward
+ if self._previous is _NO_FORWARD:
+ del self.model.forward
+ else:
+ self.model.forward = self._previous
return False
+try:
+ from unsloth_zoo.decode_graph_state import (
+ graph_generate as _zoo_graph_generate,
+ graph_decode_step as _zoo_graph_decode_step,
+ )
+except ImportError:
+ _zoo_graph_generate = _zoo_graph_decode_step = None
+
+
+def _one_token_step(kwargs):
+ ids = kwargs.get("input_ids")
+ if ids is None:
+ ids = kwargs.get("inputs_embeds")
+ return isinstance(ids, torch.Tensor) and ids.dim() >= 2 and ids.shape[1] == 1
+
+
+class _GraphDecodeSteps(_EagerDecodeSteps):
+ """One generate() call over a static cache with decode CUDA graphs on: each one-token decode
+ step replays a captured graph of the step the eager path runs (_generic_decode_graphs), else
+ runs as before (inside `unsloth_eager_decode()` for the models that compile their decode)."""
+
+ def __init__(self, model):
+ super().__init__(model)
+ self.eager_decodes = _eager_decodes(model)
+ # Diagnostic: =1 graphs the eager originals of compiled regions (not what eager decode runs).
+ self.eager_regions = self.eager_decodes or (
+ os.environ.get("UNSLOTH_DECODE_GENERIC_GRAPHS_EAGER_REGIONS", "0") == "1"
+ )
+ # Set by generate() after the call: keep the graphs and their cache for the next call.
+ self.keep = False
+ self.caches = _generic_decode_graphs.reuse_kept_cache(model)
+
+ def _forward(self, *args, **kwargs):
+ # zoo picks its graph-safe decode kernels (gpt-oss dense sink attention, routed MoE below sm80) for
+ # one-token steps in this scope: the eager warm-up, the capture and every eager step agree.
+ if _zoo_graph_decode_step is not None and _one_token_step(kwargs):
+ with _zoo_graph_decode_step():
+ return self._graph_forward(*args, **kwargs)
+ return self._graph_forward(*args, **kwargs)
+
+ def _graph_forward(self, *args, **kwargs):
+ if not args and not torch.compiler.is_compiling():
+ # Compiled regions stay compiled unless this model's eager decode leaves them: a
+ # different kernel mix rounds differently and greedy tokens drift after a few dozen steps.
+ out = _generic_decode_graphs.decode_step(
+ self.model,
+ self.forward,
+ kwargs,
+ unsloth_eager_decode if self.eager_regions else None,
+ )
+ if out is not None:
+ return out
+ if self.eager_decodes:
+ return super()._forward(*args, **kwargs)
+ return self.forward(*args, **kwargs)
+
+ def __enter__(self):
+ _generic_decode_graphs.begin(self.model)
+ self.caches.__enter__()
+ # Masks are built in prepare_inputs_for_generation, outside the forward: the call-wide scope covers them.
+ self._zoo_scope = _zoo_graph_generate() if _zoo_graph_generate is not None else None
+ if self._zoo_scope is not None:
+ self._zoo_scope.__enter__()
+ try:
+ return super().__enter__()
+ except BaseException:
+ if self._zoo_scope is not None:
+ self._zoo_scope.__exit__(None, None, None)
+ self.caches.__exit__(None, None, None)
+ raise
+
+ def __exit__(self, *exc):
+ try:
+ return super().__exit__(*exc)
+ finally:
+ if self._zoo_scope is not None:
+ self._zoo_scope.__exit__(None, None, None)
+ self.caches.__exit__(*exc)
+ _generic_decode_graphs.end(self.model, keep = self.keep and exc[0] is None)
+
+
# One-token decode is slower on flex than SDPA, but prefill must stay on flex (its block mask skips tiles a
# dense mask computes). The path is picked in prepare_inputs_for_generation, where the step's mask is built.
_FLEX_GENERATE_ENV_VAR = "UNSLOTH_FLEX_ATTENTION_GENERATE"
@@ -2155,6 +2244,9 @@ def _uses_flash_attention_for_generation(config):
def _clear_generation_caches(model):
+ # The StaticCache decode graphs keep for the next call stays. Guarded: tests exec this function alone.
+ graphs = globals().get("_generic_decode_graphs")
+ kept = graphs.kept_cache(model) if graphs is not None else None
for name, module in model.named_modules():
if hasattr(module, "_flex_attention_cache"):
try:
@@ -2162,6 +2254,8 @@ def _clear_generation_caches(model):
except:
pass
# Solves AttributeError: 'SlidingWindowLayer' object has no attribute 'max_batch_size'.
+ if kept is not None and module.__dict__.get("_cache") is kept:
+ continue
if hasattr(module, "_cache") and "cache_utils" in str(module._cache.__class__):
try:
del module._cache
@@ -2387,16 +2481,49 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
kwargs["compile_config"] = compile_config
decode_scope = _CompileDecodeOnRepeat(self) if compile_decode else contextlib.nullcontext()
- eager_scope = _EagerDecodeSteps(self) if _eager_decodes(self) else contextlib.nullcontext()
- try:
+
+ def _eager_scope():
+ return _EagerDecodeSteps(self) if _eager_decodes(self) else contextlib.nullcontext()
+
+ def _run(scope, plan):
with (
_NonFlexGenerate(self),
decode_scope,
- eager_scope,
+ scope,
+ _decode_graphs.plan_scope(plan),
_get_inference_mode_context_manager(self),
autocaster,
):
- output = self._old_generate(*args, **kwargs)
+ out = self._old_generate(*args, **kwargs)
+ if isinstance(scope, _GraphDecodeSteps):
+ scope.keep = _generic_decode_graphs.keepable(self, kwargs, out)
+ return out
+
+ try:
+ with _decode_graphs.generate_scope():
+ if cache_implementation == "static" and not force_dynamic_cache and not compile_decode:
+ plan = _generic_decode_graphs.plan(self, args, kwargs, input_ids)
+ else:
+ plan = _generic_decode_graphs.ineligible(self)
+ if plan != "graph":
+ output = _run(_eager_scope(), "eager")
+ else:
+ rng = _generic_decode_graphs.rng_state(input_ids.device)
+ out_of_memory = False
+ try:
+ output = _run(_GraphDecodeSteps(self), "graph")
+ except torch.cuda.OutOfMemoryError:
+ if not _generic_decode_graphs.replayable(kwargs):
+ raise
+ out_of_memory = True
+ if out_of_memory:
+ # Outside the handler: its traceback holds the failed call's frames (cache, graph
+ # buffers), which would stay allocated through the redo.
+ _generic_decode_graphs.redo_eagerly(self)
+ _clear_generation_caches(self)
+ _generic_decode_graphs.set_rng_state(rng)
+ with _decode_graphs.eager_scope():
+ output = _run(_eager_scope(), "eager")
finally:
_clear_generation_caches(self)
@@ -6075,6 +6202,7 @@ class FastBaseModel:
"Unsloth: I think you're passing a tokenizer, not the model to for_training!"
)
_tensor_parallel_for_training(model)
+ _decode_graphs.release_for_training(model)
# Mode last, as model.train() was: a custom train() keeps the final say over the chain below.
checkpointing, apply_training_mode = switch_training_mode(model, True, defer = True)
use_gradient_checkpointing = resolve_training_gradient_checkpointing(