Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: decode with CUDA graphs by default in the transformers chat backend (#15993)

Daniel Han9e3132637d86
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 281d3e8156e..dd6da42150d 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -695,6 +695,39 @@ class _StopSequenceStreamer:
abort()
+def _decode_graphs():
+ """Unsloth's CUDA-graph decode module, or None for an Unsloth without the Studio scope."""
+ try:
+ from unsloth.models import _decode_graphs as graphs
+ except Exception:
+ return None
+ return graphs if hasattr(graphs, "studio_generation") else None
+
+
+def _decode_graph_scope():
+ """Graphed decode on by default for this thread's chat generate(): this worker only serves
+ inference. Llama / Qwen3 on NVIDIA CUDA only; UNSLOTH_DECODE_CUDA_GRAPHS=0 still turns it off."""
+ graphs = _decode_graphs()
+ return graphs.studio_generation() if graphs is not None else contextlib.nullcontext()
+
+
+def _release_decode_graphs(model, generation_lock) -> None:
+ """Frees the model's decode graphs and decode buffers now rather than at garbage collection.
+ Skipped while a generate() still holds the lock: that thread's model reference frees them."""
+ graphs = _decode_graphs()
+ if graphs is None or not hasattr(model, "modules"):
+ return
+ if generation_lock is not None and not generation_lock.acquire(blocking = False):
+ return
+ try:
+ graphs.release_for_training(model)
+ except Exception as e:
+ logger.warning(f"Could not release decode graphs: {e}")
+ finally:
+ if generation_lock is not None:
+ generation_lock.release()
+
+
_CUSTOM_GENERATE_REJECTED_KWARGS = ("logits_processor", "stopping_criteria")
@@ -1383,7 +1416,10 @@ class InferenceBackend:
self._audio_codec_manager.unload()
logger.info(f"Unloading model '{model_name}' from memory.")
+ model = self.models[model_name].get("model")
del self.models[model_name]
+ _release_decode_graphs(model, getattr(self, "_generation_lock", None))
+ model = None
if self.active_model_name == model_name:
self.active_model_name = None
@@ -2945,9 +2981,10 @@ class InferenceBackend:
# Started after the adapter swap so only model.generate() is timed.
timer.start()
# Only the returned sequences carry an exact token count.
- gen_outputs["sequences"] = model.generate(
- **_supported_generate_kwargs(model, generation_kwargs)
- )
+ with _decode_graph_scope():
+ gen_outputs["sequences"] = model.generate(
+ **_supported_generate_kwargs(model, generation_kwargs)
+ )
except Exception as e:
err["msg"] = str(e)
stop_streamer.abort()
diff --git a/studio/backend/tests/test_transformers_decode_graphs.py b/studio/backend/tests/test_transformers_decode_graphs.py
new file mode 100644
index 00000000000..0ff105f83ad
--- /dev/null
+++ b/studio/backend/tests/test_transformers_decode_graphs.py
@@ -0,0 +1,204 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Studio chat generate() graphs decode by default; unload frees graphs. _decode_graphs.py is
+loaded by file path since importing unsloth needs an accelerator."""
+
+import importlib
+import importlib.util
+import sys
+import threading
+import types
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+_BACKEND_DIR = Path(__file__).resolve().parent.parent
+_DECODE_GRAPHS = _BACKEND_DIR.parent.parent / "unsloth" / "models" / "_decode_graphs.py"
+if str(_BACKEND_DIR) not in sys.path:
+ sys.path.insert(0, str(_BACKEND_DIR))
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(name, attrs):
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001
+ pass
+ _STUBBED.append(name)
+ mod = types.ModuleType(name)
+ mod.__spec__ = None
+ for attr in attrs:
+ setattr(mod, attr, MagicMock())
+ sys.modules[name] = mod
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, mod)
+
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
+
+try:
+ import core.inference.inference # noqa: E402,F401
+except ImportError:
+ pass
+
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+ENV = "UNSLOTH_DECODE_CUDA_GRAPHS"
+
+
+def graphs(monkeypatch):
+ """Real _decode_graphs module with the default decision on a spoofed NVIDIA CUDA host."""
+ spec = importlib.util.spec_from_file_location("_studio_decode_graphs", _DECODE_GRAPHS)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ module.ENABLED = None
+ module._DEFAULT_OK = True
+ module._STUDIO_DEFAULT_OK = True # Linux default (native Windows stays eager)
+ unsloth = types.ModuleType("unsloth")
+ models = types.ModuleType("unsloth.models")
+ unsloth.models = models
+ models._decode_graphs = module
+ monkeypatch.setitem(sys.modules, "unsloth", unsloth)
+ monkeypatch.setitem(sys.modules, "unsloth.models", models)
+ monkeypatch.setitem(sys.modules, "unsloth.models._decode_graphs", module)
+ monkeypatch.delenv(ENV, raising = False)
+ return module
+
+
+class _Batch(dict):
+ def to(self, *_args, **_kwargs):
+ return self
+
+
+class _Tokenizer:
+ all_special_tokens: list = []
+ eos_token_id = 1
+ pad_token_id = None
+ pieces = {2: "Hello", 3: " world"}
+
+ def __call__(self, *_args, **_kwargs):
+ return _Batch({"input_ids": torch.zeros((1, 1), dtype = torch.long)})
+
+ def decode(self, ids, **kwargs):
+ return "".join(self.pieces.get(int(i), "") for i in ids)
+
+
+class _Model:
+ device = "cpu"
+ generation_config = type("Cfg", (), {"eos_token_id": 1})()
+ config = generation_config
+
+ def __init__(self, graphs):
+ self.graphs = graphs
+ self.seen = []
+
+ def generate(self, streamer, max_new_tokens, **_kwargs):
+ self.seen.append(self.graphs.enabled())
+ ids = torch.tensor([[0, 2, 3]])
+ streamer.put(ids[:, :1])
+ streamer.put(torch.tensor([2]))
+ streamer.put(torch.tensor([3]))
+ streamer.end()
+ return ids
+
+
+def _backend(graphs):
+ inf = pytest.importorskip("core.inference.inference")
+ model = _Model(graphs)
+ tokenizer = _Tokenizer()
+ streamer = inf.TextIteratorStreamer(tokenizer, skip_prompt = True, timeout = 0.2)
+ backend = inf.InferenceBackend.__new__(inf.InferenceBackend)
+ backend.active_model_name = "m"
+ backend._generation_lock = threading.Lock()
+ backend.models = {"m": {"model": model, "tokenizer": tokenizer}}
+ backend._make_text_streamer = lambda *_args, **_kwargs: streamer
+ return backend, model
+
+
+def _chat(backend):
+ return list(backend.generate_stream("PROMPT", temperature = 0.0, max_new_tokens = 4))
+
+
+def test_chat_generate_runs_with_graphs_on_by_default(graphs):
+ backend, model = _backend(graphs)
+ assert _chat(backend)[-1] == "Hello world"
+ assert model.seen == [True]
+ # Scoped to the generate thread: this thread, and Core's plain generate(), stay eager.
+ assert graphs.enabled() is False
+
+
+def test_opt_out_variable_still_disables_graphs(graphs, monkeypatch):
+ monkeypatch.setenv(ENV, "0")
+ backend, model = _backend(graphs)
+ _chat(backend)
+ assert model.seen == [False]
+
+
+def test_non_nvidia_hosts_stay_eager(graphs, monkeypatch):
+ # ROCm / XPU / MPS / CPU: _default_ok() is False there.
+ monkeypatch.setattr(graphs, "_DEFAULT_OK", False)
+ backend, model = _backend(graphs)
+ _chat(backend)
+ assert model.seen == [False]
+
+
+def test_an_unsloth_without_the_scope_keeps_generate_unchanged(graphs, monkeypatch):
+ monkeypatch.delattr(graphs, "studio_generation", raising = False)
+ backend, model = _backend(graphs)
+ assert _chat(backend)[-1] == "Hello world"
+ assert model.seen == [False]
+
+
+def _graphed_model(graphs):
+ model = torch.nn.Module()
+ attn = torch.nn.Module()
+ model.attn = attn
+ graphs._state(model).graphs[(1, "cuda")] = object()
+ attn._unsloth_graph_buffers = {(1, torch.float16, "cuda"): ()}
+ attn.paged_attention = torch.zeros(1)
+ attn.attention = torch.zeros(1)
+ return model, attn
+
+
+def _unload(backend, monkeypatch):
+ inf = sys.modules["core.inference.inference"]
+ monkeypatch.setattr(inf, "clear_gpu_cache", lambda: None)
+ cleanup = pytest.importorskip("utils.cache_cleanup")
+ monkeypatch.setattr(cleanup, "clear_unsloth_compiled_cache", lambda **_kw: None)
+ return backend.unload_model("m")
+
+
+def test_unload_frees_decode_graphs_and_buffers(graphs, monkeypatch):
+ backend, _ = _backend(graphs)
+ model, attn = _graphed_model(graphs)
+ backend.models["m"]["model"] = model
+ assert _unload(backend, monkeypatch) is True
+ assert graphs.stats(model)["graphs"] == 0
+ assert "_unsloth_graph_buffers" not in attn.__dict__
+ assert "paged_attention" not in attn.__dict__ and "attention" not in attn.__dict__
+ assert backend.models == {}
+
+
+def test_unload_leaves_a_running_generation_alone(graphs, monkeypatch):
+ backend, _ = _backend(graphs)
+ model, attn = _graphed_model(graphs)
+ backend.models["m"]["model"] = model
+ with backend._generation_lock:
+ assert _unload(backend, monkeypatch) is True
+ # The generate() thread still owns the buffers; they go with its model reference.
+ assert "_unsloth_graph_buffers" in attn.__dict__
+ assert graphs.stats(model)["graphs"] == 1
+ assert backend.models == {}
diff --git a/tests/test_decode_cuda_graphs.py b/tests/test_decode_cuda_graphs.py
index 96fea0db791..b898bfc67ac 100644
--- a/tests/test_decode_cuda_graphs.py
+++ b/tests/test_decode_cuda_graphs.py
@@ -318,6 +318,58 @@ def test_release_drops_graphs_and_buffers(monkeypatch):
)
+@cuda
+def test_ineligible_generation_frees_graphs_and_recovers(monkeypatch):
+ """Compare mode: a generation the graphs cannot serve (here a DoRA adapter, faked) frees the
+ captured steps, and the next eligible generation is graphed again instead of eager for good."""
+ model, tok = _load(MODELS[0], False, lora = True)
+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")
+ causal = _causal(model)
+ eager = _generate(model, tok, enc, False, monkeypatch)
+ # _plain returns no cache, so generate() keeps the captured steps (a returned one releases them).
+ monkeypatch.setattr(G, "ENABLED", True)
+ _plain(model, tok, enc)
+ assert G.stats(causal)["graphs"] >= 1
+ plain = kernel_utils._has_active_dora_adapter
+ monkeypatch.setattr(kernel_utils, "_has_active_dora_adapter", lambda proj: True)
+ replays = G.stats(causal)["replays"]
+ _plain(model, tok, enc)
+ stats = G.stats(causal)
+ assert stats["graphs"] == 0 and stats["replays"] == replays, stats
+ assert stats["disabled"] is None, stats
+ monkeypatch.setattr(kernel_utils, "_has_active_dora_adapter", plain)
+ graph = _generate(model, tok, enc, True, monkeypatch)
+ assert G.stats(causal)["replays"] > replays
+ assert torch.equal(graph[1], eager[1])
+
+
+@cuda
+def test_failed_capture_joins_the_side_stream(monkeypatch):
+ """A capture that fails in its warmup still makes the caller's stream wait for the side stream
+ before the eager fallback runs."""
+ model, tok = _load(MODELS[0], False)
+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")
+ joins = []
+ wait = torch.cuda.Stream.wait_stream
+
+ def spy(self, other):
+ joins.append((self, other))
+ return wait(self, other)
+
+ def fail(*args, **kwargs):
+ raise RuntimeError("injected warmup failure")
+
+ monkeypatch.setattr(torch.cuda.Stream, "wait_stream", spy)
+ monkeypatch.setattr(G, "_run_step", fail)
+ eager = _generate(model, tok, enc, False, monkeypatch)
+ graph = _generate(model, tok, enc, True, monkeypatch)
+ side = G._state(_causal(model)).stream
+ assert side is not None and any(
+ other is side and self is not side for self, other in joins
+ ), joins
+ assert torch.equal(eager[1], graph[1])
+
+
@cuda
def test_steps_counter_present():
# The adapter-change check keys on this counter (bumped by every optimizer step).
diff --git a/tests/test_decode_cuda_graphs_studio.py b/tests/test_decode_cuda_graphs_studio.py
new file mode 100644
index 00000000000..21d272721c7
--- /dev/null
+++ b/tests/test_decode_cuda_graphs_studio.py
@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""studio_generation() turns graphed decode on per thread on NVIDIA CUDA; =0 wins, others stay eager.
+Loads _decode_graphs.py by file path (importing unsloth needs an accelerator)."""
+
+import importlib.util
+import threading
+from pathlib import Path
+
+import pytest
+import torch
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+
+ENV = "UNSLOTH_DECODE_CUDA_GRAPHS"
+_SOURCE = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_decode_graphs.py"
+
+
+def G(monkeypatch):
+ spec = importlib.util.spec_from_file_location("_unsloth_decode_graphs_studio", _SOURCE)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ module.ENABLED = None
+ module._STUDIO_DEFAULT_OK = True # the Linux default; the Windows test overrides it
+ monkeypatch.delenv(ENV, raising = False)
+ return module
+
+
+def test_studio_scope_turns_graphs_on_for_nvidia_cuda(G, monkeypatch):
+ monkeypatch.setattr(G, "_DEFAULT_OK", True)
+ assert G.enabled() is False
+ with G.studio_generation():
+ assert G.enabled() is True
+ with G.rl_generation():
+ assert G.enabled() is True
+ assert G.enabled() is True
+ assert G.enabled() is False
+
+
+def test_studio_scope_honours_the_variable(G, monkeypatch):
+ monkeypatch.setattr(G, "_DEFAULT_OK", True)
+ monkeypatch.setenv(ENV, "0")
+ with G.studio_generation():
+ assert G.enabled() is False
+ monkeypatch.setattr(G, "ENABLED", False)
+ monkeypatch.setenv(ENV, "1")
+ with G.studio_generation():
+ assert G.enabled() is False
+
+
+def test_studio_scope_stays_eager_off_nvidia(G, monkeypatch):
+ # ROCm (HIP), XPU, MPS and CPU hosts: _default_ok() is False.
+ monkeypatch.setattr(G, "_DEFAULT_OK", False)
+ with G.studio_generation():
+ assert G.enabled() is False
+ monkeypatch.setattr(torch.version, "hip", "6.4.0", raising = False)
+ assert G._default_ok() is False
+
+
+def test_studio_scope_is_per_thread(G, monkeypatch):
+ monkeypatch.setattr(G, "_DEFAULT_OK", True)
+ seen = []
+ with G.studio_generation():
+ t = threading.Thread(target = lambda: seen.append(G.enabled()))
+ t.start()
+ t.join()
+ assert G.enabled() is True
+ assert seen == [False]
+
+
+def test_studio_scope_unwinds_on_error(G, monkeypatch):
+ monkeypatch.setattr(G, "_DEFAULT_OK", True)
+ with pytest.raises(RuntimeError):
+ with G.studio_generation():
+ raise RuntimeError("generate failed")
+ assert G.enabled() is False
+
+
+def test_studio_scope_is_off_on_native_windows(G, monkeypatch):
+ # Native Windows CUDA capture is unverified: the Studio default stays eager there, =1 still forces.
+ monkeypatch.setattr(G, "_DEFAULT_OK", True)
+ monkeypatch.setattr(G, "_STUDIO_DEFAULT_OK", False)
+ with G.studio_generation():
+ assert G.enabled() is False
+ with G.rl_generation():
+ assert G.enabled() is True
+ assert G.enabled() is False
+ assert G.enabled() is False
+ monkeypatch.setenv(ENV, "1")
+ with G.studio_generation():
+ assert G.enabled() is True
+
+
[email protected](not has_real_cuda(), reason = "CUDA only")
+def test_studio_generate_graphed_plain_generate_eager():
+ import unsloth # noqa: F401
+ from unsloth import FastLanguageModel
+ import unsloth.models._decode_graphs as G
+
+ if not G._DEFAULT_OK:
+ pytest.skip("the default is NVIDIA CUDA only")
+ if not G._STUDIO_DEFAULT_OK:
+ pytest.skip("the Studio default stays eager on native Windows")
+ model, tok = FastLanguageModel.from_pretrained(
+ "trl-internal-testing/tiny-Qwen3ForCausalLM", max_seq_length = 2048, load_in_4bit = False
+ )
+ FastLanguageModel.for_inference(model)
+ enc = tok(["Once upon a time there was"], return_tensors = "pt").to("cuda")
+
+ def gen():
+ out = model.generate(
+ **enc, max_new_tokens = 24, min_new_tokens = 24, do_sample = False,
+ output_logits = True, return_dict_in_generate = True,
+ ) # fmt: skip
+ return out.sequences.cpu(), torch.stack(out.logits).cpu()
+
+ def replays():
+ return G.stats(model).get("replays", 0)
+
+ eager = gen()
+ assert replays() == 0
+ with G.studio_generation():
+ graphed = gen()
+ assert replays() == 24 - 2 and G.stats(model)["disabled"] is None
+ assert torch.equal(eager[0], graphed[0]) and torch.equal(eager[1], graphed[1])
+ assert torch.equal(gen()[1], eager[1]) and replays() == 24 - 2
+ G.release_for_training(model)
+ assert G.stats(model)["graphs"] == 0
+ assert not any("paged_attention" in m.__dict__ for m in model.modules())
diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.py
index ab16312b398..4a837fac823 100644
--- a/unsloth/models/_decode_graphs.py
+++ b/unsloth/models/_decode_graphs.py
@@ -2,7 +2,8 @@
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
"""CUDA-graph decode; UNSLOTH_DECODE_CUDA_GRAPHS=1 forces, =0 disables; unset, only RL-trainer
-generate on NVIDIA CUDA; graphs dropped when params, LoRA, quant or RoPE change.
+generate and Studio's chat worker on NVIDIA CUDA; graphs dropped when params, LoRA, quant or RoPE
+change.
UNSLOTH_DECODE_CUDA_GRAPHS_FULL=1 also captures the attention cores: each step is then ONE graph that
attends over a fixed span of cache rows (the next power of two >= the KV length, at least BUCKET, at
@@ -13,12 +14,13 @@ batch 1 by default (UNSLOTH_DECODE_FULL_GRAPH_MAX_BSZ): larger batches keep the
import math
import os
+import sys
import threading
import torch
# Programmatic override (tests, harnesses): True / False force graphs on / off, None (default) follows
-# the environment variable read at call time and the RL generation scope (enabled()).
+# the environment variable read at call time and the RL / Studio generation scopes (enabled()).
ENABLED = None
# Whole-step graphs (attention captured too, over a power-of-two span of cache rows).
FULL = os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_FULL", "0") == "1"
@@ -36,6 +38,7 @@ __all__ = [
"ENABLED",
"enabled",
"rl_generation",
+ "studio_generation",
"static",
"alloc_decode_buffers",
"decode_step",
@@ -105,6 +108,23 @@ class rl_generation:
return False
+# Native Windows CUDA capture is unverified: Studio default off there (=1 still forces).
+_STUDIO_DEFAULT_OK = sys.platform != "win32"
+
+
+class studio_generation(rl_generation):
+ """Per-thread scope for Studio's chat generate(): graphs default on, freed on unload (release())."""
+
+ __slots__ = ("_entered",)
+
+ def __enter__(self):
+ self._entered = _STUDIO_DEFAULT_OK
+ return super().__enter__() if self._entered else self
+
+ def __exit__(self, *exc):
+ return super().__exit__(*exc) if self._entered else False
+
+
def static():
"""The _StepStatics of a graph-mode step running on this thread, else None (eager)."""
if torch.compiler.is_compiling():
@@ -297,6 +317,7 @@ class _ModelGraphs:
self.optimizer_steps = -1
self.buffer_epoch = -1
self.disabled = None
+ self.ineligible_epoch = None
self.rope_len = None
self.captures = 0
self.replays = 0
@@ -625,6 +646,9 @@ def decode_step(
state.eager_steps += 1
state.generation_epoch = -1
return None
+ if state.ineligible_epoch == GENERATION_EPOCH:
+ state.eager_steps += 1
+ return None
if (
state.generation_epoch != GENERATION_EPOCH
or state.optimizer_steps != K._OPTIMIZER_STEPS
@@ -637,7 +661,11 @@ def decode_step(
return None
fingerprint = _fingerprint(model, layers, rope_len)
if fingerprint is None:
- state.disabled = "parameters on several / non-CUDA devices or a non-LoRA adapter"
+ # Ineligible for this generation only (several / non-CUDA devices, or a DoRA / non-LoRA
+ # adapter, e.g. one side of a compare): free the captured steps and re-check next time.
+ state.clear()
+ state.ineligible_epoch = GENERATION_EPOCH
+ state.eager_steps += 1
return None
if (
fingerprint != state.fingerprint
@@ -754,43 +782,42 @@ def _capture(
no_cache = lambda: torch.autocast("cuda", dtype = dtype, enabled = enabled, cache_enabled = False)
stream.wait_stream(torch.cuda.current_stream())
step = _Step(st)
- with torch.cuda.stream(stream), no_cache():
- # Warmup: Triton JIT, cuBLAS workspaces, lazily built RoPE rows / LoRA casts. It writes this
- # step's K / V into its cache slot, the same values the replay writes again.
- _run_step(model, fast_forward_inference, logits_fn, s, st, past_key_values)
- stream.synchronize()
-
- def split(fn):
- # Close the segment, run the attention core once (its output address is what the next
- # segment reads), open the next segment.
- _end(step.segments[-1])
- # Two calls with the first result alive: the same address means the core writes into a
- # persistent buffer (bsz == 1 writes the query buffer); else (SDPA allocates) its
- # result is copied into a fixed buffer.
- out = fn()
- fixed = out if fn().data_ptr() == out.data_ptr() else torch.empty_like(out)
- step.cores.append(fn if fixed is out else (lambda: fixed.copy_(fn())))
+ try:
+ with torch.cuda.stream(stream), no_cache():
+ # Warmup (JIT, workspaces, lazy RoPE / LoRA casts); its K / V write equals the replay's.
+ _run_step(model, fast_forward_inference, logits_fn, s, st, past_key_values)
+ stream.synchronize()
+
+ def split(fn):
+ # The next segment reads the attention core's output address.
+ _end(step.segments[-1])
+ # Same address on two calls = persistent output buffer; else (SDPA) copy to a fixed one.
+ out = fn()
+ fixed = out if fn().data_ptr() == out.data_ptr() else torch.empty_like(out)
+ step.cores.append(fn if fixed is out else (lambda: fixed.copy_(fn())))
+ graph = torch.cuda.CUDAGraph()
+ step.segments.append(graph)
+ _begin(graph, state.pool)
+ return fixed
+
graph = torch.cuda.CUDAGraph()
step.segments.append(graph)
+ if span is None:
+ st.attention = split
_begin(graph, state.pool)
- return fixed
-
- graph = torch.cuda.CUDAGraph()
- step.segments.append(graph)
- if span is None:
- st.attention = split
- _begin(graph, state.pool)
- try:
- step.logits = _run_step(
- model, fast_forward_inference, logits_fn, s, st, past_key_values
- )
- finally:
- st.attention = _run_now
- if torch.cuda.is_current_stream_capturing():
- _end(step.segments[-1])
- torch.cuda.current_stream().wait_stream(stream)
- st.mask = None
- return step
+ try:
+ step.logits = _run_step(
+ model, fast_forward_inference, logits_fn, s, st, past_key_values
+ )
+ finally:
+ st.attention = _run_now
+ if torch.cuda.is_current_stream_capturing():
+ _end(step.segments[-1])
+ return step
+ finally:
+ # Also after a failed warmup / capture: the eager fallback must not race the side stream.
+ torch.cuda.current_stream().wait_stream(stream)
+ st.mask = None
def _begin(graph, pool):