Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: decode with CUDA graphs by default in the transformers chat backend (#15993)
Daniel Han9e3132637d86
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 281d3e8156e..dd6da42150d 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -695,6 +695,39 @@ class _StopSequenceStreamer:abort()+def _decode_graphs():+ """Unsloth's CUDA-graph decode module, or None for an Unsloth without the Studio scope."""+ try:+ from unsloth.models import _decode_graphs as graphs+ except Exception:+ return None+ return graphs if hasattr(graphs, "studio_generation") else None+++def _decode_graph_scope():+ """Graphed decode on by default for this thread's chat generate(): this worker only serves+ inference. Llama / Qwen3 on NVIDIA CUDA only; UNSLOTH_DECODE_CUDA_GRAPHS=0 still turns it off."""+ graphs = _decode_graphs()+ return graphs.studio_generation() if graphs is not None else contextlib.nullcontext()+++def _release_decode_graphs(model, generation_lock) -> None:+ """Frees the model's decode graphs and decode buffers now rather than at garbage collection.+ Skipped while a generate() still holds the lock: that thread's model reference frees them."""+ graphs = _decode_graphs()+ if graphs is None or not hasattr(model, "modules"):+ return+ if generation_lock is not None and not generation_lock.acquire(blocking = False):+ return+ try:+ graphs.release_for_training(model)+ except Exception as e:+ logger.warning(f"Could not release decode graphs: {e}")+ finally:+ if generation_lock is not None:+ generation_lock.release()++_CUSTOM_GENERATE_REJECTED_KWARGS = ("logits_processor", "stopping_criteria")@@ -1383,7 +1416,10 @@ class InferenceBackend:self._audio_codec_manager.unload()logger.info(f"Unloading model '{model_name}' from memory.")+ model = self.models[model_name].get("model")del self.models[model_name]+ _release_decode_graphs(model, getattr(self, "_generation_lock", None))+ model = Noneif self.active_model_name == model_name:self.active_model_name = None@@ -2945,9 +2981,10 @@ class InferenceBackend:# Started after the adapter swap so only model.generate() is timed.timer.start()# Only the returned sequences carry an exact token count.- gen_outputs["sequences"] = model.generate(- **_supported_generate_kwargs(model, generation_kwargs)- )+ with _decode_graph_scope():+ gen_outputs["sequences"] = model.generate(+ **_supported_generate_kwargs(model, generation_kwargs)+ )except Exception as e:err["msg"] = str(e)stop_streamer.abort()diff --git a/studio/backend/tests/test_transformers_decode_graphs.py b/studio/backend/tests/test_transformers_decode_graphs.pynew file mode 100644index 00000000000..0ff105f83ad--- /dev/null+++ b/studio/backend/tests/test_transformers_decode_graphs.py@@ -0,0 +1,204 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Studio chat generate() graphs decode by default; unload frees graphs. _decode_graphs.py is+loaded by file path since importing unsloth needs an accelerator."""++import importlib+import importlib.util+import sys+import threading+import types+from pathlib import Path+from unittest.mock import MagicMock++import pytest++torch = pytest.importorskip("torch")++_BACKEND_DIR = Path(__file__).resolve().parent.parent+_DECODE_GRAPHS = _BACKEND_DIR.parent.parent / "unsloth" / "models" / "_decode_graphs.py"+if str(_BACKEND_DIR) not in sys.path:+ sys.path.insert(0, str(_BACKEND_DIR))++_STUBBED: list[str] = []+++def _stub_if_missing(name, attrs):+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001+ pass+ _STUBBED.append(name)+ mod = types.ModuleType(name)+ mod.__spec__ = None+ for attr in attrs:+ setattr(mod, attr, MagicMock())+ sys.modules[name] = mod+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, mod)+++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))++try:+ import core.inference.inference # noqa: E402,F401+except ImportError:+ pass++for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)++ENV = "UNSLOTH_DECODE_CUDA_GRAPHS"+++def graphs(monkeypatch):+ """Real _decode_graphs module with the default decision on a spoofed NVIDIA CUDA host."""+ spec = importlib.util.spec_from_file_location("_studio_decode_graphs", _DECODE_GRAPHS)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ module.ENABLED = None+ module._DEFAULT_OK = True+ module._STUDIO_DEFAULT_OK = True # Linux default (native Windows stays eager)+ unsloth = types.ModuleType("unsloth")+ models = types.ModuleType("unsloth.models")+ unsloth.models = models+ models._decode_graphs = module+ monkeypatch.setitem(sys.modules, "unsloth", unsloth)+ monkeypatch.setitem(sys.modules, "unsloth.models", models)+ monkeypatch.setitem(sys.modules, "unsloth.models._decode_graphs", module)+ monkeypatch.delenv(ENV, raising = False)+ return module+++class _Batch(dict):+ def to(self, *_args, **_kwargs):+ return self+++class _Tokenizer:+ all_special_tokens: list = []+ eos_token_id = 1+ pad_token_id = None+ pieces = {2: "Hello", 3: " world"}++ def __call__(self, *_args, **_kwargs):+ return _Batch({"input_ids": torch.zeros((1, 1), dtype = torch.long)})++ def decode(self, ids, **kwargs):+ return "".join(self.pieces.get(int(i), "") for i in ids)+++class _Model:+ device = "cpu"+ generation_config = type("Cfg", (), {"eos_token_id": 1})()+ config = generation_config++ def __init__(self, graphs):+ self.graphs = graphs+ self.seen = []++ def generate(self, streamer, max_new_tokens, **_kwargs):+ self.seen.append(self.graphs.enabled())+ ids = torch.tensor([[0, 2, 3]])+ streamer.put(ids[:, :1])+ streamer.put(torch.tensor([2]))+ streamer.put(torch.tensor([3]))+ streamer.end()+ return ids+++def _backend(graphs):+ inf = pytest.importorskip("core.inference.inference")+ model = _Model(graphs)+ tokenizer = _Tokenizer()+ streamer = inf.TextIteratorStreamer(tokenizer, skip_prompt = True, timeout = 0.2)+ backend = inf.InferenceBackend.__new__(inf.InferenceBackend)+ backend.active_model_name = "m"+ backend._generation_lock = threading.Lock()+ backend.models = {"m": {"model": model, "tokenizer": tokenizer}}+ backend._make_text_streamer = lambda *_args, **_kwargs: streamer+ return backend, model+++def _chat(backend):+ return list(backend.generate_stream("PROMPT", temperature = 0.0, max_new_tokens = 4))+++def test_chat_generate_runs_with_graphs_on_by_default(graphs):+ backend, model = _backend(graphs)+ assert _chat(backend)[-1] == "Hello world"+ assert model.seen == [True]+ # Scoped to the generate thread: this thread, and Core's plain generate(), stay eager.+ assert graphs.enabled() is False+++def test_opt_out_variable_still_disables_graphs(graphs, monkeypatch):+ monkeypatch.setenv(ENV, "0")+ backend, model = _backend(graphs)+ _chat(backend)+ assert model.seen == [False]+++def test_non_nvidia_hosts_stay_eager(graphs, monkeypatch):+ # ROCm / XPU / MPS / CPU: _default_ok() is False there.+ monkeypatch.setattr(graphs, "_DEFAULT_OK", False)+ backend, model = _backend(graphs)+ _chat(backend)+ assert model.seen == [False]+++def test_an_unsloth_without_the_scope_keeps_generate_unchanged(graphs, monkeypatch):+ monkeypatch.delattr(graphs, "studio_generation", raising = False)+ backend, model = _backend(graphs)+ assert _chat(backend)[-1] == "Hello world"+ assert model.seen == [False]+++def _graphed_model(graphs):+ model = torch.nn.Module()+ attn = torch.nn.Module()+ model.attn = attn+ graphs._state(model).graphs[(1, "cuda")] = object()+ attn._unsloth_graph_buffers = {(1, torch.float16, "cuda"): ()}+ attn.paged_attention = torch.zeros(1)+ attn.attention = torch.zeros(1)+ return model, attn+++def _unload(backend, monkeypatch):+ inf = sys.modules["core.inference.inference"]+ monkeypatch.setattr(inf, "clear_gpu_cache", lambda: None)+ cleanup = pytest.importorskip("utils.cache_cleanup")+ monkeypatch.setattr(cleanup, "clear_unsloth_compiled_cache", lambda **_kw: None)+ return backend.unload_model("m")+++def test_unload_frees_decode_graphs_and_buffers(graphs, monkeypatch):+ backend, _ = _backend(graphs)+ model, attn = _graphed_model(graphs)+ backend.models["m"]["model"] = model+ assert _unload(backend, monkeypatch) is True+ assert graphs.stats(model)["graphs"] == 0+ assert "_unsloth_graph_buffers" not in attn.__dict__+ assert "paged_attention" not in attn.__dict__ and "attention" not in attn.__dict__+ assert backend.models == {}+++def test_unload_leaves_a_running_generation_alone(graphs, monkeypatch):+ backend, _ = _backend(graphs)+ model, attn = _graphed_model(graphs)+ backend.models["m"]["model"] = model+ with backend._generation_lock:+ assert _unload(backend, monkeypatch) is True+ # The generate() thread still owns the buffers; they go with its model reference.+ assert "_unsloth_graph_buffers" in attn.__dict__+ assert graphs.stats(model)["graphs"] == 1+ assert backend.models == {}diff --git a/tests/test_decode_cuda_graphs.py b/tests/test_decode_cuda_graphs.pyindex 96fea0db791..b898bfc67ac 100644--- a/tests/test_decode_cuda_graphs.py+++ b/tests/test_decode_cuda_graphs.py@@ -318,6 +318,58 @@ def test_release_drops_graphs_and_buffers(monkeypatch):)+@cuda+def test_ineligible_generation_frees_graphs_and_recovers(monkeypatch):+ """Compare mode: a generation the graphs cannot serve (here a DoRA adapter, faked) frees the+ captured steps, and the next eligible generation is graphed again instead of eager for good."""+ model, tok = _load(MODELS[0], False, lora = True)+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")+ causal = _causal(model)+ eager = _generate(model, tok, enc, False, monkeypatch)+ # _plain returns no cache, so generate() keeps the captured steps (a returned one releases them).+ monkeypatch.setattr(G, "ENABLED", True)+ _plain(model, tok, enc)+ assert G.stats(causal)["graphs"] >= 1+ plain = kernel_utils._has_active_dora_adapter+ monkeypatch.setattr(kernel_utils, "_has_active_dora_adapter", lambda proj: True)+ replays = G.stats(causal)["replays"]+ _plain(model, tok, enc)+ stats = G.stats(causal)+ assert stats["graphs"] == 0 and stats["replays"] == replays, stats+ assert stats["disabled"] is None, stats+ monkeypatch.setattr(kernel_utils, "_has_active_dora_adapter", plain)+ graph = _generate(model, tok, enc, True, monkeypatch)+ assert G.stats(causal)["replays"] > replays+ assert torch.equal(graph[1], eager[1])+++@cuda+def test_failed_capture_joins_the_side_stream(monkeypatch):+ """A capture that fails in its warmup still makes the caller's stream wait for the side stream+ before the eager fallback runs."""+ model, tok = _load(MODELS[0], False)+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")+ joins = []+ wait = torch.cuda.Stream.wait_stream++ def spy(self, other):+ joins.append((self, other))+ return wait(self, other)++ def fail(*args, **kwargs):+ raise RuntimeError("injected warmup failure")++ monkeypatch.setattr(torch.cuda.Stream, "wait_stream", spy)+ monkeypatch.setattr(G, "_run_step", fail)+ eager = _generate(model, tok, enc, False, monkeypatch)+ graph = _generate(model, tok, enc, True, monkeypatch)+ side = G._state(_causal(model)).stream+ assert side is not None and any(+ other is side and self is not side for self, other in joins+ ), joins+ assert torch.equal(eager[1], graph[1])++@cudadef test_steps_counter_present():# The adapter-change check keys on this counter (bumped by every optimizer step).diff --git a/tests/test_decode_cuda_graphs_studio.py b/tests/test_decode_cuda_graphs_studio.pynew file mode 100644index 00000000000..21d272721c7--- /dev/null+++ b/tests/test_decode_cuda_graphs_studio.py@@ -0,0 +1,130 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""studio_generation() turns graphed decode on per thread on NVIDIA CUDA; =0 wins, others stay eager.+Loads _decode_graphs.py by file path (importing unsloth needs an accelerator)."""++import importlib.util+import threading+from pathlib import Path++import pytest+import torch+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py++ENV = "UNSLOTH_DECODE_CUDA_GRAPHS"+_SOURCE = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_decode_graphs.py"+++def G(monkeypatch):+ spec = importlib.util.spec_from_file_location("_unsloth_decode_graphs_studio", _SOURCE)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ module.ENABLED = None+ module._STUDIO_DEFAULT_OK = True # the Linux default; the Windows test overrides it+ monkeypatch.delenv(ENV, raising = False)+ return module+++def test_studio_scope_turns_graphs_on_for_nvidia_cuda(G, monkeypatch):+ monkeypatch.setattr(G, "_DEFAULT_OK", True)+ assert G.enabled() is False+ with G.studio_generation():+ assert G.enabled() is True+ with G.rl_generation():+ assert G.enabled() is True+ assert G.enabled() is True+ assert G.enabled() is False+++def test_studio_scope_honours_the_variable(G, monkeypatch):+ monkeypatch.setattr(G, "_DEFAULT_OK", True)+ monkeypatch.setenv(ENV, "0")+ with G.studio_generation():+ assert G.enabled() is False+ monkeypatch.setattr(G, "ENABLED", False)+ monkeypatch.setenv(ENV, "1")+ with G.studio_generation():+ assert G.enabled() is False+++def test_studio_scope_stays_eager_off_nvidia(G, monkeypatch):+ # ROCm (HIP), XPU, MPS and CPU hosts: _default_ok() is False.+ monkeypatch.setattr(G, "_DEFAULT_OK", False)+ with G.studio_generation():+ assert G.enabled() is False+ monkeypatch.setattr(torch.version, "hip", "6.4.0", raising = False)+ assert G._default_ok() is False+++def test_studio_scope_is_per_thread(G, monkeypatch):+ monkeypatch.setattr(G, "_DEFAULT_OK", True)+ seen = []+ with G.studio_generation():+ t = threading.Thread(target = lambda: seen.append(G.enabled()))+ t.start()+ t.join()+ assert G.enabled() is True+ assert seen == [False]+++def test_studio_scope_unwinds_on_error(G, monkeypatch):+ monkeypatch.setattr(G, "_DEFAULT_OK", True)+ with pytest.raises(RuntimeError):+ with G.studio_generation():+ raise RuntimeError("generate failed")+ assert G.enabled() is False+++def test_studio_scope_is_off_on_native_windows(G, monkeypatch):+ # Native Windows CUDA capture is unverified: the Studio default stays eager there, =1 still forces.+ monkeypatch.setattr(G, "_DEFAULT_OK", True)+ monkeypatch.setattr(G, "_STUDIO_DEFAULT_OK", False)+ with G.studio_generation():+ assert G.enabled() is False+ with G.rl_generation():+ assert G.enabled() is True+ assert G.enabled() is False+ assert G.enabled() is False+ monkeypatch.setenv(ENV, "1")+ with G.studio_generation():+ assert G.enabled() is True++[email protected](not has_real_cuda(), reason = "CUDA only")+def test_studio_generate_graphed_plain_generate_eager():+ import unsloth # noqa: F401+ from unsloth import FastLanguageModel+ import unsloth.models._decode_graphs as G++ if not G._DEFAULT_OK:+ pytest.skip("the default is NVIDIA CUDA only")+ if not G._STUDIO_DEFAULT_OK:+ pytest.skip("the Studio default stays eager on native Windows")+ model, tok = FastLanguageModel.from_pretrained(+ "trl-internal-testing/tiny-Qwen3ForCausalLM", max_seq_length = 2048, load_in_4bit = False+ )+ FastLanguageModel.for_inference(model)+ enc = tok(["Once upon a time there was"], return_tensors = "pt").to("cuda")++ def gen():+ out = model.generate(+ **enc, max_new_tokens = 24, min_new_tokens = 24, do_sample = False,+ output_logits = True, return_dict_in_generate = True,+ ) # fmt: skip+ return out.sequences.cpu(), torch.stack(out.logits).cpu()++ def replays():+ return G.stats(model).get("replays", 0)++ eager = gen()+ assert replays() == 0+ with G.studio_generation():+ graphed = gen()+ assert replays() == 24 - 2 and G.stats(model)["disabled"] is None+ assert torch.equal(eager[0], graphed[0]) and torch.equal(eager[1], graphed[1])+ assert torch.equal(gen()[1], eager[1]) and replays() == 24 - 2+ G.release_for_training(model)+ assert G.stats(model)["graphs"] == 0+ assert not any("paged_attention" in m.__dict__ for m in model.modules())diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.pyindex ab16312b398..4a837fac823 100644--- a/unsloth/models/_decode_graphs.py+++ b/unsloth/models/_decode_graphs.py@@ -2,7 +2,8 @@# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved."""CUDA-graph decode; UNSLOTH_DECODE_CUDA_GRAPHS=1 forces, =0 disables; unset, only RL-trainer-generate on NVIDIA CUDA; graphs dropped when params, LoRA, quant or RoPE change.+generate and Studio's chat worker on NVIDIA CUDA; graphs dropped when params, LoRA, quant or RoPE+change.UNSLOTH_DECODE_CUDA_GRAPHS_FULL=1 also captures the attention cores: each step is then ONE graph thatattends over a fixed span of cache rows (the next power of two >= the KV length, at least BUCKET, at@@ -13,12 +14,13 @@ batch 1 by default (UNSLOTH_DECODE_FULL_GRAPH_MAX_BSZ): larger batches keep theimport mathimport os+import sysimport threadingimport torch# Programmatic override (tests, harnesses): True / False force graphs on / off, None (default) follows-# the environment variable read at call time and the RL generation scope (enabled()).+# the environment variable read at call time and the RL / Studio generation scopes (enabled()).ENABLED = None# Whole-step graphs (attention captured too, over a power-of-two span of cache rows).FULL = os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_FULL", "0") == "1"@@ -36,6 +38,7 @@ __all__ = ["ENABLED","enabled","rl_generation",+ "studio_generation","static","alloc_decode_buffers","decode_step",@@ -105,6 +108,23 @@ class rl_generation:return False+# Native Windows CUDA capture is unverified: Studio default off there (=1 still forces).+_STUDIO_DEFAULT_OK = sys.platform != "win32"+++class studio_generation(rl_generation):+ """Per-thread scope for Studio's chat generate(): graphs default on, freed on unload (release())."""++ __slots__ = ("_entered",)++ def __enter__(self):+ self._entered = _STUDIO_DEFAULT_OK+ return super().__enter__() if self._entered else self++ def __exit__(self, *exc):+ return super().__exit__(*exc) if self._entered else False++def static():"""The _StepStatics of a graph-mode step running on this thread, else None (eager)."""if torch.compiler.is_compiling():@@ -297,6 +317,7 @@ class _ModelGraphs:self.optimizer_steps = -1self.buffer_epoch = -1self.disabled = None+ self.ineligible_epoch = Noneself.rope_len = Noneself.captures = 0self.replays = 0@@ -625,6 +646,9 @@ def decode_step(state.eager_steps += 1state.generation_epoch = -1return None+ if state.ineligible_epoch == GENERATION_EPOCH:+ state.eager_steps += 1+ return Noneif (state.generation_epoch != GENERATION_EPOCHor state.optimizer_steps != K._OPTIMIZER_STEPS@@ -637,7 +661,11 @@ def decode_step(return Nonefingerprint = _fingerprint(model, layers, rope_len)if fingerprint is None:- state.disabled = "parameters on several / non-CUDA devices or a non-LoRA adapter"+ # Ineligible for this generation only (several / non-CUDA devices, or a DoRA / non-LoRA+ # adapter, e.g. one side of a compare): free the captured steps and re-check next time.+ state.clear()+ state.ineligible_epoch = GENERATION_EPOCH+ state.eager_steps += 1return Noneif (fingerprint != state.fingerprint@@ -754,43 +782,42 @@ def _capture(no_cache = lambda: torch.autocast("cuda", dtype = dtype, enabled = enabled, cache_enabled = False)stream.wait_stream(torch.cuda.current_stream())step = _Step(st)- with torch.cuda.stream(stream), no_cache():- # Warmup: Triton JIT, cuBLAS workspaces, lazily built RoPE rows / LoRA casts. It writes this- # step's K / V into its cache slot, the same values the replay writes again.- _run_step(model, fast_forward_inference, logits_fn, s, st, past_key_values)- stream.synchronize()-- def split(fn):- # Close the segment, run the attention core once (its output address is what the next- # segment reads), open the next segment.- _end(step.segments[-1])- # Two calls with the first result alive: the same address means the core writes into a- # persistent buffer (bsz == 1 writes the query buffer); else (SDPA allocates) its- # result is copied into a fixed buffer.- out = fn()- fixed = out if fn().data_ptr() == out.data_ptr() else torch.empty_like(out)- step.cores.append(fn if fixed is out else (lambda: fixed.copy_(fn())))+ try:+ with torch.cuda.stream(stream), no_cache():+ # Warmup (JIT, workspaces, lazy RoPE / LoRA casts); its K / V write equals the replay's.+ _run_step(model, fast_forward_inference, logits_fn, s, st, past_key_values)+ stream.synchronize()++ def split(fn):+ # The next segment reads the attention core's output address.+ _end(step.segments[-1])+ # Same address on two calls = persistent output buffer; else (SDPA) copy to a fixed one.+ out = fn()+ fixed = out if fn().data_ptr() == out.data_ptr() else torch.empty_like(out)+ step.cores.append(fn if fixed is out else (lambda: fixed.copy_(fn())))+ graph = torch.cuda.CUDAGraph()+ step.segments.append(graph)+ _begin(graph, state.pool)+ return fixed+graph = torch.cuda.CUDAGraph()step.segments.append(graph)+ if span is None:+ st.attention = split_begin(graph, state.pool)- return fixed-- graph = torch.cuda.CUDAGraph()- step.segments.append(graph)- if span is None:- st.attention = split- _begin(graph, state.pool)- try:- step.logits = _run_step(- model, fast_forward_inference, logits_fn, s, st, past_key_values- )- finally:- st.attention = _run_now- if torch.cuda.is_current_stream_capturing():- _end(step.segments[-1])- torch.cuda.current_stream().wait_stream(stream)- st.mask = None- return step+ try:+ step.logits = _run_step(+ model, fast_forward_inference, logits_fn, s, st, past_key_values+ )+ finally:+ st.attention = _run_now+ if torch.cuda.is_current_stream_capturing():+ _end(step.segments[-1])+ return step+ finally:+ # Also after a failed warmup / capture: the eager fallback must not race the side stream.+ torch.cuda.current_stream().wait_stream(stream)+ st.mask = Nonedef _begin(graph, pool):