Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: faster Laya inference (marker-only head, CUDA graphs) (#12224)

Daniel Han4e8503e43779
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.py
index b0bdd6f8faf..f6cb68dcfa6 100644
--- a/studio/backend/core/systemone/laya_runtime.py
+++ b/studio/backend/core/systemone/laya_runtime.py
@@ -476,6 +476,8 @@ def _place(agent, device, fp16_checkpoint: bool) -> None:
import torch
agent.__dict__["_unsloth_fp16_checkpoint"] = fp16_checkpoint
+ # Captured graphs point at the current weights; a move or recast replaces them.
+ agent.__dict__.pop("_unsloth_graphs", None)
weights, compute = _precision(device, fp16_checkpoint)
# Cast on the CPU side of the move: before moving to an accelerator, after coming back from one.
if device.type == "cpu":
@@ -698,9 +700,7 @@ def _forward(agent, items: list[dict[str, Any]]):
global _agent, _loaded, _device_name
import torch
- collate_items = _laya().common.collate_items
-
- batch = collate_items([items], agent.tok.pad_token_id)
+ batch = _collate(items, agent.tok.pad_token_id)
if agent.device == "mlx":
try:
return agent.model.logits(batch), int(batch["attention_mask"].sum())
@@ -742,6 +742,8 @@ def _forward(agent, items: list[dict[str, Any]]):
if agent.dtype == torch.float16 and not bool(torch.isfinite(logits).all()):
# fp16 activations overflowed; the fp16 weights widen to fp32 exactly, so rerun at full precision.
logger.warning("Laya overflowed in float16; continuing in float32")
+ # The graphs read the fp16 weights that .float() is about to replace.
+ agent.__dict__.pop("_unsloth_graphs", None)
agent.model.float()
agent.dtype = torch.float32
logits = _run_model(agent, batch)
@@ -749,27 +751,316 @@ def _forward(agent, items: list[dict[str, Any]]):
def _run_model(agent, batch):
+ import os
+
import torch
device = agent.device
- with (
- torch.inference_mode(),
- torch.autocast(
- device_type = device.type,
- dtype = agent.dtype,
- enabled = device.type in ("cuda", "cpu") and agent.dtype != torch.float32,
- ),
+ fast = _marker_head(agent.model)
+ if (
+ fast
+ and device.type == "cuda"
+ and os.environ.get("UNSLOTH_SYSTEMONE_CUDA_GRAPHS", "") != "0"
+ # ROCm reports "cuda" too; as in diffusion_cuda_graph.py, graphs are CUDA only.
+ and not torch.version.hip
):
- logits, _ = agent.model(
- batch["input_ids"].to(device),
- batch["attention_mask"].to(device),
- batch["marker_pos"].to(device),
- batch["marker_mask"].to(device),
- batch["qtype"].to(device),
- )
+ graphs = agent.__dict__.get("_unsloth_graphs")
+ if graphs is None:
+ graphs = agent.__dict__["_unsloth_graphs"] = _CUDAGraphs(agent)
+ with torch.inference_mode():
+ logits = graphs.run(batch)
+ if logits is not None:
+ return logits
+ with torch.inference_mode(), _autocast(agent):
+ args = [batch[name].to(device) for name in _INPUTS]
+ if fast:
+ # No padding in the batch: no mask, so SDPA may pick its fastest kernel.
+ return _decision_logits(
+ agent.model, *args, padded = not bool(batch["attention_mask"].all())
+ )
+ logits, _ = agent.model(*args)
return logits
+_INPUTS = ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")
+# Padded rows x tokens above which a batch runs eagerly (B200: graphs win up to 8 x 1024, lose at 16 x 512).
+_GRAPH_TOKENS = 8192
+
+
+def _autocast(agent, cache_enabled = True):
+ import torch
+ device = agent.device
+ return torch.autocast(
+ device_type = device.type,
+ dtype = agent.dtype,
+ enabled = device.type in ("cuda", "cpu") and agent.dtype != torch.float32,
+ cache_enabled = cache_enabled,
+ )
+
+
+def _collate(items: list[dict[str, Any]], pad_id: int) -> dict[str, Any]:
+ """laya's ``collate_items`` for one request (same tensors), filled with numpy instead of per-row copies."""
+ import numpy as np
+ import torch
+
+ n = len(items)
+ lengths = np.fromiter((len(it["ids"]) for it in items), dtype = np.int64, count = n)
+ counts = np.fromiter((len(it["markers"]) for it in items), dtype = np.int64, count = n)
+ tokens = np.arange(int(lengths.max()))[None, :] < lengths[:, None]
+ options = np.arange(int(counts.max()))[None, :] < counts[:, None]
+ ids = np.full(tokens.shape, pad_id, dtype = np.int64)
+ ids[tokens] = np.fromiter(
+ (t for it in items for t in it["ids"]), dtype = np.int64, count = int(lengths.sum())
+ )
+ positions = np.zeros(options.shape, dtype = np.int64)
+ positions[options] = np.fromiter(
+ (m for it in items for m in it["markers"]), dtype = np.int64, count = int(counts.sum())
+ )
+ return {
+ "input_ids": torch.from_numpy(ids),
+ "attention_mask": torch.from_numpy(tokens.astype(np.int64)),
+ "marker_pos": torch.from_numpy(positions),
+ "marker_mask": torch.from_numpy(options),
+ "qtype": torch.tensor([it["qtype"] for it in items]),
+ }
+
+
+def _marker_head(model) -> bool:
+ """Whether :func:`_decision_logits` can stand in for ``model(...)``: laya's pre-norm ReLU head and GELU scorer."""
+ import os
+
+ import torch.nn as nn
+
+ if os.environ.get("UNSLOTH_SYSTEMONE_FAST", "") == "0":
+ return False
+ cached = model.__dict__.get("_unsloth_marker_head")
+ if cached is not None:
+ return cached
+ head, scorer = getattr(model, "head", None), getattr(model, "scorer", None)
+ ok = (
+ isinstance(head, nn.TransformerEncoder)
+ and len(head.layers) > 0
+ and head.norm is None
+ and all(
+ type(layer) is nn.TransformerEncoderLayer
+ and layer.norm_first
+ and layer.activation_relu_or_gelu == 1
+ and type(layer.self_attn) is nn.MultiheadAttention
+ and layer.self_attn.batch_first
+ and layer.self_attn._qkv_same_embed_dim
+ and layer.self_attn.in_proj_bias is not None
+ for layer in head.layers
+ )
+ and isinstance(scorer, nn.Sequential)
+ and [type(m) for m in scorer] == [nn.LayerNorm, nn.Linear, nn.GELU, nn.Linear]
+ and scorer[2].approximate == "none"
+ )
+ model.__dict__["_unsloth_marker_head"] = ok
+ return ok
+
+
+def _feed_forward(layer, x):
+ import torch.nn.functional as F
+
+ y = F.layer_norm(x, x.shape[-1:], layer.norm2.weight, layer.norm2.bias, layer.norm2.eps)
+ y = F.relu(F.linear(y, layer.linear1.weight, layer.linear1.bias), inplace = True)
+ return x.add_(F.linear(y, layer.linear2.weight, layer.linear2.bias))
+
+
+def _head_layer(
+ layer,
+ x,
+ mask,
+ markers = None,
+):
+ """One eval-mode pre-norm ``TransformerEncoderLayer``; with ``markers``, only the rows at those positions.
+
+ Keys and values always span every token, so each marker row is exactly what the full layer gives it.
+ """
+ import torch
+ import torch.nn.functional as F
+
+ attn = layer.self_attn
+ batch, length, width = x.shape
+ heads = attn.num_heads
+ y = F.layer_norm(x, (width,), layer.norm1.weight, layer.norm1.bias, layer.norm1.eps)
+ if markers is None:
+ q, k, v = (
+ F.linear(y, attn.in_proj_weight, attn.in_proj_bias)
+ .view(batch, length, 3, heads, width // heads)
+ .permute(2, 0, 3, 1, 4)
+ )
+ else:
+ index = markers[:, :, None].expand(-1, -1, width)
+ k, v = (
+ F.linear(y, attn.in_proj_weight[width:], attn.in_proj_bias[width:])
+ .view(batch, length, 2, heads, width // heads)
+ .permute(2, 0, 3, 1, 4)
+ )
+ q = F.linear(
+ torch.gather(y, 1, index), attn.in_proj_weight[:width], attn.in_proj_bias[:width]
+ )
+ q = q.view(batch, -1, heads, width // heads).transpose(1, 2)
+ x = torch.gather(x, 1, index)
+ # Boolean attn_mask is True where attention is allowed (the inverse of src_key_padding_mask); SDPA
+ # applies dropout_p even in eval, so it is passed as 0.
+ out = F.scaled_dot_product_attention(q, k, v, attn_mask = mask, dropout_p = 0.0)
+ out = out.transpose(1, 2).reshape(batch, -1, width)
+ x = x.add_(F.linear(out, attn.out_proj.weight, attn.out_proj.bias))
+ return _feed_forward(layer, x)
+
+
+def _decision_logits(
+ model,
+ input_ids,
+ attention_mask,
+ marker_pos,
+ marker_mask,
+ qtype,
+ *,
+ padded = True,
+):
+ """The ``logits`` of laya's ``DecisionModel.forward``, computing only what they read.
+
+ The scorer only reads the option markers, so the last head layer runs at the markers alone, and the
+ action head Studio never returns is skipped.
+ """
+ import torch.nn.functional as F
+
+ h = model.encoder(input_ids = input_ids, attention_mask = attention_mask).last_hidden_state
+ h = h + model.type_emb(qtype)[:, None, :]
+ mask = attention_mask.bool()[:, None, None, :] if padded else None
+ layers = model.head.layers
+ for layer in layers[:-1]:
+ h = _head_layer(layer, h, mask)
+ m = _head_layer(layers[-1], h, mask, marker_pos.clamp(min = 0))
+ norm, up, _, down = model.scorer
+ m = F.layer_norm(m, m.shape[-1:], norm.weight, norm.bias, norm.eps)
+ logits = (
+ F.linear(F.gelu(F.linear(m, up.weight, up.bias)), down.weight, down.bias)
+ .squeeze(-1)
+ .float()
+ )
+ return logits.masked_fill(~marker_mask, -1e4)
+
+
+class _CUDAGraphs:
+ """One captured :func:`_decision_logits` per padded (rows, tokens, options) bucket, sharing one memory pool.
+
+ A short request is bound by kernel launches (about 2.5 ms of GPU work in a 12 ms forward); replaying a
+ graph launches it all at once. Past ``_GRAPH_TOKENS`` padded tokens the GPU is the bottleneck and
+ padding up to a bucket costs more than launches save, so those batches run eagerly. Callers hold
+ ``_run_lock``, so one replay runs at a time and each output is read before the next overwrites it.
+ """
+
+ ROWS = (1, 2, 4, 8, 16)
+ TOKENS = (64, 128, 256, 512, 1024)
+ OPTIONS = (2, 4, 8, 16, 32, 64, 128, 256)
+ MAX_GRAPHS = 64
+
+ def __init__(self, agent):
+ import torch
+
+ self.agent = agent
+ # transformers 4.x wraps ModernBERT's embeddings and MLP in torch.compile on CUDA, which cannot run
+ # under capture; 5.x dropped that path, so this is what 5.x runs anyway.
+ config = getattr(getattr(agent.model, "encoder", None), "config", None)
+ if getattr(config, "reference_compile", False) is not False:
+ config.reference_compile = False
+ self.graphs: dict[tuple[int, int, int], tuple[Any, dict[str, Any], Any]] = {}
+ self.broken = False
+ self.pool = torch.cuda.graph_pool_handle()
+ self.pool_bytes = 0
+ # One warm-up stream: the allocator caches blocks per stream, so a new stream per capture would strand them.
+ self.stream = torch.cuda.Stream(agent.device)
+ free, _ = torch.cuda.mem_get_info(agent.device)
+ self.max_pool_bytes = min(1 << 30, free // 10)
+
+ @staticmethod
+ def _fit(n, sizes):
+ return next((size for size in sizes if n <= size), None)
+
+ def _forward(self, static):
+ with _autocast(self.agent, cache_enabled = False):
+ return _decision_logits(self.agent.model, *(static[name] for name in _INPUTS))
+
+ def _capture(self, key):
+ import torch
+
+ rows, tokens, options = key
+ device = self.agent.device
+ static = {
+ "input_ids": torch.zeros((rows, tokens), dtype = torch.long, device = device),
+ "attention_mask": torch.ones((rows, tokens), dtype = torch.long, device = device),
+ "marker_pos": torch.zeros((rows, options), dtype = torch.long, device = device),
+ "marker_mask": torch.ones((rows, options), dtype = torch.bool, device = device),
+ "qtype": torch.zeros((rows,), dtype = torch.long, device = device),
+ }
+ # Warm up off the default stream first, as torch.cuda.graph requires, so lazy init is not captured.
+ self.stream.wait_stream(torch.cuda.current_stream(device))
+ with torch.cuda.stream(self.stream):
+ for _ in range(2):
+ self._forward(static)
+ torch.cuda.current_stream(device).wait_stream(self.stream)
+ graph = torch.cuda.CUDAGraph()
+ # thread_local: CUDA work on other Studio threads cannot break (or be broken by) this capture.
+ with torch.cuda.graph(graph, pool = self.pool, capture_error_mode = "thread_local"):
+ logits = self._forward(static)
+ return graph, static, logits
+
+ def run(self, batch):
+ """Logits for ``batch`` from a graph replay, or None to run it eagerly."""
+ import torch
+
+ rows, tokens = batch["input_ids"].shape
+ options = batch["marker_pos"].shape[1]
+ key = (
+ self._fit(rows, self.ROWS),
+ self._fit(tokens, self.TOKENS),
+ self._fit(options, self.OPTIONS),
+ )
+ if self.broken or None in key or key[0] * key[1] > _GRAPH_TOKENS:
+ return None
+ entry = self.graphs.get(key)
+ if entry is None:
+ if len(self.graphs) >= self.MAX_GRAPHS or self.pool_bytes >= self.max_pool_bytes:
+ return None
+ before = torch.cuda.memory_reserved(self.agent.device)
+ try:
+ entry = self._capture(key)
+ except Exception:
+ # e.g. transformers 4.x builds ModernBERT's sliding-window mask on the CPU every forward.
+ logger.warning(
+ "Laya cannot run in CUDA graphs here; running it eagerly", exc_info = True
+ )
+ self.broken = True
+ return None
+ self.graphs[key] = entry
+ self.pool_bytes += max(0, torch.cuda.memory_reserved(self.agent.device) - before)
+ if self.pool_bytes > self.max_pool_bytes:
+ # Over budget: the pool is only released with every graph in it, so drop them all and stay eager.
+ logger.warning("Laya CUDA graphs need more memory than allowed; running eagerly")
+ self.graphs.clear()
+ self.broken = True
+ return None
+ graph, static, logits = entry
+ # Padding tokens are masked; padding rows repeat row 0 so every row has a real token to attend to.
+ static["input_ids"].zero_()
+ static["attention_mask"].zero_()
+ static["marker_pos"].zero_()
+ static["marker_mask"].zero_()
+ for name in _INPUTS:
+ source = batch[name]
+ if source.dim() == 1:
+ static[name][:rows].copy_(source)
+ else:
+ static[name][:rows, : source.shape[1]].copy_(source)
+ if rows < key[0]:
+ static[name][rows:] = static[name][:1]
+ graph.replay()
+ return logits[:rows, :options]
+
+
def _probabilities(agent, row, k: int, qtype: int):
import numpy as np
diff --git a/studio/backend/tests/test_systemone.py b/studio/backend/tests/test_systemone.py
index b37f28e09d4..f3404dd1e38 100644
--- a/studio/backend/tests/test_systemone.py
+++ b/studio/backend/tests/test_systemone.py
@@ -1466,6 +1466,152 @@ def test_high_special_token_ids_still_skip_the_embedding_init(tmp_path):
assert torch.equal(fast(*args)[0], reference(*args)[0])
+def _tiny_decision_batch(
+ torch,
+ rows = 4,
+ tokens = 40,
+):
+ generator = torch.Generator().manual_seed(0)
+ lengths = [tokens, 23, 9, 31][:rows]
+ ids = torch.randint(5, 300, (rows, tokens), generator = generator)
+ mask = torch.zeros(rows, tokens, dtype = torch.long)
+ for row, length in enumerate(lengths):
+ mask[row, :length] = 1
+ ids[row, length:] = 297
+ counts = [2, 5, 1, 3][:rows]
+ positions = torch.zeros(rows, 5, dtype = torch.long)
+ markers = torch.zeros(rows, 5, dtype = torch.bool)
+ for row, count in enumerate(counts):
+ positions[row, :count] = torch.randperm(lengths[row], generator = generator)[:count]
+ markers[row, :count] = True
+ return ids, mask, positions, markers, torch.tensor([0, 1, 2, 1][:rows])
+
+
[email protected]("head_layers", [1, 2, 3])
+def test_marker_head_matches_laya_forward(tmp_path, head_layers):
+ torch = pytest.importorskip("torch")
+ laya = laya_runtime._laya()
+ encoder_dir, cfg = _tiny_laya_encoder(tmp_path)
+ torch.manual_seed(0)
+ model = laya.common.build_model(
+ {**cfg, "head_layers": head_layers}, encoder_dir = encoder_dir
+ ).eval()
+ assert laya_runtime._marker_head(model)
+ args = _tiny_decision_batch(torch)
+ with torch.inference_mode():
+ want = model(*args)[0]
+ got = laya_runtime._decision_logits(model, *args)
+ assert got.dtype == want.dtype == torch.float32
+ torch.testing.assert_close(got, want, atol = 1e-5, rtol = 1e-4)
+ # Without padding the mask is dropped altogether.
+ unpadded = [arg[:1] for arg in args]
+ torch.testing.assert_close(
+ laya_runtime._decision_logits(model, *unpadded, padded = False),
+ model(*unpadded)[0],
+ atol = 1e-5,
+ rtol = 1e-4,
+ )
+
+
+def test_models_without_laya_head_keep_their_own_forward(tmp_path, monkeypatch):
+ pytest.importorskip("torch")
+ laya = laya_runtime._laya()
+ encoder_dir, cfg = _tiny_laya_encoder(tmp_path)
+ assert not laya_runtime._marker_head(
+ laya.common.build_model({**cfg, "head_layers": 0}, encoder_dir = encoder_dir)
+ )
+ model = laya.common.build_model(cfg, encoder_dir = encoder_dir)
+ model.scorer[2].approximate = "tanh"
+ assert not laya_runtime._marker_head(model)
+ model = laya.common.build_model(cfg, encoder_dir = encoder_dir)
+ monkeypatch.setenv("UNSLOTH_SYSTEMONE_FAST", "0")
+ assert not laya_runtime._marker_head(model)
+
+
+def test_collate_matches_laya():
+ torch = pytest.importorskip("torch")
+ collate_items = laya_runtime._laya().common.collate_items
+ items = [
+ {"ids": [2, 7, 9, 1], "markers": [1, 2], "qtype": 1},
+ {"ids": [2, 8, 1], "markers": [1], "qtype": 0},
+ {"ids": [2, 5, 6, 7, 8, 9, 1], "markers": [1, 3, 5, 6], "qtype": 2},
+ ]
+ want = collate_items([items], 297)
+ got = laya_runtime._collate(items, 297)
+ for name in laya_runtime._INPUTS:
+ assert got[name].dtype == want[name].dtype and torch.equal(got[name], want[name]), name
+
+
+def test_cuda_graphs_replay_the_eager_logits(tmp_path, monkeypatch):
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA GPU")
+ laya = laya_runtime._laya()
+ encoder_dir, cfg = _tiny_laya_encoder(tmp_path)
+ torch.manual_seed(0)
+ model = laya.common.build_model({**cfg, "head_layers": 2}, encoder_dir = encoder_dir).eval()
+ agent = SimpleNamespace(model = model, device = torch.device("cpu"), dtype = torch.float32)
+ monkeypatch.setattr(laya_runtime, "_precision", lambda device, fp16_checkpoint: (None, None))
+ laya_runtime._place(agent, torch.device("cuda"), False)
+ ids, mask, positions, markers, qtype = _tiny_decision_batch(torch, rows = 3, tokens = 40)
+ batch = dict(zip(laya_runtime._INPUTS, (ids, mask, positions, markers, qtype)))
+ graphed = laya_runtime._run_model(agent, batch).clone()
+ graphs = agent.__dict__["_unsloth_graphs"]
+ if graphs.broken:
+ # transformers 4.x: ModernBERT cannot be captured, so the model runs eagerly with the same logits.
+ import transformers
+
+ assert int(transformers.__version__.split(".")[0]) < 5
+ with torch.inference_mode():
+ reference = model(*(value.cuda() for value in batch.values()))[0]
+ torch.testing.assert_close(graphed, reference, atol = 1e-4, rtol = 1e-4)
+ return
+ assert list(graphs.graphs) == [(4, 64, 8)]
+ # The same padded bucket run eagerly: rows 3 -> 4 repeat row 0, tokens 40 -> 64 and options 5 -> 8 are masked.
+ padded = {
+ "input_ids": torch.zeros(4, 64, dtype = torch.long),
+ "attention_mask": torch.zeros(4, 64, dtype = torch.long),
+ "marker_pos": torch.zeros(4, 8, dtype = torch.long),
+ "marker_mask": torch.zeros(4, 8, dtype = torch.bool),
+ "qtype": torch.zeros(4, dtype = torch.long),
+ }
+ for name, value in batch.items():
+ if value.dim() == 1:
+ padded[name][:3] = value
+ else:
+ padded[name][:3, : value.shape[1]] = value
+ padded[name][3:] = padded[name][:1]
+ with torch.inference_mode():
+ eager = laya_runtime._decision_logits(
+ model, *(padded[name].cuda() for name in laya_runtime._INPUTS)
+ )
+ assert torch.equal(graphed, eager[:3, :5])
+ with torch.inference_mode():
+ reference = model(*(value.cuda() for value in batch.values()))[0]
+ torch.testing.assert_close(graphed, reference, atol = 1e-4, rtol = 1e-4)
+ # A second request with other contents replays the same graph.
+ batch["input_ids"] = torch.randint(5, 300, ids.shape)
+ with torch.inference_mode():
+ reference = model(*(value.cuda() for value in batch.values()))[0]
+ torch.testing.assert_close(
+ laya_runtime._run_model(agent, batch), reference, atol = 1e-4, rtol = 1e-4
+ )
+ assert len(graphs.graphs) == 1
+ # Moving or recasting the model drops the graphs that point at its old weights.
+ laya_runtime._place(agent, torch.device("cuda"), False)
+ assert "_unsloth_graphs" not in agent.__dict__
+ # Batches too large to be worth padding into a bucket run eagerly.
+ assert (
+ laya_runtime._CUDAGraphs(agent).run(
+ {"input_ids": torch.zeros(16, 1024), "marker_pos": torch.zeros(16, 2)}
+ )
+ is None
+ )
+ monkeypatch.setenv("UNSLOTH_SYSTEMONE_CUDA_GRAPHS", "0")
+ laya_runtime._run_model(agent, batch)
+ assert "_unsloth_graphs" not in agent.__dict__
+
+
def test_encoders_that_keep_the_padding_id_match_laya(tmp_path):
torch = pytest.importorskip("torch")
from transformers import RobertaConfig
@@ -1500,3 +1646,111 @@ def test_encoders_that_keep_the_padding_id_match_laya(tmp_path):
)
with torch.inference_mode():
assert torch.equal(fast(*args)[0], reference(*args)[0])
+
+
+def _cuda_tiny_agent(
+ tmp_path,
+ monkeypatch,
+ torch,
+ capture = True,
+):
+ if not torch.cuda.is_available() or torch.version.hip:
+ pytest.skip("needs a CUDA GPU")
+ import transformers
+
+ if capture and int(transformers.__version__.split(".")[0]) < 5:
+ pytest.skip("transformers 4.x ModernBERT cannot be captured")
+ laya = laya_runtime._laya()
+ encoder_dir, cfg = _tiny_laya_encoder(tmp_path)
+ torch.manual_seed(0)
+ model = laya.common.build_model({**cfg, "head_layers": 2}, encoder_dir = encoder_dir).eval()
+ agent = SimpleNamespace(model = model, device = torch.device("cpu"), dtype = torch.float32)
+ monkeypatch.setattr(laya_runtime, "_precision", lambda device, fp16_checkpoint: (None, None))
+ laya_runtime._place(agent, torch.device("cuda"), False)
+ return agent
+
+
+def _padded_eager(model, batch, key, torch):
+ rows = batch["input_ids"].shape[0]
+ shapes = {
+ "input_ids": key[:2],
+ "attention_mask": key[:2],
+ "marker_pos": (key[0], key[2]),
+ "marker_mask": (key[0], key[2]),
+ "qtype": key[:1],
+ }
+ padded = {name: torch.zeros(shape, dtype = batch[name].dtype) for name, shape in shapes.items()}
+ for name, value in batch.items():
+ if value.dim() == 1:
+ padded[name][:rows] = value
+ else:
+ padded[name][:rows, : value.shape[1]] = value
+ padded[name][rows:] = padded[name][:1]
+ with torch.inference_mode():
+ return laya_runtime._decision_logits(
+ model, *(padded[name].cuda() for name in laya_runtime._INPUTS)
+ )
+
+
+def test_cuda_graphs_sharing_a_pool_replay_in_any_order(tmp_path, monkeypatch):
+ torch = pytest.importorskip("torch")
+ agent = _cuda_tiny_agent(tmp_path, monkeypatch, torch)
+ generator = torch.Generator().manual_seed(1)
+ shapes = [(1, 20, 2), (3, 40, 5), (2, 100, 3), (6, 200, 9)]
+
+ def request(rows, tokens, options):
+ ids = torch.randint(5, 300, (rows, tokens), generator = generator)
+ mask = torch.ones(rows, tokens, dtype = torch.long)
+ mask[1:, tokens // 2 :] = 0
+ positions = torch.randint(0, tokens // 2, (rows, options), generator = generator)
+ markers = torch.ones(rows, options, dtype = torch.bool)
+ markers[1:, options - 1] = False
+ return dict(
+ zip(
+ laya_runtime._INPUTS,
+ (ids, mask, positions, markers, torch.randint(0, 3, (rows,), generator = generator)),
+ )
+ )
+
+ graphs = None
+ # Capture order A B C D, then replay out of order, interleaving buckets.
+ for index in [0, 1, 2, 3, 0, 2, 1, 3, 3, 0, 1, 0, 2]:
+ batch = request(*shapes[index])
+ got = laya_runtime._run_model(agent, batch).clone()
+ graphs = agent.__dict__["_unsloth_graphs"]
+ key = graphs.graphs and next(
+ k
+ for k in graphs.graphs
+ if k[0] >= shapes[index][0] and k[1] >= shapes[index][1] and k[2] >= shapes[index][2]
+ )
+ want = _padded_eager(agent.model, batch, key, torch)
+ assert torch.equal(got, want[: shapes[index][0], : shapes[index][2]]), (index, key)
+ assert len(graphs.graphs) == 4 and not graphs.broken
+
+
+def test_cuda_graphs_over_the_memory_budget_run_eagerly(tmp_path, monkeypatch):
+ torch = pytest.importorskip("torch")
+ agent = _cuda_tiny_agent(tmp_path, monkeypatch, torch)
+ batch = dict(zip(laya_runtime._INPUTS, _tiny_decision_batch(torch, rows = 3, tokens = 40)))
+ graphs = agent.__dict__["_unsloth_graphs"] = laya_runtime._CUDAGraphs(agent)
+ # The first capture alone takes more than the budget.
+ graphs.max_pool_bytes = 1 << 20
+ reserved = iter([0, 2 << 20])
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: next(reserved))
+ logits = laya_runtime._run_model(agent, batch)
+ assert graphs.broken and not graphs.graphs
+ with torch.inference_mode():
+ reference = agent.model(*(value.cuda() for value in batch.values()))[0]
+ torch.testing.assert_close(logits, reference, atol = 1e-4, rtol = 1e-4)
+
+
+def test_rocm_never_builds_cuda_graphs(tmp_path, monkeypatch):
+ torch = pytest.importorskip("torch")
+ agent = _cuda_tiny_agent(tmp_path, monkeypatch, torch, capture = False)
+ # transformers 4.x would torch.compile ModernBERT, which a faked ROCm version sends looking for HIP.
+ agent.model.encoder.config.reference_compile = False
+ monkeypatch.setattr(torch.version, "hip", "6.2")
+ monkeypatch.setattr(laya_runtime, "_CUDAGraphs", None)
+ batch = dict(zip(laya_runtime._INPUTS, _tiny_decision_batch(torch, rows = 3, tokens = 40)))
+ laya_runtime._run_model(agent, batch)
+ assert "_unsloth_graphs" not in agent.__dict__