Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: faster Laya inference (marker-only head, CUDA graphs) (#12224)
Daniel Han4e8503e43779
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.pyindex b0bdd6f8faf..f6cb68dcfa6 100644--- a/studio/backend/core/systemone/laya_runtime.py+++ b/studio/backend/core/systemone/laya_runtime.py@@ -476,6 +476,8 @@ def _place(agent, device, fp16_checkpoint: bool) -> None:import torchagent.__dict__["_unsloth_fp16_checkpoint"] = fp16_checkpoint+ # Captured graphs point at the current weights; a move or recast replaces them.+ agent.__dict__.pop("_unsloth_graphs", None)weights, compute = _precision(device, fp16_checkpoint)# Cast on the CPU side of the move: before moving to an accelerator, after coming back from one.if device.type == "cpu":@@ -698,9 +700,7 @@ def _forward(agent, items: list[dict[str, Any]]):global _agent, _loaded, _device_nameimport torch- collate_items = _laya().common.collate_items-- batch = collate_items([items], agent.tok.pad_token_id)+ batch = _collate(items, agent.tok.pad_token_id)if agent.device == "mlx":try:return agent.model.logits(batch), int(batch["attention_mask"].sum())@@ -742,6 +742,8 @@ def _forward(agent, items: list[dict[str, Any]]):if agent.dtype == torch.float16 and not bool(torch.isfinite(logits).all()):# fp16 activations overflowed; the fp16 weights widen to fp32 exactly, so rerun at full precision.logger.warning("Laya overflowed in float16; continuing in float32")+ # The graphs read the fp16 weights that .float() is about to replace.+ agent.__dict__.pop("_unsloth_graphs", None)agent.model.float()agent.dtype = torch.float32logits = _run_model(agent, batch)@@ -749,27 +751,316 @@ def _forward(agent, items: list[dict[str, Any]]):def _run_model(agent, batch):+ import os+import torchdevice = agent.device- with (- torch.inference_mode(),- torch.autocast(- device_type = device.type,- dtype = agent.dtype,- enabled = device.type in ("cuda", "cpu") and agent.dtype != torch.float32,- ),+ fast = _marker_head(agent.model)+ if (+ fast+ and device.type == "cuda"+ and os.environ.get("UNSLOTH_SYSTEMONE_CUDA_GRAPHS", "") != "0"+ # ROCm reports "cuda" too; as in diffusion_cuda_graph.py, graphs are CUDA only.+ and not torch.version.hip):- logits, _ = agent.model(- batch["input_ids"].to(device),- batch["attention_mask"].to(device),- batch["marker_pos"].to(device),- batch["marker_mask"].to(device),- batch["qtype"].to(device),- )+ graphs = agent.__dict__.get("_unsloth_graphs")+ if graphs is None:+ graphs = agent.__dict__["_unsloth_graphs"] = _CUDAGraphs(agent)+ with torch.inference_mode():+ logits = graphs.run(batch)+ if logits is not None:+ return logits+ with torch.inference_mode(), _autocast(agent):+ args = [batch[name].to(device) for name in _INPUTS]+ if fast:+ # No padding in the batch: no mask, so SDPA may pick its fastest kernel.+ return _decision_logits(+ agent.model, *args, padded = not bool(batch["attention_mask"].all())+ )+ logits, _ = agent.model(*args)return logits+_INPUTS = ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")+# Padded rows x tokens above which a batch runs eagerly (B200: graphs win up to 8 x 1024, lose at 16 x 512).+_GRAPH_TOKENS = 8192+++def _autocast(agent, cache_enabled = True):+ import torch+ device = agent.device+ return torch.autocast(+ device_type = device.type,+ dtype = agent.dtype,+ enabled = device.type in ("cuda", "cpu") and agent.dtype != torch.float32,+ cache_enabled = cache_enabled,+ )+++def _collate(items: list[dict[str, Any]], pad_id: int) -> dict[str, Any]:+ """laya's ``collate_items`` for one request (same tensors), filled with numpy instead of per-row copies."""+ import numpy as np+ import torch++ n = len(items)+ lengths = np.fromiter((len(it["ids"]) for it in items), dtype = np.int64, count = n)+ counts = np.fromiter((len(it["markers"]) for it in items), dtype = np.int64, count = n)+ tokens = np.arange(int(lengths.max()))[None, :] < lengths[:, None]+ options = np.arange(int(counts.max()))[None, :] < counts[:, None]+ ids = np.full(tokens.shape, pad_id, dtype = np.int64)+ ids[tokens] = np.fromiter(+ (t for it in items for t in it["ids"]), dtype = np.int64, count = int(lengths.sum())+ )+ positions = np.zeros(options.shape, dtype = np.int64)+ positions[options] = np.fromiter(+ (m for it in items for m in it["markers"]), dtype = np.int64, count = int(counts.sum())+ )+ return {+ "input_ids": torch.from_numpy(ids),+ "attention_mask": torch.from_numpy(tokens.astype(np.int64)),+ "marker_pos": torch.from_numpy(positions),+ "marker_mask": torch.from_numpy(options),+ "qtype": torch.tensor([it["qtype"] for it in items]),+ }+++def _marker_head(model) -> bool:+ """Whether :func:`_decision_logits` can stand in for ``model(...)``: laya's pre-norm ReLU head and GELU scorer."""+ import os++ import torch.nn as nn++ if os.environ.get("UNSLOTH_SYSTEMONE_FAST", "") == "0":+ return False+ cached = model.__dict__.get("_unsloth_marker_head")+ if cached is not None:+ return cached+ head, scorer = getattr(model, "head", None), getattr(model, "scorer", None)+ ok = (+ isinstance(head, nn.TransformerEncoder)+ and len(head.layers) > 0+ and head.norm is None+ and all(+ type(layer) is nn.TransformerEncoderLayer+ and layer.norm_first+ and layer.activation_relu_or_gelu == 1+ and type(layer.self_attn) is nn.MultiheadAttention+ and layer.self_attn.batch_first+ and layer.self_attn._qkv_same_embed_dim+ and layer.self_attn.in_proj_bias is not None+ for layer in head.layers+ )+ and isinstance(scorer, nn.Sequential)+ and [type(m) for m in scorer] == [nn.LayerNorm, nn.Linear, nn.GELU, nn.Linear]+ and scorer[2].approximate == "none"+ )+ model.__dict__["_unsloth_marker_head"] = ok+ return ok+++def _feed_forward(layer, x):+ import torch.nn.functional as F++ y = F.layer_norm(x, x.shape[-1:], layer.norm2.weight, layer.norm2.bias, layer.norm2.eps)+ y = F.relu(F.linear(y, layer.linear1.weight, layer.linear1.bias), inplace = True)+ return x.add_(F.linear(y, layer.linear2.weight, layer.linear2.bias))+++def _head_layer(+ layer,+ x,+ mask,+ markers = None,+):+ """One eval-mode pre-norm ``TransformerEncoderLayer``; with ``markers``, only the rows at those positions.++ Keys and values always span every token, so each marker row is exactly what the full layer gives it.+ """+ import torch+ import torch.nn.functional as F++ attn = layer.self_attn+ batch, length, width = x.shape+ heads = attn.num_heads+ y = F.layer_norm(x, (width,), layer.norm1.weight, layer.norm1.bias, layer.norm1.eps)+ if markers is None:+ q, k, v = (+ F.linear(y, attn.in_proj_weight, attn.in_proj_bias)+ .view(batch, length, 3, heads, width // heads)+ .permute(2, 0, 3, 1, 4)+ )+ else:+ index = markers[:, :, None].expand(-1, -1, width)+ k, v = (+ F.linear(y, attn.in_proj_weight[width:], attn.in_proj_bias[width:])+ .view(batch, length, 2, heads, width // heads)+ .permute(2, 0, 3, 1, 4)+ )+ q = F.linear(+ torch.gather(y, 1, index), attn.in_proj_weight[:width], attn.in_proj_bias[:width]+ )+ q = q.view(batch, -1, heads, width // heads).transpose(1, 2)+ x = torch.gather(x, 1, index)+ # Boolean attn_mask is True where attention is allowed (the inverse of src_key_padding_mask); SDPA+ # applies dropout_p even in eval, so it is passed as 0.+ out = F.scaled_dot_product_attention(q, k, v, attn_mask = mask, dropout_p = 0.0)+ out = out.transpose(1, 2).reshape(batch, -1, width)+ x = x.add_(F.linear(out, attn.out_proj.weight, attn.out_proj.bias))+ return _feed_forward(layer, x)+++def _decision_logits(+ model,+ input_ids,+ attention_mask,+ marker_pos,+ marker_mask,+ qtype,+ *,+ padded = True,+):+ """The ``logits`` of laya's ``DecisionModel.forward``, computing only what they read.++ The scorer only reads the option markers, so the last head layer runs at the markers alone, and the+ action head Studio never returns is skipped.+ """+ import torch.nn.functional as F++ h = model.encoder(input_ids = input_ids, attention_mask = attention_mask).last_hidden_state+ h = h + model.type_emb(qtype)[:, None, :]+ mask = attention_mask.bool()[:, None, None, :] if padded else None+ layers = model.head.layers+ for layer in layers[:-1]:+ h = _head_layer(layer, h, mask)+ m = _head_layer(layers[-1], h, mask, marker_pos.clamp(min = 0))+ norm, up, _, down = model.scorer+ m = F.layer_norm(m, m.shape[-1:], norm.weight, norm.bias, norm.eps)+ logits = (+ F.linear(F.gelu(F.linear(m, up.weight, up.bias)), down.weight, down.bias)+ .squeeze(-1)+ .float()+ )+ return logits.masked_fill(~marker_mask, -1e4)+++class _CUDAGraphs:+ """One captured :func:`_decision_logits` per padded (rows, tokens, options) bucket, sharing one memory pool.++ A short request is bound by kernel launches (about 2.5 ms of GPU work in a 12 ms forward); replaying a+ graph launches it all at once. Past ``_GRAPH_TOKENS`` padded tokens the GPU is the bottleneck and+ padding up to a bucket costs more than launches save, so those batches run eagerly. Callers hold+ ``_run_lock``, so one replay runs at a time and each output is read before the next overwrites it.+ """++ ROWS = (1, 2, 4, 8, 16)+ TOKENS = (64, 128, 256, 512, 1024)+ OPTIONS = (2, 4, 8, 16, 32, 64, 128, 256)+ MAX_GRAPHS = 64++ def __init__(self, agent):+ import torch++ self.agent = agent+ # transformers 4.x wraps ModernBERT's embeddings and MLP in torch.compile on CUDA, which cannot run+ # under capture; 5.x dropped that path, so this is what 5.x runs anyway.+ config = getattr(getattr(agent.model, "encoder", None), "config", None)+ if getattr(config, "reference_compile", False) is not False:+ config.reference_compile = False+ self.graphs: dict[tuple[int, int, int], tuple[Any, dict[str, Any], Any]] = {}+ self.broken = False+ self.pool = torch.cuda.graph_pool_handle()+ self.pool_bytes = 0+ # One warm-up stream: the allocator caches blocks per stream, so a new stream per capture would strand them.+ self.stream = torch.cuda.Stream(agent.device)+ free, _ = torch.cuda.mem_get_info(agent.device)+ self.max_pool_bytes = min(1 << 30, free // 10)++ @staticmethod+ def _fit(n, sizes):+ return next((size for size in sizes if n <= size), None)++ def _forward(self, static):+ with _autocast(self.agent, cache_enabled = False):+ return _decision_logits(self.agent.model, *(static[name] for name in _INPUTS))++ def _capture(self, key):+ import torch++ rows, tokens, options = key+ device = self.agent.device+ static = {+ "input_ids": torch.zeros((rows, tokens), dtype = torch.long, device = device),+ "attention_mask": torch.ones((rows, tokens), dtype = torch.long, device = device),+ "marker_pos": torch.zeros((rows, options), dtype = torch.long, device = device),+ "marker_mask": torch.ones((rows, options), dtype = torch.bool, device = device),+ "qtype": torch.zeros((rows,), dtype = torch.long, device = device),+ }+ # Warm up off the default stream first, as torch.cuda.graph requires, so lazy init is not captured.+ self.stream.wait_stream(torch.cuda.current_stream(device))+ with torch.cuda.stream(self.stream):+ for _ in range(2):+ self._forward(static)+ torch.cuda.current_stream(device).wait_stream(self.stream)+ graph = torch.cuda.CUDAGraph()+ # thread_local: CUDA work on other Studio threads cannot break (or be broken by) this capture.+ with torch.cuda.graph(graph, pool = self.pool, capture_error_mode = "thread_local"):+ logits = self._forward(static)+ return graph, static, logits++ def run(self, batch):+ """Logits for ``batch`` from a graph replay, or None to run it eagerly."""+ import torch++ rows, tokens = batch["input_ids"].shape+ options = batch["marker_pos"].shape[1]+ key = (+ self._fit(rows, self.ROWS),+ self._fit(tokens, self.TOKENS),+ self._fit(options, self.OPTIONS),+ )+ if self.broken or None in key or key[0] * key[1] > _GRAPH_TOKENS:+ return None+ entry = self.graphs.get(key)+ if entry is None:+ if len(self.graphs) >= self.MAX_GRAPHS or self.pool_bytes >= self.max_pool_bytes:+ return None+ before = torch.cuda.memory_reserved(self.agent.device)+ try:+ entry = self._capture(key)+ except Exception:+ # e.g. transformers 4.x builds ModernBERT's sliding-window mask on the CPU every forward.+ logger.warning(+ "Laya cannot run in CUDA graphs here; running it eagerly", exc_info = True+ )+ self.broken = True+ return None+ self.graphs[key] = entry+ self.pool_bytes += max(0, torch.cuda.memory_reserved(self.agent.device) - before)+ if self.pool_bytes > self.max_pool_bytes:+ # Over budget: the pool is only released with every graph in it, so drop them all and stay eager.+ logger.warning("Laya CUDA graphs need more memory than allowed; running eagerly")+ self.graphs.clear()+ self.broken = True+ return None+ graph, static, logits = entry+ # Padding tokens are masked; padding rows repeat row 0 so every row has a real token to attend to.+ static["input_ids"].zero_()+ static["attention_mask"].zero_()+ static["marker_pos"].zero_()+ static["marker_mask"].zero_()+ for name in _INPUTS:+ source = batch[name]+ if source.dim() == 1:+ static[name][:rows].copy_(source)+ else:+ static[name][:rows, : source.shape[1]].copy_(source)+ if rows < key[0]:+ static[name][rows:] = static[name][:1]+ graph.replay()+ return logits[:rows, :options]++def _probabilities(agent, row, k: int, qtype: int):import numpy as npdiff --git a/studio/backend/tests/test_systemone.py b/studio/backend/tests/test_systemone.pyindex b37f28e09d4..f3404dd1e38 100644--- a/studio/backend/tests/test_systemone.py+++ b/studio/backend/tests/test_systemone.py@@ -1466,6 +1466,152 @@ def test_high_special_token_ids_still_skip_the_embedding_init(tmp_path):assert torch.equal(fast(*args)[0], reference(*args)[0])+def _tiny_decision_batch(+ torch,+ rows = 4,+ tokens = 40,+):+ generator = torch.Generator().manual_seed(0)+ lengths = [tokens, 23, 9, 31][:rows]+ ids = torch.randint(5, 300, (rows, tokens), generator = generator)+ mask = torch.zeros(rows, tokens, dtype = torch.long)+ for row, length in enumerate(lengths):+ mask[row, :length] = 1+ ids[row, length:] = 297+ counts = [2, 5, 1, 3][:rows]+ positions = torch.zeros(rows, 5, dtype = torch.long)+ markers = torch.zeros(rows, 5, dtype = torch.bool)+ for row, count in enumerate(counts):+ positions[row, :count] = torch.randperm(lengths[row], generator = generator)[:count]+ markers[row, :count] = True+ return ids, mask, positions, markers, torch.tensor([0, 1, 2, 1][:rows])++[email protected]("head_layers", [1, 2, 3])+def test_marker_head_matches_laya_forward(tmp_path, head_layers):+ torch = pytest.importorskip("torch")+ laya = laya_runtime._laya()+ encoder_dir, cfg = _tiny_laya_encoder(tmp_path)+ torch.manual_seed(0)+ model = laya.common.build_model(+ {**cfg, "head_layers": head_layers}, encoder_dir = encoder_dir+ ).eval()+ assert laya_runtime._marker_head(model)+ args = _tiny_decision_batch(torch)+ with torch.inference_mode():+ want = model(*args)[0]+ got = laya_runtime._decision_logits(model, *args)+ assert got.dtype == want.dtype == torch.float32+ torch.testing.assert_close(got, want, atol = 1e-5, rtol = 1e-4)+ # Without padding the mask is dropped altogether.+ unpadded = [arg[:1] for arg in args]+ torch.testing.assert_close(+ laya_runtime._decision_logits(model, *unpadded, padded = False),+ model(*unpadded)[0],+ atol = 1e-5,+ rtol = 1e-4,+ )+++def test_models_without_laya_head_keep_their_own_forward(tmp_path, monkeypatch):+ pytest.importorskip("torch")+ laya = laya_runtime._laya()+ encoder_dir, cfg = _tiny_laya_encoder(tmp_path)+ assert not laya_runtime._marker_head(+ laya.common.build_model({**cfg, "head_layers": 0}, encoder_dir = encoder_dir)+ )+ model = laya.common.build_model(cfg, encoder_dir = encoder_dir)+ model.scorer[2].approximate = "tanh"+ assert not laya_runtime._marker_head(model)+ model = laya.common.build_model(cfg, encoder_dir = encoder_dir)+ monkeypatch.setenv("UNSLOTH_SYSTEMONE_FAST", "0")+ assert not laya_runtime._marker_head(model)+++def test_collate_matches_laya():+ torch = pytest.importorskip("torch")+ collate_items = laya_runtime._laya().common.collate_items+ items = [+ {"ids": [2, 7, 9, 1], "markers": [1, 2], "qtype": 1},+ {"ids": [2, 8, 1], "markers": [1], "qtype": 0},+ {"ids": [2, 5, 6, 7, 8, 9, 1], "markers": [1, 3, 5, 6], "qtype": 2},+ ]+ want = collate_items([items], 297)+ got = laya_runtime._collate(items, 297)+ for name in laya_runtime._INPUTS:+ assert got[name].dtype == want[name].dtype and torch.equal(got[name], want[name]), name+++def test_cuda_graphs_replay_the_eager_logits(tmp_path, monkeypatch):+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("needs a CUDA GPU")+ laya = laya_runtime._laya()+ encoder_dir, cfg = _tiny_laya_encoder(tmp_path)+ torch.manual_seed(0)+ model = laya.common.build_model({**cfg, "head_layers": 2}, encoder_dir = encoder_dir).eval()+ agent = SimpleNamespace(model = model, device = torch.device("cpu"), dtype = torch.float32)+ monkeypatch.setattr(laya_runtime, "_precision", lambda device, fp16_checkpoint: (None, None))+ laya_runtime._place(agent, torch.device("cuda"), False)+ ids, mask, positions, markers, qtype = _tiny_decision_batch(torch, rows = 3, tokens = 40)+ batch = dict(zip(laya_runtime._INPUTS, (ids, mask, positions, markers, qtype)))+ graphed = laya_runtime._run_model(agent, batch).clone()+ graphs = agent.__dict__["_unsloth_graphs"]+ if graphs.broken:+ # transformers 4.x: ModernBERT cannot be captured, so the model runs eagerly with the same logits.+ import transformers++ assert int(transformers.__version__.split(".")[0]) < 5+ with torch.inference_mode():+ reference = model(*(value.cuda() for value in batch.values()))[0]+ torch.testing.assert_close(graphed, reference, atol = 1e-4, rtol = 1e-4)+ return+ assert list(graphs.graphs) == [(4, 64, 8)]+ # The same padded bucket run eagerly: rows 3 -> 4 repeat row 0, tokens 40 -> 64 and options 5 -> 8 are masked.+ padded = {+ "input_ids": torch.zeros(4, 64, dtype = torch.long),+ "attention_mask": torch.zeros(4, 64, dtype = torch.long),+ "marker_pos": torch.zeros(4, 8, dtype = torch.long),+ "marker_mask": torch.zeros(4, 8, dtype = torch.bool),+ "qtype": torch.zeros(4, dtype = torch.long),+ }+ for name, value in batch.items():+ if value.dim() == 1:+ padded[name][:3] = value+ else:+ padded[name][:3, : value.shape[1]] = value+ padded[name][3:] = padded[name][:1]+ with torch.inference_mode():+ eager = laya_runtime._decision_logits(+ model, *(padded[name].cuda() for name in laya_runtime._INPUTS)+ )+ assert torch.equal(graphed, eager[:3, :5])+ with torch.inference_mode():+ reference = model(*(value.cuda() for value in batch.values()))[0]+ torch.testing.assert_close(graphed, reference, atol = 1e-4, rtol = 1e-4)+ # A second request with other contents replays the same graph.+ batch["input_ids"] = torch.randint(5, 300, ids.shape)+ with torch.inference_mode():+ reference = model(*(value.cuda() for value in batch.values()))[0]+ torch.testing.assert_close(+ laya_runtime._run_model(agent, batch), reference, atol = 1e-4, rtol = 1e-4+ )+ assert len(graphs.graphs) == 1+ # Moving or recasting the model drops the graphs that point at its old weights.+ laya_runtime._place(agent, torch.device("cuda"), False)+ assert "_unsloth_graphs" not in agent.__dict__+ # Batches too large to be worth padding into a bucket run eagerly.+ assert (+ laya_runtime._CUDAGraphs(agent).run(+ {"input_ids": torch.zeros(16, 1024), "marker_pos": torch.zeros(16, 2)}+ )+ is None+ )+ monkeypatch.setenv("UNSLOTH_SYSTEMONE_CUDA_GRAPHS", "0")+ laya_runtime._run_model(agent, batch)+ assert "_unsloth_graphs" not in agent.__dict__++def test_encoders_that_keep_the_padding_id_match_laya(tmp_path):torch = pytest.importorskip("torch")from transformers import RobertaConfig@@ -1500,3 +1646,111 @@ def test_encoders_that_keep_the_padding_id_match_laya(tmp_path):)with torch.inference_mode():assert torch.equal(fast(*args)[0], reference(*args)[0])+++def _cuda_tiny_agent(+ tmp_path,+ monkeypatch,+ torch,+ capture = True,+):+ if not torch.cuda.is_available() or torch.version.hip:+ pytest.skip("needs a CUDA GPU")+ import transformers++ if capture and int(transformers.__version__.split(".")[0]) < 5:+ pytest.skip("transformers 4.x ModernBERT cannot be captured")+ laya = laya_runtime._laya()+ encoder_dir, cfg = _tiny_laya_encoder(tmp_path)+ torch.manual_seed(0)+ model = laya.common.build_model({**cfg, "head_layers": 2}, encoder_dir = encoder_dir).eval()+ agent = SimpleNamespace(model = model, device = torch.device("cpu"), dtype = torch.float32)+ monkeypatch.setattr(laya_runtime, "_precision", lambda device, fp16_checkpoint: (None, None))+ laya_runtime._place(agent, torch.device("cuda"), False)+ return agent+++def _padded_eager(model, batch, key, torch):+ rows = batch["input_ids"].shape[0]+ shapes = {+ "input_ids": key[:2],+ "attention_mask": key[:2],+ "marker_pos": (key[0], key[2]),+ "marker_mask": (key[0], key[2]),+ "qtype": key[:1],+ }+ padded = {name: torch.zeros(shape, dtype = batch[name].dtype) for name, shape in shapes.items()}+ for name, value in batch.items():+ if value.dim() == 1:+ padded[name][:rows] = value+ else:+ padded[name][:rows, : value.shape[1]] = value+ padded[name][rows:] = padded[name][:1]+ with torch.inference_mode():+ return laya_runtime._decision_logits(+ model, *(padded[name].cuda() for name in laya_runtime._INPUTS)+ )+++def test_cuda_graphs_sharing_a_pool_replay_in_any_order(tmp_path, monkeypatch):+ torch = pytest.importorskip("torch")+ agent = _cuda_tiny_agent(tmp_path, monkeypatch, torch)+ generator = torch.Generator().manual_seed(1)+ shapes = [(1, 20, 2), (3, 40, 5), (2, 100, 3), (6, 200, 9)]++ def request(rows, tokens, options):+ ids = torch.randint(5, 300, (rows, tokens), generator = generator)+ mask = torch.ones(rows, tokens, dtype = torch.long)+ mask[1:, tokens // 2 :] = 0+ positions = torch.randint(0, tokens // 2, (rows, options), generator = generator)+ markers = torch.ones(rows, options, dtype = torch.bool)+ markers[1:, options - 1] = False+ return dict(+ zip(+ laya_runtime._INPUTS,+ (ids, mask, positions, markers, torch.randint(0, 3, (rows,), generator = generator)),+ )+ )++ graphs = None+ # Capture order A B C D, then replay out of order, interleaving buckets.+ for index in [0, 1, 2, 3, 0, 2, 1, 3, 3, 0, 1, 0, 2]:+ batch = request(*shapes[index])+ got = laya_runtime._run_model(agent, batch).clone()+ graphs = agent.__dict__["_unsloth_graphs"]+ key = graphs.graphs and next(+ k+ for k in graphs.graphs+ if k[0] >= shapes[index][0] and k[1] >= shapes[index][1] and k[2] >= shapes[index][2]+ )+ want = _padded_eager(agent.model, batch, key, torch)+ assert torch.equal(got, want[: shapes[index][0], : shapes[index][2]]), (index, key)+ assert len(graphs.graphs) == 4 and not graphs.broken+++def test_cuda_graphs_over_the_memory_budget_run_eagerly(tmp_path, monkeypatch):+ torch = pytest.importorskip("torch")+ agent = _cuda_tiny_agent(tmp_path, monkeypatch, torch)+ batch = dict(zip(laya_runtime._INPUTS, _tiny_decision_batch(torch, rows = 3, tokens = 40)))+ graphs = agent.__dict__["_unsloth_graphs"] = laya_runtime._CUDAGraphs(agent)+ # The first capture alone takes more than the budget.+ graphs.max_pool_bytes = 1 << 20+ reserved = iter([0, 2 << 20])+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: next(reserved))+ logits = laya_runtime._run_model(agent, batch)+ assert graphs.broken and not graphs.graphs+ with torch.inference_mode():+ reference = agent.model(*(value.cuda() for value in batch.values()))[0]+ torch.testing.assert_close(logits, reference, atol = 1e-4, rtol = 1e-4)+++def test_rocm_never_builds_cuda_graphs(tmp_path, monkeypatch):+ torch = pytest.importorskip("torch")+ agent = _cuda_tiny_agent(tmp_path, monkeypatch, torch, capture = False)+ # transformers 4.x would torch.compile ModernBERT, which a faked ROCm version sends looking for HIP.+ agent.model.encoder.config.reference_compile = False+ monkeypatch.setattr(torch.version, "hip", "6.2")+ monkeypatch.setattr(laya_runtime, "_CUDAGraphs", None)+ batch = dict(zip(laya_runtime._INPUTS, _tiny_decision_batch(torch, rows = 3, tokens = 40)))+ laya_runtime._run_model(agent, batch)+ assert "_unsloth_graphs" not in agent.__dict__