Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run large Laya requests in token-budgeted chunks (#12271)

Daniel Han1bf970223114
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.py
index b222dd45f1d..4489a5f925d 100644
--- a/studio/backend/core/systemone/laya_runtime.py
+++ b/studio/backend/core/systemone/laya_runtime.py
@@ -718,14 +718,14 @@ def _forward(agent, items: list[dict[str, Any]]):
batch = _collate(items, agent.tok.pad_token_id)
if agent.device == "mlx":
try:
- logits = agent.model.logits(batch)
+ logits = _mlx_logits(agent, batch)
if not np.isfinite(logits).all():
import mlx.core as mx
if agent.dtype != mx.float32:
logger.warning("Laya overflowed in float16; continuing in float32")
agent.model.set_dtype(mx.float32)
agent.dtype = mx.float32
- logits = agent.model.logits(batch)
+ logits = _mlx_logits(agent, batch)
return logits, int(batch["attention_mask"].sum())
except RuntimeError as exc:
# MLX reports exhausted memory as "[malloc] Unable to allocate ..." or "Insufficient Memory".
@@ -773,7 +773,64 @@ def _forward(agent, items: list[dict[str, Any]]):
return logits.float().cpu().numpy(), int(batch["attention_mask"].sum())
+def _chunks(batch, budget: int):
+ """``batch`` as consecutive row slices of at most ``budget`` padded tokens, each trimmed to its longest row.
+
+ ``budget=None`` keeps the whole batch."""
+ rows, tokens = batch["input_ids"].shape
+ step = rows if budget is None else max(1, budget // tokens)
+ if rows <= step:
+ yield batch
+ return
+ for start in range(0, rows, step):
+ part = {name: batch[name][start : start + step] for name in _INPUTS}
+ keep = int(part["attention_mask"].sum(1).max())
+ part["input_ids"] = part["input_ids"][:, :keep]
+ part["attention_mask"] = part["attention_mask"][:, :keep]
+ yield part
+
+
+def _mlx_logits(agent, batch):
+ """One MLX request, forwarded in chunks and concatenated on the host."""
+ import numpy as np
+ return np.concatenate(
+ [agent.model.logits(part) for part in _chunks(batch, _chunk_budget("mlx"))]
+ )
+
+
def _run_model(agent, batch):
+ import torch
+
+ if not _marker_head(agent.model):
+ return _run_chunk(agent, batch)
+ budget = _chunk_budget(agent.device)
+ # cat copies, so no result is a view into a CUDA graph's output buffer, which its next replay overwrites.
+ return torch.cat([_run_chunk(agent, part) for part in _chunks(batch, budget)])
+
+
+def _chunk_budget(device):
+ """Padded tokens per forward, or None for one forward.
+
+ Splitting only saves memory: each extra forward costs a fixed overhead. On a CPU host memory is rarely the
+ limit and that overhead dominates (a 16-core Strix Halo ran the 64-question request 5x slower in 4096-token
+ chunks), so the CPU runs one forward. A GPU splits only past a share of its memory, so a large card keeps
+ one forward and its speed, and a small one still bounds its peak.
+ """
+ kind = getattr(device, "type", device)
+ budget = _CHUNK_TOKENS.get(kind)
+ if kind != "cuda" or budget is None:
+ return budget
+ import torch
+
+ try:
+ total = torch.cuda.get_device_properties(device).total_memory
+ except (RuntimeError, AssertionError):
+ return budget
+ per_token = _TOKEN_BYTES["hip" if torch.version.hip else "cuda"]
+ return max(budget, int(total * _GPU_SHARE) // per_token)
+
+
+def _run_chunk(agent, batch):
import os
import torch
@@ -806,6 +863,12 @@ def _run_model(agent, batch):
_INPUTS = ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")
+# Smallest padded rows x tokens per forward; a larger request runs as several (see _chunk_budget). None: one forward.
+_CHUNK_TOKENS = {"cuda": 16384, "mlx": 8192, "cpu": None}
+# Activation bytes per padded token of the worst request (B200 fp16 41 KiB, gfx1151 137 KiB), with headroom.
+_TOKEN_BYTES = {"cuda": 48 << 10, "hip": 160 << 10}
+# Share of the GPU's total memory one Decision API forward may take before the request is split.
+_GPU_SHARE = 0.05
# Padded rows x tokens above which a batch runs eagerly (B200: graphs win up to 8 x 1024, lose at 16 x 512).
_GRAPH_TOKENS = 8192
diff --git a/studio/backend/tests/test_systemone.py b/studio/backend/tests/test_systemone.py
index 1ebefeb16b6..bca08ffb8e5 100644
--- a/studio/backend/tests/test_systemone.py
+++ b/studio/backend/tests/test_systemone.py
@@ -1545,8 +1545,8 @@ def test_collate_matches_laya():
def test_cuda_graphs_replay_the_eager_logits(tmp_path, monkeypatch):
torch = pytest.importorskip("torch")
- if not torch.cuda.is_available():
- pytest.skip("needs a CUDA GPU")
+ if not torch.cuda.is_available() or torch.version.hip:
+ pytest.skip("needs a CUDA GPU (ROCm never builds graphs)")
laya = laya_runtime._laya()
encoder_dir, cfg = _tiny_laya_encoder(tmp_path)
torch.manual_seed(0)
@@ -1757,6 +1757,106 @@ def test_rocm_never_builds_cuda_graphs(tmp_path, monkeypatch):
assert "_unsloth_graphs" not in agent.__dict__
+def test_chunks_split_rows_and_trim_padding():
+ torch = pytest.importorskip("torch")
+ lengths = [10, 3, 7, 2, 9]
+ batch = laya_runtime._collate(
+ [{"ids": list(range(2, 2 + n)), "markers": [1], "qtype": 0} for n in lengths], 0
+ )
+ parts = list(laya_runtime._chunks(batch, 20))
+ # 20 // 10 tokens = 2 rows per forward, each trimmed to its own longest row.
+ assert [tuple(p["input_ids"].shape) for p in parts] == [(2, 10), (2, 7), (1, 9)]
+ for name in laya_runtime._INPUTS:
+ if name in ("input_ids", "attention_mask"):
+ continue
+ assert torch.equal(torch.cat([p[name] for p in parts]), batch[name])
+ assert list(laya_runtime._chunks(batch, 10**6)) == [batch]
+ # A row longer than the budget still runs, one row at a time.
+ assert [p["input_ids"].shape[0] for p in laya_runtime._chunks(batch, 4)] == [1] * 5
+
+
+def test_chunked_forward_matches_one_forward(tmp_path, monkeypatch):
+ torch = pytest.importorskip("torch")
+ laya = laya_runtime._laya()
+ encoder_dir, cfg = _tiny_laya_encoder(tmp_path)
+ torch.manual_seed(0)
+ model = laya.common.build_model({**cfg, "head_layers": 2}, encoder_dir = encoder_dir).eval()
+ agent = SimpleNamespace(model = model, device = torch.device("cpu"), dtype = torch.float32)
+ batch = dict(zip(laya_runtime._INPUTS, _tiny_decision_batch(torch)))
+ with torch.inference_mode():
+ want = model(*batch.values())[0]
+ monkeypatch.setitem(laya_runtime._CHUNK_TOKENS, "cpu", 40)
+ got = laya_runtime._run_model(agent, batch)
+ torch.testing.assert_close(got, want, atol = 1e-5, rtol = 1e-4)
+ calls = []
+ monkeypatch.setattr(
+ laya_runtime,
+ "_run_chunk",
+ lambda agent, part: calls.append(part)
+ or torch.zeros(part["input_ids"].shape[0], part["marker_pos"].shape[1]),
+ )
+ assert laya_runtime._run_model(agent, batch).shape == want.shape and len(calls) == 4
+ # The kill switch keeps laya's single forward.
+ calls.clear()
+ monkeypatch.setenv("UNSLOTH_SYSTEMONE_FAST", "0")
+ laya_runtime._run_model(agent, batch)
+ assert len(calls) == 1
+
+
+def test_mlx_requests_run_in_chunks(monkeypatch):
+ np = pytest.importorskip("numpy")
+ pytest.importorskip("torch")
+ seen = []
+
+ class Model:
+ def logits(self, batch):
+ seen.append(tuple(batch["input_ids"].shape))
+ return np.full(
+ (batch["input_ids"].shape[0], batch["marker_pos"].shape[1]), float(len(seen))
+ )
+
+ agent = SimpleNamespace(device = "mlx", model = Model(), tok = SimpleNamespace(pad_token_id = 0))
+ monkeypatch.setitem(laya_runtime._CHUNK_TOKENS, "mlx", 8)
+ items = [{"ids": [2, 5, 6, 1], "markers": [1, 2], "qtype": 0} for _ in range(5)]
+ logits, usage = laya_runtime._forward(agent, items)
+ assert seen == [(2, 4), (2, 4), (1, 4)] and usage == 20
+ assert logits[:, 0].tolist() == [1, 1, 2, 2, 3]
+
+
+def test_cuda_graph_results_survive_the_next_replay(tmp_path, monkeypatch):
+ torch = pytest.importorskip("torch")
+ agent = _cuda_tiny_agent(tmp_path, monkeypatch, torch)
+ first = dict(zip(laya_runtime._INPUTS, _tiny_decision_batch(torch, rows = 3, tokens = 40)))
+ second = {**first, "input_ids": torch.randint(5, 300, first["input_ids"].shape)}
+ a = laya_runtime._run_model(agent, first)
+ kept = a.clone()
+ laya_runtime._run_model(agent, second)
+ assert torch.equal(a, kept)
+
+
+def test_chunk_budget_splits_only_where_memory_is_short(monkeypatch):
+ torch = pytest.importorskip("torch")
+ # A CPU runs one forward: every extra forward costs a fixed overhead, and host RAM is rarely the limit.
+ assert laya_runtime._chunk_budget(torch.device("cpu")) is None
+ assert laya_runtime._chunk_budget("mlx") == laya_runtime._CHUNK_TOKENS["mlx"]
+ gib = 1 << 30
+ for total, hip, want in (
+ (8 * gib, None, 16384), # a small card keeps the floor
+ (24 * gib, None, 26214),
+ (180 * gib, None, 196608), # the 64 x 1024 request stays one forward
+ (96 * gib, "7.1", 31457), # ROCm activations are larger per token
+ ):
+ monkeypatch.setattr(
+ torch.cuda,
+ "get_device_properties",
+ lambda device, t = total: SimpleNamespace(total_memory = t),
+ )
+ monkeypatch.setattr(torch.version, "hip", hip)
+ assert laya_runtime._chunk_budget(torch.device("cuda")) == want
+ monkeypatch.setitem(laya_runtime._CHUNK_TOKENS, "cuda", None)
+ assert laya_runtime._chunk_budget(torch.device("cuda")) is None
+
+
def _fake_mlx(monkeypatch, **zoo):
mx = SimpleNamespace(float16 = "float16", float32 = "float32")
monkeypatch.setitem(sys.modules, "mlx", SimpleNamespace(core = mx))
@@ -1917,3 +2017,39 @@ def test_managed_accounts_can_add_studio_decisions_but_not_other_loopback():
run_as(alice, validate_mcp_address, f"http://127.0.0.1:8888{systemone.MCP_PATH}/")
with pytest.raises(HTTPException):
run_as(alice, validate_mcp_address, "http://127.0.0.1:8888/mcp")
+
+
+def test_mlx_overflow_in_a_later_chunk_reruns_the_whole_request_in_fp32(monkeypatch):
+ np = pytest.importorskip("numpy")
+ pytest.importorskip("torch")
+ _fake_mlx(monkeypatch)
+ seen = []
+
+ class Model:
+ dtype = "float16"
+
+ def set_dtype(self, dtype):
+ self.dtype = dtype
+
+ def logits(self, batch):
+ seen.append((self.dtype, batch["input_ids"].shape[0]))
+ rows = batch["input_ids"].shape[0]
+ bad = self.dtype == "float16" and len(seen) == 3
+ return np.full((rows, batch["marker_pos"].shape[1]), np.inf if bad else 1.0)
+
+ agent = SimpleNamespace(
+ device = "mlx", dtype = "float16", model = Model(), tok = SimpleNamespace(pad_token_id = 0)
+ )
+ monkeypatch.setitem(laya_runtime._CHUNK_TOKENS, "mlx", 8)
+ items = [{"ids": [2, 5, 6, 1], "markers": [1, 2], "qtype": 0} for _ in range(5)]
+ logits, _ = laya_runtime._forward(agent, items)
+ assert np.isfinite(logits).all() and logits.shape == (5, 2)
+ # Three fp16 chunks, the last overflowing, then all three again in fp32.
+ assert seen == [
+ ("float16", 2),
+ ("float16", 2),
+ ("float16", 1),
+ ("float32", 2),
+ ("float32", 2),
+ ("float32", 1),
+ ]