Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run large Laya requests in token-budgeted chunks (#12271)
Daniel Han1bf970223114
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.pyindex b222dd45f1d..4489a5f925d 100644--- a/studio/backend/core/systemone/laya_runtime.py+++ b/studio/backend/core/systemone/laya_runtime.py@@ -718,14 +718,14 @@ def _forward(agent, items: list[dict[str, Any]]):batch = _collate(items, agent.tok.pad_token_id)if agent.device == "mlx":try:- logits = agent.model.logits(batch)+ logits = _mlx_logits(agent, batch)if not np.isfinite(logits).all():import mlx.core as mxif agent.dtype != mx.float32:logger.warning("Laya overflowed in float16; continuing in float32")agent.model.set_dtype(mx.float32)agent.dtype = mx.float32- logits = agent.model.logits(batch)+ logits = _mlx_logits(agent, batch)return logits, int(batch["attention_mask"].sum())except RuntimeError as exc:# MLX reports exhausted memory as "[malloc] Unable to allocate ..." or "Insufficient Memory".@@ -773,7 +773,64 @@ def _forward(agent, items: list[dict[str, Any]]):return logits.float().cpu().numpy(), int(batch["attention_mask"].sum())+def _chunks(batch, budget: int):+ """``batch`` as consecutive row slices of at most ``budget`` padded tokens, each trimmed to its longest row.++ ``budget=None`` keeps the whole batch."""+ rows, tokens = batch["input_ids"].shape+ step = rows if budget is None else max(1, budget // tokens)+ if rows <= step:+ yield batch+ return+ for start in range(0, rows, step):+ part = {name: batch[name][start : start + step] for name in _INPUTS}+ keep = int(part["attention_mask"].sum(1).max())+ part["input_ids"] = part["input_ids"][:, :keep]+ part["attention_mask"] = part["attention_mask"][:, :keep]+ yield part+++def _mlx_logits(agent, batch):+ """One MLX request, forwarded in chunks and concatenated on the host."""+ import numpy as np+ return np.concatenate(+ [agent.model.logits(part) for part in _chunks(batch, _chunk_budget("mlx"))]+ )++def _run_model(agent, batch):+ import torch++ if not _marker_head(agent.model):+ return _run_chunk(agent, batch)+ budget = _chunk_budget(agent.device)+ # cat copies, so no result is a view into a CUDA graph's output buffer, which its next replay overwrites.+ return torch.cat([_run_chunk(agent, part) for part in _chunks(batch, budget)])+++def _chunk_budget(device):+ """Padded tokens per forward, or None for one forward.++ Splitting only saves memory: each extra forward costs a fixed overhead. On a CPU host memory is rarely the+ limit and that overhead dominates (a 16-core Strix Halo ran the 64-question request 5x slower in 4096-token+ chunks), so the CPU runs one forward. A GPU splits only past a share of its memory, so a large card keeps+ one forward and its speed, and a small one still bounds its peak.+ """+ kind = getattr(device, "type", device)+ budget = _CHUNK_TOKENS.get(kind)+ if kind != "cuda" or budget is None:+ return budget+ import torch++ try:+ total = torch.cuda.get_device_properties(device).total_memory+ except (RuntimeError, AssertionError):+ return budget+ per_token = _TOKEN_BYTES["hip" if torch.version.hip else "cuda"]+ return max(budget, int(total * _GPU_SHARE) // per_token)+++def _run_chunk(agent, batch):import osimport torch@@ -806,6 +863,12 @@ def _run_model(agent, batch):_INPUTS = ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")+# Smallest padded rows x tokens per forward; a larger request runs as several (see _chunk_budget). None: one forward.+_CHUNK_TOKENS = {"cuda": 16384, "mlx": 8192, "cpu": None}+# Activation bytes per padded token of the worst request (B200 fp16 41 KiB, gfx1151 137 KiB), with headroom.+_TOKEN_BYTES = {"cuda": 48 << 10, "hip": 160 << 10}+# Share of the GPU's total memory one Decision API forward may take before the request is split.+_GPU_SHARE = 0.05# Padded rows x tokens above which a batch runs eagerly (B200: graphs win up to 8 x 1024, lose at 16 x 512)._GRAPH_TOKENS = 8192diff --git a/studio/backend/tests/test_systemone.py b/studio/backend/tests/test_systemone.pyindex 1ebefeb16b6..bca08ffb8e5 100644--- a/studio/backend/tests/test_systemone.py+++ b/studio/backend/tests/test_systemone.py@@ -1545,8 +1545,8 @@ def test_collate_matches_laya():def test_cuda_graphs_replay_the_eager_logits(tmp_path, monkeypatch):torch = pytest.importorskip("torch")- if not torch.cuda.is_available():- pytest.skip("needs a CUDA GPU")+ if not torch.cuda.is_available() or torch.version.hip:+ pytest.skip("needs a CUDA GPU (ROCm never builds graphs)")laya = laya_runtime._laya()encoder_dir, cfg = _tiny_laya_encoder(tmp_path)torch.manual_seed(0)@@ -1757,6 +1757,106 @@ def test_rocm_never_builds_cuda_graphs(tmp_path, monkeypatch):assert "_unsloth_graphs" not in agent.__dict__+def test_chunks_split_rows_and_trim_padding():+ torch = pytest.importorskip("torch")+ lengths = [10, 3, 7, 2, 9]+ batch = laya_runtime._collate(+ [{"ids": list(range(2, 2 + n)), "markers": [1], "qtype": 0} for n in lengths], 0+ )+ parts = list(laya_runtime._chunks(batch, 20))+ # 20 // 10 tokens = 2 rows per forward, each trimmed to its own longest row.+ assert [tuple(p["input_ids"].shape) for p in parts] == [(2, 10), (2, 7), (1, 9)]+ for name in laya_runtime._INPUTS:+ if name in ("input_ids", "attention_mask"):+ continue+ assert torch.equal(torch.cat([p[name] for p in parts]), batch[name])+ assert list(laya_runtime._chunks(batch, 10**6)) == [batch]+ # A row longer than the budget still runs, one row at a time.+ assert [p["input_ids"].shape[0] for p in laya_runtime._chunks(batch, 4)] == [1] * 5+++def test_chunked_forward_matches_one_forward(tmp_path, monkeypatch):+ torch = pytest.importorskip("torch")+ laya = laya_runtime._laya()+ encoder_dir, cfg = _tiny_laya_encoder(tmp_path)+ torch.manual_seed(0)+ model = laya.common.build_model({**cfg, "head_layers": 2}, encoder_dir = encoder_dir).eval()+ agent = SimpleNamespace(model = model, device = torch.device("cpu"), dtype = torch.float32)+ batch = dict(zip(laya_runtime._INPUTS, _tiny_decision_batch(torch)))+ with torch.inference_mode():+ want = model(*batch.values())[0]+ monkeypatch.setitem(laya_runtime._CHUNK_TOKENS, "cpu", 40)+ got = laya_runtime._run_model(agent, batch)+ torch.testing.assert_close(got, want, atol = 1e-5, rtol = 1e-4)+ calls = []+ monkeypatch.setattr(+ laya_runtime,+ "_run_chunk",+ lambda agent, part: calls.append(part)+ or torch.zeros(part["input_ids"].shape[0], part["marker_pos"].shape[1]),+ )+ assert laya_runtime._run_model(agent, batch).shape == want.shape and len(calls) == 4+ # The kill switch keeps laya's single forward.+ calls.clear()+ monkeypatch.setenv("UNSLOTH_SYSTEMONE_FAST", "0")+ laya_runtime._run_model(agent, batch)+ assert len(calls) == 1+++def test_mlx_requests_run_in_chunks(monkeypatch):+ np = pytest.importorskip("numpy")+ pytest.importorskip("torch")+ seen = []++ class Model:+ def logits(self, batch):+ seen.append(tuple(batch["input_ids"].shape))+ return np.full(+ (batch["input_ids"].shape[0], batch["marker_pos"].shape[1]), float(len(seen))+ )++ agent = SimpleNamespace(device = "mlx", model = Model(), tok = SimpleNamespace(pad_token_id = 0))+ monkeypatch.setitem(laya_runtime._CHUNK_TOKENS, "mlx", 8)+ items = [{"ids": [2, 5, 6, 1], "markers": [1, 2], "qtype": 0} for _ in range(5)]+ logits, usage = laya_runtime._forward(agent, items)+ assert seen == [(2, 4), (2, 4), (1, 4)] and usage == 20+ assert logits[:, 0].tolist() == [1, 1, 2, 2, 3]+++def test_cuda_graph_results_survive_the_next_replay(tmp_path, monkeypatch):+ torch = pytest.importorskip("torch")+ agent = _cuda_tiny_agent(tmp_path, monkeypatch, torch)+ first = dict(zip(laya_runtime._INPUTS, _tiny_decision_batch(torch, rows = 3, tokens = 40)))+ second = {**first, "input_ids": torch.randint(5, 300, first["input_ids"].shape)}+ a = laya_runtime._run_model(agent, first)+ kept = a.clone()+ laya_runtime._run_model(agent, second)+ assert torch.equal(a, kept)+++def test_chunk_budget_splits_only_where_memory_is_short(monkeypatch):+ torch = pytest.importorskip("torch")+ # A CPU runs one forward: every extra forward costs a fixed overhead, and host RAM is rarely the limit.+ assert laya_runtime._chunk_budget(torch.device("cpu")) is None+ assert laya_runtime._chunk_budget("mlx") == laya_runtime._CHUNK_TOKENS["mlx"]+ gib = 1 << 30+ for total, hip, want in (+ (8 * gib, None, 16384), # a small card keeps the floor+ (24 * gib, None, 26214),+ (180 * gib, None, 196608), # the 64 x 1024 request stays one forward+ (96 * gib, "7.1", 31457), # ROCm activations are larger per token+ ):+ monkeypatch.setattr(+ torch.cuda,+ "get_device_properties",+ lambda device, t = total: SimpleNamespace(total_memory = t),+ )+ monkeypatch.setattr(torch.version, "hip", hip)+ assert laya_runtime._chunk_budget(torch.device("cuda")) == want+ monkeypatch.setitem(laya_runtime._CHUNK_TOKENS, "cuda", None)+ assert laya_runtime._chunk_budget(torch.device("cuda")) is None++def _fake_mlx(monkeypatch, **zoo):mx = SimpleNamespace(float16 = "float16", float32 = "float32")monkeypatch.setitem(sys.modules, "mlx", SimpleNamespace(core = mx))@@ -1917,3 +2017,39 @@ def test_managed_accounts_can_add_studio_decisions_but_not_other_loopback():run_as(alice, validate_mcp_address, f"http://127.0.0.1:8888{systemone.MCP_PATH}/")with pytest.raises(HTTPException):run_as(alice, validate_mcp_address, "http://127.0.0.1:8888/mcp")+++def test_mlx_overflow_in_a_later_chunk_reruns_the_whole_request_in_fp32(monkeypatch):+ np = pytest.importorskip("numpy")+ pytest.importorskip("torch")+ _fake_mlx(monkeypatch)+ seen = []++ class Model:+ dtype = "float16"++ def set_dtype(self, dtype):+ self.dtype = dtype++ def logits(self, batch):+ seen.append((self.dtype, batch["input_ids"].shape[0]))+ rows = batch["input_ids"].shape[0]+ bad = self.dtype == "float16" and len(seen) == 3+ return np.full((rows, batch["marker_pos"].shape[1]), np.inf if bad else 1.0)++ agent = SimpleNamespace(+ device = "mlx", dtype = "float16", model = Model(), tok = SimpleNamespace(pad_token_id = 0)+ )+ monkeypatch.setitem(laya_runtime._CHUNK_TOKENS, "mlx", 8)+ items = [{"ids": [2, 5, 6, 1], "markers": [1, 2], "qtype": 0} for _ in range(5)]+ logits, _ = laya_runtime._forward(agent, items)+ assert np.isfinite(logits).all() and logits.shape == (5, 2)+ # Three fp16 chunks, the last overflowing, then all three again in fp32.+ assert seen == [+ ("float16", 2),+ ("float16", 2),+ ("float16", 1),+ ("float32", 2),+ ("float32", 2),+ ("float32", 1),+ ]