Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio int8 GEMV: wide scale groups tile inside shared memory, a grouped miss no longer demotes per-row Linears (#17106)
Daniel Han9d8cd39e8e87
diff --git a/studio/backend/core/inference/diffusion_int8_gemv.py b/studio/backend/core/inference/diffusion_int8_gemv.pyindex ae69bedbf03..854a46a7f4f 100644--- a/studio/backend/core/inference/diffusion_int8_gemv.py+++ b/studio/backend/core/inference/diffusion_int8_gemv.py@@ -10,6 +10,7 @@ torchao's forward. Opaque to Inductor and M is the batch, so no recompiles or CUfrom __future__ import annotations+import contextlibimport osimport threadingfrom functools import lru_cache@@ -25,7 +26,9 @@ _OP_NAMESPACE = "unsloth_studio"_OP_NAME = "int8_wo_gemv"# (BLOCK_N, BLOCK_K, warps, stages), first that launches wins; the first needs ~40 KB smem, the second fits any card._CONFIGS = ((128, 128, 8, 3), (64, 64, 4, 2))+# (device index, scale group or 0 per row) -> index into _plans of the plan that launched_DEVICE_CFG: dict = {}+GROUP_TILES_ENV = "UNSLOTH_DIFFUSION_INT8_GEMV_GROUP_TILES"_LOCK = threading.Lock()# Engagement evidence for benches.COUNTS = {"gemv": 0, "fallback": 0}@@ -46,7 +49,8 @@ def _kernel() -> Any:except Exception: # noqa: BLE001 - no Triton: stock pathreturn None- @triton.jit+ # M and KO only bound masks+ @triton.jit(do_not_specialize = ["M", "KO"])def i8_wo_gemv(x_ptr,q_ptr,@@ -64,6 +68,7 @@ def _kernel() -> Any:stride_sn,stride_on,stride_ym,+ GS,HAS_BIAS: tl.constexpr, # bias presentHAS_OUT: tl.constexpr, # bf16 outlier columns presentGROUPED: tl.constexpr, # per-group weight scales@@ -86,7 +91,8 @@ def _kernel() -> Any:# int8 -> bf16 is exact and bf16 x bf16 products are exact in the float32 accumulatorpart = tl.dot(x, tl.trans(q.to(tl.bfloat16)), out_dtype = tl.float32)if GROUPED:- s = tl.load(s_ptr + rn * stride_sn + k0 // BLOCK_K, mask = n_ok, other = 0.0)+ # GS (the scale group) is a multiple of BLOCK_K+ s = tl.load(s_ptr + rn * stride_sn + k0 // GS, mask = n_ok, other = 0.0)acc += part * s.to(tl.float32)[None, :]else:acc += part@@ -150,6 +156,27 @@ def reference(return y.to(torch.bfloat16)+def _group_tiles() -> bool:+ return (os.environ.get(GROUP_TILES_ENV) or "").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def _plans(group: int) -> tuple:+ """Tiles to try in order; grouped: one group per K tile first (the old tile), then _CONFIGS' K tiles inside it."""+ if not group:+ return _CONFIGS+ whole = tuple((bn, group, w, st) for bn, _bk, w, st in _CONFIGS)+ if not _group_tiles():+ return whole+ return whole + tuple(+ (bn, bk, w, st) for bn, bk, w, st in _CONFIGS if group > bk and group % bk == 0+ )++def _launch(x: Any, q: Any, s2: Any, bias: Any, oidx: Any, ow: Any, cfg: tuple) -> Any:import torch@@ -157,37 +184,39 @@ def _launch(x: Any, q: Any, s2: Any, bias: Any, oidx: Any, ow: Any, cfg: tuple)block_n, block_k, warps, stages = cfgm, k = x.shapen, groups = s2.shape- if groups > 1:- block_k = k // groupsko = 0 if oidx is None else int(oidx.numel())y = torch.empty((m, n), device = x.device, dtype = torch.bfloat16)- fn[(triton.cdiv(n, block_n),)](- x,- q,- s2,- bias if bias is not None else y,- oidx if ko else y,- ow if ko else y,- y,- m,- n,- k,- ko,- x.stride(0),- q.stride(0),- s2.stride(0),- ow.stride(0) if ko else 0,- y.stride(0),- HAS_BIAS = bias is not None,- HAS_OUT = ko > 0,- GROUPED = groups > 1,- # tl.dot wants >= 16 rows: the masked pad rows load zeros and are never stored- BLOCK_M = 16,- BLOCK_N = block_n,- BLOCK_K = block_k,- num_warps = warps,- num_stages = stages,- )+ # Triton launches on the current device, not the tensor's+ other_card = x.is_cuda and x.device.index != torch.cuda.current_device()+ with torch.cuda.device(x.device) if other_card else contextlib.nullcontext():+ fn[(triton.cdiv(n, block_n),)](+ x,+ q,+ s2,+ bias if bias is not None else y,+ oidx if ko else y,+ ow if ko else y,+ y,+ m,+ n,+ k,+ ko,+ x.stride(0),+ q.stride(0),+ s2.stride(0),+ ow.stride(0) if ko else 0,+ y.stride(0),+ k // groups,+ HAS_BIAS = bias is not None,+ HAS_OUT = ko > 0,+ GROUPED = groups > 1,+ # tl.dot wants >= 16 rows: the masked pad rows load zeros and are never stored+ BLOCK_M = 16,+ BLOCK_N = block_n,+ BLOCK_K = block_k,+ num_warps = warps,+ num_stages = stages,+ )return y@@ -205,13 +234,15 @@ def _run(block_k = k // groups if groups > 1 else _CONFIGS[-1][1]ko = 0 if oidx is None else int(oidx.numel())index = x.device.index if x.is_cuda else None- start = _DEVICE_CFG.get(index, 0)+ key = (index, block_k if groups > 1 else 0)+ plans = _plans(key[1])+ start = _DEVICE_CFG.get(key, 0)if (index is Noneor _kernel() is None- or start >= len(_CONFIGS)+ or start >= len(plans)or k % block_k- or (groups == 1 and k % _CONFIGS[start][1])+ or (groups == 1 and k % plans[start][1])or q.stride(-1) != 1or block_k & (block_k - 1)or block_k < 16@@ -224,15 +255,15 @@ def _run(s2 = s.reshape(n, groups)if s2.stride(-1) != 1:s2 = s2.contiguous()- for pos in range(start, len(_CONFIGS)):+ for pos in range(start, len(plans)):try:- y = _launch(x, q, s2, bias, oidx, ow, _CONFIGS[pos])+ y = _launch(x, q, s2, bias, oidx, ow, plans[pos])except Exception: # noqa: BLE001 - a tile this card cannot hold (shared memory): the next, smaller onecontinue- _DEVICE_CFG[index] = pos+ _DEVICE_CFG[key] = posCOUNTS["gemv"] += 1return y- _DEVICE_CFG[index] = len(_CONFIGS)+ _DEVICE_CFG[key] = len(plans)COUNTS["fallback"] += 1return reference(x, q, s, bias, oidx, ow)diff --git a/studio/backend/tests/test_diffusion_int8_gemv_tiles.py b/studio/backend/tests/test_diffusion_int8_gemv_tiles.pynew file mode 100644index 00000000000..045b04ff7ed--- /dev/null+++ b/studio/backend/tests/test_diffusion_int8_gemv_tiles.py@@ -0,0 +1,109 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""int8 weight-only GEMV tile choice: a grouped layer tries its one-group tile first, then the configs' own K tiles+inside a wider group; the config that launched is remembered per (card, layout), so a grouped layer whose tile does+not fit never demotes the card's per-row Linears. CPU only: the launch is replaced by a recorder."""++from __future__ import annotations++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_int8_gemv as G # noqa: E402++_REAL_KERNEL = G._kernel++[email protected](autouse = True)+def _clean(monkeypatch):+ monkeypatch.delenv(G.GROUP_TILES_ENV, raising = False)+ monkeypatch.setattr(G, "_DEVICE_CFG", {})+ monkeypatch.setattr(G, "COUNTS", {"gemv": 0, "fallback": 0})+ monkeypatch.setattr(G, "_kernel", lambda: object())+++class _CudaX:+ """A CPU tensor that _run treats as living on cuda:0 (the launch itself is stubbed)."""++ def __init__(self, t):+ self.t, self.shape, self.is_cuda, self.device = t, t.shape, True, torch.device("cuda", 0)++ def stride(self, *dim):+ return self.t.stride(*dim)+++def _layer(n, k, group):+ q = torch.zeros(n, k, dtype = torch.int8)+ s = torch.ones(n, k // group if group else 1, dtype = torch.bfloat16)+ return _CudaX(torch.zeros(2, k, dtype = torch.bfloat16)), q, (s if group else s.reshape(-1))+++def _smem_limited(monkeypatch, fits):+ tried = []++ def launch(x, q, s2, bias, oidx, ow, plan):+ tried.append(plan)+ if not fits(plan):+ raise RuntimeError("OutOfResources: shared memory")+ return ("y", plan)++ monkeypatch.setattr(G, "_launch", launch)+ return tried+++def test_plans():+ assert G._plans(0) == G._CONFIGS+ assert G._plans(512) == ((128, 512, 8, 3), (64, 512, 4, 2), (128, 128, 8, 3), (64, 64, 4, 2))+ assert G._plans(128) == ((128, 128, 8, 3), (64, 128, 4, 2), (64, 64, 4, 2))+ assert G._plans(64) == ((128, 64, 8, 3), (64, 64, 4, 2))+++def test_kill_switch_keeps_one_group_tiles(monkeypatch):+ monkeypatch.setenv(G.GROUP_TILES_ENV, "0")+ assert G._plans(512) == ((128, 512, 8, 3), (64, 512, 4, 2))+++def test_wide_group_runs_split_tiles_and_leaves_per_row_alone(monkeypatch):+ # a T4-sized card: only tiles of at most 128 K columns fit+ tried = _smem_limited(monkeypatch, lambda plan: plan[1] <= 128)+ y = G._run(*_layer(256, 4096, 1024), None)+ assert y == ("y", (128, 128, 8, 3))+ assert tried == [(128, 1024, 8, 3), (64, 1024, 4, 2), (128, 128, 8, 3)]+ assert G._DEVICE_CFG == {(0, 1024): 2}+ tried.clear()+ assert G._run(*_layer(256, 4096, 0), None) == ("y", (128, 128, 8, 3))+ assert tried == [(128, 128, 8, 3)]+ assert G._DEVICE_CFG == {(0, 1024): 2, (0, 0): 0}+ assert G.COUNTS == {"gemv": 2, "fallback": 0}+++def test_a_group_that_fits_keeps_its_one_group_tile(monkeypatch):+ tried = _smem_limited(monkeypatch, lambda plan: True)+ assert G._run(*_layer(256, 4096, 512), None) == ("y", (128, 512, 8, 3))+ assert tried == [(128, 512, 8, 3)]+++def test_remembered_plan_is_reused(monkeypatch):+ tried = _smem_limited(monkeypatch, lambda plan: plan[0] == 64)+ G._run(*_layer(256, 4096, 512), None)+ tried.clear()+ G._run(*_layer(256, 4096, 512), None)+ assert tried == [(64, 512, 4, 2)]+++def test_nothing_fits_falls_back_for_that_layout_only(monkeypatch):+ _smem_limited(monkeypatch, lambda plan: plan[1] < 128 and plan[0] == 0)+ seen = []+ monkeypatch.setattr(G, "reference", lambda *a, **k: seen.append(1) or "ref")+ assert G._run(*_layer(256, 4096, 512), None) == "ref"+ assert G._DEVICE_CFG == {(0, 512): 4}+ assert G._DEVICE_CFG.get((0, 0), 0) == 0+++def test_rows_and_outliers_not_specialized():+ """M is the batch (1..MAX_ROWS) and KO the outlier count: neither may compile a binary of its own."""+ pytest.importorskip("triton")+ fn = _REAL_KERNEL()[1]+ assert [p.name for p in fn.params if p.do_not_specialize] == ["M", "KO"]