Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio int8 GEMM: stop the device probe from reserving 0.7 to 1.4 GB of spill memory (#12525)

Daniel Han331cfda9fb42
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index 4cb6b1bbb1a..111b98d42a1 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -241,12 +241,25 @@ def _launch(a: Any, w: Any, xs: Any, ws: Any, bias: Any, cfg: tuple) -> Any:
return out
+def _aligned(a: Any, w: Any) -> bool:
+ """K, N, both row strides and both int8 base pointers on 16 bytes. Triton specialises on these, and an off-16 variant
+ spills; the driver then reserves that local memory device-wide for the process, outside PyTorch's allocator."""
+ return (
+ a.shape[1] % 16 == 0
+ and w.shape[0] % 16 == 0
+ and a.stride(0) % 16 == 0
+ and w.stride(0) % 16 == 0
+ and a.data_ptr() % 16 == 0
+ and w.data_ptr() % 16 == 0
+ )
+
+
def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
- """Op body: the probed tile for this device, the stock epilogue if the launch fails."""
+ """Op body: the probed tile for this device, else (launch failure, ``_aligned`` false) the stock epilogue."""
_CALLS[0] += 1
a = a if a.stride(-1) == 1 else a.contiguous()
cfg = _DEVICE_CFG.get(a.device.index)
- if cfg is not None:
+ if cfg is not None and w.stride(-1) == 1 and _aligned(a, w):
try:
return _launch(a, w, xs, ws, bias, cfg)
except Exception: # noqa: BLE001 - a failed launch keeps the stock math
@@ -311,17 +324,21 @@ def device_config(index: int) -> Optional[tuple]:
return cfg
+# (M, N, K, bias, fp32 scales). Ragged N / K stay on 16: an off-16 probe compiles the spilling variant (see ``_aligned``).
+_PROBE_SHAPES = (
+ (257, 384, 512, False, False),
+ (33, 208, 144, True, True),
+ (300, 528, 1040, True, False),
+)
+
+
def _probe(index: int, cfg: tuple) -> bool:
import torch
dev = torch.device("cuda", index)
g = torch.Generator(device = "cpu").manual_seed(0)
try:
- for m, n, k, bias, xs32 in (
- (257, 384, 512, False, False),
- (33, 200, 136, True, True),
- (300, 520, 1000, True, False),
- ):
+ for m, n, k, bias, xs32 in _PROBE_SHAPES:
a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)
w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)
xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(torch.bfloat16)
diff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.py
index afb19cae355..5c1853f7560 100644
--- a/studio/backend/tests/test_diffusion_int8_gemm.py
+++ b/studio/backend/tests/test_diffusion_int8_gemm.py
@@ -79,6 +79,48 @@ def test_dense_linear_is_not_eligible():
assert g8._eligible(torch.nn.Linear(128, 128)) is None
+def test_probe_shapes_stay_on_the_16_byte_grid():
+ ns = [n for _m, n, _k, _b, _x in g8._PROBE_SHAPES]
+ ks = [k for _m, _n, k, _b, _x in g8._PROBE_SHAPES]
+ assert all(n % 16 == 0 for n in ns) and all(k % 16 == 0 for k in ks)
+ for bk in (64, 128): # every shipped BLOCK_K still sees a ragged K
+ assert any(k % bk for k in ks)
+ assert any(n % 128 for n in ns) # and BLOCK_N a ragged N
+
+
+def test_misaligned_operands_take_the_stock_epilogue(monkeypatch):
+ launched, stock = [], []
+ monkeypatch.setattr(g8, "_launch", lambda a, w, *rest: launched.append(a.shape) or "fused")
+ monkeypatch.setattr(g8, "reference", lambda a, w, *rest: stock.append(a.shape) or "stock")
+ g8._DEVICE_CFG[None] = g8._FALLBACK_CONFIG # CPU tensors report device index None
+ xs, ws = torch.ones(32), torch.ones(64)
+
+ def run(a, w):
+ return g8._run(a, w, xs, ws, None)
+
+ assert (
+ run(torch.zeros(32, 1024, dtype = torch.int8), torch.zeros(64, 1024, dtype = torch.int8))
+ == "fused"
+ )
+ assert (
+ run(torch.zeros(32, 1000, dtype = torch.int8), torch.zeros(64, 1000, dtype = torch.int8))
+ == "stock"
+ )
+ flat = torch.zeros(64 * 1024 + 8, dtype = torch.int8)
+ assert (
+ run(flat[8 : 8 + 32 * 1024].view(32, 1024), torch.zeros(64, 1024, dtype = torch.int8))
+ == "stock"
+ )
+ assert run(torch.zeros(32, 1024, dtype = torch.int8), flat[8:].view(64, 1024)) == "stock"
+ wide = torch.zeros(64, 1032, dtype = torch.int8)[:, :1024] # row stride 1032: off 16
+ assert run(torch.zeros(32, 1024, dtype = torch.int8), wide) == "stock"
+ assert (
+ run(torch.zeros(32, 1024, dtype = torch.int8), torch.zeros(72, 1024, dtype = torch.int8))
+ == "stock"
+ )
+ assert len(launched) == 1 and len(stock) == 5
+
+
def _cuda_ready() -> bool:
if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
return False
@@ -125,7 +167,7 @@ def _int8_linear(k, n, bias, version):
"m, k, n, bias, xs32",
[
(4096, 4096, 4096, False, False),
- (1037, 520, 1400, True, False),
+ (1037, 528, 1400, True, False),
(17, 256, 1024, True, True),
(300, 12288, 256, False, False),
],
@@ -195,6 +237,26 @@ def test_prequant_style_fp32_weight_scale(forced, bias):
assert torch.equal(out, torch.compile(stock, fullgraph = True)(x))
+@needs_cuda
+def test_no_compiled_variant_spills_to_local_memory(forced):
+ g = torch.Generator().manual_seed(0)
+ for m, k, n in ((4096, 3072, 3072), (300, 1040, 528), (300, 1040, 520), (300, 1000, 384)):
+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).cuda()
+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).cuda()
+ xs = (torch.rand(m, generator = g) * 0.02).to(torch.bfloat16).cuda()
+ ws = (torch.rand(n, generator = g) * 0.002).to(torch.bfloat16).cuda()
+ assert torch.equal(g8._op()(a, w, xs, ws, None), g8.reference(a, w, xs, ws, None))
+ torch.cuda.synchronize()
+ caches = getattr(g8._kernels().i8mm_dq, "device_caches", None)
+ if caches is None:
+ pytest.skip("Triton without per-device kernel caches")
+ variants = [ck for cache in caches.values() for ck in cache[0].values()]
+ assert variants
+ for ck in variants:
+ ck._init_handles()
+ assert ck.n_spills == 0, ck.n_spills
+
+
@needs_cuda
def test_small_m_and_misaligned_keep_stock(forced):
lin = _int8_linear(1000, 768, False, None) # K off the 64 grid