Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio int8 GEMM: stop the device probe from reserving 0.7 to 1.4 GB of spill memory (#12525)
Daniel Han331cfda9fb42
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex 4cb6b1bbb1a..111b98d42a1 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -241,12 +241,25 @@ def _launch(a: Any, w: Any, xs: Any, ws: Any, bias: Any, cfg: tuple) -> Any:return out+def _aligned(a: Any, w: Any) -> bool:+ """K, N, both row strides and both int8 base pointers on 16 bytes. Triton specialises on these, and an off-16 variant+ spills; the driver then reserves that local memory device-wide for the process, outside PyTorch's allocator."""+ return (+ a.shape[1] % 16 == 0+ and w.shape[0] % 16 == 0+ and a.stride(0) % 16 == 0+ and w.stride(0) % 16 == 0+ and a.data_ptr() % 16 == 0+ and w.data_ptr() % 16 == 0+ )++def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:- """Op body: the probed tile for this device, the stock epilogue if the launch fails."""+ """Op body: the probed tile for this device, else (launch failure, ``_aligned`` false) the stock epilogue."""_CALLS[0] += 1a = a if a.stride(-1) == 1 else a.contiguous()cfg = _DEVICE_CFG.get(a.device.index)- if cfg is not None:+ if cfg is not None and w.stride(-1) == 1 and _aligned(a, w):try:return _launch(a, w, xs, ws, bias, cfg)except Exception: # noqa: BLE001 - a failed launch keeps the stock math@@ -311,17 +324,21 @@ def device_config(index: int) -> Optional[tuple]:return cfg+# (M, N, K, bias, fp32 scales). Ragged N / K stay on 16: an off-16 probe compiles the spilling variant (see ``_aligned``).+_PROBE_SHAPES = (+ (257, 384, 512, False, False),+ (33, 208, 144, True, True),+ (300, 528, 1040, True, False),+)++def _probe(index: int, cfg: tuple) -> bool:import torchdev = torch.device("cuda", index)g = torch.Generator(device = "cpu").manual_seed(0)try:- for m, n, k, bias, xs32 in (- (257, 384, 512, False, False),- (33, 200, 136, True, True),- (300, 520, 1000, True, False),- ):+ for m, n, k, bias, xs32 in _PROBE_SHAPES:a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(torch.bfloat16)diff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.pyindex afb19cae355..5c1853f7560 100644--- a/studio/backend/tests/test_diffusion_int8_gemm.py+++ b/studio/backend/tests/test_diffusion_int8_gemm.py@@ -79,6 +79,48 @@ def test_dense_linear_is_not_eligible():assert g8._eligible(torch.nn.Linear(128, 128)) is None+def test_probe_shapes_stay_on_the_16_byte_grid():+ ns = [n for _m, n, _k, _b, _x in g8._PROBE_SHAPES]+ ks = [k for _m, _n, k, _b, _x in g8._PROBE_SHAPES]+ assert all(n % 16 == 0 for n in ns) and all(k % 16 == 0 for k in ks)+ for bk in (64, 128): # every shipped BLOCK_K still sees a ragged K+ assert any(k % bk for k in ks)+ assert any(n % 128 for n in ns) # and BLOCK_N a ragged N+++def test_misaligned_operands_take_the_stock_epilogue(monkeypatch):+ launched, stock = [], []+ monkeypatch.setattr(g8, "_launch", lambda a, w, *rest: launched.append(a.shape) or "fused")+ monkeypatch.setattr(g8, "reference", lambda a, w, *rest: stock.append(a.shape) or "stock")+ g8._DEVICE_CFG[None] = g8._FALLBACK_CONFIG # CPU tensors report device index None+ xs, ws = torch.ones(32), torch.ones(64)++ def run(a, w):+ return g8._run(a, w, xs, ws, None)++ assert (+ run(torch.zeros(32, 1024, dtype = torch.int8), torch.zeros(64, 1024, dtype = torch.int8))+ == "fused"+ )+ assert (+ run(torch.zeros(32, 1000, dtype = torch.int8), torch.zeros(64, 1000, dtype = torch.int8))+ == "stock"+ )+ flat = torch.zeros(64 * 1024 + 8, dtype = torch.int8)+ assert (+ run(flat[8 : 8 + 32 * 1024].view(32, 1024), torch.zeros(64, 1024, dtype = torch.int8))+ == "stock"+ )+ assert run(torch.zeros(32, 1024, dtype = torch.int8), flat[8:].view(64, 1024)) == "stock"+ wide = torch.zeros(64, 1032, dtype = torch.int8)[:, :1024] # row stride 1032: off 16+ assert run(torch.zeros(32, 1024, dtype = torch.int8), wide) == "stock"+ assert (+ run(torch.zeros(32, 1024, dtype = torch.int8), torch.zeros(72, 1024, dtype = torch.int8))+ == "stock"+ )+ assert len(launched) == 1 and len(stock) == 5++def _cuda_ready() -> bool:if not torch.cuda.is_available() or getattr(torch.version, "hip", None):return False@@ -125,7 +167,7 @@ def _int8_linear(k, n, bias, version):"m, k, n, bias, xs32",[(4096, 4096, 4096, False, False),- (1037, 520, 1400, True, False),+ (1037, 528, 1400, True, False),(17, 256, 1024, True, True),(300, 12288, 256, False, False),],@@ -195,6 +237,26 @@ def test_prequant_style_fp32_weight_scale(forced, bias):assert torch.equal(out, torch.compile(stock, fullgraph = True)(x))+@needs_cuda+def test_no_compiled_variant_spills_to_local_memory(forced):+ g = torch.Generator().manual_seed(0)+ for m, k, n in ((4096, 3072, 3072), (300, 1040, 528), (300, 1040, 520), (300, 1000, 384)):+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).cuda()+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).cuda()+ xs = (torch.rand(m, generator = g) * 0.02).to(torch.bfloat16).cuda()+ ws = (torch.rand(n, generator = g) * 0.002).to(torch.bfloat16).cuda()+ assert torch.equal(g8._op()(a, w, xs, ws, None), g8.reference(a, w, xs, ws, None))+ torch.cuda.synchronize()+ caches = getattr(g8._kernels().i8mm_dq, "device_caches", None)+ if caches is None:+ pytest.skip("Triton without per-device kernel caches")+ variants = [ck for cache in caches.values() for ck in cache[0].values()]+ assert variants+ for ck in variants:+ ck._init_handles()+ assert ck.n_spills == 0, ck.n_spills++@needs_cudadef test_small_m_and_misaligned_keep_stock(forced):lin = _int8_linear(1000, 768, False, None) # K off the 64 grid