Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio int8 GEMM: keep sm8x GPUs on cuBLAS under Triton 3.8+, where every shipped tile spills (#17036)
Daniel Han711e29e9f59a
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex c67e5fa61e3..3e26248e44e 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -1,10 +1,10 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""int8 GEMM with torchao dequant epilogue in Triton, bf16 out; sm80/sm89/sm120 measured, other sm80+ behind a-bit-exact probe and a latency check (UNSLOTH_DIFFUSION_INT8_GEMM_UNLISTED=0 keeps them stock), sm100 / sm103 stock-(ROCm and CPU never reach it); Linear output bit-identical, compiled block not end to end;-sm120 loads its operands through TMA descriptors (UNSLOTH_DIFFUSION_INT8_GEMM_TMA=0 keeps pointer loads);+"""int8 GEMM with torchao dequant epilogue in Triton, bf16 out; sm80/sm89/sm120 measured (every sm8x stock on+Triton >= 3.8, whose int8 dot spills there), other sm80+ behind a bit-exact probe and a latency check+(UNSLOTH_DIFFUSION_INT8_GEMM_UNLISTED=0 keeps them stock), sm100 / sm103 stock (ROCm and CPU never reach it);+Linear output bit-identical, compiled block not end to end; sm120 loads its operands through TMA descriptors (UNSLOTH_DIFFUSION_INT8_GEMM_TMA=0 keeps pointer loads);UNSLOTH_DIFFUSION_INT8_GEMM=0 disables."""from __future__ import annotations@@ -137,6 +137,23 @@ def _triton_version_ok(version: Optional[str] = None) -> bool:return bool(match) and (int(match.group(1)), int(match.group(2))) >= _MIN_TRITON+@lru_cache(maxsize = None)+def _triton_at_least(major: int, minor: int) -> bool:+ try:+ import triton+ except Exception: # noqa: BLE001+ return False+ import re++ match = re.match(r"(\d+)\.(\d+)", str(triton.__version__))+ return bool(match) and (int(match.group(1)), int(match.group(2))) >= (major, minor)+++def _triton38_sm8x(cap: tuple) -> bool:+ """Triton >= 3.8 spills sm8x int8 tiles (triton-lang/triton#12486); even spill-free ones trail cuBLAS."""+ return cap[0] == 8 and _triton_at_least(3, 8)++def unlisted_mode() -> str:"""'off', 'force' (no latency check) or 'auto' for the sm80+ archs without an ``_ARCH_CONFIG`` tile."""gemm = int8_gemm_mode()@@ -161,7 +178,10 @@ def unlisted_wanted(capability: Optional[tuple], mode: Optional[str] = None) ->mode = unlisted_mode() if mode is None else modeif mode == "off" or not unlisted_arch(capability):return False- return mode == "force" or (int(capability[0]), int(capability[1])) not in _UNLISTED_SKIP+ if mode == "force":+ return True+ cap = (int(capability[0]), int(capability[1]))+ return cap not in _UNLISTED_SKIP and not _triton38_sm8x(cap)def arch_config(capability: Optional[tuple], mode: Optional[str] = None) -> Optional[tuple]:@@ -171,6 +191,8 @@ def arch_config(capability: Optional[tuple], mode: Optional[str] = None) -> Optireturn Nonecap = (int(capability[0]), int(capability[1]))cfg = _ARCH_CONFIG.get(cap)+ if mode == "auto" and _triton38_sm8x(cap):+ return Noneif cfg is None and mode == "force" and cap >= (8, 0):return _FALLBACK_CONFIGreturn cfgdiff --git a/studio/backend/tests/test_diffusion_int8_gemm.py b/studio/backend/tests/test_diffusion_int8_gemm.pyindex ac33a406ab4..808f53cc82a 100644--- a/studio/backend/tests/test_diffusion_int8_gemm.py+++ b/studio/backend/tests/test_diffusion_int8_gemm.py@@ -50,7 +50,8 @@ def _clean(monkeypatch):((12, 0), True),],)-def test_arch_gate_auto(cap, on):+def test_arch_gate_auto(monkeypatch, cap, on):+ monkeypatch.setattr(g8, "_triton_at_least", lambda major, minor: False) # the pre-3.8 tableassert (g8.arch_config(cap, "auto") is not None) is on@@ -68,6 +69,52 @@ def test_force_enables_unmeasured_sm80_plus_only(monkeypatch):assert g8.arch_config((7, 5)) is None[email protected]("newer", [False, True])+def test_triton38_keeps_every_sm8x_stock(monkeypatch, newer):+ monkeypatch.setattr(g8, "_triton_at_least", lambda major, minor: newer)+ for cap, cfg in g8._ARCH_CONFIG.items():+ stock = newer and cap[0] == 8+ assert g8.arch_config(cap, "auto") == (None if stock else cfg)+ assert g8.arch_config(cap, "force") == cfg # forcing still runs the kernel+ assert not g8.unlisted_arch(cap) and not g8.unlisted_wanted(cap, "auto")+ for cap in ((8, 6), (8, 7)): # unlisted sm8x: no probe / latency check either+ assert g8.unlisted_wanted(cap, "auto") is not newer+ assert g8.unlisted_wanted(cap, "force")+ for cap in ((9, 0), (12, 1)): # no spill there: unchanged+ assert g8.unlisted_wanted(cap, "auto")+ assert g8.arch_config((12, 0), "auto") == g8._ARCH_CONFIG[(12, 0)]+ assert g8._gates((8, 9))[0] is not newer+++ "version, newer", [("3.7.1", False), ("3.8.0", True), ("3.9.0+git1", True), ("4.0", True)]+)+def test_triton_at_least_parses_the_installed_version(monkeypatch, version, newer):+ import sys++ monkeypatch.setitem(sys.modules, "triton", types.SimpleNamespace(__version__ = version))+ g8._triton_at_least.cache_clear()+ try:+ assert g8._triton_at_least(3, 8) is newer+ finally:+ g8._triton_at_least.cache_clear()+++def test_triton38_runtime_turns_sm80_sm89_stock(monkeypatch):+ import sys++ monkeypatch.setitem(sys.modules, "triton", types.SimpleNamespace(__version__ = "3.8.0"))+ clear = getattr(getattr(g8, "_triton_at_least", None), "cache_clear", lambda: None)+ clear()+ try:+ assert g8.arch_config((8, 0), "auto") is None+ assert g8.arch_config((8, 9), "auto") is None+ assert not g8.unlisted_wanted((8, 6), "auto")+ assert g8.arch_config((12, 0), "auto") == g8._ARCH_CONFIG[(12, 0)]+ finally:+ clear()++def test_install_is_a_noop_when_killed(monkeypatch):monkeypatch.setenv(g8.INT8_GEMM_ENV, "0")lin = torch.nn.Linear(64, 64)diff --git a/studio/backend/tests/test_diffusion_int8_gemm_unlisted.py b/studio/backend/tests/test_diffusion_int8_gemm_unlisted.pyindex 1bdf4a7b274..d6e0240be46 100644--- a/studio/backend/tests/test_diffusion_int8_gemm_unlisted.py+++ b/studio/backend/tests/test_diffusion_int8_gemm_unlisted.py@@ -42,6 +42,8 @@ def _reset():def _clean(monkeypatch, tmp_path):for name in _ENVS:monkeypatch.delenv(name, raising = False)+ # pre-Triton-3.8 gate; 3.8+ is in test_diffusion_int8_gemm.py+ monkeypatch.setattr(g8, "_triton_at_least", lambda major, minor: False)monkeypatch.setattr(g8, "_race_cache_file", lambda: tmp_path / g8._RACE_CACHE_NAME)monkeypatch.setattr(g8, "_race_key", lambda index: f"card{index}")_reset()