Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio int8: fused MLPs and v2 Linears quantize their input with the probed one-pass kernel (#17114)
Daniel Han86a41b05766c
diff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.pyindex c4b5499d04f..c4258b231ce 100644--- a/studio/backend/core/inference/diffusion_int8_fused.py+++ b/studio/backend/core/inference/diffusion_int8_fused.py@@ -16,6 +16,7 @@ from typing import Any, Optionalfrom .rocm_bf16 import is_rocm_torchINT8_FUSED_ENV = "UNSLOTH_DIFFUSION_INT8_FUSED"+MLP_FAST_ACT_QUANT_ENV = "UNSLOTH_DIFFUSION_INT8_MLP_FAST_ACT_QUANT"_MIN_TRITON = (3, 2)# torch._int_mm needs more than 16 rows; torchao's own safe_int_mm handles the rest, so small M keeps the stock path._MIN_ROWS = 17@@ -36,6 +37,8 @@ _RQ_HANDLE: Any = None_RQ_DEVICES: frozenset = frozenset()# UNSLOTH_DIFFUSION_INT8_FUSED=0 at the last install: int8_linear is then module(x)_LINEAR_OFF = False+# MLP_FAST_ACT_QUANT_ENV at the last install: a global, so traced forwards never read os.environ+_MLP_FAST_ACTQ = True# torchao's safe_int_mm home, resolved at install (outside any trace) for the same reason._INTMM_MODULE: Any = None# Marker on each patched module (no global registry: it would pin an unloaded transformer).@@ -825,6 +828,13 @@ def _fast_act_quant(x2d: Any, weight: Any) -> tuple:return _act_quant(x2d, weight)+def _mlp_act_quant(x2d: Any, weight: Any) -> tuple:+ """``_fast_act_quant`` for ``_plain_int8_weight`` weights, torchao's quant under the kill switch."""+ if _MLP_FAST_ACTQ:+ return _fast_act_quant(x2d, weight)+ return _act_quant(x2d, weight)++@lru_cache(maxsize = 8)def _device_ok(index: int) -> bool:"""CUDA NVIDIA, Triton new enough, toolchain present, kernel compiles and matches the reference once."""@@ -1098,7 +1108,7 @@ def _ff_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Anyor hidden_states.dtype != torch.bfloat16):return type(self).forward(self, hidden_states, *args, **kwargs)- xq, xs = _act_quant(x2d, proj.weight)+ xq, xs = _mlp_act_quant(x2d, proj.weight)c = _int_mm(xq.reshape(-1, xq.shape[-1]), proj.weight)q, s = _OP_HANDLE.gelu(c, xs.reshape(-1), proj.weight.scale.flatten(), proj.bias, None)y = _linear_from_q(q, s, down.weight, down.bias, hidden_states.dtype)@@ -1134,7 +1144,7 @@ def _flux_single_forward(residual = hidden_statesnorm_hidden_states, gate = self.norm(hidden_states, emb = temb)x2d = norm_hidden_states.reshape(rows, -1)- xq, xs = _act_quant(x2d, self.proj_mlp.weight)+ xq, xs = _mlp_act_quant(x2d, self.proj_mlp.weight)c = _int_mm(xq.reshape(rows, -1), self.proj_mlp.weight)joint_attention_kwargs = joint_attention_kwargs or {}attn_output = self.attn(@@ -1373,7 +1383,7 @@ def _swiglu_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) ->if in_group:xq, xs = _rotated_act_quant(x2d, in_group, fused_in.weight)else:- xq, xs = _act_quant(x2d, fused_in.weight)+ xq, xs = _mlp_act_quant(x2d, fused_in.weight)c = _int_mm(xq.reshape(-1, xq.shape[-1]), fused_in.weight)if down_group:ws = fused_in.weight.scale.flatten()@@ -1581,7 +1591,13 @@ def install(offload_active: bool = False,) -> int:"""Idempotent; returns the (candidate) count. Must run before the first compiled forward, which traces ``forward``."""- global _ACTQ_HANDLE, _ACTQ_DEVICES, _RQ_DEVICES, _LINEAR_OFF+ global _ACTQ_HANDLE, _ACTQ_DEVICES, _RQ_DEVICES, _LINEAR_OFF, _MLP_FAST_ACTQ+ _MLP_FAST_ACTQ = (os.environ.get(MLP_FAST_ACT_QUANT_ENV) or "").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )if int8_fused_disabled():# process-global: a previous load's probe must not outlive the kill switch_ACTQ_HANDLE = Nonediff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex 4e6ad042687..5036753c4bf 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -996,8 +996,8 @@ def _act_quant_v1(x2d: Any) -> tuple:def _act_quant_v2(x2d: Any, weight: Any) -> tuple:- from .diffusion_int8_fused import _act_quant- return _act_quant(x2d, weight)+ from .diffusion_int8_fused import _mlp_act_quant+ return _mlp_act_quant(x2d, weight)@lru_cache(maxsize = 4)diff --git a/studio/backend/tests/test_diffusion_int8_fused.py b/studio/backend/tests/test_diffusion_int8_fused.pyindex 388d0837749..873bc27ca5e 100644--- a/studio/backend/tests/test_diffusion_int8_fused.py+++ b/studio/backend/tests/test_diffusion_int8_fused.py@@ -5,6 +5,8 @@from __future__ import annotations+import textwrap+import pytestfrom core.inference import diffusion_int8_fused as fused@@ -1094,3 +1096,44 @@ def test_int8_linear_leaves_an_installed_int8_gemm_forward_alone(monkeypatch):lin.__dict__[g8._MARK] = object() # what diffusion_int8_gemm.install leaves on a swapped Linearlin.forward = lambda x: "int8_gemm forward"assert fused.int8_linear(lin, torch.randn(32, 8)) == "int8_gemm forward"+++def test_fused_mlps_and_v2_linears_quantize_through_the_probed_kernel():+ """The three fused MLP forwards and diffusion_int8_gemm's v2 Linears use the probed one-pass act quant (bit+ identical to torchao's where it engages), not torchao's own quant."""+ import ast+ import inspect++ from core.inference import diffusion_int8_gemm as g8++ for fn in (+ fused._ff_forward,+ fused._flux_single_forward,+ fused._swiglu_forward,+ g8._act_quant_v2,+ ):+ calls = {+ n.func.id+ for n in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(fn))))+ if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)+ }+ assert "_act_quant" not in calls, fn.__name__+ assert "_mlp_act_quant" in calls, fn.__name__+++def test_mlp_act_quant_kill_switch(monkeypatch):+ monkeypatch.setattr(fused, "_fast_act_quant", lambda x, w: "fast")+ monkeypatch.setattr(fused, "_act_quant", lambda x, w: "torchao")+ monkeypatch.delenv(fused.MLP_FAST_ACT_QUANT_ENV, raising = False)+ monkeypatch.setenv(+ fused.INT8_FUSED_ENV, "0"+ ) # install returns right after reading the switches+ fused.install(None)+ assert fused._mlp_act_quant(None, None) == "fast"+ monkeypatch.setenv(fused.MLP_FAST_ACT_QUANT_ENV, "0")+ fused.install(None)+ assert fused._mlp_act_quant(None, None) == "torchao"+ monkeypatch.delenv(fused.MLP_FAST_ACT_QUANT_ENV)+ monkeypatch.delenv(fused.INT8_FUSED_ENV)+ fused.install(None)+ assert fused._MLP_FAST_ACTQ is True