Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio int8: fused MLPs and v2 Linears quantize their input with the probed one-pass kernel (#17114)

Daniel Han86a41b05766c
diff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.py
index c4b5499d04f..c4258b231ce 100644
--- a/studio/backend/core/inference/diffusion_int8_fused.py
+++ b/studio/backend/core/inference/diffusion_int8_fused.py
@@ -16,6 +16,7 @@ from typing import Any, Optional
from .rocm_bf16 import is_rocm_torch
INT8_FUSED_ENV = "UNSLOTH_DIFFUSION_INT8_FUSED"
+MLP_FAST_ACT_QUANT_ENV = "UNSLOTH_DIFFUSION_INT8_MLP_FAST_ACT_QUANT"
_MIN_TRITON = (3, 2)
# torch._int_mm needs more than 16 rows; torchao's own safe_int_mm handles the rest, so small M keeps the stock path.
_MIN_ROWS = 17
@@ -36,6 +37,8 @@ _RQ_HANDLE: Any = None
_RQ_DEVICES: frozenset = frozenset()
# UNSLOTH_DIFFUSION_INT8_FUSED=0 at the last install: int8_linear is then module(x)
_LINEAR_OFF = False
+# MLP_FAST_ACT_QUANT_ENV at the last install: a global, so traced forwards never read os.environ
+_MLP_FAST_ACTQ = True
# torchao's safe_int_mm home, resolved at install (outside any trace) for the same reason.
_INTMM_MODULE: Any = None
# Marker on each patched module (no global registry: it would pin an unloaded transformer).
@@ -825,6 +828,13 @@ def _fast_act_quant(x2d: Any, weight: Any) -> tuple:
return _act_quant(x2d, weight)
+def _mlp_act_quant(x2d: Any, weight: Any) -> tuple:
+ """``_fast_act_quant`` for ``_plain_int8_weight`` weights, torchao's quant under the kill switch."""
+ if _MLP_FAST_ACTQ:
+ return _fast_act_quant(x2d, weight)
+ return _act_quant(x2d, weight)
+
+
@lru_cache(maxsize = 8)
def _device_ok(index: int) -> bool:
"""CUDA NVIDIA, Triton new enough, toolchain present, kernel compiles and matches the reference once."""
@@ -1098,7 +1108,7 @@ def _ff_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) -> Any
or hidden_states.dtype != torch.bfloat16
):
return type(self).forward(self, hidden_states, *args, **kwargs)
- xq, xs = _act_quant(x2d, proj.weight)
+ xq, xs = _mlp_act_quant(x2d, proj.weight)
c = _int_mm(xq.reshape(-1, xq.shape[-1]), proj.weight)
q, s = _OP_HANDLE.gelu(c, xs.reshape(-1), proj.weight.scale.flatten(), proj.bias, None)
y = _linear_from_q(q, s, down.weight, down.bias, hidden_states.dtype)
@@ -1134,7 +1144,7 @@ def _flux_single_forward(
residual = hidden_states
norm_hidden_states, gate = self.norm(hidden_states, emb = temb)
x2d = norm_hidden_states.reshape(rows, -1)
- xq, xs = _act_quant(x2d, self.proj_mlp.weight)
+ xq, xs = _mlp_act_quant(x2d, self.proj_mlp.weight)
c = _int_mm(xq.reshape(rows, -1), self.proj_mlp.weight)
joint_attention_kwargs = joint_attention_kwargs or {}
attn_output = self.attn(
@@ -1373,7 +1383,7 @@ def _swiglu_forward(self: Any, hidden_states: Any, *args: Any, **kwargs: Any) ->
if in_group:
xq, xs = _rotated_act_quant(x2d, in_group, fused_in.weight)
else:
- xq, xs = _act_quant(x2d, fused_in.weight)
+ xq, xs = _mlp_act_quant(x2d, fused_in.weight)
c = _int_mm(xq.reshape(-1, xq.shape[-1]), fused_in.weight)
if down_group:
ws = fused_in.weight.scale.flatten()
@@ -1581,7 +1591,13 @@ def install(
offload_active: bool = False,
) -> int:
"""Idempotent; returns the (candidate) count. Must run before the first compiled forward, which traces ``forward``."""
- global _ACTQ_HANDLE, _ACTQ_DEVICES, _RQ_DEVICES, _LINEAR_OFF
+ global _ACTQ_HANDLE, _ACTQ_DEVICES, _RQ_DEVICES, _LINEAR_OFF, _MLP_FAST_ACTQ
+ _MLP_FAST_ACTQ = (os.environ.get(MLP_FAST_ACT_QUANT_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
if int8_fused_disabled():
# process-global: a previous load's probe must not outlive the kill switch
_ACTQ_HANDLE = None
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index 4e6ad042687..5036753c4bf 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -996,8 +996,8 @@ def _act_quant_v1(x2d: Any) -> tuple:
def _act_quant_v2(x2d: Any, weight: Any) -> tuple:
- from .diffusion_int8_fused import _act_quant
- return _act_quant(x2d, weight)
+ from .diffusion_int8_fused import _mlp_act_quant
+ return _mlp_act_quant(x2d, weight)
@lru_cache(maxsize = 4)
diff --git a/studio/backend/tests/test_diffusion_int8_fused.py b/studio/backend/tests/test_diffusion_int8_fused.py
index 388d0837749..873bc27ca5e 100644
--- a/studio/backend/tests/test_diffusion_int8_fused.py
+++ b/studio/backend/tests/test_diffusion_int8_fused.py
@@ -5,6 +5,8 @@
from __future__ import annotations
+import textwrap
+
import pytest
from core.inference import diffusion_int8_fused as fused
@@ -1094,3 +1096,44 @@ def test_int8_linear_leaves_an_installed_int8_gemm_forward_alone(monkeypatch):
lin.__dict__[g8._MARK] = object() # what diffusion_int8_gemm.install leaves on a swapped Linear
lin.forward = lambda x: "int8_gemm forward"
assert fused.int8_linear(lin, torch.randn(32, 8)) == "int8_gemm forward"
+
+
+def test_fused_mlps_and_v2_linears_quantize_through_the_probed_kernel():
+ """The three fused MLP forwards and diffusion_int8_gemm's v2 Linears use the probed one-pass act quant (bit
+ identical to torchao's where it engages), not torchao's own quant."""
+ import ast
+ import inspect
+
+ from core.inference import diffusion_int8_gemm as g8
+
+ for fn in (
+ fused._ff_forward,
+ fused._flux_single_forward,
+ fused._swiglu_forward,
+ g8._act_quant_v2,
+ ):
+ calls = {
+ n.func.id
+ for n in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(fn))))
+ if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
+ }
+ assert "_act_quant" not in calls, fn.__name__
+ assert "_mlp_act_quant" in calls, fn.__name__
+
+
+def test_mlp_act_quant_kill_switch(monkeypatch):
+ monkeypatch.setattr(fused, "_fast_act_quant", lambda x, w: "fast")
+ monkeypatch.setattr(fused, "_act_quant", lambda x, w: "torchao")
+ monkeypatch.delenv(fused.MLP_FAST_ACT_QUANT_ENV, raising = False)
+ monkeypatch.setenv(
+ fused.INT8_FUSED_ENV, "0"
+ ) # install returns right after reading the switches
+ fused.install(None)
+ assert fused._mlp_act_quant(None, None) == "fast"
+ monkeypatch.setenv(fused.MLP_FAST_ACT_QUANT_ENV, "0")
+ fused.install(None)
+ assert fused._mlp_act_quant(None, None) == "torchao"
+ monkeypatch.delenv(fused.MLP_FAST_ACT_QUANT_ENV)
+ monkeypatch.delenv(fused.INT8_FUSED_ENV)
+ fused.install(None)
+ assert fused._MLP_FAST_ACTQ is True