Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: resolve torchao's int8 GEMM home before the int8 fused install returns (#14771)

Daniel Han445599f3de51
diff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.py
index 908c14b2609..0e95d97389c 100644
--- a/studio/backend/core/inference/diffusion_int8_fused.py
+++ b/studio/backend/core/inference/diffusion_int8_fused.py
@@ -1063,6 +1063,20 @@ def _resolve_intmm() -> Any:
return None
+def ensure_intmm_module() -> Any:
+ """Resolve and cache ``_INTMM_MODULE`` (idempotent). Must run eagerly, outside any trace: ``int8_linear`` runs on
+ every int8 Linear of a fused attention processor whether or not ``_finalize`` installed anything (an offloaded or
+ not yet resident denoiser), and an ``importlib.import_module`` reached from ``_int_mm`` under ``torch.compile`` is
+ a graph break in every one of them."""
+ global _INTMM_MODULE
+ module = _INTMM_MODULE
+ if module is None:
+ module = _resolve_intmm()
+ if module is not None:
+ _INTMM_MODULE = module
+ return module
+
+
def _int_mm(a: Any, weight: Any) -> Any:
module = _INTMM_MODULE
if module is None: # a direct call before any install (tests): resolve eagerly
@@ -1562,6 +1576,11 @@ def install(
_LINEAR_OFF = True
return 0
_LINEAR_OFF = False
+ # Before the offload / residency returns: int8_linear (fused attention processors) runs either way.
+ try:
+ ensure_intmm_module()
+ except Exception: # noqa: BLE001 - _int_mm resolves on its own
+ pass
if transformer is None or offload_active:
return 0
if resident_cuda_device(transformer) is not None:
@@ -1612,7 +1631,9 @@ def _has_eligible(transformer: Any) -> bool:
def _finalize(transformer: Any, logger: Any = None) -> int:
- global _OP_HANDLE, _ACTQ_HANDLE, _ACTQ_DEVICES, _INTMM_MODULE
+ global _OP_HANDLE, _ACTQ_HANDLE, _ACTQ_DEVICES
+ # Before every early return below: an offloaded denoiser keeps int8_linear in its attention processors.
+ intmm = ensure_intmm_module()
dev = resident_cuda_device(transformer)
cards: tuple = ()
if dev is None:
@@ -1623,12 +1644,10 @@ def _finalize(transformer: Any, logger: Any = None) -> int:
# Before the probe: it compiles and launches both Triton kernels, pure latency for a bf16 / fp16 model.
if not _has_eligible(transformer):
return 0
- intmm = _resolve_intmm()
if (
intmm is None
): # an int8 GEMM home this file does not know: keep the stock path rather than fail the render
return 0
- _INTMM_MODULE = intmm
import torch
if cards:
diff --git a/studio/backend/core/inference/diffusion_qwenimage21_fused.py b/studio/backend/core/inference/diffusion_qwenimage21_fused.py
index b75e64047c6..39d9035aaf0 100644
--- a/studio/backend/core/inference/diffusion_qwenimage21_fused.py
+++ b/studio/backend/core/inference/diffusion_qwenimage21_fused.py
@@ -99,7 +99,9 @@ def _qkv_intact(attn: Any) -> Any:
def _make_prepare(stock: Any, mod: Any) -> Any:
import torch
- from .diffusion_int8_fused import int8_linear
+ from .diffusion_int8_fused import ensure_intmm_module, int8_linear
+
+ ensure_intmm_module() # resolved here, eagerly: an import under the trace is a graph break per Linear
def _qwenimage21_prepare_qkv(
attn, hidden_states, rotary_emb, layer_cache, kv_cache_mode, cache_write_slice
diff --git a/studio/backend/core/inference/diffusion_zimage_fused.py b/studio/backend/core/inference/diffusion_zimage_fused.py
index ea283ee7194..ed14c1bb064 100644
--- a/studio/backend/core/inference/diffusion_zimage_fused.py
+++ b/studio/backend/core/inference/diffusion_zimage_fused.py
@@ -82,7 +82,9 @@ def _make_call(stock: Any) -> Any:
import torch
# bound here, not inside the traced __call__
- from .diffusion_int8_fused import int8_linear
+ from .diffusion_int8_fused import ensure_intmm_module, int8_linear
+
+ ensure_intmm_module() # resolved here, eagerly: an import under the trace is a graph break per Linear
def __call__(
self,
diff --git a/studio/backend/tests/test_diffusion_int8_fused.py b/studio/backend/tests/test_diffusion_int8_fused.py
index 66e517cfc97..c55fb4b0183 100644
--- a/studio/backend/tests/test_diffusion_int8_fused.py
+++ b/studio/backend/tests/test_diffusion_int8_fused.py
@@ -112,6 +112,46 @@ def test_int_mm_prefers_the_released_home(monkeypatch):
assert fused._int_mm(torch.ones(4, 2, dtype = torch.int8), weight) == "old"
+def _intmm_graph_breaks(monkeypatch, prepare) -> int:
+ """Graph breaks dynamo counts in ``_int_mm`` after ``prepare`` ran with no int8 GEMM home cached yet."""
+ import sys
+ import types
+
+ home = types.ModuleType("torchao.kernel.intmm")
+ home.safe_int_mm = lambda a, b: a.float() @ b.float()
+ monkeypatch.setitem(sys.modules, "torchao.kernel.intmm", home)
+ monkeypatch.setattr(fused, "_INTMM_MODULE", None, raising = False)
+ prepare()
+ weight = types.SimpleNamespace(qdata = torch.ones(3, 2, dtype = torch.int8))
+
+ def fn(a):
+ return fused._int_mm(a, weight) * 2
+
+ torch._dynamo.reset()
+ try:
+ explained = torch._dynamo.explain(fn)(torch.ones(4, 2, dtype = torch.int8))
+ finally:
+ torch._dynamo.reset()
+ return explained.graph_break_count
+
+
+def test_int_mm_no_graph_break_after_offloaded_install(monkeypatch):
+ """An offloaded denoiser: install returns before _finalize, yet the fused attention processors still run
+ int8_linear on every int8 Linear. The GEMM home must already be cached, or each ``_int_mm`` under compile
+ imports it (``importlib.import_module`` is a dynamo graph break per Linear)."""
+ model = torch.nn.Linear(2, 3)
+ monkeypatch.setattr(fused, "int8_fused_disabled", lambda: False)
+ assert _intmm_graph_breaks(monkeypatch, lambda: fused.install(model, offload_active = True)) == 0
+
+
+def test_int_mm_no_graph_break_after_finalize_early_return(monkeypatch):
+ """_finalize on weights that are on no CUDA card (group offload, CPU) returns early; the GEMM home it resolves
+ must be cached before that return."""
+ model = torch.nn.Linear(2, 3)
+ assert fused.resident_cuda_device(model) is None
+ assert _intmm_graph_breaks(monkeypatch, lambda: fused._finalize(model)) == 0
+
+
@pytest.mark.parametrize(
"version, ok",
[