Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: resolve torchao's int8 GEMM home before the int8 fused install returns (#14771)
Daniel Han445599f3de51
diff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.pyindex 908c14b2609..0e95d97389c 100644--- a/studio/backend/core/inference/diffusion_int8_fused.py+++ b/studio/backend/core/inference/diffusion_int8_fused.py@@ -1063,6 +1063,20 @@ def _resolve_intmm() -> Any:return None+def ensure_intmm_module() -> Any:+ """Resolve and cache ``_INTMM_MODULE`` (idempotent). Must run eagerly, outside any trace: ``int8_linear`` runs on+ every int8 Linear of a fused attention processor whether or not ``_finalize`` installed anything (an offloaded or+ not yet resident denoiser), and an ``importlib.import_module`` reached from ``_int_mm`` under ``torch.compile`` is+ a graph break in every one of them."""+ global _INTMM_MODULE+ module = _INTMM_MODULE+ if module is None:+ module = _resolve_intmm()+ if module is not None:+ _INTMM_MODULE = module+ return module++def _int_mm(a: Any, weight: Any) -> Any:module = _INTMM_MODULEif module is None: # a direct call before any install (tests): resolve eagerly@@ -1562,6 +1576,11 @@ def install(_LINEAR_OFF = Truereturn 0_LINEAR_OFF = False+ # Before the offload / residency returns: int8_linear (fused attention processors) runs either way.+ try:+ ensure_intmm_module()+ except Exception: # noqa: BLE001 - _int_mm resolves on its own+ passif transformer is None or offload_active:return 0if resident_cuda_device(transformer) is not None:@@ -1612,7 +1631,9 @@ def _has_eligible(transformer: Any) -> bool:def _finalize(transformer: Any, logger: Any = None) -> int:- global _OP_HANDLE, _ACTQ_HANDLE, _ACTQ_DEVICES, _INTMM_MODULE+ global _OP_HANDLE, _ACTQ_HANDLE, _ACTQ_DEVICES+ # Before every early return below: an offloaded denoiser keeps int8_linear in its attention processors.+ intmm = ensure_intmm_module()dev = resident_cuda_device(transformer)cards: tuple = ()if dev is None:@@ -1623,12 +1644,10 @@ def _finalize(transformer: Any, logger: Any = None) -> int:# Before the probe: it compiles and launches both Triton kernels, pure latency for a bf16 / fp16 model.if not _has_eligible(transformer):return 0- intmm = _resolve_intmm()if (intmm is None): # an int8 GEMM home this file does not know: keep the stock path rather than fail the renderreturn 0- _INTMM_MODULE = intmmimport torchif cards:diff --git a/studio/backend/core/inference/diffusion_qwenimage21_fused.py b/studio/backend/core/inference/diffusion_qwenimage21_fused.pyindex b75e64047c6..39d9035aaf0 100644--- a/studio/backend/core/inference/diffusion_qwenimage21_fused.py+++ b/studio/backend/core/inference/diffusion_qwenimage21_fused.py@@ -99,7 +99,9 @@ def _qkv_intact(attn: Any) -> Any:def _make_prepare(stock: Any, mod: Any) -> Any:import torch- from .diffusion_int8_fused import int8_linear+ from .diffusion_int8_fused import ensure_intmm_module, int8_linear++ ensure_intmm_module() # resolved here, eagerly: an import under the trace is a graph break per Lineardef _qwenimage21_prepare_qkv(attn, hidden_states, rotary_emb, layer_cache, kv_cache_mode, cache_write_slicediff --git a/studio/backend/core/inference/diffusion_zimage_fused.py b/studio/backend/core/inference/diffusion_zimage_fused.pyindex ea283ee7194..ed14c1bb064 100644--- a/studio/backend/core/inference/diffusion_zimage_fused.py+++ b/studio/backend/core/inference/diffusion_zimage_fused.py@@ -82,7 +82,9 @@ def _make_call(stock: Any) -> Any:import torch# bound here, not inside the traced __call__- from .diffusion_int8_fused import int8_linear+ from .diffusion_int8_fused import ensure_intmm_module, int8_linear++ ensure_intmm_module() # resolved here, eagerly: an import under the trace is a graph break per Lineardef __call__(self,diff --git a/studio/backend/tests/test_diffusion_int8_fused.py b/studio/backend/tests/test_diffusion_int8_fused.pyindex 66e517cfc97..c55fb4b0183 100644--- a/studio/backend/tests/test_diffusion_int8_fused.py+++ b/studio/backend/tests/test_diffusion_int8_fused.py@@ -112,6 +112,46 @@ def test_int_mm_prefers_the_released_home(monkeypatch):assert fused._int_mm(torch.ones(4, 2, dtype = torch.int8), weight) == "old"+def _intmm_graph_breaks(monkeypatch, prepare) -> int:+ """Graph breaks dynamo counts in ``_int_mm`` after ``prepare`` ran with no int8 GEMM home cached yet."""+ import sys+ import types++ home = types.ModuleType("torchao.kernel.intmm")+ home.safe_int_mm = lambda a, b: a.float() @ b.float()+ monkeypatch.setitem(sys.modules, "torchao.kernel.intmm", home)+ monkeypatch.setattr(fused, "_INTMM_MODULE", None, raising = False)+ prepare()+ weight = types.SimpleNamespace(qdata = torch.ones(3, 2, dtype = torch.int8))++ def fn(a):+ return fused._int_mm(a, weight) * 2++ torch._dynamo.reset()+ try:+ explained = torch._dynamo.explain(fn)(torch.ones(4, 2, dtype = torch.int8))+ finally:+ torch._dynamo.reset()+ return explained.graph_break_count+++def test_int_mm_no_graph_break_after_offloaded_install(monkeypatch):+ """An offloaded denoiser: install returns before _finalize, yet the fused attention processors still run+ int8_linear on every int8 Linear. The GEMM home must already be cached, or each ``_int_mm`` under compile+ imports it (``importlib.import_module`` is a dynamo graph break per Linear)."""+ model = torch.nn.Linear(2, 3)+ monkeypatch.setattr(fused, "int8_fused_disabled", lambda: False)+ assert _intmm_graph_breaks(monkeypatch, lambda: fused.install(model, offload_active = True)) == 0+++def test_int_mm_no_graph_break_after_finalize_early_return(monkeypatch):+ """_finalize on weights that are on no CUDA card (group offload, CPU) returns early; the GEMM home it resolves+ must be cached before that return."""+ model = torch.nn.Linear(2, 3)+ assert fused.resident_cuda_device(model) is None+ assert _intmm_graph_breaks(monkeypatch, lambda: fused._finalize(model)) == 0++@pytest.mark.parametrize("version, ok",[