Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep torchao quantisation from mutating inductor config, and make its int8 GEMM capture-safe (#10525)
Daniel Hanff9206a03acc
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 64f1f8b6ee8..60cd1cfb8e5 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -106,6 +106,7 @@ from .diffusion_memory import (snapshot_device_memory,unified_memory_shortfall_message,)+from .diffusion_torchao_patches import install_torchao_int_mm_patchfrom .diffusion_speed import (SPEED_DEFAULT,SPEED_MAX,@@ -193,6 +194,7 @@ logger = get_logger(__name__)# diffusers imports xformers on sight, its quantizers torchao.install_xformers_windows_rocm_stub()install_torchao_windows_rocm_stub()+install_torchao_int_mm_patch()# "gguf" and "single_file" take companions from the base repo; "pipeline" is a full diffusers repo.@@ -4324,6 +4326,7 @@ class DiffusionBackend:and compile_eligible(target, is_gguf = False, family = fam))# Speed optims run BEFORE placement, so snapshot the global backend flags first for unload restore.+ # The dense transformer quant above builds quiet configs, so it mutated none of these flags.backend_flags_before = snapshot_backend_flags()# Pick the attention kernel BEFORE compile: auto upgrades to cuDNN fused attention on NVIDIA (~1.18x)attention_engaged = apply_attention_backend(diff --git a/studio/backend/core/inference/diffusion_precision.py b/studio/backend/core/inference/diffusion_precision.pyindex b21d4c29291..a26b47604eb 100644--- a/studio/backend/core/inference/diffusion_precision.py+++ b/studio/backend/core/inference/diffusion_precision.py@@ -459,12 +459,17 @@ def _cast_nvfp4(encoder: Any, target: Any) -> None:# exclusions as the int8/fp8 TE modes; require_bf16 skips non-bf16 Linears so the cast engages instead of aborting.from torchao.quantization import quantize_from torchao.prototype.mx_formats import NVFP4WeightOnlyConfig- from .diffusion_transformer_quant import DEFAULT_MIN_LINEAR_FEATURES, make_filter_fn+ from .diffusion_transformer_quant import (+ DEFAULT_MIN_LINEAR_FEATURES,+ _quiet_config,+ make_filter_fn,+ )filter_fn = make_filter_fn(DEFAULT_MIN_LINEAR_FEATURES, _te_exclude_tokens(encoder), require_bf16 = True)- quantize_(encoder, NVFP4WeightOnlyConfig(), filter_fn = filter_fn)+ # No-op today (the prototype config has no set_inductor_config knob), but no torchao config is built bare.+ quantize_(encoder, _quiet_config(NVFP4WeightOnlyConfig), filter_fn = filter_fn)def _warn(logger: Any, what: str, exc: Exception) -> None:diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 888a0829c8f..5d4b7af6f9c 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -47,15 +47,25 @@ SPEED_MAX = "max"SPEED_MODES = (SPEED_OFF, SPEED_EAGER, SPEED_DEFAULT, SPEED_MAX)+# (attribute, snapshot key). All but the first are what torchao's recommended_inductor_config_setter() flips.+_INDUCTOR_FLAGS = (+ ("emulate_precision_casts", "inductor_emulate_precision_casts"),+ ("coordinate_descent_tuning", "inductor_coordinate_descent_tuning"),+ ("coordinate_descent_check_all_directions", "inductor_coordinate_descent_check_all_directions"),+ ("force_fuse_int_mm_with_mul", "inductor_force_fuse_int_mm_with_mul"),+ ("fx_graph_cache", "inductor_fx_graph_cache"),+)+_INDUCTOR_TRITON_FLAGS = (("unique_kernel_names", "inductor_triton_unique_kernel_names"),)++def snapshot_backend_flags() -> Optional[dict]:- """Capture the process-wide torch backend flags this layer may mutate, for restore on unload.- None if torch is unavailable. Each flag is read defensively so a build missing one (e.g. no- cuda.matmul on CPU/MPS) still captures the rest, instead of leaking a real mutated flag."""+ """Capture the process-wide torch backend flags this layer may mutate, for restore on unload. None+ without torch. Each flag is read defensively: a build missing one still captures the rest."""try:import torchexcept Exception: # noqa: BLE001 - no torch -> nothing to snapshot/restorereturn None- state: dict[str, bool] = {}+ state: dict[str, Any] = {}matmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)if matmul is not None and hasattr(matmul, "allow_tf32"):state["matmul_tf32"] = bool(matmul.allow_tf32)@@ -68,8 +78,21 @@ def snapshot_backend_flags() -> Optional[dict]:if hasattr(cudnn, "benchmark"):state["cudnn_benchmark"] = bool(cudnn.benchmark)inductor_cfg = _inductor_config()- if inductor_cfg is not None and hasattr(inductor_cfg, "emulate_precision_casts"):- state["inductor_emulate_precision_casts"] = bool(inductor_cfg.emulate_precision_casts)+ if inductor_cfg is not None:+ for attr, key in _INDUCTOR_FLAGS:+ if hasattr(inductor_cfg, attr):+ state[key] = bool(getattr(inductor_cfg, attr))+ triton_cfg = getattr(inductor_cfg, "triton", None)+ if triton_cfg is not None:+ for attr, key in _INDUCTOR_TRITON_FLAGS:+ if hasattr(triton_cfg, attr):+ state[key] = bool(getattr(triton_cfg, attr))+ getter = getattr(torch, "get_float32_matmul_precision", None)+ if callable(getter):+ try:+ state["matmul_precision"] = str(getter())+ except Exception: # noqa: BLE001 - unreadable on this build: restore the rest+ passreturn state@@ -90,13 +113,25 @@ def restore_backend_flags(state: Optional[dict]) -> None:except Exception: # noqa: BLE001 - best-effort per-flag restorepass+ # FIRST: on some builds set_float32_matmul_precision also writes matmul.allow_tf32.+ setter = getattr(torch, "set_float32_matmul_precision", None)+ if state.get("matmul_precision") and callable(setter):+ try:+ setter(state["matmul_precision"])+ except Exception: # noqa: BLE001 - best-effort per-flag restore+ passmatmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)_set(matmul, "allow_tf32", "matmul_tf32")_set(matmul, "allow_fp16_accumulation", "matmul_fp16_accum")cudnn = getattr(torch.backends, "cudnn", None)_set(cudnn, "allow_tf32", "cudnn_tf32")_set(cudnn, "benchmark", "cudnn_benchmark")- _set(_inductor_config(), "emulate_precision_casts", "inductor_emulate_precision_casts")+ inductor_cfg = _inductor_config()+ for attr, key in _INDUCTOR_FLAGS:+ _set(inductor_cfg, attr, key)+ triton_cfg = getattr(inductor_cfg, "triton", None) if inductor_cfg is not None else None+ for attr, key in _INDUCTOR_TRITON_FLAGS:+ _set(triton_cfg, attr, key)def _inductor_config() -> Any:diff --git a/studio/backend/core/inference/diffusion_torchao_patches.py b/studio/backend/core/inference/diffusion_torchao_patches.pynew file mode 100644index 00000000000..dacb1922e05--- /dev/null+++ b/studio/backend/core/inference/diffusion_torchao_patches.py@@ -0,0 +1,234 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Studio's copy of ``unsloth/import_fixes.py::fix_torchao_safe_int_mm_repr_probe``, not an import+of it: the diffusion process must not execute ``unsloth/__init__``. The two copies are mutually inert.++torchao's int8 GEMM asks whether it is being traced by formatting its input's repr, which on a real+CUDA tensor calls ``.item()``: a device sync per eager int8 linear, and an outright+``cudaErrorStreamCaptureUnsupported`` inside ``torch.cuda.graph``. Installed from a meta path finder+at import time, not at quantise time, because the int8 prequant path only ``torch.load``s torchao+subclasses and never calls ``quantize_``.+"""++from __future__ import annotations++import functools+import importlib.abc+import importlib.util+import inspect+import logging+import os+import sys++logger = logging.getLogger(__name__)+++_TORCHAO_INTMM_MODULES = (+ "torchao.kernel.intmm", # every release up to and including 0.18.0+ "torchao.quantization.quantize_.workflows.int8.kernels", # main after pytorch/ao#4718+)+_TORCHAO_INTMM_MODULE = _TORCHAO_INTMM_MODULES[0] # kept for callers that named the old home+_TORCHAO_INTMM_SENTINEL = "__unsloth_torchao_intmm_patch__"+_TORCHAO_INT_MM_ENV = "UNSLOTH_TORCHAO_INT_MM_FIX"++# The copy below hard-codes torchao's cuBLAS guards and fp32 fallback: ALL must be in the installed source.+_TORCHAO_SAFE_INT_MM_MARKERS = (+ "input.__repr__()",+ "dynamo_is_compiling()",+ "out_dtype(torch.ops.aten.mm.default",+ "j_is_nonzero_multiple_of_8",+ "k_is_nonzero_multiple_of_8",+ "mat2.is_contiguous()",+)+++def _is_fake_tensor(x):+ """Covers exactly the set the substring repr probe did: ``is_fake`` unwraps FunctionalTensor too."""+ try:+ from torch._subclasses.fake_tensor import is_fake+ return bool(is_fake(x))+ except Exception:+ pass+ try:+ from torch._subclasses.fake_tensor import FakeTensor+ return isinstance(x, FakeTensor)+ except Exception:+ return False+++def _make_safe_int_mm(mod, original):+ """A FULL copy of torchao 0.17.0's body, not a wrapper: its very first statement IS the probe."""+ import torch++ out_dtype = mod.out_dtype+ dynamo_is_compiling = mod.dynamo_is_compiling++ @functools.wraps(original)+ def safe_int_mm(input: torch.Tensor, mat2: torch.Tensor) -> torch.Tensor:+ if dynamo_is_compiling() or _is_fake_tensor(input):+ if input.device.type == "cpu":+ # Matmul in int32 is slow on CPU and not supported well by Inductor cpp backend+ return out_dtype(+ torch.ops.aten.mm.default, torch.int32, input.float(), mat2.float()+ )+ return out_dtype(torch.ops.aten.mm.default, torch.int32, input, mat2)++ assert (+ mat2.device == input.device+ ), f"need both tensors to be on the same device but got {mat2.device} and {input.device}"+ device_cpu = "cpu" in [mat2.device.type, input.device.type]+ j_is_nonzero_multiple_of_8 = (input.shape[1] % 8 == 0) and (input.shape[1] > 0)+ k_is_nonzero_multiple_of_8 = (mat2.shape[1] % 8 == 0) and (mat2.shape[1] > 0)+ bad_dimensions_for_cublas = not (j_is_nonzero_multiple_of_8 and k_is_nonzero_multiple_of_8)++ if device_cpu or bad_dimensions_for_cublas:+ return torch.matmul(input.cpu().to(torch.int32), mat2.cpu().to(torch.int32)).to(+ input.device.type+ )++ if not mat2.is_contiguous(): # silently gives incorrect result without this+ mat2 = mat2.contiguous()+ if (not input.is_contiguous()) and (+ input.shape[0] % 8 != 0+ ): # gives cryptic error without this+ input = input.contiguous()+ try:+ return out_dtype(torch.ops.aten.mm.default, torch.int32, input, mat2)+ except Exception:+ # H100 float8: "addmm_cuda" not implemented for 'Float8_e4m3fn'+ return torch.matmul(input.to(torch.float32), mat2.to(torch.float32)).to(torch.int32)++ safe_int_mm.__unsloth_patched__ = True+ safe_int_mm.__unsloth_original__ = original+ return safe_int_mm+++def _patch_torchao_intmm_module(mod):+ """Rebind ``safe_int_mm`` on either torchao home; True when this call installed it."""+ original = getattr(mod, "safe_int_mm", None)+ if original is None or not callable(original):+ return False+ if getattr(original, "__unsloth_patched__", False):+ return False+ # Off the module, never imported here, so this fails closed rather than borrowing our own operators.+ if not hasattr(mod, "out_dtype") or not hasattr(mod, "dynamo_is_compiling"):+ return False+ try:+ source = inspect.getsource(original)+ except Exception:+ return False+ missing = [marker for marker in _TORCHAO_SAFE_INT_MM_MARKERS if marker not in source]+ if missing:+ if "input.__repr__()" not in missing:+ logger.warning(+ "Unsloth: torchao's safe_int_mm still probes input.__repr__() but its body "+ "changed (%s missing), so the capture-safe replacement was not installed. "+ "Eager int8 keeps syncing the device on every linear.",+ ", ".join(missing),+ )+ return False+ patched = _make_safe_int_mm(mod, original)+ try:+ mod.safe_int_mm = patched+ except Exception:+ return False+ # `torchao.kernel` and `torchao.quantization` re-export the function OBJECT, so they need a sweep.+ for name, other in tuple(sys.modules.items()):+ if other is None or not (name == "torchao" or name.startswith("torchao.")):+ continue+ if getattr(other, "safe_int_mm", None) is original:+ try:+ setattr(other, "safe_int_mm", patched)+ except Exception:+ pass+ return True+++class _TorchaoIntmmLoader(importlib.abc.Loader):+ """The real loader, plus the patch once the module body finishes. Failures here leave torchao unpatched."""++ __slots__ = ("_loader",)++ def __init__(self, loader):+ self._loader = loader++ def create_module(self, spec):+ create = getattr(self._loader, "create_module", None)+ if create is None:+ return None+ return create(spec)++ def exec_module(self, module):+ self._loader.exec_module(module)+ try:+ _patch_torchao_intmm_module(module)+ except Exception:+ pass++ def __getattr__(self, attribute):+ return getattr(self._loader, attribute)+++class _TorchaoIntmmPatchFinder(importlib.abc.MetaPathFinder):+ """Inserted at the FRONT of sys.meta_path: the module really exists, so PathFinder would answer first."""++ __slots__ = (_TORCHAO_INTMM_SENTINEL, "_finding")++ def __init__(self):+ setattr(self, _TORCHAO_INTMM_SENTINEL, True)+ self._finding = False # find_spec below walks sys.meta_path again++ def find_spec(+ self,+ fullname,+ path = None,+ target = None,+ ):+ if fullname not in _TORCHAO_INTMM_MODULES or self._finding:+ return None+ self._finding = True+ try:+ spec = importlib.util.find_spec(fullname)+ except Exception:+ return None+ finally:+ self._finding = False+ if spec is None or spec.loader is None:+ return None+ if not hasattr(spec.loader, "exec_module"):+ return None # a loader from before PEP 451; leave the import entirely alone+ try:+ spec.loader = _TorchaoIntmmLoader(spec.loader)+ except Exception:+ return None+ return spec+++def install_torchao_int_mm_patch():+ """Install the capture-safe ``safe_int_mm`` if torchao is present; idempotent, so several modules may each+ ask for it. ``UNSLOTH_TORCHAO_INT_MM_FIX=0`` keeps upstream's behaviour. True when patched or the finder was+ installed, False when there is nothing to do, None when torchao is absent or the fix is off."""+ if os.environ.get(_TORCHAO_INT_MM_ENV, "1").strip() == "0":+ return None+ try:+ if importlib.util.find_spec("torchao") is None:+ return None+ except Exception:+ return None+ patched_now = False+ for name in _TORCHAO_INTMM_MODULES:+ module = sys.modules.get(name)+ if module is None:+ continue+ try:+ patched_now = _patch_torchao_intmm_module(module) or patched_now+ except Exception:+ pass+ if all(name in sys.modules for name in _TORCHAO_INTMM_MODULES):+ return patched_now # nothing left for a finder to catch+ for finder in sys.meta_path:+ if getattr(finder, _TORCHAO_INTMM_SENTINEL, False):+ return patched_now+ sys.meta_path.insert(0, _TorchaoIntmmPatchFinder())+ return Truediff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 75cd063ed4c..267fd623d2a 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -23,6 +23,8 @@ probe is best-effort: an unsupported scheme yields None and the caller loads GGUfrom __future__ import annotations+import inspect as _inspect+import os as _osimport re as _reimport sys as _sysimport threading as _threading@@ -30,6 +32,10 @@ from typing import Any, Optional# stdlib-only module (no torch), so this stays inside the "imported lazily" promise above.from core._torchao_stub import is_stubbed, torch_is_rocm+from .diffusion_torchao_patches import install_torchao_int_mm_patch++# Also runs in the spawned smoke-probe child, which imports this module and nothing else of the backend.+install_torchao_int_mm_patch()TQ_INT8 = "int8"TQ_FP8 = "fp8"@@ -964,6 +970,37 @@ def _resolve_fast_accum(fast_accum: Optional[bool]) -> bool:return True if fast_accum is None else bool(fast_accum)+# torchao's config handlers call the PROCESS-WIDE ``recommended_inductor_config_setter()`` unless+# ``set_inductor_config`` is False, and two of its flags change results rather than only speed: coordinate-descent+# tuning makes the render differ between processes on one seed, and set_float32_matmul_precision("high") reaches+# every fp32 op in the pipeline (VAE, norms). UNSLOTH_TORCHAO_INDUCTOR_CONFIG=1 restores it, for A/B benchmarking.+_TORCHAO_INDUCTOR_CONFIG_ENV = "UNSLOTH_TORCHAO_INDUCTOR_CONFIG"+++def _torchao_may_set_inductor_config() -> bool:+ return (_os.environ.get(_TORCHAO_INDUCTOR_CONFIG_ENV) or "").strip().lower() in (+ "1",+ "on",+ "true",+ "yes",+ )+++def _quiet_config(cls: Any, **kwargs: Any) -> Any:+ """Build a torchao config with ``set_inductor_config = False`` when the class accepts it. Guarded by+ signature, not version: the mx_formats configs lack the kwarg and never call the setter anyway."""+ if not _torchao_may_set_inductor_config():+ try:+ if "set_inductor_config" in _inspect.signature(cls).parameters:+ kwargs = {**kwargs, "set_inductor_config": False}+ except (+ TypeError,+ ValueError,+ ): # C-implemented or otherwise unintrospectable: leave it alone+ pass+ return cls(**kwargs)++def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:"""The torchao dynamic-activation config for ``scheme`` (lazy imports per branch).@@ -974,7 +1011,7 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:)if scheme == TQ_INT8:- return Int8DynamicActivationInt8WeightConfig()+ return _quiet_config(Int8DynamicActivationInt8WeightConfig)if scheme == TQ_FP8:# Per-ROW granularity (per-token activation + per-channel weight scale) is REQUIRED: torchao defaults to# per-TENSOR, where one Z-Image outlier near 6.6e4 forces a tensor-wide scale that pushes normal values below@@ -982,11 +1019,10 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:# it falls to int8. fast accumulate (fp8 only) follows GPU class unless forced: consumer cards run fp8 ~2x# faster with FP16 accumulate. activation_value_lb floors the per-row scale: an ALL-ZERO row otherwise yields# scale 0, NaN qdata, black frames.- import inspectfrom torchao.quantization import PerRowfp8_kwargs: dict = {"granularity": PerRow()}- config_params = inspect.signature(Float8DynamicActivationFloat8WeightConfig).parameters+ config_params = _inspect.signature(Float8DynamicActivationFloat8WeightConfig).parametersif "activation_value_lb" in config_params:fp8_kwargs["activation_value_lb"] = 1e-12# Pin the plain-torch quantize kernel: the default AUTO switches to the MSLK kernel whenever an mslk package is@@ -1002,32 +1038,35 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:passtry:from torchao.float8 import Float8MMConfig- return Float8DynamicActivationFloat8WeightConfig(+ return _quiet_config(+ Float8DynamicActivationFloat8WeightConfig,mm_config = Float8MMConfig(use_fast_accum = _resolve_fast_accum(fast_accum)),**fp8_kwargs,)except Exception: # noqa: BLE001 - older torchao without the explicit mm knob- return Float8DynamicActivationFloat8WeightConfig(**fp8_kwargs)+ return _quiet_config(Float8DynamicActivationFloat8WeightConfig, **fp8_kwargs)if scheme == TQ_NVFP4:from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig# Select the CUTLASS FP4 path, not the default Triton kernel (which needs MSLK): on a Blackwell box with CUTLASS# FP4 but no MSLK the default fails the smoke probe and falls back to GGUF.try:- return NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False)+ return _quiet_config(NVFP4DynamicActivationNVFP4WeightConfig, use_triton_kernel = False)except TypeError: # older torchao without the knob- return NVFP4DynamicActivationNVFP4WeightConfig()+ return _quiet_config(NVFP4DynamicActivationNVFP4WeightConfig)if scheme == TQ_MXFP8:import torchfrom torchao.prototype.mx_formats import MXDynamicActivationMXWeightConfigtry:- return MXDynamicActivationMXWeightConfig(- activation_dtype = torch.float8_e4m3fn, weight_dtype = torch.float8_e4m3fn+ return _quiet_config(+ MXDynamicActivationMXWeightConfig,+ activation_dtype = torch.float8_e4m3fn,+ weight_dtype = torch.float8_e4m3fn,)except (TypeError, AttributeError):# TypeError: older torchao without the explicit dtype knobs. AttributeError: a torch build without# torch.float8_e4m3fn.- return MXDynamicActivationMXWeightConfig()+ return _quiet_config(MXDynamicActivationMXWeightConfig)raise ValueError(f"unknown transformer quant scheme '{scheme}'")diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex d0c8486b56c..73ef1031983 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -82,6 +82,7 @@ from .diffusion_memory import (reclaim_offload_host_memory,settled_snapshot_device_memory,)+from .diffusion_torchao_patches import install_torchao_int_mm_patchfrom .diffusion_speed import (SPEED_DEFAULT,SPEED_EAGER,@@ -170,6 +171,9 @@ from core.inference.diffusion import hub_cache_dirlogger = get_logger(__name__)+# Asked for here as well as in the image backend, so the video path never depends on that import order.+install_torchao_int_mm_patch()+# Load kinds (mirror the image backend): gguf (single-file GGUF DiT + base repo), single_file (safetensors DiT),# pipeline (full diffusers repo)_MODEL_KINDS = frozenset({"gguf", "single_file", "pipeline"})@@ -3791,6 +3795,10 @@ class VideoBackend:# _SecondDiTView(pipe)); single-DiT resolves to (pipe,).views = _views_for(pipe, fam)+ # Before the transformer quant, the first mutator; until the state commit a failure restores them itself.+ backend_flags = snapshot_backend_flags()+ self._precommit_globals = (_load_token, backend_flags)+# dense transformer quant (opt-in, pipeline-kind only): torchao-quantise the dense bf16 DiT in place onto the# low-precision tensor cores. CUDA + bf16 only, best-effort. Quant must precede compile (eager is ~30x slower).transformer_quant_engaged: Optional[str] = None@@ -3938,10 +3946,6 @@ class VideoBackend:"(quantized transformer must be compiled; eager is ~30x slower)")effective_speed = SPEED_DEFAULT- backend_flags = snapshot_backend_flags()- # Until the state commit hands ownership to _teardown_state_locked, a failure has to restore these process-wide- # flags itself. Registered BEFORE the first mutation, as above.- self._precommit_globals = (_load_token, backend_flags)# Step cache tri-state: unset/"auto" -> FBCACHE_MIN_STEPS policy (re-checked per generation); "off"/"fbcache"# pinned. Run per expert.cache_request = normalize_transformer_cache(transformer_cache)diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.pyindex 4baf6ea478a..6f8382c5dca 100644--- a/studio/backend/core/training/diffusion_dit_trainer.py+++ b/studio/backend/core/training/diffusion_dit_trainer.py@@ -356,15 +356,17 @@ def _int8_quantize_base(transformer, family: Optional[str] = None) -> None:generic nor the family exclusions, so without this they are quantized bare and raise on thefirst forward exactly like an unexcluded one."""from core.inference.diffusion_transformer_quant import (+ _quiet_config,apply_small_m_padding,exclude_tokens_for_scheme,make_filter_fn,)from torchao.quantization import Int8WeightOnlyConfig, quantize_+ # Int8WeightOnlyConfig defaults to set_inductor_config=True, process-wide, and the trainer compiles next.quantize_(transformer,- Int8WeightOnlyConfig(),+ _quiet_config(Int8WeightOnlyConfig),filter_fn = make_filter_fn(512, exclude_name_tokens = exclude_tokens_for_scheme("int8", family)),diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.pyindex 8bc24dc9e29..fd04aaedee5 100644--- a/studio/backend/core/training/diffusion_train_common.py+++ b/studio/backend/core/training/diffusion_train_common.py@@ -37,6 +37,7 @@ from core.inference.diffusion_families import (supported_family_names,trainable_family_names,)+from core.inference.diffusion_torchao_patches import install_torchao_int_mm_patchfrom core.inference.video_families import detect_video_family, supported_video_family_namesfrom utils.paths.path_utils import drop_appledouble_metadata@@ -44,6 +45,8 @@ from utils.paths.path_utils import drop_appledouble_metadata# does not carry over. Both import this module first.install_xformers_windows_rocm_stub()install_torchao_windows_rocm_stub()+# Same child: the DiT trainer's int8 base-weight quantisation goes through torchao.+install_torchao_int_mm_patch()# Default LoRA target modules: the attention projections common to the SDXL U-Net and the DiTs (the# diffusers/kohya convention). A family may override this.diff --git a/studio/backend/tests/test_diffusion_precision.py b/studio/backend/tests/test_diffusion_precision.pyindex d8ad0adccc2..7278ecac6d0 100644--- a/studio/backend/tests/test_diffusion_precision.py+++ b/studio/backend/tests/test_diffusion_precision.py@@ -16,6 +16,7 @@ import pytestimport core.inference.diffusion_precision as dpfrom core.inference.diffusion_precision import (+ _cast_fp8_dynamic,TE_QUANT_FP8,TE_QUANT_FP8_DYNAMIC,TE_QUANT_INT8,@@ -81,6 +82,7 @@ def _stub_casters(monkeypatch, recorder):dtq.make_filter_fn = lambda min_features, exclude = (), *, require_bf16 = False: (lambda module, fqn = "": True)+ dtq._quiet_config = lambda cls, **kw: cls(**kw)monkeypatch.setitem(sys.modules, "core.inference.diffusion_transformer_quant", dtq)@@ -321,6 +323,7 @@ def _stub_transformer_quant(monkeypatch, captured):dtq.TQ_FP8 = "fp8"dtq.DEFAULT_MIN_LINEAR_FEATURES = 512dtq._make_quant_config = lambda scheme, *a, **k: f"cfg:{scheme}"+ dtq._quiet_config = lambda cls, **kw: cls(**kw)dtq.exclude_tokens_for_scheme = lambda scheme: ("modulation",)def _make_filter_fn(@@ -519,3 +522,64 @@ def test_int8_without_a_schedule_reports_fp8_as_the_effective_mode():# Every other mode is its own effective mode, and absent stays absent.assert effective_te_quant(TE_QUANT_FP8_DYNAMIC, "z-image-turbo") == TE_QUANT_FP8_DYNAMICassert effective_te_quant(None, "qwen-image") is None+++def test_nvfp4_te_cast_builds_its_config_through_quiet_config(monkeypatch):+ """NVFP4 has no set_inductor_config knob, so the risk of routing it through _quiet_config is a TypeError."""+ _stub_torch(monkeypatch, cc = (10, 0))+ captured: dict = {}+ _stub_transformer_quant(monkeypatch, captured)+ seen: list = []+ dtq = sys.modules["core.inference.diffusion_transformer_quant"]+ dtq._quiet_config = lambda cls, **kw: seen.append((cls, kw)) or cls(**kw)+ outcome = quantize_text_encoders(+ types.SimpleNamespace(text_encoder = object()), _target(), mode = "nvfp4"+ )+ assert outcome.mode == TE_QUANT_NVFP4+ assert len(seen) == 1 and seen[0][1] == {}+ assert captured["config"] == "nvfp4cfg"+++def test_int8_and_fp8_dynamic_te_casts_reuse_the_quiet_factory(monkeypatch):+ torch = _stub_torch(monkeypatch, cc = (10, 0))+ captured: dict = {}+ _stub_transformer_quant(monkeypatch, captured)+ layers = torch.nn.ModuleList([object() for _ in range(8)])+ enc = types.SimpleNamespace(_keep_in_fp32_modules = ["wo"])+ enc.named_modules = lambda: [("model.layers", layers)]+ _cast_int8_selective(enc, _target(), 3, 0)+ assert captured["config"] == "cfg:int8"+ _cast_fp8_dynamic(enc, _target())+ assert captured["config"] == "cfg:fp8"+++def test_no_torchao_config_is_constructed_outside_quiet_config():+ """Every torchao config must be built through ``_quiet_config``, whose default makes renders differ."""+ import ast+ from pathlib import Path++ backend = Path(__file__).resolve().parents[1]+ files = sorted((backend / "core" / "inference").glob("*.py")) + [+ backend / "core" / "training" / "diffusion_dit_trainer.py",+ ]+ offenders: list[str] = []+ for path in files:+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ for node in ast.walk(tree):+ if not isinstance(node, ast.Call):+ continue+ func = node.func+ name = (+ func.id+ if isinstance(func, ast.Name)+ else func.attr+ if isinstance(func, ast.Attribute)+ else ""+ )+ if not (name.endswith("WeightConfig") or name.endswith("WeightOnlyConfig")):+ continue+ offenders.append(f"{path.relative_to(backend)}:{node.lineno} {name}(...)")+ assert not offenders, (+ "torchao config constructed outside _quiet_config (pass the CLASS as its first argument instead):\n "+ + "\n ".join(offenders)+ )diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.pyindex 9943c4f35ee..455fb0b965e 100644--- a/studio/backend/tests/test_diffusion_speed.py+++ b/studio/backend/tests/test_diffusion_speed.py@@ -138,6 +138,7 @@ def test_compile_eligible_requires_bf16_cuda_friendly(monkeypatch):def test_snapshot_restore_backend_flags(monkeypatch):torch = _stub_torch(monkeypatch)snap = snapshot_backend_flags()+ # The plain stub torch has no _inductor and no get_float32_matmul_precision, so none of those keys appear.assert snap == {"matmul_tf32": False, "cudnn_tf32": False, "cudnn_benchmark": False}# An opt-in max run flips the globals on...torch.backends.cuda.matmul.allow_tf32 = True@@ -852,3 +853,139 @@ def test_linux_and_mac_are_not_asked_about_triton(monkeypatch):_clear_runtime_cache()assert ds_mod.torch_compile_runtime_available() is True_clear_runtime_cache()+++_TORCHAO_FLAGS = (+ "coordinate_descent_tuning",+ "coordinate_descent_check_all_directions",+ "force_fuse_int_mm_with_mul",+ "fx_graph_cache",+)+++def _stub_full_inductor_config(torch):+ cfg = types.SimpleNamespace(+ emulate_precision_casts = False,+ triton = types.SimpleNamespace(unique_kernel_names = False),+ **{name: False for name in _TORCHAO_FLAGS},+ )+ torch._inductor = types.SimpleNamespace(config = cfg)+ return cfg+++def _stub_matmul_precision(+ torch,+ calls: list,+ initial = "highest",+):+ cell = {"v": initial}++ def _get():+ return cell["v"]++ def _set(v):+ calls.append(("precision", v))+ cell["v"] = v++ torch.get_float32_matmul_precision = _get+ torch.set_float32_matmul_precision = _set+ return cell+++def test_snapshot_restores_torchao_inductor_flags(monkeypatch):+ torch = _stub_torch(monkeypatch)+ cfg = _stub_full_inductor_config(torch)+ snap = snapshot_backend_flags()+ for name in _TORCHAO_FLAGS:+ assert snap[f"inductor_{name}"] is False+ assert snap["inductor_triton_unique_kernel_names"] is False+ for name in _TORCHAO_FLAGS:+ setattr(cfg, name, True)+ cfg.triton.unique_kernel_names = True+ cfg.emulate_precision_casts = True+ restore_backend_flags(snap)+ for name in _TORCHAO_FLAGS:+ assert getattr(cfg, name) is False, name+ assert cfg.triton.unique_kernel_names is False+ assert cfg.emulate_precision_casts is False+++def test_snapshot_restores_float32_matmul_precision(monkeypatch):+ torch = _stub_torch(monkeypatch)+ calls: list = []+ cell = _stub_matmul_precision(torch, calls, initial = "highest")+ snap = snapshot_backend_flags()+ assert snap["matmul_precision"] == "highest"+ cell["v"] = "high"+ restore_backend_flags(snap)+ assert cell["v"] == "highest"+++def test_matmul_precision_is_restored_before_tf32(monkeypatch):+ torch = _stub_torch(monkeypatch)+ calls: list = []+ _stub_matmul_precision(torch, calls, initial = "highest")++ class _Matmul:+ def __init__(self):+ self._tf32 = False++ @property+ def allow_tf32(self):+ return self._tf32++ @allow_tf32.setter+ def allow_tf32(self, v):+ calls.append(("tf32", v))+ self._tf32 = v++ torch.backends.cuda.matmul = _Matmul()+ snap = snapshot_backend_flags()+ calls.clear()+ restore_backend_flags(snap)+ kinds = [k for k, _ in calls]+ assert kinds.index("precision") < kinds.index("tf32"), calls+++def test_snapshot_skips_inductor_flags_a_build_lacks(monkeypatch):+ torch = _stub_torch(monkeypatch)+ cfg = _stub_inductor_config(monkeypatch, torch, emulate = False)+ snap = snapshot_backend_flags()+ assert snap["inductor_emulate_precision_casts"] is False+ for name in _TORCHAO_FLAGS:+ assert f"inductor_{name}" not in snap+ assert "inductor_triton_unique_kernel_names" not in snap+ assert "matmul_precision" not in snap+ cfg.emulate_precision_casts = True+ restore_backend_flags(snap)+ assert cfg.emulate_precision_casts is False+++def test_video_snapshot_precedes_transformer_quant():+ """A failed load and an unload must both restore the pre-quant backend flags."""+ import ast+ from pathlib import Path++ path = Path(__file__).resolve().parents[1] / "core" / "inference" / "video.py"+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ for node in ast.walk(tree):+ if not isinstance(node, ast.FunctionDef):+ continue+ snaps = [+ c.lineno+ for c in ast.walk(node)+ if isinstance(c, ast.Call) and getattr(c.func, "id", None) == "snapshot_backend_flags"+ ]+ quants = [+ c.lineno+ for c in ast.walk(node)+ if isinstance(c, ast.Call) and getattr(c.func, "id", None) == "quantize_transformer"+ ]+ if snaps and quants:+ assert (+ min(snaps) < min(quants)+ ), f"{node.name}: snapshot_backend_flags at {snaps} must precede quantize_transformer at {quants}"+ return+ raise AssertionError(+ "no video.py function calls both snapshot_backend_flags and quantize_transformer"+ )diff --git a/studio/backend/tests/test_diffusion_transformer_quant.py b/studio/backend/tests/test_diffusion_transformer_quant.pyindex 6777fdc49f2..8bcebbe38bb 100644--- a/studio/backend/tests/test_diffusion_transformer_quant.py+++ b/studio/backend/tests/test_diffusion_transformer_quant.py@@ -1494,3 +1494,107 @@ def test_paths_are_stripped_without_eating_dotted_module_names():assert "C:\\Users" not in tq._strip_paths(r"ImportError: DLL load failed: C:\Users\me\.venv\Lib\site-packages\torchao\_C.pyd")+++class _RecordingConfig:+ """Stands in for a torchao config dataclass: records every kwarg it was built with."""++ built: list = []++ def __init__(+ self,+ *,+ set_inductor_config = True,+ **kw,+ ):+ self.set_inductor_config = set_inductor_config+ self.kw = kw+ type(self).built.append(self)+++def _stub_torchao_configs(+ monkeypatch,+ *,+ int8 = None,+ fp8 = None,+):+ tqz = types.ModuleType("torchao.quantization")+ tqz.quantize_ = lambda *a, **k: None+ tqz.PerRow = lambda: "per_row"+ tqz.Int8DynamicActivationInt8WeightConfig = int8 or _RecordingConfig+ tqz.Float8DynamicActivationFloat8WeightConfig = fp8 or _RecordingConfig+ monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)+ # _make_quant_config(fp8) also reaches for these two; the older-torchao shape is absent.+ monkeypatch.setitem(sys.modules, "torchao.float8", None)+ monkeypatch.setitem(sys.modules, "torchao.quantization.quantize_", None)+ return tqz+++def test_int8_config_disables_torchao_inductor_config(monkeypatch):+ _RecordingConfig.built = []+ _stub_torchao_configs(monkeypatch)+ cfg = tq._make_quant_config(TQ_INT8)+ assert isinstance(cfg, _RecordingConfig)+ assert cfg.set_inductor_config is False+ assert cfg.kw == {}+++def test_fp8_config_disables_torchao_inductor_config_and_keeps_its_kwargs(monkeypatch):+ _RecordingConfig.built = []+ _stub_torchao_configs(monkeypatch)+ cfg = tq._make_quant_config(TQ_FP8)+ assert isinstance(cfg, _RecordingConfig)+ assert cfg.set_inductor_config is False+ assert cfg.kw["granularity"] == "per_row"+++def test_quiet_config_omits_the_kwarg_when_the_class_does_not_accept_it():+ """The mx_formats configs have no set_inductor_config, so the helper must not raise TypeError on them."""++ class _NoKnob:+ def __init__(self, use_triton_kernel = True):+ self.use_triton_kernel = use_triton_kernel++ cfg = tq._quiet_config(_NoKnob, use_triton_kernel = False)+ assert cfg.use_triton_kernel is False and not hasattr(cfg, "set_inductor_config")+ assert tq._quiet_config(lambda: "cfg") == "cfg"+++def test_quiet_config_is_reenabled_by_env(monkeypatch):+ """UNSLOTH_TORCHAO_INDUCTOR_CONFIG=1 restores torchao's default so the two behaviours can be benchmarked A/B."""+ monkeypatch.setenv("UNSLOTH_TORCHAO_INDUCTOR_CONFIG", "1")+ assert tq._quiet_config(_RecordingConfig).set_inductor_config is True+ monkeypatch.setenv("UNSLOTH_TORCHAO_INDUCTOR_CONFIG", "0")+ assert tq._quiet_config(_RecordingConfig).set_inductor_config is False+ monkeypatch.delenv("UNSLOTH_TORCHAO_INDUCTOR_CONFIG")+ assert tq._quiet_config(_RecordingConfig).set_inductor_config is False+++def test_quiet_config_tolerates_an_unintrospectable_class(monkeypatch):+ import inspect++ real = inspect.signature++ def _boom(obj):+ if obj is _RecordingConfig:+ raise ValueError("no signature found for builtin")+ return real(obj)++ monkeypatch.setattr(tq._inspect, "signature", _boom)+ cfg = tq._quiet_config(_RecordingConfig)+ # The kwarg could not be proven to exist, so the class is built with its own default rather than crashing.+ assert cfg.set_inductor_config is True+++def test_real_torchao_configs_carry_set_inductor_config_false():+ pytest.importorskip("torchao.quantization")+ torch = pytest.importorskip("torch")+ ic = getattr(getattr(torch, "_inductor", None), "config", None)+ before = getattr(ic, "coordinate_descent_tuning", None) if ic is not None else None+ for scheme in (TQ_INT8, TQ_FP8):+ cfg = tq._make_quant_config(scheme)+ if not hasattr(cfg, "set_inductor_config"):+ pytest.skip("torchao config without set_inductor_config")+ assert cfg.set_inductor_config is False, scheme+ if ic is not None:+ assert getattr(ic, "coordinate_descent_tuning", None) == beforediff --git a/studio/backend/tests/test_torchao_intmm_patch_wiring.py b/studio/backend/tests/test_torchao_intmm_patch_wiring.pynew file mode 100644index 00000000000..163a0e9be16--- /dev/null+++ b/studio/backend/tests/test_torchao_intmm_patch_wiring.py@@ -0,0 +1,192 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Invariants for Studio's copy of the torchao ``safe_int_mm`` fix, none of which raise on their own."""++from __future__ import annotations++import ast+import types+from pathlib import Path++import pytest++from core.inference.diffusion_torchao_patches import (+ _TORCHAO_INTMM_MODULES,+ _TorchaoIntmmLoader,+ _TorchaoIntmmPatchFinder,+ _patch_torchao_intmm_module,+ install_torchao_int_mm_patch,+)++_BACKEND = Path(__file__).resolve().parent.parent+_CORE = _BACKEND / "core"+_REPO_ROOT = Path(__file__).resolve().parents[3]+_IMPORT_FIXES = _REPO_ROOT / "unsloth" / "import_fixes.py"+_PATCH_MODULE = _CORE / "inference" / "diffusion_torchao_patches.py"+_INSTALL = install_torchao_int_mm_patch.__name__++# Every module that can be the first thing in its process to reach torchao.+_ENTRYPOINTS = [+ _CORE / "inference" / "diffusion.py",+ _CORE / "inference" / "video.py",+ _CORE / "inference" / "diffusion_transformer_quant.py",+ _CORE / "training" / "diffusion_train_common.py",+]++_SHARED_FUNCTIONS = ("_is_fake_tensor", "_make_safe_int_mm", "_patch_torchao_intmm_module")+++def _install_call_linenos(node) -> list[int]:+ return [+ c.lineno+ for c in ast.walk(node)+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == _INSTALL+ ]+++def test_all_entrypoints_install_the_patch():+ for path in _ENTRYPOINTS:+ assert _install_call_linenos(ast.parse(path.read_text(encoding = "utf-8"))), (+ f"{path.relative_to(_BACKEND)} never calls {_INSTALL}() -- torchao's int8 GEMM "+ "keeps syncing the device on every linear and cannot be CUDA-graph captured."+ )+++def _strip_docstrings(node):+ for child in ast.walk(node):+ body = getattr(child, "body", None)+ if not isinstance(body, list) or not body:+ continue+ first = body[0]+ if (+ isinstance(first, ast.Expr)+ and isinstance(first.value, ast.Constant)+ and isinstance(first.value.value, str)+ ):+ del body[0]+ return node+++def _function_dump(path: Path, name: str) -> str:+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ for node in tree.body:+ if isinstance(node, ast.FunctionDef) and node.name == name:+ return ast.dump(_strip_docstrings(node))+ raise AssertionError(f"{path} has no top-level def {name}")++[email protected]("name", _SHARED_FUNCTIONS)+def test_studio_copy_matches_unsloth_import_fixes(name):+ if not _IMPORT_FIXES.is_file():+ pytest.skip("unsloth/import_fixes.py is not in this checkout")+ assert _function_dump(_PATCH_MODULE, name) == _function_dump(_IMPORT_FIXES, name), (+ f"DRIFT: {name}() differs between core/inference/diffusion_torchao_patches.py and "+ "unsloth/import_fixes.py. The two copies must stay interchangeable, because whichever "+ "one runs first in a process is the one that patches torchao."+ )+++def _fake_intmm_module(safe_int_mm) -> types.ModuleType:+ module = types.ModuleType("torchao_intmm_stand_in")+ module.safe_int_mm = safe_int_mm+ module.out_dtype = lambda *args, **kwargs: None+ module.dynamo_is_compiling = lambda: False+ return module+++def test_patch_leaves_an_already_patched_module_alone():+ """Why the two copies do not fight: both mark their replacement ``__unsloth_patched__``."""++ def already_patched(input, mat2):+ return None++ already_patched.__unsloth_patched__ = True+ module = _fake_intmm_module(already_patched)++ assert _patch_torchao_intmm_module(module) is False+ assert module.safe_int_mm is already_patched+++def test_patch_refuses_an_unrecognised_body():+ def rewritten_upstream(input, mat2):+ # None of the markers the gate looks for+ return input @ mat2++ module = _fake_intmm_module(rewritten_upstream)++ assert _patch_torchao_intmm_module(module) is False+ assert module.safe_int_mm is rewritten_upstream+++def test_finder_answers_for_both_torchao_homes(monkeypatch):+ """torchao main moved ``safe_int_mm`` into the int8 workflow module (pytorch/ao#4718), probe intact."""+ import importlib.machinery+ import importlib.util++ assert _TORCHAO_INTMM_MODULES == (+ "torchao.kernel.intmm",+ "torchao.quantization.quantize_.workflows.int8.kernels",+ )+ if _IMPORT_FIXES.is_file():+ text = _IMPORT_FIXES.read_text(encoding = "utf-8")+ for name in _TORCHAO_INTMM_MODULES:+ assert f'"{name}"' in text, f"unsloth/import_fixes.py does not list {name}"++ class _Loader:+ def exec_module(self, module):+ pass++ monkeypatch.setattr(+ importlib.util,+ "find_spec",+ lambda fullname, *a, **k: importlib.machinery.ModuleSpec(fullname, _Loader()),+ )+ finder = _TorchaoIntmmPatchFinder()+ for name in _TORCHAO_INTMM_MODULES:+ spec = finder.find_spec(name)+ assert spec is not None and isinstance(spec.loader, _TorchaoIntmmLoader), name+ assert finder.find_spec("torchao.kernel.other") is None+++def test_real_torchao_int_mm_is_patched_and_bit_identical():+ pytest.importorskip("torchao")+ torch = pytest.importorskip("torch")+ import importlib++ intmm = None+ for name in _TORCHAO_INTMM_MODULES:+ try:+ candidate = importlib.import_module(name)+ except ImportError:+ continue+ if callable(getattr(candidate, "safe_int_mm", None)):+ intmm = candidate+ break+ if intmm is None:+ pytest.skip("torchao does not define safe_int_mm under any known module name")++ install_torchao_int_mm_patch()+ patched = intmm.safe_int_mm+ assert getattr(patched, "__unsloth_patched__", False) is True+ # int_scaled_matmul resolves the name through module globals, so the rebind must reach it+ assert intmm.int_scaled_matmul.__globals__["safe_int_mm"] is patched+ original = patched.__unsloth_original__++ generator = torch.Generator().manual_seed(0)++ def randint8(*shape):+ return torch.randint(-127, 127, shape, dtype = torch.int8, generator = generator)++ cases = [+ (randint8(64, 64), randint8(64, 64)),+ (randint8(40, 24), randint8(24, 72)),+ # mat2 non-contiguous: torchao silently returns a wrong answer without the .contiguous()+ (randint8(64, 64), randint8(64, 64).t().contiguous().t()),+ # j = 20 is not a nonzero multiple of 8, so cuBLAS is skipped for the fallback path+ (randint8(40, 20), randint8(20, 64)),+ ]+ for a, b in cases:+ assert torch.equal(+ patched(a, b), original(a, b)+ ), f"patched safe_int_mm diverged from torchao's on {tuple(a.shape)} x {tuple(b.shape)}"diff --git a/tests/test_import_fixes_drift.py b/tests/test_import_fixes_drift.pyindex 208ac46eba3..b59280bcbef 100644--- a/tests/test_import_fixes_drift.py+++ b/tests/test_import_fixes_drift.py@@ -755,3 +755,276 @@ def test_psutil_cpu_freq_shape_and_wiring():"DRIFT DETECTED: patch_psutil_cpu_freq is defined but never called in ""_gpu_init.py, so real imports never install it.")+++def _import_torchao_intmm_home():+ from unsloth.import_fixes import _TORCHAO_INTMM_MODULES+ for name in _TORCHAO_INTMM_MODULES:+ try:+ module = importlib.import_module(name)+ except ImportError:+ continue+ if callable(getattr(module, "safe_int_mm", None)):+ return module+ pytest.skip("torchao does not define safe_int_mm under any known module name")+++def _torchao_intmm_original_source():+ """Once the fix has run, upstream's body is reachable only through ``__unsloth_original__``."""+ pytest.importorskip("torchao")+ intmm = _import_torchao_intmm_home()+ function = intmm.safe_int_mm+ if getattr(function, "__unsloth_patched__", False):+ function = function.__unsloth_original__+ return inspect.getsource(function)+++def test_torchao_safe_int_mm_still_uses_the_repr_probe():+ """The pathology itself: a repr that formats the tensor's values, and so calls ``.item()``."""+ source = _torchao_intmm_original_source()+ if "__repr__" not in source:+ pytest.fail(+ "upstream fixed the probe, delete fix_torchao_safe_int_mm_repr_probe: "+ "torchao's safe_int_mm no longer reprs its input, so the replacement in "+ "unsloth/import_fixes.py (and its Studio copy in "+ "studio/backend/core/inference/diffusion_torchao_patches.py) is dead weight."+ )+++def test_torchao_safe_int_mm_body_matches_the_verified_shape():+ from unsloth.import_fixes import _TORCHAO_SAFE_INT_MM_MARKERS++ source = _torchao_intmm_original_source()+ missing = [marker for marker in _TORCHAO_SAFE_INT_MM_MARKERS if marker not in source]+ if missing:+ pytest.fail(+ "DRIFT DETECTED: torchao's safe_int_mm body changed "+ f"({', '.join(missing)} missing); re-verify bit-identity before keeping the patch. "+ "_make_safe_int_mm copies the cuBLAS dimension guards, the contiguity fixes and the "+ "fp32 fallback, so a moved body is one the copy must not impersonate."+ )+++def _patched_torchao_safe_int_mm():+ pytest.importorskip("torchao")+ intmm = _import_torchao_intmm_home()+ from unsloth.import_fixes import (+ _TORCHAO_SAFE_INT_MM_MARKERS,+ fix_torchao_safe_int_mm_repr_probe,+ )++ fix_torchao_safe_int_mm_repr_probe()+ function = intmm.safe_int_mm+ if not getattr(function, "__unsloth_patched__", False):+ source = inspect.getsource(function)+ if any(marker not in source for marker in _TORCHAO_SAFE_INT_MM_MARKERS):+ pytest.skip(+ "this torchao's safe_int_mm is not the verified body, so the fix declined to "+ "patch it (see test_torchao_safe_int_mm_body_matches_the_verified_shape)"+ )+ pytest.fail(+ "DRIFT DETECTED: fix_torchao_safe_int_mm_repr_probe left a recognised "+ "safe_int_mm unpatched."+ )+ return intmm, function+++def test_torchao_intmm_patch_is_bit_identical_on_cpu():+ torch = pytest.importorskip("torch")+ intmm, patched = _patched_torchao_safe_int_mm()+ # int_scaled_matmul resolves the name through module globals, so the rebind must reach it+ assert intmm.int_scaled_matmul.__globals__["safe_int_mm"] is patched+ original = patched.__unsloth_original__++ generator = torch.Generator().manual_seed(0)++ def randint8(*shape):+ return torch.randint(-127, 127, shape, dtype = torch.int8, generator = generator)++ cases = [+ (randint8(64, 64), randint8(64, 64)),+ (randint8(40, 24), randint8(24, 72)),+ (randint8(64, 64), randint8(64, 64).t().contiguous().t()),+ (randint8(40, 20), randint8(20, 64)),+ ]+ for a, b in cases:+ assert torch.equal(patched(a, b), original(a, b)), (+ "DRIFT DETECTED: the patched safe_int_mm no longer matches torchao's on "+ f"{tuple(a.shape)} x {tuple(b.shape)}."+ )+++def test_torchao_intmm_patch_is_idempotent():+ import types++ from unsloth.import_fixes import (+ _patch_torchao_intmm_module,+ fix_torchao_safe_int_mm_repr_probe,+ )++ intmm, patched = _patched_torchao_safe_int_mm()+ fix_torchao_safe_int_mm_repr_probe()+ assert intmm.safe_int_mm is patched, "DRIFT DETECTED: safe_int_mm was replaced twice."++ def already_patched(input, mat2):+ return None++ already_patched.__unsloth_patched__ = True+ stand_in = types.ModuleType("torchao_intmm_stand_in")+ stand_in.safe_int_mm = already_patched+ stand_in.out_dtype = lambda *args, **kwargs: None+ stand_in.dynamo_is_compiling = lambda: False+ assert _patch_torchao_intmm_module(stand_in) is False+ assert stand_in.safe_int_mm is already_patched+++def test_torchao_intmm_patch_refuses_an_unrecognised_body():+ import types++ from unsloth.import_fixes import _patch_torchao_intmm_module++ def rewritten_upstream(input, mat2):+ # None of the markers the gate looks for+ return input @ mat2++ stand_in = types.ModuleType("torchao_intmm_stand_in")+ stand_in.safe_int_mm = rewritten_upstream+ stand_in.out_dtype = lambda *args, **kwargs: None+ stand_in.dynamo_is_compiling = lambda: False+ assert _patch_torchao_intmm_module(stand_in) is False+ assert stand_in.safe_int_mm is rewritten_upstream+++def test_torchao_intmm_patch_covers_a_later_import():+ """The point of the finder: the prequant path imports torchao well after ``import unsloth``."""+ if importlib.util.find_spec("torchao") is None:+ pytest.skip("torchao not installed -- nothing to patch.")+ import subprocess++ import_fixes_path = Path(__file__).resolve().parent.parent / "unsloth" / "import_fixes.py"+ program = (+ "import importlib.util, sys\n"+ f"spec = importlib.util.spec_from_file_location('unsloth_import_fixes_under_test', {str(import_fixes_path)!r})\n"+ "module = importlib.util.module_from_spec(spec)\n"+ "spec.loader.exec_module(module)\n"+ "assert 'torchao' not in sys.modules, 'torchao was imported before the fix ran'\n"+ "module.fix_torchao_safe_int_mm_repr_probe()\n"+ "import torchao.quantization\n"+ "import importlib\n"+ "intmm = None\n"+ "for name in module._TORCHAO_INTMM_MODULES:\n"+ " try:\n"+ " candidate = importlib.import_module(name)\n"+ " except ImportError:\n"+ " continue\n"+ " if callable(getattr(candidate, 'safe_int_mm', None)):\n"+ " intmm = candidate\n"+ " break\n"+ "assert intmm is not None, 'no torchao module defines safe_int_mm'\n"+ "print('PATCHED=' + str(bool(getattr(intmm.safe_int_mm, '__unsloth_patched__', False))))\n"+ )+ env = dict(os.environ)+ env.pop("UNSLOTH_TORCHAO_INT_MM_FIX", None)+ result = subprocess.run(+ [sys.executable, "-c", program],+ capture_output = True,+ text = True,+ timeout = 600,+ env = env,+ )+ assert result.returncode == 0, f"child failed:\n{result.stdout}\n{result.stderr}"+ assert "PATCHED=True" in result.stdout, (+ "DRIFT DETECTED: a torchao imported AFTER fix_torchao_safe_int_mm_repr_probe ran was "+ f"left unpatched, so the meta path finder no longer fires.\n{result.stdout}"+ )+++def test_torchao_intmm_finder_covers_both_module_homes(monkeypatch):+ """A finder covering only the old name loses the fix once torchao ships pytorch/ao#4718."""+ import importlib.machinery++ from unsloth.import_fixes import (+ _TORCHAO_INTMM_MODULES,+ _TorchaoIntmmLoader,+ _TorchaoIntmmPatchFinder,+ )++ assert "torchao.kernel.intmm" in _TORCHAO_INTMM_MODULES+ assert "torchao.quantization.quantize_.workflows.int8.kernels" in _TORCHAO_INTMM_MODULES++ class _Loader:+ def exec_module(self, module):+ pass++ def fake_find_spec(fullname, *args, **kwargs):+ return importlib.machinery.ModuleSpec(fullname, _Loader())++ monkeypatch.setattr(importlib.util, "find_spec", fake_find_spec)+ finder = _TorchaoIntmmPatchFinder()+ for name in _TORCHAO_INTMM_MODULES:+ spec = finder.find_spec(name)+ assert spec is not None and isinstance(spec.loader, _TorchaoIntmmLoader), name+ assert finder.find_spec("torchao.kernel.somewhere_else") is None+ assert finder.find_spec("torchao.quantization.quantize_.workflows.int8") is None+++def test_torchao_intmm_installer_patches_the_new_home_when_already_imported(monkeypatch):+ """A module ALREADY in ``sys.modules`` must be patched too: the finder only sees later imports."""+ import types++ torch = pytest.importorskip("torch")+ from unsloth.import_fixes import (+ _TORCHAO_INTMM_MODULES,+ _TORCHAO_INTMM_SENTINEL,+ fix_torchao_safe_int_mm_repr_probe,+ )++ source = _torchao_intmm_original_source()+ module = types.ModuleType("torchao.quantization.quantize_.workflows.int8.kernels")+ module.__dict__["torch"] = torch+ from torch._dynamo import is_compiling as dynamo_is_compiling+ from torch._higher_order_ops.out_dtype import out_dtype++ module.out_dtype = out_dtype+ module.dynamo_is_compiling = dynamo_is_compiling+ import linecache+ import textwrap++ # The gate reads the body through inspect.getsource, so the copy needs a linecache entry.+ text = textwrap.dedent(source)+ filename = "<torchao safe_int_mm copy>"+ linecache.cache[filename] = (len(text), None, text.splitlines(True), filename)+ exec(compile(text, filename, "exec"), module.__dict__)+ assert callable(module.safe_int_mm)++ # Hide every real torchao home so only the stand-in is visible, and drop any finder.+ for name in _TORCHAO_INTMM_MODULES:+ monkeypatch.delitem(sys.modules, name, raising = False)+ monkeypatch.setitem(sys.modules, module.__name__, module)+ monkeypatch.setattr(+ sys,+ "meta_path",+ [f for f in sys.meta_path if not getattr(f, _TORCHAO_INTMM_SENTINEL, False)],+ )+ monkeypatch.delenv("UNSLOTH_TORCHAO_INT_MM_FIX", raising = False)++ assert fix_torchao_safe_int_mm_repr_probe() is True+ assert getattr(+ module.safe_int_mm, "__unsloth_patched__", False+ ), "the installer ignored a safe_int_mm registered under torchao's new module name"+ a = torch.randint(-128, 127, (16, 32), dtype = torch.int8)+ b = torch.randint(-128, 127, (32, 24), dtype = torch.int8)+ assert torch.equal(module.safe_int_mm(a, b), module.safe_int_mm.__unsloth_original__(a, b))+++def test_torchao_intmm_patch_wired_into_gpu_init():+ source = Path(__file__).resolve().parent.parent / "unsloth" / "_gpu_init.py"+ source = source.read_text(encoding = "utf-8")+ assert "fix_torchao_safe_int_mm_repr_probe()" in source, (+ "DRIFT DETECTED: fix_torchao_safe_int_mm_repr_probe is defined but never called in "+ "_gpu_init.py, so real imports never install it."+ )+ assert "del fix_torchao_safe_int_mm_repr_probe" in source, (+ "DRIFT DETECTED: fix_torchao_safe_int_mm_repr_probe is left bound on the unsloth "+ "namespace; _gpu_init.py deletes every fix it calls."+ )diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex b4d09653b5f..f6f4fcd4cb5 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -233,6 +233,7 @@ from .import_fixes import (fix_vllm_aimv2_issue,fix_vllm_lora_tokenizer_module,fix_torchao_nf4tensor_move,+ fix_torchao_safe_int_mm_repr_probe,check_vllm_torch_sm100_compatibility,fix_vllm_guided_decoding_params,fix_vllm_pdl_blackwell,@@ -276,6 +277,7 @@ fix_vllm_lora_tokenizer_module()# torchao 0.18.0 moved nf4tensor; torchtune (via xcodec2) still imports the old path. Lazy alias, so# it costs nothing unless asked for.fix_torchao_nf4tensor_move()+fix_torchao_safe_int_mm_repr_probe()# Check vLLM + torch < 2.9.0 + SM100 compatibility BEFORE importing vLLMcheck_vllm_torch_sm100_compatibility()fix_vllm_guided_decoding_params()@@ -318,6 +320,7 @@ del fix_flash_attn_4_namespace_shadowdel fix_vllm_aimv2_issuedel fix_vllm_lora_tokenizer_moduledel fix_torchao_nf4tensor_move+del fix_torchao_safe_int_mm_repr_probedel check_vllm_torch_sm100_compatibilitydel fix_vllm_guided_decoding_paramsdel fix_trl_vllm_ascenddiff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 2ac2362e274..8cf8b8d175b 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -5755,6 +5755,221 @@ def fix_torchao_nf4tensor_move():sys.meta_path.append(_TorchaoNF4AliasFinder())+_TORCHAO_INTMM_MODULES = (+ "torchao.kernel.intmm", # every release up to and including 0.18.0+ "torchao.quantization.quantize_.workflows.int8.kernels", # main after pytorch/ao#4718+)+_TORCHAO_INTMM_MODULE = _TORCHAO_INTMM_MODULES[0] # kept for callers that named the old home+_TORCHAO_INTMM_SENTINEL = "__unsloth_torchao_intmm_patch__"+_TORCHAO_INT_MM_ENV = "UNSLOTH_TORCHAO_INT_MM_FIX"++# The copy below hard-codes torchao's cuBLAS guards and fp32 fallback: ALL must be in the installed source.+_TORCHAO_SAFE_INT_MM_MARKERS = (+ "input.__repr__()",+ "dynamo_is_compiling()",+ "out_dtype(torch.ops.aten.mm.default",+ "j_is_nonzero_multiple_of_8",+ "k_is_nonzero_multiple_of_8",+ "mat2.is_contiguous()",+)+++def _is_fake_tensor(x):+ """Covers exactly the set the substring repr probe did: ``is_fake`` unwraps FunctionalTensor too."""+ try:+ from torch._subclasses.fake_tensor import is_fake+ return bool(is_fake(x))+ except Exception:+ pass+ try:+ from torch._subclasses.fake_tensor import FakeTensor+ return isinstance(x, FakeTensor)+ except Exception:+ return False+++def _make_safe_int_mm(mod, original):+ """A FULL copy of torchao 0.17.0's body, not a wrapper: its very first statement IS the probe."""+ import torch++ out_dtype = mod.out_dtype+ dynamo_is_compiling = mod.dynamo_is_compiling++ @functools.wraps(original)+ def safe_int_mm(input: torch.Tensor, mat2: torch.Tensor) -> torch.Tensor:+ if dynamo_is_compiling() or _is_fake_tensor(input):+ if input.device.type == "cpu":+ # Matmul in int32 is slow on CPU and not supported well by Inductor cpp backend+ return out_dtype(+ torch.ops.aten.mm.default, torch.int32, input.float(), mat2.float()+ )+ return out_dtype(torch.ops.aten.mm.default, torch.int32, input, mat2)++ assert (+ mat2.device == input.device+ ), f"need both tensors to be on the same device but got {mat2.device} and {input.device}"+ device_cpu = "cpu" in [mat2.device.type, input.device.type]+ j_is_nonzero_multiple_of_8 = (input.shape[1] % 8 == 0) and (input.shape[1] > 0)+ k_is_nonzero_multiple_of_8 = (mat2.shape[1] % 8 == 0) and (mat2.shape[1] > 0)+ bad_dimensions_for_cublas = not (j_is_nonzero_multiple_of_8 and k_is_nonzero_multiple_of_8)++ if device_cpu or bad_dimensions_for_cublas:+ return torch.matmul(input.cpu().to(torch.int32), mat2.cpu().to(torch.int32)).to(+ input.device.type+ )++ if not mat2.is_contiguous(): # silently gives incorrect result without this+ mat2 = mat2.contiguous()+ if (not input.is_contiguous()) and (+ input.shape[0] % 8 != 0+ ): # gives cryptic error without this+ input = input.contiguous()+ try:+ return out_dtype(torch.ops.aten.mm.default, torch.int32, input, mat2)+ except Exception:+ # H100 float8: "addmm_cuda" not implemented for 'Float8_e4m3fn'+ return torch.matmul(input.to(torch.float32), mat2.to(torch.float32)).to(torch.int32)++ safe_int_mm.__unsloth_patched__ = True+ safe_int_mm.__unsloth_original__ = original+ return safe_int_mm+++def _patch_torchao_intmm_module(mod):+ """Rebind ``safe_int_mm`` on either torchao home; True when this call installed it."""+ original = getattr(mod, "safe_int_mm", None)+ if original is None or not callable(original):+ return False+ if getattr(original, "__unsloth_patched__", False):+ return False+ # Off the module, never imported here, so this fails closed rather than borrowing our own operators.+ if not hasattr(mod, "out_dtype") or not hasattr(mod, "dynamo_is_compiling"):+ return False+ try:+ source = inspect.getsource(original)+ except Exception:+ return False+ missing = [marker for marker in _TORCHAO_SAFE_INT_MM_MARKERS if marker not in source]+ if missing:+ if "input.__repr__()" not in missing:+ logger.warning(+ "Unsloth: torchao's safe_int_mm still probes input.__repr__() but its body "+ "changed (%s missing), so the capture-safe replacement was not installed. "+ "Eager int8 keeps syncing the device on every linear.",+ ", ".join(missing),+ )+ return False+ patched = _make_safe_int_mm(mod, original)+ try:+ mod.safe_int_mm = patched+ except Exception:+ return False+ # `torchao.kernel` and `torchao.quantization` re-export the function OBJECT, so they need a sweep.+ for name, other in tuple(sys.modules.items()):+ if other is None or not (name == "torchao" or name.startswith("torchao.")):+ continue+ if getattr(other, "safe_int_mm", None) is original:+ try:+ setattr(other, "safe_int_mm", patched)+ except Exception:+ pass+ return True+++class _TorchaoIntmmLoader(importlib.abc.Loader):+ """The real loader, plus the patch once the module body finishes. Failures here leave torchao unpatched."""++ __slots__ = ("_loader",)++ def __init__(self, loader):+ self._loader = loader++ def create_module(self, spec):+ create = getattr(self._loader, "create_module", None)+ if create is None:+ return None+ return create(spec)++ def exec_module(self, module):+ self._loader.exec_module(module)+ try:+ _patch_torchao_intmm_module(module)+ except Exception:+ pass++ def __getattr__(self, attribute):+ return getattr(self._loader, attribute)+++class _TorchaoIntmmPatchFinder(importlib.abc.MetaPathFinder):+ """Inserted at the FRONT of sys.meta_path: the module really exists, so PathFinder would answer first."""++ __slots__ = (_TORCHAO_INTMM_SENTINEL, "_finding")++ def __init__(self):+ setattr(self, _TORCHAO_INTMM_SENTINEL, True)+ self._finding = False # find_spec below walks sys.meta_path again++ def find_spec(+ self,+ fullname,+ path = None,+ target = None,+ ):+ if fullname not in _TORCHAO_INTMM_MODULES or self._finding:+ return None+ self._finding = True+ try:+ spec = importlib.util.find_spec(fullname)+ except Exception:+ return None+ finally:+ self._finding = False+ if spec is None or spec.loader is None:+ return None+ if not hasattr(spec.loader, "exec_module"):+ return None # a loader from before PEP 451; leave the import entirely alone+ try:+ spec.loader = _TorchaoIntmmLoader(spec.loader)+ except Exception:+ return None+ return spec+++def fix_torchao_safe_int_mm_repr_probe():+ """Stop torchao's int8 GEMM from syncing the device to ask whether it is being traced.++ ``safe_int_mm`` branches on ``"FakeTensor" in input.__repr__()``; a real CUDA tensor's repr calls+ ``.item()``, so eager int8 syncs per linear and ``torch.cuda.graph`` capture fails outright. The+ gate is structural, not version-based, and the meta path finder covers a torchao imported after+ this call, which the int8 prequant path needs since it never calls ``quantize_``.+ ``UNSLOTH_TORCHAO_INT_MM_FIX=0`` keeps upstream's behaviour. True when patched or the finder was+ installed, False when there is nothing to do, None when torchao is absent or the fix is off."""+ if os.environ.get(_TORCHAO_INT_MM_ENV, "1").strip() == "0":+ return None+ try:+ if importlib.util.find_spec("torchao") is None:+ return None+ except Exception:+ return None+ patched_now = False+ for name in _TORCHAO_INTMM_MODULES:+ module = sys.modules.get(name)+ if module is None:+ continue+ try:+ patched_now = _patch_torchao_intmm_module(module) or patched_now+ except Exception:+ pass+ if all(name in sys.modules for name in _TORCHAO_INTMM_MODULES):+ return patched_now # nothing left for a finder to catch+ for finder in sys.meta_path:+ if getattr(finder, _TORCHAO_INTMM_SENTINEL, False):+ return patched_now+ sys.meta_path.insert(0, _TorchaoIntmmPatchFinder())+ return True++# `datasets` fingerprints through dill, and dill._dill._is_builtin_module pickles a module by# reference only if its __file__ starts with a sys prefix, ends with an extension suffix, or# contains the literal `site-packages`. An install matching none (pip install --target, a