Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep torchao quantisation from mutating inductor config, and make its int8 GEMM capture-safe (#10525)

Daniel Hanff9206a03acc
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 64f1f8b6ee8..60cd1cfb8e5 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -106,6 +106,7 @@ from .diffusion_memory import (
snapshot_device_memory,
unified_memory_shortfall_message,
)
+from .diffusion_torchao_patches import install_torchao_int_mm_patch
from .diffusion_speed import (
SPEED_DEFAULT,
SPEED_MAX,
@@ -193,6 +194,7 @@ logger = get_logger(__name__)
# diffusers imports xformers on sight, its quantizers torchao.
install_xformers_windows_rocm_stub()
install_torchao_windows_rocm_stub()
+install_torchao_int_mm_patch()
# "gguf" and "single_file" take companions from the base repo; "pipeline" is a full diffusers repo.
@@ -4324,6 +4326,7 @@ class DiffusionBackend:
and compile_eligible(target, is_gguf = False, family = fam)
)
# Speed optims run BEFORE placement, so snapshot the global backend flags first for unload restore.
+ # The dense transformer quant above builds quiet configs, so it mutated none of these flags.
backend_flags_before = snapshot_backend_flags()
# Pick the attention kernel BEFORE compile: auto upgrades to cuDNN fused attention on NVIDIA (~1.18x)
attention_engaged = apply_attention_backend(
diff --git a/studio/backend/core/inference/diffusion_precision.py b/studio/backend/core/inference/diffusion_precision.py
index b21d4c29291..a26b47604eb 100644
--- a/studio/backend/core/inference/diffusion_precision.py
+++ b/studio/backend/core/inference/diffusion_precision.py
@@ -459,12 +459,17 @@ def _cast_nvfp4(encoder: Any, target: Any) -> None:
# exclusions as the int8/fp8 TE modes; require_bf16 skips non-bf16 Linears so the cast engages instead of aborting.
from torchao.quantization import quantize_
from torchao.prototype.mx_formats import NVFP4WeightOnlyConfig
- from .diffusion_transformer_quant import DEFAULT_MIN_LINEAR_FEATURES, make_filter_fn
+ from .diffusion_transformer_quant import (
+ DEFAULT_MIN_LINEAR_FEATURES,
+ _quiet_config,
+ make_filter_fn,
+ )
filter_fn = make_filter_fn(
DEFAULT_MIN_LINEAR_FEATURES, _te_exclude_tokens(encoder), require_bf16 = True
)
- quantize_(encoder, NVFP4WeightOnlyConfig(), filter_fn = filter_fn)
+ # No-op today (the prototype config has no set_inductor_config knob), but no torchao config is built bare.
+ quantize_(encoder, _quiet_config(NVFP4WeightOnlyConfig), filter_fn = filter_fn)
def _warn(logger: Any, what: str, exc: Exception) -> None:
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 888a0829c8f..5d4b7af6f9c 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -47,15 +47,25 @@ SPEED_MAX = "max"
SPEED_MODES = (SPEED_OFF, SPEED_EAGER, SPEED_DEFAULT, SPEED_MAX)
+# (attribute, snapshot key). All but the first are what torchao's recommended_inductor_config_setter() flips.
+_INDUCTOR_FLAGS = (
+ ("emulate_precision_casts", "inductor_emulate_precision_casts"),
+ ("coordinate_descent_tuning", "inductor_coordinate_descent_tuning"),
+ ("coordinate_descent_check_all_directions", "inductor_coordinate_descent_check_all_directions"),
+ ("force_fuse_int_mm_with_mul", "inductor_force_fuse_int_mm_with_mul"),
+ ("fx_graph_cache", "inductor_fx_graph_cache"),
+)
+_INDUCTOR_TRITON_FLAGS = (("unique_kernel_names", "inductor_triton_unique_kernel_names"),)
+
+
def snapshot_backend_flags() -> Optional[dict]:
- """Capture the process-wide torch backend flags this layer may mutate, for restore on unload.
- None if torch is unavailable. Each flag is read defensively so a build missing one (e.g. no
- cuda.matmul on CPU/MPS) still captures the rest, instead of leaking a real mutated flag."""
+ """Capture the process-wide torch backend flags this layer may mutate, for restore on unload. None
+ without torch. Each flag is read defensively: a build missing one still captures the rest."""
try:
import torch
except Exception: # noqa: BLE001 - no torch -> nothing to snapshot/restore
return None
- state: dict[str, bool] = {}
+ state: dict[str, Any] = {}
matmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)
if matmul is not None and hasattr(matmul, "allow_tf32"):
state["matmul_tf32"] = bool(matmul.allow_tf32)
@@ -68,8 +78,21 @@ def snapshot_backend_flags() -> Optional[dict]:
if hasattr(cudnn, "benchmark"):
state["cudnn_benchmark"] = bool(cudnn.benchmark)
inductor_cfg = _inductor_config()
- if inductor_cfg is not None and hasattr(inductor_cfg, "emulate_precision_casts"):
- state["inductor_emulate_precision_casts"] = bool(inductor_cfg.emulate_precision_casts)
+ if inductor_cfg is not None:
+ for attr, key in _INDUCTOR_FLAGS:
+ if hasattr(inductor_cfg, attr):
+ state[key] = bool(getattr(inductor_cfg, attr))
+ triton_cfg = getattr(inductor_cfg, "triton", None)
+ if triton_cfg is not None:
+ for attr, key in _INDUCTOR_TRITON_FLAGS:
+ if hasattr(triton_cfg, attr):
+ state[key] = bool(getattr(triton_cfg, attr))
+ getter = getattr(torch, "get_float32_matmul_precision", None)
+ if callable(getter):
+ try:
+ state["matmul_precision"] = str(getter())
+ except Exception: # noqa: BLE001 - unreadable on this build: restore the rest
+ pass
return state
@@ -90,13 +113,25 @@ def restore_backend_flags(state: Optional[dict]) -> None:
except Exception: # noqa: BLE001 - best-effort per-flag restore
pass
+ # FIRST: on some builds set_float32_matmul_precision also writes matmul.allow_tf32.
+ setter = getattr(torch, "set_float32_matmul_precision", None)
+ if state.get("matmul_precision") and callable(setter):
+ try:
+ setter(state["matmul_precision"])
+ except Exception: # noqa: BLE001 - best-effort per-flag restore
+ pass
matmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)
_set(matmul, "allow_tf32", "matmul_tf32")
_set(matmul, "allow_fp16_accumulation", "matmul_fp16_accum")
cudnn = getattr(torch.backends, "cudnn", None)
_set(cudnn, "allow_tf32", "cudnn_tf32")
_set(cudnn, "benchmark", "cudnn_benchmark")
- _set(_inductor_config(), "emulate_precision_casts", "inductor_emulate_precision_casts")
+ inductor_cfg = _inductor_config()
+ for attr, key in _INDUCTOR_FLAGS:
+ _set(inductor_cfg, attr, key)
+ triton_cfg = getattr(inductor_cfg, "triton", None) if inductor_cfg is not None else None
+ for attr, key in _INDUCTOR_TRITON_FLAGS:
+ _set(triton_cfg, attr, key)
def _inductor_config() -> Any:
diff --git a/studio/backend/core/inference/diffusion_torchao_patches.py b/studio/backend/core/inference/diffusion_torchao_patches.py
new file mode 100644
index 00000000000..dacb1922e05
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_torchao_patches.py
@@ -0,0 +1,234 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Studio's copy of ``unsloth/import_fixes.py::fix_torchao_safe_int_mm_repr_probe``, not an import
+of it: the diffusion process must not execute ``unsloth/__init__``. The two copies are mutually inert.
+
+torchao's int8 GEMM asks whether it is being traced by formatting its input's repr, which on a real
+CUDA tensor calls ``.item()``: a device sync per eager int8 linear, and an outright
+``cudaErrorStreamCaptureUnsupported`` inside ``torch.cuda.graph``. Installed from a meta path finder
+at import time, not at quantise time, because the int8 prequant path only ``torch.load``s torchao
+subclasses and never calls ``quantize_``.
+"""
+
+from __future__ import annotations
+
+import functools
+import importlib.abc
+import importlib.util
+import inspect
+import logging
+import os
+import sys
+
+logger = logging.getLogger(__name__)
+
+
+_TORCHAO_INTMM_MODULES = (
+ "torchao.kernel.intmm", # every release up to and including 0.18.0
+ "torchao.quantization.quantize_.workflows.int8.kernels", # main after pytorch/ao#4718
+)
+_TORCHAO_INTMM_MODULE = _TORCHAO_INTMM_MODULES[0] # kept for callers that named the old home
+_TORCHAO_INTMM_SENTINEL = "__unsloth_torchao_intmm_patch__"
+_TORCHAO_INT_MM_ENV = "UNSLOTH_TORCHAO_INT_MM_FIX"
+
+# The copy below hard-codes torchao's cuBLAS guards and fp32 fallback: ALL must be in the installed source.
+_TORCHAO_SAFE_INT_MM_MARKERS = (
+ "input.__repr__()",
+ "dynamo_is_compiling()",
+ "out_dtype(torch.ops.aten.mm.default",
+ "j_is_nonzero_multiple_of_8",
+ "k_is_nonzero_multiple_of_8",
+ "mat2.is_contiguous()",
+)
+
+
+def _is_fake_tensor(x):
+ """Covers exactly the set the substring repr probe did: ``is_fake`` unwraps FunctionalTensor too."""
+ try:
+ from torch._subclasses.fake_tensor import is_fake
+ return bool(is_fake(x))
+ except Exception:
+ pass
+ try:
+ from torch._subclasses.fake_tensor import FakeTensor
+ return isinstance(x, FakeTensor)
+ except Exception:
+ return False
+
+
+def _make_safe_int_mm(mod, original):
+ """A FULL copy of torchao 0.17.0's body, not a wrapper: its very first statement IS the probe."""
+ import torch
+
+ out_dtype = mod.out_dtype
+ dynamo_is_compiling = mod.dynamo_is_compiling
+
+ @functools.wraps(original)
+ def safe_int_mm(input: torch.Tensor, mat2: torch.Tensor) -> torch.Tensor:
+ if dynamo_is_compiling() or _is_fake_tensor(input):
+ if input.device.type == "cpu":
+ # Matmul in int32 is slow on CPU and not supported well by Inductor cpp backend
+ return out_dtype(
+ torch.ops.aten.mm.default, torch.int32, input.float(), mat2.float()
+ )
+ return out_dtype(torch.ops.aten.mm.default, torch.int32, input, mat2)
+
+ assert (
+ mat2.device == input.device
+ ), f"need both tensors to be on the same device but got {mat2.device} and {input.device}"
+ device_cpu = "cpu" in [mat2.device.type, input.device.type]
+ j_is_nonzero_multiple_of_8 = (input.shape[1] % 8 == 0) and (input.shape[1] > 0)
+ k_is_nonzero_multiple_of_8 = (mat2.shape[1] % 8 == 0) and (mat2.shape[1] > 0)
+ bad_dimensions_for_cublas = not (j_is_nonzero_multiple_of_8 and k_is_nonzero_multiple_of_8)
+
+ if device_cpu or bad_dimensions_for_cublas:
+ return torch.matmul(input.cpu().to(torch.int32), mat2.cpu().to(torch.int32)).to(
+ input.device.type
+ )
+
+ if not mat2.is_contiguous(): # silently gives incorrect result without this
+ mat2 = mat2.contiguous()
+ if (not input.is_contiguous()) and (
+ input.shape[0] % 8 != 0
+ ): # gives cryptic error without this
+ input = input.contiguous()
+ try:
+ return out_dtype(torch.ops.aten.mm.default, torch.int32, input, mat2)
+ except Exception:
+ # H100 float8: "addmm_cuda" not implemented for 'Float8_e4m3fn'
+ return torch.matmul(input.to(torch.float32), mat2.to(torch.float32)).to(torch.int32)
+
+ safe_int_mm.__unsloth_patched__ = True
+ safe_int_mm.__unsloth_original__ = original
+ return safe_int_mm
+
+
+def _patch_torchao_intmm_module(mod):
+ """Rebind ``safe_int_mm`` on either torchao home; True when this call installed it."""
+ original = getattr(mod, "safe_int_mm", None)
+ if original is None or not callable(original):
+ return False
+ if getattr(original, "__unsloth_patched__", False):
+ return False
+ # Off the module, never imported here, so this fails closed rather than borrowing our own operators.
+ if not hasattr(mod, "out_dtype") or not hasattr(mod, "dynamo_is_compiling"):
+ return False
+ try:
+ source = inspect.getsource(original)
+ except Exception:
+ return False
+ missing = [marker for marker in _TORCHAO_SAFE_INT_MM_MARKERS if marker not in source]
+ if missing:
+ if "input.__repr__()" not in missing:
+ logger.warning(
+ "Unsloth: torchao's safe_int_mm still probes input.__repr__() but its body "
+ "changed (%s missing), so the capture-safe replacement was not installed. "
+ "Eager int8 keeps syncing the device on every linear.",
+ ", ".join(missing),
+ )
+ return False
+ patched = _make_safe_int_mm(mod, original)
+ try:
+ mod.safe_int_mm = patched
+ except Exception:
+ return False
+ # `torchao.kernel` and `torchao.quantization` re-export the function OBJECT, so they need a sweep.
+ for name, other in tuple(sys.modules.items()):
+ if other is None or not (name == "torchao" or name.startswith("torchao.")):
+ continue
+ if getattr(other, "safe_int_mm", None) is original:
+ try:
+ setattr(other, "safe_int_mm", patched)
+ except Exception:
+ pass
+ return True
+
+
+class _TorchaoIntmmLoader(importlib.abc.Loader):
+ """The real loader, plus the patch once the module body finishes. Failures here leave torchao unpatched."""
+
+ __slots__ = ("_loader",)
+
+ def __init__(self, loader):
+ self._loader = loader
+
+ def create_module(self, spec):
+ create = getattr(self._loader, "create_module", None)
+ if create is None:
+ return None
+ return create(spec)
+
+ def exec_module(self, module):
+ self._loader.exec_module(module)
+ try:
+ _patch_torchao_intmm_module(module)
+ except Exception:
+ pass
+
+ def __getattr__(self, attribute):
+ return getattr(self._loader, attribute)
+
+
+class _TorchaoIntmmPatchFinder(importlib.abc.MetaPathFinder):
+ """Inserted at the FRONT of sys.meta_path: the module really exists, so PathFinder would answer first."""
+
+ __slots__ = (_TORCHAO_INTMM_SENTINEL, "_finding")
+
+ def __init__(self):
+ setattr(self, _TORCHAO_INTMM_SENTINEL, True)
+ self._finding = False # find_spec below walks sys.meta_path again
+
+ def find_spec(
+ self,
+ fullname,
+ path = None,
+ target = None,
+ ):
+ if fullname not in _TORCHAO_INTMM_MODULES or self._finding:
+ return None
+ self._finding = True
+ try:
+ spec = importlib.util.find_spec(fullname)
+ except Exception:
+ return None
+ finally:
+ self._finding = False
+ if spec is None or spec.loader is None:
+ return None
+ if not hasattr(spec.loader, "exec_module"):
+ return None # a loader from before PEP 451; leave the import entirely alone
+ try:
+ spec.loader = _TorchaoIntmmLoader(spec.loader)
+ except Exception:
+ return None
+ return spec
+
+
+def install_torchao_int_mm_patch():
+ """Install the capture-safe ``safe_int_mm`` if torchao is present; idempotent, so several modules may each
+ ask for it. ``UNSLOTH_TORCHAO_INT_MM_FIX=0`` keeps upstream's behaviour. True when patched or the finder was
+ installed, False when there is nothing to do, None when torchao is absent or the fix is off."""
+ if os.environ.get(_TORCHAO_INT_MM_ENV, "1").strip() == "0":
+ return None
+ try:
+ if importlib.util.find_spec("torchao") is None:
+ return None
+ except Exception:
+ return None
+ patched_now = False
+ for name in _TORCHAO_INTMM_MODULES:
+ module = sys.modules.get(name)
+ if module is None:
+ continue
+ try:
+ patched_now = _patch_torchao_intmm_module(module) or patched_now
+ except Exception:
+ pass
+ if all(name in sys.modules for name in _TORCHAO_INTMM_MODULES):
+ return patched_now # nothing left for a finder to catch
+ for finder in sys.meta_path:
+ if getattr(finder, _TORCHAO_INTMM_SENTINEL, False):
+ return patched_now
+ sys.meta_path.insert(0, _TorchaoIntmmPatchFinder())
+ return True
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 75cd063ed4c..267fd623d2a 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -23,6 +23,8 @@ probe is best-effort: an unsupported scheme yields None and the caller loads GGU
from __future__ import annotations
+import inspect as _inspect
+import os as _os
import re as _re
import sys as _sys
import threading as _threading
@@ -30,6 +32,10 @@ from typing import Any, Optional
# stdlib-only module (no torch), so this stays inside the "imported lazily" promise above.
from core._torchao_stub import is_stubbed, torch_is_rocm
+from .diffusion_torchao_patches import install_torchao_int_mm_patch
+
+# Also runs in the spawned smoke-probe child, which imports this module and nothing else of the backend.
+install_torchao_int_mm_patch()
TQ_INT8 = "int8"
TQ_FP8 = "fp8"
@@ -964,6 +970,37 @@ def _resolve_fast_accum(fast_accum: Optional[bool]) -> bool:
return True if fast_accum is None else bool(fast_accum)
+# torchao's config handlers call the PROCESS-WIDE ``recommended_inductor_config_setter()`` unless
+# ``set_inductor_config`` is False, and two of its flags change results rather than only speed: coordinate-descent
+# tuning makes the render differ between processes on one seed, and set_float32_matmul_precision("high") reaches
+# every fp32 op in the pipeline (VAE, norms). UNSLOTH_TORCHAO_INDUCTOR_CONFIG=1 restores it, for A/B benchmarking.
+_TORCHAO_INDUCTOR_CONFIG_ENV = "UNSLOTH_TORCHAO_INDUCTOR_CONFIG"
+
+
+def _torchao_may_set_inductor_config() -> bool:
+ return (_os.environ.get(_TORCHAO_INDUCTOR_CONFIG_ENV) or "").strip().lower() in (
+ "1",
+ "on",
+ "true",
+ "yes",
+ )
+
+
+def _quiet_config(cls: Any, **kwargs: Any) -> Any:
+ """Build a torchao config with ``set_inductor_config = False`` when the class accepts it. Guarded by
+ signature, not version: the mx_formats configs lack the kwarg and never call the setter anyway."""
+ if not _torchao_may_set_inductor_config():
+ try:
+ if "set_inductor_config" in _inspect.signature(cls).parameters:
+ kwargs = {**kwargs, "set_inductor_config": False}
+ except (
+ TypeError,
+ ValueError,
+ ): # C-implemented or otherwise unintrospectable: leave it alone
+ pass
+ return cls(**kwargs)
+
+
def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
"""The torchao dynamic-activation config for ``scheme`` (lazy imports per branch).
@@ -974,7 +1011,7 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
)
if scheme == TQ_INT8:
- return Int8DynamicActivationInt8WeightConfig()
+ return _quiet_config(Int8DynamicActivationInt8WeightConfig)
if scheme == TQ_FP8:
# Per-ROW granularity (per-token activation + per-channel weight scale) is REQUIRED: torchao defaults to
# per-TENSOR, where one Z-Image outlier near 6.6e4 forces a tensor-wide scale that pushes normal values below
@@ -982,11 +1019,10 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
# it falls to int8. fast accumulate (fp8 only) follows GPU class unless forced: consumer cards run fp8 ~2x
# faster with FP16 accumulate. activation_value_lb floors the per-row scale: an ALL-ZERO row otherwise yields
# scale 0, NaN qdata, black frames.
- import inspect
from torchao.quantization import PerRow
fp8_kwargs: dict = {"granularity": PerRow()}
- config_params = inspect.signature(Float8DynamicActivationFloat8WeightConfig).parameters
+ config_params = _inspect.signature(Float8DynamicActivationFloat8WeightConfig).parameters
if "activation_value_lb" in config_params:
fp8_kwargs["activation_value_lb"] = 1e-12
# Pin the plain-torch quantize kernel: the default AUTO switches to the MSLK kernel whenever an mslk package is
@@ -1002,32 +1038,35 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
pass
try:
from torchao.float8 import Float8MMConfig
- return Float8DynamicActivationFloat8WeightConfig(
+ return _quiet_config(
+ Float8DynamicActivationFloat8WeightConfig,
mm_config = Float8MMConfig(use_fast_accum = _resolve_fast_accum(fast_accum)),
**fp8_kwargs,
)
except Exception: # noqa: BLE001 - older torchao without the explicit mm knob
- return Float8DynamicActivationFloat8WeightConfig(**fp8_kwargs)
+ return _quiet_config(Float8DynamicActivationFloat8WeightConfig, **fp8_kwargs)
if scheme == TQ_NVFP4:
from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
# Select the CUTLASS FP4 path, not the default Triton kernel (which needs MSLK): on a Blackwell box with CUTLASS
# FP4 but no MSLK the default fails the smoke probe and falls back to GGUF.
try:
- return NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False)
+ return _quiet_config(NVFP4DynamicActivationNVFP4WeightConfig, use_triton_kernel = False)
except TypeError: # older torchao without the knob
- return NVFP4DynamicActivationNVFP4WeightConfig()
+ return _quiet_config(NVFP4DynamicActivationNVFP4WeightConfig)
if scheme == TQ_MXFP8:
import torch
from torchao.prototype.mx_formats import MXDynamicActivationMXWeightConfig
try:
- return MXDynamicActivationMXWeightConfig(
- activation_dtype = torch.float8_e4m3fn, weight_dtype = torch.float8_e4m3fn
+ return _quiet_config(
+ MXDynamicActivationMXWeightConfig,
+ activation_dtype = torch.float8_e4m3fn,
+ weight_dtype = torch.float8_e4m3fn,
)
except (TypeError, AttributeError):
# TypeError: older torchao without the explicit dtype knobs. AttributeError: a torch build without
# torch.float8_e4m3fn.
- return MXDynamicActivationMXWeightConfig()
+ return _quiet_config(MXDynamicActivationMXWeightConfig)
raise ValueError(f"unknown transformer quant scheme '{scheme}'")
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index d0c8486b56c..73ef1031983 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -82,6 +82,7 @@ from .diffusion_memory import (
reclaim_offload_host_memory,
settled_snapshot_device_memory,
)
+from .diffusion_torchao_patches import install_torchao_int_mm_patch
from .diffusion_speed import (
SPEED_DEFAULT,
SPEED_EAGER,
@@ -170,6 +171,9 @@ from core.inference.diffusion import hub_cache_dir
logger = get_logger(__name__)
+# Asked for here as well as in the image backend, so the video path never depends on that import order.
+install_torchao_int_mm_patch()
+
# Load kinds (mirror the image backend): gguf (single-file GGUF DiT + base repo), single_file (safetensors DiT),
# pipeline (full diffusers repo)
_MODEL_KINDS = frozenset({"gguf", "single_file", "pipeline"})
@@ -3791,6 +3795,10 @@ class VideoBackend:
# _SecondDiTView(pipe)); single-DiT resolves to (pipe,).
views = _views_for(pipe, fam)
+ # Before the transformer quant, the first mutator; until the state commit a failure restores them itself.
+ backend_flags = snapshot_backend_flags()
+ self._precommit_globals = (_load_token, backend_flags)
+
# dense transformer quant (opt-in, pipeline-kind only): torchao-quantise the dense bf16 DiT in place onto the
# low-precision tensor cores. CUDA + bf16 only, best-effort. Quant must precede compile (eager is ~30x slower).
transformer_quant_engaged: Optional[str] = None
@@ -3938,10 +3946,6 @@ class VideoBackend:
"(quantized transformer must be compiled; eager is ~30x slower)"
)
effective_speed = SPEED_DEFAULT
- backend_flags = snapshot_backend_flags()
- # Until the state commit hands ownership to _teardown_state_locked, a failure has to restore these process-wide
- # flags itself. Registered BEFORE the first mutation, as above.
- self._precommit_globals = (_load_token, backend_flags)
# Step cache tri-state: unset/"auto" -> FBCACHE_MIN_STEPS policy (re-checked per generation); "off"/"fbcache"
# pinned. Run per expert.
cache_request = normalize_transformer_cache(transformer_cache)
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index 4baf6ea478a..6f8382c5dca 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -356,15 +356,17 @@ def _int8_quantize_base(transformer, family: Optional[str] = None) -> None:
generic nor the family exclusions, so without this they are quantized bare and raise on the
first forward exactly like an unexcluded one."""
from core.inference.diffusion_transformer_quant import (
+ _quiet_config,
apply_small_m_padding,
exclude_tokens_for_scheme,
make_filter_fn,
)
from torchao.quantization import Int8WeightOnlyConfig, quantize_
+ # Int8WeightOnlyConfig defaults to set_inductor_config=True, process-wide, and the trainer compiles next.
quantize_(
transformer,
- Int8WeightOnlyConfig(),
+ _quiet_config(Int8WeightOnlyConfig),
filter_fn = make_filter_fn(
512, exclude_name_tokens = exclude_tokens_for_scheme("int8", family)
),
diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.py
index 8bc24dc9e29..fd04aaedee5 100644
--- a/studio/backend/core/training/diffusion_train_common.py
+++ b/studio/backend/core/training/diffusion_train_common.py
@@ -37,6 +37,7 @@ from core.inference.diffusion_families import (
supported_family_names,
trainable_family_names,
)
+from core.inference.diffusion_torchao_patches import install_torchao_int_mm_patch
from core.inference.video_families import detect_video_family, supported_video_family_names
from utils.paths.path_utils import drop_appledouble_metadata
@@ -44,6 +45,8 @@ from utils.paths.path_utils import drop_appledouble_metadata
# does not carry over. Both import this module first.
install_xformers_windows_rocm_stub()
install_torchao_windows_rocm_stub()
+# Same child: the DiT trainer's int8 base-weight quantisation goes through torchao.
+install_torchao_int_mm_patch()
# Default LoRA target modules: the attention projections common to the SDXL U-Net and the DiTs (the
# diffusers/kohya convention). A family may override this.
diff --git a/studio/backend/tests/test_diffusion_precision.py b/studio/backend/tests/test_diffusion_precision.py
index d8ad0adccc2..7278ecac6d0 100644
--- a/studio/backend/tests/test_diffusion_precision.py
+++ b/studio/backend/tests/test_diffusion_precision.py
@@ -16,6 +16,7 @@ import pytest
import core.inference.diffusion_precision as dp
from core.inference.diffusion_precision import (
+ _cast_fp8_dynamic,
TE_QUANT_FP8,
TE_QUANT_FP8_DYNAMIC,
TE_QUANT_INT8,
@@ -81,6 +82,7 @@ def _stub_casters(monkeypatch, recorder):
dtq.make_filter_fn = lambda min_features, exclude = (), *, require_bf16 = False: (
lambda module, fqn = "": True
)
+ dtq._quiet_config = lambda cls, **kw: cls(**kw)
monkeypatch.setitem(sys.modules, "core.inference.diffusion_transformer_quant", dtq)
@@ -321,6 +323,7 @@ def _stub_transformer_quant(monkeypatch, captured):
dtq.TQ_FP8 = "fp8"
dtq.DEFAULT_MIN_LINEAR_FEATURES = 512
dtq._make_quant_config = lambda scheme, *a, **k: f"cfg:{scheme}"
+ dtq._quiet_config = lambda cls, **kw: cls(**kw)
dtq.exclude_tokens_for_scheme = lambda scheme: ("modulation",)
def _make_filter_fn(
@@ -519,3 +522,64 @@ def test_int8_without_a_schedule_reports_fp8_as_the_effective_mode():
# Every other mode is its own effective mode, and absent stays absent.
assert effective_te_quant(TE_QUANT_FP8_DYNAMIC, "z-image-turbo") == TE_QUANT_FP8_DYNAMIC
assert effective_te_quant(None, "qwen-image") is None
+
+
+def test_nvfp4_te_cast_builds_its_config_through_quiet_config(monkeypatch):
+ """NVFP4 has no set_inductor_config knob, so the risk of routing it through _quiet_config is a TypeError."""
+ _stub_torch(monkeypatch, cc = (10, 0))
+ captured: dict = {}
+ _stub_transformer_quant(monkeypatch, captured)
+ seen: list = []
+ dtq = sys.modules["core.inference.diffusion_transformer_quant"]
+ dtq._quiet_config = lambda cls, **kw: seen.append((cls, kw)) or cls(**kw)
+ outcome = quantize_text_encoders(
+ types.SimpleNamespace(text_encoder = object()), _target(), mode = "nvfp4"
+ )
+ assert outcome.mode == TE_QUANT_NVFP4
+ assert len(seen) == 1 and seen[0][1] == {}
+ assert captured["config"] == "nvfp4cfg"
+
+
+def test_int8_and_fp8_dynamic_te_casts_reuse_the_quiet_factory(monkeypatch):
+ torch = _stub_torch(monkeypatch, cc = (10, 0))
+ captured: dict = {}
+ _stub_transformer_quant(monkeypatch, captured)
+ layers = torch.nn.ModuleList([object() for _ in range(8)])
+ enc = types.SimpleNamespace(_keep_in_fp32_modules = ["wo"])
+ enc.named_modules = lambda: [("model.layers", layers)]
+ _cast_int8_selective(enc, _target(), 3, 0)
+ assert captured["config"] == "cfg:int8"
+ _cast_fp8_dynamic(enc, _target())
+ assert captured["config"] == "cfg:fp8"
+
+
+def test_no_torchao_config_is_constructed_outside_quiet_config():
+ """Every torchao config must be built through ``_quiet_config``, whose default makes renders differ."""
+ import ast
+ from pathlib import Path
+
+ backend = Path(__file__).resolve().parents[1]
+ files = sorted((backend / "core" / "inference").glob("*.py")) + [
+ backend / "core" / "training" / "diffusion_dit_trainer.py",
+ ]
+ offenders: list[str] = []
+ for path in files:
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call):
+ continue
+ func = node.func
+ name = (
+ func.id
+ if isinstance(func, ast.Name)
+ else func.attr
+ if isinstance(func, ast.Attribute)
+ else ""
+ )
+ if not (name.endswith("WeightConfig") or name.endswith("WeightOnlyConfig")):
+ continue
+ offenders.append(f"{path.relative_to(backend)}:{node.lineno} {name}(...)")
+ assert not offenders, (
+ "torchao config constructed outside _quiet_config (pass the CLASS as its first argument instead):\n "
+ + "\n ".join(offenders)
+ )
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index 9943c4f35ee..455fb0b965e 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -138,6 +138,7 @@ def test_compile_eligible_requires_bf16_cuda_friendly(monkeypatch):
def test_snapshot_restore_backend_flags(monkeypatch):
torch = _stub_torch(monkeypatch)
snap = snapshot_backend_flags()
+ # The plain stub torch has no _inductor and no get_float32_matmul_precision, so none of those keys appear.
assert snap == {"matmul_tf32": False, "cudnn_tf32": False, "cudnn_benchmark": False}
# An opt-in max run flips the globals on...
torch.backends.cuda.matmul.allow_tf32 = True
@@ -852,3 +853,139 @@ def test_linux_and_mac_are_not_asked_about_triton(monkeypatch):
_clear_runtime_cache()
assert ds_mod.torch_compile_runtime_available() is True
_clear_runtime_cache()
+
+
+_TORCHAO_FLAGS = (
+ "coordinate_descent_tuning",
+ "coordinate_descent_check_all_directions",
+ "force_fuse_int_mm_with_mul",
+ "fx_graph_cache",
+)
+
+
+def _stub_full_inductor_config(torch):
+ cfg = types.SimpleNamespace(
+ emulate_precision_casts = False,
+ triton = types.SimpleNamespace(unique_kernel_names = False),
+ **{name: False for name in _TORCHAO_FLAGS},
+ )
+ torch._inductor = types.SimpleNamespace(config = cfg)
+ return cfg
+
+
+def _stub_matmul_precision(
+ torch,
+ calls: list,
+ initial = "highest",
+):
+ cell = {"v": initial}
+
+ def _get():
+ return cell["v"]
+
+ def _set(v):
+ calls.append(("precision", v))
+ cell["v"] = v
+
+ torch.get_float32_matmul_precision = _get
+ torch.set_float32_matmul_precision = _set
+ return cell
+
+
+def test_snapshot_restores_torchao_inductor_flags(monkeypatch):
+ torch = _stub_torch(monkeypatch)
+ cfg = _stub_full_inductor_config(torch)
+ snap = snapshot_backend_flags()
+ for name in _TORCHAO_FLAGS:
+ assert snap[f"inductor_{name}"] is False
+ assert snap["inductor_triton_unique_kernel_names"] is False
+ for name in _TORCHAO_FLAGS:
+ setattr(cfg, name, True)
+ cfg.triton.unique_kernel_names = True
+ cfg.emulate_precision_casts = True
+ restore_backend_flags(snap)
+ for name in _TORCHAO_FLAGS:
+ assert getattr(cfg, name) is False, name
+ assert cfg.triton.unique_kernel_names is False
+ assert cfg.emulate_precision_casts is False
+
+
+def test_snapshot_restores_float32_matmul_precision(monkeypatch):
+ torch = _stub_torch(monkeypatch)
+ calls: list = []
+ cell = _stub_matmul_precision(torch, calls, initial = "highest")
+ snap = snapshot_backend_flags()
+ assert snap["matmul_precision"] == "highest"
+ cell["v"] = "high"
+ restore_backend_flags(snap)
+ assert cell["v"] == "highest"
+
+
+def test_matmul_precision_is_restored_before_tf32(monkeypatch):
+ torch = _stub_torch(monkeypatch)
+ calls: list = []
+ _stub_matmul_precision(torch, calls, initial = "highest")
+
+ class _Matmul:
+ def __init__(self):
+ self._tf32 = False
+
+ @property
+ def allow_tf32(self):
+ return self._tf32
+
+ @allow_tf32.setter
+ def allow_tf32(self, v):
+ calls.append(("tf32", v))
+ self._tf32 = v
+
+ torch.backends.cuda.matmul = _Matmul()
+ snap = snapshot_backend_flags()
+ calls.clear()
+ restore_backend_flags(snap)
+ kinds = [k for k, _ in calls]
+ assert kinds.index("precision") < kinds.index("tf32"), calls
+
+
+def test_snapshot_skips_inductor_flags_a_build_lacks(monkeypatch):
+ torch = _stub_torch(monkeypatch)
+ cfg = _stub_inductor_config(monkeypatch, torch, emulate = False)
+ snap = snapshot_backend_flags()
+ assert snap["inductor_emulate_precision_casts"] is False
+ for name in _TORCHAO_FLAGS:
+ assert f"inductor_{name}" not in snap
+ assert "inductor_triton_unique_kernel_names" not in snap
+ assert "matmul_precision" not in snap
+ cfg.emulate_precision_casts = True
+ restore_backend_flags(snap)
+ assert cfg.emulate_precision_casts is False
+
+
+def test_video_snapshot_precedes_transformer_quant():
+ """A failed load and an unload must both restore the pre-quant backend flags."""
+ import ast
+ from pathlib import Path
+
+ path = Path(__file__).resolve().parents[1] / "core" / "inference" / "video.py"
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.FunctionDef):
+ continue
+ snaps = [
+ c.lineno
+ for c in ast.walk(node)
+ if isinstance(c, ast.Call) and getattr(c.func, "id", None) == "snapshot_backend_flags"
+ ]
+ quants = [
+ c.lineno
+ for c in ast.walk(node)
+ if isinstance(c, ast.Call) and getattr(c.func, "id", None) == "quantize_transformer"
+ ]
+ if snaps and quants:
+ assert (
+ min(snaps) < min(quants)
+ ), f"{node.name}: snapshot_backend_flags at {snaps} must precede quantize_transformer at {quants}"
+ return
+ raise AssertionError(
+ "no video.py function calls both snapshot_backend_flags and quantize_transformer"
+ )
diff --git a/studio/backend/tests/test_diffusion_transformer_quant.py b/studio/backend/tests/test_diffusion_transformer_quant.py
index 6777fdc49f2..8bcebbe38bb 100644
--- a/studio/backend/tests/test_diffusion_transformer_quant.py
+++ b/studio/backend/tests/test_diffusion_transformer_quant.py
@@ -1494,3 +1494,107 @@ def test_paths_are_stripped_without_eating_dotted_module_names():
assert "C:\\Users" not in tq._strip_paths(
r"ImportError: DLL load failed: C:\Users\me\.venv\Lib\site-packages\torchao\_C.pyd"
)
+
+
+class _RecordingConfig:
+ """Stands in for a torchao config dataclass: records every kwarg it was built with."""
+
+ built: list = []
+
+ def __init__(
+ self,
+ *,
+ set_inductor_config = True,
+ **kw,
+ ):
+ self.set_inductor_config = set_inductor_config
+ self.kw = kw
+ type(self).built.append(self)
+
+
+def _stub_torchao_configs(
+ monkeypatch,
+ *,
+ int8 = None,
+ fp8 = None,
+):
+ tqz = types.ModuleType("torchao.quantization")
+ tqz.quantize_ = lambda *a, **k: None
+ tqz.PerRow = lambda: "per_row"
+ tqz.Int8DynamicActivationInt8WeightConfig = int8 or _RecordingConfig
+ tqz.Float8DynamicActivationFloat8WeightConfig = fp8 or _RecordingConfig
+ monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
+ # _make_quant_config(fp8) also reaches for these two; the older-torchao shape is absent.
+ monkeypatch.setitem(sys.modules, "torchao.float8", None)
+ monkeypatch.setitem(sys.modules, "torchao.quantization.quantize_", None)
+ return tqz
+
+
+def test_int8_config_disables_torchao_inductor_config(monkeypatch):
+ _RecordingConfig.built = []
+ _stub_torchao_configs(monkeypatch)
+ cfg = tq._make_quant_config(TQ_INT8)
+ assert isinstance(cfg, _RecordingConfig)
+ assert cfg.set_inductor_config is False
+ assert cfg.kw == {}
+
+
+def test_fp8_config_disables_torchao_inductor_config_and_keeps_its_kwargs(monkeypatch):
+ _RecordingConfig.built = []
+ _stub_torchao_configs(monkeypatch)
+ cfg = tq._make_quant_config(TQ_FP8)
+ assert isinstance(cfg, _RecordingConfig)
+ assert cfg.set_inductor_config is False
+ assert cfg.kw["granularity"] == "per_row"
+
+
+def test_quiet_config_omits_the_kwarg_when_the_class_does_not_accept_it():
+ """The mx_formats configs have no set_inductor_config, so the helper must not raise TypeError on them."""
+
+ class _NoKnob:
+ def __init__(self, use_triton_kernel = True):
+ self.use_triton_kernel = use_triton_kernel
+
+ cfg = tq._quiet_config(_NoKnob, use_triton_kernel = False)
+ assert cfg.use_triton_kernel is False and not hasattr(cfg, "set_inductor_config")
+ assert tq._quiet_config(lambda: "cfg") == "cfg"
+
+
+def test_quiet_config_is_reenabled_by_env(monkeypatch):
+ """UNSLOTH_TORCHAO_INDUCTOR_CONFIG=1 restores torchao's default so the two behaviours can be benchmarked A/B."""
+ monkeypatch.setenv("UNSLOTH_TORCHAO_INDUCTOR_CONFIG", "1")
+ assert tq._quiet_config(_RecordingConfig).set_inductor_config is True
+ monkeypatch.setenv("UNSLOTH_TORCHAO_INDUCTOR_CONFIG", "0")
+ assert tq._quiet_config(_RecordingConfig).set_inductor_config is False
+ monkeypatch.delenv("UNSLOTH_TORCHAO_INDUCTOR_CONFIG")
+ assert tq._quiet_config(_RecordingConfig).set_inductor_config is False
+
+
+def test_quiet_config_tolerates_an_unintrospectable_class(monkeypatch):
+ import inspect
+
+ real = inspect.signature
+
+ def _boom(obj):
+ if obj is _RecordingConfig:
+ raise ValueError("no signature found for builtin")
+ return real(obj)
+
+ monkeypatch.setattr(tq._inspect, "signature", _boom)
+ cfg = tq._quiet_config(_RecordingConfig)
+ # The kwarg could not be proven to exist, so the class is built with its own default rather than crashing.
+ assert cfg.set_inductor_config is True
+
+
+def test_real_torchao_configs_carry_set_inductor_config_false():
+ pytest.importorskip("torchao.quantization")
+ torch = pytest.importorskip("torch")
+ ic = getattr(getattr(torch, "_inductor", None), "config", None)
+ before = getattr(ic, "coordinate_descent_tuning", None) if ic is not None else None
+ for scheme in (TQ_INT8, TQ_FP8):
+ cfg = tq._make_quant_config(scheme)
+ if not hasattr(cfg, "set_inductor_config"):
+ pytest.skip("torchao config without set_inductor_config")
+ assert cfg.set_inductor_config is False, scheme
+ if ic is not None:
+ assert getattr(ic, "coordinate_descent_tuning", None) == before
diff --git a/studio/backend/tests/test_torchao_intmm_patch_wiring.py b/studio/backend/tests/test_torchao_intmm_patch_wiring.py
new file mode 100644
index 00000000000..163a0e9be16
--- /dev/null
+++ b/studio/backend/tests/test_torchao_intmm_patch_wiring.py
@@ -0,0 +1,192 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Invariants for Studio's copy of the torchao ``safe_int_mm`` fix, none of which raise on their own."""
+
+from __future__ import annotations
+
+import ast
+import types
+from pathlib import Path
+
+import pytest
+
+from core.inference.diffusion_torchao_patches import (
+ _TORCHAO_INTMM_MODULES,
+ _TorchaoIntmmLoader,
+ _TorchaoIntmmPatchFinder,
+ _patch_torchao_intmm_module,
+ install_torchao_int_mm_patch,
+)
+
+_BACKEND = Path(__file__).resolve().parent.parent
+_CORE = _BACKEND / "core"
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+_IMPORT_FIXES = _REPO_ROOT / "unsloth" / "import_fixes.py"
+_PATCH_MODULE = _CORE / "inference" / "diffusion_torchao_patches.py"
+_INSTALL = install_torchao_int_mm_patch.__name__
+
+# Every module that can be the first thing in its process to reach torchao.
+_ENTRYPOINTS = [
+ _CORE / "inference" / "diffusion.py",
+ _CORE / "inference" / "video.py",
+ _CORE / "inference" / "diffusion_transformer_quant.py",
+ _CORE / "training" / "diffusion_train_common.py",
+]
+
+_SHARED_FUNCTIONS = ("_is_fake_tensor", "_make_safe_int_mm", "_patch_torchao_intmm_module")
+
+
+def _install_call_linenos(node) -> list[int]:
+ return [
+ c.lineno
+ for c in ast.walk(node)
+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == _INSTALL
+ ]
+
+
+def test_all_entrypoints_install_the_patch():
+ for path in _ENTRYPOINTS:
+ assert _install_call_linenos(ast.parse(path.read_text(encoding = "utf-8"))), (
+ f"{path.relative_to(_BACKEND)} never calls {_INSTALL}() -- torchao's int8 GEMM "
+ "keeps syncing the device on every linear and cannot be CUDA-graph captured."
+ )
+
+
+def _strip_docstrings(node):
+ for child in ast.walk(node):
+ body = getattr(child, "body", None)
+ if not isinstance(body, list) or not body:
+ continue
+ first = body[0]
+ if (
+ isinstance(first, ast.Expr)
+ and isinstance(first.value, ast.Constant)
+ and isinstance(first.value.value, str)
+ ):
+ del body[0]
+ return node
+
+
+def _function_dump(path: Path, name: str) -> str:
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ for node in tree.body:
+ if isinstance(node, ast.FunctionDef) and node.name == name:
+ return ast.dump(_strip_docstrings(node))
+ raise AssertionError(f"{path} has no top-level def {name}")
+
+
[email protected]("name", _SHARED_FUNCTIONS)
+def test_studio_copy_matches_unsloth_import_fixes(name):
+ if not _IMPORT_FIXES.is_file():
+ pytest.skip("unsloth/import_fixes.py is not in this checkout")
+ assert _function_dump(_PATCH_MODULE, name) == _function_dump(_IMPORT_FIXES, name), (
+ f"DRIFT: {name}() differs between core/inference/diffusion_torchao_patches.py and "
+ "unsloth/import_fixes.py. The two copies must stay interchangeable, because whichever "
+ "one runs first in a process is the one that patches torchao."
+ )
+
+
+def _fake_intmm_module(safe_int_mm) -> types.ModuleType:
+ module = types.ModuleType("torchao_intmm_stand_in")
+ module.safe_int_mm = safe_int_mm
+ module.out_dtype = lambda *args, **kwargs: None
+ module.dynamo_is_compiling = lambda: False
+ return module
+
+
+def test_patch_leaves_an_already_patched_module_alone():
+ """Why the two copies do not fight: both mark their replacement ``__unsloth_patched__``."""
+
+ def already_patched(input, mat2):
+ return None
+
+ already_patched.__unsloth_patched__ = True
+ module = _fake_intmm_module(already_patched)
+
+ assert _patch_torchao_intmm_module(module) is False
+ assert module.safe_int_mm is already_patched
+
+
+def test_patch_refuses_an_unrecognised_body():
+ def rewritten_upstream(input, mat2):
+ # None of the markers the gate looks for
+ return input @ mat2
+
+ module = _fake_intmm_module(rewritten_upstream)
+
+ assert _patch_torchao_intmm_module(module) is False
+ assert module.safe_int_mm is rewritten_upstream
+
+
+def test_finder_answers_for_both_torchao_homes(monkeypatch):
+ """torchao main moved ``safe_int_mm`` into the int8 workflow module (pytorch/ao#4718), probe intact."""
+ import importlib.machinery
+ import importlib.util
+
+ assert _TORCHAO_INTMM_MODULES == (
+ "torchao.kernel.intmm",
+ "torchao.quantization.quantize_.workflows.int8.kernels",
+ )
+ if _IMPORT_FIXES.is_file():
+ text = _IMPORT_FIXES.read_text(encoding = "utf-8")
+ for name in _TORCHAO_INTMM_MODULES:
+ assert f'"{name}"' in text, f"unsloth/import_fixes.py does not list {name}"
+
+ class _Loader:
+ def exec_module(self, module):
+ pass
+
+ monkeypatch.setattr(
+ importlib.util,
+ "find_spec",
+ lambda fullname, *a, **k: importlib.machinery.ModuleSpec(fullname, _Loader()),
+ )
+ finder = _TorchaoIntmmPatchFinder()
+ for name in _TORCHAO_INTMM_MODULES:
+ spec = finder.find_spec(name)
+ assert spec is not None and isinstance(spec.loader, _TorchaoIntmmLoader), name
+ assert finder.find_spec("torchao.kernel.other") is None
+
+
+def test_real_torchao_int_mm_is_patched_and_bit_identical():
+ pytest.importorskip("torchao")
+ torch = pytest.importorskip("torch")
+ import importlib
+
+ intmm = None
+ for name in _TORCHAO_INTMM_MODULES:
+ try:
+ candidate = importlib.import_module(name)
+ except ImportError:
+ continue
+ if callable(getattr(candidate, "safe_int_mm", None)):
+ intmm = candidate
+ break
+ if intmm is None:
+ pytest.skip("torchao does not define safe_int_mm under any known module name")
+
+ install_torchao_int_mm_patch()
+ patched = intmm.safe_int_mm
+ assert getattr(patched, "__unsloth_patched__", False) is True
+ # int_scaled_matmul resolves the name through module globals, so the rebind must reach it
+ assert intmm.int_scaled_matmul.__globals__["safe_int_mm"] is patched
+ original = patched.__unsloth_original__
+
+ generator = torch.Generator().manual_seed(0)
+
+ def randint8(*shape):
+ return torch.randint(-127, 127, shape, dtype = torch.int8, generator = generator)
+
+ cases = [
+ (randint8(64, 64), randint8(64, 64)),
+ (randint8(40, 24), randint8(24, 72)),
+ # mat2 non-contiguous: torchao silently returns a wrong answer without the .contiguous()
+ (randint8(64, 64), randint8(64, 64).t().contiguous().t()),
+ # j = 20 is not a nonzero multiple of 8, so cuBLAS is skipped for the fallback path
+ (randint8(40, 20), randint8(20, 64)),
+ ]
+ for a, b in cases:
+ assert torch.equal(
+ patched(a, b), original(a, b)
+ ), f"patched safe_int_mm diverged from torchao's on {tuple(a.shape)} x {tuple(b.shape)}"
diff --git a/tests/test_import_fixes_drift.py b/tests/test_import_fixes_drift.py
index 208ac46eba3..b59280bcbef 100644
--- a/tests/test_import_fixes_drift.py
+++ b/tests/test_import_fixes_drift.py
@@ -755,3 +755,276 @@ def test_psutil_cpu_freq_shape_and_wiring():
"DRIFT DETECTED: patch_psutil_cpu_freq is defined but never called in "
"_gpu_init.py, so real imports never install it."
)
+
+
+def _import_torchao_intmm_home():
+ from unsloth.import_fixes import _TORCHAO_INTMM_MODULES
+ for name in _TORCHAO_INTMM_MODULES:
+ try:
+ module = importlib.import_module(name)
+ except ImportError:
+ continue
+ if callable(getattr(module, "safe_int_mm", None)):
+ return module
+ pytest.skip("torchao does not define safe_int_mm under any known module name")
+
+
+def _torchao_intmm_original_source():
+ """Once the fix has run, upstream's body is reachable only through ``__unsloth_original__``."""
+ pytest.importorskip("torchao")
+ intmm = _import_torchao_intmm_home()
+ function = intmm.safe_int_mm
+ if getattr(function, "__unsloth_patched__", False):
+ function = function.__unsloth_original__
+ return inspect.getsource(function)
+
+
+def test_torchao_safe_int_mm_still_uses_the_repr_probe():
+ """The pathology itself: a repr that formats the tensor's values, and so calls ``.item()``."""
+ source = _torchao_intmm_original_source()
+ if "__repr__" not in source:
+ pytest.fail(
+ "upstream fixed the probe, delete fix_torchao_safe_int_mm_repr_probe: "
+ "torchao's safe_int_mm no longer reprs its input, so the replacement in "
+ "unsloth/import_fixes.py (and its Studio copy in "
+ "studio/backend/core/inference/diffusion_torchao_patches.py) is dead weight."
+ )
+
+
+def test_torchao_safe_int_mm_body_matches_the_verified_shape():
+ from unsloth.import_fixes import _TORCHAO_SAFE_INT_MM_MARKERS
+
+ source = _torchao_intmm_original_source()
+ missing = [marker for marker in _TORCHAO_SAFE_INT_MM_MARKERS if marker not in source]
+ if missing:
+ pytest.fail(
+ "DRIFT DETECTED: torchao's safe_int_mm body changed "
+ f"({', '.join(missing)} missing); re-verify bit-identity before keeping the patch. "
+ "_make_safe_int_mm copies the cuBLAS dimension guards, the contiguity fixes and the "
+ "fp32 fallback, so a moved body is one the copy must not impersonate."
+ )
+
+
+def _patched_torchao_safe_int_mm():
+ pytest.importorskip("torchao")
+ intmm = _import_torchao_intmm_home()
+ from unsloth.import_fixes import (
+ _TORCHAO_SAFE_INT_MM_MARKERS,
+ fix_torchao_safe_int_mm_repr_probe,
+ )
+
+ fix_torchao_safe_int_mm_repr_probe()
+ function = intmm.safe_int_mm
+ if not getattr(function, "__unsloth_patched__", False):
+ source = inspect.getsource(function)
+ if any(marker not in source for marker in _TORCHAO_SAFE_INT_MM_MARKERS):
+ pytest.skip(
+ "this torchao's safe_int_mm is not the verified body, so the fix declined to "
+ "patch it (see test_torchao_safe_int_mm_body_matches_the_verified_shape)"
+ )
+ pytest.fail(
+ "DRIFT DETECTED: fix_torchao_safe_int_mm_repr_probe left a recognised "
+ "safe_int_mm unpatched."
+ )
+ return intmm, function
+
+
+def test_torchao_intmm_patch_is_bit_identical_on_cpu():
+ torch = pytest.importorskip("torch")
+ intmm, patched = _patched_torchao_safe_int_mm()
+ # int_scaled_matmul resolves the name through module globals, so the rebind must reach it
+ assert intmm.int_scaled_matmul.__globals__["safe_int_mm"] is patched
+ original = patched.__unsloth_original__
+
+ generator = torch.Generator().manual_seed(0)
+
+ def randint8(*shape):
+ return torch.randint(-127, 127, shape, dtype = torch.int8, generator = generator)
+
+ cases = [
+ (randint8(64, 64), randint8(64, 64)),
+ (randint8(40, 24), randint8(24, 72)),
+ (randint8(64, 64), randint8(64, 64).t().contiguous().t()),
+ (randint8(40, 20), randint8(20, 64)),
+ ]
+ for a, b in cases:
+ assert torch.equal(patched(a, b), original(a, b)), (
+ "DRIFT DETECTED: the patched safe_int_mm no longer matches torchao's on "
+ f"{tuple(a.shape)} x {tuple(b.shape)}."
+ )
+
+
+def test_torchao_intmm_patch_is_idempotent():
+ import types
+
+ from unsloth.import_fixes import (
+ _patch_torchao_intmm_module,
+ fix_torchao_safe_int_mm_repr_probe,
+ )
+
+ intmm, patched = _patched_torchao_safe_int_mm()
+ fix_torchao_safe_int_mm_repr_probe()
+ assert intmm.safe_int_mm is patched, "DRIFT DETECTED: safe_int_mm was replaced twice."
+
+ def already_patched(input, mat2):
+ return None
+
+ already_patched.__unsloth_patched__ = True
+ stand_in = types.ModuleType("torchao_intmm_stand_in")
+ stand_in.safe_int_mm = already_patched
+ stand_in.out_dtype = lambda *args, **kwargs: None
+ stand_in.dynamo_is_compiling = lambda: False
+ assert _patch_torchao_intmm_module(stand_in) is False
+ assert stand_in.safe_int_mm is already_patched
+
+
+def test_torchao_intmm_patch_refuses_an_unrecognised_body():
+ import types
+
+ from unsloth.import_fixes import _patch_torchao_intmm_module
+
+ def rewritten_upstream(input, mat2):
+ # None of the markers the gate looks for
+ return input @ mat2
+
+ stand_in = types.ModuleType("torchao_intmm_stand_in")
+ stand_in.safe_int_mm = rewritten_upstream
+ stand_in.out_dtype = lambda *args, **kwargs: None
+ stand_in.dynamo_is_compiling = lambda: False
+ assert _patch_torchao_intmm_module(stand_in) is False
+ assert stand_in.safe_int_mm is rewritten_upstream
+
+
+def test_torchao_intmm_patch_covers_a_later_import():
+ """The point of the finder: the prequant path imports torchao well after ``import unsloth``."""
+ if importlib.util.find_spec("torchao") is None:
+ pytest.skip("torchao not installed -- nothing to patch.")
+ import subprocess
+
+ import_fixes_path = Path(__file__).resolve().parent.parent / "unsloth" / "import_fixes.py"
+ program = (
+ "import importlib.util, sys\n"
+ f"spec = importlib.util.spec_from_file_location('unsloth_import_fixes_under_test', {str(import_fixes_path)!r})\n"
+ "module = importlib.util.module_from_spec(spec)\n"
+ "spec.loader.exec_module(module)\n"
+ "assert 'torchao' not in sys.modules, 'torchao was imported before the fix ran'\n"
+ "module.fix_torchao_safe_int_mm_repr_probe()\n"
+ "import torchao.quantization\n"
+ "import importlib\n"
+ "intmm = None\n"
+ "for name in module._TORCHAO_INTMM_MODULES:\n"
+ " try:\n"
+ " candidate = importlib.import_module(name)\n"
+ " except ImportError:\n"
+ " continue\n"
+ " if callable(getattr(candidate, 'safe_int_mm', None)):\n"
+ " intmm = candidate\n"
+ " break\n"
+ "assert intmm is not None, 'no torchao module defines safe_int_mm'\n"
+ "print('PATCHED=' + str(bool(getattr(intmm.safe_int_mm, '__unsloth_patched__', False))))\n"
+ )
+ env = dict(os.environ)
+ env.pop("UNSLOTH_TORCHAO_INT_MM_FIX", None)
+ result = subprocess.run(
+ [sys.executable, "-c", program],
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ env = env,
+ )
+ assert result.returncode == 0, f"child failed:\n{result.stdout}\n{result.stderr}"
+ assert "PATCHED=True" in result.stdout, (
+ "DRIFT DETECTED: a torchao imported AFTER fix_torchao_safe_int_mm_repr_probe ran was "
+ f"left unpatched, so the meta path finder no longer fires.\n{result.stdout}"
+ )
+
+
+def test_torchao_intmm_finder_covers_both_module_homes(monkeypatch):
+ """A finder covering only the old name loses the fix once torchao ships pytorch/ao#4718."""
+ import importlib.machinery
+
+ from unsloth.import_fixes import (
+ _TORCHAO_INTMM_MODULES,
+ _TorchaoIntmmLoader,
+ _TorchaoIntmmPatchFinder,
+ )
+
+ assert "torchao.kernel.intmm" in _TORCHAO_INTMM_MODULES
+ assert "torchao.quantization.quantize_.workflows.int8.kernels" in _TORCHAO_INTMM_MODULES
+
+ class _Loader:
+ def exec_module(self, module):
+ pass
+
+ def fake_find_spec(fullname, *args, **kwargs):
+ return importlib.machinery.ModuleSpec(fullname, _Loader())
+
+ monkeypatch.setattr(importlib.util, "find_spec", fake_find_spec)
+ finder = _TorchaoIntmmPatchFinder()
+ for name in _TORCHAO_INTMM_MODULES:
+ spec = finder.find_spec(name)
+ assert spec is not None and isinstance(spec.loader, _TorchaoIntmmLoader), name
+ assert finder.find_spec("torchao.kernel.somewhere_else") is None
+ assert finder.find_spec("torchao.quantization.quantize_.workflows.int8") is None
+
+
+def test_torchao_intmm_installer_patches_the_new_home_when_already_imported(monkeypatch):
+ """A module ALREADY in ``sys.modules`` must be patched too: the finder only sees later imports."""
+ import types
+
+ torch = pytest.importorskip("torch")
+ from unsloth.import_fixes import (
+ _TORCHAO_INTMM_MODULES,
+ _TORCHAO_INTMM_SENTINEL,
+ fix_torchao_safe_int_mm_repr_probe,
+ )
+
+ source = _torchao_intmm_original_source()
+ module = types.ModuleType("torchao.quantization.quantize_.workflows.int8.kernels")
+ module.__dict__["torch"] = torch
+ from torch._dynamo import is_compiling as dynamo_is_compiling
+ from torch._higher_order_ops.out_dtype import out_dtype
+
+ module.out_dtype = out_dtype
+ module.dynamo_is_compiling = dynamo_is_compiling
+ import linecache
+ import textwrap
+
+ # The gate reads the body through inspect.getsource, so the copy needs a linecache entry.
+ text = textwrap.dedent(source)
+ filename = "<torchao safe_int_mm copy>"
+ linecache.cache[filename] = (len(text), None, text.splitlines(True), filename)
+ exec(compile(text, filename, "exec"), module.__dict__)
+ assert callable(module.safe_int_mm)
+
+ # Hide every real torchao home so only the stand-in is visible, and drop any finder.
+ for name in _TORCHAO_INTMM_MODULES:
+ monkeypatch.delitem(sys.modules, name, raising = False)
+ monkeypatch.setitem(sys.modules, module.__name__, module)
+ monkeypatch.setattr(
+ sys,
+ "meta_path",
+ [f for f in sys.meta_path if not getattr(f, _TORCHAO_INTMM_SENTINEL, False)],
+ )
+ monkeypatch.delenv("UNSLOTH_TORCHAO_INT_MM_FIX", raising = False)
+
+ assert fix_torchao_safe_int_mm_repr_probe() is True
+ assert getattr(
+ module.safe_int_mm, "__unsloth_patched__", False
+ ), "the installer ignored a safe_int_mm registered under torchao's new module name"
+ a = torch.randint(-128, 127, (16, 32), dtype = torch.int8)
+ b = torch.randint(-128, 127, (32, 24), dtype = torch.int8)
+ assert torch.equal(module.safe_int_mm(a, b), module.safe_int_mm.__unsloth_original__(a, b))
+
+
+def test_torchao_intmm_patch_wired_into_gpu_init():
+ source = Path(__file__).resolve().parent.parent / "unsloth" / "_gpu_init.py"
+ source = source.read_text(encoding = "utf-8")
+ assert "fix_torchao_safe_int_mm_repr_probe()" in source, (
+ "DRIFT DETECTED: fix_torchao_safe_int_mm_repr_probe is defined but never called in "
+ "_gpu_init.py, so real imports never install it."
+ )
+ assert "del fix_torchao_safe_int_mm_repr_probe" in source, (
+ "DRIFT DETECTED: fix_torchao_safe_int_mm_repr_probe is left bound on the unsloth "
+ "namespace; _gpu_init.py deletes every fix it calls."
+ )
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index b4d09653b5f..f6f4fcd4cb5 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -233,6 +233,7 @@ from .import_fixes import (
fix_vllm_aimv2_issue,
fix_vllm_lora_tokenizer_module,
fix_torchao_nf4tensor_move,
+ fix_torchao_safe_int_mm_repr_probe,
check_vllm_torch_sm100_compatibility,
fix_vllm_guided_decoding_params,
fix_vllm_pdl_blackwell,
@@ -276,6 +277,7 @@ fix_vllm_lora_tokenizer_module()
# torchao 0.18.0 moved nf4tensor; torchtune (via xcodec2) still imports the old path. Lazy alias, so
# it costs nothing unless asked for.
fix_torchao_nf4tensor_move()
+fix_torchao_safe_int_mm_repr_probe()
# Check vLLM + torch < 2.9.0 + SM100 compatibility BEFORE importing vLLM
check_vllm_torch_sm100_compatibility()
fix_vllm_guided_decoding_params()
@@ -318,6 +320,7 @@ del fix_flash_attn_4_namespace_shadow
del fix_vllm_aimv2_issue
del fix_vllm_lora_tokenizer_module
del fix_torchao_nf4tensor_move
+del fix_torchao_safe_int_mm_repr_probe
del check_vllm_torch_sm100_compatibility
del fix_vllm_guided_decoding_params
del fix_trl_vllm_ascend
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 2ac2362e274..8cf8b8d175b 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -5755,6 +5755,221 @@ def fix_torchao_nf4tensor_move():
sys.meta_path.append(_TorchaoNF4AliasFinder())
+_TORCHAO_INTMM_MODULES = (
+ "torchao.kernel.intmm", # every release up to and including 0.18.0
+ "torchao.quantization.quantize_.workflows.int8.kernels", # main after pytorch/ao#4718
+)
+_TORCHAO_INTMM_MODULE = _TORCHAO_INTMM_MODULES[0] # kept for callers that named the old home
+_TORCHAO_INTMM_SENTINEL = "__unsloth_torchao_intmm_patch__"
+_TORCHAO_INT_MM_ENV = "UNSLOTH_TORCHAO_INT_MM_FIX"
+
+# The copy below hard-codes torchao's cuBLAS guards and fp32 fallback: ALL must be in the installed source.
+_TORCHAO_SAFE_INT_MM_MARKERS = (
+ "input.__repr__()",
+ "dynamo_is_compiling()",
+ "out_dtype(torch.ops.aten.mm.default",
+ "j_is_nonzero_multiple_of_8",
+ "k_is_nonzero_multiple_of_8",
+ "mat2.is_contiguous()",
+)
+
+
+def _is_fake_tensor(x):
+ """Covers exactly the set the substring repr probe did: ``is_fake`` unwraps FunctionalTensor too."""
+ try:
+ from torch._subclasses.fake_tensor import is_fake
+ return bool(is_fake(x))
+ except Exception:
+ pass
+ try:
+ from torch._subclasses.fake_tensor import FakeTensor
+ return isinstance(x, FakeTensor)
+ except Exception:
+ return False
+
+
+def _make_safe_int_mm(mod, original):
+ """A FULL copy of torchao 0.17.0's body, not a wrapper: its very first statement IS the probe."""
+ import torch
+
+ out_dtype = mod.out_dtype
+ dynamo_is_compiling = mod.dynamo_is_compiling
+
+ @functools.wraps(original)
+ def safe_int_mm(input: torch.Tensor, mat2: torch.Tensor) -> torch.Tensor:
+ if dynamo_is_compiling() or _is_fake_tensor(input):
+ if input.device.type == "cpu":
+ # Matmul in int32 is slow on CPU and not supported well by Inductor cpp backend
+ return out_dtype(
+ torch.ops.aten.mm.default, torch.int32, input.float(), mat2.float()
+ )
+ return out_dtype(torch.ops.aten.mm.default, torch.int32, input, mat2)
+
+ assert (
+ mat2.device == input.device
+ ), f"need both tensors to be on the same device but got {mat2.device} and {input.device}"
+ device_cpu = "cpu" in [mat2.device.type, input.device.type]
+ j_is_nonzero_multiple_of_8 = (input.shape[1] % 8 == 0) and (input.shape[1] > 0)
+ k_is_nonzero_multiple_of_8 = (mat2.shape[1] % 8 == 0) and (mat2.shape[1] > 0)
+ bad_dimensions_for_cublas = not (j_is_nonzero_multiple_of_8 and k_is_nonzero_multiple_of_8)
+
+ if device_cpu or bad_dimensions_for_cublas:
+ return torch.matmul(input.cpu().to(torch.int32), mat2.cpu().to(torch.int32)).to(
+ input.device.type
+ )
+
+ if not mat2.is_contiguous(): # silently gives incorrect result without this
+ mat2 = mat2.contiguous()
+ if (not input.is_contiguous()) and (
+ input.shape[0] % 8 != 0
+ ): # gives cryptic error without this
+ input = input.contiguous()
+ try:
+ return out_dtype(torch.ops.aten.mm.default, torch.int32, input, mat2)
+ except Exception:
+ # H100 float8: "addmm_cuda" not implemented for 'Float8_e4m3fn'
+ return torch.matmul(input.to(torch.float32), mat2.to(torch.float32)).to(torch.int32)
+
+ safe_int_mm.__unsloth_patched__ = True
+ safe_int_mm.__unsloth_original__ = original
+ return safe_int_mm
+
+
+def _patch_torchao_intmm_module(mod):
+ """Rebind ``safe_int_mm`` on either torchao home; True when this call installed it."""
+ original = getattr(mod, "safe_int_mm", None)
+ if original is None or not callable(original):
+ return False
+ if getattr(original, "__unsloth_patched__", False):
+ return False
+ # Off the module, never imported here, so this fails closed rather than borrowing our own operators.
+ if not hasattr(mod, "out_dtype") or not hasattr(mod, "dynamo_is_compiling"):
+ return False
+ try:
+ source = inspect.getsource(original)
+ except Exception:
+ return False
+ missing = [marker for marker in _TORCHAO_SAFE_INT_MM_MARKERS if marker not in source]
+ if missing:
+ if "input.__repr__()" not in missing:
+ logger.warning(
+ "Unsloth: torchao's safe_int_mm still probes input.__repr__() but its body "
+ "changed (%s missing), so the capture-safe replacement was not installed. "
+ "Eager int8 keeps syncing the device on every linear.",
+ ", ".join(missing),
+ )
+ return False
+ patched = _make_safe_int_mm(mod, original)
+ try:
+ mod.safe_int_mm = patched
+ except Exception:
+ return False
+ # `torchao.kernel` and `torchao.quantization` re-export the function OBJECT, so they need a sweep.
+ for name, other in tuple(sys.modules.items()):
+ if other is None or not (name == "torchao" or name.startswith("torchao.")):
+ continue
+ if getattr(other, "safe_int_mm", None) is original:
+ try:
+ setattr(other, "safe_int_mm", patched)
+ except Exception:
+ pass
+ return True
+
+
+class _TorchaoIntmmLoader(importlib.abc.Loader):
+ """The real loader, plus the patch once the module body finishes. Failures here leave torchao unpatched."""
+
+ __slots__ = ("_loader",)
+
+ def __init__(self, loader):
+ self._loader = loader
+
+ def create_module(self, spec):
+ create = getattr(self._loader, "create_module", None)
+ if create is None:
+ return None
+ return create(spec)
+
+ def exec_module(self, module):
+ self._loader.exec_module(module)
+ try:
+ _patch_torchao_intmm_module(module)
+ except Exception:
+ pass
+
+ def __getattr__(self, attribute):
+ return getattr(self._loader, attribute)
+
+
+class _TorchaoIntmmPatchFinder(importlib.abc.MetaPathFinder):
+ """Inserted at the FRONT of sys.meta_path: the module really exists, so PathFinder would answer first."""
+
+ __slots__ = (_TORCHAO_INTMM_SENTINEL, "_finding")
+
+ def __init__(self):
+ setattr(self, _TORCHAO_INTMM_SENTINEL, True)
+ self._finding = False # find_spec below walks sys.meta_path again
+
+ def find_spec(
+ self,
+ fullname,
+ path = None,
+ target = None,
+ ):
+ if fullname not in _TORCHAO_INTMM_MODULES or self._finding:
+ return None
+ self._finding = True
+ try:
+ spec = importlib.util.find_spec(fullname)
+ except Exception:
+ return None
+ finally:
+ self._finding = False
+ if spec is None or spec.loader is None:
+ return None
+ if not hasattr(spec.loader, "exec_module"):
+ return None # a loader from before PEP 451; leave the import entirely alone
+ try:
+ spec.loader = _TorchaoIntmmLoader(spec.loader)
+ except Exception:
+ return None
+ return spec
+
+
+def fix_torchao_safe_int_mm_repr_probe():
+ """Stop torchao's int8 GEMM from syncing the device to ask whether it is being traced.
+
+ ``safe_int_mm`` branches on ``"FakeTensor" in input.__repr__()``; a real CUDA tensor's repr calls
+ ``.item()``, so eager int8 syncs per linear and ``torch.cuda.graph`` capture fails outright. The
+ gate is structural, not version-based, and the meta path finder covers a torchao imported after
+ this call, which the int8 prequant path needs since it never calls ``quantize_``.
+ ``UNSLOTH_TORCHAO_INT_MM_FIX=0`` keeps upstream's behaviour. True when patched or the finder was
+ installed, False when there is nothing to do, None when torchao is absent or the fix is off."""
+ if os.environ.get(_TORCHAO_INT_MM_ENV, "1").strip() == "0":
+ return None
+ try:
+ if importlib.util.find_spec("torchao") is None:
+ return None
+ except Exception:
+ return None
+ patched_now = False
+ for name in _TORCHAO_INTMM_MODULES:
+ module = sys.modules.get(name)
+ if module is None:
+ continue
+ try:
+ patched_now = _patch_torchao_intmm_module(module) or patched_now
+ except Exception:
+ pass
+ if all(name in sys.modules for name in _TORCHAO_INTMM_MODULES):
+ return patched_now # nothing left for a finder to catch
+ for finder in sys.meta_path:
+ if getattr(finder, _TORCHAO_INTMM_SENTINEL, False):
+ return patched_now
+ sys.meta_path.insert(0, _TorchaoIntmmPatchFinder())
+ return True
+
+
# `datasets` fingerprints through dill, and dill._dill._is_builtin_module pickles a module by
# reference only if its __file__ starts with a sys prefix, ends with an extension suffix, or
# contains the literal `site-packages`. An install matching none (pip install --target, a