Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio (AMD/Windows): real torchao INT8 / FP8 export on Windows ROCm (#7102)

Daniel Han7b07d860992f
diff --git a/studio/backend/core/_torchao_stub.py b/studio/backend/core/_torchao_stub.py
index 00c0beb79e2..54943908fbe 100644
--- a/studio/backend/core/_torchao_stub.py
+++ b/studio/backend/core/_torchao_stub.py
@@ -22,6 +22,7 @@ import re
import sys
import types
import importlib.abc
+import importlib.metadata
import importlib.machinery
import importlib.util
from typing import Optional
@@ -227,6 +228,71 @@ def install_torchao_windows_rocm_stub() -> None:
sys.modules[_tao_name] = _make_mod_stub(_tao_name)
+def _load_torchao_nodist():
+ """unsloth/_torchao_nodist.py, loaded by path: importing unsloth here would start its GPU
+ stack before the worker is ready. None on an unsloth that predates it."""
+ try:
+ spec = importlib.util.find_spec("unsloth")
+ locations = list(spec.submodule_search_locations or ()) if spec else []
+ path = os.path.join(locations[0], "_torchao_nodist.py") if locations else None
+ if not path or not os.path.isfile(path):
+ return None
+ module_spec = importlib.util.spec_from_file_location("_studio_torchao_nodist", path)
+ module = importlib.util.module_from_spec(module_spec)
+ module_spec.loader.exec_module(module)
+ return module
+ except Exception:
+ return None
+
+
+_TORCHAO_EXPORT_MIN = (0, 15)
+
+
+def torchao_export_loadable() -> bool:
+ """Whether an export worker can get real torchao: always off Windows ROCm; there, torchao must
+ be installed and unsloth must ship the shim. Searches sys.path, so a stub already in
+ sys.modules does not count, and imports nothing."""
+ if not _is_windows_rocm():
+ return True
+ try:
+ if importlib.machinery.PathFinder.find_spec("torchao") is None:
+ return False
+ # transformers 5's TorchAoConfig minimum; torch <= 2.9 is paired with torchao 0.14.
+ found = re.match(r"(\d+)\.(\d+)", importlib.metadata.version("torchao"))
+ if not found or (int(found[1]), int(found[2])) < _TORCHAO_EXPORT_MIN:
+ return False
+ spec = importlib.util.find_spec("unsloth")
+ locations = list(spec.submodule_search_locations or ()) if spec else []
+ return bool(locations) and os.path.isfile(os.path.join(locations[0], "_torchao_nodist.py"))
+ except Exception:
+ return False
+
+
+_STUB_CONSUMERS = ("transformers", "peft", "diffusers", "accelerate", "unsloth", "unsloth_zoo")
+
+
+def install_torchao_windows_rocm_real_or_stub() -> bool:
+ """Export worker: real torchao on Windows ROCm when it is installed and unsloth can import it
+ without torch.distributed, else the stub. True iff real torchao is loaded. No-op elsewhere."""
+ if not _is_windows_rocm():
+ return False
+ # A spawn child re-runs run.py as __mp_main__, which stubs torchao first. Drop that stub
+ # while nothing that could have bound it is loaded yet.
+ if is_stubbed("torchao") and not any(m in sys.modules for m in _STUB_CONSUMERS):
+ for name in [n for n in sys.modules if n == "torchao" or n.startswith("torchao.")]:
+ if getattr(sys.modules[name], "_unsloth_stub", None) is _STUB_SENTINEL:
+ del sys.modules[name]
+ if "torchao" not in sys.modules and torchao_export_loadable():
+ module = _load_torchao_nodist()
+ try:
+ if module is not None and module.fix_torchao_without_torch_distributed():
+ return True
+ except Exception:
+ pass
+ install_torchao_windows_rocm_stub()
+ return not is_stubbed("torchao") and "torchao" in sys.modules
+
+
def install_xformers_windows_rocm_stub() -> None:
"""Pre-stub xformers on Windows ROCm so diffusers can import at all. No-op elsewhere, and must
precede diffusers: the Windows xformers pin is CUDA-only, so against a ROCm torch (no
diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py
index f8ea88facf9..7e16f3a8f4b 100644
--- a/studio/backend/core/export/export.py
+++ b/studio/backend/core/export/export.py
@@ -9,6 +9,7 @@ import structlog
import tempfile
from loggers import get_logger
import os
+import sys
import shutil
import contextlib
from pathlib import Path
@@ -368,14 +369,42 @@ def _compressed_export_supported():
def _torchao_export_supported():
- """True if the installed unsloth build has the portable torchao FP8/INT8 export path."""
+ """True if the installed unsloth build has the portable torchao FP8/INT8 export path and a real
+ torchao to run it; False where torchao is stubbed (its config classes return None)."""
try:
+ if _torchao_runtime_unavailable():
+ return False
import unsloth.save as _us
return hasattr(_us, "_normalize_torchao_method")
except Exception:
return False
+def _torchao_runtime_unavailable():
+ try:
+ from core._torchao_stub import _is_windows_rocm, is_stubbed
+ if is_stubbed("torchao"):
+ return True
+ # Windows ROCm only gets real torchao through install_torchao_windows_rocm_real_or_stub().
+ return _is_windows_rocm() and "torchao" not in sys.modules
+ except Exception:
+ return False
+
+
+def _is_torchao_alias(alias):
+ """Any torchao spelling unsloth's normalizer accepts, else a torchao_ prefix, so the Windows
+ ROCm guard catches it before it is misread as compressed-tensors."""
+ if not alias:
+ return False
+ try:
+ import unsloth.save as _us
+ if _us._normalize_torchao_method(alias) is not None:
+ return True
+ except Exception:
+ pass
+ return str(alias).lower().startswith("torchao")
+
+
def _has_nvidia_gpu():
"""True only on a real NVIDIA CUDA box (not ROCm/XPU/CPU/MLX); compressed-tensors needs it."""
try:
@@ -943,6 +972,17 @@ class ExportBackend:
"NVFP4 (compressed-tensors)": "nvfp4",
}
compressed_alias = compressed_method or _LABEL_TO_ALIAS.get(format_type)
+
+ # Fail fast: a stubbed torchao otherwise crashes in transformers with TorchAoConfig(quant_type=None).
+ if _is_torchao_alias(compressed_alias) and _torchao_runtime_unavailable():
+ return (
+ False,
+ "Portable torchao FP8/INT8 export needs torchao, which could not be loaded "
+ "on this Windows ROCm build. Update Unsloth to install it, or use "
+ "16-bit merged or GGUF quantization instead.",
+ None,
+ )
+
compressed_suffix: Optional[str] = None
# Classify the alias: torchao-portable vs compressed-tensors.
torchao_info = None
diff --git a/studio/backend/core/export/worker.py b/studio/backend/core/export/worker.py
index 663961c6e04..e965370c986 100644
--- a/studio/backend/core/export/worker.py
+++ b/studio/backend/core/export/worker.py
@@ -596,11 +596,10 @@ def run_export_process(*, cmd_queue: Any, resp_queue: Any, config: dict) -> None
from core._msvc_env import gate_torch_compile_on_windows
gate_torch_compile_on_windows(logger)
- # See core/_torchao_stub.py: torchao crashes on Windows ROCm (RCCL absent). No-op off Windows ROCm. Must run
- # before importing transformers / unsloth_zoo.
- from core._torchao_stub import install_torchao_windows_rocm_stub
+ # Before transformers / unsloth_zoo: real torchao via unsloth's shim on Windows ROCm, else the stub.
+ from core._torchao_stub import install_torchao_windows_rocm_real_or_stub
- install_torchao_windows_rocm_stub()
+ install_torchao_windows_rocm_real_or_stub()
try:
_send_response(
diff --git a/studio/backend/tests/test_torchao_select.py b/studio/backend/tests/test_torchao_select.py
index c42613a11a0..550ab85f152 100644
--- a/studio/backend/tests/test_torchao_select.py
+++ b/studio/backend/tests/test_torchao_select.py
@@ -10,6 +10,8 @@ venv (otherwise the cpp kernels are skipped). This pins that mapping.
from __future__ import annotations
+import importlib
+import importlib.machinery
import sys
from pathlib import Path
from unittest.mock import MagicMock
@@ -185,7 +187,7 @@ def test_the_torchao_step_pins_the_index_and_retries_without_it():
after this ships can lag a release, so the pin must not be able to fail an install.
Unlike torchcodec the retry stays FATAL if it also fails: torchao is not optional."""
body = _torchao_installer_source()
- assert "index = _torch_accelerator_index_url(torch_version)" in body
+ assert "index = None if default_index else _torch_accelerator_index_url(torch_version)" in body
assert '"--index-url", index, spec' in body
assert "retrying from the default index" in body
# The unpinned attempt is pip_install, not pip_install_try: still fatal on failure.
@@ -327,18 +329,23 @@ def test_windows_first_hop_uses_einx_wheel_without_shared_test_tree():
(True, True),
],
)
-def test_skips_torchao_on_windows_rocm(
- monkeypatch, tmp_path, rocm_windows_torch_installed, installed_torch_is_windows_rocm
[email protected]("torchao_install_ok", [True, False])
+def test_installs_pypi_torchao_on_windows_rocm(
+ monkeypatch,
+ tmp_path,
+ rocm_windows_torch_installed,
+ installed_torch_is_windows_rocm,
+ torchao_install_ok,
):
- """The overrides step must skip torchao on Windows ROCm: no working build exists
- there (it imports an absent c10d backend and crashes transformers.quantizers),
- so the installer skips it and relies on the runtime stub instead."""
+ """Windows ROCm gets the torch-matched torchao from PyPI (download.pytorch.org's rocm leaves
+ serve Linux only); the export worker loads it through unsloth/_torchao_nodist.py. A failed
+ torchao install does not fail the install: only export needs it."""
mod = _load_module(monkeypatch)
- installed_specs: list[str] = []
+ pip_calls: list[list[str]] = []
progress_labels: list[str] = []
def _record_pip_install(*args, **kwargs):
- installed_specs.extend(str(arg) for arg in args)
+ pip_calls.append([str(arg) for arg in args])
return 0
unstructured_plugin = tmp_path / "unstructured"
@@ -385,7 +392,23 @@ def test_skips_torchao_on_windows_rocm(
mod, "_probe_torch_runtime", lambda *args, **kwargs: (True, True, "2.9.1+cpu", "", "")
)
monkeypatch.setattr(mod, "run", lambda *args, **kwargs: None)
- monkeypatch.setattr(mod, "pip_install", _record_pip_install)
+
+ def _fatal_pip_install(*args, **kwargs):
+ # pip_install exits the installer on failure.
+ _record_pip_install(*args, **kwargs)
+ if not torchao_install_ok and any(str(a).startswith("torchao") for a in args):
+ raise SystemExit(1)
+ return 0
+
+ monkeypatch.setattr(mod, "pip_install", _fatal_pip_install)
+ monkeypatch.setattr(
+ mod,
+ "pip_install_try",
+ lambda *a, **k: (
+ _record_pip_install(*a, **k),
+ torchao_install_ok or not any(str(x).startswith("torchao") for x in a),
+ )[1],
+ )
monkeypatch.setattr(mod, "_progress", lambda label: progress_labels.append(label))
monkeypatch.setattr(mod, "LOCAL_DD_UNSTRUCTURED_PLUGIN", unstructured_plugin)
monkeypatch.setattr(mod, "LOCAL_DD_GITHUB_PLUGIN", github_plugin)
@@ -402,5 +425,325 @@ def test_skips_torchao_on_windows_rocm(
assert mod.install_python_stack() == 0
- assert not any(spec.startswith("torchao") for spec in installed_specs)
- assert "dependency overrides (skipped, Windows ROCm)" in progress_labels
+ torchao_calls = [c for c in pip_calls if any(a.startswith("torchao") for a in c)]
+ assert torchao_calls
+ assert all("--index-url" not in c for c in torchao_calls)
+ assert "dependency overrides (Windows ROCm)" in progress_labels
+
+
+# Windows ROCm torchao export: real torchao via unsloth/_torchao_nodist.py, the stub as fallback.
+
+import types
+
+import core._torchao_stub as _stub
+
+_BACKEND = Path(__file__).resolve().parents[1]
+_EXPORT_HELPERS = ("_torchao_export_supported", "_torchao_runtime_unavailable", "_is_torchao_alias")
+
+
+def _export_helpers():
+ """Exec the gate helpers alone, avoiding export.py's heavy import chain."""
+ import ast
+
+ src = (_BACKEND / "core" / "export" / "export.py").read_text(encoding = "utf-8")
+ ns: dict = {}
+ for node in ast.parse(src).body:
+ if isinstance(node, ast.FunctionDef) and node.name in _EXPORT_HELPERS:
+ exec(ast.get_source_segment(src, node), ns)
+ return ns
+
+
+def _fake_normalize_torchao(save_method):
+ if not isinstance(save_method, str):
+ return None
+ key = save_method.lower().strip().replace("-", "_").replace(" ", "_")
+ ok = key in {"torchao_fp8", "torchao_int8", "portable_fp8", "portable_int8"}
+ return ("fp8", "torchao-fp8") if ok else None
+
+
+def _install_fake_unsloth_save(monkeypatch, *, has_method):
+ unsloth = types.ModuleType("unsloth")
+ save = types.ModuleType("unsloth.save")
+ if has_method:
+ save._normalize_torchao_method = _fake_normalize_torchao
+ unsloth.save = save
+ monkeypatch.setitem(sys.modules, "unsloth", unsloth)
+ monkeypatch.setitem(sys.modules, "unsloth.save", save)
+
+
+def _set_torchao(monkeypatch, state):
+ if state == "absent":
+ monkeypatch.delitem(sys.modules, "torchao", raising = False)
+ elif state == "stub":
+ monkeypatch.setitem(sys.modules, "torchao", _stub._make_mod_stub("torchao"))
+ else:
+ monkeypatch.setitem(sys.modules, "torchao", types.ModuleType("torchao"))
+
+
+ ("win_rocm", "torchao", "has_method", "expected"),
+ [
+ (True, "stub", True, False),
+ (True, "absent", True, False),
+ (True, "real", True, True),
+ (False, "real", True, True),
+ (False, "absent", True, True),
+ (False, "real", False, False),
+ ],
+)
+def test_torchao_export_gate(monkeypatch, win_rocm, torchao, has_method, expected):
+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: win_rocm)
+ _set_torchao(monkeypatch, torchao)
+ _install_fake_unsloth_save(monkeypatch, has_method = has_method)
+ ns = _export_helpers()
+ ns["sys"] = sys
+ assert ns["_torchao_export_supported"]() is expected
+
+
+def test_is_torchao_alias_recognizes_all_forms(monkeypatch):
+ _install_fake_unsloth_save(monkeypatch, has_method = True)
+ fn = _export_helpers()["_is_torchao_alias"]
+ for alias in ("torchao_fp8", "portable_int8", "portable-fp8", "Portable FP8"):
+ assert fn(alias) is True
+ for alias in ("fp8", "nvfp4", "w8a8", "", None):
+ assert fn(alias) is False
+
+
+def _load_export_module_no_torch(monkeypatch):
+ """Import core.export.export with torch/unsloth blocked, so the guard runs without a GPU."""
+ import builtins
+ import importlib
+
+ real_import = builtins.__import__
+
+ def blocking_import(name, *args, **kwargs):
+ top = name.split(".")[0]
+ if top in {"torch", "unsloth"} and top not in sys.modules:
+ raise ImportError(f"blocked: {name}")
+ return real_import(name, *args, **kwargs)
+
+ for m in [k for k in list(sys.modules) if k.split(".")[0] in {"torch", "unsloth"}]:
+ monkeypatch.delitem(sys.modules, m, raising = False)
+ monkeypatch.delitem(sys.modules, "core.export.export", raising = False)
+ monkeypatch.setattr(builtins, "__import__", blocking_import)
+ return importlib.import_module("core.export.export")
+
+
+def _bare_backend(mod):
+ be = mod.ExportBackend.__new__(mod.ExportBackend)
+ be.current_model = object()
+ be.current_tokenizer = object()
+ be._audio_type = None
+ be.is_peft = True
+ return be
+
+
[email protected]("alias", ["torchao_fp8", "portable_fp8"])
+def test_torchao_export_rejected_early_when_stubbed(monkeypatch, alias):
+ mod = _load_export_module_no_torch(monkeypatch)
+ monkeypatch.setattr(mod, "_export_runtime_available", lambda: True)
+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)
+ _set_torchao(monkeypatch, "stub")
+ _install_fake_unsloth_save(monkeypatch, has_method = True)
+
+ ok, message, out = _bare_backend(mod).export_merged_model("/tmp/x", compressed_method = alias)
+ assert ok is False and out is None
+ assert "Windows ROCm" in message and "torchao" in message.lower()
+
+
+def test_torchao_export_not_rejected_with_real_torchao_on_windows_rocm(monkeypatch):
+ mod = _load_export_module_no_torch(monkeypatch)
+ monkeypatch.setattr(mod, "_export_runtime_available", lambda: True)
+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)
+ _set_torchao(monkeypatch, "real")
+ _install_fake_unsloth_save(monkeypatch, has_method = True)
+
+ ok, message, _ = _bare_backend(mod).export_merged_model(
+ "/tmp/x", compressed_method = "torchao_int8"
+ )
+ # Past the guard: the placeholder model is what fails now.
+ assert ok is False and "Windows ROCm" not in message
+
+
+ ("platform", "is_rocm", "loadable", "expected"),
+ [
+ ("win32", True, False, False),
+ ("win32", True, True, True),
+ ("win32", False, False, True),
+ ("linux", True, False, True),
+ ],
+)
+def test_export_capability_torchao_flag(monkeypatch, platform, is_rocm, loadable, expected):
+ import utils.hardware.hardware as hw
+
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
+ monkeypatch.setattr(sys, "platform", platform)
+ monkeypatch.setattr(hw, "IS_ROCM", is_rocm)
+ monkeypatch.setattr(_stub, "torchao_export_loadable", lambda: loadable)
+ assert hw.export_capability()["torchao_export_supported"] is expected
+
+
+def test_torchao_export_loadable_ignores_the_stub(monkeypatch):
+ # The main process has already stubbed torchao; only an installed torchao counts.
+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)
+ _set_torchao(monkeypatch, "stub")
+ real = importlib.machinery.PathFinder.find_spec
+ monkeypatch.setattr(
+ importlib.machinery.PathFinder,
+ "find_spec",
+ classmethod(lambda cls, name, *a, **k: None if name == "torchao" else real(name, *a, **k)),
+ )
+ assert _stub.torchao_export_loadable() is False
+
+
+ ("installed", "expect"),
+ [("0.14.0", False), ("0.15.0", True), ("0.17.0", True), ("0.18.0+rocm", True)],
+)
+def test_torchao_export_loadable_needs_transformers_minimum(
+ monkeypatch, tmp_path, installed, expect
+):
+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)
+ (tmp_path / "unsloth").mkdir()
+ (tmp_path / "unsloth" / "_torchao_nodist.py").write_text("")
+ specs = {
+ "torchao": importlib.machinery.ModuleSpec("torchao", None),
+ "unsloth": importlib.machinery.ModuleSpec("unsloth", None, is_package = True),
+ }
+ specs["unsloth"].submodule_search_locations = [str(tmp_path / "unsloth")]
+ monkeypatch.setattr(
+ importlib.machinery.PathFinder,
+ "find_spec",
+ classmethod(lambda cls, name, *a, **k: specs.get(name)),
+ )
+ monkeypatch.setattr(importlib.util, "find_spec", lambda name, *a, **k: specs.get(name))
+ monkeypatch.setattr(importlib.metadata, "version", lambda name: installed)
+ assert _stub.torchao_export_loadable() is expect
+
+
[email protected](("fix_result", "expect_real"), [(True, True), (False, False)])
+def test_real_or_stub(monkeypatch, fix_result, expect_real):
+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)
+ monkeypatch.setattr(_stub, "torchao_export_loadable", lambda: True)
+ monkeypatch.delitem(sys.modules, "torchao", raising = False)
+ calls = []
+
+ def fix():
+ calls.append(1)
+ if fix_result:
+ sys.modules["torchao"] = types.ModuleType("torchao")
+ return fix_result
+
+ monkeypatch.setattr(
+ _stub,
+ "_load_torchao_nodist",
+ lambda: types.SimpleNamespace(fix_torchao_without_torch_distributed = fix),
+ )
+ stubbed = []
+ monkeypatch.setattr(_stub, "install_torchao_windows_rocm_stub", lambda: stubbed.append(1))
+ try:
+ assert _stub.install_torchao_windows_rocm_real_or_stub() is expect_real
+ assert calls == [1]
+ assert bool(stubbed) is (not expect_real)
+ finally:
+ sys.modules.pop("torchao", None)
+
+
[email protected]("consumer_loaded", [False, True])
+def test_real_or_stub_replaces_a_stub_inherited_from_run_py(monkeypatch, consumer_loaded):
+ """spawn re-runs run.py as __mp_main__, which stubs torchao before the export worker starts."""
+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)
+ monkeypatch.setattr(_stub, "torchao_export_loadable", lambda: True)
+ # Simulated "already imported" consumer: json is always loaded; none of the real ones is required.
+ monkeypatch.setattr(_stub, "_STUB_CONSUMERS", ("json",) if consumer_loaded else ())
+ saved = {n: m for n, m in sys.modules.items() if n == "torchao" or n.startswith("torchao.")}
+ for name in saved:
+ monkeypatch.delitem(sys.modules, name)
+ calls = []
+
+ def fix():
+ calls.append("torchao" in sys.modules)
+ sys.modules["torchao"] = types.ModuleType("torchao")
+ return True
+
+ monkeypatch.setattr(
+ _stub,
+ "_load_torchao_nodist",
+ lambda: types.SimpleNamespace(fix_torchao_without_torch_distributed = fix),
+ )
+ try:
+ _stub.install_torchao_windows_rocm_stub()
+ assert _stub.is_stubbed("torchao")
+ result = _stub.install_torchao_windows_rocm_real_or_stub()
+ if consumer_loaded:
+ assert result is False and calls == [] and _stub.is_stubbed("torchao")
+ else:
+ assert result is True and calls == [False] and not _stub.is_stubbed("torchao")
+ assert not any(n.startswith("torchao.") for n in sys.modules)
+ finally:
+ for name in [n for n in sys.modules if n == "torchao" or n.startswith("torchao.")]:
+ del sys.modules[name]
+ sys.modules.update(saved)
+
+
+def test_real_or_stub_keeps_the_stub_when_torchao_is_not_loadable(monkeypatch):
+ """torch <= 2.9 pairs with torchao 0.14, which transformers 5 rejects: never load it."""
+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)
+ monkeypatch.setattr(_stub, "torchao_export_loadable", lambda: False)
+ monkeypatch.delitem(sys.modules, "torchao", raising = False)
+ monkeypatch.setattr(
+ _stub, "_load_torchao_nodist", lambda: pytest.fail("must not load the shim")
+ )
+ stubbed = []
+ monkeypatch.setattr(_stub, "install_torchao_windows_rocm_stub", lambda: stubbed.append(1))
+ assert _stub.install_torchao_windows_rocm_real_or_stub() is False
+ assert stubbed == [1]
+
+
+def test_shim_found_through_an_editable_finder(monkeypatch, tmp_path):
+ """setuptools editable installs can expose unsloth only through a meta-path finder."""
+ (tmp_path / "unsloth").mkdir()
+ (tmp_path / "unsloth" / "_torchao_nodist.py").write_text(
+ "def fix_torchao_without_torch_distributed():\n return 'ok'\n"
+ )
+
+ class EditableFinder:
+ def find_spec(
+ self,
+ name,
+ path = None,
+ target = None,
+ ):
+ if name != "unsloth":
+ return None
+ spec = importlib.machinery.ModuleSpec("unsloth", None, is_package = True)
+ spec.submodule_search_locations = [str(tmp_path / "unsloth")]
+ return spec
+
+ monkeypatch.delitem(sys.modules, "unsloth", raising = False)
+ real = importlib.machinery.PathFinder.find_spec
+ monkeypatch.setattr(
+ importlib.machinery.PathFinder,
+ "find_spec",
+ classmethod(lambda cls, name, *a, **k: None if name == "unsloth" else real(name, *a, **k)),
+ )
+ monkeypatch.setattr(sys, "meta_path", [EditableFinder(), *sys.meta_path])
+ module = _stub._load_torchao_nodist()
+ assert module is not None and module.fix_torchao_without_torch_distributed() == "ok"
+
+
+def test_real_or_stub_noop_off_windows_rocm(monkeypatch):
+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: False)
+ monkeypatch.setattr(
+ _stub, "_load_torchao_nodist", lambda: pytest.fail("must not load the shim")
+ )
+ assert _stub.install_torchao_windows_rocm_real_or_stub() is False
+
+
+def test_load_torchao_nodist_reads_unsloths_file():
+ module = _stub._load_torchao_nodist()
+ if importlib.util.find_spec("unsloth") is None:
+ assert module is None
+ else:
+ assert callable(module.fix_torchao_without_torch_distributed)
diff --git a/studio/backend/tests/test_torchao_stub_worker_parity.py b/studio/backend/tests/test_torchao_stub_worker_parity.py
index bb743385f1f..8f2f0b3f86b 100644
--- a/studio/backend/tests/test_torchao_stub_worker_parity.py
+++ b/studio/backend/tests/test_torchao_stub_worker_parity.py
@@ -18,11 +18,16 @@ from __future__ import annotations
import ast
from pathlib import Path
-from core._torchao_stub import install_torchao_windows_rocm_stub
+from core._torchao_stub import (
+ install_torchao_windows_rocm_real_or_stub,
+ install_torchao_windows_rocm_stub,
+)
_BACKEND = Path(__file__).resolve().parent.parent # studio/backend
_CORE = _BACKEND / "core"
_STUB = install_torchao_windows_rocm_stub.__name__ # a rename breaks the import loudly
+# The export worker loads real torchao when it can, falling back to the same stub.
+_REAL_OR_STUB = install_torchao_windows_rocm_real_or_stub.__name__
_ENTRYPOINTS = [
_CORE / "training" / "worker.py",
@@ -37,7 +42,9 @@ def _stub_call_linenos(node) -> list[int]:
return [
c.lineno
for c in ast.walk(node)
- if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == _STUB
+ if isinstance(c, ast.Call)
+ and isinstance(c.func, ast.Name)
+ and c.func.id in (_STUB, _REAL_OR_STUB)
]
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 4383d301ce0..685299c26e3 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -1719,12 +1719,22 @@ def _gpu_present_but_unusable_message(
def export_capability() -> dict:
- """Whether model export can run here, with a torch-aware reason when it cannot. Export runs through Unsloth, which hard-requires an accelerator (it calls ``torch.cuda`` at import and has no CPU path), so it is supported iff ``get_device() in {CUDA, XPU, MLX}``. The reason distinguishes a --no-torch install from a bare-CPU host. Safe to call without torch. Returns {export_supported, export_unsupported_reason, export_unsupported_message}."""
- if get_device() in (DeviceType.CUDA, DeviceType.XPU, DeviceType.MLX):
+ """Whether model export can run here, with a torch-aware reason when it cannot. Export runs through Unsloth, which hard-requires an accelerator (it calls ``torch.cuda`` at import and has no CPU path), so it is supported iff ``get_device() in {CUDA, XPU, MLX}``. The reason distinguishes a --no-torch install from a bare-CPU host. Safe to call without torch. Returns {export_supported, export_unsupported_reason, export_unsupported_message, torchao_export_supported}; the last is False only on Windows ROCm without a loadable torchao, where the portable FP8/INT8 formats are hidden."""
+ device = get_device()
+ torchao_export_supported = True
+ # get_device() ran detect_hardware(), so IS_ROCM (hip field OR "rocm" tag) is settled.
+ if sys.platform == "win32" and IS_ROCM:
+ try:
+ from core._torchao_stub import torchao_export_loadable
+ torchao_export_supported = torchao_export_loadable()
+ except Exception:
+ torchao_export_supported = False
+ if device in (DeviceType.CUDA, DeviceType.XPU, DeviceType.MLX):
return {
"export_supported": True,
"export_unsupported_reason": None,
"export_unsupported_message": None,
+ "torchao_export_supported": torchao_export_supported,
}
verdict = current_chat_only_verdict()
# No accelerator: name the blocker. Detection failure first, since the branches below all describe a measured host, so a broken probe would tell a GPU box to install PyTorch.
@@ -1767,6 +1777,7 @@ def export_capability() -> dict:
"export_supported": False,
"export_unsupported_reason": reason,
"export_unsupported_message": message,
+ "torchao_export_supported": torchao_export_supported,
}
diff --git a/studio/frontend/src/features/export/export-page.tsx b/studio/frontend/src/features/export/export-page.tsx
index 2118298deda..96881ec8a4a 100644
--- a/studio/frontend/src/features/export/export-page.tsx
+++ b/studio/frontend/src/features/export/export-page.tsx
@@ -232,6 +232,7 @@ export function ExportPage() {
const hardware = useHardwareInfo();
// GGUF LoRA conversion is rejected on the macOS / MLX path, so gate it out on a Mac host.
const isMacHost = usePlatformStore((s) => s.deviceType) === "mac";
+ const torchaoUnavailable = !hardware.torchaoExportSupported;
// Real CUDA (not ROCm); gates the NVIDIA-only compressed-tensors formats.
const hasNvidia = hardware.cuda != null && hardware.rocm == null;
// Only gray out on an authoritative unsupported response; the backend supplies the reason.
@@ -246,19 +247,27 @@ export function ExportPage() {
// compressed-tensors (llm-compressor) is the NVIDIA path; shown only on an NVIDIA GPU.
if (f.backend === "compressed") return hasNvidia;
// Portable torchao is the fallback for hosts without the NVIDIA compressed path. Hidden on
- // NVIDIA (use compressed-tensors) and on macOS/MLX (the backend rejects quantized export).
- if (f.backend === "torchao") return !hasNvidia && !isMacHost;
+ // NVIDIA (use compressed-tensors), macOS/MLX (rejected) and where torchao cannot load.
+ if (f.backend === "torchao") return !hasNvidia && !isMacHost && !torchaoUnavailable;
// Plain 16-bit is available everywhere.
return true;
}),
- [hasNvidia, isMacHost],
+ [hasNvidia, isMacHost, torchaoUnavailable],
);
const toggleFormat = useCallback((value: string) => {
setSelectedFormats((prev) =>
prev.includes(value) ? prev.filter((v) => v !== value) : [...prev, value],
);
}, []);
- // availableFormats already drops NVIDIA-only formats on other hardware, so no pruning needed.
+ // Drop picks the gate removed, only once hardware is loaded: before that hasNvidia reads false.
+ useEffect(() => {
+ if (!hardware.loaded) return;
+ const allowed = new Set(availableFormats.map((f) => f.value));
+ setSelectedFormats((prev) => {
+ const next = prev.filter((v) => allowed.has(v));
+ return next.length === prev.length ? prev : next;
+ });
+ }, [availableFormats, hardware.loaded]);
// IQ quants are imatrix-only: force imatrix on when one is selected, else llama.cpp rejects it.
const requiresImatrix = quantLevels.some(
(q) => QUANT_OPTIONS.find((o) => o.value === q)?.imatrix,
@@ -1523,13 +1532,21 @@ export function ExportPage() {
</div>
)}
- {!hasNvidia && (
+ {!hasNvidia && !torchaoUnavailable && (
<div className="text-ui-11 text-muted-foreground">
No NVIDIA GPU detected: compressed-tensors formats are
hidden. 16-bit and portable FP8/INT8 (torchao) still
work here and load in vLLM.
</div>
)}
+
+ {torchaoUnavailable && (
+ <div className="text-ui-11 text-muted-foreground">
+ Portable FP8/INT8 (torchao) export needs torchao, which
+ is not installed here. Update Unsloth, or use
+ 16-bit or GGUF.
+ </div>
+ )}
</div>
</div>
)}
diff --git a/studio/frontend/src/hooks/use-hardware-info.ts b/studio/frontend/src/hooks/use-hardware-info.ts
index 36f42f94054..71e5f47a419 100644
--- a/studio/frontend/src/hooks/use-hardware-info.ts
+++ b/studio/frontend/src/hooks/use-hardware-info.ts
@@ -34,6 +34,8 @@ export interface HardwareInfo {
exportSupported: boolean | null;
exportUnsupportedReason: string | null;
exportUnsupportedMessage: string | null;
+ // False only on Windows ROCm without a loadable torchao; absent from older backends = true.
+ torchaoExportSupported: boolean;
// Whether video generation can run here. Same tri-state as export: `null` until the
// authoritative response lands, and `null` too against a backend that predates the field,
// so only an explicit `false` hides the generator.
@@ -58,6 +60,7 @@ const DEFAULT: HardwareInfo = {
exportSupported: null,
exportUnsupportedReason: null,
exportUnsupportedMessage: null,
+ torchaoExportSupported: true,
videoSupported: null,
videoUnsupportedReason: null,
videoUnsupportedMessage: null,
@@ -118,6 +121,7 @@ async function fetchOnce(): Promise<HardwareInfo> {
exportSupported: data?.export_supported ?? null,
exportUnsupportedReason: data?.export_unsupported_reason ?? null,
exportUnsupportedMessage: data?.export_unsupported_message ?? null,
+ torchaoExportSupported: data?.torchao_export_supported ?? true,
videoSupported: data?.video_supported ?? null,
videoUnsupportedReason: data?.video_unsupported_reason ?? null,
videoUnsupportedMessage: data?.video_unsupported_message ?? null,
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py
index 581a1b1cd83..664d9f81d7b 100644
--- a/studio/install_python_stack.py
+++ b/studio/install_python_stack.py
@@ -1026,8 +1026,7 @@ def _installed_torch_is_windows_rocm() -> bool:
This is a belt-and-suspenders guard for the torchao override step: if the
earlier ROCm install path failed to set _rocm_windows_torch_installed but the
- venv already contains a ROCm torch wheel, still skip torchao because it
- crashes on import on Windows ROCm.
+ venv already contains a ROCm torch wheel, torchao still comes from PyPI.
"""
if not IS_WINDOWS:
return False
@@ -5153,8 +5152,9 @@ def _resident_xformers_build_torch() -> "str | None":
return recorded.strip() if isinstance(recorded, str) and recorded.strip() else None
-def _install_torchao_for_torch(torch_version: "str | None") -> None:
- """Select the torchao matching torch_version and install it from its own index.
+def _install_torchao_for_torch(torch_version: "str | None", default_index: bool = False) -> None:
+ """Select the torchao matching torch_version and install it from its own index (PyPI's with
+ default_index: download.pytorch.org's rocm leaves serve Linux only, so not Windows ROCm).
Called twice: as step 4, and again after the Linux torch repair, which can move torch
across families and releases underneath the first call.
@@ -5162,7 +5162,7 @@ def _install_torchao_for_torch(torch_version: "str | None") -> None:
spec = _select_torchao_spec(torch_version)
# See _TORCHAO_DEFAULT_SPEC. rocm is included here, unlike torchcodec: the rocm leaves
# really do publish torchao.
- index = _torch_accelerator_index_url(torch_version)
+ index = None if default_index else _torch_accelerator_index_url(torch_version)
# --no-deps skips nothing today (no torchao release declares a runtime torch dependency)
# and guards the second caller, which runs right after the torch repair.
args = ["--no-deps", "--no-cache-dir"]
@@ -5183,6 +5183,11 @@ def _install_torchao_for_torch(torch_version: "str | None") -> None:
# Redacted for display only; the installer below still gets the exact URL.
+ (f" from {_strip_index_url_credentials(index)}" if index else "")
)
+ if default_index:
+ # Optional on Windows ROCm: only export uses it, and hides its formats without it.
+ if not pip_install_try("Installing dependency overrides", *args, spec):
+ _note(f"could not install {spec}; torchao export stays unavailable")
+ return
if not index:
pip_install("Installing dependency overrides", *args, spec)
return
@@ -11755,13 +11760,13 @@ def install_python_stack() -> int:
# 4. Install the torch-matched torchao override. Reinstall only when the pin
# changes, since Windows can remove shared files during replacement.
- # Skip when torch is unavailable or Windows ROCm has no working build.
+ # Skip when torch is unavailable.
if NO_TORCH:
_progress("dependency overrides (skipped, no torch)")
elif _rocm_windows_torch_installed or _installed_torch_is_windows_rocm():
- # No working Windows ROCm torchao build (crashes on import; stubbed at runtime).
- _progress("dependency overrides (skipped, Windows ROCm)")
- _note("Windows ROCm -- skipping torchao (no working build; stubbed at runtime)")
+ # Stock torchao dies on import here; only the export worker loads it (unsloth/_torchao_nodist.py).
+ _progress("dependency overrides (Windows ROCm)")
+ _install_torchao_for_torch(_probe_installed_torch_version(), default_index = True)
else:
_progress("dependency overrides")
_install_torchao_for_torch(_probe_installed_torch_version())
diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py
index 02591746258..1a19895278b 100644
--- a/tests/studio/install/test_rocm_support.py
+++ b/tests/studio/install/test_rocm_support.py
@@ -5852,7 +5852,7 @@ class TestWindowsRocmTorchaoGuard:
@patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = True)
@patch.object(stack_mod, "run")
@patch.object(stack_mod, "pip_install")
- def test_install_python_stack_skips_torchao_when_windows_rocm_torch_is_installed(
+ def test_install_python_stack_installs_pypi_torchao_when_windows_rocm_torch_is_installed(
self,
mock_pip,
mock_run,
@@ -5887,11 +5887,18 @@ class TestWindowsRocmTorchaoGuard:
patch.object(stack_mod.install_manifest, "installed_versions", return_value = ["0"]),
patch.object(stack_mod.install_manifest, "damaged_payload_files", return_value = []),
patch.object(stack_mod.subprocess, "run", return_value = subprocess_result),
+ patch.object(stack_mod, "pip_install_try", return_value = True) as mock_try,
):
assert stack_mod.install_python_stack() == 0
- installed_specs = [str(arg) for call in mock_pip.call_args_list for arg in call.args]
- assert not any("torchao" in arg for arg in installed_specs)
+ torchao_calls = [
+ [str(arg) for arg in call.args]
+ for call in mock_pip.call_args_list + mock_try.call_args_list
+ if any(str(arg).startswith("torchao") for arg in call.args)
+ ]
+ # From PyPI: download.pytorch.org's rocm leaves serve Linux only.
+ assert torchao_calls
+ assert all("--index-url" not in c for c in torchao_calls)
class TestProgressStepCountMatchesTotal:
@@ -6116,9 +6123,9 @@ class TestWorkerWindowsRocmPatches:
assert "install_torchao_windows_rocm_stub()" in source
def test_export_worker_calls_shared_torchao_stub(self):
- """export/worker.py must invoke the same shared torchao stub entrypoint."""
+ """export/worker.py loads real torchao when it can and otherwise the same shared stub."""
source = _EXPORT_WORKER_PATH.read_text(encoding = "utf-8")
- assert "install_torchao_windows_rocm_stub()" in source
+ assert "install_torchao_windows_rocm_real_or_stub()" in source
def test_embedder_calls_shared_torchao_stub(self):
"""embeddings.py must install the stub before importing sentence-transformers:
diff --git a/tests/test_torchao_nodist.py b/tests/test_torchao_nodist.py
new file mode 100644
index 00000000000..c6046040800
--- /dev/null
+++ b/tests/test_torchao_nodist.py
@@ -0,0 +1,161 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""unsloth/_torchao_nodist.py on a simulated torch without torch.distributed (AMD's Windows ROCm
+wheels): is_available() False, every torch.distributed.* submodule unimportable, and the c10d ops
+absent, as in pytorch/ao#4761's test plus the missing ops. Out of process so the parent's module
+cache is untouched. Verified on real hardware separately (Windows 11, gfx1151, torch 2.11 ROCm)."""
+
+import importlib.util
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+_SHIM = Path(__file__).resolve().parents[1] / "unsloth" / "_torchao_nodist.py"
+
+_SIMULATE = r"""
+import importlib.util, json, sys
+import torch
+
+torch.distributed.is_available = lambda: False
+for name in [n for n in sys.modules if n.startswith("torch.distributed.")]:
+ del sys.modules[name]
+for attr, value in list(vars(torch.distributed).items()):
+ if isinstance(value, type(sys)) and value.__name__.startswith("torch.distributed."):
+ delattr(torch.distributed, attr)
+
+
+class BlockDistributed:
+ def find_spec(self, fullname, path = None, target = None):
+ if fullname.startswith("torch.distributed."):
+ raise ModuleNotFoundError(f"No module named 'torch._C._distributed_c10d' ({fullname})", name = fullname)
+ return None
+
+
+sys.meta_path.insert(0, BlockDistributed())
+_HIDDEN = {"c10d_functional", "_c10d_functional", "c10d", "_dtensor"}
+_ns = torch._ops._OpNamespace
+_real = _ns.__getattr__
+
+
+def _hiding(self, op):
+ if self.name in _HIDDEN and not op.startswith("__"):
+ raise AttributeError(f"'_OpNamespace' '{self.name}' object has no attribute '{op}'")
+ return _real(self, op)
+
+
+_ns.__getattr__ = _hiding
+for ns in _HIDDEN:
+ obj = torch.ops.__dict__.get(ns)
+ for attr in list(vars(obj)) if obj is not None else []:
+ if not attr.startswith("__"):
+ try:
+ delattr(obj, attr)
+ except Exception:
+ pass
+
+r = {}
+if sys.argv[2] == "fix":
+ spec = importlib.util.spec_from_file_location("_nodist", sys.argv[1])
+ shim = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(shim)
+ r["fixed"] = shim.fix_torchao_without_torch_distributed()
+for stmt in ("import torchao", "from torchao.quantization import Int8WeightOnlyConfig, Float8WeightOnlyConfig",
+ "import transformers.quantizers.quantizer_torchao", "import peft.tuners.lora.torchao",
+ "import torch.distributed.tensor"):
+ try:
+ exec(stmt)
+ r[stmt] = "ok"
+ except BaseException as e:
+ r[stmt] = f"{type(e).__name__}: {e}"[:200]
+held = []
+for name, module in list(sys.modules.items()):
+ if module is None or name == "torchao" or name.startswith(("torchao.", "_nodist")):
+ continue
+ try:
+ values = list(vars(module).values())
+ except Exception:
+ continue
+ for value in values:
+ try:
+ if isinstance(value, type(sys)) and value.__dict__.get("__unsloth_nodist_stub__"):
+ held.append(name)
+ except Exception:
+ pass
+r["stand_ins_held_outside_torchao"] = sorted(set(held))
+print("RESULT=" + json.dumps(r))
+"""
+
+pytestmark = pytest.mark.skipif(
+ importlib.util.find_spec("torchao") is None or importlib.util.find_spec("transformers") is None,
+ reason = "needs torchao and transformers installed",
+)
+
+
+def _run(mode):
+ out = subprocess.run(
+ [sys.executable, "-c", _SIMULATE, str(_SHIM), mode],
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ )
+ lines = [l for l in out.stdout.splitlines() if l.startswith("RESULT=")]
+ assert lines, out.stderr[-3000:]
+ return json.loads(lines[-1][len("RESULT=") :])
+
+
+def test_torchao_fails_without_the_fix():
+ r = _run("none")
+ assert r["import torchao"] != "ok"
+
+
+def test_torchao_and_its_consumers_import_with_the_fix():
+ r = _run("fix")
+ assert r["fixed"] is True
+ assert r["import torchao"] == "ok"
+ assert (
+ r["from torchao.quantization import Int8WeightOnlyConfig, Float8WeightOnlyConfig"] == "ok"
+ )
+ # transformers imports torchao.prototype.* lazily, long after `import torchao` returned.
+ assert r["import transformers.quantizers.quantizer_torchao"] == "ok"
+ assert r["import peft.tuners.lora.torchao"] == "ok"
+
+
+def test_nothing_outside_torchao_sees_a_fake_torch_distributed():
+ r = _run("fix")
+ assert r["import torch.distributed.tensor"].startswith("ModuleNotFoundError")
+ assert r["stand_ins_held_outside_torchao"] == []
+
+
+def test_noop_where_torch_distributed_exists():
+ code = (
+ "import importlib.util, sys, torch\n"
+ "spec = importlib.util.spec_from_file_location('_nodist', sys.argv[1])\n"
+ "m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)\n"
+ "print('RESULT', torch.distributed.is_available(), m.fix_torchao_without_torch_distributed(), "
+ "any(type(f).__name__ == '_TorchaoWindowFinder' for f in sys.meta_path))\n"
+ )
+ out = subprocess.run(
+ [sys.executable, "-c", code, str(_SHIM)], capture_output = True, text = True, timeout = 600
+ )
+ line = next(l for l in out.stdout.splitlines() if l.startswith("RESULT"))
+ available, fixed, hooked = line.split()[1:]
+ if available != "True":
+ pytest.skip("this torch has no torch.distributed")
+ assert (fixed, hooked) == ("False", "False")
+
+
+def test_gpu_init_runs_the_fix_before_anything_imports_torchao():
+ """The torch symbol-skew fix must come first (torchao 0.18 on torch < 2.10), and the shim
+ must precede vllm's probe and unsloth_zoo, both of which import transformers."""
+ src = (_SHIM.parent / "_gpu_init.py").read_text(encoding = "utf-8")
+ order = [
+ src.index("\nfix_torchao_torch_symbol_skew()"),
+ src.index("\nfix_torchao_without_torch_distributed()"),
+ src.index("\ndisable_broken_vllm()"),
+ src.index("\n import unsloth_zoo\n"),
+ ]
+ assert order == sorted(order)
diff --git a/tests/test_torchao_subprocess_fix.py b/tests/test_torchao_subprocess_fix.py
index 23f4c5a21c5..ed5b30cb981 100644
--- a/tests/test_torchao_subprocess_fix.py
+++ b/tests/test_torchao_subprocess_fix.py
@@ -505,7 +505,7 @@ if __name__ == "__main__":
def test_the_in_process_fix_does_not_disable_the_subprocess_fix(monkeypatch, tmp_path):
- """_gpu_init.py runs fix_torchao_torch_symbol_skew() immediately before
+ """_gpu_init.py runs fix_torchao_torch_symbol_skew() before
this one, so a gate asking only `hasattr` would read its placeholders as a
healthy torch and stage nothing, in exactly the environments vLLM's
inspector child needs it."""
diff --git a/tests/test_xformers_stub_without_metadata.py b/tests/test_xformers_stub_without_metadata.py
new file mode 100644
index 00000000000..77d1f029448
--- /dev/null
+++ b/tests/test_xformers_stub_without_metadata.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Studio's Windows ROCm run.py puts an xformers stub in sys.modules without installing xformers;
+a spawned worker re-runs run.py, so `import unsloth` meets that stub."""
+
+import importlib.machinery
+import importlib.metadata
+import sys
+import types
+
+from unsloth import import_fixes
+
+
+def test_xformers_stub_without_metadata_is_skipped(monkeypatch):
+ stub = types.ModuleType("xformers")
+ stub.__spec__ = importlib.machinery.ModuleSpec("xformers", loader = None, is_package = True)
+ monkeypatch.setitem(sys.modules, "xformers", stub)
+
+ def missing(name):
+ raise importlib.metadata.PackageNotFoundError(name)
+
+ monkeypatch.setattr(import_fixes, "importlib_version", missing)
+ assert import_fixes.fix_xformers_performance_issue() is None
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index ba918a498a5..15cc1180a73 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -82,11 +82,17 @@ configure_amdgpu_asic_id_table_path()
patch_torch_missing_attribute_error()
# Must precede `import unsloth_zoo` below, which imports bnb on ROCm.
fix_bitsandbytes_rocm_arch_detection()
+# Torch-only, so it can run first; torchao 0.18 on torch < 2.10 needs it before any torchao import.
+fix_torchao_torch_symbol_skew()
+# Before unsloth_zoo and transformers (and vllm, below): real torchao on a torch without torch.distributed.
+from ._torchao_nodist import fix_torchao_without_torch_distributed
+
+fix_torchao_without_torch_distributed()
+del fix_torchao_without_torch_distributed
disable_broken_causal_conv1d()
disable_broken_vllm()
fix_message_factory_issue()
fix_torch_check_is_size()
-fix_torchao_torch_symbol_skew()
# The above fixes THIS process only; vLLM's model-architecture inspector is a subprocess that
# imports torchao itself and hits the same ImportError.
propagate_torchao_fix_to_subprocesses()
diff --git a/unsloth/_torchao_nodist.py b/unsloth/_torchao_nodist.py
new file mode 100644
index 00000000000..681132a67f9
--- /dev/null
+++ b/unsloth/_torchao_nodist.py
@@ -0,0 +1,257 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Import real torchao on a torch built without torch.distributed (AMD's Windows ROCm wheels).
+
+torchao imports DTensor / DeviceMesh / the functional collectives at module scope and keys
+dispatch tables on c10d ops, none of which exist on such a build, so ``import torchao`` dies
+(pytorch/ao#3452; the upstream guard, pytorch/ao#4761, is unmerged). All of it only serves
+isinstance() checks and collective handlers that cannot run without distributed. So each torchao
+module body executes inside a window where imports made BY torchao code get inert stand-ins for
+missing torch.distributed modules and lookups made BY torchao code get inert keys for missing
+c10d ops. Everything else, torch internals imported meanwhile included, sees the real (absent)
+torch.distributed. Stdlib + torch only: Studio loads this file without importing unsloth.
+"""
+
+import importlib
+import importlib.abc
+import importlib.machinery
+import importlib.util
+import sys
+import threading
+import types
+
+__all__ = ["fix_torchao_without_torch_distributed"]
+
+_OP_NAMESPACES = frozenset({"c10d_functional", "_c10d_functional", "c10d", "_dtensor"})
+_IMPORTING = [0]
+
+
+def _is_torchao(name):
+ return isinstance(name, str) and (name == "torchao" or name.startswith("torchao."))
+
+
+def _importer():
+ frame = sys._getframe(1)
+ while frame is not None:
+ name = frame.f_globals.get("__name__", "")
+ if not name.startswith(("importlib", "_frozen_importlib")) and name != __name__:
+ return name
+ frame = frame.f_back
+ return None
+
+
+def _passthrough(fn):
+ return fn
+
+
+def _unavailable(qualname):
+ def fn(*args, **kwargs):
+ # torchao applies distributed decorators (register_sharding(op), ...) while loading;
+ # there they wrap nothing. Any later call is real distributed work.
+ if _IMPORTING[0]:
+ if len(args) == 1 and not kwargs and isinstance(args[0], types.FunctionType):
+ return args[0]
+ return _passthrough
+ raise RuntimeError(f"{qualname} needs torch.distributed, which this torch build lacks.")
+
+ return fn
+
+
+class _NeverMeta(type):
+ def __instancecheck__(cls, obj):
+ return False
+
+ def __subclasscheck__(cls, sub):
+ return False
+
+ def __call__(cls, *args, **kwargs):
+ raise RuntimeError(f"{cls.__name__} needs torch.distributed, which this torch build lacks.")
+
+
+class _StubLoader(importlib.abc.Loader):
+ def create_module(self, spec):
+ return types.ModuleType(spec.name)
+
+ def exec_module(self, module):
+ name = module.__name__
+ cache = {}
+
+ def __getattr__(attr):
+ if attr.startswith("__"):
+ raise AttributeError(attr)
+ if attr not in cache:
+ cache[attr] = (
+ _NeverMeta(attr, (), {"__module__": name})
+ if attr[:1].isupper()
+ else _unavailable(f"{name}.{attr}")
+ )
+ return cache[attr]
+
+ module.__getattr__ = __getattr__
+ module.__path__ = []
+ module.__unsloth_nodist_stub__ = True
+
+
+class _DistStubFinder(importlib.abc.MetaPathFinder):
+ def find_spec(
+ self,
+ fullname,
+ path = None,
+ target = None,
+ ):
+ if not fullname.startswith("torch.distributed.") or fullname in sys.modules:
+ return None
+ if not _is_torchao(_importer()):
+ return None
+ return importlib.machinery.ModuleSpec(fullname, _StubLoader(), is_package = True)
+
+
+class _InertPacket:
+ """A missing c10d OpOverloadPacket: each overload is a unique key no real op equals."""
+
+ def __init__(self, qualname):
+ self._qualname = qualname
+ self._overloads = {}
+
+ def __getattr__(self, overload):
+ if overload.startswith("__"):
+ raise AttributeError(overload)
+ return self._overloads.setdefault(overload, _unavailable(f"{self._qualname}.{overload}"))
+
+ def __call__(self, *args, **kwargs):
+ raise RuntimeError(
+ f"{self._qualname} needs torch.distributed, which this torch build lacks."
+ )
+
+
+class _Window:
+ def __init__(self, torch):
+ self.torch = torch
+ self.lock = threading.RLock()
+ self.depth = 0
+ self.snapshots = []
+ self.packets = {}
+ self.dist_finder = _DistStubFinder()
+
+ def __enter__(self):
+ self.lock.acquire()
+ self.depth += 1
+ _IMPORTING[0] = self.depth
+ self.snapshots.append(set(sys.modules))
+ if self.depth == 1:
+ sys.meta_path.insert(0, self.dist_finder)
+ ns_cls = self.torch._ops._OpNamespace
+ self.original_getattr = original = ns_cls.__getattr__
+ packets = self.packets
+
+ def __getattr__(ns, op_name):
+ try:
+ return original(ns, op_name)
+ except AttributeError:
+ if (
+ ns.name not in _OP_NAMESPACES
+ or op_name.startswith("__")
+ or not _is_torchao(sys._getframe(1).f_globals.get("__name__"))
+ ):
+ raise
+ key = f"{ns.name}.{op_name}"
+ return packets.setdefault(key, _InertPacket(key))
+
+ ns_cls.__getattr__ = __getattr__
+ return self
+
+ def __exit__(self, *exc):
+ try:
+ self.depth -= 1
+ _IMPORTING[0] = self.depth
+ # This module keeps its references; nobody else may find the stand-ins.
+ for name in set(sys.modules) - self.snapshots.pop():
+ module = sys.modules.get(name)
+ if getattr(module, "__unsloth_nodist_stub__", False):
+ del sys.modules[name]
+ parent_name, _, attr = name.rpartition(".")
+ parent = sys.modules.get(parent_name)
+ if parent is not None and getattr(parent, attr, None) is module:
+ delattr(parent, attr)
+ if self.depth == 0:
+ self.torch._ops._OpNamespace.__getattr__ = self.original_getattr
+ sys.meta_path.remove(self.dist_finder)
+ finally:
+ self.lock.release()
+ return False
+
+
+class _WindowedLoader(importlib.abc.Loader):
+ def __init__(self, loader, window):
+ self.loader = loader
+ self.window = window
+
+ def create_module(self, spec):
+ return self.loader.create_module(spec)
+
+ def exec_module(self, module):
+ with self.window:
+ self.loader.exec_module(module)
+
+ def __getattr__(self, name):
+ return getattr(self.loader, name)
+
+
+class _TorchaoWindowFinder(importlib.abc.MetaPathFinder):
+ """Every torchao module, including ones transformers imports lazily later
+ (quantizer_torchao -> torchao.prototype.mx_formats), loads inside the window."""
+
+ def __init__(self, window):
+ self.window = window
+
+ def find_spec(
+ self,
+ fullname,
+ path = None,
+ target = None,
+ ):
+ if not _is_torchao(fullname):
+ return None
+ for finder in sys.meta_path:
+ if finder is self or not hasattr(finder, "find_spec"):
+ continue
+ spec = finder.find_spec(fullname, path, target)
+ if spec is not None:
+ break
+ else:
+ return None
+ if spec.loader is not None and not isinstance(spec.loader, _WindowedLoader):
+ spec.loader = _WindowedLoader(spec.loader, self.window)
+ return spec
+
+
+def fix_torchao_without_torch_distributed():
+ """True once real torchao is imported through the window. False, leaving nothing behind,
+ where torch.distributed exists, torchao is absent or already imported (e.g. Studio's
+ stub), or torchao still fails to import."""
+ if any(type(f).__name__ == "_TorchaoWindowFinder" for f in sys.meta_path):
+ return "torchao" in sys.modules
+ if "torchao" in sys.modules:
+ return False
+ try:
+ if importlib.util.find_spec("torchao") is None:
+ return False
+ import torch
+ if torch.distributed.is_available():
+ return False
+ except Exception:
+ return False
+
+ finder = _TorchaoWindowFinder(_Window(torch))
+ sys.meta_path.insert(0, finder)
+ before = set(sys.modules)
+ try:
+ importlib.import_module("torchao")
+ return True
+ except Exception:
+ sys.meta_path.remove(finder)
+ for name in set(sys.modules) - before:
+ if _is_torchao(name):
+ sys.modules.pop(name, None)
+ return False
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 6ca081fab70..a0467ad8c58 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -333,7 +333,11 @@ def fix_xformers_performance_issue():
spec = importlib.util.find_spec("xformers")
if spec is None:
return
- xformers_version = importlib_version("xformers")
+ try:
+ xformers_version = importlib_version("xformers")
+ except Exception:
+ # Studio's Windows ROCm xformers stub: in sys.modules, not installed.
+ return
if Version(xformers_version) < Version("0.0.29"):
xformers_location = spec.origin
if xformers_location is None:
diff --git a/unsloth/save.py b/unsloth/save.py
index 04adfe65de5..9b147872e21 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -6719,7 +6719,11 @@ def _unsloth_save_torchao(
quant_type = Float8WeightOnlyConfig()
safe_serialization = True
elif kind == "int8":
- quant_type = Int8WeightOnlyConfig()
+ # version 2 (Int8Tensor) is what transformers serializes; torchao 0.16 / 0.17 default to 1.
+ _int8_fields = getattr(Int8WeightOnlyConfig, "__dataclass_fields__", {})
+ quant_type = (
+ Int8WeightOnlyConfig(version = 2) if "version" in _int8_fields else Int8WeightOnlyConfig()
+ )
safe_serialization = False # torchao only supports safetensors for float8 configs
else:
raise RuntimeError(f"Unsloth: unknown torchao export kind '{kind}' (expected fp8/int8).")