Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio (AMD/Windows): real torchao INT8 / FP8 export on Windows ROCm (#7102)
Daniel Han7b07d860992f
diff --git a/studio/backend/core/_torchao_stub.py b/studio/backend/core/_torchao_stub.pyindex 00c0beb79e2..54943908fbe 100644--- a/studio/backend/core/_torchao_stub.py+++ b/studio/backend/core/_torchao_stub.py@@ -22,6 +22,7 @@ import reimport sysimport typesimport importlib.abc+import importlib.metadataimport importlib.machineryimport importlib.utilfrom typing import Optional@@ -227,6 +228,71 @@ def install_torchao_windows_rocm_stub() -> None:sys.modules[_tao_name] = _make_mod_stub(_tao_name)+def _load_torchao_nodist():+ """unsloth/_torchao_nodist.py, loaded by path: importing unsloth here would start its GPU+ stack before the worker is ready. None on an unsloth that predates it."""+ try:+ spec = importlib.util.find_spec("unsloth")+ locations = list(spec.submodule_search_locations or ()) if spec else []+ path = os.path.join(locations[0], "_torchao_nodist.py") if locations else None+ if not path or not os.path.isfile(path):+ return None+ module_spec = importlib.util.spec_from_file_location("_studio_torchao_nodist", path)+ module = importlib.util.module_from_spec(module_spec)+ module_spec.loader.exec_module(module)+ return module+ except Exception:+ return None+++_TORCHAO_EXPORT_MIN = (0, 15)+++def torchao_export_loadable() -> bool:+ """Whether an export worker can get real torchao: always off Windows ROCm; there, torchao must+ be installed and unsloth must ship the shim. Searches sys.path, so a stub already in+ sys.modules does not count, and imports nothing."""+ if not _is_windows_rocm():+ return True+ try:+ if importlib.machinery.PathFinder.find_spec("torchao") is None:+ return False+ # transformers 5's TorchAoConfig minimum; torch <= 2.9 is paired with torchao 0.14.+ found = re.match(r"(\d+)\.(\d+)", importlib.metadata.version("torchao"))+ if not found or (int(found[1]), int(found[2])) < _TORCHAO_EXPORT_MIN:+ return False+ spec = importlib.util.find_spec("unsloth")+ locations = list(spec.submodule_search_locations or ()) if spec else []+ return bool(locations) and os.path.isfile(os.path.join(locations[0], "_torchao_nodist.py"))+ except Exception:+ return False+++_STUB_CONSUMERS = ("transformers", "peft", "diffusers", "accelerate", "unsloth", "unsloth_zoo")+++def install_torchao_windows_rocm_real_or_stub() -> bool:+ """Export worker: real torchao on Windows ROCm when it is installed and unsloth can import it+ without torch.distributed, else the stub. True iff real torchao is loaded. No-op elsewhere."""+ if not _is_windows_rocm():+ return False+ # A spawn child re-runs run.py as __mp_main__, which stubs torchao first. Drop that stub+ # while nothing that could have bound it is loaded yet.+ if is_stubbed("torchao") and not any(m in sys.modules for m in _STUB_CONSUMERS):+ for name in [n for n in sys.modules if n == "torchao" or n.startswith("torchao.")]:+ if getattr(sys.modules[name], "_unsloth_stub", None) is _STUB_SENTINEL:+ del sys.modules[name]+ if "torchao" not in sys.modules and torchao_export_loadable():+ module = _load_torchao_nodist()+ try:+ if module is not None and module.fix_torchao_without_torch_distributed():+ return True+ except Exception:+ pass+ install_torchao_windows_rocm_stub()+ return not is_stubbed("torchao") and "torchao" in sys.modules++def install_xformers_windows_rocm_stub() -> None:"""Pre-stub xformers on Windows ROCm so diffusers can import at all. No-op elsewhere, and mustprecede diffusers: the Windows xformers pin is CUDA-only, so against a ROCm torch (nodiff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.pyindex f8ea88facf9..7e16f3a8f4b 100644--- a/studio/backend/core/export/export.py+++ b/studio/backend/core/export/export.py@@ -9,6 +9,7 @@ import structlogimport tempfilefrom loggers import get_loggerimport os+import sysimport shutilimport contextlibfrom pathlib import Path@@ -368,14 +369,42 @@ def _compressed_export_supported():def _torchao_export_supported():- """True if the installed unsloth build has the portable torchao FP8/INT8 export path."""+ """True if the installed unsloth build has the portable torchao FP8/INT8 export path and a real+ torchao to run it; False where torchao is stubbed (its config classes return None)."""try:+ if _torchao_runtime_unavailable():+ return Falseimport unsloth.save as _usreturn hasattr(_us, "_normalize_torchao_method")except Exception:return False+def _torchao_runtime_unavailable():+ try:+ from core._torchao_stub import _is_windows_rocm, is_stubbed+ if is_stubbed("torchao"):+ return True+ # Windows ROCm only gets real torchao through install_torchao_windows_rocm_real_or_stub().+ return _is_windows_rocm() and "torchao" not in sys.modules+ except Exception:+ return False+++def _is_torchao_alias(alias):+ """Any torchao spelling unsloth's normalizer accepts, else a torchao_ prefix, so the Windows+ ROCm guard catches it before it is misread as compressed-tensors."""+ if not alias:+ return False+ try:+ import unsloth.save as _us+ if _us._normalize_torchao_method(alias) is not None:+ return True+ except Exception:+ pass+ return str(alias).lower().startswith("torchao")++def _has_nvidia_gpu():"""True only on a real NVIDIA CUDA box (not ROCm/XPU/CPU/MLX); compressed-tensors needs it."""try:@@ -943,6 +972,17 @@ class ExportBackend:"NVFP4 (compressed-tensors)": "nvfp4",}compressed_alias = compressed_method or _LABEL_TO_ALIAS.get(format_type)++ # Fail fast: a stubbed torchao otherwise crashes in transformers with TorchAoConfig(quant_type=None).+ if _is_torchao_alias(compressed_alias) and _torchao_runtime_unavailable():+ return (+ False,+ "Portable torchao FP8/INT8 export needs torchao, which could not be loaded "+ "on this Windows ROCm build. Update Unsloth to install it, or use "+ "16-bit merged or GGUF quantization instead.",+ None,+ )+compressed_suffix: Optional[str] = None# Classify the alias: torchao-portable vs compressed-tensors.torchao_info = Nonediff --git a/studio/backend/core/export/worker.py b/studio/backend/core/export/worker.pyindex 663961c6e04..e965370c986 100644--- a/studio/backend/core/export/worker.py+++ b/studio/backend/core/export/worker.py@@ -596,11 +596,10 @@ def run_export_process(*, cmd_queue: Any, resp_queue: Any, config: dict) -> Nonefrom core._msvc_env import gate_torch_compile_on_windowsgate_torch_compile_on_windows(logger)- # See core/_torchao_stub.py: torchao crashes on Windows ROCm (RCCL absent). No-op off Windows ROCm. Must run- # before importing transformers / unsloth_zoo.- from core._torchao_stub import install_torchao_windows_rocm_stub+ # Before transformers / unsloth_zoo: real torchao via unsloth's shim on Windows ROCm, else the stub.+ from core._torchao_stub import install_torchao_windows_rocm_real_or_stub- install_torchao_windows_rocm_stub()+ install_torchao_windows_rocm_real_or_stub()try:_send_response(diff --git a/studio/backend/tests/test_torchao_select.py b/studio/backend/tests/test_torchao_select.pyindex c42613a11a0..550ab85f152 100644--- a/studio/backend/tests/test_torchao_select.py+++ b/studio/backend/tests/test_torchao_select.py@@ -10,6 +10,8 @@ venv (otherwise the cpp kernels are skipped). This pins that mapping.from __future__ import annotations+import importlib+import importlib.machineryimport sysfrom pathlib import Pathfrom unittest.mock import MagicMock@@ -185,7 +187,7 @@ def test_the_torchao_step_pins_the_index_and_retries_without_it():after this ships can lag a release, so the pin must not be able to fail an install.Unlike torchcodec the retry stays FATAL if it also fails: torchao is not optional."""body = _torchao_installer_source()- assert "index = _torch_accelerator_index_url(torch_version)" in body+ assert "index = None if default_index else _torch_accelerator_index_url(torch_version)" in bodyassert '"--index-url", index, spec' in bodyassert "retrying from the default index" in body# The unpinned attempt is pip_install, not pip_install_try: still fatal on failure.@@ -327,18 +329,23 @@ def test_windows_first_hop_uses_einx_wheel_without_shared_test_tree():(True, True),],)-def test_skips_torchao_on_windows_rocm(- monkeypatch, tmp_path, rocm_windows_torch_installed, installed_torch_is_windows_rocm[email protected]("torchao_install_ok", [True, False])+def test_installs_pypi_torchao_on_windows_rocm(+ monkeypatch,+ tmp_path,+ rocm_windows_torch_installed,+ installed_torch_is_windows_rocm,+ torchao_install_ok,):- """The overrides step must skip torchao on Windows ROCm: no working build exists- there (it imports an absent c10d backend and crashes transformers.quantizers),- so the installer skips it and relies on the runtime stub instead."""+ """Windows ROCm gets the torch-matched torchao from PyPI (download.pytorch.org's rocm leaves+ serve Linux only); the export worker loads it through unsloth/_torchao_nodist.py. A failed+ torchao install does not fail the install: only export needs it."""mod = _load_module(monkeypatch)- installed_specs: list[str] = []+ pip_calls: list[list[str]] = []progress_labels: list[str] = []def _record_pip_install(*args, **kwargs):- installed_specs.extend(str(arg) for arg in args)+ pip_calls.append([str(arg) for arg in args])return 0unstructured_plugin = tmp_path / "unstructured"@@ -385,7 +392,23 @@ def test_skips_torchao_on_windows_rocm(mod, "_probe_torch_runtime", lambda *args, **kwargs: (True, True, "2.9.1+cpu", "", ""))monkeypatch.setattr(mod, "run", lambda *args, **kwargs: None)- monkeypatch.setattr(mod, "pip_install", _record_pip_install)++ def _fatal_pip_install(*args, **kwargs):+ # pip_install exits the installer on failure.+ _record_pip_install(*args, **kwargs)+ if not torchao_install_ok and any(str(a).startswith("torchao") for a in args):+ raise SystemExit(1)+ return 0++ monkeypatch.setattr(mod, "pip_install", _fatal_pip_install)+ monkeypatch.setattr(+ mod,+ "pip_install_try",+ lambda *a, **k: (+ _record_pip_install(*a, **k),+ torchao_install_ok or not any(str(x).startswith("torchao") for x in a),+ )[1],+ )monkeypatch.setattr(mod, "_progress", lambda label: progress_labels.append(label))monkeypatch.setattr(mod, "LOCAL_DD_UNSTRUCTURED_PLUGIN", unstructured_plugin)monkeypatch.setattr(mod, "LOCAL_DD_GITHUB_PLUGIN", github_plugin)@@ -402,5 +425,325 @@ def test_skips_torchao_on_windows_rocm(assert mod.install_python_stack() == 0- assert not any(spec.startswith("torchao") for spec in installed_specs)- assert "dependency overrides (skipped, Windows ROCm)" in progress_labels+ torchao_calls = [c for c in pip_calls if any(a.startswith("torchao") for a in c)]+ assert torchao_calls+ assert all("--index-url" not in c for c in torchao_calls)+ assert "dependency overrides (Windows ROCm)" in progress_labels+++# Windows ROCm torchao export: real torchao via unsloth/_torchao_nodist.py, the stub as fallback.++import types++import core._torchao_stub as _stub++_BACKEND = Path(__file__).resolve().parents[1]+_EXPORT_HELPERS = ("_torchao_export_supported", "_torchao_runtime_unavailable", "_is_torchao_alias")+++def _export_helpers():+ """Exec the gate helpers alone, avoiding export.py's heavy import chain."""+ import ast++ src = (_BACKEND / "core" / "export" / "export.py").read_text(encoding = "utf-8")+ ns: dict = {}+ for node in ast.parse(src).body:+ if isinstance(node, ast.FunctionDef) and node.name in _EXPORT_HELPERS:+ exec(ast.get_source_segment(src, node), ns)+ return ns+++def _fake_normalize_torchao(save_method):+ if not isinstance(save_method, str):+ return None+ key = save_method.lower().strip().replace("-", "_").replace(" ", "_")+ ok = key in {"torchao_fp8", "torchao_int8", "portable_fp8", "portable_int8"}+ return ("fp8", "torchao-fp8") if ok else None+++def _install_fake_unsloth_save(monkeypatch, *, has_method):+ unsloth = types.ModuleType("unsloth")+ save = types.ModuleType("unsloth.save")+ if has_method:+ save._normalize_torchao_method = _fake_normalize_torchao+ unsloth.save = save+ monkeypatch.setitem(sys.modules, "unsloth", unsloth)+ monkeypatch.setitem(sys.modules, "unsloth.save", save)+++def _set_torchao(monkeypatch, state):+ if state == "absent":+ monkeypatch.delitem(sys.modules, "torchao", raising = False)+ elif state == "stub":+ monkeypatch.setitem(sys.modules, "torchao", _stub._make_mod_stub("torchao"))+ else:+ monkeypatch.setitem(sys.modules, "torchao", types.ModuleType("torchao"))+++ ("win_rocm", "torchao", "has_method", "expected"),+ [+ (True, "stub", True, False),+ (True, "absent", True, False),+ (True, "real", True, True),+ (False, "real", True, True),+ (False, "absent", True, True),+ (False, "real", False, False),+ ],+)+def test_torchao_export_gate(monkeypatch, win_rocm, torchao, has_method, expected):+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: win_rocm)+ _set_torchao(monkeypatch, torchao)+ _install_fake_unsloth_save(monkeypatch, has_method = has_method)+ ns = _export_helpers()+ ns["sys"] = sys+ assert ns["_torchao_export_supported"]() is expected+++def test_is_torchao_alias_recognizes_all_forms(monkeypatch):+ _install_fake_unsloth_save(monkeypatch, has_method = True)+ fn = _export_helpers()["_is_torchao_alias"]+ for alias in ("torchao_fp8", "portable_int8", "portable-fp8", "Portable FP8"):+ assert fn(alias) is True+ for alias in ("fp8", "nvfp4", "w8a8", "", None):+ assert fn(alias) is False+++def _load_export_module_no_torch(monkeypatch):+ """Import core.export.export with torch/unsloth blocked, so the guard runs without a GPU."""+ import builtins+ import importlib++ real_import = builtins.__import__++ def blocking_import(name, *args, **kwargs):+ top = name.split(".")[0]+ if top in {"torch", "unsloth"} and top not in sys.modules:+ raise ImportError(f"blocked: {name}")+ return real_import(name, *args, **kwargs)++ for m in [k for k in list(sys.modules) if k.split(".")[0] in {"torch", "unsloth"}]:+ monkeypatch.delitem(sys.modules, m, raising = False)+ monkeypatch.delitem(sys.modules, "core.export.export", raising = False)+ monkeypatch.setattr(builtins, "__import__", blocking_import)+ return importlib.import_module("core.export.export")+++def _bare_backend(mod):+ be = mod.ExportBackend.__new__(mod.ExportBackend)+ be.current_model = object()+ be.current_tokenizer = object()+ be._audio_type = None+ be.is_peft = True+ return be++[email protected]("alias", ["torchao_fp8", "portable_fp8"])+def test_torchao_export_rejected_early_when_stubbed(monkeypatch, alias):+ mod = _load_export_module_no_torch(monkeypatch)+ monkeypatch.setattr(mod, "_export_runtime_available", lambda: True)+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)+ _set_torchao(monkeypatch, "stub")+ _install_fake_unsloth_save(monkeypatch, has_method = True)++ ok, message, out = _bare_backend(mod).export_merged_model("/tmp/x", compressed_method = alias)+ assert ok is False and out is None+ assert "Windows ROCm" in message and "torchao" in message.lower()+++def test_torchao_export_not_rejected_with_real_torchao_on_windows_rocm(monkeypatch):+ mod = _load_export_module_no_torch(monkeypatch)+ monkeypatch.setattr(mod, "_export_runtime_available", lambda: True)+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)+ _set_torchao(monkeypatch, "real")+ _install_fake_unsloth_save(monkeypatch, has_method = True)++ ok, message, _ = _bare_backend(mod).export_merged_model(+ "/tmp/x", compressed_method = "torchao_int8"+ )+ # Past the guard: the placeholder model is what fails now.+ assert ok is False and "Windows ROCm" not in message+++ ("platform", "is_rocm", "loadable", "expected"),+ [+ ("win32", True, False, False),+ ("win32", True, True, True),+ ("win32", False, False, True),+ ("linux", True, False, True),+ ],+)+def test_export_capability_torchao_flag(monkeypatch, platform, is_rocm, loadable, expected):+ import utils.hardware.hardware as hw++ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)+ monkeypatch.setattr(sys, "platform", platform)+ monkeypatch.setattr(hw, "IS_ROCM", is_rocm)+ monkeypatch.setattr(_stub, "torchao_export_loadable", lambda: loadable)+ assert hw.export_capability()["torchao_export_supported"] is expected+++def test_torchao_export_loadable_ignores_the_stub(monkeypatch):+ # The main process has already stubbed torchao; only an installed torchao counts.+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)+ _set_torchao(monkeypatch, "stub")+ real = importlib.machinery.PathFinder.find_spec+ monkeypatch.setattr(+ importlib.machinery.PathFinder,+ "find_spec",+ classmethod(lambda cls, name, *a, **k: None if name == "torchao" else real(name, *a, **k)),+ )+ assert _stub.torchao_export_loadable() is False+++ ("installed", "expect"),+ [("0.14.0", False), ("0.15.0", True), ("0.17.0", True), ("0.18.0+rocm", True)],+)+def test_torchao_export_loadable_needs_transformers_minimum(+ monkeypatch, tmp_path, installed, expect+):+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)+ (tmp_path / "unsloth").mkdir()+ (tmp_path / "unsloth" / "_torchao_nodist.py").write_text("")+ specs = {+ "torchao": importlib.machinery.ModuleSpec("torchao", None),+ "unsloth": importlib.machinery.ModuleSpec("unsloth", None, is_package = True),+ }+ specs["unsloth"].submodule_search_locations = [str(tmp_path / "unsloth")]+ monkeypatch.setattr(+ importlib.machinery.PathFinder,+ "find_spec",+ classmethod(lambda cls, name, *a, **k: specs.get(name)),+ )+ monkeypatch.setattr(importlib.util, "find_spec", lambda name, *a, **k: specs.get(name))+ monkeypatch.setattr(importlib.metadata, "version", lambda name: installed)+ assert _stub.torchao_export_loadable() is expect++[email protected](("fix_result", "expect_real"), [(True, True), (False, False)])+def test_real_or_stub(monkeypatch, fix_result, expect_real):+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)+ monkeypatch.setattr(_stub, "torchao_export_loadable", lambda: True)+ monkeypatch.delitem(sys.modules, "torchao", raising = False)+ calls = []++ def fix():+ calls.append(1)+ if fix_result:+ sys.modules["torchao"] = types.ModuleType("torchao")+ return fix_result++ monkeypatch.setattr(+ _stub,+ "_load_torchao_nodist",+ lambda: types.SimpleNamespace(fix_torchao_without_torch_distributed = fix),+ )+ stubbed = []+ monkeypatch.setattr(_stub, "install_torchao_windows_rocm_stub", lambda: stubbed.append(1))+ try:+ assert _stub.install_torchao_windows_rocm_real_or_stub() is expect_real+ assert calls == [1]+ assert bool(stubbed) is (not expect_real)+ finally:+ sys.modules.pop("torchao", None)++[email protected]("consumer_loaded", [False, True])+def test_real_or_stub_replaces_a_stub_inherited_from_run_py(monkeypatch, consumer_loaded):+ """spawn re-runs run.py as __mp_main__, which stubs torchao before the export worker starts."""+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)+ monkeypatch.setattr(_stub, "torchao_export_loadable", lambda: True)+ # Simulated "already imported" consumer: json is always loaded; none of the real ones is required.+ monkeypatch.setattr(_stub, "_STUB_CONSUMERS", ("json",) if consumer_loaded else ())+ saved = {n: m for n, m in sys.modules.items() if n == "torchao" or n.startswith("torchao.")}+ for name in saved:+ monkeypatch.delitem(sys.modules, name)+ calls = []++ def fix():+ calls.append("torchao" in sys.modules)+ sys.modules["torchao"] = types.ModuleType("torchao")+ return True++ monkeypatch.setattr(+ _stub,+ "_load_torchao_nodist",+ lambda: types.SimpleNamespace(fix_torchao_without_torch_distributed = fix),+ )+ try:+ _stub.install_torchao_windows_rocm_stub()+ assert _stub.is_stubbed("torchao")+ result = _stub.install_torchao_windows_rocm_real_or_stub()+ if consumer_loaded:+ assert result is False and calls == [] and _stub.is_stubbed("torchao")+ else:+ assert result is True and calls == [False] and not _stub.is_stubbed("torchao")+ assert not any(n.startswith("torchao.") for n in sys.modules)+ finally:+ for name in [n for n in sys.modules if n == "torchao" or n.startswith("torchao.")]:+ del sys.modules[name]+ sys.modules.update(saved)+++def test_real_or_stub_keeps_the_stub_when_torchao_is_not_loadable(monkeypatch):+ """torch <= 2.9 pairs with torchao 0.14, which transformers 5 rejects: never load it."""+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: True)+ monkeypatch.setattr(_stub, "torchao_export_loadable", lambda: False)+ monkeypatch.delitem(sys.modules, "torchao", raising = False)+ monkeypatch.setattr(+ _stub, "_load_torchao_nodist", lambda: pytest.fail("must not load the shim")+ )+ stubbed = []+ monkeypatch.setattr(_stub, "install_torchao_windows_rocm_stub", lambda: stubbed.append(1))+ assert _stub.install_torchao_windows_rocm_real_or_stub() is False+ assert stubbed == [1]+++def test_shim_found_through_an_editable_finder(monkeypatch, tmp_path):+ """setuptools editable installs can expose unsloth only through a meta-path finder."""+ (tmp_path / "unsloth").mkdir()+ (tmp_path / "unsloth" / "_torchao_nodist.py").write_text(+ "def fix_torchao_without_torch_distributed():\n return 'ok'\n"+ )++ class EditableFinder:+ def find_spec(+ self,+ name,+ path = None,+ target = None,+ ):+ if name != "unsloth":+ return None+ spec = importlib.machinery.ModuleSpec("unsloth", None, is_package = True)+ spec.submodule_search_locations = [str(tmp_path / "unsloth")]+ return spec++ monkeypatch.delitem(sys.modules, "unsloth", raising = False)+ real = importlib.machinery.PathFinder.find_spec+ monkeypatch.setattr(+ importlib.machinery.PathFinder,+ "find_spec",+ classmethod(lambda cls, name, *a, **k: None if name == "unsloth" else real(name, *a, **k)),+ )+ monkeypatch.setattr(sys, "meta_path", [EditableFinder(), *sys.meta_path])+ module = _stub._load_torchao_nodist()+ assert module is not None and module.fix_torchao_without_torch_distributed() == "ok"+++def test_real_or_stub_noop_off_windows_rocm(monkeypatch):+ monkeypatch.setattr(_stub, "_is_windows_rocm", lambda: False)+ monkeypatch.setattr(+ _stub, "_load_torchao_nodist", lambda: pytest.fail("must not load the shim")+ )+ assert _stub.install_torchao_windows_rocm_real_or_stub() is False+++def test_load_torchao_nodist_reads_unsloths_file():+ module = _stub._load_torchao_nodist()+ if importlib.util.find_spec("unsloth") is None:+ assert module is None+ else:+ assert callable(module.fix_torchao_without_torch_distributed)diff --git a/studio/backend/tests/test_torchao_stub_worker_parity.py b/studio/backend/tests/test_torchao_stub_worker_parity.pyindex bb743385f1f..8f2f0b3f86b 100644--- a/studio/backend/tests/test_torchao_stub_worker_parity.py+++ b/studio/backend/tests/test_torchao_stub_worker_parity.py@@ -18,11 +18,16 @@ from __future__ import annotationsimport astfrom pathlib import Path-from core._torchao_stub import install_torchao_windows_rocm_stub+from core._torchao_stub import (+ install_torchao_windows_rocm_real_or_stub,+ install_torchao_windows_rocm_stub,+)_BACKEND = Path(__file__).resolve().parent.parent # studio/backend_CORE = _BACKEND / "core"_STUB = install_torchao_windows_rocm_stub.__name__ # a rename breaks the import loudly+# The export worker loads real torchao when it can, falling back to the same stub.+_REAL_OR_STUB = install_torchao_windows_rocm_real_or_stub.__name___ENTRYPOINTS = [_CORE / "training" / "worker.py",@@ -37,7 +42,9 @@ def _stub_call_linenos(node) -> list[int]:return [c.linenofor c in ast.walk(node)- if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == _STUB+ if isinstance(c, ast.Call)+ and isinstance(c.func, ast.Name)+ and c.func.id in (_STUB, _REAL_OR_STUB)]diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 4383d301ce0..685299c26e3 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -1719,12 +1719,22 @@ def _gpu_present_but_unusable_message(def export_capability() -> dict:- """Whether model export can run here, with a torch-aware reason when it cannot. Export runs through Unsloth, which hard-requires an accelerator (it calls ``torch.cuda`` at import and has no CPU path), so it is supported iff ``get_device() in {CUDA, XPU, MLX}``. The reason distinguishes a --no-torch install from a bare-CPU host. Safe to call without torch. Returns {export_supported, export_unsupported_reason, export_unsupported_message}."""- if get_device() in (DeviceType.CUDA, DeviceType.XPU, DeviceType.MLX):+ """Whether model export can run here, with a torch-aware reason when it cannot. Export runs through Unsloth, which hard-requires an accelerator (it calls ``torch.cuda`` at import and has no CPU path), so it is supported iff ``get_device() in {CUDA, XPU, MLX}``. The reason distinguishes a --no-torch install from a bare-CPU host. Safe to call without torch. Returns {export_supported, export_unsupported_reason, export_unsupported_message, torchao_export_supported}; the last is False only on Windows ROCm without a loadable torchao, where the portable FP8/INT8 formats are hidden."""+ device = get_device()+ torchao_export_supported = True+ # get_device() ran detect_hardware(), so IS_ROCM (hip field OR "rocm" tag) is settled.+ if sys.platform == "win32" and IS_ROCM:+ try:+ from core._torchao_stub import torchao_export_loadable+ torchao_export_supported = torchao_export_loadable()+ except Exception:+ torchao_export_supported = False+ if device in (DeviceType.CUDA, DeviceType.XPU, DeviceType.MLX):return {"export_supported": True,"export_unsupported_reason": None,"export_unsupported_message": None,+ "torchao_export_supported": torchao_export_supported,}verdict = current_chat_only_verdict()# No accelerator: name the blocker. Detection failure first, since the branches below all describe a measured host, so a broken probe would tell a GPU box to install PyTorch.@@ -1767,6 +1777,7 @@ def export_capability() -> dict:"export_supported": False,"export_unsupported_reason": reason,"export_unsupported_message": message,+ "torchao_export_supported": torchao_export_supported,}diff --git a/studio/frontend/src/features/export/export-page.tsx b/studio/frontend/src/features/export/export-page.tsxindex 2118298deda..96881ec8a4a 100644--- a/studio/frontend/src/features/export/export-page.tsx+++ b/studio/frontend/src/features/export/export-page.tsx@@ -232,6 +232,7 @@ export function ExportPage() {const hardware = useHardwareInfo();// GGUF LoRA conversion is rejected on the macOS / MLX path, so gate it out on a Mac host.const isMacHost = usePlatformStore((s) => s.deviceType) === "mac";+ const torchaoUnavailable = !hardware.torchaoExportSupported;// Real CUDA (not ROCm); gates the NVIDIA-only compressed-tensors formats.const hasNvidia = hardware.cuda != null && hardware.rocm == null;// Only gray out on an authoritative unsupported response; the backend supplies the reason.@@ -246,19 +247,27 @@ export function ExportPage() {// compressed-tensors (llm-compressor) is the NVIDIA path; shown only on an NVIDIA GPU.if (f.backend === "compressed") return hasNvidia;// Portable torchao is the fallback for hosts without the NVIDIA compressed path. Hidden on- // NVIDIA (use compressed-tensors) and on macOS/MLX (the backend rejects quantized export).- if (f.backend === "torchao") return !hasNvidia && !isMacHost;+ // NVIDIA (use compressed-tensors), macOS/MLX (rejected) and where torchao cannot load.+ if (f.backend === "torchao") return !hasNvidia && !isMacHost && !torchaoUnavailable;// Plain 16-bit is available everywhere.return true;}),- [hasNvidia, isMacHost],+ [hasNvidia, isMacHost, torchaoUnavailable],);const toggleFormat = useCallback((value: string) => {setSelectedFormats((prev) =>prev.includes(value) ? prev.filter((v) => v !== value) : [...prev, value],);}, []);- // availableFormats already drops NVIDIA-only formats on other hardware, so no pruning needed.+ // Drop picks the gate removed, only once hardware is loaded: before that hasNvidia reads false.+ useEffect(() => {+ if (!hardware.loaded) return;+ const allowed = new Set(availableFormats.map((f) => f.value));+ setSelectedFormats((prev) => {+ const next = prev.filter((v) => allowed.has(v));+ return next.length === prev.length ? prev : next;+ });+ }, [availableFormats, hardware.loaded]);// IQ quants are imatrix-only: force imatrix on when one is selected, else llama.cpp rejects it.const requiresImatrix = quantLevels.some((q) => QUANT_OPTIONS.find((o) => o.value === q)?.imatrix,@@ -1523,13 +1532,21 @@ export function ExportPage() {</div>)}- {!hasNvidia && (+ {!hasNvidia && !torchaoUnavailable && (<div className="text-ui-11 text-muted-foreground">No NVIDIA GPU detected: compressed-tensors formats arehidden. 16-bit and portable FP8/INT8 (torchao) stillwork here and load in vLLM.</div>)}++ {torchaoUnavailable && (+ <div className="text-ui-11 text-muted-foreground">+ Portable FP8/INT8 (torchao) export needs torchao, which+ is not installed here. Update Unsloth, or use+ 16-bit or GGUF.+ </div>+ )}</div></div>)}diff --git a/studio/frontend/src/hooks/use-hardware-info.ts b/studio/frontend/src/hooks/use-hardware-info.tsindex 36f42f94054..71e5f47a419 100644--- a/studio/frontend/src/hooks/use-hardware-info.ts+++ b/studio/frontend/src/hooks/use-hardware-info.ts@@ -34,6 +34,8 @@ export interface HardwareInfo {exportSupported: boolean | null;exportUnsupportedReason: string | null;exportUnsupportedMessage: string | null;+ // False only on Windows ROCm without a loadable torchao; absent from older backends = true.+ torchaoExportSupported: boolean;// Whether video generation can run here. Same tri-state as export: `null` until the// authoritative response lands, and `null` too against a backend that predates the field,// so only an explicit `false` hides the generator.@@ -58,6 +60,7 @@ const DEFAULT: HardwareInfo = {exportSupported: null,exportUnsupportedReason: null,exportUnsupportedMessage: null,+ torchaoExportSupported: true,videoSupported: null,videoUnsupportedReason: null,videoUnsupportedMessage: null,@@ -118,6 +121,7 @@ async function fetchOnce(): Promise<HardwareInfo> {exportSupported: data?.export_supported ?? null,exportUnsupportedReason: data?.export_unsupported_reason ?? null,exportUnsupportedMessage: data?.export_unsupported_message ?? null,+ torchaoExportSupported: data?.torchao_export_supported ?? true,videoSupported: data?.video_supported ?? null,videoUnsupportedReason: data?.video_unsupported_reason ?? null,videoUnsupportedMessage: data?.video_unsupported_message ?? null,diff --git a/studio/install_python_stack.py b/studio/install_python_stack.pyindex 581a1b1cd83..664d9f81d7b 100644--- a/studio/install_python_stack.py+++ b/studio/install_python_stack.py@@ -1026,8 +1026,7 @@ def _installed_torch_is_windows_rocm() -> bool:This is a belt-and-suspenders guard for the torchao override step: if theearlier ROCm install path failed to set _rocm_windows_torch_installed but the- venv already contains a ROCm torch wheel, still skip torchao because it- crashes on import on Windows ROCm.+ venv already contains a ROCm torch wheel, torchao still comes from PyPI."""if not IS_WINDOWS:return False@@ -5153,8 +5152,9 @@ def _resident_xformers_build_torch() -> "str | None":return recorded.strip() if isinstance(recorded, str) and recorded.strip() else None-def _install_torchao_for_torch(torch_version: "str | None") -> None:- """Select the torchao matching torch_version and install it from its own index.+def _install_torchao_for_torch(torch_version: "str | None", default_index: bool = False) -> None:+ """Select the torchao matching torch_version and install it from its own index (PyPI's with+ default_index: download.pytorch.org's rocm leaves serve Linux only, so not Windows ROCm).Called twice: as step 4, and again after the Linux torch repair, which can move torchacross families and releases underneath the first call.@@ -5162,7 +5162,7 @@ def _install_torchao_for_torch(torch_version: "str | None") -> None:spec = _select_torchao_spec(torch_version)# See _TORCHAO_DEFAULT_SPEC. rocm is included here, unlike torchcodec: the rocm leaves# really do publish torchao.- index = _torch_accelerator_index_url(torch_version)+ index = None if default_index else _torch_accelerator_index_url(torch_version)# --no-deps skips nothing today (no torchao release declares a runtime torch dependency)# and guards the second caller, which runs right after the torch repair.args = ["--no-deps", "--no-cache-dir"]@@ -5183,6 +5183,11 @@ def _install_torchao_for_torch(torch_version: "str | None") -> None:# Redacted for display only; the installer below still gets the exact URL.+ (f" from {_strip_index_url_credentials(index)}" if index else ""))+ if default_index:+ # Optional on Windows ROCm: only export uses it, and hides its formats without it.+ if not pip_install_try("Installing dependency overrides", *args, spec):+ _note(f"could not install {spec}; torchao export stays unavailable")+ returnif not index:pip_install("Installing dependency overrides", *args, spec)return@@ -11755,13 +11760,13 @@ def install_python_stack() -> int:# 4. Install the torch-matched torchao override. Reinstall only when the pin# changes, since Windows can remove shared files during replacement.- # Skip when torch is unavailable or Windows ROCm has no working build.+ # Skip when torch is unavailable.if NO_TORCH:_progress("dependency overrides (skipped, no torch)")elif _rocm_windows_torch_installed or _installed_torch_is_windows_rocm():- # No working Windows ROCm torchao build (crashes on import; stubbed at runtime).- _progress("dependency overrides (skipped, Windows ROCm)")- _note("Windows ROCm -- skipping torchao (no working build; stubbed at runtime)")+ # Stock torchao dies on import here; only the export worker loads it (unsloth/_torchao_nodist.py).+ _progress("dependency overrides (Windows ROCm)")+ _install_torchao_for_torch(_probe_installed_torch_version(), default_index = True)else:_progress("dependency overrides")_install_torchao_for_torch(_probe_installed_torch_version())diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.pyindex 02591746258..1a19895278b 100644--- a/tests/studio/install/test_rocm_support.py+++ b/tests/studio/install/test_rocm_support.py@@ -5852,7 +5852,7 @@ class TestWindowsRocmTorchaoGuard:@patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = True)@patch.object(stack_mod, "run")@patch.object(stack_mod, "pip_install")- def test_install_python_stack_skips_torchao_when_windows_rocm_torch_is_installed(+ def test_install_python_stack_installs_pypi_torchao_when_windows_rocm_torch_is_installed(self,mock_pip,mock_run,@@ -5887,11 +5887,18 @@ class TestWindowsRocmTorchaoGuard:patch.object(stack_mod.install_manifest, "installed_versions", return_value = ["0"]),patch.object(stack_mod.install_manifest, "damaged_payload_files", return_value = []),patch.object(stack_mod.subprocess, "run", return_value = subprocess_result),+ patch.object(stack_mod, "pip_install_try", return_value = True) as mock_try,):assert stack_mod.install_python_stack() == 0- installed_specs = [str(arg) for call in mock_pip.call_args_list for arg in call.args]- assert not any("torchao" in arg for arg in installed_specs)+ torchao_calls = [+ [str(arg) for arg in call.args]+ for call in mock_pip.call_args_list + mock_try.call_args_list+ if any(str(arg).startswith("torchao") for arg in call.args)+ ]+ # From PyPI: download.pytorch.org's rocm leaves serve Linux only.+ assert torchao_calls+ assert all("--index-url" not in c for c in torchao_calls)class TestProgressStepCountMatchesTotal:@@ -6116,9 +6123,9 @@ class TestWorkerWindowsRocmPatches:assert "install_torchao_windows_rocm_stub()" in sourcedef test_export_worker_calls_shared_torchao_stub(self):- """export/worker.py must invoke the same shared torchao stub entrypoint."""+ """export/worker.py loads real torchao when it can and otherwise the same shared stub."""source = _EXPORT_WORKER_PATH.read_text(encoding = "utf-8")- assert "install_torchao_windows_rocm_stub()" in source+ assert "install_torchao_windows_rocm_real_or_stub()" in sourcedef test_embedder_calls_shared_torchao_stub(self):"""embeddings.py must install the stub before importing sentence-transformers:diff --git a/tests/test_torchao_nodist.py b/tests/test_torchao_nodist.pynew file mode 100644index 00000000000..c6046040800--- /dev/null+++ b/tests/test_torchao_nodist.py@@ -0,0 +1,161 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""unsloth/_torchao_nodist.py on a simulated torch without torch.distributed (AMD's Windows ROCm+wheels): is_available() False, every torch.distributed.* submodule unimportable, and the c10d ops+absent, as in pytorch/ao#4761's test plus the missing ops. Out of process so the parent's module+cache is untouched. Verified on real hardware separately (Windows 11, gfx1151, torch 2.11 ROCm)."""++import importlib.util+import json+import subprocess+import sys+from pathlib import Path++import pytest++_SHIM = Path(__file__).resolve().parents[1] / "unsloth" / "_torchao_nodist.py"++_SIMULATE = r"""+import importlib.util, json, sys+import torch++torch.distributed.is_available = lambda: False+for name in [n for n in sys.modules if n.startswith("torch.distributed.")]:+ del sys.modules[name]+for attr, value in list(vars(torch.distributed).items()):+ if isinstance(value, type(sys)) and value.__name__.startswith("torch.distributed."):+ delattr(torch.distributed, attr)+++class BlockDistributed:+ def find_spec(self, fullname, path = None, target = None):+ if fullname.startswith("torch.distributed."):+ raise ModuleNotFoundError(f"No module named 'torch._C._distributed_c10d' ({fullname})", name = fullname)+ return None+++sys.meta_path.insert(0, BlockDistributed())+_HIDDEN = {"c10d_functional", "_c10d_functional", "c10d", "_dtensor"}+_ns = torch._ops._OpNamespace+_real = _ns.__getattr__+++def _hiding(self, op):+ if self.name in _HIDDEN and not op.startswith("__"):+ raise AttributeError(f"'_OpNamespace' '{self.name}' object has no attribute '{op}'")+ return _real(self, op)+++_ns.__getattr__ = _hiding+for ns in _HIDDEN:+ obj = torch.ops.__dict__.get(ns)+ for attr in list(vars(obj)) if obj is not None else []:+ if not attr.startswith("__"):+ try:+ delattr(obj, attr)+ except Exception:+ pass++r = {}+if sys.argv[2] == "fix":+ spec = importlib.util.spec_from_file_location("_nodist", sys.argv[1])+ shim = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(shim)+ r["fixed"] = shim.fix_torchao_without_torch_distributed()+for stmt in ("import torchao", "from torchao.quantization import Int8WeightOnlyConfig, Float8WeightOnlyConfig",+ "import transformers.quantizers.quantizer_torchao", "import peft.tuners.lora.torchao",+ "import torch.distributed.tensor"):+ try:+ exec(stmt)+ r[stmt] = "ok"+ except BaseException as e:+ r[stmt] = f"{type(e).__name__}: {e}"[:200]+held = []+for name, module in list(sys.modules.items()):+ if module is None or name == "torchao" or name.startswith(("torchao.", "_nodist")):+ continue+ try:+ values = list(vars(module).values())+ except Exception:+ continue+ for value in values:+ try:+ if isinstance(value, type(sys)) and value.__dict__.get("__unsloth_nodist_stub__"):+ held.append(name)+ except Exception:+ pass+r["stand_ins_held_outside_torchao"] = sorted(set(held))+print("RESULT=" + json.dumps(r))+"""++pytestmark = pytest.mark.skipif(+ importlib.util.find_spec("torchao") is None or importlib.util.find_spec("transformers") is None,+ reason = "needs torchao and transformers installed",+)+++def _run(mode):+ out = subprocess.run(+ [sys.executable, "-c", _SIMULATE, str(_SHIM), mode],+ capture_output = True,+ text = True,+ timeout = 600,+ )+ lines = [l for l in out.stdout.splitlines() if l.startswith("RESULT=")]+ assert lines, out.stderr[-3000:]+ return json.loads(lines[-1][len("RESULT=") :])+++def test_torchao_fails_without_the_fix():+ r = _run("none")+ assert r["import torchao"] != "ok"+++def test_torchao_and_its_consumers_import_with_the_fix():+ r = _run("fix")+ assert r["fixed"] is True+ assert r["import torchao"] == "ok"+ assert (+ r["from torchao.quantization import Int8WeightOnlyConfig, Float8WeightOnlyConfig"] == "ok"+ )+ # transformers imports torchao.prototype.* lazily, long after `import torchao` returned.+ assert r["import transformers.quantizers.quantizer_torchao"] == "ok"+ assert r["import peft.tuners.lora.torchao"] == "ok"+++def test_nothing_outside_torchao_sees_a_fake_torch_distributed():+ r = _run("fix")+ assert r["import torch.distributed.tensor"].startswith("ModuleNotFoundError")+ assert r["stand_ins_held_outside_torchao"] == []+++def test_noop_where_torch_distributed_exists():+ code = (+ "import importlib.util, sys, torch\n"+ "spec = importlib.util.spec_from_file_location('_nodist', sys.argv[1])\n"+ "m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)\n"+ "print('RESULT', torch.distributed.is_available(), m.fix_torchao_without_torch_distributed(), "+ "any(type(f).__name__ == '_TorchaoWindowFinder' for f in sys.meta_path))\n"+ )+ out = subprocess.run(+ [sys.executable, "-c", code, str(_SHIM)], capture_output = True, text = True, timeout = 600+ )+ line = next(l for l in out.stdout.splitlines() if l.startswith("RESULT"))+ available, fixed, hooked = line.split()[1:]+ if available != "True":+ pytest.skip("this torch has no torch.distributed")+ assert (fixed, hooked) == ("False", "False")+++def test_gpu_init_runs_the_fix_before_anything_imports_torchao():+ """The torch symbol-skew fix must come first (torchao 0.18 on torch < 2.10), and the shim+ must precede vllm's probe and unsloth_zoo, both of which import transformers."""+ src = (_SHIM.parent / "_gpu_init.py").read_text(encoding = "utf-8")+ order = [+ src.index("\nfix_torchao_torch_symbol_skew()"),+ src.index("\nfix_torchao_without_torch_distributed()"),+ src.index("\ndisable_broken_vllm()"),+ src.index("\n import unsloth_zoo\n"),+ ]+ assert order == sorted(order)diff --git a/tests/test_torchao_subprocess_fix.py b/tests/test_torchao_subprocess_fix.pyindex 23f4c5a21c5..ed5b30cb981 100644--- a/tests/test_torchao_subprocess_fix.py+++ b/tests/test_torchao_subprocess_fix.py@@ -505,7 +505,7 @@ if __name__ == "__main__":def test_the_in_process_fix_does_not_disable_the_subprocess_fix(monkeypatch, tmp_path):- """_gpu_init.py runs fix_torchao_torch_symbol_skew() immediately before+ """_gpu_init.py runs fix_torchao_torch_symbol_skew() beforethis one, so a gate asking only `hasattr` would read its placeholders as ahealthy torch and stage nothing, in exactly the environments vLLM'sinspector child needs it."""diff --git a/tests/test_xformers_stub_without_metadata.py b/tests/test_xformers_stub_without_metadata.pynew file mode 100644index 00000000000..77d1f029448--- /dev/null+++ b/tests/test_xformers_stub_without_metadata.py@@ -0,0 +1,24 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Studio's Windows ROCm run.py puts an xformers stub in sys.modules without installing xformers;+a spawned worker re-runs run.py, so `import unsloth` meets that stub."""++import importlib.machinery+import importlib.metadata+import sys+import types++from unsloth import import_fixes+++def test_xformers_stub_without_metadata_is_skipped(monkeypatch):+ stub = types.ModuleType("xformers")+ stub.__spec__ = importlib.machinery.ModuleSpec("xformers", loader = None, is_package = True)+ monkeypatch.setitem(sys.modules, "xformers", stub)++ def missing(name):+ raise importlib.metadata.PackageNotFoundError(name)++ monkeypatch.setattr(import_fixes, "importlib_version", missing)+ assert import_fixes.fix_xformers_performance_issue() is Nonediff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex ba918a498a5..15cc1180a73 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -82,11 +82,17 @@ configure_amdgpu_asic_id_table_path()patch_torch_missing_attribute_error()# Must precede `import unsloth_zoo` below, which imports bnb on ROCm.fix_bitsandbytes_rocm_arch_detection()+# Torch-only, so it can run first; torchao 0.18 on torch < 2.10 needs it before any torchao import.+fix_torchao_torch_symbol_skew()+# Before unsloth_zoo and transformers (and vllm, below): real torchao on a torch without torch.distributed.+from ._torchao_nodist import fix_torchao_without_torch_distributed++fix_torchao_without_torch_distributed()+del fix_torchao_without_torch_distributeddisable_broken_causal_conv1d()disable_broken_vllm()fix_message_factory_issue()fix_torch_check_is_size()-fix_torchao_torch_symbol_skew()# The above fixes THIS process only; vLLM's model-architecture inspector is a subprocess that# imports torchao itself and hits the same ImportError.propagate_torchao_fix_to_subprocesses()diff --git a/unsloth/_torchao_nodist.py b/unsloth/_torchao_nodist.pynew file mode 100644index 00000000000..681132a67f9--- /dev/null+++ b/unsloth/_torchao_nodist.py@@ -0,0 +1,257 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Import real torchao on a torch built without torch.distributed (AMD's Windows ROCm wheels).++torchao imports DTensor / DeviceMesh / the functional collectives at module scope and keys+dispatch tables on c10d ops, none of which exist on such a build, so ``import torchao`` dies+(pytorch/ao#3452; the upstream guard, pytorch/ao#4761, is unmerged). All of it only serves+isinstance() checks and collective handlers that cannot run without distributed. So each torchao+module body executes inside a window where imports made BY torchao code get inert stand-ins for+missing torch.distributed modules and lookups made BY torchao code get inert keys for missing+c10d ops. Everything else, torch internals imported meanwhile included, sees the real (absent)+torch.distributed. Stdlib + torch only: Studio loads this file without importing unsloth.+"""++import importlib+import importlib.abc+import importlib.machinery+import importlib.util+import sys+import threading+import types++__all__ = ["fix_torchao_without_torch_distributed"]++_OP_NAMESPACES = frozenset({"c10d_functional", "_c10d_functional", "c10d", "_dtensor"})+_IMPORTING = [0]+++def _is_torchao(name):+ return isinstance(name, str) and (name == "torchao" or name.startswith("torchao."))+++def _importer():+ frame = sys._getframe(1)+ while frame is not None:+ name = frame.f_globals.get("__name__", "")+ if not name.startswith(("importlib", "_frozen_importlib")) and name != __name__:+ return name+ frame = frame.f_back+ return None+++def _passthrough(fn):+ return fn+++def _unavailable(qualname):+ def fn(*args, **kwargs):+ # torchao applies distributed decorators (register_sharding(op), ...) while loading;+ # there they wrap nothing. Any later call is real distributed work.+ if _IMPORTING[0]:+ if len(args) == 1 and not kwargs and isinstance(args[0], types.FunctionType):+ return args[0]+ return _passthrough+ raise RuntimeError(f"{qualname} needs torch.distributed, which this torch build lacks.")++ return fn+++class _NeverMeta(type):+ def __instancecheck__(cls, obj):+ return False++ def __subclasscheck__(cls, sub):+ return False++ def __call__(cls, *args, **kwargs):+ raise RuntimeError(f"{cls.__name__} needs torch.distributed, which this torch build lacks.")+++class _StubLoader(importlib.abc.Loader):+ def create_module(self, spec):+ return types.ModuleType(spec.name)++ def exec_module(self, module):+ name = module.__name__+ cache = {}++ def __getattr__(attr):+ if attr.startswith("__"):+ raise AttributeError(attr)+ if attr not in cache:+ cache[attr] = (+ _NeverMeta(attr, (), {"__module__": name})+ if attr[:1].isupper()+ else _unavailable(f"{name}.{attr}")+ )+ return cache[attr]++ module.__getattr__ = __getattr__+ module.__path__ = []+ module.__unsloth_nodist_stub__ = True+++class _DistStubFinder(importlib.abc.MetaPathFinder):+ def find_spec(+ self,+ fullname,+ path = None,+ target = None,+ ):+ if not fullname.startswith("torch.distributed.") or fullname in sys.modules:+ return None+ if not _is_torchao(_importer()):+ return None+ return importlib.machinery.ModuleSpec(fullname, _StubLoader(), is_package = True)+++class _InertPacket:+ """A missing c10d OpOverloadPacket: each overload is a unique key no real op equals."""++ def __init__(self, qualname):+ self._qualname = qualname+ self._overloads = {}++ def __getattr__(self, overload):+ if overload.startswith("__"):+ raise AttributeError(overload)+ return self._overloads.setdefault(overload, _unavailable(f"{self._qualname}.{overload}"))++ def __call__(self, *args, **kwargs):+ raise RuntimeError(+ f"{self._qualname} needs torch.distributed, which this torch build lacks."+ )+++class _Window:+ def __init__(self, torch):+ self.torch = torch+ self.lock = threading.RLock()+ self.depth = 0+ self.snapshots = []+ self.packets = {}+ self.dist_finder = _DistStubFinder()++ def __enter__(self):+ self.lock.acquire()+ self.depth += 1+ _IMPORTING[0] = self.depth+ self.snapshots.append(set(sys.modules))+ if self.depth == 1:+ sys.meta_path.insert(0, self.dist_finder)+ ns_cls = self.torch._ops._OpNamespace+ self.original_getattr = original = ns_cls.__getattr__+ packets = self.packets++ def __getattr__(ns, op_name):+ try:+ return original(ns, op_name)+ except AttributeError:+ if (+ ns.name not in _OP_NAMESPACES+ or op_name.startswith("__")+ or not _is_torchao(sys._getframe(1).f_globals.get("__name__"))+ ):+ raise+ key = f"{ns.name}.{op_name}"+ return packets.setdefault(key, _InertPacket(key))++ ns_cls.__getattr__ = __getattr__+ return self++ def __exit__(self, *exc):+ try:+ self.depth -= 1+ _IMPORTING[0] = self.depth+ # This module keeps its references; nobody else may find the stand-ins.+ for name in set(sys.modules) - self.snapshots.pop():+ module = sys.modules.get(name)+ if getattr(module, "__unsloth_nodist_stub__", False):+ del sys.modules[name]+ parent_name, _, attr = name.rpartition(".")+ parent = sys.modules.get(parent_name)+ if parent is not None and getattr(parent, attr, None) is module:+ delattr(parent, attr)+ if self.depth == 0:+ self.torch._ops._OpNamespace.__getattr__ = self.original_getattr+ sys.meta_path.remove(self.dist_finder)+ finally:+ self.lock.release()+ return False+++class _WindowedLoader(importlib.abc.Loader):+ def __init__(self, loader, window):+ self.loader = loader+ self.window = window++ def create_module(self, spec):+ return self.loader.create_module(spec)++ def exec_module(self, module):+ with self.window:+ self.loader.exec_module(module)++ def __getattr__(self, name):+ return getattr(self.loader, name)+++class _TorchaoWindowFinder(importlib.abc.MetaPathFinder):+ """Every torchao module, including ones transformers imports lazily later+ (quantizer_torchao -> torchao.prototype.mx_formats), loads inside the window."""++ def __init__(self, window):+ self.window = window++ def find_spec(+ self,+ fullname,+ path = None,+ target = None,+ ):+ if not _is_torchao(fullname):+ return None+ for finder in sys.meta_path:+ if finder is self or not hasattr(finder, "find_spec"):+ continue+ spec = finder.find_spec(fullname, path, target)+ if spec is not None:+ break+ else:+ return None+ if spec.loader is not None and not isinstance(spec.loader, _WindowedLoader):+ spec.loader = _WindowedLoader(spec.loader, self.window)+ return spec+++def fix_torchao_without_torch_distributed():+ """True once real torchao is imported through the window. False, leaving nothing behind,+ where torch.distributed exists, torchao is absent or already imported (e.g. Studio's+ stub), or torchao still fails to import."""+ if any(type(f).__name__ == "_TorchaoWindowFinder" for f in sys.meta_path):+ return "torchao" in sys.modules+ if "torchao" in sys.modules:+ return False+ try:+ if importlib.util.find_spec("torchao") is None:+ return False+ import torch+ if torch.distributed.is_available():+ return False+ except Exception:+ return False++ finder = _TorchaoWindowFinder(_Window(torch))+ sys.meta_path.insert(0, finder)+ before = set(sys.modules)+ try:+ importlib.import_module("torchao")+ return True+ except Exception:+ sys.meta_path.remove(finder)+ for name in set(sys.modules) - before:+ if _is_torchao(name):+ sys.modules.pop(name, None)+ return Falsediff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 6ca081fab70..a0467ad8c58 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -333,7 +333,11 @@ def fix_xformers_performance_issue():spec = importlib.util.find_spec("xformers")if spec is None:return- xformers_version = importlib_version("xformers")+ try:+ xformers_version = importlib_version("xformers")+ except Exception:+ # Studio's Windows ROCm xformers stub: in sys.modules, not installed.+ returnif Version(xformers_version) < Version("0.0.29"):xformers_location = spec.originif xformers_location is None:diff --git a/unsloth/save.py b/unsloth/save.pyindex 04adfe65de5..9b147872e21 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -6719,7 +6719,11 @@ def _unsloth_save_torchao(quant_type = Float8WeightOnlyConfig()safe_serialization = Trueelif kind == "int8":- quant_type = Int8WeightOnlyConfig()+ # version 2 (Int8Tensor) is what transformers serializes; torchao 0.16 / 0.17 default to 1.+ _int8_fields = getattr(Int8WeightOnlyConfig, "__dataclass_fields__", {})+ quant_type = (+ Int8WeightOnlyConfig(version = 2) if "version" in _int8_fields else Int8WeightOnlyConfig()+ )safe_serialization = False # torchao only supports safetensors for float8 configselse:raise RuntimeError(f"Unsloth: unknown torchao export kind '{kind}' (expected fp8/int8).")