Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Intel XPU: 4bit loads without a C compiler by falling back to bitsandbytes' PyTorch quantize (#13309)

Daniel Han956430af9fbb
diff --git a/tests/test_bnb_triton_quantize_fallback.py b/tests/test_bnb_triton_quantize_fallback.py
new file mode 100644
index 00000000000..66a1778c7e0
--- /dev/null
+++ b/tests/test_bnb_triton_quantize_fallback.py
@@ -0,0 +1,184 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""#10844: bitsandbytes' Triton quantize ops with no host C compiler. One subprocess per case:
+kernel registrations are process-wide."""
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+
+pytest.importorskip("bitsandbytes")
+pytest.importorskip("triton")
+torch = pytest.importorskip("torch")
+
+_FIXES = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "unsloth", "import_fixes.py"
+)
+
+# Stand-ins raise from a triton/runtime/build.py frame, as Triton's launcher build does.
+_CPU_RUNNER = textwrap.dedent(
+ """
+ import importlib.util, json, sys
+ import torch, bitsandbytes as bnb, bitsandbytes.functional as F
+ if torch.accelerator.current_accelerator() is None:
+ # triton/ops.py reads the accelerator's type at import; the stand-ins below never use it.
+ torch.accelerator.current_accelerator = lambda *a, **k: torch.device("cpu")
+ from bitsandbytes.backends.triton import ops as triton_ops
+ behaviour, patch = sys.argv[1], sys.argv[2] == "1"
+
+ build = {}
+ exec(compile(
+ "def _build():\\n raise RuntimeError('Failed to find C compiler. Please specify via CC environment variable')\\n",
+ "/site-packages/triton/runtime/build.py", "exec"), build)
+ calls = []
+ default_keys = torch._C.DispatchKeySet(torch._C.DispatchKey.CompositeExplicitAutograd)
+
+ def stand_in(name):
+ op = getattr(torch.ops.bitsandbytes, name).default
+ def kernel(*args):
+ calls.append(name)
+ if behaviour == "no_compiler":
+ build["_build"]()
+ if behaviour == "bad_input":
+ raise ValueError("not a build failure")
+ return op.redispatch(default_keys, *args)
+ return kernel
+
+ lib = torch.library.Library("bitsandbytes", "FRAGMENT")
+ for name in ("quantize_4bit", "quantize_blockwise"):
+ setattr(triton_ops, name, stand_in(name))
+ lib.impl(name, getattr(triton_ops, name), "CPU", allow_override = True)
+ if patch:
+ spec = importlib.util.spec_from_file_location("fixes_under_test", sys.argv[3])
+ fixes = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(fixes)
+ fixes.patch_bitsandbytes_triton_quantize_fallback("cpu")
+ state = triton_ops._unsloth_quantize_fallback
+ fixes.patch_bitsandbytes_triton_quantize_fallback("cpu")
+ assert triton_ops._unsloth_quantize_fallback is state
+
+ torch.manual_seed(3407)
+ W = torch.randn(256, 192, dtype = torch.bfloat16)
+ out = {"calls": calls}
+ try:
+ q, qs = F.quantize_4bit(W, quant_type = "nf4", compress_statistics = True)
+ F.quantize_4bit(W, quant_type = "nf4", compress_statistics = True)
+ dq = F.dequantize_4bit(q, qs)
+ out.update(ok = True, rel_err = float((dq.float() - W.float()).norm() / W.float().norm()))
+ except Exception as e:
+ out.update(ok = False, error = f"{type(e).__name__}: {e}")
+ state = getattr(triton_ops, "_unsloth_quantize_fallback", None)
+ out["fallback_error"] = state and state["error"]
+ print("RESULT " + json.dumps(out))
+ """
+)
+
+
+def _run(argv, env = None):
+ proc = subprocess.run(
+ [sys.executable, "-c", *argv], capture_output = True, text = True, timeout = 600, env = env
+ )
+ lines = [l for l in proc.stdout.splitlines() if l.startswith("RESULT ")]
+ assert lines, proc.stdout[-2000:] + proc.stderr[-4000:]
+ return json.loads(lines[-1][len("RESULT ") :]), proc.stdout
+
+
+def _cpu(behaviour, patch):
+ return _run([_CPU_RUNNER, behaviour, "1" if patch else "0", _FIXES])
+
+
+def test_no_compiler_fails_without_the_fallback():
+ result, _ = _cpu("no_compiler", patch = False)
+ assert not result["ok"]
+ assert "Failed to find C compiler" in result["error"]
+
+
+def test_no_compiler_falls_back_to_the_pytorch_kernels_once():
+ result, stdout = _cpu("no_compiler", patch = True)
+ assert result["ok"], result
+ assert result["rel_err"] < 0.15
+ assert "Failed to find C compiler" in result["fallback_error"]
+ # Triton is tried once, then never again: the second quantize goes straight to PyTorch.
+ assert result["calls"] == ["quantize_4bit"]
+ assert stdout.count("Triton could not build its kernel launcher") == 1
+
+
+def test_working_compiler_keeps_triton():
+ result, stdout = _cpu("works", patch = True)
+ assert result["ok"], result
+ assert result["fallback_error"] is None
+ assert result["calls"] == ["quantize_4bit", "quantize_blockwise"] * 2
+ assert "Triton could not build" not in stdout
+
+
+def test_other_errors_are_not_swallowed():
+ result, _ = _cpu("bad_input", patch = True)
+ assert not result["ok"]
+ assert result["error"] == "ValueError: not a build failure"
+ assert result["fallback_error"] is None
+
+
+_GPU_RUNNER = textwrap.dedent(
+ """
+ import importlib.util, json, sys
+ import torch, bitsandbytes.functional as F
+ from bitsandbytes.backends.triton import ops as triton_ops
+ torch.manual_seed(3407)
+ W = torch.randn(1024, 768, device = "cuda", dtype = torch.bfloat16)
+ ref_q, ref_s = F.quantize_4bit(W, quant_type = "nf4", compress_statistics = True)
+ lib = torch.library.Library("bitsandbytes", "FRAGMENT")
+ for name in ("quantize_blockwise", "quantize_4bit"):
+ lib.impl(name, getattr(triton_ops, name), "CUDA", allow_override = True)
+ if sys.argv[1] == "1":
+ spec = importlib.util.spec_from_file_location("fixes_under_test", sys.argv[2])
+ fixes = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(fixes)
+ fixes.patch_bitsandbytes_triton_quantize_fallback("cuda")
+ out = {}
+ try:
+ q, s = F.quantize_4bit(W, quant_type = "nf4", compress_statistics = True)
+ out.update(ok = True, packed_equal = bool(torch.equal(q, ref_q)),
+ absmax_equal = bool(torch.equal(s.absmax, ref_s.absmax)))
+ except Exception as e:
+ out.update(ok = False, error = f"{type(e).__name__}: {e}")
+ print("RESULT " + json.dumps(out))
+ """
+)
+
+
[email protected](not has_real_cuda(), reason = "real Triton launches need a GPU")
[email protected]("patch", [False, True])
+def test_real_triton_without_a_compiler(tmp_path, patch):
+ """Real Triton, with gcc / clang hidden and a cold cache: its launcher build raises."""
+ env = dict(os.environ, TRITON_CACHE_DIR = str(tmp_path), PATH = os.path.dirname(sys.executable))
+ env.pop("CC", None)
+ env.pop("CXX", None)
+ if shutil.which("gcc", path = env["PATH"]) or shutil.which("clang", path = env["PATH"]):
+ pytest.skip("a C compiler sits beside the interpreter")
+ result, _ = _run([_GPU_RUNNER, "1" if patch else "0", _FIXES], env = env)
+ if not patch:
+ assert not result["ok"] and "Failed to find C compiler" in result["error"], result
+ else:
+ # bitsandbytes' PyTorch kernels match its native CUDA ones bit for bit.
+ assert result == {"ok": True, "packed_equal": True, "absmax_equal": True}, result
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 33bdaad2980..220967ee5ce 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -716,7 +716,10 @@ elif DEVICE_TYPE == "xpu":
)
bnb = None
- pass
+ from .import_fixes import patch_bitsandbytes_triton_quantize_fallback
+
+ patch_bitsandbytes_triton_quantize_fallback("xpu")
+ del patch_bitsandbytes_triton_quantize_fallback
# After the bitsandbytes import above, never before: it only patches an already-imported bitsandbytes.
from .import_fixes import patch_bitsandbytes_paged_optimizer_resume
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 44828cfee61..88e14dbb251 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -5219,6 +5219,78 @@ def patch_bitsandbytes_paged_optimizer_resume():
Optimizer8bit.load_state_dict = _unsloth_load_state_dict
+def _raised_in_triton_host_build(exc):
+ """Triton builds each kernel's launcher with the host C compiler in triton/runtime/build.py."""
+ seen = set()
+ while exc is not None and id(exc) not in seen:
+ seen.add(id(exc))
+ tb = exc.__traceback__
+ while tb is not None:
+ path = tb.tb_frame.f_code.co_filename.replace("\\", "/")
+ if path.endswith("triton/runtime/build.py"):
+ return True
+ tb = tb.tb_next
+ exc = exc.__cause__ or exc.__context__
+ return False
+
+
+def patch_bitsandbytes_triton_quantize_fallback(device_type = "xpu"):
+ """bitsandbytes' XPU quantize ops are Triton's, whose launchers need a host C compiler that a stock
+ Windows box lacks (#10844). On that build failure use bitsandbytes' PyTorch kernels instead."""
+ if "bitsandbytes" not in sys.modules:
+ return
+ try:
+ import torch
+ from bitsandbytes.backends.utils import triton_available
+
+ if not triton_available:
+ return
+ from bitsandbytes.backends.triton import ops as triton_ops
+ except Exception:
+ return
+ if getattr(triton_ops, "_unsloth_quantize_fallback", None) is not None:
+ return
+ state = {"error": None}
+ # bitsandbytes registers its PyTorch kernels under "default", i.e. this key.
+ default_keys = torch._C.DispatchKeySet(torch._C.DispatchKey.CompositeExplicitAutograd)
+
+ def _wrap(name):
+ triton_kernel = getattr(triton_ops, name)
+ op = getattr(torch.ops.bitsandbytes, name).default
+
+ @functools.wraps(triton_kernel)
+ def kernel(*args, **kwargs):
+ if state["error"] is None:
+ try:
+ return triton_kernel(*args, **kwargs)
+ except Exception as exc:
+ if not _raised_in_triton_host_build(exc):
+ raise
+ state["error"] = f"{type(exc).__name__}: {str(exc).strip().splitlines()[0]}"
+ print(
+ "Unsloth: Triton could not build its kernel launcher, as there is no working "
+ f"C/C++ compiler ({state['error']}). bitsandbytes 4bit quantization will use "
+ "its PyTorch kernels instead - only model loading is slower."
+ )
+ return op.redispatch(default_keys, *args, **kwargs)
+
+ return kernel
+
+ try:
+ kernels = {name: _wrap(name) for name in ("quantize_4bit", "quantize_blockwise")}
+ key = torch._C._dispatch_key_for_device(device_type)
+ lib = torch.library.Library("bitsandbytes", "FRAGMENT")
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ for name, kernel in kernels.items():
+ # torch < 2.13 refuses to override a Python-registered kernel unless asked to.
+ lib.impl(name, torch._disable_dynamo(kernel), key, allow_override = True)
+ except Exception:
+ return
+ state["lib"] = lib
+ triton_ops._unsloth_quantize_fallback = state
+
+
_PT2_UNSAFE_LOAD_ENV = "UNSLOTH_ALLOW_UNSAFE_PT2_LOAD"
# torch.export's .pt2 readers: their weights_only=False loads unpickle archive bytes.
_PT2_LOADER_MODULES = frozenset(