Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Core: decide bfloat16, compile and TMA options from every visible NVIDIA GPU on mixed hosts (#13923)

Daniel Han2b724677b77b
diff --git a/tests/conftest.py b/tests/conftest.py
index 2b931854d43..9f9d1e8c3e0 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -164,6 +164,8 @@ def _install_device_type_stub(name: str) -> None:
stub.apply_gfx101x_triton_workaround = lambda *a, **k: False
stub.gfx101x_triton_workaround_applied = lambda: False
stub.hip_visible_archs = lambda: []
+ stub.cuda_visible_capabilities = lambda *a, **k: []
+ stub.cuda_gate_capabilities = lambda: []
sys.modules[name] = stub
diff --git a/tests/python/test_rocm_bf16_capability.py b/tests/python/test_rocm_bf16_capability.py
index bda84d76006..b00e56d4731 100644
--- a/tests/python/test_rocm_bf16_capability.py
+++ b/tests/python/test_rocm_bf16_capability.py
@@ -117,7 +117,7 @@ def _fake_torch(
get_device_properties = get_device_properties,
is_bf16_supported = is_bf16_supported,
is_available = lambda: True,
- get_device_capability = lambda: (9, 0),
+ get_device_capability = lambda *a, **k: (9, 0),
),
xpu = types.SimpleNamespace(is_bf16_supported = lambda: True),
)
diff --git a/tests/test_mixed_nvidia_capabilities.py b/tests/test_mixed_nvidia_capabilities.py
new file mode 100644
index 00000000000..f63ab7afb52
--- /dev/null
+++ b/tests/test_mixed_nvidia_capabilities.py
@@ -0,0 +1,375 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""bf16 and the torch.compile kill switches are process-wide, so on a mixed host (e.g. RTX 3090 +
+T4 with device_map="auto") they must hold on the weakest visible card, not on device 0. The
+capability cache must also follow torch.cuda.set_device."""
+
+import functools
+import inspect
+import types
+from pathlib import Path
+
+import pytest
+
+pytest.importorskip("torch")
+
+import unsloth.device_type as dt
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+GPU_INIT = REPO_ROOT / "unsloth" / "_gpu_init.py"
+MODEL_UTILS = REPO_ROOT / "unsloth" / "models" / "_utils.py"
+DEVICE_TYPE_PY = REPO_ROOT / "unsloth" / "device_type.py"
+
+if not hasattr(dt, "torch"):
+ pytest.skip(
+ "device_type stub or MLX host; the real probes are not loaded", allow_module_level = True
+ )
+
+SM80, SM86, SM75, SM90, SM61 = (8, 0), (8, 6), (7, 5), (9, 0), (6, 1)
+
+CASES = [
+ pytest.param([SM80, SM80], True, None, id = "sm80x2"),
+ pytest.param([SM86, SM75], False, None, id = "sm86+sm75"),
+ pytest.param([SM75, SM86], False, None, id = "sm75+sm86"),
+ pytest.param([SM90, SM61], False, 6, id = "sm90+sm61"),
+ pytest.param([SM61], False, 6, id = "sm61"),
+]
+
+
+def _fake_torch(caps, archs = None):
+ state = {"current": 0}
+
+ def get_device_capability(device = None):
+ return caps[state["current"] if device is None else device]
+
+ def get_device_properties(device = None):
+ i = state["current"] if device is None else device
+ major, minor = caps[i] if caps else (9, 0)
+ return types.SimpleNamespace(
+ major = major, minor = minor, gcnArchName = (archs or [""] * len(caps))[i]
+ )
+
+ # Real torch answers for the current device only.
+ def is_bf16_supported(including_emulation = True):
+ return including_emulation or caps[state["current"]][0] >= 8
+
+ def set_device(i):
+ state["current"] = i
+
+ return types.SimpleNamespace(
+ device = __import__("torch").device,
+ version = types.SimpleNamespace(hip = None, cuda = "12.8"),
+ cuda = types.SimpleNamespace(
+ device_count = lambda: len(caps),
+ current_device = lambda: state["current"],
+ set_device = set_device,
+ is_available = lambda: True,
+ get_device_capability = get_device_capability,
+ get_device_properties = get_device_properties,
+ is_bf16_supported = is_bf16_supported,
+ ),
+ )
+
+
+def _block(path, start, end):
+ source = path.read_text(encoding = "utf-8")
+ return start + source.split(start, 1)[1].split(end, 1)[0]
+
+
+def _module_namespace(module, fake, monkeypatch, **overrides):
+ # Helpers read their own module's torch, so point that at the fake too.
+ monkeypatch.setattr(dt, "torch", fake)
+ namespace = dict(vars(module))
+ namespace.update(torch = fake, DEVICE_TYPE = "cuda", inspect = inspect, functools = functools)
+ namespace.update(overrides)
+ return namespace
+
+
+def _run_gpu_init_chain(fake, monkeypatch):
+ import unsloth._gpu_init as gpu_init
+
+ body = _block(
+ GPU_INIT,
+ 'if DEVICE_TYPE == "cuda" and not torch.cuda.is_available():',
+ '\nelif DEVICE_TYPE == "hip":',
+ )
+ namespace = _module_namespace(gpu_init, fake, monkeypatch)
+ exec(compile(body, str(GPU_INIT), "exec"), namespace)
+ return namespace
+
+
+def _run_model_utils_chain(fake, monkeypatch):
+ import unsloth.models._utils as model_utils
+
+ body = _block(
+ MODEL_UTILS,
+ 'if DEVICE_TYPE == "cuda" and not torch.cuda.is_available():',
+ '\nelif DEVICE_TYPE == "hip":',
+ )
+ monkeypatch.setattr(model_utils, "torch", fake)
+ namespace = _module_namespace(
+ model_utils, fake, monkeypatch, _package_available = lambda name: False
+ )
+ exec(compile(body, str(MODEL_UTILS), "exec"), namespace)
+ return namespace
+
+
[email protected]("caps,bf16,_floor", CASES)
+def test_gpu_init_bf16_needs_every_visible_card(monkeypatch, caps, bf16, _floor):
+ fake = _fake_torch(caps)
+ namespace = _run_gpu_init_chain(fake, monkeypatch)
+ assert namespace["SUPPORTS_BFLOAT16"] is bf16
+ assert bool(fake.cuda.is_bf16_supported(False)) is bf16
+ assert bool(fake.cuda.is_bf16_supported()) is bf16
+ # Emulation is torch's own per-device answer and stays untouched.
+ assert fake.cuda.is_bf16_supported(including_emulation = True) is True
+
+
[email protected]("caps,bf16,_floor", CASES)
+def test_mixed_host_says_why_bf16_is_off(monkeypatch, capsys, caps, bf16, _floor):
+ _run_gpu_init_chain(_fake_torch(caps), monkeypatch)
+ out = capsys.readouterr().out
+ mixed = any(c[0] >= 8 for c in caps) and any(c[0] < 8 for c in caps)
+ assert ("lack bfloat16" in out) is mixed
+ if mixed:
+ assert str([i for i, c in enumerate(caps) if c[0] < 8]) in out
+
+
[email protected]("caps,bf16,_floor", CASES)
+def test_model_utils_bf16_needs_every_visible_card(monkeypatch, caps, bf16, _floor):
+ namespace = _run_model_utils_chain(_fake_torch(caps), monkeypatch)
+ assert namespace["SUPPORTS_BFLOAT16"] is bf16
+
+
+def test_capability_cache_follows_set_device(monkeypatch):
+ fake = _fake_torch([SM86, SM75])
+ _run_model_utils_chain(fake, monkeypatch)
+ patched = fake.cuda.get_device_capability
+ assert patched() == SM86
+ fake.cuda.set_device(1)
+ assert patched() == SM75
+ assert patched(None) == SM75
+ assert patched(0) == SM86
+ assert patched(1) == SM75
+ fake.cuda.set_device(0)
+ assert patched() == SM86
+
+
+def test_capability_cache_resolves_device_spellings(monkeypatch):
+ import torch
+
+ import unsloth.models._utils as model_utils
+
+ if not hasattr(model_utils, "_cache_device_capability"):
+ pytest.fail("no per-device capability cache")
+ calls = []
+
+ def probe(device = None):
+ calls.append(device)
+ return {0: SM86, 1: SM75}[device]
+
+ fake = _fake_torch([SM86, SM75])
+ monkeypatch.setattr(model_utils, "torch", fake)
+ cached = model_utils._cache_device_capability(probe)
+ assert cached(1) == SM75
+ assert cached("cuda:1") == SM75
+ assert cached(torch.device("cuda", 1)) == SM75
+ assert cached() == SM86
+ assert cached("cuda") == SM86
+ assert calls == [1, 0] # one real probe per index
+ with pytest.raises(KeyError):
+ cached("cpu") # not a CUDA device: passed through uncached, as torch would reject it
+ assert model_utils._cache_device_capability(cached) is cached # re-import does not stack
+ assert inspect.signature(cached) == inspect.signature(probe)
+
+
+def _run_pre_volta(fake, monkeypatch, capsys):
+ applied = []
+
+ def record(major, *args, **kwargs):
+ applied.append(major)
+ return major is not None and major < 7
+
+ body = _block(
+ DEVICE_TYPE_PY,
+ 'if DEVICE_TYPE == "cuda" and torch.cuda.is_available():',
+ '\nif DEVICE_TYPE == "hip":\n try:\n import bitsandbytes',
+ )
+ namespace = _module_namespace(dt, fake, monkeypatch, apply_pre_volta_compile_workaround = record)
+ exec(compile(body, str(DEVICE_TYPE_PY), "exec"), namespace)
+ return applied, capsys.readouterr().out
+
+
[email protected]("caps,_bf16,floor", CASES)
+def test_pre_volta_switch_reads_the_weakest_visible_card(monkeypatch, capsys, caps, _bf16, floor):
+ applied, out = _run_pre_volta(_fake_torch(caps), monkeypatch, capsys)
+ assert applied == [min(c[0] for c in caps)]
+ if floor is None:
+ assert out == ""
+ else:
+ assert out == (
+ "Unsloth: GPUs older than Volta (compute capability < 7.0) cannot run "
+ "torch.compile - training without it.\n"
+ )
+
+
+def test_pre_volta_switch_skips_an_unreadable_card(monkeypatch, capsys):
+ fake = _fake_torch([SM61, SM90])
+ real = fake.cuda.get_device_capability
+
+ def flaky(device = None):
+ if device == 1:
+ raise RuntimeError("device wedged")
+ return real(device)
+
+ fake.cuda.get_device_capability = flaky
+ applied, _ = _run_pre_volta(fake, monkeypatch, capsys)
+ assert applied == [6]
+
+
+ "archs,applied",
+ [
+ (["gfx906:sramecc+:xnack-"], True),
+ (["gfx1100", "gfx906:sramecc+:xnack-"], True),
+ (["gfx906", "gfx1100"], True),
+ (["gfx90a", "gfx942"], False),
+ ],
+)
+def test_gfx906_switch_reads_every_visible_card(monkeypatch, capsys, archs, applied):
+ fake = _fake_torch([SM90] * len(archs), archs = archs)
+ environ = {}
+ dynamo_config = types.SimpleNamespace(disable = False)
+ body = _block(
+ DEVICE_TYPE_PY,
+ "# gfx906 (MI50",
+ '\n\nif DEVICE_TYPE == "cuda" and torch.cuda.is_available():',
+ )
+ namespace = _module_namespace(
+ dt,
+ fake,
+ monkeypatch,
+ DEVICE_TYPE = "hip",
+ os = types.SimpleNamespace(environ = environ),
+ apply_pre_volta_compile_workaround = lambda major: dt.apply_pre_volta_compile_workaround(
+ major, environ = environ, dynamo_config = dynamo_config
+ ),
+ )
+ exec(compile(body, str(DEVICE_TYPE_PY), "exec"), namespace)
+ out = capsys.readouterr().out
+ # Regions compiled before this check read the live config, not the env vars.
+ assert dynamo_config.disable is applied
+ if applied:
+ assert environ == {
+ "TORCHDYNAMO_DISABLE": "1",
+ "TORCH_COMPILE_DISABLE": "1",
+ "UNSLOTH_COMPILE_DISABLE": "1",
+ }
+ assert out == (
+ "Unsloth: gfx906 (MI50 / Radeon VII) detected - torch.compile disabled "
+ "(community-maintained legacy GCN path).\n"
+ )
+ else:
+ assert environ == {} and out == ""
+
+
+def _grpo_all_sm90_expression():
+ source = (REPO_ROOT / "unsloth" / "models" / "rl.py").read_text(encoding = "utf-8")
+ block = source.split("all_sm90 = (", 1)[1].split("\n )", 1)[0]
+ return eval("(" + block + ")")
+
+
+ "caps,unreadable,expected",
+ [
+ ([(10, 0), (10, 0)], set(), True),
+ ([(10, 0), (8, 0)], set(), False),
+ ([(8, 0), (10, 0)], set(), False),
+ # A card whose capability query raises must turn TMA off, not abort the import.
+ ([(10, 0), (10, 0)], {1}, False),
+ ],
+)
+def test_grpo_tma_options_need_every_visible_card(caps, unreadable, expected):
+ import builtins
+
+ fake = _fake_torch(caps)
+ real_cap = fake.cuda.get_device_capability
+
+ def get_device_capability(device = None):
+ if device in unreadable:
+ raise RuntimeError("unreadable")
+ return real_cap(device)
+
+ fake.cuda.get_device_capability = get_device_capability
+
+ def visible(with_index = False):
+ out = []
+ for i in range(fake.cuda.device_count()):
+ try:
+ cap = tuple(fake.cuda.get_device_capability(i))
+ except Exception:
+ continue
+ out.append((i, cap) if with_index else cap)
+ return out
+
+ stub = types.SimpleNamespace(cuda_visible_capabilities = visible)
+ safe_builtins = dict(vars(builtins))
+ safe_builtins["__import__"] = lambda name, *a, **k: stub
+ expression = _grpo_all_sm90_expression()
+ assert "get_device_capability()" not in expression
+ assert eval(expression, {"__builtins__": safe_builtins, "torch": fake}) is expected
+
+
+ "caps,current,bf16",
+ [
+ ([(9, 0), (9, 0), (7, 5)], 0, False), # a known old card still counts
+ ([(9, 0), (9, 0), (7, 5)], 2, False),
+ ([(9, 0), (9, 0), (9, 0)], 0, True), # current device is sm90, as on main
+ ],
+)
+def test_a_partial_scan_judges_every_card_it_read(monkeypatch, capsys, caps, current, bf16):
+ # Card 1 unreadable: the gate judges the readable cards, which include the current device,
+ # so it is never less strict than main's current-device rule.
+ fake = _fake_torch(caps)
+ real_cap = fake.cuda.get_device_capability
+
+ def get_device_capability(device = None):
+ if device == 1:
+ raise RuntimeError("unreadable")
+ return real_cap(device)
+
+ fake.cuda.get_device_capability = get_device_capability
+ fake.cuda.set_device(current)
+ namespace = _run_gpu_init_chain(fake, monkeypatch)
+ assert namespace["SUPPORTS_BFLOAT16"] is bf16
+ out = capsys.readouterr().out
+ assert ("[2]" in out) is (not bf16) and "[1]" not in out
+ assert _run_model_utils_chain(fake, monkeypatch)["SUPPORTS_BFLOAT16"] is bf16
+
+
+def test_bf16_notice_names_real_device_indices(monkeypatch, capsys):
+ # Device 2 is the pre-Ampere card while device 0 is current: the notice names 2.
+ _run_gpu_init_chain(_fake_torch([(9, 0), (8, 6), (7, 5)]), monkeypatch)
+ out = capsys.readouterr().out
+ assert "[2]" in out and "[0]" not in out and "[1]" not in out
+
+
[email protected](not __import__("torch").cuda.is_available(), reason = "needs a CUDA device")
[email protected]("spelling", [None, 0, "cuda:0", "device", "tensor"])
+def test_capability_cache_traces_in_fullgraph(spelling):
+ # Attention helpers call get_device_capability(Q.device) inside compiled regions.
+ import torch
+
+ import unsloth.models._utils as model_utils
+
+ cached = model_utils._cache_device_capability(inspect.unwrap(torch.cuda.get_device_capability))
+
+ def f(x):
+ arg = {"device": torch.device("cuda"), "tensor": x.device}.get(spelling, spelling)
+ return x * (cached() if arg is None else cached(arg))[0]
+
+ torch._dynamo.reset()
+ x = torch.ones(2, device = "cuda")
+ assert torch.compile(f, fullgraph = True)(x)[0].item() == torch.cuda.get_device_capability(0)[0]
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 7ae16996db7..5d06655d61d 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -315,6 +315,8 @@ try:
arch_lacks_bf16,
arch_lacks_buffer_ops,
apply_gfx101x_triton_workaround,
+ cuda_gate_capabilities,
+ cuda_visible_capabilities,
hip_visible_archs,
)
except NotImplementedError as device_type_error:
@@ -546,14 +548,24 @@ if DEVICE_TYPE == "cuda" and not torch.cuda.is_available():
SUPPORTS_BFLOAT16 = False
torch.cuda.is_bf16_supported = lambda *args, **kwargs: False
elif DEVICE_TYPE == "cuda":
- major_version, minor_version = torch.cuda.get_device_capability()
- SUPPORTS_BFLOAT16 = major_version >= 8
+ # SUPPORTS_BFLOAT16 is process-wide, so one pre-Ampere card in the visible set disables it for all.
+ _cuda_caps = cuda_gate_capabilities()
+ SUPPORTS_BFLOAT16 = bool(_cuda_caps) and all(major >= 8 for major, _ in _cuda_caps)
+ if not SUPPORTS_BFLOAT16 and any(major >= 8 for major, _ in _cuda_caps):
+ _old = [i for i, (major, _) in cuda_visible_capabilities(with_index = True) if major < 8]
+ print(
+ f"Unsloth: visible GPU(s) {_old} lack bfloat16, so float16 is used on every GPU. "
+ "Hide them with CUDA_VISIBLE_DEVICES to train in bfloat16."
+ )
+ del _old
+ del _cuda_caps
old_is_bf16_supported = torch.cuda.is_bf16_supported
if "including_emulation" in str(inspect.signature(old_is_bf16_supported)):
def is_bf16_supported(including_emulation = False):
- return old_is_bf16_supported(including_emulation)
+ supported = old_is_bf16_supported(including_emulation)
+ return supported if including_emulation else (supported and SUPPORTS_BFLOAT16)
torch.cuda.is_bf16_supported = is_bf16_supported
else:
@@ -562,7 +574,6 @@ elif DEVICE_TYPE == "cuda":
return SUPPORTS_BFLOAT16
torch.cuda.is_bf16_supported = is_bf16_supported
- del major_version, minor_version
elif DEVICE_TYPE == "hip":
old_is_bf16_supported = torch.cuda.is_bf16_supported
diff --git a/unsloth/device_type.py b/unsloth/device_type.py
index 8c73dc56cb8..fc1532cbcf5 100644
--- a/unsloth/device_type.py
+++ b/unsloth/device_type.py
@@ -167,21 +167,53 @@ else:
ALLOW_PREQUANTIZED_MODELS = False
ALLOW_BITSANDBYTES = False
del _bnb_probe
-# gfx906 (MI50 / Radeon VII / Vega 20): Dynamo/Inductor codegen is broken on this legacy GCN arch
-# (ROCm dropped it after 6.3), so compiled graphs crash or miscompile while eager trains fine.
-if DEVICE_TYPE == "hip":
+
+
+def hip_visible_archs():
+ """Guarded per device: one unreadable device must not discard the archs beside it, or a
+ gfx10 keeps bf16 and dies in Triton (#7922). Only an unreadable count returns []."""
try:
- _gcn_arch = torch.cuda.get_device_properties(0).gcnArchName.split(":")[0].strip().lower()
+ count = torch.cuda.device_count()
except Exception:
- _gcn_arch = ""
- if _gcn_arch == "gfx906":
- os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")
- os.environ.setdefault("TORCH_COMPILE_DISABLE", "1")
- os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")
- print(
- "Unsloth: gfx906 (MI50 / Radeon VII) detected - torch.compile disabled "
- "(community-maintained legacy GCN path)."
- )
+ return []
+ archs = []
+ for i in range(count):
+ try:
+ archs.append(str(getattr(torch.cuda.get_device_properties(i), "gcnArchName", "")))
+ except Exception:
+ continue
+ return archs
+
+
+def cuda_visible_capabilities(with_index = False):
+ """(major, minor) of every visible CUDA device, guarded per device like hip_visible_archs:
+ process-wide gates (bf16, compile) must hold on the weakest card layers may land on.
+ with_index yields (device_index, (major, minor)), since unreadable cards are skipped."""
+ try:
+ count = torch.cuda.device_count()
+ except Exception:
+ return []
+ caps = []
+ for i in range(count):
+ try:
+ cap = tuple(torch.cuda.get_device_capability(i))
+ except Exception:
+ continue
+ caps.append((i, cap) if with_index else cap)
+ return caps
+
+
+def cuda_gate_capabilities():
+ """Capabilities a process-wide gate should judge: every card that could be read, which
+ always includes the current device (the pre-mixed-host rule) when it is readable, so a
+ gate is never less strict than before and still sees a known old card beside it."""
+ caps = cuda_visible_capabilities()
+ if caps:
+ return caps
+ try:
+ return [tuple(torch.cuda.get_device_capability())]
+ except Exception:
+ return []
# Pre-Volta NVIDIA (Maxwell / Pascal, sm < 7.0): Inductor refuses to emit Triton for these
@@ -217,11 +249,25 @@ def apply_pre_volta_compile_workaround(
return True
+# gfx906 (MI50 / Radeon VII / Vega 20): Dynamo/Inductor codegen is broken on this legacy GCN arch
+# (ROCm dropped it after 6.3), so compiled graphs crash or miscompile while eager trains fine.
+# Any visible gfx906 counts: compile is process-wide and layers may land on it. Same switch as
+# pre-Volta, so regions compiled while unsloth_zoo imported are turned off too.
+if DEVICE_TYPE == "hip":
+ if any(arch.split(":")[0].strip().lower() == "gfx906" for arch in hip_visible_archs()):
+ os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")
+ os.environ.setdefault("TORCH_COMPILE_DISABLE", "1")
+ os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")
+ apply_pre_volta_compile_workaround(0) # the live Dynamo config, unless explicitly enabled
+ print(
+ "Unsloth: gfx906 (MI50 / Radeon VII) detected - torch.compile disabled "
+ "(community-maintained legacy GCN path)."
+ )
+
+
if DEVICE_TYPE == "cuda" and torch.cuda.is_available():
- try:
- _cuda_major = torch.cuda.get_device_capability()[0]
- except Exception:
- _cuda_major = None
+ # Weakest visible card: compile is process-wide and layers may land on any of them.
+ _cuda_major = min((cap[0] for cap in cuda_gate_capabilities()), default = None)
if apply_pre_volta_compile_workaround(_cuda_major):
print(
"Unsloth: GPUs older than Volta (compute capability < 7.0) cannot run "
@@ -323,22 +369,6 @@ def _default_inductor_cache_dir():
return os.path.join(tempfile.gettempdir(), "torchinductor_" + user)
-def hip_visible_archs():
- """Guarded per device: one unreadable device must not discard the archs beside it, or a
- gfx10 keeps bf16 and dies in Triton (#7922). Only an unreadable count returns []."""
- try:
- count = torch.cuda.device_count()
- except Exception:
- return []
- archs = []
- for i in range(count):
- try:
- archs.append(str(getattr(torch.cuda.get_device_properties(i), "gcnArchName", "")))
- except Exception:
- continue
- return archs
-
-
def resolve_hip_gpu_stats_name(gpu_stats):
name = str(getattr(gpu_stats, "name", "") or "").strip()
name = re.sub(r"\s*\([^)]*\)\s*$", "", name).strip()
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 304c4a6a6b5..b1ebe0ac311 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -167,6 +167,7 @@ from ..device_type import (
DEVICE_COUNT,
ALLOW_PREQUANTIZED_MODELS,
apply_gfx101x_triton_workaround,
+ cuda_gate_capabilities,
gfx101x_triton_workaround_applied,
)
from ..import_fixes import UNSLOTH_ENABLE_LOGGING, stale_kernel_hint
@@ -3846,6 +3847,38 @@ def _package_available(pkg_name: str) -> bool:
return bool(result)
+def _cache_device_capability(get_device_capability):
+ """Cache per resolved device index: a no-arg call means the current device, so caching the
+ call itself returned device 0's answer after torch.cuda.set_device(1)."""
+ if getattr(get_device_capability, "_unsloth_per_device_cache", False):
+ return get_device_capability
+
+ cache = {}
+
+ @functools.wraps(get_device_capability)
+ def cached_get_device_capability(device = None):
+ if device is None:
+ index = torch.cuda.current_device()
+ elif isinstance(device, int):
+ index = device
+ else:
+ # torch.device, not torch.cuda._utils._get_device_index: Dynamo traces it (fullgraph).
+ try:
+ device = torch.device(device)
+ except Exception:
+ return get_device_capability(device)
+ if device.type != "cuda":
+ return get_device_capability(device)
+ index = torch.cuda.current_device() if device.index is None else device.index
+ capability = cache.get(index)
+ if capability is None:
+ capability = cache[index] = get_device_capability(index)
+ return capability
+
+ cached_get_device_capability._unsloth_per_device_cache = True
+ return cached_get_device_capability
+
+
SUPPORTS_BFLOAT16 = False
HAS_FLASH_ATTENTION = False
HAS_FLASH_ATTENTION_SOFTCAPPING = False
@@ -3857,10 +3890,13 @@ if DEVICE_TYPE == "cuda" and not torch.cuda.is_available():
HAS_FLASH_ATTENTION_SOFTCAPPING = False
elif DEVICE_TYPE == "cuda":
major_version, minor_version = torch.cuda.get_device_capability()
- torch.cuda.get_device_capability = functools.cache(torch.cuda.get_device_capability)
+ torch.cuda.get_device_capability = _cache_device_capability(torch.cuda.get_device_capability)
+ # SUPPORTS_BFLOAT16 is process-wide, so one pre-Ampere card in the visible set disables it for all.
+ _cuda_caps = cuda_gate_capabilities()
+ SUPPORTS_BFLOAT16 = bool(_cuda_caps) and all(major >= 8 for major, _ in _cuda_caps)
+ del _cuda_caps
if major_version >= 8:
- SUPPORTS_BFLOAT16 = True
if _package_available("flash_attn"):
# Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl" and for softcapping; see #1437.
try:
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index dcd054682df..3cac0f24602 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -3624,13 +3624,21 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"trace.enabled" : False,"""
if DEVICE_TYPE == "cuda":
- cuda_options = """
- "triton.enable_persistent_tma_matmul": torch.cuda.get_device_capability()[0] >= 9,"""
+ # TMA only if every visible card is sm90+ (compile options are process-wide); an
+ # unreadable card is skipped by the probe, so a short list also means False.
+ all_sm90 = (
+ "(lambda caps: bool(caps) and len(caps) == torch.cuda.device_count() "
+ "and min(c[0] for c in caps) >= 9)"
+ '(__import__("unsloth.device_type", fromlist = ["cuda_visible_capabilities"])'
+ ".cuda_visible_capabilities())"
+ )
+ cuda_options = f"""
+ "triton.enable_persistent_tma_matmul": {all_sm90},"""
# cutlass options were added in PyTorch 2.8.0.
if torch_version >= Version("2.8.0"):
- cuda_options += """
- "cuda.cutlass_epilogue_fusion_enabled": torch.cuda.get_device_capability()[0] >= 9,
- "cuda.cutlass_tma_only": torch.cuda.get_device_capability()[0] >= 9,"""
+ cuda_options += f"""
+ "cuda.cutlass_epilogue_fusion_enabled": {all_sm90},
+ "cuda.cutlass_tma_only": {all_sm90},"""
cuda_options += """
"cuda.compile_opt_level" : "-O2",
"cuda.enable_cuda_lto" : True,