Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Core: decide bfloat16, compile and TMA options from every visible NVIDIA GPU on mixed hosts (#13923)
Daniel Han2b724677b77b
diff --git a/tests/conftest.py b/tests/conftest.pyindex 2b931854d43..9f9d1e8c3e0 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -164,6 +164,8 @@ def _install_device_type_stub(name: str) -> None:stub.apply_gfx101x_triton_workaround = lambda *a, **k: Falsestub.gfx101x_triton_workaround_applied = lambda: Falsestub.hip_visible_archs = lambda: []+ stub.cuda_visible_capabilities = lambda *a, **k: []+ stub.cuda_gate_capabilities = lambda: []sys.modules[name] = stubdiff --git a/tests/python/test_rocm_bf16_capability.py b/tests/python/test_rocm_bf16_capability.pyindex bda84d76006..b00e56d4731 100644--- a/tests/python/test_rocm_bf16_capability.py+++ b/tests/python/test_rocm_bf16_capability.py@@ -117,7 +117,7 @@ def _fake_torch(get_device_properties = get_device_properties,is_bf16_supported = is_bf16_supported,is_available = lambda: True,- get_device_capability = lambda: (9, 0),+ get_device_capability = lambda *a, **k: (9, 0),),xpu = types.SimpleNamespace(is_bf16_supported = lambda: True),)diff --git a/tests/test_mixed_nvidia_capabilities.py b/tests/test_mixed_nvidia_capabilities.pynew file mode 100644index 00000000000..f63ab7afb52--- /dev/null+++ b/tests/test_mixed_nvidia_capabilities.py@@ -0,0 +1,375 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""bf16 and the torch.compile kill switches are process-wide, so on a mixed host (e.g. RTX 3090 ++T4 with device_map="auto") they must hold on the weakest visible card, not on device 0. The+capability cache must also follow torch.cuda.set_device."""++import functools+import inspect+import types+from pathlib import Path++import pytest++pytest.importorskip("torch")++import unsloth.device_type as dt++REPO_ROOT = Path(__file__).resolve().parents[1]+GPU_INIT = REPO_ROOT / "unsloth" / "_gpu_init.py"+MODEL_UTILS = REPO_ROOT / "unsloth" / "models" / "_utils.py"+DEVICE_TYPE_PY = REPO_ROOT / "unsloth" / "device_type.py"++if not hasattr(dt, "torch"):+ pytest.skip(+ "device_type stub or MLX host; the real probes are not loaded", allow_module_level = True+ )++SM80, SM86, SM75, SM90, SM61 = (8, 0), (8, 6), (7, 5), (9, 0), (6, 1)++CASES = [+ pytest.param([SM80, SM80], True, None, id = "sm80x2"),+ pytest.param([SM86, SM75], False, None, id = "sm86+sm75"),+ pytest.param([SM75, SM86], False, None, id = "sm75+sm86"),+ pytest.param([SM90, SM61], False, 6, id = "sm90+sm61"),+ pytest.param([SM61], False, 6, id = "sm61"),+]+++def _fake_torch(caps, archs = None):+ state = {"current": 0}++ def get_device_capability(device = None):+ return caps[state["current"] if device is None else device]++ def get_device_properties(device = None):+ i = state["current"] if device is None else device+ major, minor = caps[i] if caps else (9, 0)+ return types.SimpleNamespace(+ major = major, minor = minor, gcnArchName = (archs or [""] * len(caps))[i]+ )++ # Real torch answers for the current device only.+ def is_bf16_supported(including_emulation = True):+ return including_emulation or caps[state["current"]][0] >= 8++ def set_device(i):+ state["current"] = i++ return types.SimpleNamespace(+ device = __import__("torch").device,+ version = types.SimpleNamespace(hip = None, cuda = "12.8"),+ cuda = types.SimpleNamespace(+ device_count = lambda: len(caps),+ current_device = lambda: state["current"],+ set_device = set_device,+ is_available = lambda: True,+ get_device_capability = get_device_capability,+ get_device_properties = get_device_properties,+ is_bf16_supported = is_bf16_supported,+ ),+ )+++def _block(path, start, end):+ source = path.read_text(encoding = "utf-8")+ return start + source.split(start, 1)[1].split(end, 1)[0]+++def _module_namespace(module, fake, monkeypatch, **overrides):+ # Helpers read their own module's torch, so point that at the fake too.+ monkeypatch.setattr(dt, "torch", fake)+ namespace = dict(vars(module))+ namespace.update(torch = fake, DEVICE_TYPE = "cuda", inspect = inspect, functools = functools)+ namespace.update(overrides)+ return namespace+++def _run_gpu_init_chain(fake, monkeypatch):+ import unsloth._gpu_init as gpu_init++ body = _block(+ GPU_INIT,+ 'if DEVICE_TYPE == "cuda" and not torch.cuda.is_available():',+ '\nelif DEVICE_TYPE == "hip":',+ )+ namespace = _module_namespace(gpu_init, fake, monkeypatch)+ exec(compile(body, str(GPU_INIT), "exec"), namespace)+ return namespace+++def _run_model_utils_chain(fake, monkeypatch):+ import unsloth.models._utils as model_utils++ body = _block(+ MODEL_UTILS,+ 'if DEVICE_TYPE == "cuda" and not torch.cuda.is_available():',+ '\nelif DEVICE_TYPE == "hip":',+ )+ monkeypatch.setattr(model_utils, "torch", fake)+ namespace = _module_namespace(+ model_utils, fake, monkeypatch, _package_available = lambda name: False+ )+ exec(compile(body, str(MODEL_UTILS), "exec"), namespace)+ return namespace++[email protected]("caps,bf16,_floor", CASES)+def test_gpu_init_bf16_needs_every_visible_card(monkeypatch, caps, bf16, _floor):+ fake = _fake_torch(caps)+ namespace = _run_gpu_init_chain(fake, monkeypatch)+ assert namespace["SUPPORTS_BFLOAT16"] is bf16+ assert bool(fake.cuda.is_bf16_supported(False)) is bf16+ assert bool(fake.cuda.is_bf16_supported()) is bf16+ # Emulation is torch's own per-device answer and stays untouched.+ assert fake.cuda.is_bf16_supported(including_emulation = True) is True++[email protected]("caps,bf16,_floor", CASES)+def test_mixed_host_says_why_bf16_is_off(monkeypatch, capsys, caps, bf16, _floor):+ _run_gpu_init_chain(_fake_torch(caps), monkeypatch)+ out = capsys.readouterr().out+ mixed = any(c[0] >= 8 for c in caps) and any(c[0] < 8 for c in caps)+ assert ("lack bfloat16" in out) is mixed+ if mixed:+ assert str([i for i, c in enumerate(caps) if c[0] < 8]) in out++[email protected]("caps,bf16,_floor", CASES)+def test_model_utils_bf16_needs_every_visible_card(monkeypatch, caps, bf16, _floor):+ namespace = _run_model_utils_chain(_fake_torch(caps), monkeypatch)+ assert namespace["SUPPORTS_BFLOAT16"] is bf16+++def test_capability_cache_follows_set_device(monkeypatch):+ fake = _fake_torch([SM86, SM75])+ _run_model_utils_chain(fake, monkeypatch)+ patched = fake.cuda.get_device_capability+ assert patched() == SM86+ fake.cuda.set_device(1)+ assert patched() == SM75+ assert patched(None) == SM75+ assert patched(0) == SM86+ assert patched(1) == SM75+ fake.cuda.set_device(0)+ assert patched() == SM86+++def test_capability_cache_resolves_device_spellings(monkeypatch):+ import torch++ import unsloth.models._utils as model_utils++ if not hasattr(model_utils, "_cache_device_capability"):+ pytest.fail("no per-device capability cache")+ calls = []++ def probe(device = None):+ calls.append(device)+ return {0: SM86, 1: SM75}[device]++ fake = _fake_torch([SM86, SM75])+ monkeypatch.setattr(model_utils, "torch", fake)+ cached = model_utils._cache_device_capability(probe)+ assert cached(1) == SM75+ assert cached("cuda:1") == SM75+ assert cached(torch.device("cuda", 1)) == SM75+ assert cached() == SM86+ assert cached("cuda") == SM86+ assert calls == [1, 0] # one real probe per index+ with pytest.raises(KeyError):+ cached("cpu") # not a CUDA device: passed through uncached, as torch would reject it+ assert model_utils._cache_device_capability(cached) is cached # re-import does not stack+ assert inspect.signature(cached) == inspect.signature(probe)+++def _run_pre_volta(fake, monkeypatch, capsys):+ applied = []++ def record(major, *args, **kwargs):+ applied.append(major)+ return major is not None and major < 7++ body = _block(+ DEVICE_TYPE_PY,+ 'if DEVICE_TYPE == "cuda" and torch.cuda.is_available():',+ '\nif DEVICE_TYPE == "hip":\n try:\n import bitsandbytes',+ )+ namespace = _module_namespace(dt, fake, monkeypatch, apply_pre_volta_compile_workaround = record)+ exec(compile(body, str(DEVICE_TYPE_PY), "exec"), namespace)+ return applied, capsys.readouterr().out++[email protected]("caps,_bf16,floor", CASES)+def test_pre_volta_switch_reads_the_weakest_visible_card(monkeypatch, capsys, caps, _bf16, floor):+ applied, out = _run_pre_volta(_fake_torch(caps), monkeypatch, capsys)+ assert applied == [min(c[0] for c in caps)]+ if floor is None:+ assert out == ""+ else:+ assert out == (+ "Unsloth: GPUs older than Volta (compute capability < 7.0) cannot run "+ "torch.compile - training without it.\n"+ )+++def test_pre_volta_switch_skips_an_unreadable_card(monkeypatch, capsys):+ fake = _fake_torch([SM61, SM90])+ real = fake.cuda.get_device_capability++ def flaky(device = None):+ if device == 1:+ raise RuntimeError("device wedged")+ return real(device)++ fake.cuda.get_device_capability = flaky+ applied, _ = _run_pre_volta(fake, monkeypatch, capsys)+ assert applied == [6]+++ "archs,applied",+ [+ (["gfx906:sramecc+:xnack-"], True),+ (["gfx1100", "gfx906:sramecc+:xnack-"], True),+ (["gfx906", "gfx1100"], True),+ (["gfx90a", "gfx942"], False),+ ],+)+def test_gfx906_switch_reads_every_visible_card(monkeypatch, capsys, archs, applied):+ fake = _fake_torch([SM90] * len(archs), archs = archs)+ environ = {}+ dynamo_config = types.SimpleNamespace(disable = False)+ body = _block(+ DEVICE_TYPE_PY,+ "# gfx906 (MI50",+ '\n\nif DEVICE_TYPE == "cuda" and torch.cuda.is_available():',+ )+ namespace = _module_namespace(+ dt,+ fake,+ monkeypatch,+ DEVICE_TYPE = "hip",+ os = types.SimpleNamespace(environ = environ),+ apply_pre_volta_compile_workaround = lambda major: dt.apply_pre_volta_compile_workaround(+ major, environ = environ, dynamo_config = dynamo_config+ ),+ )+ exec(compile(body, str(DEVICE_TYPE_PY), "exec"), namespace)+ out = capsys.readouterr().out+ # Regions compiled before this check read the live config, not the env vars.+ assert dynamo_config.disable is applied+ if applied:+ assert environ == {+ "TORCHDYNAMO_DISABLE": "1",+ "TORCH_COMPILE_DISABLE": "1",+ "UNSLOTH_COMPILE_DISABLE": "1",+ }+ assert out == (+ "Unsloth: gfx906 (MI50 / Radeon VII) detected - torch.compile disabled "+ "(community-maintained legacy GCN path).\n"+ )+ else:+ assert environ == {} and out == ""+++def _grpo_all_sm90_expression():+ source = (REPO_ROOT / "unsloth" / "models" / "rl.py").read_text(encoding = "utf-8")+ block = source.split("all_sm90 = (", 1)[1].split("\n )", 1)[0]+ return eval("(" + block + ")")+++ "caps,unreadable,expected",+ [+ ([(10, 0), (10, 0)], set(), True),+ ([(10, 0), (8, 0)], set(), False),+ ([(8, 0), (10, 0)], set(), False),+ # A card whose capability query raises must turn TMA off, not abort the import.+ ([(10, 0), (10, 0)], {1}, False),+ ],+)+def test_grpo_tma_options_need_every_visible_card(caps, unreadable, expected):+ import builtins++ fake = _fake_torch(caps)+ real_cap = fake.cuda.get_device_capability++ def get_device_capability(device = None):+ if device in unreadable:+ raise RuntimeError("unreadable")+ return real_cap(device)++ fake.cuda.get_device_capability = get_device_capability++ def visible(with_index = False):+ out = []+ for i in range(fake.cuda.device_count()):+ try:+ cap = tuple(fake.cuda.get_device_capability(i))+ except Exception:+ continue+ out.append((i, cap) if with_index else cap)+ return out++ stub = types.SimpleNamespace(cuda_visible_capabilities = visible)+ safe_builtins = dict(vars(builtins))+ safe_builtins["__import__"] = lambda name, *a, **k: stub+ expression = _grpo_all_sm90_expression()+ assert "get_device_capability()" not in expression+ assert eval(expression, {"__builtins__": safe_builtins, "torch": fake}) is expected+++ "caps,current,bf16",+ [+ ([(9, 0), (9, 0), (7, 5)], 0, False), # a known old card still counts+ ([(9, 0), (9, 0), (7, 5)], 2, False),+ ([(9, 0), (9, 0), (9, 0)], 0, True), # current device is sm90, as on main+ ],+)+def test_a_partial_scan_judges_every_card_it_read(monkeypatch, capsys, caps, current, bf16):+ # Card 1 unreadable: the gate judges the readable cards, which include the current device,+ # so it is never less strict than main's current-device rule.+ fake = _fake_torch(caps)+ real_cap = fake.cuda.get_device_capability++ def get_device_capability(device = None):+ if device == 1:+ raise RuntimeError("unreadable")+ return real_cap(device)++ fake.cuda.get_device_capability = get_device_capability+ fake.cuda.set_device(current)+ namespace = _run_gpu_init_chain(fake, monkeypatch)+ assert namespace["SUPPORTS_BFLOAT16"] is bf16+ out = capsys.readouterr().out+ assert ("[2]" in out) is (not bf16) and "[1]" not in out+ assert _run_model_utils_chain(fake, monkeypatch)["SUPPORTS_BFLOAT16"] is bf16+++def test_bf16_notice_names_real_device_indices(monkeypatch, capsys):+ # Device 2 is the pre-Ampere card while device 0 is current: the notice names 2.+ _run_gpu_init_chain(_fake_torch([(9, 0), (8, 6), (7, 5)]), monkeypatch)+ out = capsys.readouterr().out+ assert "[2]" in out and "[0]" not in out and "[1]" not in out++[email protected](not __import__("torch").cuda.is_available(), reason = "needs a CUDA device")[email protected]("spelling", [None, 0, "cuda:0", "device", "tensor"])+def test_capability_cache_traces_in_fullgraph(spelling):+ # Attention helpers call get_device_capability(Q.device) inside compiled regions.+ import torch++ import unsloth.models._utils as model_utils++ cached = model_utils._cache_device_capability(inspect.unwrap(torch.cuda.get_device_capability))++ def f(x):+ arg = {"device": torch.device("cuda"), "tensor": x.device}.get(spelling, spelling)+ return x * (cached() if arg is None else cached(arg))[0]++ torch._dynamo.reset()+ x = torch.ones(2, device = "cuda")+ assert torch.compile(f, fullgraph = True)(x)[0].item() == torch.cuda.get_device_capability(0)[0]diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 7ae16996db7..5d06655d61d 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -315,6 +315,8 @@ try:arch_lacks_bf16,arch_lacks_buffer_ops,apply_gfx101x_triton_workaround,+ cuda_gate_capabilities,+ cuda_visible_capabilities,hip_visible_archs,)except NotImplementedError as device_type_error:@@ -546,14 +548,24 @@ if DEVICE_TYPE == "cuda" and not torch.cuda.is_available():SUPPORTS_BFLOAT16 = Falsetorch.cuda.is_bf16_supported = lambda *args, **kwargs: Falseelif DEVICE_TYPE == "cuda":- major_version, minor_version = torch.cuda.get_device_capability()- SUPPORTS_BFLOAT16 = major_version >= 8+ # SUPPORTS_BFLOAT16 is process-wide, so one pre-Ampere card in the visible set disables it for all.+ _cuda_caps = cuda_gate_capabilities()+ SUPPORTS_BFLOAT16 = bool(_cuda_caps) and all(major >= 8 for major, _ in _cuda_caps)+ if not SUPPORTS_BFLOAT16 and any(major >= 8 for major, _ in _cuda_caps):+ _old = [i for i, (major, _) in cuda_visible_capabilities(with_index = True) if major < 8]+ print(+ f"Unsloth: visible GPU(s) {_old} lack bfloat16, so float16 is used on every GPU. "+ "Hide them with CUDA_VISIBLE_DEVICES to train in bfloat16."+ )+ del _old+ del _cuda_capsold_is_bf16_supported = torch.cuda.is_bf16_supportedif "including_emulation" in str(inspect.signature(old_is_bf16_supported)):def is_bf16_supported(including_emulation = False):- return old_is_bf16_supported(including_emulation)+ supported = old_is_bf16_supported(including_emulation)+ return supported if including_emulation else (supported and SUPPORTS_BFLOAT16)torch.cuda.is_bf16_supported = is_bf16_supportedelse:@@ -562,7 +574,6 @@ elif DEVICE_TYPE == "cuda":return SUPPORTS_BFLOAT16torch.cuda.is_bf16_supported = is_bf16_supported- del major_version, minor_versionelif DEVICE_TYPE == "hip":old_is_bf16_supported = torch.cuda.is_bf16_supporteddiff --git a/unsloth/device_type.py b/unsloth/device_type.pyindex 8c73dc56cb8..fc1532cbcf5 100644--- a/unsloth/device_type.py+++ b/unsloth/device_type.py@@ -167,21 +167,53 @@ else:ALLOW_PREQUANTIZED_MODELS = FalseALLOW_BITSANDBYTES = Falsedel _bnb_probe-# gfx906 (MI50 / Radeon VII / Vega 20): Dynamo/Inductor codegen is broken on this legacy GCN arch-# (ROCm dropped it after 6.3), so compiled graphs crash or miscompile while eager trains fine.-if DEVICE_TYPE == "hip":+++def hip_visible_archs():+ """Guarded per device: one unreadable device must not discard the archs beside it, or a+ gfx10 keeps bf16 and dies in Triton (#7922). Only an unreadable count returns []."""try:- _gcn_arch = torch.cuda.get_device_properties(0).gcnArchName.split(":")[0].strip().lower()+ count = torch.cuda.device_count()except Exception:- _gcn_arch = ""- if _gcn_arch == "gfx906":- os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")- os.environ.setdefault("TORCH_COMPILE_DISABLE", "1")- os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")- print(- "Unsloth: gfx906 (MI50 / Radeon VII) detected - torch.compile disabled "- "(community-maintained legacy GCN path)."- )+ return []+ archs = []+ for i in range(count):+ try:+ archs.append(str(getattr(torch.cuda.get_device_properties(i), "gcnArchName", "")))+ except Exception:+ continue+ return archs+++def cuda_visible_capabilities(with_index = False):+ """(major, minor) of every visible CUDA device, guarded per device like hip_visible_archs:+ process-wide gates (bf16, compile) must hold on the weakest card layers may land on.+ with_index yields (device_index, (major, minor)), since unreadable cards are skipped."""+ try:+ count = torch.cuda.device_count()+ except Exception:+ return []+ caps = []+ for i in range(count):+ try:+ cap = tuple(torch.cuda.get_device_capability(i))+ except Exception:+ continue+ caps.append((i, cap) if with_index else cap)+ return caps+++def cuda_gate_capabilities():+ """Capabilities a process-wide gate should judge: every card that could be read, which+ always includes the current device (the pre-mixed-host rule) when it is readable, so a+ gate is never less strict than before and still sees a known old card beside it."""+ caps = cuda_visible_capabilities()+ if caps:+ return caps+ try:+ return [tuple(torch.cuda.get_device_capability())]+ except Exception:+ return []# Pre-Volta NVIDIA (Maxwell / Pascal, sm < 7.0): Inductor refuses to emit Triton for these@@ -217,11 +249,25 @@ def apply_pre_volta_compile_workaround(return True+# gfx906 (MI50 / Radeon VII / Vega 20): Dynamo/Inductor codegen is broken on this legacy GCN arch+# (ROCm dropped it after 6.3), so compiled graphs crash or miscompile while eager trains fine.+# Any visible gfx906 counts: compile is process-wide and layers may land on it. Same switch as+# pre-Volta, so regions compiled while unsloth_zoo imported are turned off too.+if DEVICE_TYPE == "hip":+ if any(arch.split(":")[0].strip().lower() == "gfx906" for arch in hip_visible_archs()):+ os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")+ os.environ.setdefault("TORCH_COMPILE_DISABLE", "1")+ os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")+ apply_pre_volta_compile_workaround(0) # the live Dynamo config, unless explicitly enabled+ print(+ "Unsloth: gfx906 (MI50 / Radeon VII) detected - torch.compile disabled "+ "(community-maintained legacy GCN path)."+ )++if DEVICE_TYPE == "cuda" and torch.cuda.is_available():- try:- _cuda_major = torch.cuda.get_device_capability()[0]- except Exception:- _cuda_major = None+ # Weakest visible card: compile is process-wide and layers may land on any of them.+ _cuda_major = min((cap[0] for cap in cuda_gate_capabilities()), default = None)if apply_pre_volta_compile_workaround(_cuda_major):print("Unsloth: GPUs older than Volta (compute capability < 7.0) cannot run "@@ -323,22 +369,6 @@ def _default_inductor_cache_dir():return os.path.join(tempfile.gettempdir(), "torchinductor_" + user)-def hip_visible_archs():- """Guarded per device: one unreadable device must not discard the archs beside it, or a- gfx10 keeps bf16 and dies in Triton (#7922). Only an unreadable count returns []."""- try:- count = torch.cuda.device_count()- except Exception:- return []- archs = []- for i in range(count):- try:- archs.append(str(getattr(torch.cuda.get_device_properties(i), "gcnArchName", "")))- except Exception:- continue- return archs--def resolve_hip_gpu_stats_name(gpu_stats):name = str(getattr(gpu_stats, "name", "") or "").strip()name = re.sub(r"\s*\([^)]*\)\s*$", "", name).strip()diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 304c4a6a6b5..b1ebe0ac311 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -167,6 +167,7 @@ from ..device_type import (DEVICE_COUNT,ALLOW_PREQUANTIZED_MODELS,apply_gfx101x_triton_workaround,+ cuda_gate_capabilities,gfx101x_triton_workaround_applied,)from ..import_fixes import UNSLOTH_ENABLE_LOGGING, stale_kernel_hint@@ -3846,6 +3847,38 @@ def _package_available(pkg_name: str) -> bool:return bool(result)+def _cache_device_capability(get_device_capability):+ """Cache per resolved device index: a no-arg call means the current device, so caching the+ call itself returned device 0's answer after torch.cuda.set_device(1)."""+ if getattr(get_device_capability, "_unsloth_per_device_cache", False):+ return get_device_capability++ cache = {}++ @functools.wraps(get_device_capability)+ def cached_get_device_capability(device = None):+ if device is None:+ index = torch.cuda.current_device()+ elif isinstance(device, int):+ index = device+ else:+ # torch.device, not torch.cuda._utils._get_device_index: Dynamo traces it (fullgraph).+ try:+ device = torch.device(device)+ except Exception:+ return get_device_capability(device)+ if device.type != "cuda":+ return get_device_capability(device)+ index = torch.cuda.current_device() if device.index is None else device.index+ capability = cache.get(index)+ if capability is None:+ capability = cache[index] = get_device_capability(index)+ return capability++ cached_get_device_capability._unsloth_per_device_cache = True+ return cached_get_device_capability++SUPPORTS_BFLOAT16 = FalseHAS_FLASH_ATTENTION = FalseHAS_FLASH_ATTENTION_SOFTCAPPING = False@@ -3857,10 +3890,13 @@ if DEVICE_TYPE == "cuda" and not torch.cuda.is_available():HAS_FLASH_ATTENTION_SOFTCAPPING = Falseelif DEVICE_TYPE == "cuda":major_version, minor_version = torch.cuda.get_device_capability()- torch.cuda.get_device_capability = functools.cache(torch.cuda.get_device_capability)+ torch.cuda.get_device_capability = _cache_device_capability(torch.cuda.get_device_capability)+ # SUPPORTS_BFLOAT16 is process-wide, so one pre-Ampere card in the visible set disables it for all.+ _cuda_caps = cuda_gate_capabilities()+ SUPPORTS_BFLOAT16 = bool(_cuda_caps) and all(major >= 8 for major, _ in _cuda_caps)+ del _cuda_capsif major_version >= 8:- SUPPORTS_BFLOAT16 = Trueif _package_available("flash_attn"):# Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl" and for softcapping; see #1437.try:diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex dcd054682df..3cac0f24602 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -3624,13 +3624,21 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"trace.enabled" : False,"""if DEVICE_TYPE == "cuda":- cuda_options = """- "triton.enable_persistent_tma_matmul": torch.cuda.get_device_capability()[0] >= 9,"""+ # TMA only if every visible card is sm90+ (compile options are process-wide); an+ # unreadable card is skipped by the probe, so a short list also means False.+ all_sm90 = (+ "(lambda caps: bool(caps) and len(caps) == torch.cuda.device_count() "+ "and min(c[0] for c in caps) >= 9)"+ '(__import__("unsloth.device_type", fromlist = ["cuda_visible_capabilities"])'+ ".cuda_visible_capabilities())"+ )+ cuda_options = f"""+ "triton.enable_persistent_tma_matmul": {all_sm90},"""# cutlass options were added in PyTorch 2.8.0.if torch_version >= Version("2.8.0"):- cuda_options += """- "cuda.cutlass_epilogue_fusion_enabled": torch.cuda.get_device_capability()[0] >= 9,- "cuda.cutlass_tma_only": torch.cuda.get_device_capability()[0] >= 9,"""+ cuda_options += f"""+ "cuda.cutlass_epilogue_fusion_enabled": {all_sm90},+ "cuda.cutlass_tma_only": {all_sm90},"""cuda_options += """"cuda.compile_opt_level" : "-O2","cuda.enable_cuda_lto" : True,