Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio (AMD/ROCm): refuse a GPU the installed PyTorch has no kernels for (#11571)

Leo Borcherding4c17ada8db2b
diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.py
index fdb0ed1831e..3828c7981b0 100644
--- a/studio/backend/tests/test_gpu_selection.py
+++ b/studio/backend/tests/test_gpu_selection.py
@@ -901,6 +901,66 @@ class TestGpuAutoSelection(_GpuCacheResetMixin, unittest.TestCase):
self.assertIsNone(metadata["selected_gpu_ids"])
+class TestExplicitPickWithoutTorchKernels(unittest.TestCase):
+ def test_an_uncovered_card_is_rejected_with_the_arch_list(self):
+ with (
+ patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),
+ patch("utils.hardware.hardware.resolve_requested_gpu_ids", return_value = [1]),
+ patch("utils.hardware.hardware.rocm_gpu_ids_without_torch_kernels", return_value = {1}),
+ patch(
+ "utils.hardware.hardware._describe_rocm_gpus",
+ return_value = ["GPU 1 (AMD Radeon RX 5700 XT, gfx1010)"],
+ ),
+ patch(
+ "utils.hardware.hardware._torch_kernel_arch_tokens",
+ return_value = ["gfx1030", "gfx1034"],
+ ),
+ patch("utils.hardware.hardware.auto_select_gpu_ids") as mock_auto_select,
+ ):
+ with self.assertRaisesRegex(
+ ValueError,
+ r"GPU 1 \(AMD Radeon RX 5700 XT, gfx1010\) cannot run the PyTorch build this Unsloth Studio installed, "
+ r"which has kernels for gfx1030, gfx1034 only",
+ ):
+ prepare_gpu_selection([1], model_name = "unsloth/test")
+ mock_auto_select.assert_not_called()
+
+ def test_a_covered_card_beside_an_uncovered_one_is_accepted(self):
+ with (
+ patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),
+ patch("utils.hardware.hardware.resolve_requested_gpu_ids", return_value = [0]),
+ patch("utils.hardware.hardware.rocm_gpu_ids_without_torch_kernels", return_value = {1}),
+ patch("utils.hardware.hardware.auto_select_gpu_ids") as mock_auto_select,
+ ):
+ selected, metadata = prepare_gpu_selection([0], model_name = "unsloth/test")
+ self.assertEqual(selected, [0])
+ self.assertEqual(metadata["selection_mode"], "explicit")
+ mock_auto_select.assert_not_called()
+
+ def test_the_refusal_names_the_arch_on_wheels_without_gcnArchName(self):
+ props = SimpleNamespace(name = "AMD Radeon RX 5700 XT", gcnArchName = "", gfx_arch_name = "gfx1010:xnack-")
+ with (
+ patch("torch.cuda.device_count", return_value = 2),
+ patch("torch.cuda.get_device_properties", return_value = props),
+ patch(
+ "utils.hardware.hardware._get_parent_visible_gpu_spec",
+ return_value = {"numeric_ids": [0, 1], "raw": None},
+ ),
+ ):
+ self.assertEqual(
+ _hw_module._describe_rocm_gpus([1]), ["GPU 1 (AMD Radeon RX 5700 XT, gfx1010)"]
+ )
+
+ def test_a_host_where_every_card_is_covered_is_untouched(self):
+ with (
+ patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),
+ patch("utils.hardware.hardware.resolve_requested_gpu_ids", return_value = [1]),
+ patch("utils.hardware.hardware.rocm_gpu_ids_without_torch_kernels", return_value = set()),
+ ):
+ selected, _ = prepare_gpu_selection([1], model_name = "unsloth/test")
+ self.assertEqual(selected, [1])
+
+
class TestPreSpawnGpuResolution(_GpuCacheResetMixin, unittest.TestCase):
def test_training_backend_resolves_explicit_gpu_ids_before_spawn(self):
backend = TrainingBackend()
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 7a0e310e648..3bedb9243b7 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -5324,6 +5324,15 @@ def estimate_required_model_memory_gb(
_CONCRETE_GFX_ARCH = re.compile(r"^gfx[0-9][0-9a-f]{2,4}$")
+def _props_gfx_arch(props) -> str:
+ # gcnArchName alone leaves the map empty on AMD SDK / Radeon wheels.
+ for attr in ("gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"):
+ arch = (getattr(props, attr, "") or "").split(":")[0].strip().lower()
+ if arch:
+ return arch
+ return ""
+
+
def rocm_gpu_ids_without_torch_kernels() -> set[int]:
"""PHYSICAL ids of visible ROCm GPUs the installed torch wheel has no kernels for. Compares what the device PRESENTS, not its silicon, so HSA_OVERRIDE_GFX_VERSION keeps working (#7624). Every uncertainty fails OPEN, the opposite of the bf16 gate: one unreadable device is skipped rather than voiding the probe, which would restore the known-uncovered card and re-break #8792."""
try:
@@ -5388,12 +5397,7 @@ def rocm_gpu_ids_without_torch_kernels() -> set[int]:
props = torch.cuda.get_device_properties(ordinal)
except Exception:
continue
- # gcnArchName alone leaves the map empty on AMD SDK / Radeon wheels.
- arch = ""
- for attr in ("gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"):
- arch = (getattr(props, attr, "") or "").split(":")[0].strip().lower()
- if arch:
- break
+ arch = _props_gfx_arch(props)
if not arch:
logger.debug("Torch arch gate: device %s reports no arch; not gating it", ordinal)
continue
@@ -5416,6 +5420,60 @@ def rocm_gpu_ids_without_torch_kernels() -> set[int]:
return set()
+def _torch_kernel_arch_tokens() -> list[str]:
+ try:
+ import torch
+ return sorted(
+ {
+ str(arch).split(":")[0].strip().lower()
+ for arch in (torch.cuda.get_arch_list() or ())
+ if str(arch).strip()
+ }
+ )
+ except Exception:
+ return []
+
+
+def _describe_rocm_gpus(gpu_ids) -> list[str]:
+ """Best-effort labels keyed by PHYSICAL id, for an error message only; never a gate."""
+ wanted = {int(gpu_id) for gpu_id in gpu_ids}
+ labels: Dict[int, str] = {}
+ try:
+ import torch
+
+ count = torch.cuda.device_count()
+ physical_ids = _get_parent_visible_gpu_spec()["numeric_ids"]
+ if physical_ids is None or count > len(physical_ids):
+ physical_ids = list(range(count))
+ for ordinal, physical in enumerate(physical_ids[:count]):
+ if physical not in wanted:
+ continue
+ props = torch.cuda.get_device_properties(ordinal)
+ arch = _props_gfx_arch(props)
+ detail = ", ".join(
+ part for part in (str(getattr(props, "name", "") or ""), arch) if part
+ )
+ labels[physical] = f"GPU {physical} ({detail})" if detail else f"GPU {physical}"
+ except Exception as e:
+ logger.debug("Could not describe GPUs %s: %s", sorted(wanted), e)
+ return [labels.get(gpu_id, f"GPU {gpu_id}") for gpu_id in sorted(wanted)]
+
+
+def reject_gpu_ids_without_torch_kernels(gpu_ids) -> None:
+ """Explicit picks bypass the #8792 auto-select skip; without this the worker dies with hipErrorInvalidImage."""
+ uncovered = sorted(
+ set(int(gpu_id) for gpu_id in gpu_ids) & rocm_gpu_ids_without_torch_kernels()
+ )
+ if not uncovered:
+ return
+ built_for = ", ".join(_torch_kernel_arch_tokens()) or "other GPU architectures"
+ raise ValueError(
+ f"{', '.join(_describe_rocm_gpus(uncovered))} cannot run the PyTorch build this Unsloth Studio installed, "
+ f"which has kernels for {built_for} only. Pick another GPU, or reinstall Unsloth "
+ f"Studio for that card."
+ )
+
+
def auto_select_gpu_ids(
model_name: str,
*,
@@ -5605,6 +5663,7 @@ def prepare_gpu_selection(
if gpu_ids:
resolved = resolve_requested_gpu_ids(gpu_ids)
+ reject_gpu_ids_without_torch_kernels(resolved)
metadata = {
"selection_mode": "explicit",
"selected_gpu_ids": resolved,