Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio (AMD/ROCm): refuse a GPU the installed PyTorch has no kernels for (#11571)
Leo Borcherding4c17ada8db2b
diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.pyindex fdb0ed1831e..3828c7981b0 100644--- a/studio/backend/tests/test_gpu_selection.py+++ b/studio/backend/tests/test_gpu_selection.py@@ -901,6 +901,66 @@ class TestGpuAutoSelection(_GpuCacheResetMixin, unittest.TestCase):self.assertIsNone(metadata["selected_gpu_ids"])+class TestExplicitPickWithoutTorchKernels(unittest.TestCase):+ def test_an_uncovered_card_is_rejected_with_the_arch_list(self):+ with (+ patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),+ patch("utils.hardware.hardware.resolve_requested_gpu_ids", return_value = [1]),+ patch("utils.hardware.hardware.rocm_gpu_ids_without_torch_kernels", return_value = {1}),+ patch(+ "utils.hardware.hardware._describe_rocm_gpus",+ return_value = ["GPU 1 (AMD Radeon RX 5700 XT, gfx1010)"],+ ),+ patch(+ "utils.hardware.hardware._torch_kernel_arch_tokens",+ return_value = ["gfx1030", "gfx1034"],+ ),+ patch("utils.hardware.hardware.auto_select_gpu_ids") as mock_auto_select,+ ):+ with self.assertRaisesRegex(+ ValueError,+ r"GPU 1 \(AMD Radeon RX 5700 XT, gfx1010\) cannot run the PyTorch build this Unsloth Studio installed, "+ r"which has kernels for gfx1030, gfx1034 only",+ ):+ prepare_gpu_selection([1], model_name = "unsloth/test")+ mock_auto_select.assert_not_called()++ def test_a_covered_card_beside_an_uncovered_one_is_accepted(self):+ with (+ patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),+ patch("utils.hardware.hardware.resolve_requested_gpu_ids", return_value = [0]),+ patch("utils.hardware.hardware.rocm_gpu_ids_without_torch_kernels", return_value = {1}),+ patch("utils.hardware.hardware.auto_select_gpu_ids") as mock_auto_select,+ ):+ selected, metadata = prepare_gpu_selection([0], model_name = "unsloth/test")+ self.assertEqual(selected, [0])+ self.assertEqual(metadata["selection_mode"], "explicit")+ mock_auto_select.assert_not_called()++ def test_the_refusal_names_the_arch_on_wheels_without_gcnArchName(self):+ props = SimpleNamespace(name = "AMD Radeon RX 5700 XT", gcnArchName = "", gfx_arch_name = "gfx1010:xnack-")+ with (+ patch("torch.cuda.device_count", return_value = 2),+ patch("torch.cuda.get_device_properties", return_value = props),+ patch(+ "utils.hardware.hardware._get_parent_visible_gpu_spec",+ return_value = {"numeric_ids": [0, 1], "raw": None},+ ),+ ):+ self.assertEqual(+ _hw_module._describe_rocm_gpus([1]), ["GPU 1 (AMD Radeon RX 5700 XT, gfx1010)"]+ )++ def test_a_host_where_every_card_is_covered_is_untouched(self):+ with (+ patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),+ patch("utils.hardware.hardware.resolve_requested_gpu_ids", return_value = [1]),+ patch("utils.hardware.hardware.rocm_gpu_ids_without_torch_kernels", return_value = set()),+ ):+ selected, _ = prepare_gpu_selection([1], model_name = "unsloth/test")+ self.assertEqual(selected, [1])++class TestPreSpawnGpuResolution(_GpuCacheResetMixin, unittest.TestCase):def test_training_backend_resolves_explicit_gpu_ids_before_spawn(self):backend = TrainingBackend()diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 7a0e310e648..3bedb9243b7 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -5324,6 +5324,15 @@ def estimate_required_model_memory_gb(_CONCRETE_GFX_ARCH = re.compile(r"^gfx[0-9][0-9a-f]{2,4}$")+def _props_gfx_arch(props) -> str:+ # gcnArchName alone leaves the map empty on AMD SDK / Radeon wheels.+ for attr in ("gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"):+ arch = (getattr(props, attr, "") or "").split(":")[0].strip().lower()+ if arch:+ return arch+ return ""++def rocm_gpu_ids_without_torch_kernels() -> set[int]:"""PHYSICAL ids of visible ROCm GPUs the installed torch wheel has no kernels for. Compares what the device PRESENTS, not its silicon, so HSA_OVERRIDE_GFX_VERSION keeps working (#7624). Every uncertainty fails OPEN, the opposite of the bf16 gate: one unreadable device is skipped rather than voiding the probe, which would restore the known-uncovered card and re-break #8792."""try:@@ -5388,12 +5397,7 @@ def rocm_gpu_ids_without_torch_kernels() -> set[int]:props = torch.cuda.get_device_properties(ordinal)except Exception:continue- # gcnArchName alone leaves the map empty on AMD SDK / Radeon wheels.- arch = ""- for attr in ("gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"):- arch = (getattr(props, attr, "") or "").split(":")[0].strip().lower()- if arch:- break+ arch = _props_gfx_arch(props)if not arch:logger.debug("Torch arch gate: device %s reports no arch; not gating it", ordinal)continue@@ -5416,6 +5420,60 @@ def rocm_gpu_ids_without_torch_kernels() -> set[int]:return set()+def _torch_kernel_arch_tokens() -> list[str]:+ try:+ import torch+ return sorted(+ {+ str(arch).split(":")[0].strip().lower()+ for arch in (torch.cuda.get_arch_list() or ())+ if str(arch).strip()+ }+ )+ except Exception:+ return []+++def _describe_rocm_gpus(gpu_ids) -> list[str]:+ """Best-effort labels keyed by PHYSICAL id, for an error message only; never a gate."""+ wanted = {int(gpu_id) for gpu_id in gpu_ids}+ labels: Dict[int, str] = {}+ try:+ import torch++ count = torch.cuda.device_count()+ physical_ids = _get_parent_visible_gpu_spec()["numeric_ids"]+ if physical_ids is None or count > len(physical_ids):+ physical_ids = list(range(count))+ for ordinal, physical in enumerate(physical_ids[:count]):+ if physical not in wanted:+ continue+ props = torch.cuda.get_device_properties(ordinal)+ arch = _props_gfx_arch(props)+ detail = ", ".join(+ part for part in (str(getattr(props, "name", "") or ""), arch) if part+ )+ labels[physical] = f"GPU {physical} ({detail})" if detail else f"GPU {physical}"+ except Exception as e:+ logger.debug("Could not describe GPUs %s: %s", sorted(wanted), e)+ return [labels.get(gpu_id, f"GPU {gpu_id}") for gpu_id in sorted(wanted)]+++def reject_gpu_ids_without_torch_kernels(gpu_ids) -> None:+ """Explicit picks bypass the #8792 auto-select skip; without this the worker dies with hipErrorInvalidImage."""+ uncovered = sorted(+ set(int(gpu_id) for gpu_id in gpu_ids) & rocm_gpu_ids_without_torch_kernels()+ )+ if not uncovered:+ return+ built_for = ", ".join(_torch_kernel_arch_tokens()) or "other GPU architectures"+ raise ValueError(+ f"{', '.join(_describe_rocm_gpus(uncovered))} cannot run the PyTorch build this Unsloth Studio installed, "+ f"which has kernels for {built_for} only. Pick another GPU, or reinstall Unsloth "+ f"Studio for that card."+ )++def auto_select_gpu_ids(model_name: str,*,@@ -5605,6 +5663,7 @@ def prepare_gpu_selection(if gpu_ids:resolved = resolve_requested_gpu_ids(gpu_ids)+ reject_gpu_ids_without_torch_kernels(resolved)metadata = {"selection_mode": "explicit","selected_gpu_ids": resolved,