Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(amd): gate training GPU selection on the installed torch build's arch coverage (#8791)

Leo Borcherdingad73e5057708
diff --git a/studio/backend/tests/test_training_gpu_arch_gate.py b/studio/backend/tests/test_training_gpu_arch_gate.py
new file mode 100644
index 00000000000..6787625e5c6
--- /dev/null
+++ b/studio/backend/tests/test_training_gpu_arch_gate.py
@@ -0,0 +1,520 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""ROCm arch gate: selection must not pick a device the wheel lacks kernels for (#7669)."""
+
+from __future__ import annotations
+
+import os
+import sys
+import types
+from unittest.mock import patch
+
+import pytest
+
+import utils.hardware.hardware as _hw_module
+from utils.hardware.hardware import (
+ DeviceType,
+ apply_gpu_ids,
+ auto_select_gpu_ids,
+ rocm_gpu_ids_without_torch_kernels,
+)
+
+# gfx1036 (the Raphael iGPU in #7669) is deliberately absent: that is the bug.
+GFX110X = ["gfx1100", "gfx1101", "gfx1102", "gfx1103"]
+
+# Cleared wholesale so a developer's own ROCm env cannot decide what these assert.
+_MASK_VARS = (
+ "CUDA_VISIBLE_DEVICES",
+ "HIP_VISIBLE_DEVICES",
+ "ROCR_VISIBLE_DEVICES",
+ "GPU_DEVICE_ORDINAL",
+ "ZE_AFFINITY_MASK",
+)
+
+
+def _props(
+ arch = "",
+ *,
+ attr = "gcnArchName",
+ name = "AMD Radeon RX 7700 XT",
+):
+ p = types.SimpleNamespace(name = name)
+ if arch:
+ setattr(p, attr, arch)
+ return p
+
+
+def _fake_torch(
+ devices,
+ *,
+ arch_list = GFX110X,
+ vendor = "amd",
+ available = True,
+ device_count = None,
+):
+ torch = types.ModuleType("torch")
+ if vendor == "amd":
+ torch.version = types.SimpleNamespace(hip = "7.13.99004", cuda = None)
+ torch.__version__ = "2.11.0+rocm7.13.0"
+ elif vendor == "amd_sdk":
+ torch.version = types.SimpleNamespace()
+ torch.__version__ = "2.6.0+rocm6.4"
+ else:
+ torch.version = types.SimpleNamespace(hip = None, cuda = "12.4")
+ torch.__version__ = "2.6.0+cu124"
+
+ def _get_device_properties(ordinal):
+ entry = devices[ordinal]
+ if isinstance(entry, Exception):
+ raise entry
+ return entry
+
+ def _get_arch_list():
+ if isinstance(arch_list, Exception):
+ raise arch_list
+ return list(arch_list)
+
+ torch.cuda = types.SimpleNamespace(
+ is_available = lambda: available,
+ device_count = lambda: len(devices) if device_count is None else device_count,
+ get_arch_list = _get_arch_list,
+ get_device_properties = _get_device_properties,
+ )
+ return torch
+
+
+def no_mask(monkeypatch):
+ for var in _MASK_VARS:
+ monkeypatch.delenv(var, raising = False)
+ monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 2)
+
+
[email protected](autouse = True)
+def _no_device_ordinal(monkeypatch):
+ monkeypatch.delenv("GPU_DEVICE_ORDINAL", raising = False)
+
+
[email protected](autouse = True)
+def _detection_is_declared_not_detected(monkeypatch):
+ """Keeps detect_hardware() from latching IS_ROCM off the fake AMD torch session-wide."""
+ monkeypatch.setattr(_hw_module, "DEVICE", DeviceType.CUDA)
+ monkeypatch.setattr(_hw_module, "IS_ROCM", _hw_module.IS_ROCM)
+
+
+def _install(monkeypatch, torch):
+ monkeypatch.setitem(sys.modules, "torch", torch)
+
+
+class TestTheReportedHost:
+ """#7669: RX 7700 XT (gfx1101) + Raphael iGPU (gfx1036), gfx110X wheel."""
+
+ def test_the_uncovered_igpu_is_dropped(self, monkeypatch, no_mask):
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
+ assert rocm_gpu_ids_without_torch_kernels() == {1}
+
+ def test_a_fully_covered_host_drops_nothing(self, monkeypatch, no_mask):
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1100")]))
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ def test_the_xnack_suffix_does_not_defeat_the_match(self, monkeypatch, no_mask):
+ _install(
+ monkeypatch,
+ _fake_torch([_props("gfx1101:sramecc-:xnack-"), _props("gfx1036:xnack-")]),
+ )
+ assert rocm_gpu_ids_without_torch_kernels() == {1}
+
+
+class TestHsaOverrideKeepsWorking:
+ """The override makes a device PRESENT a supported arch; reading silicon breaks it."""
+
+ def test_a_spoofed_device_is_kept(self, monkeypatch, no_mask):
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1100")]))
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+
+class TestArchSpellings:
+ """Reading only gcnArchName makes the gate a no-op on AMD SDK wheels."""
+
+ @pytest.mark.parametrize("attr", ["gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"])
+ def test_every_spelling_is_read(self, monkeypatch, no_mask, attr):
+ _install(
+ monkeypatch,
+ _fake_torch([_props("gfx1101", attr = attr), _props("gfx1036", attr = attr)]),
+ )
+ assert rocm_gpu_ids_without_torch_kernels() == {1}
+
+
+class TestFailsOpen:
+ """Uncertainty keeps the pre-gate selection: a working machine on CPU is worse."""
+
+ def test_a_cuda_wheel_is_inert(self, monkeypatch, no_mask):
+ # PTX JIT covers archs not listed, so filtering on NVIDIA drops working cards.
+ _install(
+ monkeypatch,
+ _fake_torch([_props("gfx1036")], arch_list = ["sm_80", "sm_90"], vendor = "nvidia"),
+ )
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ def test_an_amd_sdk_wheel_still_gates(self, monkeypatch, no_mask):
+ _install(
+ monkeypatch,
+ _fake_torch([_props("gfx1101"), _props("gfx1036")], vendor = "amd_sdk"),
+ )
+ assert rocm_gpu_ids_without_torch_kernels() == {1}
+
+ def test_every_device_uncovered_keeps_them_all(self, monkeypatch, no_mask):
+ # Dropping all of them hands the caller an empty selection: silent CPU.
+ _install(monkeypatch, _fake_torch([_props("gfx900"), _props("gfx906")]))
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ def test_an_unreadable_arch_list_is_unknown(self, monkeypatch, no_mask):
+ _install(
+ monkeypatch,
+ _fake_torch([_props("gfx1036")], arch_list = RuntimeError("no arch list")),
+ )
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ @pytest.mark.parametrize(
+ "arch_list",
+ [[], ["gfx11-generic"], ["gfx110X"], ["", " "], ["garbage"], ["sm_90"]],
+ ids = ["empty", "generic", "family_label", "blank", "garbage", "cuda_tokens"],
+ )
+ def test_a_non_concrete_arch_list_is_unknown(self, monkeypatch, no_mask, arch_list):
+ _install(monkeypatch, _fake_torch([_props("gfx1101")], arch_list = arch_list))
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ @pytest.mark.parametrize(
+ "arch_list",
+ [
+ ["gfx1100", "gfx11-generic"],
+ ["gfx11-generic", "gfx1100"],
+ ["gfx900", "gfx110X", "gfx1100"],
+ ],
+ ids = ["generic_last", "generic_first", "family_label_between"],
+ )
+ def test_one_non_concrete_token_disables_the_whole_list(self, monkeypatch, no_mask, arch_list):
+ # The concrete subset alone marks gfx1101 uncovered and drops it (#7624).
+ _install(
+ monkeypatch,
+ _fake_torch([_props("gfx1100"), _props("gfx1101")], arch_list = arch_list),
+ )
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ def test_a_device_with_no_readable_arch_is_kept(self, monkeypatch, no_mask):
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("")]))
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ def test_properties_raising_is_not_fatal(self, monkeypatch, no_mask):
+ _install(
+ monkeypatch,
+ _fake_torch([_props("gfx1101"), RuntimeError("cannot describe device")]),
+ )
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ @pytest.mark.parametrize(
+ "third",
+ [_props(""), RuntimeError("cannot describe device")],
+ ids = ["no_arch_attribute", "properties_raise"],
+ )
+ def test_one_unreadable_device_does_not_spare_a_known_uncovered_one(
+ self, monkeypatch, no_mask, third
+ ):
+ # Discarding the whole answer would put the uncovered GPU 1 back (#8792).
+ monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 3)
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036"), third]))
+ assert rocm_gpu_ids_without_torch_kernels() == {1}
+
+ @pytest.mark.parametrize(
+ "second",
+ [_props(""), RuntimeError("cannot describe device")],
+ ids = ["no_arch_attribute", "properties_raise"],
+ )
+ def test_an_unreadable_device_is_not_an_all_uncovered_host(self, monkeypatch, no_mask, second):
+ # The unread device is still selectable, so this is not "every GPU" (#8792).
+ _install(monkeypatch, _fake_torch([_props("gfx1036"), second]))
+ assert rocm_gpu_ids_without_torch_kernels() == {0}
+
+ def test_no_cuda_runtime_is_inert(self, monkeypatch, no_mask):
+ _install(monkeypatch, _fake_torch([_props("gfx1036")], available = False))
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ def test_a_uuid_mask_cannot_be_named_back(self, monkeypatch, no_mask):
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "GPU-DEADBEEFDEADBEEF,0")
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ def test_gpu_device_ordinal_renumbers_the_map_away(self, monkeypatch, no_mask):
+ # No visibility spec reads it, so ordinal 1 is not physical 1.
+ monkeypatch.setenv("GPU_DEVICE_ORDINAL", "1,0")
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ def test_stacked_rocr_and_cuda_masks_renumber_the_map_away(self, monkeypatch, no_mask):
+ # Ordinal 1 is phys1, but the spec sees only ROCr and would name it physical 0.
+ monkeypatch.setattr(sys, "platform", "linux")
+ monkeypatch.setattr("utils.hardware.hardware.IS_ROCM", True)
+ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "2,0,1")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1,2")
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+
+class TestIdSpace:
+ """Results are consumed as physical ids, so ordinals map through the mask."""
+
+ def test_ordinals_map_through_the_mask(self, monkeypatch, no_mask):
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2,3")
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
+ assert rocm_gpu_ids_without_torch_kernels() == {3}
+
+ def _rocr_only(self, monkeypatch, platform):
+ monkeypatch.setattr(sys, "platform", platform)
+ monkeypatch.setattr("utils.hardware.hardware.IS_ROCM", True)
+ monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 2)
+ for var in _MASK_VARS:
+ monkeypatch.delenv(var, raising = False)
+ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "2,3")
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
+
+ def test_a_rocr_mask_maps_the_ordinals_on_linux(self, monkeypatch):
+ self._rocr_only(monkeypatch, "linux")
+ assert rocm_gpu_ids_without_torch_kernels() == {3}
+
+ def test_windows_ignores_a_stray_rocr_mask(self, monkeypatch):
+ # Windows HIP has no ROCr layer, so reading this mask excludes a nonexistent card.
+ self._rocr_only(monkeypatch, "win32")
+ assert rocm_gpu_ids_without_torch_kernels() == {1}
+
+ def test_windows_still_honours_a_hip_mask(self, monkeypatch):
+ self._rocr_only(monkeypatch, "win32")
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2,3")
+ assert rocm_gpu_ids_without_torch_kernels() == {3}
+
+
+class TestTheOrdinalToIdMapMustBeTotal:
+ """device_count() freezes at torch init while the visible spec re-reads the env, so
+ naming an overflow ordinal into the physical namespace collides with a real id."""
+
+ def test_more_ordinals_than_ids_gates_nothing(self, monkeypatch, no_mask):
+ # Ordinal 2 has no id; reusing it as one would drop physical 2, the good card.
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2,0")
+ _install(
+ monkeypatch,
+ _fake_torch(
+ [_props("gfx1101"), _props("gfx1036"), _props("gfx1036")],
+ device_count = 3,
+ ),
+ )
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ def test_the_selector_keeps_the_covered_card(self, monkeypatch, no_mask):
+ # The regression this guards: an empty list is "no GPU", not "inherit".
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2,0")
+ _install(
+ monkeypatch,
+ _fake_torch(
+ [_props("gfx1101"), _props("gfx1036"), _props("gfx1036")],
+ device_count = 3,
+ ),
+ )
+ monkeypatch.setattr(_hw_module, "get_device", lambda: DeviceType.CUDA)
+ monkeypatch.setattr(
+ _hw_module,
+ "get_visible_gpu_utilization",
+ lambda: {
+ "devices": [
+ {"index": 0, "vram_total_gb": 16.0, "vram_used_gb": 1.0},
+ {"index": 2, "vram_total_gb": 32.0, "vram_used_gb": 1.0},
+ ]
+ },
+ )
+ gpu_ids, _ = auto_select_gpu_ids("m", required_override_gb = 8.0)
+ assert gpu_ids == [2]
+
+ def test_amd_smi_undercounting_still_gates(self, monkeypatch, no_mask):
+ # No mask, so the short list is only amd-smi missing the iGPU; bailing out here
+ # would disable the fix on the very host #8792 reports.
+ monkeypatch.setattr(_hw_module, "get_physical_gpu_count", lambda: 1)
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
+ assert rocm_gpu_ids_without_torch_kernels() == {1}
+
+ def test_an_id_named_twice_still_trips_the_all_uncovered_guard(self, monkeypatch, no_mask):
+ # Both ordinals are physical 0, so a deduplicated set would read as a partial drop.
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0,0")
+ _install(
+ monkeypatch,
+ _fake_torch([_props("gfx1036"), _props("gfx1036")]),
+ )
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+
+class TestSelectorWiring:
+ DEVICES = {
+ "devices": [
+ {"index": 0, "vram_total_gb": 12.0, "vram_used_gb": 0.0},
+ {"index": 1, "vram_total_gb": 12.0, "vram_used_gb": 0.0},
+ ]
+ }
+
+ def _run(
+ self,
+ *,
+ uncovered,
+ devices = None,
+ required = (14.0, {"required_gb": 14.0}),
+ ):
+ with (
+ patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),
+ patch(
+ "utils.hardware.hardware.rocm_gpu_ids_without_torch_kernels",
+ return_value = uncovered,
+ ),
+ patch(
+ "utils.hardware.hardware.estimate_required_model_memory_gb",
+ return_value = required,
+ ),
+ patch(
+ "utils.hardware.hardware.get_visible_gpu_utilization",
+ return_value = self.DEVICES if devices is None else devices,
+ ),
+ patch(
+ "utils.hardware.hardware.get_parent_visible_gpu_ids",
+ return_value = [0, 1],
+ ),
+ ):
+ return auto_select_gpu_ids("unsloth/test")
+
+ def test_the_ranked_path_never_offers_the_uncovered_gpu(self):
+ selected, _meta = self._run(uncovered = {1})
+ assert selected == [0]
+
+ def test_the_unestimatable_fallback_is_filtered(self):
+ selected, metadata = self._run(uncovered = {1}, required = (None, {}))
+ assert metadata["selection_mode"] == "fallback_all"
+ assert selected == [0]
+
+ def test_the_no_telemetry_fallback_is_filtered(self):
+ selected, metadata = self._run(uncovered = {1}, devices = {"devices": []})
+ assert metadata["selection_mode"] == "fallback_all"
+ assert selected == [0]
+
+ def test_a_covered_host_selects_exactly_as_before(self):
+ assert self._run(uncovered = set())[0] == self._run_ungated()
+
+ def _run_ungated(self):
+ with (
+ patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),
+ patch(
+ "utils.hardware.hardware.estimate_required_model_memory_gb",
+ return_value = (14.0, {"required_gb": 14.0}),
+ ),
+ patch(
+ "utils.hardware.hardware.get_visible_gpu_utilization",
+ return_value = self.DEVICES,
+ ),
+ patch(
+ "utils.hardware.hardware.get_parent_visible_gpu_ids",
+ return_value = [0, 1],
+ ),
+ ):
+ return auto_select_gpu_ids("unsloth/test")[0]
+
+
+class TestThePinLandsOnTheKeptCard:
+ """HIP indexes the agents an inherited ROCr mask left, so a raw physical id misses."""
+
+ def _rocr(
+ self,
+ monkeypatch,
+ mask,
+ *,
+ devices = ("gfx1101", "gfx1036"),
+ ):
+ monkeypatch.setattr(sys, "platform", "linux")
+ monkeypatch.setattr("utils.hardware.hardware.IS_ROCM", True)
+ monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 2)
+ _install(monkeypatch, _fake_torch([_props(arch) for arch in devices]))
+ os.environ.pop("HIP_VISIBLE_DEVICES", None)
+ os.environ.pop("CUDA_VISIBLE_DEVICES", None)
+ os.environ["ROCR_VISIBLE_DEVICES"] = mask
+
+ def test_a_reordered_mask_pins_the_covered_card(self, monkeypatch):
+ # Physical id 1 verbatim picks ROCr agent 1, physical 0: the card just excluded.
+ with patch.dict(os.environ):
+ self._rocr(monkeypatch, "1,0")
+ assert rocm_gpu_ids_without_torch_kernels() == {0}
+ apply_gpu_ids([1], backend = DeviceType.CUDA.value)
+ assert os.environ["HIP_VISIBLE_DEVICES"] == "0"
+ assert os.environ["CUDA_VISIBLE_DEVICES"] == "0"
+ # Clearing it would hand the child every agent, the gfx1036 among them.
+ assert os.environ["ROCR_VISIBLE_DEVICES"] == "1,0"
+
+ def test_a_nonzero_mask_pins_in_range(self, monkeypatch):
+ # ROCR=2,3 leaves two agents: an untranslated HIP="3" is out of range.
+ with patch.dict(os.environ):
+ self._rocr(monkeypatch, "2,3")
+ assert rocm_gpu_ids_without_torch_kernels() == {3}
+ apply_gpu_ids([2], backend = DeviceType.CUDA.value)
+ assert os.environ["HIP_VISIBLE_DEVICES"] == "0"
+
+ def test_an_identity_mask_is_written_unchanged(self, monkeypatch):
+ with patch.dict(os.environ):
+ self._rocr(monkeypatch, "0,1")
+ apply_gpu_ids([1], backend = DeviceType.CUDA.value)
+ assert os.environ["HIP_VISIBLE_DEVICES"] == "1"
+ assert os.environ["CUDA_VISIBLE_DEVICES"] == "1"
+
+ def test_an_inherited_hip_mask_is_already_relative(self, monkeypatch):
+ # HIP produced the ids, so translating again would map them twice.
+ with patch.dict(os.environ):
+ self._rocr(monkeypatch, "1,0")
+ os.environ["HIP_VISIBLE_DEVICES"] = "1,0"
+ apply_gpu_ids([1], backend = DeviceType.CUDA.value)
+ assert os.environ["HIP_VISIBLE_DEVICES"] == "1"
+
+ def test_an_inherited_cuda_mask_is_already_relative(self, monkeypatch):
+ # rocclr already read this as the HIP mask, so translating again writes "0":
+ # ROCr agent 0, physical 1, the card the parent hid.
+ with patch.dict(os.environ):
+ self._rocr(monkeypatch, "1,0")
+ os.environ["CUDA_VISIBLE_DEVICES"] = "1"
+ apply_gpu_ids([1], backend = DeviceType.CUDA.value)
+ assert os.environ["HIP_VISIBLE_DEVICES"] == "1"
+ assert os.environ["CUDA_VISIBLE_DEVICES"] == "1"
+
+ def test_a_uuid_rocr_mask_is_left_alone(self, monkeypatch):
+ with patch.dict(os.environ):
+ self._rocr(monkeypatch, "GPU-DEADBEEFDEADBEEF,1")
+ apply_gpu_ids([1], backend = DeviceType.CUDA.value)
+ assert os.environ["HIP_VISIBLE_DEVICES"] == "1"
+
+ def test_windows_writes_physical_ids(self, monkeypatch):
+ with patch.dict(os.environ):
+ self._rocr(monkeypatch, "1,0")
+ monkeypatch.setattr(sys, "platform", "win32")
+ apply_gpu_ids([1], backend = DeviceType.CUDA.value)
+ assert os.environ["HIP_VISIBLE_DEVICES"] == "1"
+
+ def test_a_cuda_host_is_untouched(self, monkeypatch):
+ with patch.dict(os.environ):
+ monkeypatch.setattr(sys, "platform", "linux")
+ monkeypatch.setattr("utils.hardware.hardware.IS_ROCM", False)
+ _install(monkeypatch, _fake_torch([_props("gfx1101")], vendor = "nvidia"))
+ for var in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):
+ os.environ.pop(var, None)
+ apply_gpu_ids([1, 3], backend = DeviceType.CUDA.value)
+ assert os.environ["CUDA_VISIBLE_DEVICES"] == "1,3"
+ assert "HIP_VISIBLE_DEVICES" not in os.environ
+
+ def test_a_stale_rocr_var_on_nvidia_does_not_move_the_pin(self, monkeypatch):
+ # Reading the var's presence as "this is ROCm" writes CUDA="0", not GPU 1.
+ with patch.dict(os.environ):
+ monkeypatch.setattr(sys, "platform", "linux")
+ monkeypatch.setattr("utils.hardware.hardware.IS_ROCM", False)
+ _install(monkeypatch, _fake_torch([_props("")], vendor = "nvidia"))
+ os.environ.pop("HIP_VISIBLE_DEVICES", None)
+ os.environ["ROCR_VISIBLE_DEVICES"] = "1,0"
+ apply_gpu_ids([1], backend = DeviceType.CUDA.value)
+ assert os.environ["CUDA_VISIBLE_DEVICES"] == "1"
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index ea64c4200cc..a092b43f995 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -4289,6 +4289,113 @@ def estimate_required_model_memory_gb(
return required_gb, metadata
+# Excludes generic code objects and family labels, which no device reports.
+_CONCRETE_GFX_ARCH = re.compile(r"^gfx[0-9][0-9a-f]{2,4}$")
+
+
+def rocm_gpu_ids_without_torch_kernels() -> set[int]:
+ """PHYSICAL ids of visible ROCm GPUs the installed torch wheel has no kernels for.
+
+ Compares what the device PRESENTS, not its silicon, so ``HSA_OVERRIDE_GFX_VERSION``
+ keeps working (#7624). Every uncertainty fails OPEN, the opposite of the bf16 gate;
+ one unreadable device is skipped, not treated as voiding the probe, which would
+ restore the known-uncovered card and re-break #8792."""
+ try:
+ import torch
+
+ if not (
+ getattr(torch.version, "hip", None) is not None
+ or "rocm" in getattr(torch, "__version__", "").lower()
+ ):
+ return set()
+ if not (hasattr(torch, "cuda") and torch.cuda.is_available()):
+ return set()
+
+ tokens = [
+ token
+ for token in (
+ str(arch).split(":")[0].strip().lower()
+ for arch in (torch.cuda.get_arch_list() or ())
+ )
+ if token
+ ]
+ # A generic code object covers devices no exact token names; a partial read drops a card.
+ unknown = sorted(t for t in tokens if not _CONCRETE_GFX_ARCH.match(t))
+ if unknown:
+ logger.debug("torch arch list carries non-concrete tokens %s; not gating", unknown)
+ return set()
+ if not tokens:
+ return set()
+ supported = set(tokens)
+
+ # Under ROCR="1,0" + CUDA="1" the spec reports [1,0] but ordinal 0 is physical 0.
+ if _rocm_device_ordinal_active() or _rocm_visibility_masks_are_stacked():
+ logger.debug("Skipping torch arch gate: torch ordinals are renumbered by a mask")
+ return set()
+
+ # None means UUID/MIG entries: no ordinal to name a device back to.
+ visible_spec = _get_parent_visible_gpu_spec()
+ physical_ids = visible_spec["numeric_ids"]
+ if physical_ids is None:
+ return set()
+
+ device_count = torch.cuda.device_count()
+ if device_count > len(physical_ids):
+ # Unmasked, so ordinal IS physical id and a short list is only amd-smi missing
+ # the iGPU #8792 is about: EXTEND, or the gate dies on the reported host.
+ if visible_spec["raw"] is None:
+ physical_ids = list(range(device_count))
+ else:
+ # A real mask that device_count disagrees with: _device_count_amdsmi runs
+ # before init and cannot see the HIP layer. Naming the overflow into the
+ # physical namespace collides with ids other ordinals own.
+ logger.debug(
+ "Skipping torch arch gate: %s torch devices but mask %r names %s ids",
+ device_count,
+ visible_spec["raw"],
+ len(physical_ids),
+ )
+ return set()
+
+ unsupported: set[int] = set()
+ unsupported_ordinals = 0
+ readable = 0
+ for ordinal in range(device_count):
+ try:
+ props = torch.cuda.get_device_properties(ordinal)
+ except Exception:
+ continue
+ # gcnArchName alone leaves the map empty on AMD SDK / Radeon wheels.
+ arch = ""
+ for attr in ("gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"):
+ arch = (getattr(props, attr, "") or "").split(":")[0].strip().lower()
+ if arch:
+ break
+ if not arch:
+ logger.debug("Torch arch gate: device %s reports no arch; not gating it", ordinal)
+ continue
+ readable += 1
+ if arch not in supported:
+ unsupported_ordinals += 1
+ unsupported.add(physical_ids[ordinal])
+
+ # Dropping every device would leave the caller no GPU and silently force CPU.
+ # Ordinals, not ids: a mask may name one id twice and the deduplicated set then
+ # reads as fewer devices than were rejected. And readable == device_count, because
+ # an unread device is still selectable, so this is not "every device".
+ if readable and readable == device_count and unsupported_ordinals >= readable:
+ logger.warning(
+ "The installed PyTorch build has no kernels for any GPU on this host "
+ "(built for %s); leaving device selection alone.",
+ sorted(supported),
+ )
+ return set()
+ return unsupported
+ except Exception as e:
+ logger.debug("torch arch coverage probe failed: %s", e)
+ return set()
+
+
def auto_select_gpu_ids(
model_name: str,
*,
@@ -4341,16 +4448,28 @@ def auto_select_gpu_ids(
metadata["selected_gpu_ids"] = None
return None, metadata
+ # Before the free-memory rank AND every fallback, which return the whole visible set.
+ _uncovered = rocm_gpu_ids_without_torch_kernels()
+ if _uncovered:
+ logger.warning(
+ "Excluding GPU(s) %s from automatic selection: the installed PyTorch "
+ "build has no kernels for their architecture.",
+ sorted(_uncovered),
+ )
+
+ def _covered(ids):
+ return [gpu_id for gpu_id in ids if gpu_id not in _uncovered]
+
if required_gb is None:
# Can't estimate size -- use all visible GPUs rather than risk one too small.
- parent_ids = get_parent_visible_gpu_ids()
+ parent_ids = _covered(get_parent_visible_gpu_ids())
metadata["selection_mode"] = "fallback_all"
metadata["selected_gpu_ids"] = parent_ids
return parent_ids, metadata
utilization = get_visible_gpu_utilization()
- devices = utilization.get("devices", [])
- parent_ids = get_parent_visible_gpu_ids()
+ devices = [d for d in utilization.get("devices", []) if d.get("index") not in _uncovered]
+ parent_ids = _covered(get_parent_visible_gpu_ids())
if not devices:
metadata["selection_mode"] = "fallback_all"
@@ -4825,6 +4944,33 @@ def get_visible_gpu_count() -> int:
return _visible_gpu_count
+def _rocr_relative_visibility(value: str) -> Optional[str]:
+ """``value`` (physical ids) as ordinals into the ROCr-filtered agent list, else None.
+
+ HIP indexes what ROCr left: under ``ROCR_VISIBLE_DEVICES=1,0`` a HIP mask of "1" is
+ physical GPU 0. Clearing the ROCr mask instead would re-expose the agents it hides,
+ which HSA can crash on merely enumerating."""
+ if sys.platform == "win32" or "HIP_VISIBLE_DEVICES" in os.environ:
+ return None
+ rocr = os.environ.get("ROCR_VISIBLE_DEVICES")
+ if rocr is None:
+ return None
+ # rocclr reads CUDA_VISIBLE_DEVICES as the HIP-layer mask when HIP_VISIBLE_DEVICES is
+ # empty (rocclr/device/rocm/rocdevice.cpp:391), so the ids already went through it
+ # once and translating again resolves to a GPU the parent deliberately hid.
+ if _rocm_visibility_masks_are_stacked():
+ return None
+ try:
+ agents = [int(token.strip()) for token in rocr.split(",") if token.strip()]
+ wanted = [int(token.strip()) for token in value.split(",") if token.strip()]
+ except ValueError:
+ # UUID/MIG tokens on either side: no ordinal to name them back to.
+ return None
+ if not agents or not wanted or any(gpu_id not in agents for gpu_id in wanted):
+ return None
+ return ",".join(str(agents.index(gpu_id)) for gpu_id in wanted)
+
+
def apply_gpu_ids(gpu_ids, backend: Optional[str] = None) -> None:
if gpu_ids is None:
return
@@ -4897,20 +5043,12 @@ def apply_gpu_ids(gpu_ids, backend: Optional[str] = None) -> None:
logger.info("Applied gpu_ids: ZE_AFFINITY_MASK='%s'", value)
return
- os.environ["CUDA_VISIBLE_DEVICES"] = value
- # Keep ROCm visibility env vars in sync. Workers may call apply_gpu_ids()
- # before detect_hardware() (IS_ROCM still False), so also mirror when the
- # parent set a ROCm visibility var, with a torch.version.hip probe fallback.
- _inherits_rocm_visibility = (
- "HIP_VISIBLE_DEVICES" in os.environ or "ROCR_VISIBLE_DEVICES" in os.environ
- )
- _is_rocm = IS_ROCM or _inherits_rocm_visibility
- if not _is_rocm:
- # torch.version.hip is set on ROCm, None on CUDA; AMD SDK wheels may leave
- # it unset but encode "rocm" in __version__. Broad except: never crash a worker.
+ # From the BUILD: a stale ROCR var on an NVIDIA node would move a CUDA pin elsewhere.
+ _rocm_build = IS_ROCM
+ if not _rocm_build:
try:
import torch as _torch
- _is_rocm = (
+ _rocm_build = (
getattr(_torch.version, "hip", None) is not None
or "rocm" in getattr(_torch, "__version__", "").lower()
)
@@ -4920,15 +5058,21 @@ def apply_gpu_ids(gpu_ids, backend: Optional[str] = None) -> None:
type(e).__name__,
e,
)
+
+ if _rocm_build:
+ _relative = _rocr_relative_visibility(value)
+ if _relative is not None:
+ value = _relative
+
+ os.environ["CUDA_VISIBLE_DEVICES"] = value
+ # Wider than the build check: writing HIP on NVIDIA is inert, a missed mirror is not.
+ _inherits_rocm_visibility = (
+ "HIP_VISIBLE_DEVICES" in os.environ or "ROCR_VISIBLE_DEVICES" in os.environ
+ )
+ _is_rocm = _rocm_build or _inherits_rocm_visibility
if _is_rocm:
os.environ["HIP_VISIBLE_DEVICES"] = value
- # ROCR_VISIBLE_DEVICES operates at the HSA agent level and uses
- # different indexing semantics to HIP_VISIBLE_DEVICES. Setting it
- # to a physical GPU index breaks multi-GPU ROCm systems where the
- # parent already set ROCR_VISIBLE_DEVICES (e.g. "0,1"): narrowing
- # to "1" causes torch.cuda.is_available() to return False in the
- # worker subprocess. HIP_VISIBLE_DEVICES is sufficient for GPU
- # selection on ROCm -- leave ROCR_VISIBLE_DEVICES inherited.
+ # ROCR stays inherited: narrowing it too made torch.cuda.is_available() False.
_visible_gpu_count = None
if _is_rocm:
logger.info("Applied gpu_ids: CUDA_VISIBLE_DEVICES='%s' (rocm)", value)