Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio (AMD): treat a multi-arch arch whose kernel pack is not installed as uncovered (#13231)

Daniel Han5d96b95b2fd5
diff --git a/studio/backend/tests/test_training_gpu_arch_gate.py b/studio/backend/tests/test_training_gpu_arch_gate.py
index bbd0303a395..b420aef0226 100644
--- a/studio/backend/tests/test_training_gpu_arch_gate.py
+++ b/studio/backend/tests/test_training_gpu_arch_gate.py
@@ -254,6 +254,60 @@ class TestFailsOpen:
assert rocm_gpu_ids_without_torch_kernels() == set()
+# torch.cuda.get_arch_list() of AMD's 2.11.0+rocm7.14.0 multi-arch wheel, read from its libtorch_python.
+MULTIARCH = (
+ "gfx1010 gfx1011 gfx1012 gfx1030 gfx1031 gfx1032 gfx1033 gfx1034 gfx1035 gfx1036 "
+ "gfx1100 gfx1101 gfx1102 gfx1103 gfx1200 gfx1201 gfx1150 gfx1151 gfx1152 gfx1153 "
+ "gfx908 gfx90a gfx942 gfx950 gfx1250"
+).split()
+
+
+class TestMultiArchKernelPacks:
+ """#8847: Windows multi-arch torch installed with only the dGPU's device pack."""
+
+ def _torch(self, tmp_path, devices, packs):
+ torch = _fake_torch(devices, arch_list = MULTIARCH)
+ root = tmp_path / "torch"
+ root.mkdir()
+ torch.__file__ = str(root / "__init__.py")
+ if packs is not None:
+ (root / ".kpack").mkdir()
+ for name in packs:
+ (root / ".kpack" / name).write_bytes(b"")
+ return torch
+
+ def test_an_igpu_without_its_pack_is_dropped(self, monkeypatch, no_mask, tmp_path):
+ torch = self._torch(
+ tmp_path, [_props("gfx1036"), _props("gfx1101")], ["torch_gfx1101.kpack"]
+ )
+ _install(monkeypatch, torch)
+ assert rocm_gpu_ids_without_torch_kernels() == {0}
+
+ def test_both_packs_installed_drops_nothing(self, monkeypatch, no_mask, tmp_path):
+ packs = ["torch_gfx1036.kpack", "torch_gfx1101.kpack"]
+ _install(monkeypatch, self._torch(tmp_path, [_props("gfx1036"), _props("gfx1101")], packs))
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ @pytest.mark.parametrize(
+ "packs",
+ [None, [], ["torch_gfx110x.kpack", "torch_gfx1101.kpack"], ["README.txt"]],
+ ids = ["no_pack_dir", "empty_pack_dir", "family_pack", "no_pack_files"],
+ )
+ def test_an_unreadable_pack_layout_keeps_the_arch_list(
+ self, monkeypatch, no_mask, tmp_path, packs
+ ):
+ _install(monkeypatch, self._torch(tmp_path, [_props("gfx1036"), _props("gfx1101")], packs))
+ assert rocm_gpu_ids_without_torch_kernels() == set()
+
+ def test_an_explicit_pick_names_only_the_installed_packs(self, monkeypatch, no_mask, tmp_path):
+ torch = self._torch(
+ tmp_path, [_props("gfx1036"), _props("gfx1101")], ["torch_gfx1101.kpack"]
+ )
+ _install(monkeypatch, torch)
+ with pytest.raises(ValueError, match = r"kernels for gfx1101 only"):
+ _hw_module.reject_gpu_ids_without_torch_kernels([0])
+
+
class TestIdSpace:
"""Results are consumed as physical ids, so ordinals map through the mask."""
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 88824c991a5..c54c589052a 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -5684,6 +5684,31 @@ def estimate_required_model_memory_gb(
# Excludes generic code objects and family labels, which no device reports.
_CONCRETE_GFX_ARCH = re.compile(r"^gfx[0-9][0-9a-f]{2,4}$")
+_KERNEL_PACK_NAME = re.compile(r"^torch_(.+)\.kpack$", re.IGNORECASE)
+
+
+def _torch_kernel_pack_arches(torch) -> Optional[set[str]]:
+ """Arches whose kernel pack is installed, for a torch that ships its kernels as per-arch packs; None when it does not, or when the packs cannot be read.
+
+ AMD's multi-arch wheels (the Windows route) put each arch's kernels in its own `torch[device-gfxNNNN]` package, which drops `torch/.kpack/torch_gfxNNNN.kpack` (libtorch_hip loads `../.kpack/torch_@[email protected]`). get_arch_list() still names every arch the build supports, so a gfx1036 iGPU beside the gfx1101 card the install chose reads as covered and dies at its first kernel launch."""
+ try:
+ pack_dir = Path(torch.__file__).parent / ".kpack"
+ if not pack_dir.is_dir():
+ return None
+ arches: set[str] = set()
+ for entry in pack_dir.iterdir():
+ match = _KERNEL_PACK_NAME.match(entry.name)
+ if not match:
+ continue
+ arch = match.group(1).lower()
+ # A pack naming a family or generic target could cover devices no token names.
+ if not _CONCRETE_GFX_ARCH.match(arch):
+ return None
+ arches.add(arch)
+ return arches or None
+ except Exception as e:
+ logger.debug("torch kernel pack probe failed: %s", e)
+ return None
def _props_gfx_arch(props) -> str:
@@ -5744,6 +5769,9 @@ def rocm_gpu_ids_without_torch_kernels() -> set[int]:
if not tokens:
return set()
supported = set(tokens)
+ packs = _torch_kernel_pack_arches(torch)
+ if packs is not None:
+ supported &= packs
# Under ROCR="1,0" + CUDA="1" the spec reports [1,0] but ordinal 0 is physical 0.
if _rocm_device_ordinal_active() or _rocm_visibility_masks_are_stacked():
@@ -5789,13 +5817,14 @@ def rocm_gpu_ids_without_torch_kernels() -> set[int]:
def _torch_kernel_arch_tokens() -> list[str]:
try:
import torch
- return sorted(
- {
- str(arch).split(":")[0].strip().lower()
- for arch in (torch.cuda.get_arch_list() or ())
- if str(arch).strip()
- }
- )
+
+ tokens = {
+ str(arch).split(":")[0].strip().lower()
+ for arch in (torch.cuda.get_arch_list() or ())
+ if str(arch).strip()
+ }
+ packs = _torch_kernel_pack_arches(torch)
+ return sorted(tokens & packs if packs is not None else tokens)
except Exception:
return []