Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio (AMD): keep export off a GPU PyTorch has no kernels for, like the iGPU (#11946)

oobabooga52f2e77be3e9
diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.py
index d8e712bb05a..e1742e40648 100644
--- a/studio/backend/core/export/orchestrator.py
+++ b/studio/backend/core/export/orchestrator.py
@@ -506,6 +506,11 @@ class ExportOrchestrator:
"hf_token": hf_token,
"allow_ambient": allow_ambient,
}
+ from utils.hardware import get_device, gpu_ids_with_torch_kernels
+
+ # Prevent export from sharding onto GPUs with missing kernels (#11870).
+ sub_config["resolved_gpu_ids"] = gpu_ids_with_torch_kernels()
+ sub_config["device_backend"] = get_device().value
with self._lock:
# Fresh log buffer so the UI sees only this run's output.
diff --git a/studio/backend/core/export/worker.py b/studio/backend/core/export/worker.py
index dd65cefc364..663961c6e04 100644
--- a/studio/backend/core/export/worker.py
+++ b/studio/backend/core/export/worker.py
@@ -33,6 +33,8 @@ from utils.native_tls import activate_native_tls
activate_native_tls()
+from utils.hardware import apply_gpu_ids
+
# Gate controlling whether captured stdout/stderr lines are forwarded to the
# parent's resp_queue (and on to the export-dialog SSE stream). Closed by default
@@ -558,6 +560,8 @@ def run_export_process(*, cmd_queue: Any, resp_queue: Any, config: dict) -> None
quiet_progress_bars = False,
)
+ apply_gpu_ids(config.get("resolved_gpu_ids"), backend = config.get("device_backend"))
+
checkpoint_path = config["checkpoint_path"]
# Before the huggingface_hub import: it latches HF_HUB_DISABLE_IMPLICIT_TOKEN into a
diff --git a/studio/backend/tests/test_training_gpu_arch_gate.py b/studio/backend/tests/test_training_gpu_arch_gate.py
index 6787625e5c6..ac8fbaa7880 100644
--- a/studio/backend/tests/test_training_gpu_arch_gate.py
+++ b/studio/backend/tests/test_training_gpu_arch_gate.py
@@ -17,6 +17,7 @@ from utils.hardware.hardware import (
DeviceType,
apply_gpu_ids,
auto_select_gpu_ids,
+ gpu_ids_with_torch_kernels,
rocm_gpu_ids_without_torch_kernels,
)
@@ -422,6 +423,66 @@ class TestSelectorWiring:
return auto_select_gpu_ids("unsloth/test")[0]
+class TestExportWorkerVisibility:
+ """Export workers hide GPUs with missing torch kernels (#11870)."""
+
+ def test_the_reported_host_keeps_only_the_discrete_card(self, monkeypatch, no_mask):
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
+ assert gpu_ids_with_torch_kernels() == [0]
+
+ def test_a_covered_host_inherits_the_parent_visibility(self, monkeypatch, no_mask):
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1100")]))
+ assert gpu_ids_with_torch_kernels() is None
+
+ def test_a_cuda_host_inherits_the_parent_visibility(self, monkeypatch, no_mask):
+ _install(
+ monkeypatch,
+ _fake_torch([_props(""), _props("")], arch_list = ["sm_80"], vendor = "nvidia"),
+ )
+ assert gpu_ids_with_torch_kernels() is None
+
+ def test_only_the_uncovered_card_is_hidden(self, monkeypatch, no_mask):
+ monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 3)
+ _install(
+ monkeypatch,
+ _fake_torch([_props("gfx1101"), _props("gfx1036"), _props("gfx1100")]),
+ )
+ assert gpu_ids_with_torch_kernels() == [0, 2]
+
+ def test_a_card_amd_smi_missed_is_kept(self, monkeypatch, no_mask):
+ monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 1)
+ _install(monkeypatch, _fake_torch([_props("gfx1036"), _props("gfx1101")]))
+ assert rocm_gpu_ids_without_torch_kernels() == {0}
+ assert gpu_ids_with_torch_kernels() == [1]
+
+ def test_an_inherited_mask_keeps_its_physical_ids(self, monkeypatch, no_mask):
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2,3")
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
+ assert gpu_ids_with_torch_kernels() == [2]
+
+ def test_load_checkpoint_hands_the_covered_ids_to_the_worker(
+ self, monkeypatch, no_mask, tmp_path
+ ):
+ from core.export.orchestrator import ExportOrchestrator
+
+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))
+ backend = ExportOrchestrator()
+ spawned = {}
+ monkeypatch.setattr(backend, "_ensure_subprocess_alive", lambda: False)
+ monkeypatch.setattr(backend, "_spawn_subprocess", spawned.update)
+ monkeypatch.setattr(
+ backend, "_wait_response", lambda *a, **k: {"success": True, "message": "loaded"}
+ )
+ assert backend.load_checkpoint(str(tmp_path), load_in_4bit = True)[0]
+ assert spawned["resolved_gpu_ids"] == [0]
+ assert spawned["device_backend"] == DeviceType.CUDA.value
+
+ with patch.dict(os.environ):
+ monkeypatch.setattr(sys, "platform", "win32")
+ apply_gpu_ids(spawned["resolved_gpu_ids"], backend = spawned["device_backend"])
+ assert os.environ["HIP_VISIBLE_DEVICES"] == "0"
+
+
class TestThePinLandsOnTheKeptCard:
"""HIP indexes the agents an inherited ROCr mask left, so a raw physical id misses."""
diff --git a/studio/backend/utils/hardware/__init__.py b/studio/backend/utils/hardware/__init__.py
index a69df539fc2..eb7eaea868d 100644
--- a/studio/backend/utils/hardware/__init__.py
+++ b/studio/backend/utils/hardware/__init__.py
@@ -28,6 +28,7 @@ from .hardware import (
get_physical_gpu_count,
get_visible_gpu_count,
get_parent_visible_gpu_ids,
+ gpu_ids_with_torch_kernels,
resolve_requested_gpu_ids,
estimate_fp16_model_size_bytes,
estimate_required_model_memory_gb,
@@ -108,6 +109,7 @@ __all__ = [
"get_physical_gpu_count",
"get_visible_gpu_count",
"get_parent_visible_gpu_ids",
+ "gpu_ids_with_torch_kernels",
"resolve_requested_gpu_ids",
"estimate_fp16_model_size_bytes",
"estimate_required_model_memory_gb",
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index f25895dc695..2c8f081dca0 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -5380,6 +5380,26 @@ def _props_gfx_arch(props) -> str:
return ""
+def _torch_ordinal_physical_ids(device_count: int) -> Optional[list[int]]:
+ """Map torch ordinals to physical GPU IDs, or return None if uncertain."""
+ visible_spec = _get_parent_visible_gpu_spec()
+ physical_ids = visible_spec["numeric_ids"]
+ if physical_ids is None:
+ return None
+ if device_count > len(physical_ids):
+ # Without a mask, torch ordinals include GPUs AMD SMI may miss (#8792).
+ if visible_spec["raw"] is None:
+ return list(range(device_count))
+ logger.debug(
+ "Skipping torch arch gate: %s torch devices but mask %r names %s ids",
+ device_count,
+ visible_spec["raw"],
+ len(physical_ids),
+ )
+ return None
+ return list(physical_ids)
+
+
def rocm_gpu_ids_without_torch_kernels() -> set[int]:
"""PHYSICAL ids of visible ROCm GPUs the installed torch wheel has no kernels for. Compares what the device PRESENTS, not its silicon, so HSA_OVERRIDE_GFX_VERSION keeps working (#7624). Every uncertainty fails OPEN, the opposite of the bf16 gate: one unreadable device is skipped rather than voiding the probe, which would restore the known-uncovered card and re-break #8792."""
try:
@@ -5415,27 +5435,11 @@ def rocm_gpu_ids_without_torch_kernels() -> set[int]:
logger.debug("Skipping torch arch gate: torch ordinals are renumbered by a mask")
return set()
- # None means UUID/MIG entries: no ordinal to name a device back to.
- visible_spec = _get_parent_visible_gpu_spec()
- physical_ids = visible_spec["numeric_ids"]
+ device_count = torch.cuda.device_count()
+ physical_ids = _torch_ordinal_physical_ids(device_count)
if physical_ids is None:
return set()
- device_count = torch.cuda.device_count()
- if device_count > len(physical_ids):
- # Unmasked, so ordinal IS physical id and a short list is only amd-smi missing the iGPU #8792 is about: EXTEND, or the gate dies on the reported host.
- if visible_spec["raw"] is None:
- physical_ids = list(range(device_count))
- else:
- # A real mask that device_count disagrees with: _device_count_amdsmi runs before init and cannot see the HIP layer, and naming the overflow into the physical namespace collides with ids other ordinals own.
- logger.debug(
- "Skipping torch arch gate: %s torch devices but mask %r names %s ids",
- device_count,
- visible_spec["raw"],
- len(physical_ids),
- )
- return set()
-
unsupported: set[int] = set()
unsupported_ordinals = 0
readable = 0
@@ -5521,6 +5525,28 @@ def reject_gpu_ids_without_torch_kernels(gpu_ids) -> None:
)
+def gpu_ids_with_torch_kernels() -> Optional[list[int]]:
+ """Exclude GPUs with known missing torch kernels; None preserves visibility."""
+ uncovered = rocm_gpu_ids_without_torch_kernels()
+ if not uncovered:
+ return None
+ try:
+ import torch
+ visible = _torch_ordinal_physical_ids(torch.cuda.device_count())
+ except Exception as e:
+ logger.debug("Could not map torch devices to physical ids: %s", e)
+ return None
+ covered = [gpu_id for gpu_id in visible or () if gpu_id not in uncovered]
+ if not covered:
+ return None
+ logger.warning(
+ "Hiding GPU(s) %s from this worker: the installed PyTorch build has no kernels "
+ "for their architecture.",
+ sorted(uncovered),
+ )
+ return covered
+
+
def auto_select_gpu_ids(
model_name: str,
*,