Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio (AMD): keep export off a GPU PyTorch has no kernels for, like the iGPU (#11946)
oobabooga52f2e77be3e9
diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.pyindex d8e712bb05a..e1742e40648 100644--- a/studio/backend/core/export/orchestrator.py+++ b/studio/backend/core/export/orchestrator.py@@ -506,6 +506,11 @@ class ExportOrchestrator:"hf_token": hf_token,"allow_ambient": allow_ambient,}+ from utils.hardware import get_device, gpu_ids_with_torch_kernels++ # Prevent export from sharding onto GPUs with missing kernels (#11870).+ sub_config["resolved_gpu_ids"] = gpu_ids_with_torch_kernels()+ sub_config["device_backend"] = get_device().valuewith self._lock:# Fresh log buffer so the UI sees only this run's output.diff --git a/studio/backend/core/export/worker.py b/studio/backend/core/export/worker.pyindex dd65cefc364..663961c6e04 100644--- a/studio/backend/core/export/worker.py+++ b/studio/backend/core/export/worker.py@@ -33,6 +33,8 @@ from utils.native_tls import activate_native_tlsactivate_native_tls()+from utils.hardware import apply_gpu_ids+# Gate controlling whether captured stdout/stderr lines are forwarded to the# parent's resp_queue (and on to the export-dialog SSE stream). Closed by default@@ -558,6 +560,8 @@ def run_export_process(*, cmd_queue: Any, resp_queue: Any, config: dict) -> Nonequiet_progress_bars = False,)+ apply_gpu_ids(config.get("resolved_gpu_ids"), backend = config.get("device_backend"))+checkpoint_path = config["checkpoint_path"]# Before the huggingface_hub import: it latches HF_HUB_DISABLE_IMPLICIT_TOKEN into adiff --git a/studio/backend/tests/test_training_gpu_arch_gate.py b/studio/backend/tests/test_training_gpu_arch_gate.pyindex 6787625e5c6..ac8fbaa7880 100644--- a/studio/backend/tests/test_training_gpu_arch_gate.py+++ b/studio/backend/tests/test_training_gpu_arch_gate.py@@ -17,6 +17,7 @@ from utils.hardware.hardware import (DeviceType,apply_gpu_ids,auto_select_gpu_ids,+ gpu_ids_with_torch_kernels,rocm_gpu_ids_without_torch_kernels,)@@ -422,6 +423,66 @@ class TestSelectorWiring:return auto_select_gpu_ids("unsloth/test")[0]+class TestExportWorkerVisibility:+ """Export workers hide GPUs with missing torch kernels (#11870)."""++ def test_the_reported_host_keeps_only_the_discrete_card(self, monkeypatch, no_mask):+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))+ assert gpu_ids_with_torch_kernels() == [0]++ def test_a_covered_host_inherits_the_parent_visibility(self, monkeypatch, no_mask):+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1100")]))+ assert gpu_ids_with_torch_kernels() is None++ def test_a_cuda_host_inherits_the_parent_visibility(self, monkeypatch, no_mask):+ _install(+ monkeypatch,+ _fake_torch([_props(""), _props("")], arch_list = ["sm_80"], vendor = "nvidia"),+ )+ assert gpu_ids_with_torch_kernels() is None++ def test_only_the_uncovered_card_is_hidden(self, monkeypatch, no_mask):+ monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 3)+ _install(+ monkeypatch,+ _fake_torch([_props("gfx1101"), _props("gfx1036"), _props("gfx1100")]),+ )+ assert gpu_ids_with_torch_kernels() == [0, 2]++ def test_a_card_amd_smi_missed_is_kept(self, monkeypatch, no_mask):+ monkeypatch.setattr("utils.hardware.hardware.get_physical_gpu_count", lambda: 1)+ _install(monkeypatch, _fake_torch([_props("gfx1036"), _props("gfx1101")]))+ assert rocm_gpu_ids_without_torch_kernels() == {0}+ assert gpu_ids_with_torch_kernels() == [1]++ def test_an_inherited_mask_keeps_its_physical_ids(self, monkeypatch, no_mask):+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2,3")+ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))+ assert gpu_ids_with_torch_kernels() == [2]++ def test_load_checkpoint_hands_the_covered_ids_to_the_worker(+ self, monkeypatch, no_mask, tmp_path+ ):+ from core.export.orchestrator import ExportOrchestrator++ _install(monkeypatch, _fake_torch([_props("gfx1101"), _props("gfx1036")]))+ backend = ExportOrchestrator()+ spawned = {}+ monkeypatch.setattr(backend, "_ensure_subprocess_alive", lambda: False)+ monkeypatch.setattr(backend, "_spawn_subprocess", spawned.update)+ monkeypatch.setattr(+ backend, "_wait_response", lambda *a, **k: {"success": True, "message": "loaded"}+ )+ assert backend.load_checkpoint(str(tmp_path), load_in_4bit = True)[0]+ assert spawned["resolved_gpu_ids"] == [0]+ assert spawned["device_backend"] == DeviceType.CUDA.value++ with patch.dict(os.environ):+ monkeypatch.setattr(sys, "platform", "win32")+ apply_gpu_ids(spawned["resolved_gpu_ids"], backend = spawned["device_backend"])+ assert os.environ["HIP_VISIBLE_DEVICES"] == "0"++class TestThePinLandsOnTheKeptCard:"""HIP indexes the agents an inherited ROCr mask left, so a raw physical id misses."""diff --git a/studio/backend/utils/hardware/__init__.py b/studio/backend/utils/hardware/__init__.pyindex a69df539fc2..eb7eaea868d 100644--- a/studio/backend/utils/hardware/__init__.py+++ b/studio/backend/utils/hardware/__init__.py@@ -28,6 +28,7 @@ from .hardware import (get_physical_gpu_count,get_visible_gpu_count,get_parent_visible_gpu_ids,+ gpu_ids_with_torch_kernels,resolve_requested_gpu_ids,estimate_fp16_model_size_bytes,estimate_required_model_memory_gb,@@ -108,6 +109,7 @@ __all__ = ["get_physical_gpu_count","get_visible_gpu_count","get_parent_visible_gpu_ids",+ "gpu_ids_with_torch_kernels","resolve_requested_gpu_ids","estimate_fp16_model_size_bytes","estimate_required_model_memory_gb",diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex f25895dc695..2c8f081dca0 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -5380,6 +5380,26 @@ def _props_gfx_arch(props) -> str:return ""+def _torch_ordinal_physical_ids(device_count: int) -> Optional[list[int]]:+ """Map torch ordinals to physical GPU IDs, or return None if uncertain."""+ visible_spec = _get_parent_visible_gpu_spec()+ physical_ids = visible_spec["numeric_ids"]+ if physical_ids is None:+ return None+ if device_count > len(physical_ids):+ # Without a mask, torch ordinals include GPUs AMD SMI may miss (#8792).+ if visible_spec["raw"] is None:+ return list(range(device_count))+ logger.debug(+ "Skipping torch arch gate: %s torch devices but mask %r names %s ids",+ device_count,+ visible_spec["raw"],+ len(physical_ids),+ )+ return None+ return list(physical_ids)++def rocm_gpu_ids_without_torch_kernels() -> set[int]:"""PHYSICAL ids of visible ROCm GPUs the installed torch wheel has no kernels for. Compares what the device PRESENTS, not its silicon, so HSA_OVERRIDE_GFX_VERSION keeps working (#7624). Every uncertainty fails OPEN, the opposite of the bf16 gate: one unreadable device is skipped rather than voiding the probe, which would restore the known-uncovered card and re-break #8792."""try:@@ -5415,27 +5435,11 @@ def rocm_gpu_ids_without_torch_kernels() -> set[int]:logger.debug("Skipping torch arch gate: torch ordinals are renumbered by a mask")return set()- # None means UUID/MIG entries: no ordinal to name a device back to.- visible_spec = _get_parent_visible_gpu_spec()- physical_ids = visible_spec["numeric_ids"]+ device_count = torch.cuda.device_count()+ physical_ids = _torch_ordinal_physical_ids(device_count)if physical_ids is None:return set()- device_count = torch.cuda.device_count()- if device_count > len(physical_ids):- # Unmasked, so ordinal IS physical id and a short list is only amd-smi missing the iGPU #8792 is about: EXTEND, or the gate dies on the reported host.- if visible_spec["raw"] is None:- physical_ids = list(range(device_count))- else:- # A real mask that device_count disagrees with: _device_count_amdsmi runs before init and cannot see the HIP layer, and naming the overflow into the physical namespace collides with ids other ordinals own.- logger.debug(- "Skipping torch arch gate: %s torch devices but mask %r names %s ids",- device_count,- visible_spec["raw"],- len(physical_ids),- )- return set()-unsupported: set[int] = set()unsupported_ordinals = 0readable = 0@@ -5521,6 +5525,28 @@ def reject_gpu_ids_without_torch_kernels(gpu_ids) -> None:)+def gpu_ids_with_torch_kernels() -> Optional[list[int]]:+ """Exclude GPUs with known missing torch kernels; None preserves visibility."""+ uncovered = rocm_gpu_ids_without_torch_kernels()+ if not uncovered:+ return None+ try:+ import torch+ visible = _torch_ordinal_physical_ids(torch.cuda.device_count())+ except Exception as e:+ logger.debug("Could not map torch devices to physical ids: %s", e)+ return None+ covered = [gpu_id for gpu_id in visible or () if gpu_id not in uncovered]+ if not covered:+ return None+ logger.warning(+ "Hiding GPU(s) %s from this worker: the installed PyTorch build has no kernels "+ "for their architecture.",+ sorted(uncovered),+ )+ return covered++def auto_select_gpu_ids(model_name: str,*,