Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio: AMD: skip LFM3's Triton conv and flex attention on ROCm cards Triton cannot build for (#16974)
Leo Borcherding6c525184197a
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex fc81f07b920..958d61b98bf 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -40,6 +40,7 @@ from core.inference.runtime_context import ()from core.inference.message_content import content_to_text, named_turnfrom core.inference.model_ids import is_deepseek_ocr+from core.inference.lfm3_conv_fallback import disable_lfm3_triton_conv_where_unsupportedfrom core.inference.chat_eos import (chat_eos_repair,resolve_chat_turn_end_eos_ids_using,@@ -1377,6 +1378,9 @@ class InferenceBackend:)raise_if_offloaded(self.models[model_name]["model"], device_map, "Inference")+ disable_lfm3_triton_conv_where_unsupported(+ self.models[model_name]["model"], torch, logger+ )self.models[model_name]["context_length"] = runtime_context_length(self.models[model_name].get("model"),max_seq_length,@@ -1566,6 +1570,9 @@ class InferenceBackend:)raise_if_offloaded(self.models[model_name]["model"], device_map, "Inference")+ disable_lfm3_triton_conv_where_unsupported(+ self.models[model_name]["model"], torch, logger+ )self.models[model_name]["context_length"] = runtime_context_length(self.models[model_name].get("model"),max_seq_length,diff --git a/studio/backend/core/inference/lfm3_conv_fallback.py b/studio/backend/core/inference/lfm3_conv_fallback.pynew file mode 100644index 00000000000..f030f7fb1db--- /dev/null+++ b/studio/backend/core/inference/lfm3_conv_fallback.py@@ -0,0 +1,75 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""LFM3's remote Triton conv dies with ``PassManager::run failed`` where Triton cannot build for the card (gfx101x on Triton 3.9); use its torch path there."""++from __future__ import annotations++import sys+from typing import Any, Callable, Optional++from core.inference.rocm_bf16 import _device_gfx_arch, is_rocm_torch++_GATE = "can_use_lfm3_conv"+++def _core_probe() -> Callable[[int], bool]:+ try:+ from unsloth.models._utils import triton_runs_on_device+ except Exception: # noqa: BLE001 -- no probe: keep the remote code's own choice+ return lambda index: True+ return triton_runs_on_device+++def _model_cuda_indices(model: Any) -> set[int]:+ indices = set()+ try:+ for param in model.parameters():+ device = param.device+ if device.type == "cuda":+ indices.add(device.index or 0)+ except Exception: # noqa: BLE001 -- no parameters to read: nothing on a GPU to gate+ pass+ return indices+++def _guarded(original: Any, runs_on: Callable[[int], bool]):+ def gate(hidden_states, weight, *args, **kwargs):+ # Module-wide gate: a model on a card Triton does support keeps the kernel.+ device = getattr(hidden_states, "device", None)+ if getattr(device, "type", None) != "cuda" or not runs_on(device.index or 0):+ return False+ return original(hidden_states, weight, *args, **kwargs)++ gate._unsloth_triton_guard = True+ return gate+++def disable_lfm3_triton_conv_where_unsupported(+ model: Any,+ torch: Any,+ log: Any = None,+ runs_on: Optional[Callable[[int], bool]] = None,+) -> list[str]:+ """Make every loaded remote-code module's ``can_use_lfm3_conv`` refuse tensors on a ROCm card Triton cannot run on."""+ if not is_rocm_torch(torch):+ return []+ runs_on = runs_on or _core_probe()+ failing = sorted(index for index in _model_cuda_indices(model) if not runs_on(index))+ if not failing:+ return []+ patched = []+ for name, module in list(sys.modules.items()):+ if not name.startswith("transformers_modules.") or module is None:+ continue+ original = getattr(module, _GATE, None)+ if callable(original) and not getattr(original, "_unsloth_triton_guard", False):+ setattr(module, _GATE, _guarded(original, runs_on))+ patched.append(name)+ if patched and log is not None:+ archs = ", ".join(_device_gfx_arch(torch, index) or f"cuda:{index}" for index in failing)+ log.info(+ f"LFM3 Triton conv disabled on {archs} (Triton cannot build for it), "+ f"using the torch conv: {', '.join(patched)}"+ )+ return patcheddiff --git a/studio/backend/tests/test_lfm3_conv_fallback.py b/studio/backend/tests/test_lfm3_conv_fallback.pynew file mode 100644index 00000000000..87b91517250--- /dev/null+++ b/studio/backend/tests/test_lfm3_conv_fallback.py@@ -0,0 +1,149 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""LFM3's remote Triton conv is turned off only on a ROCm card Triton cannot run on."""++from __future__ import annotations++import sys+import types++import pytest++from core.inference import lfm3_conv_fallback as fallback+from core.inference.lfm3_conv_fallback import disable_lfm3_triton_conv_where_unsupported++LFM3 = "transformers_modules.LiquidAI.LFM3_hyphen_1_dot_7B.abc.modeling_lfm3"+OMNI_TEXT = "transformers_modules.LiquidAI.LFM3_hyphen_Omni_hyphen_2B.def.modeling_lfm3"+OTHER = "transformers_modules.someone.other.ghi.modeling_other"++_KERNEL = object()+++def _original(hidden_states, weight):+ return _KERNEL+++def _fake_torch(*, hip, arch = "gfx1010"):+ torch = types.ModuleType("torch")+ torch.__version__ = "2.11.0+rocm7.14.0" if hip else "2.11.0+cu130"+ torch.version = types.SimpleNamespace(hip = "7.14" if hip else None)+ torch.cuda = types.SimpleNamespace(+ current_device = lambda: 0,+ get_device_properties = lambda index = None: types.SimpleNamespace(gcnArchName = arch),+ )+ return torch+++def _model(device_type, index = 0):+ param = types.SimpleNamespace(device = types.SimpleNamespace(type = device_type, index = index))+ return types.SimpleNamespace(parameters = lambda: iter([param]))+++def _tensor(index, device_type = "cuda"):+ return types.SimpleNamespace(device = types.SimpleNamespace(type = device_type, index = index))+++def _probe(results):+ calls = []++ def runs_on(index):+ calls.append(index)+ return results[index]++ runs_on.calls = calls+ return runs_on+++def remote_modules(monkeypatch):+ mods = {}+ for name in (LFM3, OMNI_TEXT):+ mod = types.ModuleType(name)+ mod.can_use_lfm3_conv = _original+ mods[name] = mod+ other = types.ModuleType(OTHER)+ other.something_else = _original+ mods[OTHER] = other+ for name, mod in mods.items():+ monkeypatch.setitem(sys.modules, name, mod)+ return mods++[email protected]("arch", ["gfx1010", "gfx1030", "gfx1036"])+def test_gate_patched_where_triton_cannot_run(remote_modules, arch):+ log = types.SimpleNamespace(lines = [])+ log.info = log.lines.append+ patched = disable_lfm3_triton_conv_where_unsupported(+ _model("cuda"), _fake_torch(hip = True, arch = arch), log, _probe({0: False})+ )+ assert sorted(patched) == sorted([LFM3, OMNI_TEXT])+ for name in (LFM3, OMNI_TEXT):+ assert remote_modules[name].can_use_lfm3_conv(_tensor(0), None) is False+ assert not hasattr(remote_modules[OTHER], "can_use_lfm3_conv")+ assert len(log.lines) == 1 and arch in log.lines[0]+++ "hip, device",+ [+ (+ True,+ "cuda",+ ), # Triton runs on the card (gfx1010 on ROCm Triton 3.5, gfx1100, gfx1151 ...)+ (True, "cpu"),+ (False, "cuda"),+ (False, "mps"),+ (False, "cpu"),+ ],+)+def test_gate_untouched_elsewhere(remote_modules, hip, device):+ probe = _probe({0: True})+ patched = disable_lfm3_triton_conv_where_unsupported(+ _model(device), _fake_torch(hip = hip), None, probe+ )+ assert patched == []+ if not hip:+ assert probe.calls == []+ for name in (LFM3, OMNI_TEXT):+ assert remote_modules[name].can_use_lfm3_conv is _original+++def test_model_without_gate_is_a_noop(monkeypatch):+ for name in [n for n in sys.modules if n.startswith("transformers_modules.")]:+ monkeypatch.delitem(sys.modules, name)+ monkeypatch.setitem(sys.modules, OTHER, types.ModuleType(OTHER))+ assert (+ disable_lfm3_triton_conv_where_unsupported(+ _model("cuda"), _fake_torch(hip = True), None, _probe({0: False})+ )+ == []+ )+++def test_unreadable_model_is_a_noop(remote_modules):+ assert (+ disable_lfm3_triton_conv_where_unsupported(+ object(), _fake_torch(hip = True), None, _probe({0: False})+ )+ == []+ )+ assert remote_modules[LFM3].can_use_lfm3_conv is _original+++def test_missing_core_probe_keeps_the_kernel(remote_modules, monkeypatch):+ monkeypatch.setitem(sys.modules, "unsloth.models._utils", None)+ assert fallback._core_probe()(0) is True+ assert disable_lfm3_triton_conv_where_unsupported(_model("cuda"), _fake_torch(hip = True)) == []+++def test_a_supported_card_in_the_same_process_keeps_the_kernel(remote_modules):+ probe = _probe({0: False, 1: True})+ torch = _fake_torch(hip = True)+ disable_lfm3_triton_conv_where_unsupported(_model("cuda", 0), torch, None, probe)+ assert disable_lfm3_triton_conv_where_unsupported(_model("cuda", 0), torch, None, probe) == []+ for name in (LFM3, OMNI_TEXT):+ gate = remote_modules[name].can_use_lfm3_conv+ assert gate(_tensor(0), None) is False+ assert gate(_tensor(1), None) is _KERNEL+ assert gate(_tensor(None, "cpu"), None) is Falsediff --git a/tests/test_flex_attention_needs_ampere.py b/tests/test_flex_attention_needs_ampere.pyindex 33206a34f5d..db55ffcfe31 100644--- a/tests/test_flex_attention_needs_ampere.py+++ b/tests/test_flex_attention_needs_ampere.py@@ -70,7 +70,7 @@ def test_rocm_is_not_judged_by_a_cuda_capability():"""`get_device_capability` answers on ROCm too, with numbers that are notCUDA's, so reading them would disable flex on AMD for no reason."""cuda, hip = _cuda([(7, 5)], hip = "6.2.0")- with cuda, hip:+ with cuda, hip, mock.patch.object(U, "triton_runs_on_device", lambda index = 0: True):assert U._flex_attention_gpu_is_supported() is Truediff --git a/tests/test_triton_runs_on_device.py b/tests/test_triton_runs_on_device.pynew file mode 100644index 00000000000..d01594042ff--- /dev/null+++ b/tests/test_triton_runs_on_device.py@@ -0,0 +1,103 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Flex attention is dropped on a ROCm card this Triton cannot build for (gfx101x on Triton 3.9)."""++import sys+import types+from unittest import mock++import pytest++import unsloth.models._utils as U++[email protected](autouse = True)+def _fresh_cache():+ # macOS / Windows runners have no Triton; the probe itself is stubbed per test.+ triton = sys.modules.get("triton") or types.ModuleType("triton")+ with (+ mock.patch.dict(U._TRITON_RUNS_ON_DEVICE, clear = True),+ mock.patch.dict(sys.modules, {"triton": triton}),+ ):+ yield+++def _hip(+ on,+ count = 1,+ current = 0,+):+ return mock.patch.multiple(+ U.torch.cuda,+ is_available = lambda: count > 0,+ device_count = lambda: count,+ current_device = lambda: current,+ ), mock.patch.object(U.torch.version, "hip", "7.14" if on else None, create = True)+++def _probe(results):+ calls = []++ def probe(index):+ calls.append(index)+ if isinstance(results[index], Exception):+ raise results[index]+ return results[index]++ probe.calls = calls+ return mock.patch.object(U, "_triton_probe", probe), calls+++def test_cuda_builds_never_probe():+ cuda, hip = _hip(False)+ patch, calls = _probe({0: RuntimeError("unreachable")})+ with cuda, hip, patch, mock.patch.object(U.torch, "__version__", "2.14.1+cu130"):+ assert U.triton_runs_on_device(0) is True+ assert calls == []+++def test_a_target_triton_rejects_is_refused_once():+ cuda, hip = _hip(True)+ patch, calls = _probe(+ {0: RuntimeError("error: unsupported target: 'gfx1010'\nPassManager::run failed")}+ )+ with cuda, hip, patch:+ assert U.triton_runs_on_device(0) is False+ assert U.triton_runs_on_device(0) is False+ assert U._flex_attention_gpu_is_supported() is False+ assert calls == [0]+++def test_a_card_triton_runs_on_keeps_flex():+ cuda, hip = _hip(True)+ patch, calls = _probe({0: True})+ with cuda, hip, patch:+ assert U._flex_attention_gpu_is_supported() is True+ assert calls == [0]++[email protected]("current, expected", [(0, True), (1, False)])+def test_only_the_current_card_is_probed(current, expected):+ """Probing every visible card would open a HIP context on each of them."""+ cuda, hip = _hip(True, count = 2, current = current)+ patch, calls = _probe({0: True, 1: RuntimeError("PassManager::run failed")})+ with cuda, hip, patch:+ assert U._flex_attention_gpu_is_supported() is expected+ assert calls == [current]+++def test_a_rocm_build_without_a_card_is_left_alone():+ cuda, hip = _hip(True, count = 0)+ patch, calls = _probe({})+ with cuda, hip, patch:+ assert U._flex_attention_gpu_is_supported() is True+ assert calls == []+++def test_no_triton_keeps_the_old_answer():+ cuda, hip = _hip(True)+ patch, calls = _probe({0: RuntimeError("unreachable")})+ with cuda, hip, patch, mock.patch.dict(sys.modules, {"triton": None}):+ assert U.triton_runs_on_device(0) is True+ assert calls == []diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 910299cbb77..0de8d497a03 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -582,11 +582,66 @@ def _is_eager_only(model_type):_FLEX_ATTENTION_MIN_CAPABILITY = (8, 0)+_TRITON_RUNS_ON_DEVICE = {}+++def _is_hip_build():+ return bool(+ getattr(torch.version, "hip", None)+ or "rocm" in str(getattr(torch, "__version__", "")).lower()+ )+++def _triton_probe(index):+ import triton+ import triton.language as tl++ @triton.jit+ def _add_one(X, N, BLOCK: tl.constexpr):+ offsets = tl.arange(0, BLOCK)+ tl.store(X + offsets, tl.load(X + offsets, offsets < N) + 1, offsets < N)++ x = torch.zeros(16, device = f"cuda:{index}", dtype = torch.float32)+ with torch.cuda.device(index):+ _add_one[(1,)](x, 16, BLOCK = 16)+ torch.cuda.synchronize(index)+ return bool((x == 1).all().item())+++def triton_runs_on_device(index = 0):+ """False when this Triton build cannot build or run a kernel on ROCm card ``index``.++ Triton 3.9 rejects every gfx10xx target (``unsupported target`` -> ``PassManager::run failed``),+ while ROCm Triton 3.5 builds them, so the answer is the toolchain's, not an arch list.+ NVIDIA, CPU and a missing Triton keep True: nothing to take away there.+ """+ if not _is_hip_build():+ return True+ if index not in _TRITON_RUNS_ON_DEVICE:+ try:+ import triton # noqa: F401+ except Exception:+ return True+ try:+ ok = _triton_probe(index)+ except Exception as error:+ ok = False+ logger.info(+ f"Unsloth: Triton cannot run on ROCm device {index} ({type(error).__name__}: "+ f"{(str(error).strip().splitlines() or [''])[-1]}); its Triton kernels are skipped."+ )+ _TRITON_RUNS_ON_DEVICE[index] = ok+ return _TRITON_RUNS_ON_DEVICE[index]++def _flex_attention_gpu_is_supported():- """False only for NVIDIA cards below Ampere. ROCm, XPU, MPS, CPU and an unreadable device are left alone, so this can only ever remove a path that was already broken."""+ """False for NVIDIA cards below Ampere and ROCm cards this Triton cannot build for. XPU, MPS, CPU and an unreadable device are left alone, so this can only ever remove a path that was already broken."""try:- if getattr(torch.version, "hip", None):- return True+ if _is_hip_build():+ # Current card only: probing the others would open a HIP context on every visible GPU.+ return not torch.cuda.device_count() or triton_runs_on_device(+ torch.cuda.current_device()+ )if not torch.cuda.is_available():return Truereturn all(