Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio: AMD: skip LFM3's Triton conv and flex attention on ROCm cards Triton cannot build for (#16974)

Leo Borcherding6c525184197a
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index fc81f07b920..958d61b98bf 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -40,6 +40,7 @@ from core.inference.runtime_context import (
)
from core.inference.message_content import content_to_text, named_turn
from core.inference.model_ids import is_deepseek_ocr
+from core.inference.lfm3_conv_fallback import disable_lfm3_triton_conv_where_unsupported
from core.inference.chat_eos import (
chat_eos_repair,
resolve_chat_turn_end_eos_ids_using,
@@ -1377,6 +1378,9 @@ class InferenceBackend:
)
raise_if_offloaded(self.models[model_name]["model"], device_map, "Inference")
+ disable_lfm3_triton_conv_where_unsupported(
+ self.models[model_name]["model"], torch, logger
+ )
self.models[model_name]["context_length"] = runtime_context_length(
self.models[model_name].get("model"),
max_seq_length,
@@ -1566,6 +1570,9 @@ class InferenceBackend:
)
raise_if_offloaded(self.models[model_name]["model"], device_map, "Inference")
+ disable_lfm3_triton_conv_where_unsupported(
+ self.models[model_name]["model"], torch, logger
+ )
self.models[model_name]["context_length"] = runtime_context_length(
self.models[model_name].get("model"),
max_seq_length,
diff --git a/studio/backend/core/inference/lfm3_conv_fallback.py b/studio/backend/core/inference/lfm3_conv_fallback.py
new file mode 100644
index 00000000000..f030f7fb1db
--- /dev/null
+++ b/studio/backend/core/inference/lfm3_conv_fallback.py
@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""LFM3's remote Triton conv dies with ``PassManager::run failed`` where Triton cannot build for the card (gfx101x on Triton 3.9); use its torch path there."""
+
+from __future__ import annotations
+
+import sys
+from typing import Any, Callable, Optional
+
+from core.inference.rocm_bf16 import _device_gfx_arch, is_rocm_torch
+
+_GATE = "can_use_lfm3_conv"
+
+
+def _core_probe() -> Callable[[int], bool]:
+ try:
+ from unsloth.models._utils import triton_runs_on_device
+ except Exception: # noqa: BLE001 -- no probe: keep the remote code's own choice
+ return lambda index: True
+ return triton_runs_on_device
+
+
+def _model_cuda_indices(model: Any) -> set[int]:
+ indices = set()
+ try:
+ for param in model.parameters():
+ device = param.device
+ if device.type == "cuda":
+ indices.add(device.index or 0)
+ except Exception: # noqa: BLE001 -- no parameters to read: nothing on a GPU to gate
+ pass
+ return indices
+
+
+def _guarded(original: Any, runs_on: Callable[[int], bool]):
+ def gate(hidden_states, weight, *args, **kwargs):
+ # Module-wide gate: a model on a card Triton does support keeps the kernel.
+ device = getattr(hidden_states, "device", None)
+ if getattr(device, "type", None) != "cuda" or not runs_on(device.index or 0):
+ return False
+ return original(hidden_states, weight, *args, **kwargs)
+
+ gate._unsloth_triton_guard = True
+ return gate
+
+
+def disable_lfm3_triton_conv_where_unsupported(
+ model: Any,
+ torch: Any,
+ log: Any = None,
+ runs_on: Optional[Callable[[int], bool]] = None,
+) -> list[str]:
+ """Make every loaded remote-code module's ``can_use_lfm3_conv`` refuse tensors on a ROCm card Triton cannot run on."""
+ if not is_rocm_torch(torch):
+ return []
+ runs_on = runs_on or _core_probe()
+ failing = sorted(index for index in _model_cuda_indices(model) if not runs_on(index))
+ if not failing:
+ return []
+ patched = []
+ for name, module in list(sys.modules.items()):
+ if not name.startswith("transformers_modules.") or module is None:
+ continue
+ original = getattr(module, _GATE, None)
+ if callable(original) and not getattr(original, "_unsloth_triton_guard", False):
+ setattr(module, _GATE, _guarded(original, runs_on))
+ patched.append(name)
+ if patched and log is not None:
+ archs = ", ".join(_device_gfx_arch(torch, index) or f"cuda:{index}" for index in failing)
+ log.info(
+ f"LFM3 Triton conv disabled on {archs} (Triton cannot build for it), "
+ f"using the torch conv: {', '.join(patched)}"
+ )
+ return patched
diff --git a/studio/backend/tests/test_lfm3_conv_fallback.py b/studio/backend/tests/test_lfm3_conv_fallback.py
new file mode 100644
index 00000000000..87b91517250
--- /dev/null
+++ b/studio/backend/tests/test_lfm3_conv_fallback.py
@@ -0,0 +1,149 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""LFM3's remote Triton conv is turned off only on a ROCm card Triton cannot run on."""
+
+from __future__ import annotations
+
+import sys
+import types
+
+import pytest
+
+from core.inference import lfm3_conv_fallback as fallback
+from core.inference.lfm3_conv_fallback import disable_lfm3_triton_conv_where_unsupported
+
+LFM3 = "transformers_modules.LiquidAI.LFM3_hyphen_1_dot_7B.abc.modeling_lfm3"
+OMNI_TEXT = "transformers_modules.LiquidAI.LFM3_hyphen_Omni_hyphen_2B.def.modeling_lfm3"
+OTHER = "transformers_modules.someone.other.ghi.modeling_other"
+
+_KERNEL = object()
+
+
+def _original(hidden_states, weight):
+ return _KERNEL
+
+
+def _fake_torch(*, hip, arch = "gfx1010"):
+ torch = types.ModuleType("torch")
+ torch.__version__ = "2.11.0+rocm7.14.0" if hip else "2.11.0+cu130"
+ torch.version = types.SimpleNamespace(hip = "7.14" if hip else None)
+ torch.cuda = types.SimpleNamespace(
+ current_device = lambda: 0,
+ get_device_properties = lambda index = None: types.SimpleNamespace(gcnArchName = arch),
+ )
+ return torch
+
+
+def _model(device_type, index = 0):
+ param = types.SimpleNamespace(device = types.SimpleNamespace(type = device_type, index = index))
+ return types.SimpleNamespace(parameters = lambda: iter([param]))
+
+
+def _tensor(index, device_type = "cuda"):
+ return types.SimpleNamespace(device = types.SimpleNamespace(type = device_type, index = index))
+
+
+def _probe(results):
+ calls = []
+
+ def runs_on(index):
+ calls.append(index)
+ return results[index]
+
+ runs_on.calls = calls
+ return runs_on
+
+
+def remote_modules(monkeypatch):
+ mods = {}
+ for name in (LFM3, OMNI_TEXT):
+ mod = types.ModuleType(name)
+ mod.can_use_lfm3_conv = _original
+ mods[name] = mod
+ other = types.ModuleType(OTHER)
+ other.something_else = _original
+ mods[OTHER] = other
+ for name, mod in mods.items():
+ monkeypatch.setitem(sys.modules, name, mod)
+ return mods
+
+
[email protected]("arch", ["gfx1010", "gfx1030", "gfx1036"])
+def test_gate_patched_where_triton_cannot_run(remote_modules, arch):
+ log = types.SimpleNamespace(lines = [])
+ log.info = log.lines.append
+ patched = disable_lfm3_triton_conv_where_unsupported(
+ _model("cuda"), _fake_torch(hip = True, arch = arch), log, _probe({0: False})
+ )
+ assert sorted(patched) == sorted([LFM3, OMNI_TEXT])
+ for name in (LFM3, OMNI_TEXT):
+ assert remote_modules[name].can_use_lfm3_conv(_tensor(0), None) is False
+ assert not hasattr(remote_modules[OTHER], "can_use_lfm3_conv")
+ assert len(log.lines) == 1 and arch in log.lines[0]
+
+
+ "hip, device",
+ [
+ (
+ True,
+ "cuda",
+ ), # Triton runs on the card (gfx1010 on ROCm Triton 3.5, gfx1100, gfx1151 ...)
+ (True, "cpu"),
+ (False, "cuda"),
+ (False, "mps"),
+ (False, "cpu"),
+ ],
+)
+def test_gate_untouched_elsewhere(remote_modules, hip, device):
+ probe = _probe({0: True})
+ patched = disable_lfm3_triton_conv_where_unsupported(
+ _model(device), _fake_torch(hip = hip), None, probe
+ )
+ assert patched == []
+ if not hip:
+ assert probe.calls == []
+ for name in (LFM3, OMNI_TEXT):
+ assert remote_modules[name].can_use_lfm3_conv is _original
+
+
+def test_model_without_gate_is_a_noop(monkeypatch):
+ for name in [n for n in sys.modules if n.startswith("transformers_modules.")]:
+ monkeypatch.delitem(sys.modules, name)
+ monkeypatch.setitem(sys.modules, OTHER, types.ModuleType(OTHER))
+ assert (
+ disable_lfm3_triton_conv_where_unsupported(
+ _model("cuda"), _fake_torch(hip = True), None, _probe({0: False})
+ )
+ == []
+ )
+
+
+def test_unreadable_model_is_a_noop(remote_modules):
+ assert (
+ disable_lfm3_triton_conv_where_unsupported(
+ object(), _fake_torch(hip = True), None, _probe({0: False})
+ )
+ == []
+ )
+ assert remote_modules[LFM3].can_use_lfm3_conv is _original
+
+
+def test_missing_core_probe_keeps_the_kernel(remote_modules, monkeypatch):
+ monkeypatch.setitem(sys.modules, "unsloth.models._utils", None)
+ assert fallback._core_probe()(0) is True
+ assert disable_lfm3_triton_conv_where_unsupported(_model("cuda"), _fake_torch(hip = True)) == []
+
+
+def test_a_supported_card_in_the_same_process_keeps_the_kernel(remote_modules):
+ probe = _probe({0: False, 1: True})
+ torch = _fake_torch(hip = True)
+ disable_lfm3_triton_conv_where_unsupported(_model("cuda", 0), torch, None, probe)
+ assert disable_lfm3_triton_conv_where_unsupported(_model("cuda", 0), torch, None, probe) == []
+ for name in (LFM3, OMNI_TEXT):
+ gate = remote_modules[name].can_use_lfm3_conv
+ assert gate(_tensor(0), None) is False
+ assert gate(_tensor(1), None) is _KERNEL
+ assert gate(_tensor(None, "cpu"), None) is False
diff --git a/tests/test_flex_attention_needs_ampere.py b/tests/test_flex_attention_needs_ampere.py
index 33206a34f5d..db55ffcfe31 100644
--- a/tests/test_flex_attention_needs_ampere.py
+++ b/tests/test_flex_attention_needs_ampere.py
@@ -70,7 +70,7 @@ def test_rocm_is_not_judged_by_a_cuda_capability():
"""`get_device_capability` answers on ROCm too, with numbers that are not
CUDA's, so reading them would disable flex on AMD for no reason."""
cuda, hip = _cuda([(7, 5)], hip = "6.2.0")
- with cuda, hip:
+ with cuda, hip, mock.patch.object(U, "triton_runs_on_device", lambda index = 0: True):
assert U._flex_attention_gpu_is_supported() is True
diff --git a/tests/test_triton_runs_on_device.py b/tests/test_triton_runs_on_device.py
new file mode 100644
index 00000000000..d01594042ff
--- /dev/null
+++ b/tests/test_triton_runs_on_device.py
@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Flex attention is dropped on a ROCm card this Triton cannot build for (gfx101x on Triton 3.9)."""
+
+import sys
+import types
+from unittest import mock
+
+import pytest
+
+import unsloth.models._utils as U
+
+
[email protected](autouse = True)
+def _fresh_cache():
+ # macOS / Windows runners have no Triton; the probe itself is stubbed per test.
+ triton = sys.modules.get("triton") or types.ModuleType("triton")
+ with (
+ mock.patch.dict(U._TRITON_RUNS_ON_DEVICE, clear = True),
+ mock.patch.dict(sys.modules, {"triton": triton}),
+ ):
+ yield
+
+
+def _hip(
+ on,
+ count = 1,
+ current = 0,
+):
+ return mock.patch.multiple(
+ U.torch.cuda,
+ is_available = lambda: count > 0,
+ device_count = lambda: count,
+ current_device = lambda: current,
+ ), mock.patch.object(U.torch.version, "hip", "7.14" if on else None, create = True)
+
+
+def _probe(results):
+ calls = []
+
+ def probe(index):
+ calls.append(index)
+ if isinstance(results[index], Exception):
+ raise results[index]
+ return results[index]
+
+ probe.calls = calls
+ return mock.patch.object(U, "_triton_probe", probe), calls
+
+
+def test_cuda_builds_never_probe():
+ cuda, hip = _hip(False)
+ patch, calls = _probe({0: RuntimeError("unreachable")})
+ with cuda, hip, patch, mock.patch.object(U.torch, "__version__", "2.14.1+cu130"):
+ assert U.triton_runs_on_device(0) is True
+ assert calls == []
+
+
+def test_a_target_triton_rejects_is_refused_once():
+ cuda, hip = _hip(True)
+ patch, calls = _probe(
+ {0: RuntimeError("error: unsupported target: 'gfx1010'\nPassManager::run failed")}
+ )
+ with cuda, hip, patch:
+ assert U.triton_runs_on_device(0) is False
+ assert U.triton_runs_on_device(0) is False
+ assert U._flex_attention_gpu_is_supported() is False
+ assert calls == [0]
+
+
+def test_a_card_triton_runs_on_keeps_flex():
+ cuda, hip = _hip(True)
+ patch, calls = _probe({0: True})
+ with cuda, hip, patch:
+ assert U._flex_attention_gpu_is_supported() is True
+ assert calls == [0]
+
+
[email protected]("current, expected", [(0, True), (1, False)])
+def test_only_the_current_card_is_probed(current, expected):
+ """Probing every visible card would open a HIP context on each of them."""
+ cuda, hip = _hip(True, count = 2, current = current)
+ patch, calls = _probe({0: True, 1: RuntimeError("PassManager::run failed")})
+ with cuda, hip, patch:
+ assert U._flex_attention_gpu_is_supported() is expected
+ assert calls == [current]
+
+
+def test_a_rocm_build_without_a_card_is_left_alone():
+ cuda, hip = _hip(True, count = 0)
+ patch, calls = _probe({})
+ with cuda, hip, patch:
+ assert U._flex_attention_gpu_is_supported() is True
+ assert calls == []
+
+
+def test_no_triton_keeps_the_old_answer():
+ cuda, hip = _hip(True)
+ patch, calls = _probe({0: RuntimeError("unreachable")})
+ with cuda, hip, patch, mock.patch.dict(sys.modules, {"triton": None}):
+ assert U.triton_runs_on_device(0) is True
+ assert calls == []
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 910299cbb77..0de8d497a03 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -582,11 +582,66 @@ def _is_eager_only(model_type):
_FLEX_ATTENTION_MIN_CAPABILITY = (8, 0)
+_TRITON_RUNS_ON_DEVICE = {}
+
+
+def _is_hip_build():
+ return bool(
+ getattr(torch.version, "hip", None)
+ or "rocm" in str(getattr(torch, "__version__", "")).lower()
+ )
+
+
+def _triton_probe(index):
+ import triton
+ import triton.language as tl
+
+ @triton.jit
+ def _add_one(X, N, BLOCK: tl.constexpr):
+ offsets = tl.arange(0, BLOCK)
+ tl.store(X + offsets, tl.load(X + offsets, offsets < N) + 1, offsets < N)
+
+ x = torch.zeros(16, device = f"cuda:{index}", dtype = torch.float32)
+ with torch.cuda.device(index):
+ _add_one[(1,)](x, 16, BLOCK = 16)
+ torch.cuda.synchronize(index)
+ return bool((x == 1).all().item())
+
+
+def triton_runs_on_device(index = 0):
+ """False when this Triton build cannot build or run a kernel on ROCm card ``index``.
+
+ Triton 3.9 rejects every gfx10xx target (``unsupported target`` -> ``PassManager::run failed``),
+ while ROCm Triton 3.5 builds them, so the answer is the toolchain's, not an arch list.
+ NVIDIA, CPU and a missing Triton keep True: nothing to take away there.
+ """
+ if not _is_hip_build():
+ return True
+ if index not in _TRITON_RUNS_ON_DEVICE:
+ try:
+ import triton # noqa: F401
+ except Exception:
+ return True
+ try:
+ ok = _triton_probe(index)
+ except Exception as error:
+ ok = False
+ logger.info(
+ f"Unsloth: Triton cannot run on ROCm device {index} ({type(error).__name__}: "
+ f"{(str(error).strip().splitlines() or [''])[-1]}); its Triton kernels are skipped."
+ )
+ _TRITON_RUNS_ON_DEVICE[index] = ok
+ return _TRITON_RUNS_ON_DEVICE[index]
+
+
def _flex_attention_gpu_is_supported():
- """False only for NVIDIA cards below Ampere. ROCm, XPU, MPS, CPU and an unreadable device are left alone, so this can only ever remove a path that was already broken."""
+ """False for NVIDIA cards below Ampere and ROCm cards this Triton cannot build for. XPU, MPS, CPU and an unreadable device are left alone, so this can only ever remove a path that was already broken."""
try:
- if getattr(torch.version, "hip", None):
- return True
+ if _is_hip_build():
+ # Current card only: probing the others would open a HIP context on every visible GPU.
+ return not torch.cuda.device_count() or triton_runs_on_device(
+ torch.cuda.current_device()
+ )
if not torch.cuda.is_available():
return True
return all(