Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

AMD RDNA1: turn Triton's buffer ops off on gfx101x before Triton loads (#11615)

Leo Borcherding41c0d446c869
diff --git a/tests/test_rdna1_triton_buffer_ops.py b/tests/test_rdna1_triton_buffer_ops.py
new file mode 100644
index 00000000000..50eb160c02c
--- /dev/null
+++ b/tests/test_rdna1_triton_buffer_ops.py
@@ -0,0 +1,170 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""On RDNA1 (gfx101x) Triton's AMD buffer ops silently do nothing.
+
+Every Triton kernel launches with hipSuccess and leaves its outputs untouched, because the
+buffer resource descriptor Triton builds is laid out for gfx10.3+ and gfx10.1 reads it
+differently. Found on an RX 5700 XT (gfx1010): a ten-line ``x * 2`` kernel returned 0 of
+1024 correct values, and every value with ``AMDGCN_USE_BUFFER_OPS=0``. unsloth sets that
+variable at import when such a GPU is visible. These are decision-table checks, no GPU.
+"""
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from unsloth import device_type
+
+
+ "arch, lacks",
+ [
+ ("gfx1010", True), # RDNA1, RX 5700 XT: where it was measured
+ ("gfx1010:xnack-", True), # the suffixed form torch reports
+ ("gfx1011", True), # RDNA1, Radeon Pro V520
+ ("gfx1012", True), # RDNA1, RX 5500 XT
+ ("gfx1013", True), # RDNA1, Cyan Skillfish
+ ("gfx1030", False), # RDNA2 reads the newer descriptor, buffer ops are fine
+ ("gfx1034", False), # RDNA2, RX 6500 XT: measured correct with buffer ops on
+ ("gfx1034:sramecc-:xnack-", False),
+ ("gfx1100", False), # RDNA3
+ ("gfx1201", False), # RDNA4
+ ("gfx90a", False), # CDNA
+ ("gfx942", False), # CDNA3
+ ("", False), # unreadable: fail open, never guess
+ (None, False),
+ ],
+)
+def test_only_gfx101x_loses_buffer_ops(arch, lacks):
+ """The prefix has to be exactly ``gfx101``. ``gfx10`` would also switch RDNA2 to global
+ loads, which is slower for no reason; ``gfx1`` would reach RDNA3 and RDNA4."""
+ assert device_type.arch_lacks_buffer_ops(arch) is lacks
+
+
+def test_an_unreadable_arch_does_not_narrow_anything():
+ for unreadable in ("", None, "unknown", " "):
+ assert device_type.arch_lacks_buffer_ops(unreadable) is False
+
+
+def test_bf16_gate_and_buffer_ops_gate_disagree_on_rdna2():
+ """Both gates exist because the two problems have different scopes: all of gfx10 lacks
+ bf16, only gfx10.1 mishandles the descriptor. A refactor that merges them breaks RDNA2."""
+ assert device_type.arch_lacks_bf16("gfx1034") is True
+ assert device_type.arch_lacks_buffer_ops("gfx1034") is False
+ assert device_type.arch_lacks_bf16("gfx1010") is True
+ assert device_type.arch_lacks_buffer_ops("gfx1010") is True
+
+
+def test_workaround_sets_knob_and_a_separate_cache_dir():
+ """The knob alone is not enough: Inductor's cache does not key on AMDGCN_USE_BUFFER_OPS,
+ so kernels compiled with buffer ops on are reused with them off. Measured on an RX 5700
+ XT: the same run trained cleanly with empty caches and went -inf / nan once a
+ buffer-ops-on process had populated them."""
+ env = {}
+ assert device_type.apply_gfx101x_triton_workaround(env, triton_home = "/th") is True
+ assert env["AMDGCN_USE_BUFFER_OPS"] == "0"
+ assert env["TRITON_CACHE_DIR"].replace("\\", "/") == "/th/.triton/cache-no-buffer-ops"
+ # Inductor caches the Triton kernels it generates on its own, keyed the same blind way.
+ assert env["TORCHINDUCTOR_CACHE_DIR"].endswith("_no_buffer_ops")
+ assert "torchinductor_" in env["TORCHINDUCTOR_CACHE_DIR"]
+
+
+def test_workaround_honours_a_user_who_already_chose():
+ env = {"AMDGCN_USE_BUFFER_OPS": "1"}
+ assert device_type.apply_gfx101x_triton_workaround(env, triton_home = "/th") is False
+ assert env == {
+ "AMDGCN_USE_BUFFER_OPS": "1"
+ }, "an explicit choice is not overridden and no cache dir is forced"
+ env = {"TRITON_CACHE_DIR": "/mine", "TORCHINDUCTOR_CACHE_DIR": "/mine-inductor"}
+ assert device_type.apply_gfx101x_triton_workaround(env, triton_home = "/th") is True
+ assert env["TRITON_CACHE_DIR"] == "/mine"
+ assert env["TORCHINDUCTOR_CACHE_DIR"] == "/mine-inductor"
+
+
+def test_workaround_follows_triton_home():
+ env = {"TRITON_HOME": "/custom/triton"}
+ device_type.apply_gfx101x_triton_workaround(env)
+ assert (
+ env["TRITON_CACHE_DIR"].replace("\\", "/") == "/custom/triton/.triton/cache-no-buffer-ops"
+ )
+
+
[email protected]("preset", ["0", "false", "off", ""])
+def test_a_user_who_already_turned_buffer_ops_off_still_gets_clean_caches(preset):
+ """AMDGCN_USE_BUFFER_OPS=0 is the manual workaround from #11614. Without separate caches
+ that user keeps loading the kernels an earlier buffer-ops-on run left behind."""
+ env = {"AMDGCN_USE_BUFFER_OPS": preset}
+ assert device_type.apply_gfx101x_triton_workaround(env, triton_home = "/th") is True
+ assert env["AMDGCN_USE_BUFFER_OPS"] == preset
+ assert env["TRITON_CACHE_DIR"].replace("\\", "/") == "/th/.triton/cache-no-buffer-ops"
+ assert env["TORCHINDUCTOR_CACHE_DIR"].endswith("_no_buffer_ops")
+
+
[email protected]("preset", ["1", "true", "ON", "yes", "y"])
+def test_every_spelling_triton_reads_as_on_is_left_alone(preset):
+ env = {"AMDGCN_USE_BUFFER_OPS": preset}
+ assert device_type.apply_gfx101x_triton_workaround(env, triton_home = "/th") is False
+ assert env == {"AMDGCN_USE_BUFFER_OPS": preset}
+
+
+def test_inductor_default_already_in_environ_is_still_redirected():
+ """`import torch._dynamo` writes Inductor's default cache dir into os.environ before
+ unsloth gets here, so "already set" usually means "torch set the shared default"."""
+ default = device_type._default_inductor_cache_dir()
+ env = {"TORCHINDUCTOR_CACHE_DIR": default}
+ device_type.apply_gfx101x_triton_workaround(env, triton_home = "/th")
+ assert env["TORCHINDUCTOR_CACHE_DIR"] == default + "_no_buffer_ops"
+
+
+def test_import_time_env_is_redirected_after_torch_dynamo_import(monkeypatch):
+ import torch._dynamo # noqa: F401 populates TORCHINDUCTOR_CACHE_DIR as a side effect
+ import os
+
+ monkeypatch.delenv("AMDGCN_USE_BUFFER_OPS", raising = False)
+ monkeypatch.delenv("TRITON_CACHE_DIR", raising = False)
+ monkeypatch.setenv("TORCHINDUCTOR_CACHE_DIR", device_type._default_inductor_cache_dir())
+ assert device_type.apply_gfx101x_triton_workaround() is True
+ assert os.environ["TORCHINDUCTOR_CACHE_DIR"].endswith("_no_buffer_ops")
+
+
+def test_missing_passwd_entry_does_not_break_import(monkeypatch):
+ """getpass.getuser raises in a container whose uid has no passwd entry."""
+ import getpass
+ import sys
+
+ def boom():
+ raise KeyError("getpwuid(): uid not found: 12345")
+
+ monkeypatch.setattr(getpass, "getuser", boom)
+ monkeypatch.setitem(sys.modules, "torch._inductor.runtime.cache_dir_utils", None)
+ env = {}
+ assert device_type.apply_gfx101x_triton_workaround(env, triton_home = "/th") is True
+ assert "torchinductor_" in env["TORCHINDUCTOR_CACHE_DIR"]
+
+
+def test_reapplying_after_patch_torch_compile_restores_the_inductor_dir(monkeypatch):
+ """unsloth_zoo's patch_torch_compile pops TORCHINDUCTOR_CACHE_DIR after _gpu_init ran;
+ models/_utils.py re-applies the workaround, which must put the separate dir back and
+ leave everything else as the first call set it."""
+ import os
+
+ for name in ("AMDGCN_USE_BUFFER_OPS", "TRITON_CACHE_DIR", "TORCHINDUCTOR_CACHE_DIR"):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setattr(device_type, "_GFX101X_TRITON_WORKAROUND_APPLIED", False)
+ assert device_type.gfx101x_triton_workaround_applied() is False
+ assert device_type.apply_gfx101x_triton_workaround() is True
+ assert device_type.gfx101x_triton_workaround_applied() is True
+ first = {
+ k: os.environ[k]
+ for k in ("AMDGCN_USE_BUFFER_OPS", "TRITON_CACHE_DIR", "TORCHINDUCTOR_CACHE_DIR")
+ }
+ os.environ.pop("TORCHINDUCTOR_CACHE_DIR") # what patch_torch_compile does
+ assert device_type.apply_gfx101x_triton_workaround() is True
+ assert {k: os.environ[k] for k in first} == first
+
+
+def test_a_dict_environ_does_not_mark_the_process():
+ before = device_type.gfx101x_triton_workaround_applied()
+ device_type.apply_gfx101x_triton_workaround({}, triton_home = "/th")
+ assert device_type.gfx101x_triton_workaround_applied() is before
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 917d1436c87..cc9485736bb 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -241,7 +241,12 @@ from unsloth_zoo.device_type import (
DEVICE_COUNT,
ALLOW_PREQUANTIZED_MODELS,
)
-from .device_type import arch_lacks_bf16, hip_visible_archs
+from .device_type import (
+ arch_lacks_bf16,
+ arch_lacks_buffer_ops,
+ apply_gfx101x_triton_workaround,
+ hip_visible_archs,
+)
from .import_fixes import (
fix_transformers5_bare_annotation_configs,
@@ -449,6 +454,10 @@ elif DEVICE_TYPE == "npu":
# No arm left the name unbound, so consumers fell back to their own False.
SUPPORTS_BFLOAT16 = torch.npu.is_bf16_supported()
+# gfx101x: Triton buffer-op kernels silently write nothing; must be set before the first compile.
+if DEVICE_TYPE == "hip" and any(arch_lacks_buffer_ops(arch) for arch in hip_visible_archs()):
+ apply_gfx101x_triton_workaround()
+
# For Gradio HF Spaces?
# if "SPACE_AUTHOR_NAME" not in os.environ and "SPACE_REPO_NAME" not in os.environ:
# `triton` is optional here, like bitsandbytes below. The PyPI `triton` project publishes no
diff --git a/unsloth/device_type.py b/unsloth/device_type.py
index 86dd0f39078..85d2596aa36 100644
--- a/unsloth/device_type.py
+++ b/unsloth/device_type.py
@@ -220,6 +220,63 @@ def arch_lacks_bf16(gcn_arch):
return str(gcn_arch or "").split(":", 1)[0].strip().lower().startswith("gfx10")
+def arch_lacks_buffer_ops(gcn_arch):
+ """RDNA1 reads Triton's gfx10.3-layout buffer descriptors wrongly: kernels launch and write
+ nothing. gfx103x (RDNA2) is fine and must not match (#11614)."""
+ return str(gcn_arch or "").split(":", 1)[0].strip().lower().startswith("gfx101")
+
+
+_GFX101X_TRITON_WORKAROUND_APPLIED = False
+
+
+def gfx101x_triton_workaround_applied():
+ return _GFX101X_TRITON_WORKAROUND_APPLIED
+
+
+def apply_gfx101x_triton_workaround(environ = None, triton_home = None):
+ """Turn Triton's buffer ops off and give Triton and Inductor separate caches: Inductor's cache
+ key ignores the knob, so stale buffer-op kernels gave -inf/nan. A user-set value that Triton
+ reads as on is left alone; user-chosen cache dirs are kept. Returns whether ops end up off."""
+ global _GFX101X_TRITON_WORKAROUND_APPLIED
+ is_process_env = environ is None
+ environ = os.environ if environ is None else environ
+ current = environ.get("AMDGCN_USE_BUFFER_OPS")
+ # Triton's getenv_bool: only these spellings mean on, anything else is off.
+ if current is not None and current.strip().lower() in ("1", "true", "on", "yes", "y"):
+ return False
+ environ.setdefault("AMDGCN_USE_BUFFER_OPS", "0")
+ if "TRITON_CACHE_DIR" not in environ:
+ home = triton_home or environ.get("TRITON_HOME") or os.path.expanduser("~")
+ environ["TRITON_CACHE_DIR"] = os.path.join(home, ".triton", "cache-no-buffer-ops")
+ default_inductor = _default_inductor_cache_dir()
+ inductor = environ.get("TORCHINDUCTOR_CACHE_DIR")
+ # `import torch._dynamo` already wrote the shared default into os.environ; not a user choice.
+ if inductor is None or os.path.abspath(inductor) == os.path.abspath(default_inductor):
+ environ["TORCHINDUCTOR_CACHE_DIR"] = default_inductor + "_no_buffer_ops"
+ if is_process_env:
+ _GFX101X_TRITON_WORKAROUND_APPLIED = True
+ return True
+
+
+def _default_inductor_cache_dir():
+ try:
+ from torch._inductor.runtime.cache_dir_utils import default_cache_dir
+ return default_cache_dir()
+ except Exception:
+ pass
+ import getpass
+ import tempfile
+
+ # getuser raises for a uid with no passwd entry (containers); same fallback as torch.
+ try:
+ user = getpass.getuser()
+ except (KeyError, ModuleNotFoundError, OSError):
+ getuid = getattr(os, "getuid", None)
+ user = f"uid_{getuid()}" if callable(getuid) else "unknown_user"
+ user = re.sub(r'[\\/:*?"<>|]', "_", user)
+ return os.path.join(tempfile.gettempdir(), "torchinductor_" + user)
+
+
def hip_visible_archs():
"""Guarded per device: one unreadable device must not discard the archs beside it, or a
gfx10 keeps bf16 and dies in Triton (#7922). Only an unreadable count returns []."""
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index f30583e32db..c14b96bd86f 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -129,6 +129,8 @@ from ..device_type import (
DEVICE_TYPE_TORCH,
DEVICE_COUNT,
ALLOW_PREQUANTIZED_MODELS,
+ apply_gfx101x_triton_workaround,
+ gfx101x_triton_workaround_applied,
)
from ..import_fixes import UNSLOTH_ENABLE_LOGGING
from unsloth_zoo.log import logger
@@ -2662,6 +2664,9 @@ patch_torch_compile(
O3 = UNSLOTH_COMPILE_MAXIMUM,
ignore_errors = UNSLOTH_COMPILE_IGNORE_ERRORS,
)
+# patch_torch_compile popped the gfx101x Inductor cache dir chosen in _gpu_init.
+if gfx101x_triton_workaround_applied():
+ apply_gfx101x_triton_workaround()
torch_compile_options = {
"epilogue_fusion": True,