Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: contiguous k/v before SDPA on gfx1151 when they outgrow L2 (#15096)

Daniel Han2c2e04474bcf
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index b9b78223af8..2eaec192a07 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -512,6 +512,95 @@ def _apply_rocm_guard(available: tuple[str, ...], logger: Any = None) -> tuple[s
return tuple(disabled)
+# gfx1151 (Strix Halo): AOTriton's SDPA tuning covers contiguous [B, H, S, D] only, and diffusers' native backend
+# hands SDPA permuted [B, S, H, D] views. Once one head of the strided k/v outgrows L2 the strided call is far slower
+# than copying k/v first (ComfyUI #16726, pytorch/pytorch#190154). Output is bit-identical. 0 disables.
+ROCM_CONTIGUOUS_KV_ENV = "UNSLOTH_DIFFUSION_ROCM_CONTIGUOUS_KV"
+ROCM_CONTIGUOUS_KV_ARCHES = ("gfx1151",)
+ROCM_CONTIGUOUS_KV_ATTR = "_unsloth_rocm_contiguous_kv"
+_ROCM_CONTIGUOUS_KV_L2: dict[int, int] = {} # device index -> L2 bytes, gfx1151 cards only
+
+
+def _rocm_contiguous_kv_wanted(key: Any, value: Any) -> bool:
+ """True when one head of the strided ``[B, S, H, D]`` k/v exceeds L2."""
+ try:
+ if key.dim() != 4 or value.dim() != 4:
+ return False
+ l2 = _ROCM_CONTIGUOUS_KV_L2.get(key.device.index)
+ if not l2:
+ return False
+ strided = [t for t in (key, value) if not t.permute(0, 2, 1, 3).is_contiguous()]
+ return sum(t.shape[1] * t.shape[3] for t in strided) * key.element_size() > l2
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _heads_major(t: Any) -> Any:
+ """``[B, H, S, D]`` memory so the native backend's permute hands SDPA a dense tensor."""
+ if t.permute(0, 2, 1, 3).is_contiguous():
+ return t
+ return t.permute(0, 2, 1, 3).contiguous().permute(0, 2, 1, 3)
+
+
+def install_rocm_contiguous_kv(target: Any) -> bool:
+ """gfx1151 only: k/v over L2 reach SDPA contiguous (q is not copied). False elsewhere: other GPUs untouched."""
+ if os.environ.get(ROCM_CONTIGUOUS_KV_ENV, "").strip().lower() in ("0", "false", "no", "off"):
+ return False
+ if not _is_cuda_rocm(target) or _rocm_gfx_arch(target) not in ROCM_CONTIGUOUS_KV_ARCHES:
+ return False
+ try:
+ import torch
+ from diffusers.models.attention_dispatch import (
+ AttentionBackendName,
+ _AttentionBackendRegistry,
+ )
+
+ device = torch.device(
+ _indexed_cuda_device(
+ str(
+ getattr(target, "torch_device", None)
+ or getattr(target, "device", None)
+ or "cuda"
+ )
+ )
+ )
+ index = device.index if device.index is not None else torch.cuda.current_device()
+ l2 = int(getattr(torch.cuda.get_device_properties(index), "L2_cache_size", 0) or 0)
+ backends = _AttentionBackendRegistry._backends
+ native_fn = backends[AttentionBackendName.NATIVE]
+ except Exception: # noqa: BLE001
+ return False
+ if l2 <= 0 or not callable(native_fn):
+ return False
+ _ROCM_CONTIGUOUS_KV_L2[index] = l2
+ if getattr(native_fn, ROCM_CONTIGUOUS_KV_ATTR, False):
+ return True
+
+ def _contiguous_kv(*args: Any, **kwargs: Any) -> Any:
+ args = list(args)
+ key = args[1] if len(args) > 1 else kwargs.get("key")
+ value = args[2] if len(args) > 2 else kwargs.get("value")
+ if kwargs.get("_parallel_config") is None and _rocm_contiguous_kv_wanted(key, value):
+ key, value = _heads_major(key), _heads_major(value)
+ if len(args) > 1:
+ args[1] = key
+ else:
+ kwargs["key"] = key
+ if len(args) > 2:
+ args[2] = value
+ else:
+ kwargs["value"] = value
+ return native_fn(*args, **kwargs)
+
+ setattr(_contiguous_kv, ROCM_CONTIGUOUS_KV_ATTR, True)
+ _contiguous_kv.__wrapped__ = native_fn # type: ignore[attr-defined]
+ try:
+ backends[AttentionBackendName.NATIVE] = _contiguous_kv
+ except Exception: # noqa: BLE001
+ return False
+ return True
+
+
def select_attention_backend(
target: Any,
requested: Optional[str],
@@ -3436,6 +3525,7 @@ def apply_attention_backend(
guard_rocm_fused_sdpa(target, logger)
except Exception as exc:
_warn(logger, "ROCm fused SDPA check", exc)
+ install_rocm_contiguous_kv(target)
setters = [
s
for s in (getattr(t, "set_attention_backend", None) for t in _attention_dits(pipe))
diff --git a/studio/backend/tests/test_diffusion_attention_rocm_contiguous_kv.py b/studio/backend/tests/test_diffusion_attention_rocm_contiguous_kv.py
new file mode 100644
index 00000000000..bfd9c291a78
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_attention_rocm_contiguous_kv.py
@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""gfx1151 contiguous k/v wrapper. CPU tensors stand in for the card (L2 table key is None on CPU)."""
+
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("diffusers")
+
+from diffusers.models.attention_dispatch import ( # noqa: E402
+ AttentionBackendName,
+ _AttentionBackendRegistry,
+)
+
+from core.inference import diffusion_attention as da # noqa: E402
+
+TARGET = SimpleNamespace(device = "cuda", torch_device = "cuda:0", ordinal = 0)
+
+
+def registry(monkeypatch):
+ backends = _AttentionBackendRegistry._backends
+ monkeypatch.setitem(
+ backends, AttentionBackendName.NATIVE, backends[AttentionBackendName.NATIVE]
+ )
+ monkeypatch.setattr(da, "_ROCM_CONTIGUOUS_KV_L2", {})
+ monkeypatch.delenv(da.ROCM_CONTIGUOUS_KV_ENV, raising = False)
+ return backends
+
+
+def _fake_card(
+ monkeypatch,
+ arch = "gfx1151",
+ rocm = True,
+ l2 = 2 * 1024 * 1024,
+):
+ monkeypatch.setattr(da, "_is_cuda_rocm", lambda target: rocm)
+ monkeypatch.setattr(da, "_rocm_gfx_arch", lambda target: arch)
+ monkeypatch.setattr(
+ torch.cuda, "get_device_properties", lambda index: SimpleNamespace(L2_cache_size = l2)
+ )
+
+
+ "arch, rocm", [("gfx1100", True), ("gfx1201", True), ("gfx942", True), ("", False)]
+)
+def test_other_gpus_keep_the_stock_native_backend(registry, monkeypatch, arch, rocm):
+ _fake_card(monkeypatch, arch = arch, rocm = rocm)
+ native = registry[AttentionBackendName.NATIVE]
+ assert da.install_rocm_contiguous_kv(TARGET) is False
+ assert registry[AttentionBackendName.NATIVE] is native
+
+
+def test_kill_switch(registry, monkeypatch):
+ _fake_card(monkeypatch)
+ monkeypatch.setenv(da.ROCM_CONTIGUOUS_KV_ENV, "0")
+ native = registry[AttentionBackendName.NATIVE]
+ assert da.install_rocm_contiguous_kv(TARGET) is False
+ assert registry[AttentionBackendName.NATIVE] is native
+
+
+def test_installs_once_on_gfx1151(registry, monkeypatch):
+ _fake_card(monkeypatch)
+ native = registry[AttentionBackendName.NATIVE]
+ assert da.install_rocm_contiguous_kv(TARGET) is True
+ wrapped = registry[AttentionBackendName.NATIVE]
+ assert wrapped is not native and wrapped.__wrapped__ is native
+ assert da.install_rocm_contiguous_kv(TARGET) is True
+ assert registry[AttentionBackendName.NATIVE] is wrapped
+ assert da._ROCM_CONTIGUOUS_KV_L2 == {0: 2 * 1024 * 1024}
+
+
+def test_size_gate_matches_one_head_of_strided_kv_against_l2(monkeypatch):
+ l2 = 1 << 16
+ monkeypatch.setattr(da, "_ROCM_CONTIGUOUS_KV_L2", {None: l2})
+ d = 32
+ edge = l2 // (2 * d * 4) # fp32 k + v of one head exactly fill L2
+ at = torch.zeros(1, edge, 2, d)
+ over = torch.zeros(1, edge + 1, 2, d)
+ assert not da._rocm_contiguous_kv_wanted(at, at)
+ assert da._rocm_contiguous_kv_wanted(over, over)
+ dense = over.permute(0, 2, 1, 3).contiguous().permute(0, 2, 1, 3)
+ assert not da._rocm_contiguous_kv_wanted(dense, dense)
+ monkeypatch.setattr(da, "_ROCM_CONTIGUOUS_KV_L2", {5: l2})
+ assert not da._rocm_contiguous_kv_wanted(over, over) # no L2 entry: not gfx1151
+
+
+def test_wrapper_hands_sdpa_contiguous_kv_and_keeps_outputs(registry, monkeypatch):
+ _fake_card(monkeypatch)
+ da.install_rocm_contiguous_kv(TARGET)
+ monkeypatch.setattr(da, "_ROCM_CONTIGUOUS_KV_L2", {None: 1 << 12})
+ wrapped = registry[AttentionBackendName.NATIVE]
+ seen = []
+ real = torch.nn.functional.scaled_dot_product_attention
+
+ def spy(*args, **kwargs):
+ seen.append((kwargs["key"].is_contiguous(), kwargs["value"].is_contiguous()))
+ return real(*args, **kwargs)
+
+ monkeypatch.setattr(torch.nn.functional, "scaled_dot_product_attention", spy)
+ g = torch.Generator().manual_seed(0)
+ q, k, v = (torch.randn(1, 64, 2, 16, generator = g) for _ in range(3))
+ ref = wrapped.__wrapped__(q, k, v)
+ assert seen.pop() == (False, False)
+ assert torch.equal(wrapped(query = q, key = k, value = v), ref)
+ assert seen.pop() == (True, True)
+ assert torch.equal(wrapped(q, k, v), ref) # positional, as Studio's dispatch guards call it
+ assert seen.pop() == (True, True)
+ small = (t[:, :8] for t in (q, k, v))
+ wrapped(*small)
+ assert seen.pop() == (False, False)