Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: contiguous k/v before SDPA on gfx1151 when they outgrow L2 (#15096)
Daniel Han2c2e04474bcf
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex b9b78223af8..2eaec192a07 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -512,6 +512,95 @@ def _apply_rocm_guard(available: tuple[str, ...], logger: Any = None) -> tuple[sreturn tuple(disabled)+# gfx1151 (Strix Halo): AOTriton's SDPA tuning covers contiguous [B, H, S, D] only, and diffusers' native backend+# hands SDPA permuted [B, S, H, D] views. Once one head of the strided k/v outgrows L2 the strided call is far slower+# than copying k/v first (ComfyUI #16726, pytorch/pytorch#190154). Output is bit-identical. 0 disables.+ROCM_CONTIGUOUS_KV_ENV = "UNSLOTH_DIFFUSION_ROCM_CONTIGUOUS_KV"+ROCM_CONTIGUOUS_KV_ARCHES = ("gfx1151",)+ROCM_CONTIGUOUS_KV_ATTR = "_unsloth_rocm_contiguous_kv"+_ROCM_CONTIGUOUS_KV_L2: dict[int, int] = {} # device index -> L2 bytes, gfx1151 cards only+++def _rocm_contiguous_kv_wanted(key: Any, value: Any) -> bool:+ """True when one head of the strided ``[B, S, H, D]`` k/v exceeds L2."""+ try:+ if key.dim() != 4 or value.dim() != 4:+ return False+ l2 = _ROCM_CONTIGUOUS_KV_L2.get(key.device.index)+ if not l2:+ return False+ strided = [t for t in (key, value) if not t.permute(0, 2, 1, 3).is_contiguous()]+ return sum(t.shape[1] * t.shape[3] for t in strided) * key.element_size() > l2+ except Exception: # noqa: BLE001+ return False+++def _heads_major(t: Any) -> Any:+ """``[B, H, S, D]`` memory so the native backend's permute hands SDPA a dense tensor."""+ if t.permute(0, 2, 1, 3).is_contiguous():+ return t+ return t.permute(0, 2, 1, 3).contiguous().permute(0, 2, 1, 3)+++def install_rocm_contiguous_kv(target: Any) -> bool:+ """gfx1151 only: k/v over L2 reach SDPA contiguous (q is not copied). False elsewhere: other GPUs untouched."""+ if os.environ.get(ROCM_CONTIGUOUS_KV_ENV, "").strip().lower() in ("0", "false", "no", "off"):+ return False+ if not _is_cuda_rocm(target) or _rocm_gfx_arch(target) not in ROCM_CONTIGUOUS_KV_ARCHES:+ return False+ try:+ import torch+ from diffusers.models.attention_dispatch import (+ AttentionBackendName,+ _AttentionBackendRegistry,+ )++ device = torch.device(+ _indexed_cuda_device(+ str(+ getattr(target, "torch_device", None)+ or getattr(target, "device", None)+ or "cuda"+ )+ )+ )+ index = device.index if device.index is not None else torch.cuda.current_device()+ l2 = int(getattr(torch.cuda.get_device_properties(index), "L2_cache_size", 0) or 0)+ backends = _AttentionBackendRegistry._backends+ native_fn = backends[AttentionBackendName.NATIVE]+ except Exception: # noqa: BLE001+ return False+ if l2 <= 0 or not callable(native_fn):+ return False+ _ROCM_CONTIGUOUS_KV_L2[index] = l2+ if getattr(native_fn, ROCM_CONTIGUOUS_KV_ATTR, False):+ return True++ def _contiguous_kv(*args: Any, **kwargs: Any) -> Any:+ args = list(args)+ key = args[1] if len(args) > 1 else kwargs.get("key")+ value = args[2] if len(args) > 2 else kwargs.get("value")+ if kwargs.get("_parallel_config") is None and _rocm_contiguous_kv_wanted(key, value):+ key, value = _heads_major(key), _heads_major(value)+ if len(args) > 1:+ args[1] = key+ else:+ kwargs["key"] = key+ if len(args) > 2:+ args[2] = value+ else:+ kwargs["value"] = value+ return native_fn(*args, **kwargs)++ setattr(_contiguous_kv, ROCM_CONTIGUOUS_KV_ATTR, True)+ _contiguous_kv.__wrapped__ = native_fn # type: ignore[attr-defined]+ try:+ backends[AttentionBackendName.NATIVE] = _contiguous_kv+ except Exception: # noqa: BLE001+ return False+ return True++def select_attention_backend(target: Any,requested: Optional[str],@@ -3436,6 +3525,7 @@ def apply_attention_backend(guard_rocm_fused_sdpa(target, logger)except Exception as exc:_warn(logger, "ROCm fused SDPA check", exc)+ install_rocm_contiguous_kv(target)setters = [sfor s in (getattr(t, "set_attention_backend", None) for t in _attention_dits(pipe))diff --git a/studio/backend/tests/test_diffusion_attention_rocm_contiguous_kv.py b/studio/backend/tests/test_diffusion_attention_rocm_contiguous_kv.pynew file mode 100644index 00000000000..bfd9c291a78--- /dev/null+++ b/studio/backend/tests/test_diffusion_attention_rocm_contiguous_kv.py@@ -0,0 +1,114 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""gfx1151 contiguous k/v wrapper. CPU tensors stand in for the card (L2 table key is None on CPU)."""++from types import SimpleNamespace++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("diffusers")++from diffusers.models.attention_dispatch import ( # noqa: E402+ AttentionBackendName,+ _AttentionBackendRegistry,+)++from core.inference import diffusion_attention as da # noqa: E402++TARGET = SimpleNamespace(device = "cuda", torch_device = "cuda:0", ordinal = 0)+++def registry(monkeypatch):+ backends = _AttentionBackendRegistry._backends+ monkeypatch.setitem(+ backends, AttentionBackendName.NATIVE, backends[AttentionBackendName.NATIVE]+ )+ monkeypatch.setattr(da, "_ROCM_CONTIGUOUS_KV_L2", {})+ monkeypatch.delenv(da.ROCM_CONTIGUOUS_KV_ENV, raising = False)+ return backends+++def _fake_card(+ monkeypatch,+ arch = "gfx1151",+ rocm = True,+ l2 = 2 * 1024 * 1024,+):+ monkeypatch.setattr(da, "_is_cuda_rocm", lambda target: rocm)+ monkeypatch.setattr(da, "_rocm_gfx_arch", lambda target: arch)+ monkeypatch.setattr(+ torch.cuda, "get_device_properties", lambda index: SimpleNamespace(L2_cache_size = l2)+ )+++ "arch, rocm", [("gfx1100", True), ("gfx1201", True), ("gfx942", True), ("", False)]+)+def test_other_gpus_keep_the_stock_native_backend(registry, monkeypatch, arch, rocm):+ _fake_card(monkeypatch, arch = arch, rocm = rocm)+ native = registry[AttentionBackendName.NATIVE]+ assert da.install_rocm_contiguous_kv(TARGET) is False+ assert registry[AttentionBackendName.NATIVE] is native+++def test_kill_switch(registry, monkeypatch):+ _fake_card(monkeypatch)+ monkeypatch.setenv(da.ROCM_CONTIGUOUS_KV_ENV, "0")+ native = registry[AttentionBackendName.NATIVE]+ assert da.install_rocm_contiguous_kv(TARGET) is False+ assert registry[AttentionBackendName.NATIVE] is native+++def test_installs_once_on_gfx1151(registry, monkeypatch):+ _fake_card(monkeypatch)+ native = registry[AttentionBackendName.NATIVE]+ assert da.install_rocm_contiguous_kv(TARGET) is True+ wrapped = registry[AttentionBackendName.NATIVE]+ assert wrapped is not native and wrapped.__wrapped__ is native+ assert da.install_rocm_contiguous_kv(TARGET) is True+ assert registry[AttentionBackendName.NATIVE] is wrapped+ assert da._ROCM_CONTIGUOUS_KV_L2 == {0: 2 * 1024 * 1024}+++def test_size_gate_matches_one_head_of_strided_kv_against_l2(monkeypatch):+ l2 = 1 << 16+ monkeypatch.setattr(da, "_ROCM_CONTIGUOUS_KV_L2", {None: l2})+ d = 32+ edge = l2 // (2 * d * 4) # fp32 k + v of one head exactly fill L2+ at = torch.zeros(1, edge, 2, d)+ over = torch.zeros(1, edge + 1, 2, d)+ assert not da._rocm_contiguous_kv_wanted(at, at)+ assert da._rocm_contiguous_kv_wanted(over, over)+ dense = over.permute(0, 2, 1, 3).contiguous().permute(0, 2, 1, 3)+ assert not da._rocm_contiguous_kv_wanted(dense, dense)+ monkeypatch.setattr(da, "_ROCM_CONTIGUOUS_KV_L2", {5: l2})+ assert not da._rocm_contiguous_kv_wanted(over, over) # no L2 entry: not gfx1151+++def test_wrapper_hands_sdpa_contiguous_kv_and_keeps_outputs(registry, monkeypatch):+ _fake_card(monkeypatch)+ da.install_rocm_contiguous_kv(TARGET)+ monkeypatch.setattr(da, "_ROCM_CONTIGUOUS_KV_L2", {None: 1 << 12})+ wrapped = registry[AttentionBackendName.NATIVE]+ seen = []+ real = torch.nn.functional.scaled_dot_product_attention++ def spy(*args, **kwargs):+ seen.append((kwargs["key"].is_contiguous(), kwargs["value"].is_contiguous()))+ return real(*args, **kwargs)++ monkeypatch.setattr(torch.nn.functional, "scaled_dot_product_attention", spy)+ g = torch.Generator().manual_seed(0)+ q, k, v = (torch.randn(1, 64, 2, 16, generator = g) for _ in range(3))+ ref = wrapped.__wrapped__(q, k, v)+ assert seen.pop() == (False, False)+ assert torch.equal(wrapped(query = q, key = k, value = v), ref)+ assert seen.pop() == (True, True)+ assert torch.equal(wrapped(q, k, v), ref) # positional, as Studio's dispatch guards call it+ assert seen.pop() == (True, True)+ small = (t[:, :8] for t in (q, k, v))+ wrapped(*small)+ assert seen.pop() == (False, False)