Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio ROCm: tell self from cross attention without guarding the query length on the key length (#15458)

Daniel Han88ff2b0b14a0
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index 8a72791e73c..5b93ad9056c 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -974,6 +974,21 @@ _ROCM_FLASH_GATE_LOCK = threading.Lock()
# The native SDPA baseline the in-memory verdicts were timed against; when it moves (a later load re-enables or disables
# the fused SDPA kernels), both in-memory caches are dropped and the disk cache, keyed on it, decides again.
_ROCM_FLASH_BASELINE: list = [None]
+# ``torch.fx.experimental.symbolic_shapes.statically_known_true`` when this torch's Dynamo traces it in graph (no
+# guard), else None. Resolved outside tracing, when the dispatch guard installs.
+_ROCM_FLASH_STATICALLY_KNOWN_TRUE: list = [None]
+
+
+def _resolve_statically_known_true() -> None:
+ try:
+ from torch._dynamo import trace_rules
+ from torch.fx.experimental.symbolic_shapes import statically_known_true
+
+ name = "torch.fx.experimental.symbolic_shapes.statically_known_true"
+ if name in trace_rules.manual_torch_name_rule_map:
+ _ROCM_FLASH_STATICALLY_KNOWN_TRUE[0] = statically_known_true
+ except Exception: # noqa: BLE001 - older torch: the direct comparison stays
+ pass
def _sync_rocm_flash_baseline() -> None:
@@ -1050,7 +1065,11 @@ def _rocm_flash_length_reason(query: Any, key: Any) -> Optional[str]:
return None
h, d, geometries = match[0][2], match[0][3], tuple(sorted(match[1]))
b_bucket = _rocm_flash_batch_bucket(batch)
- joint = kv == seq # self / joint attention: one symbol, so no guard
+ # Self / joint attention: keys and queries share one length symbol. ``statically_known_true`` answers that
+ # without a guard, so cross attention (an independent key length) never specialises on the query length.
+ # A torch whose Dynamo cannot trace it (2.6) compares directly, which guards.
+ known = _ROCM_FLASH_STATICALLY_KNOWN_TRUE[0]
+ joint = known(kv == seq) if known is not None else kv == seq
kv_bucket = None if joint else _rocm_flash_bucket(kv)
# Guard only where the route changes, so neighbouring same-verdict buckets reuse this graph.
segments = _rocm_flash_traced_segments(
@@ -2182,6 +2201,7 @@ def _rocm_flash_reroute_reason(query: Any, key: Any, value: Any, attn_mask: Any)
def _install_rocm_flash_dispatch_guard() -> bool:
+ _resolve_statically_known_true()
try:
import torch
torch.compiler.assume_constant_result(_rocm_flash_traced_reason)
diff --git a/studio/backend/tests/test_diffusion_attention_rocm_flash.py b/studio/backend/tests/test_diffusion_attention_rocm_flash.py
index f0abfa8fbe6..1484bcbb546 100644
--- a/studio/backend/tests/test_diffusion_attention_rocm_flash.py
+++ b/studio/backend/tests/test_diffusion_attention_rocm_flash.py
@@ -1964,7 +1964,12 @@ class _CompiledDit(__import__("torch").nn.Module):
return self._guarded(q, q, q, None)
-def _compiled_env(monkeypatch, auto, flash_ms):
+def _compiled_env(
+ monkeypatch,
+ auto,
+ flash_ms,
+ dit_cls = None,
+):
torch = pytest.importorskip("torch")
_rocm(monkeypatch)
ad = pytest.importorskip("diffusers.models.attention_dispatch")
@@ -1997,7 +2002,7 @@ def _compiled_env(monkeypatch, auto, flash_ms):
return {**_GOOD, "flash_ms": flash_ms.get(tokens, 90.0), "tokens": tokens}
monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)
- dit = _CompiledDit(guarded)
+ dit = (dit_cls or _CompiledDit)(guarded)
backend = att.ROCM_FLASH_CANDIDATE if auto else "flash"
assert (
att.apply_attention_backend(
@@ -2054,6 +2059,63 @@ def test_compiled_dynamic_lengths_recompile_per_bucket_not_per_length(monkeypatc
torch._dynamo.reset()
+class _CompiledCrossDit(_CompiledDit):
+ def forward(self, q, k):
+ return self._guarded(q, k, k, None)
+
+
+def _traces_statically_known_true():
+ try:
+ from torch._dynamo import trace_rules
+ except Exception: # noqa: BLE001
+ return False
+ name = "torch.fx.experimental.symbolic_shapes.statically_known_true"
+ return name in trace_rules.manual_torch_name_rule_map
+
+
+# Cross attention: the key length is its own symbol. Telling self from cross attention must not compare the two
+# lengths with a guard, or a query length equal to the key length recompiles and a marked dynamic range is rejected.
+_needs_skt = pytest.mark.skipif(
+ not _traces_statically_known_true(),
+ reason = "this torch's Dynamo cannot trace statically_known_true",
+)
+
+
+@_needs_skt
+def test_compiled_cross_attention_does_not_guard_query_length_on_key_length(monkeypatch):
+ torch, dit, timed = _compiled_env(monkeypatch, True, {4096: 70.0}, _CompiledCrossDit)
+ torch._dynamo.reset()
+ frames = []
+ compiled = torch.compile(
+ dit,
+ backend = lambda gm, inputs: frames.append(gm) or gm.forward,
+ dynamic = True,
+ fullgraph = True,
+ )
+ bf16 = torch.bfloat16
+ k = torch.zeros(1, 6000, 2, 16, dtype = bf16)
+ counts = []
+ for n in (5000, 6000, 7000, 5500):
+ assert float(compiled(torch.zeros(1, n, 2, 16, dtype = bf16), k).flatten()[0]) == 1.0
+ counts.append(len(frames))
+ assert counts == [1, 1, 1, 1]
+ torch._dynamo.reset()
+
+
+@_needs_skt
+def test_compiled_cross_attention_accepts_a_marked_query_range_spanning_the_key_length(monkeypatch):
+ torch, dit, timed = _compiled_env(monkeypatch, True, {4096: 70.0}, _CompiledCrossDit)
+ torch._dynamo.reset()
+ bf16 = torch.bfloat16
+ q = torch.zeros(1, 5000, 2, 16, dtype = bf16)
+ k = torch.zeros(1, 6000, 2, 16, dtype = bf16)
+ torch._dynamo.mark_dynamic(q, 1, min = 4500, max = 7500)
+ torch._dynamo.mark_static(k, 1)
+ compiled = torch.compile(dit, backend = "eager", dynamic = True, fullgraph = True)
+ assert float(compiled(q, k).flatten()[0]) == 1.0
+ torch._dynamo.reset()
+
+
@pytest.mark.parametrize("native_8192", [False, True])
def test_compiled_lengths_across_known_buckets_guard_only_where_the_route_changes(
monkeypatch, native_8192