Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio ROCm: tell self from cross attention without guarding the query length on the key length (#15458)
Daniel Han88ff2b0b14a0
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex 8a72791e73c..5b93ad9056c 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -974,6 +974,21 @@ _ROCM_FLASH_GATE_LOCK = threading.Lock()# The native SDPA baseline the in-memory verdicts were timed against; when it moves (a later load re-enables or disables# the fused SDPA kernels), both in-memory caches are dropped and the disk cache, keyed on it, decides again._ROCM_FLASH_BASELINE: list = [None]+# ``torch.fx.experimental.symbolic_shapes.statically_known_true`` when this torch's Dynamo traces it in graph (no+# guard), else None. Resolved outside tracing, when the dispatch guard installs.+_ROCM_FLASH_STATICALLY_KNOWN_TRUE: list = [None]+++def _resolve_statically_known_true() -> None:+ try:+ from torch._dynamo import trace_rules+ from torch.fx.experimental.symbolic_shapes import statically_known_true++ name = "torch.fx.experimental.symbolic_shapes.statically_known_true"+ if name in trace_rules.manual_torch_name_rule_map:+ _ROCM_FLASH_STATICALLY_KNOWN_TRUE[0] = statically_known_true+ except Exception: # noqa: BLE001 - older torch: the direct comparison stays+ passdef _sync_rocm_flash_baseline() -> None:@@ -1050,7 +1065,11 @@ def _rocm_flash_length_reason(query: Any, key: Any) -> Optional[str]:return Noneh, d, geometries = match[0][2], match[0][3], tuple(sorted(match[1]))b_bucket = _rocm_flash_batch_bucket(batch)- joint = kv == seq # self / joint attention: one symbol, so no guard+ # Self / joint attention: keys and queries share one length symbol. ``statically_known_true`` answers that+ # without a guard, so cross attention (an independent key length) never specialises on the query length.+ # A torch whose Dynamo cannot trace it (2.6) compares directly, which guards.+ known = _ROCM_FLASH_STATICALLY_KNOWN_TRUE[0]+ joint = known(kv == seq) if known is not None else kv == seqkv_bucket = None if joint else _rocm_flash_bucket(kv)# Guard only where the route changes, so neighbouring same-verdict buckets reuse this graph.segments = _rocm_flash_traced_segments(@@ -2182,6 +2201,7 @@ def _rocm_flash_reroute_reason(query: Any, key: Any, value: Any, attn_mask: Any)def _install_rocm_flash_dispatch_guard() -> bool:+ _resolve_statically_known_true()try:import torchtorch.compiler.assume_constant_result(_rocm_flash_traced_reason)diff --git a/studio/backend/tests/test_diffusion_attention_rocm_flash.py b/studio/backend/tests/test_diffusion_attention_rocm_flash.pyindex f0abfa8fbe6..1484bcbb546 100644--- a/studio/backend/tests/test_diffusion_attention_rocm_flash.py+++ b/studio/backend/tests/test_diffusion_attention_rocm_flash.py@@ -1964,7 +1964,12 @@ class _CompiledDit(__import__("torch").nn.Module):return self._guarded(q, q, q, None)-def _compiled_env(monkeypatch, auto, flash_ms):+def _compiled_env(+ monkeypatch,+ auto,+ flash_ms,+ dit_cls = None,+):torch = pytest.importorskip("torch")_rocm(monkeypatch)ad = pytest.importorskip("diffusers.models.attention_dispatch")@@ -1997,7 +2002,7 @@ def _compiled_env(monkeypatch, auto, flash_ms):return {**_GOOD, "flash_ms": flash_ms.get(tokens, 90.0), "tokens": tokens}monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)- dit = _CompiledDit(guarded)+ dit = (dit_cls or _CompiledDit)(guarded)backend = att.ROCM_FLASH_CANDIDATE if auto else "flash"assert (att.apply_attention_backend(@@ -2054,6 +2059,63 @@ def test_compiled_dynamic_lengths_recompile_per_bucket_not_per_length(monkeypatctorch._dynamo.reset()+class _CompiledCrossDit(_CompiledDit):+ def forward(self, q, k):+ return self._guarded(q, k, k, None)+++def _traces_statically_known_true():+ try:+ from torch._dynamo import trace_rules+ except Exception: # noqa: BLE001+ return False+ name = "torch.fx.experimental.symbolic_shapes.statically_known_true"+ return name in trace_rules.manual_torch_name_rule_map+++# Cross attention: the key length is its own symbol. Telling self from cross attention must not compare the two+# lengths with a guard, or a query length equal to the key length recompiles and a marked dynamic range is rejected.+_needs_skt = pytest.mark.skipif(+ not _traces_statically_known_true(),+ reason = "this torch's Dynamo cannot trace statically_known_true",+)+++@_needs_skt+def test_compiled_cross_attention_does_not_guard_query_length_on_key_length(monkeypatch):+ torch, dit, timed = _compiled_env(monkeypatch, True, {4096: 70.0}, _CompiledCrossDit)+ torch._dynamo.reset()+ frames = []+ compiled = torch.compile(+ dit,+ backend = lambda gm, inputs: frames.append(gm) or gm.forward,+ dynamic = True,+ fullgraph = True,+ )+ bf16 = torch.bfloat16+ k = torch.zeros(1, 6000, 2, 16, dtype = bf16)+ counts = []+ for n in (5000, 6000, 7000, 5500):+ assert float(compiled(torch.zeros(1, n, 2, 16, dtype = bf16), k).flatten()[0]) == 1.0+ counts.append(len(frames))+ assert counts == [1, 1, 1, 1]+ torch._dynamo.reset()+++@_needs_skt+def test_compiled_cross_attention_accepts_a_marked_query_range_spanning_the_key_length(monkeypatch):+ torch, dit, timed = _compiled_env(monkeypatch, True, {4096: 70.0}, _CompiledCrossDit)+ torch._dynamo.reset()+ bf16 = torch.bfloat16+ q = torch.zeros(1, 5000, 2, 16, dtype = bf16)+ k = torch.zeros(1, 6000, 2, 16, dtype = bf16)+ torch._dynamo.mark_dynamic(q, 1, min = 4500, max = 7500)+ torch._dynamo.mark_static(k, 1)+ compiled = torch.compile(dit, backend = "eager", dynamic = True, fullgraph = True)+ assert float(compiled(q, k).flatten()[0]) == 1.0+ torch._dynamo.reset()++@pytest.mark.parametrize("native_8192", [False, True])def test_compiled_lengths_across_known_buckets_guard_only_where_the_route_changes(monkeypatch, native_8192