Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep FlashAttention 4 inside a fullgraph compile (#12686)

Daniel Han5ac49d41d899
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index d7f6ae43f63..75d21429d4b 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -573,7 +573,7 @@ _SAGE_OP_LOCK = threading.Lock()
def _sage_custom_op(sage_fn: Any, op_name: str = _SAGE_OP_NAME) -> Optional[Any]:
- """An opaque ``torch.library`` op around diffusers' sage function (NHD), or None."""
+ """An opaque ``torch.library`` op around a diffusers attention function (NHD), or None. Used for Sage and FA4."""
with _SAGE_OP_LOCK:
slot = _SAGE_OPS.setdefault(op_name, {})
if "op" in slot:
@@ -712,9 +712,16 @@ def _note_fa4_reroute(reason: str) -> None:
_note_reroute("FlashAttention 4", reason, _FA4_ROUTED, _FA4_ROUTED_LOGGED)
+# The hub FA4 launch (CuTe DSL, tvm-ffi) breaks a fullgraph compile 5 times per Flux block: keep it opaque, like Sage.
+_FA4_OP_NAME = "unsloth_studio::flash_4_hub_attention_nhd"
+
+
def _install_fa4_dispatch_guard() -> bool:
return _install_dispatch_guard(
- "flash_4_hub", lambda *a: _fa4_reroute_reason(*a), _note_fa4_reroute
+ "flash_4_hub",
+ lambda *a: _fa4_reroute_reason(*a),
+ _note_fa4_reroute,
+ lambda fn: _sage_custom_op(fn, _FA4_OP_NAME),
)
diff --git a/studio/backend/tests/test_diffusion_attention_sage_gate.py b/studio/backend/tests/test_diffusion_attention_sage_gate.py
index b3d508b1ca9..7dc68298a48 100644
--- a/studio/backend/tests/test_diffusion_attention_sage_gate.py
+++ b/studio/backend/tests/test_diffusion_attention_sage_gate.py
@@ -412,6 +412,59 @@ def test_unguarded_sage_call_breaks_a_fullgraph_compile():
torch._dynamo.reset()
+def _untraceable_fa4(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ scale = None,
+ is_causal = False,
+ return_lse = False,
+ _parallel_config = None,
+):
+ torch._dynamo.graph_break() # stands in for the CuTe DSL / tvm-ffi launch Dynamo cannot trace
+ return _native(query.float(), key.float(), value.float()).to(query.dtype)
+
+
[email protected]("head_dim", [64, 128])
+def test_fa4_call_compiles_fullgraph_through_the_guard(monkeypatch, head_dim):
+ backends = dispatch._AttentionBackendRegistry._backends
+ backends[dispatch.AttentionBackendName.FLASH_4_HUB] = _untraceable_fa4
+ assert att._install_fa4_dispatch_guard() is True
+ monkeypatch.setattr(att, "_fa4_reroute_reason", lambda *a: None)
+ guarded = backends[dispatch.AttentionBackendName.FLASH_4_HUB]
+ q, k, v = _qkv(head_dim = head_dim, dtype = torch.bfloat16)
+
+ def block(q, k, v):
+ return guarded(query = q * 1.0, key = k, value = v).sum(-1)
+
+ torch._dynamo.reset()
+ compiled = torch.compile(block, backend = "aot_eager", fullgraph = True)
+ torch.testing.assert_close(
+ compiled(q, k, v), _native(q.float(), k.float(), v.float()).to(q.dtype).sum(-1)
+ )
+ torch._dynamo.reset()
+
+
+def test_fa4_and_sage_ops_are_separate():
+ sage = att._sage_custom_op(_arch_reading_sage)
+ fa4 = att._sage_custom_op(_untraceable_fa4, att._FA4_OP_NAME)
+ assert fa4 is not None and fa4 is not sage
+ assert att._SAGE_OPS[att._FA4_OP_NAME]["fn"] is _untraceable_fa4
+ assert att._SAGE_OPS[att._SAGE_OP_NAME]["fn"] is _arch_reading_sage
+
+
+def test_unguarded_fa4_call_breaks_a_fullgraph_compile():
+ q, k, v = _qkv(head_dim = 64, dtype = torch.bfloat16)
+ torch._dynamo.reset()
+ compiled = torch.compile(
+ lambda q, k, v: _untraceable_fa4(q, k, v), backend = "aot_eager", fullgraph = True
+ )
+ with pytest.raises(Exception):
+ compiled(q, k, v)
+ torch._dynamo.reset()
+
+
def test_engaged_backend_is_tagged_on_every_dit(monkeypatch):
monkeypatch.setattr(att, "_run_sage_probe", lambda d, dt, hd = 128: "")
t, t2 = _Transformer(128), _Transformer(128)