Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio CUDA graphs: the sm_89 fp8 stream-K guard reads every split card, not only the primary (follow-up to #17166) (#17194)

Daniel Han64ccbc2dfc1a
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 0753a42a2ae..67621629622 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -2666,19 +2666,38 @@ def fp8_rowwise_capture_refusal(modules: Any, device: Any = None) -> Optional[st
cap = tuple(torch.cuda.get_device_capability(device))
except Exception: # noqa: BLE001
return None
- if cap not in FP8_ROWWISE_STREAMK_CAPS:
- return None
+ caps: dict = {}
+
+ def _weight_cap(w: Any) -> Any:
+ # Layer split: an sm_89 secondary card hangs too.
+ if cap in FP8_ROWWISE_STREAMK_CAPS:
+ return cap
+ d = getattr(w, "device", None)
+ if getattr(d, "type", None) != "cuda":
+ return cap
+ key = str(d)
+ if key not in caps:
+ try:
+ caps[key] = tuple(torch.cuda.get_device_capability(d))
+ except Exception: # noqa: BLE001
+ caps[key] = None
+ return caps[key]
+
for module in modules or ():
try:
weights = list(module.parameters())
except Exception: # noqa: BLE001
continue
- if any(_fp8_rowwise_weight(w) for w in weights):
- return (
- f"the row-wise fp8 GEMM on this sm_{cap[0]}{cap[1]} card cannot replay in a CUDA graph under torch "
- f"{major}.{minor} (it hangs; fixed in torch {FP8_ROWWISE_STREAMK_FIXED[0]}."
- f"{FP8_ROWWISE_STREAMK_FIXED[1]}, pytorch/pytorch#177573)"
- )
+ for w in weights:
+ if not _fp8_rowwise_weight(w):
+ continue
+ hit = _weight_cap(w)
+ if hit in FP8_ROWWISE_STREAMK_CAPS:
+ return (
+ f"the row-wise fp8 GEMM on this sm_{hit[0]}{hit[1]} card cannot replay in a CUDA graph under "
+ f"torch {major}.{minor} (it hangs; fixed in torch {FP8_ROWWISE_STREAMK_FIXED[0]}."
+ f"{FP8_ROWWISE_STREAMK_FIXED[1]}, pytorch/pytorch#177573)"
+ )
return None
diff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.py
index ae154291d37..d11d960cc8c 100644
--- a/studio/backend/tests/test_diffusion_cuda_graph.py
+++ b/studio/backend/tests/test_diffusion_cuda_graph.py
@@ -920,6 +920,42 @@ def test_fp8_streamk_check_reads_the_onload_card(stub_torch, monkeypatch):
assert seen == ["cuda:1"]
+def test_fp8_streamk_check_reads_each_split_card(stub_torch, monkeypatch):
+ """An sm_89 split card under an sm_90 primary is refused too."""
+ _hazard_torch(stub_torch, monkeypatch)
+ caps = {"cuda:0": (9, 0), "cuda:1": (8, 9), "cuda:2": (8, 6)}
+ stub_torch.cuda.get_device_capability = lambda device = None: caps[str(device)]
+ target = types.SimpleNamespace(
+ device = "cuda", backend = "cuda", dtype = "bfloat16", torch_device = "cuda:0"
+ )
+
+ def on(card, w):
+ w.device = _Dev(f"cuda:{card}")
+ return w
+
+ ada = types.SimpleNamespace(transformer = _QuantDiT(on(0, _fp8_weight()), on(1, _fp8_weight())))
+ ok, reason = _eligible(monkeypatch, target = target, pipe = ada)
+ assert ok is False and "sm_89" in reason
+ ampere = types.SimpleNamespace(
+ transformer = _QuantDiT(on(0, _fp8_weight()), on(2, _fp8_weight()))
+ )
+ assert _eligible(monkeypatch, target = target, pipe = ampere) == (True, "eligible")
+ plain = Int8Tensor()
+ plain.device = _Dev("cuda:1")
+ mixed = types.SimpleNamespace(transformer = _QuantDiT(on(0, _fp8_weight()), plain))
+ assert _eligible(monkeypatch, target = target, pipe = mixed) == (True, "eligible")
+
+
+class _Dev:
+ type = "cuda"
+
+ def __init__(self, name):
+ self.name = name
+
+ def __str__(self):
+ return self.name
+
+
def test_graph_eligible_needs_torch_cuda(stub_torch, monkeypatch):
stub_torch.cuda.is_available = lambda: False
assert _eligible(monkeypatch)[1] == "torch.cuda unavailable"