Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio CUDA graphs: the sm_89 fp8 stream-K guard reads every split card, not only the primary (follow-up to #17166) (#17194)
Daniel Han64ccbc2dfc1a
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex 0753a42a2ae..67621629622 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -2666,19 +2666,38 @@ def fp8_rowwise_capture_refusal(modules: Any, device: Any = None) -> Optional[stcap = tuple(torch.cuda.get_device_capability(device))except Exception: # noqa: BLE001return None- if cap not in FP8_ROWWISE_STREAMK_CAPS:- return None+ caps: dict = {}++ def _weight_cap(w: Any) -> Any:+ # Layer split: an sm_89 secondary card hangs too.+ if cap in FP8_ROWWISE_STREAMK_CAPS:+ return cap+ d = getattr(w, "device", None)+ if getattr(d, "type", None) != "cuda":+ return cap+ key = str(d)+ if key not in caps:+ try:+ caps[key] = tuple(torch.cuda.get_device_capability(d))+ except Exception: # noqa: BLE001+ caps[key] = None+ return caps[key]+for module in modules or ():try:weights = list(module.parameters())except Exception: # noqa: BLE001continue- if any(_fp8_rowwise_weight(w) for w in weights):- return (- f"the row-wise fp8 GEMM on this sm_{cap[0]}{cap[1]} card cannot replay in a CUDA graph under torch "- f"{major}.{minor} (it hangs; fixed in torch {FP8_ROWWISE_STREAMK_FIXED[0]}."- f"{FP8_ROWWISE_STREAMK_FIXED[1]}, pytorch/pytorch#177573)"- )+ for w in weights:+ if not _fp8_rowwise_weight(w):+ continue+ hit = _weight_cap(w)+ if hit in FP8_ROWWISE_STREAMK_CAPS:+ return (+ f"the row-wise fp8 GEMM on this sm_{hit[0]}{hit[1]} card cannot replay in a CUDA graph under "+ f"torch {major}.{minor} (it hangs; fixed in torch {FP8_ROWWISE_STREAMK_FIXED[0]}."+ f"{FP8_ROWWISE_STREAMK_FIXED[1]}, pytorch/pytorch#177573)"+ )return Nonediff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.pyindex ae154291d37..d11d960cc8c 100644--- a/studio/backend/tests/test_diffusion_cuda_graph.py+++ b/studio/backend/tests/test_diffusion_cuda_graph.py@@ -920,6 +920,42 @@ def test_fp8_streamk_check_reads_the_onload_card(stub_torch, monkeypatch):assert seen == ["cuda:1"]+def test_fp8_streamk_check_reads_each_split_card(stub_torch, monkeypatch):+ """An sm_89 split card under an sm_90 primary is refused too."""+ _hazard_torch(stub_torch, monkeypatch)+ caps = {"cuda:0": (9, 0), "cuda:1": (8, 9), "cuda:2": (8, 6)}+ stub_torch.cuda.get_device_capability = lambda device = None: caps[str(device)]+ target = types.SimpleNamespace(+ device = "cuda", backend = "cuda", dtype = "bfloat16", torch_device = "cuda:0"+ )++ def on(card, w):+ w.device = _Dev(f"cuda:{card}")+ return w++ ada = types.SimpleNamespace(transformer = _QuantDiT(on(0, _fp8_weight()), on(1, _fp8_weight())))+ ok, reason = _eligible(monkeypatch, target = target, pipe = ada)+ assert ok is False and "sm_89" in reason+ ampere = types.SimpleNamespace(+ transformer = _QuantDiT(on(0, _fp8_weight()), on(2, _fp8_weight()))+ )+ assert _eligible(monkeypatch, target = target, pipe = ampere) == (True, "eligible")+ plain = Int8Tensor()+ plain.device = _Dev("cuda:1")+ mixed = types.SimpleNamespace(transformer = _QuantDiT(on(0, _fp8_weight()), plain))+ assert _eligible(monkeypatch, target = target, pipe = mixed) == (True, "eligible")+++class _Dev:+ type = "cuda"++ def __init__(self, name):+ self.name = name++ def __str__(self):+ return self.name++def test_graph_eligible_needs_torch_cuda(stub_torch, monkeypatch):stub_torch.cuda.is_available = lambda: Falseassert _eligible(monkeypatch)[1] == "torch.cuda unavailable"