Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio ROCm layer split: probe secondary cards read-only, at the attention dtype (#14655)
Daniel Han10c52ed46359
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex c3d8eaceb7c..3dc1afbe126 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -8,6 +8,8 @@ lazy."""from __future__ import annotations+import contextlib+import contextvarsimport functoolsimport osimport re@@ -115,6 +117,20 @@ _SDPA_PROBE_CACHE: dict[tuple[str, str], tuple[str, ...]] = {}_ROCM_PROBE_RETRY_S = 120.0_ROCM_PROBE_INCOMPLETE: dict[tuple[str, str], float] = {}+# Candidate-card probes must not flip the process-wide SDPA switches the running card also uses.+_SDPA_PROBE_READ_ONLY: contextvars.ContextVar[bool] = contextvars.ContextVar(+ "_sdpa_probe_read_only", default = False+)+++def read_only_sdpa_probe():+ token = _SDPA_PROBE_READ_ONLY.set(True)+ try:+ yield+ finally:+ _SDPA_PROBE_READ_ONLY.reset(token)+def _probe_rocm_sdpa_kernels(device: str, dtype: Any) -> tuple[str, ...]:import time@@ -126,8 +142,11 @@ def _probe_rocm_sdpa_kernels(device: str, dtype: Any) -> tuple[str, ...]:available = _run_rocm_sdpa_children(device, dtype)if available:_ROCM_PROBE_INCOMPLETE.pop(key, None)- # Whichever check completes the probe applies its answer; fp32 never runs fused kernels.- if _rocm_guard_allowed() and str(dtype) in ("torch.float16", "torch.bfloat16"):+ if (+ not _SDPA_PROBE_READ_ONLY.get()+ and _rocm_guard_allowed()+ and str(dtype) in ("torch.float16", "torch.bfloat16")+ ):try:_apply_rocm_guard(available)except Exception: # noqa: BLE001 - a diagnostic may never fail a load@@ -425,6 +444,33 @@ def guard_rocm_fused_sdpa(target: Any, logger: Any = None) -> tuple[str, ...]:return _apply_rocm_guard(available, logger)+def guard_rocm_fused_sdpa_cards(+ devices: Any,+ dtype: Any,+ logger: Any = None,+) -> tuple[str, ...]:+ """Switches are process-wide, so only kernels that ran on ALL cards stay on."""+ from types import SimpleNamespace++ if not _is_cuda_rocm(SimpleNamespace(device = "cuda")) or not _rocm_guard_allowed():+ return ()+ if dtype is None:+ import torch+ dtype = torch.float16 # _sdpa_capability's default+ if str(dtype) not in ("torch.float16", "torch.bfloat16"):+ return ()+ answers = []+ with read_only_sdpa_probe():+ for device in devices:+ available = _sdpa_capability(SimpleNamespace(device = str(device), dtype = dtype))+ if available:+ answers.append(available)+ if not answers:+ return ()+ common = tuple(k for k in answers[0] if all(k in a for a in answers[1:]))+ return _apply_rocm_guard(common, logger)++def _rocm_guard_allowed() -> bool:return os.environ.get(ROCM_FUSED_SDPA_ALLOW_ENV, "").strip().lower() not in ("1",diff --git a/studio/backend/core/inference/diffusion_layer_split_plan.py b/studio/backend/core/inference/diffusion_layer_split_plan.pyindex c5944d07509..e2a8dac1f3e 100644--- a/studio/backend/core/inference/diffusion_layer_split_plan.py+++ b/studio/backend/core/inference/diffusion_layer_split_plan.py@@ -701,6 +701,61 @@ def _denoiser_dtype(module: Any) -> Any:return None+def _probe_dtype(dtype: Any) -> Any:+ return dtype if str(dtype) in _PROBE_DTYPES else None+++def _projection_dtype(proj: Any) -> Any:+ registry = getattr(proj, "_diffusers_hook", None)+ get_hook = getattr(registry, "get_hook", None)+ if callable(get_hook):+ with contextlib.suppress(Exception):+ dtype = _probe_dtype(getattr(get_hook("layerwise_casting"), "compute_dtype", None))+ if dtype is not None:+ return dtype+ for dtype in (+ getattr(proj, "compute_dtype", None),+ getattr(getattr(proj, "weight", None), "dtype", None),+ getattr(getattr(proj, "bias", None), "dtype", None),+ ):+ dtype = _probe_dtype(dtype)+ if dtype is not None:+ return dtype+ return None+++def _attention_dtype(module: Any) -> Any:+ # Only to_q / to_qkv: MiniMax-H3 proj_in and Krea-2 / FLUX.3 norm_q/norm_k stay fp32 under bf16 attention.+ with contextlib.suppress(Exception):+ for sub in module.modules():+ proj = getattr(sub, "to_q", None)+ if proj is None:+ proj = getattr(sub, "to_qkv", None)+ if proj is None:+ continue+ dtype = _projection_dtype(proj)+ if dtype is not None:+ return dtype+ return None+++def _unpinned_param_dtype(module: Any) -> Any:+ # diffusers' ModelMixin.dtype rule: skip _keep_in_fp32_modules.+ keep = tuple(getattr(module, "_keep_in_fp32_modules", None) or ())+ with contextlib.suppress(Exception):+ for name, param in module.named_parameters():+ if keep and any(k in name for k in keep):+ continue+ dtype = _probe_dtype(param.dtype)+ if dtype is not None:+ return dtype+ return None+++def _module_compute_dtype(module: Any) -> Any:+ return _attention_dtype(module) or _unpinned_param_dtype(module)++def _rocm_flash_runs_on(ordinal: int, dtype: Any) -> bool:try:from types import SimpleNamespace@@ -720,36 +775,70 @@ _PROBE_DTYPES = ("torch.float16", "torch.bfloat16", "torch.float32")def _rocm_attention_dtype(pipe: Any) -> Any:- """The compute dtype the SDPA probe should run at: the denoiser's, else the pipeline's; None (the probe's fp16- default) when only a quantized storage dtype is readable."""from . import diffusion_multigpu as mg- for name in mg.DENOISER_ATTRS:- module = getattr(pipe, name, None)- if module is not None:- dtype = _denoiser_dtype(module)- if str(dtype) in _PROBE_DTYPES:- return dtype- dtype = getattr(pipe, "dtype", None)- return dtype if str(dtype) in _PROBE_DTYPES else None+ modules = [m for m in (getattr(pipe, n, None) for n in mg.DENOISER_ATTRS) if m is not None]+ for module in modules:+ dtype = _module_compute_dtype(module)+ if dtype is not None:+ return dtype+ dtype = _probe_dtype(getattr(pipe, "dtype", None))+ if dtype is not None:+ return dtype+ for module in modules:+ dtype = _probe_dtype(_denoiser_dtype(module))+ if dtype is not None:+ return dtype+ return None-def _rocm_sdpa_math_only(ordinal: int, dtype: Any) -> bool:+def _rocm_sdpa_math_only(+ ordinal: int,+ dtype: Any,+ *,+ read_only: bool = False,+) -> bool:"""diffusion_attention's per-device SDPA probe: True only when math ran and every fused kernel refused on this- card at this dtype (an unanswered probe is False)."""+ card at this dtype (an unanswered probe is False). ``read_only``: do not flip the primary's process-wide switches."""try:from types import SimpleNamespace- from .diffusion_attention import sdpa_math_only- return bool(- sdpa_math_only(- SimpleNamespace(device = "cuda", torch_device = f"cuda:{int(ordinal)}", dtype = dtype)+ from .diffusion_attention import read_only_sdpa_probe, sdpa_math_only+ with read_only_sdpa_probe() if read_only else contextlib.nullcontext():+ return bool(+ sdpa_math_only(+ SimpleNamespace(device = "cuda", torch_device = f"cuda:{int(ordinal)}", dtype = dtype)+ ))- )except Exception: # noqa: BLE001return False+def _rocm_fused_sdpa_kernels(ordinal: int, dtype: Any) -> Optional[tuple[str, ...]]:+ try:+ from types import SimpleNamespace++ from .diffusion_attention import (+ _SDPA_SUBQUADRATIC,+ _is_cuda_rocm,+ _rocm_guard_allowed,+ _sdpa_capability,+ read_only_sdpa_probe,+ )++ if not _is_cuda_rocm(SimpleNamespace(device = "cuda")) or not _rocm_guard_allowed():+ return None+ with read_only_sdpa_probe():+ available = _sdpa_capability(+ SimpleNamespace(device = f"cuda:{int(ordinal)}", dtype = dtype)+ )+ except Exception: # noqa: BLE001+ return None+ if not available:+ return None+ return tuple(k for k in available if k in _SDPA_SUBQUADRATIC)++def _rocm_drop_math_only(pipe: Any,primary: int,@@ -758,29 +847,49 @@ def _rocm_drop_math_only() -> tuple[int, ...]:"""Drop a secondary whose native SDPA is math-only when the primary's is not: the plan's activation figures werepriced for the primary's fused kernel, and math SDPA materialises the full score matrix on the card running the- block. A math-only primary priced that peak already, so cards matching it are kept."""+ block. A math-only primary priced that peak already, so cards matching it are kept. A secondary sharing no fused+ kernel with the cards kept so far is dropped too (the guard keeps only common kernels)."""if not cards:return cardsdtype = _rocm_attention_dtype(pipe)if _rocm_sdpa_math_only(primary, dtype):return cards- dropped = tuple(o for o in cards if _rocm_sdpa_math_only(o, dtype))+ dropped = tuple(o for o in cards if _rocm_sdpa_math_only(o, dtype, read_only = True))+ mismatched: list[int] = []+ common = _rocm_fused_sdpa_kernels(primary, dtype)+ if common:+ for o in cards:+ if o in dropped:+ continue+ fused = _rocm_fused_sdpa_kernels(o, dtype)+ if fused is None:+ continue+ if any(k in fused for k in common):+ common = tuple(k for k in common if k in fused)+ else:+ mismatched.append(o)+ dropped = tuple(o for o in cards if o in dropped or o in mismatched)if dropped and logger is not None:logger.info("diffusion.multigpu: layer split skips %s (%s at %s)",", ".join(f"cuda:{o}" for o in dropped),- MATH_ONLY_DECLINE,+ "no fused SDPA kernel shared with the primary" if mismatched else MATH_ONLY_DECLINE,dtype,)- return tuple(o for o in cards if o not in dropped)+ kept = tuple(o for o in cards if o not in dropped)+ if kept:+ with contextlib.suppress(Exception):+ from .diffusion_attention import guard_rocm_fused_sdpa_cards+ guard_rocm_fused_sdpa_cards([f"cuda:{int(o)}" for o in (primary, *kept)], dtype, logger)+ return keptdef _math_only_reason(primary: int, cards: Sequence[int]) -> str:names = ", ".join(f"cuda:{int(o)}" for o in cards)has = "has" if len(cards) == 1 else "have"return (- f"{MATH_ONLY_DECLINE}: {names} {has} no fused SDPA kernel at the model dtype (cuda:{int(primary)} does), so its "- "attention would materialise the full score matrix"+ f"{MATH_ONLY_DECLINE}: {names} {has} no fused SDPA kernel shared with cuda:{int(primary)} at the model dtype, so "+ "its attention would materialise the full score matrix")@@ -813,7 +922,7 @@ def _rocm_backend_cards(return (), backendkept = tuple(o for o in cards if _rocm_gfx(o) == need)if backend == "flash":- dtype = _denoiser_dtype(module)+ dtype = _module_compute_dtype(module) or _denoiser_dtype(module)kept = tuple(o for o in kept if _rocm_flash_runs_on(o, dtype))return kept, backenddiff --git a/studio/backend/tests/test_diffusion_layer_split_plan.py b/studio/backend/tests/test_diffusion_layer_split_plan.pyindex f19e7e8ac81..fb7e53e252a 100644--- a/studio/backend/tests/test_diffusion_layer_split_plan.py+++ b/studio/backend/tests/test_diffusion_layer_split_plan.py@@ -1183,3 +1183,234 @@ def test_rocm_second_expert_moves_only_to_a_card_that_runs_the_attention_backendmonkeypatch.setattr(da, "sdpa_math_only", lambda t: t.torch_device == "cuda:1")rocm, _ = _refine(_plan(), pipe = pipe)assert "transformer_2" not in (rocm.secondary_components or ())+++def _real_rocm_sdpa_probe(monkeypatch):+ """Real probe + guard + switches; only ROCm detection and child answers faked (cuda:0 fused, cuda:1 math-only)."""+ from core import _torchao_stub+ from core.inference import diffusion_attention as da++ switches = (+ (torch.backends.cuda.flash_sdp_enabled, torch.backends.cuda.enable_flash_sdp),+ (+ torch.backends.cuda.mem_efficient_sdp_enabled,+ torch.backends.cuda.enable_mem_efficient_sdp,+ ),+ )+ before = [read() for read, _ in switches]+ for _, write in switches:+ write(True)+ caches = (da._SDPA_PROBE_CACHE, da._ROCM_GUARD_DISABLED, da._ROCM_PROBE_INCOMPLETE)+ for cache in caches:+ cache.clear()+ monkeypatch.delenv(da.ROCM_FUSED_SDPA_ALLOW_ENV, raising = False)+ monkeypatch.setattr(_torchao_stub, "_module_is_rocm", lambda t: True)+ monkeypatch.setattr(+ da,+ "_run_rocm_sdpa_children",+ lambda device, dtype: (+ (da.SDPA_MATH, da.SDPA_FLASH, da.SDPA_MEM_EFFICIENT)+ if device == "cuda:0"+ else (da.SDPA_MATH,)+ ),+ )+ yield lambda: tuple(read() for read, _ in switches)+ for (_, write), value in zip(switches, before):+ write(value)+ for cache in caches:+ cache.clear()+++def test_rocm_secondary_probe_leaves_the_primary_sdpa_switches_alone(_real_rocm_sdpa_probe):+ # A rejected math-only secondary must not turn fused SDPA off process-wide.+ from types import SimpleNamespace++ from core.inference import diffusion_attention as da++ flags = _real_rocm_sdpa_probe+ pipe = _Pipe()+ pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)+ assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == ()+ assert flags() == (True, True)+ assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == ()+ assert flags() == (True, True)+ assert not da.sdpa_math_only(+ SimpleNamespace(device = "cuda", torch_device = "cuda:0", dtype = torch.bfloat16)+ )+ assert da.guard_rocm_fused_sdpa(+ SimpleNamespace(device = "cuda", torch_device = "cuda:1", dtype = torch.bfloat16)+ ) == (da.SDPA_FLASH, da.SDPA_MEM_EFFICIENT)+ assert flags() == (False, False)+++def test_rocm_kept_secondary_still_gets_the_fused_sdpa_guard(_real_rocm_sdpa_probe, monkeypatch):+ # cuda:1 is kept despite a failed flash launch, so flash goes off process-wide.+ from core.inference import diffusion_attention as da++ flags = _real_rocm_sdpa_probe+ monkeypatch.setattr(+ da,+ "_run_rocm_sdpa_children",+ lambda device, dtype: (+ (da.SDPA_MATH, da.SDPA_FLASH, da.SDPA_MEM_EFFICIENT)+ if device == "cuda:0"+ else (da.SDPA_MATH, da.SDPA_MEM_EFFICIENT)+ ),+ )+ pipe = _Pipe()+ pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)+ assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == (1,)+ assert flags() == (False, True)+ assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == (1,)+ assert flags() == (False, True)+++def test_rocm_kept_secondaries_share_a_fused_sdpa_kernel(_real_rocm_sdpa_probe, monkeypatch):+ # Keeping both would leave only math in common; cuda:2 goes.+ from core.inference import diffusion_attention as da++ flags = _real_rocm_sdpa_probe+ answers = {+ "cuda:0": (da.SDPA_MATH, da.SDPA_FLASH, da.SDPA_MEM_EFFICIENT),+ "cuda:1": (da.SDPA_MATH, da.SDPA_FLASH),+ "cuda:2": (da.SDPA_MATH, da.SDPA_MEM_EFFICIENT),+ }+ monkeypatch.setattr(da, "_run_rocm_sdpa_children", lambda device, dtype: answers[device])+ pipe = _Pipe()+ pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)+ assert lsp._rocm_drop_math_only(pipe, 0, (1, 2)) == (1,)+ assert flags() == (True, False)+ assert lsp._rocm_drop_math_only(pipe, 0, (1, 2)) == (1,)+ assert flags() == (True, False)+++class _MixedPrecisionDiT(torch.nn.Module):+ """MiniMax-H3: fp32 input stem before bf16 attention blocks."""++ def __init__(self):+ super().__init__()+ self.proj_in = torch.nn.Linear(2, 4)+ attn = torch.nn.Module()+ attn.to_q = torch.nn.Linear(4, 4).to(torch.bfloat16)+ self.blocks = torch.nn.ModuleList([torch.nn.ModuleDict({"attn": attn})])+++def test_rocm_probe_runs_at_the_attention_dtype_not_an_fp32_stem(monkeypatch):+ from core.inference import diffusion_attention as da++ pipe = _Pipe()+ pipe.transformer = _MixedPrecisionDiT()+ assert next(pipe.transformer.parameters()).dtype == torch.float32+ assert lsp._rocm_attention_dtype(pipe) == torch.bfloat16+ probed = []+ monkeypatch.setattr(+ da,+ "sdpa_math_only",+ lambda t: probed.append(t.dtype) or t.dtype == torch.float32 or t.torch_device == "cuda:1",+ )+ assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == ()+ assert set(probed) == {torch.bfloat16}+++class _PackedLinear(torch.nn.Linear):+ """diffusers' GGUFLinear surface: packed uint8 weight, ``compute_dtype``, no bias."""++ def __init__(+ self,+ fin,+ fout,+ compute_dtype = None,+ ):+ super().__init__(fin, fout, bias = False)+ self.compute_dtype = compute_dtype+ self.weight = torch.nn.Parameter(+ torch.zeros(fout, fin, dtype = torch.uint8), requires_grad = False+ )+++def _pack_attention(model, compute_dtype):+ for module in list(model.modules()):+ for attr in ("to_q", "to_k", "to_v", "to_gate", "to_qkv"):+ linear = getattr(module, attr, None)+ if isinstance(linear, torch.nn.Linear):+ packed = _PackedLinear(linear.in_features, linear.out_features, compute_dtype)+ setattr(module, attr, packed)+ return model+++def _math_at_fp32_and_on_cuda1(monkeypatch):+ from core.inference import diffusion_attention as da++ probed = []+ monkeypatch.setattr(+ da,+ "sdpa_math_only",+ lambda t: probed.append(t.dtype) or t.dtype == torch.float32 or t.torch_device == "cuda:1",+ )+ return probed++[email protected](scope = "session")+def krea2():+ # Must import before the 8-card fixture: xformers / torchao device checks fail against it.+ return pytest.importorskip("diffusers.models.transformers.transformer_krea2")+++def _tiny_krea2(krea2):+ model = krea2.Krea2Transformer2DModel(+ in_channels = 4,+ num_layers = 2,+ attention_head_dim = 16,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ intermediate_size = 32,+ timestep_embed_dim = 8,+ text_hidden_dim = 8,+ num_text_layers = 2,+ text_num_attention_heads = 2,+ text_num_key_value_heads = 2,+ text_intermediate_size = 16,+ num_layerwise_text_blocks = 1,+ num_refiner_text_blocks = 1,+ axes_dims_rope = (4, 6, 6),+ ).to(torch.bfloat16)+ for name, module in model.named_modules():+ if name and any(k in name.split(".")[-1] for k in model._keep_in_fp32_modules):+ module.to(torch.float32)+ return model+++def test_rocm_probe_reads_a_quantized_projection_not_its_fp32_qk_norm(monkeypatch, krea2):+ model = _tiny_krea2(krea2)+ attn = next(m for m in model.modules() if getattr(m, "to_q", None) is not None)+ assert attn.norm_q.weight.dtype == torch.float32+ pipe = _Pipe()+ pipe.transformer = _pack_attention(model, torch.bfloat16)+ assert lsp._rocm_attention_dtype(pipe) == torch.bfloat16+ probed = _math_at_fp32_and_on_cuda1(monkeypatch)+ assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == ()+ assert set(probed) == {torch.bfloat16}+++def test_rocm_probe_dtype_skips_fp32_pinned_modules_when_projections_carry_no_dtype(+ monkeypatch, krea2+):+ pipe = _Pipe()+ pipe.transformer = _pack_attention(_tiny_krea2(krea2), None)+ assert lsp._attention_dtype(pipe.transformer) is None+ assert lsp._rocm_attention_dtype(pipe) == torch.bfloat16+ probed = _math_at_fp32_and_on_cuda1(monkeypatch)+ assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == ()+ assert set(probed) == {torch.bfloat16}+++def test_rocm_probe_dtype_packed_projection_with_fp32_norm_synthetic():+ attn = torch.nn.Module()+ attn.norm_q = torch.nn.RMSNorm(4)+ attn.to_q = _PackedLinear(4, 4, torch.bfloat16)+ dit = torch.nn.Module()+ dit.attn = attn+ pipe = _Pipe()+ pipe.transformer = dit+ assert lsp._rocm_attention_dtype(pipe) == torch.bfloat16diff --git a/studio/backend/tests/test_video_minimax_h3_pdd.py b/studio/backend/tests/test_video_minimax_h3_pdd.pyindex 390d1ba52b5..f9d3a0125e4 100644--- a/studio/backend/tests/test_video_minimax_h3_pdd.py+++ b/studio/backend/tests/test_video_minimax_h3_pdd.py@@ -743,6 +743,27 @@ def test_the_whole_pdd_denoiser_forward_compiles_fullgraph_without_recompiles():)+def test_studio_regional_fullgraph_compile_keeps_the_pdd_hook_eager():+ """Regional compile keeps the root eager, so the pre-hook (``is_inference()``, untraceable) stays out of fullgraph."""+ import torch._dynamo as dynamo++ model = _tiny()+ eager = _tiny()+ sigmas = _bound(model)+ _bound(eager)+ dynamo.reset()+ model.compile_repeated_blocks(fullgraph = True, backend = "eager")+ assert getattr(model, "_compiled_call_impl", None) is None+ assert any(getattr(m, "_compiled_call_impl", None) is not None for m in model.modules())+ with torch.no_grad():+ for i in range(1, 8):+ kw = _inputs("cpu", 1.0 - float(sigmas[i]), seed = i)+ out, ref = model(**kw), eager(**kw)+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1]), i+ assert float(model.proj_out.pdd_state[1]) == float(sigmas[i + 1]), i+ dynamo.reset()++def test_the_denoiser_under_model_cpu_offload_matches_resident():_cuda()accelerate = pytest.importorskip("accelerate")