Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: allow the multi-GPU layer split on discrete ROCm cards (#14479)
Daniel Han04d2beec9b68
diff --git a/studio/backend/core/inference/diffusion_layer_split_plan.py b/studio/backend/core/inference/diffusion_layer_split_plan.pyindex 3a8656a119a..0c66d3ebbaa 100644--- a/studio/backend/core/inference/diffusion_layer_split_plan.py+++ b/studio/backend/core/inference/diffusion_layer_split_plan.py@@ -7,8 +7,9 @@ Order of preference on a multi-card selection: companion split (diffusion_multigone-card offload plan. A layer split is only considered when the one-card plan offloads the DENOISER itself and thecompanion assignment found nothing; it is priced with the same per-card reserves as the companion split. The VAE andevery non-text-encoder companion stay on the primary (the decode and its tile verdict stay where today's guard reads-them); the text encoders may move to any used card. One card, Keep multiple models loaded, ROCm and the kill switches-keep today's plan unchanged.+them); the text encoders may move to any used card. One card, Keep multiple models loaded and the kill switches keep+today's plan unchanged. ROCm splits across discrete cards only (an APU's memory is the host's: a split across "cards"+sharing one pool frees nothing), and a per-gfx-target attention kernel keeps blocks on cards of the primary's target.The engine (block placement + device-crossing hooks) is a separate piece; this module only defines the interface itmust provide (``split_blocks``, ``engine_available``, ``place_layer_split``, ``undo_layer_split``) with stubs that keep@@ -545,11 +546,67 @@ def _record_decline(pipe: Any, reason: str, logger: Any) -> None:def _is_rocm() -> bool:+ from .diffusion_multigpu import _rocm_torch+ return _rocm_torch()+++UNIFIED_DECLINE = "unified memory"+++def _rocm_card_unified(ordinal: int) -> Optional[bool]:+ """True for a ROCm APU (Strix Halo / Point, Phoenix: the GPU's memory is the host's), False for a discrete card,+ None when unreadable. Studio's own classifier (the driver's integrated flag, then the gfx target)."""try:import torch- return bool(getattr(torch.version, "hip", None))++ from utils.hardware.hardware import _rocm_props_unified_status+ return _rocm_props_unified_status(torch.cuda.get_device_properties(int(ordinal)))except Exception: # noqa: BLE001- return False+ return None+++def _unified_reason(cards: Sequence[int]) -> str:+ names = ", ".join(f"cuda:{int(o)}" for o in cards)+ what = "is an integrated GPU (APU)" if len(cards) == 1 else "are integrated GPUs (APUs)"+ return (+ f"{UNIFIED_DECLINE}: {names} {what} sharing system memory, so cutting the denoiser across cards frees no "+ "memory"+ )+++def note_unified_primary(+ pipe: Any,+ selection: Any,+ logger: Any = None,+) -> None:+ """``refine_plan_for_multigpu`` stopped on a unified-memory primary: on ROCm say why in the layer split status (a+ multi-APU selection is otherwise silent). CUDA keeps today's silent no-op."""+ if selection is None or layer_split_mode() == "off" or not _is_rocm():+ return+ _record_decline(pipe, _unified_reason((int(selection.primary),)), logger)+++def _rocm_discrete_secondaries(+ pipe: Any, primary: int, secondaries: Sequence[int], logger: Any+) -> Optional[tuple[int, ...]]:+ """ROCm: the secondaries a split may use (discrete cards); None after recording a decline. A unified primary+ declines (its blocks and the host share one pool); a unified secondary is dropped (its blocks would sit in host+ memory). An unreadable card is kept: the per-card budgets below still price it."""+ if _rocm_card_unified(primary) is True:+ _record_decline(pipe, _unified_reason((int(primary),)), logger)+ return None+ unified = tuple(int(o) for o in secondaries if _rocm_card_unified(int(o)) is True)+ kept = tuple(int(o) for o in secondaries if int(o) not in unified)+ if not kept:+ _record_decline(pipe, _unified_reason(unified), logger)+ return None+ if unified and logger is not None:+ logger.info(+ "diffusion.multigpu: layer split skips %s (%s)",+ ", ".join(f"cuda:{o}" for o in unified),+ UNIFIED_DECLINE,+ )+ return keptdef plan_offloads_denoiser(plan: Any) -> bool:@@ -615,7 +672,10 @@ _PRIMARY_ARCH_BACKENDS = frozenset({"_flash_3_hub", "flash_4_hub", "sage", "sagedef _backend_cards(- pipe: Any, primary: int, secondaries: Sequence[int]+ pipe: Any,+ primary: int,+ secondaries: Sequence[int],+ logger: Any = None,) -> tuple[tuple[int, ...], Optional[str]]:"""``(secondaries the engaged attention backend runs on, that backend)``; a card of unknown capability is kept."""from . import diffusion_multigpu as mg@@ -630,6 +690,8 @@ def _backend_cards(backend = tagbreakcards = tuple(int(o) for o in secondaries)+ if _is_rocm():+ return _rocm_backend_cards(pipe, primary, cards, logger)if backend is None:return cards, Noneneed = mg._capability(primary)@@ -638,6 +700,145 @@ def _backend_cards(return tuple(o for o in cards if (mg._capability(o) or need) == need), backend+# ROCm kernels built per gfx target: ROCm flash_attn (CK / Triton, #13792) and AITER.+_ROCM_ARCH_BACKENDS = frozenset({"flash", "aiter"})+++def _rocm_gfx(ordinal: int) -> str:+ try:+ import torch++ from .rocm_bf16 import _device_gfx_arch+ return _device_gfx_arch(torch, int(ordinal))+ except Exception: # noqa: BLE001+ return ""+++def _denoiser_dtype(module: Any) -> Any:+ try:+ return next(module.parameters()).dtype+ except Exception: # noqa: BLE001+ return None+++def _rocm_flash_runs_on(ordinal: int, dtype: Any) -> bool:+ try:+ from types import SimpleNamespace++ from .diffusion_attention import _rocm_flash_attn_runs+ return bool(+ _rocm_flash_attn_runs(+ SimpleNamespace(device = "cuda", torch_device = f"cuda:{int(ordinal)}", dtype = dtype)+ )+ )+ except Exception: # noqa: BLE001+ return False+++MATH_ONLY_DECLINE = "math-only attention"+_PROBE_DTYPES = ("torch.float16", "torch.bfloat16", "torch.float32")+++def _rocm_attention_dtype(pipe: Any) -> Any:+ """The compute dtype the SDPA probe should run at: the denoiser's, else the pipeline's; None (the probe's fp16+ default) when only a quantized storage dtype is readable."""+ from . import diffusion_multigpu as mg++ for name in mg.DENOISER_ATTRS:+ module = getattr(pipe, name, None)+ if module is not None:+ dtype = _denoiser_dtype(module)+ if str(dtype) in _PROBE_DTYPES:+ return dtype+ dtype = getattr(pipe, "dtype", None)+ return dtype if str(dtype) in _PROBE_DTYPES else None+++def _rocm_sdpa_math_only(ordinal: int, dtype: Any) -> bool:+ """diffusion_attention's per-device SDPA probe: True only when math ran and every fused kernel refused on this+ card at this dtype (an unanswered probe is False)."""+ try:+ from types import SimpleNamespace++ from .diffusion_attention import sdpa_math_only+ return bool(+ sdpa_math_only(+ SimpleNamespace(device = "cuda", torch_device = f"cuda:{int(ordinal)}", dtype = dtype)+ )+ )+ except Exception: # noqa: BLE001+ return False+++def _rocm_drop_math_only(+ pipe: Any,+ primary: int,+ cards: tuple[int, ...],+ logger: Any = None,+) -> tuple[int, ...]:+ """Drop a secondary whose native SDPA is math-only when the primary's is not: the plan's activation figures were+ priced for the primary's fused kernel, and math SDPA materialises the full score matrix on the card running the+ block. A math-only primary priced that peak already, so cards matching it are kept."""+ if not cards:+ return cards+ dtype = _rocm_attention_dtype(pipe)+ if _rocm_sdpa_math_only(primary, dtype):+ return cards+ dropped = tuple(o for o in cards if _rocm_sdpa_math_only(o, dtype))+ if dropped and logger is not None:+ logger.info(+ "diffusion.multigpu: layer split skips %s (%s at %s)",+ ", ".join(f"cuda:{o}" for o in dropped),+ MATH_ONLY_DECLINE,+ dtype,+ )+ return tuple(o for o in cards if o not in dropped)+++def _math_only_reason(primary: int, cards: Sequence[int]) -> str:+ names = ", ".join(f"cuda:{int(o)}" for o in cards)+ has = "has" if len(cards) == 1 else "have"+ return (+ f"{MATH_ONLY_DECLINE}: {names} {has} no fused SDPA kernel at the model dtype (cuda:{int(primary)} does), so its "+ "attention would materialise the full score matrix"+ )+++def _rocm_backend_cards(+ pipe: Any,+ primary: int,+ cards: tuple[int, ...],+ logger: Any = None,+) -> tuple[tuple[int, ...], Optional[str]]:+ """ROCm: native SDPA runs on any gfx target (the load dtype is already matched by ``make_selection``), so every+ card whose SDPA is not math-only where the primary's is fused is kept. A per-target kernel (flash_attn, AITER) the+ primary verified keeps only cards of the primary's gfx target, and flash only where its own probe runs and matches+ on that card (as the primary's did at load)."""+ from . import diffusion_multigpu as mg++ backend = module = None+ with contextlib.suppress(Exception):+ from .diffusion_attention import ATTENTION_BACKEND_ATTR+ for name in mg.DENOISER_ATTRS:+ candidate = getattr(pipe, name, None)+ tag = (+ getattr(candidate, ATTENTION_BACKEND_ATTR, None) if candidate is not None else None+ )+ if tag in _ROCM_ARCH_BACKENDS:+ backend, module = tag, candidate+ break+ if backend is None:+ return _rocm_drop_math_only(pipe, primary, cards, logger), None+ need = _rocm_gfx(primary)+ if not need:+ return (), backend+ kept = tuple(o for o in cards if _rocm_gfx(o) == need)+ if backend == "flash":+ dtype = _denoiser_dtype(module)+ kept = tuple(o for o in kept if _rocm_flash_runs_on(o, dtype))+ return kept, backend++def refine_plan_for_layer_split(pipe: Any,plan: Any,@@ -658,19 +859,23 @@ def refine_plan_for_layer_split(return planif not plan_offloads_denoiser(plan) and mode != "force":return plan+ candidates: Sequence[int] = selection.secondariesif _is_rocm():- _record_decline(pipe, "ROCm (layer split is CUDA only)", logger)- return plan+ candidates = _rocm_discrete_secondaries(pipe, int(selection.primary), candidates, logger)+ if candidates is None:+ return planlayouts = _denoiser_layouts(pipe)if layouts is None:_record_decline(pipe, f"denoiser cannot be split ({_layout_refusal(pipe)})", logger)return planlimits: dict[int, int] = {int(selection.primary): int(primary_limit_mib)}- secondaries, backend = _backend_cards(pipe, int(selection.primary), selection.secondaries)+ secondaries, backend = _backend_cards(pipe, int(selection.primary), candidates, logger)if not secondaries:_record_decline(pipe,- f"attention backend {backend} was verified for cuda:{int(selection.primary)}'s architecture only",+ _math_only_reason(int(selection.primary), candidates)+ if backend is None and _is_rocm()+ else f"attention backend {backend} was verified for cuda:{int(selection.primary)}'s architecture only",logger,)return plandiff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.pyindex e0d0fb48ecd..418f92ee509 100644--- a/studio/backend/core/inference/diffusion_multigpu.py+++ b/studio/backend/core/inference/diffusion_multigpu.py@@ -125,6 +125,15 @@ def make_selection(passif not others:return None+ if _rocm_torch():+ # ROCm's capability is the gfx target (gfx942 (9, 4)), which does not order; match the primary's load dtype+ # instead (arch-pinned attention kernels are gated per card in diffusion_layer_split_plan).+ others = tuple(o for o in others if _rocm_dtype_parity(int(primary), o))+ return (+ MultiGpuSelection(primary = int(primary), secondaries = others, automatic = automatic)+ if others+ else None+ )# The load dtype and the device-specific patches are resolved for the primary (bf16 only from Ampere, FP8 kernels# from Ada): a card of a lower compute capability may not run them, so it takes no component.need = _capability(int(primary))@@ -173,6 +182,35 @@ def _capability(ordinal: int) -> Optional[tuple[int, int]]:return None+def _rocm_torch() -> bool:+ """A ROCm torch build (AMD SDK / Radeon wheels included, which leave ``torch.version.hip`` unset)."""+ try:+ import torch++ from .rocm_bf16 import is_rocm_torch+ return is_rocm_torch(torch)+ except Exception: # noqa: BLE001+ return False+++def _rocm_bf16(ordinal: int) -> Optional[bool]:+ try:+ import torch++ from .rocm_bf16 import rocm_bf16_supported+ return bool(rocm_bf16_supported(torch, int(ordinal)))+ except Exception: # noqa: BLE001+ return None+++def _rocm_dtype_parity(primary: int, ordinal: int) -> bool:+ """A ROCm card runs the dtype the load resolves for the primary: bf16 when the primary has native bf16 (RDNA1/2,+ GCN and Vega emulate it), anything otherwise. An unreadable card is kept, like an unknown CUDA capability."""+ if _rocm_bf16(primary) is not True:+ return True+ return _rocm_bf16(ordinal) is not False++@contextlib.contextmanagerdef selection_scope(selection: Optional[MultiGpuSelection],@@ -383,12 +421,14 @@ def choose_component_assignment(overhead_mib: int,primary_limit_mib: int,secondary_limit_mib: int,+ move_denoisers: bool = True,) -> Optional[tuple[str, ...]]:"""Names to place on the secondary so each card holds its share resident, or None. ``sizes`` maps component name-> (MiB, role) with role 'dit' / 'text_encoder' / 'other'. A card running a denoiser or the VAE (any 'other'component) is priced with the runtime headroom; a card holding only text encoders with an encoder forward's room.Tried in order: every companion; the text encoders alone; the second denoiser; the second denoiser plus the- companions. The first denoiser always stays on the primary."""+ companions. The first denoiser always stays on the primary; ``move_denoisers=False`` (the secondary cannot run the+ denoiser's attention backend) keeps every denoiser there."""dits = [n for n, (_m, r) in sizes.items() if r == "dit"]if not dits:return None@@ -405,7 +445,7 @@ def choose_component_assignment(candidates.append(tuple(companions))if encoders and len(encoders) < len(companions):candidates.append(tuple(encoders))- if seconds:+ if seconds and move_denoisers:candidates.append(tuple(seconds))if companions:candidates.append(tuple(seconds + companions))@@ -422,6 +462,32 @@ def choose_component_assignment(return None+def _secondary_runs_denoiser(+ pipe: Any, selection: MultiGpuSelection, secondary: int, sizes: dict, logger: Any+) -> bool:+ """Whether a second denoiser may move wholesale to ``secondary``. ROCm admits mixed gfx targets (make_selection),+ so the card must pass the layer split's per-card attention gate: an arch-pinned kernel (flash_attn, AITER) only on+ the primary's gfx target, native SDPA not math-only beside a fused primary. CUDA's capability order already holds."""+ if not _rocm_torch() or sum(1 for _m, role in sizes.values() if role == "dit") < 2:+ return True+ try:+ from .diffusion_layer_split_plan import _rocm_backend_cards+ kept, _backend = _rocm_backend_cards(+ pipe, int(selection.primary), (int(secondary),), logger+ )+ except Exception: # noqa: BLE001 - unverifiable: keep the denoisers on the primary+ return False+ if int(secondary) in kept:+ return True+ if logger is not None:+ logger.info(+ "diffusion.multigpu: second denoiser stays on cuda:%s (attention backend not verified on cuda:%s)",+ selection.primary,+ secondary,+ )+ return False++def refine_plan_for_multigpu(pipe: Any,plan: Any,@@ -451,6 +517,9 @@ def refine_plan_for_multigpu(return planmemory = plan.device_memoryif memory.is_unified or getattr(memory, "device", None) != "cuda":+ if memory.is_unified:+ from .diffusion_layer_split_plan import note_unified_primary+ note_unified_primary(pipe, selection, logger)return planbudget = plan.estimates.get("safe_device_budget_mib")if budget is None:@@ -503,6 +572,7 @@ def refine_plan_for_multigpu(overhead_mib = overhead,primary_limit_mib = int(primary_limit),secondary_limit_mib = int(sec_limit),+ move_denoisers = _secondary_runs_denoiser(pipe, selection, secondary, sizes, logger),)if moved is None:return layer_split_or_plan()diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 4288a6f51d3..2bbcc8108ee 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -1925,6 +1925,12 @@ def _ensure_mamba_ssm()+# Integrated-only Radeon names for the other APU targets, as whole tokens (a substring would read "RX 7600M" as "600M"):+# 820M Krackan Point 2 (gfx1153), 8040S Strix Halo (gfx1151), 780M / 760M / 740M Phoenix / Hawk Point (gfx1103),+# 680M / 660M Rembrandt (gfx1035), 610M Raphael / Dragon Range / Mendocino (gfx1036 / gfx1037).+_ROCM_APU_NAME_RE = re.compile(r"(?<![\w-])(?:820m|8040s|780m|760m|740m|680m|660m|610m)(?!\w)")++def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:"""Classify a ROCm device as unified-memory (APU) or discrete, returning ``(gcn_arch,is_unified)``. ``gcn_arch`` is the canonical arch string (e.g. ``"gfx1151"``) when a known@@ -1970,6 +1976,7 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:or "8050s" in dev_loweror "860m" in dev_loweror "840m" in dev_lower+ or _ROCM_APU_NAME_RE.search(dev_lower) is not None)return gcn_arch, is_unifieddiff --git a/studio/backend/tests/test_diffusion_layer_split_order.py b/studio/backend/tests/test_diffusion_layer_split_order.pyindex be875001f78..fd7abbf0351 100644--- a/studio/backend/tests/test_diffusion_layer_split_order.py+++ b/studio/backend/tests/test_diffusion_layer_split_order.py@@ -30,6 +30,7 @@ def _env(monkeypatch):monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)monkeypatch.setattr(mg, "_capability", lambda o: None)monkeypatch.setattr(lsp, "_is_rocm", lambda: False)+ monkeypatch.setattr(mg, "_rocm_torch", lambda: False)yielddiff --git a/studio/backend/tests/test_diffusion_layer_split_plan.py b/studio/backend/tests/test_diffusion_layer_split_plan.pyindex a0d7c6bbe20..a47bd05928b 100644--- a/studio/backend/tests/test_diffusion_layer_split_plan.py+++ b/studio/backend/tests/test_diffusion_layer_split_plan.py@@ -29,6 +29,7 @@ def _env(monkeypatch):monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)monkeypatch.setattr(mg, "_capability", lambda o: None)monkeypatch.setattr(lsp, "_is_rocm", lambda: False)+ monkeypatch.setattr(mg, "_rocm_torch", lambda: False)yield@@ -587,12 +588,16 @@ def test_cards_too_small_keep_the_offload_plan_and_record_why(monkeypatch, enginassert mg.placement_status(pipe)["layer_split"] == status-def test_rocm_explicit_modes_and_legacy_cpu_offload_keep_the_plan(monkeypatch, engine):+def test_rocm_discrete_splits_and_explicit_modes_and_legacy_cpu_offload_keep_the_plan(+ monkeypatch, engine+):_cards(monkeypatch)plan = _plan()- monkeypatch.setattr(lsp, "_is_rocm", lambda: True)- assert _refine(plan)[0] is plan+ _rocm_host(monkeypatch, {0: _RX7900, 1: _RX7900})+ refined, pipe = _refine(plan)+ assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)monkeypatch.setattr(lsp, "_is_rocm", lambda: False)+ monkeypatch.setattr(mg, "_rocm_torch", lambda: False)balanced = _plan(requested_mode = "balanced")assert _refine(balanced)[0] is balancedlegacy = _plan(explicit_offload = True)@@ -945,3 +950,236 @@ def test_an_arch_pinned_attention_backend_keeps_blocks_off_other_architectures(m) # runs on every admitted cardrefined, _ = _refine(plan, pipe = pipe, ordinals = (0, 1))assert refined.layer_split is not None and 1 in refined.layer_split.cards+++# ROCm: real classifiers (utils.hardware's unified test, rocm_bf16's gfx table) over fake device properties.+_RX7900 = ("gfx1100", 0) # 7900 XTX, discrete+_MI300 = ("gfx942", 0)+_RX6900 = ("gfx1030", 0) # RDNA2: no native bf16+_STRIX_HALO = ("gfx1151", 1) # APU: the driver's integrated flag set+++def _rocm_host(monkeypatch, cards):+ from types import SimpleNamespace++ monkeypatch.setattr(lsp, "_is_rocm", lambda: True)+ monkeypatch.setattr(mg, "_rocm_torch", lambda: True)+ monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda *a, **k: True)+ monkeypatch.delenv("UNSLOTH_STUDIO_ROCM_BF16", raising = False)++ def props(o = None):+ arch, integrated = cards[int(o)]+ return SimpleNamespace(+ gcnArchName = f"{arch}:sramecc+:xnack-", is_integrated = integrated, name = "AMD Radeon"+ )++ monkeypatch.setattr(torch.cuda, "get_device_properties", props)+ # the per-card SDPA probe spawns HIP children; every card has a fused kernel unless a test says otherwise+ from core.inference import diffusion_attention as da++ monkeypatch.setattr(da, "sdpa_math_only", lambda target: False)+++def test_rocm_two_discrete_cards_plan_the_same_split_as_cuda(monkeypatch, engine):+ _cards(monkeypatch)+ cuda, _ = _refine(_plan())+ _rocm_host(monkeypatch, {0: _RX7900, 1: _RX7900})+ rocm, pipe = _refine(_plan())+ assert rocm.layer_split is not None+ assert rocm.layer_split.ranges == cuda.layer_split.ranges+ assert rocm.layer_split.card_required_mib == cuda.layer_split.card_required_mib+ assert lsp.layer_split_status(pipe) is None # no decline recorded+++def test_rocm_two_apus_decline_with_the_unified_reason(monkeypatch, engine):+ _cards(monkeypatch)+ _rocm_host(monkeypatch, {0: _STRIX_HALO, 1: _STRIX_HALO})+ plan = _plan()+ refined, pipe = _refine(plan)+ assert refined is plan and engine.placed == []+ declined = lsp.layer_split_status(pipe)["declined"]+ assert declined.startswith(lsp.UNIFIED_DECLINE) and "cuda:0" in declined+++def test_rocm_unified_primary_plan_records_why_and_cuda_stays_silent(monkeypatch, engine):+ _cards(monkeypatch)+ unified = dm.DeviceMemory(+ backend = "cuda", device = "cuda", memory_kind = "unified_memory", free_mib = 90000, total_mib = 98000+ )+ plan = dataclasses.replace(_plan(), device_memory = unified)+ monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "force")+ refined, pipe = _refine(plan)+ assert refined is plan and lsp.layer_split_status(pipe) is None # CUDA: unchanged+ _rocm_host(monkeypatch, {0: _STRIX_HALO, 1: _STRIX_HALO})+ refined, pipe = _refine(plan)+ assert refined is plan and engine.placed == []+ assert lsp.layer_split_status(pipe)["declined"].startswith(lsp.UNIFIED_DECLINE)+++def test_rocm_discrete_primary_skips_an_apu_secondary(monkeypatch, engine):+ _cards(monkeypatch)+ _rocm_host(monkeypatch, {0: _RX7900, 1: _STRIX_HALO, 2: _RX7900})+ refined, _ = _refine(_plan(), ordinals = (0, 1, 2))+ assert refined.layer_split is not None and 1 not in refined.layer_split.cards+ assert 2 in refined.layer_split.cards+ refined, pipe = _refine(_plan(), ordinals = (0, 1))+ assert refined.layer_split is None+ assert "cuda:1" in lsp.layer_split_status(pipe)["declined"]+++def test_rocm_kill_switch_keeps_the_plan_and_records_nothing(monkeypatch, engine):+ _cards(monkeypatch)+ _rocm_host(monkeypatch, {0: _RX7900, 1: _RX7900})+ monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "0")+ plan = _plan()+ refined, pipe = _refine(plan)+ assert refined is plan and engine.placed == [] and lsp.layer_split_status(pipe) is None+++def test_rocm_mixed_arch_selection_compares_dtype_not_gfx_order(monkeypatch):+ # CUDA's capability order would drop MI300 (9, 4) under a 7900 XTX (11, 0) primary+ _rocm_host(monkeypatch, {0: _RX7900, 1: _MI300, 2: _RX6900})+ monkeypatch.setattr(mg, "_capability", lambda o: {0: (11, 0), 1: (9, 4), 2: (10, 3)}[int(o)])+ assert mg.make_selection(0, [0, 1]) == mg.MultiGpuSelection(primary = 0, secondaries = (1,))+ assert mg.make_selection(1, [1, 0]) == mg.MultiGpuSelection(primary = 1, secondaries = (0,))+ # a bf16 primary keeps no RDNA2 card (it would emulate the primary's dtype); an fp16 primary keeps any+ assert mg.make_selection(0, [0, 2]) is None+ assert mg.make_selection(2, [2, 0, 1]) == mg.MultiGpuSelection(primary = 2, secondaries = (0, 1))+++def test_rocm_mixed_arch_native_sdpa_splits_and_flash_stays_on_the_primary_target(+ monkeypatch, engine+):+ from core.inference.diffusion_attention import ATTENTION_BACKEND_ATTR++ _cards(monkeypatch)+ _rocm_host(monkeypatch, {0: _RX7900, 1: _MI300, 2: _RX7900})+ pipe = _Pipe()+ pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1))+ assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)+ probed = []+ monkeypatch.setattr(+ lsp, "_rocm_flash_runs_on", lambda o, dtype: probed.append((o, dtype)) or o != 2+ )+ for tag in ("flash", "aiter"):+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, tag)+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1))+ assert refined.layer_split is None+ assert f"attention backend {tag}" in lsp._DECLINES[pipe]+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, "aiter")+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1, 2))+ assert refined.layer_split is not None and 1 not in refined.layer_split.cards+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, "flash")+ probed.clear()+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1, 2))+ assert probed == [+ (2, torch.bfloat16)+ ] # the same-target card is probed; the MI300 never reaches the probe+ assert refined.layer_split is None # flash failed on cuda:2: no secondary left+++def test_windows_rocm_secondary_budget_uses_the_capped_free(monkeypatch):+ """WDDM: mem_get_info reports free ~ total whatever is resident; the secondary is priced from what torch has not+ reserved there (utils.hardware.trusted_mem_get_info)."""+ import sys++ from utils.hardware import hardware as hw++ gib = 1 << 30+ monkeypatch.setattr(sys, "platform", "win32")+ monkeypatch.setattr(hw, "IS_ROCM", True)+ monkeypatch.setattr(+ torch.cuda, "mem_get_info", lambda o = None: (24 * gib - (64 << 20), 24 * gib)+ )+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda o = None: 10 * gib)+ memory = mg.secondary_device_memory(1)+ assert memory.total_mib == 24 * 1024 and memory.free_mib == 14 * 1024+ assert mg.secondary_resident_limit_mib(1, "auto") < 14 * 1024+ monkeypatch.setattr(hw, "IS_ROCM", False)+ assert mg.secondary_device_memory(1).free_mib == 24 * 1024 - 64+++def test_rocm_apu_without_the_integrated_flag_gets_no_blocks(monkeypatch, engine):+ # Windows gfx1153 Krackan Point 2 on a runtime that leaves is_integrated 0: still an APU by its gfx target+ _cards(monkeypatch)+ _rocm_host(monkeypatch, {0: _RX7900, 1: ("gfx1153", 0), 2: _RX7900})+ refined, _ = _refine(_plan(), ordinals = (0, 1, 2))+ assert refined.layer_split is not None and 1 not in refined.layer_split.cards+ refined, pipe = _refine(_plan(), ordinals = (0, 1))+ assert refined.layer_split is None+ assert lsp.layer_split_status(pipe)["declined"].startswith(lsp.UNIFIED_DECLINE)+++def test_rocm_math_only_secondary_gets_no_blocks_beside_a_fused_primary(monkeypatch, engine):+ # mixed gfx, native SDPA: cuda:1 has no fused kernel at bf16, so math SDPA would hold the full score matrix there+ _cards(monkeypatch)+ _rocm_host(monkeypatch, {0: _RX7900, 1: _MI300, 2: _RX7900})+ pipe = _Pipe()+ pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)+ from core.inference import diffusion_attention as da++ probed = []+ monkeypatch.setattr(+ da,+ "sdpa_math_only",+ lambda t: probed.append((t.torch_device, t.dtype)) or t.torch_device == "cuda:1",+ )+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1, 2))+ assert refined.layer_split is not None and 1 not in refined.layer_split.cards+ assert 2 in refined.layer_split.cards+ assert ("cuda:1", torch.bfloat16) in probed # probed per card at the denoiser's dtype+ refined, pipe = _refine(_plan(), pipe = pipe, ordinals = (0, 1))+ assert refined.layer_split is None+ declined = lsp.layer_split_status(pipe)["declined"]+ assert declined.startswith(lsp.MATH_ONLY_DECLINE) and "cuda:1" in declined+ # a math-only primary already priced the quadratic peak: matching cards keep the split+ monkeypatch.setattr(da, "sdpa_math_only", lambda t: True)+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1))+ assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)+++def test_cuda_split_never_runs_the_rocm_sdpa_probe(monkeypatch, engine):+ _cards(monkeypatch)+ from core.inference import diffusion_attention as da++ monkeypatch.setattr(da, "sdpa_math_only", lambda t: pytest.fail("ROCm probe on CUDA"))+ refined, _ = _refine(_plan())+ assert refined.layer_split is not None+++_TWO_EXPERTS = { # only moving the second expert fits both cards+ "transformer": (9000, "dit"),+ "transformer_2": (9000, "dit"),+ "text_encoder": (4000, "text_encoder"),+ "vae": (500, "other"),+}+++def test_rocm_second_expert_moves_only_to_a_card_that_runs_the_attention_backend(+ monkeypatch, engine+):+ from core.inference import diffusion_attention as da+ from core.inference.diffusion_attention import ATTENTION_BACKEND_ATTR++ _cards(monkeypatch, sizes = _TWO_EXPERTS, sec_limit = 16000)+ plan = _plan()+ cuda, _ = _refine(plan)+ # the component split moves the expert+ assert "transformer_2" in (cuda.secondary_components or ())+ _rocm_host(monkeypatch, {0: _RX7900, 1: _MI300})+ pipe = _Pipe()+ pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)+ pipe.transformer_2 = torch.nn.Linear(2, 2).to(torch.bfloat16)+ rocm, _ = _refine(_plan(), pipe = pipe)+ assert "transformer_2" in (rocm.secondary_components or ()) # native SDPA, fused on both cards+ for tag in ("flash", "aiter"): # pinned to gfx1100: the gfx942 card must not run the expert+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, tag)+ setattr(pipe.transformer_2, ATTENTION_BACKEND_ATTR, tag)+ rocm, _ = _refine(_plan(), pipe = pipe)+ assert "transformer_2" not in (rocm.secondary_components or ())+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, None)+ setattr(pipe.transformer_2, ATTENTION_BACKEND_ATTR, None)+ monkeypatch.setattr(da, "sdpa_math_only", lambda t: t.torch_device == "cuda:1")+ rocm, _ = _refine(_plan(), pipe = pipe)+ assert "transformer_2" not in (rocm.secondary_components or ())diff --git a/studio/backend/tests/test_diffusion_multigpu.py b/studio/backend/tests/test_diffusion_multigpu.pyindex 47461fdcad7..31891fcc68c 100644--- a/studio/backend/tests/test_diffusion_multigpu.py+++ b/studio/backend/tests/test_diffusion_multigpu.py@@ -39,6 +39,7 @@ def _env(monkeypatch):# Keep make_selection's real-host filters from collapsing fake [0, 1] selections on a one-GPU host.monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)monkeypatch.setattr(mg, "_capability", lambda o: None)+ monkeypatch.setattr(mg, "_rocm_torch", lambda: False)yield@@ -1846,6 +1847,8 @@ def test_two_discrete_rocm_cards_split_like_cuda(monkeypatch):monkeypatch.setattr(torch.cuda, "is_available", lambda: True)monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)monkeypatch.setattr(mg, "_capability", lambda o: (11, 0)) # gfx1100, as ROCm reports it+ monkeypatch.setattr(mg, "_rocm_torch", lambda: True)+ monkeypatch.setattr(mg, "_rocm_bf16", lambda o: True)monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)selection = mg.make_selection(0, [0, 1])assert selection == mg.MultiGpuSelection(primary = 0, secondaries = (1,))diff --git a/studio/backend/tests/test_rocm_oom_guard.py b/studio/backend/tests/test_rocm_oom_guard.pyindex 8519d65a543..c2e1e12dd40 100644--- a/studio/backend/tests/test_rocm_oom_guard.py+++ b/studio/backend/tests/test_rocm_oom_guard.py@@ -205,6 +205,19 @@ class TestDeviceNameFallback:"AMD Radeon 860M Graphics","Radeon 840M","AMD Radeon 840M Graphics",+ # gfx1153 Krackan Point 2 (Ryzen AI 5 330)+ "AMD Radeon(TM) 820M Graphics",+ # gfx1151 Strix Halo (Ryzen AI Max 380)+ "AMD Radeon 8040S Graphics",+ # gfx1103 Phoenix / Hawk Point+ "AMD Radeon(TM) 780M",+ "AMD Radeon 760M Graphics",+ "AMD Radeon 740M",+ # gfx1035 Rembrandt+ "AMD Radeon 680M",+ "AMD Radeon(TM) 660M",+ # gfx1036 Raphael / Dragon Range, gfx1037 Mendocino+ "AMD Radeon(TM) 610M",# case variants"RADEON 8060S GRAPHICS","radeon 8050s",@@ -230,6 +243,12 @@ class TestDeviceNameFallback:# Superficially similar substrings but discrete"Radeon RX 580","Radeon VII",+ # discrete mobile parts whose digits contain an APU name+ "AMD Radeon RX 7600M XT",+ "AMD Radeon RX 6800M",+ "AMD Radeon RX 6650M",+ "AMD Radeon Pro W6600M",+ "AMD Radeon RX 7800M",],)def test_discrete_not_misclassified(self, device_name: str) -> None:diff --git a/studio/backend/tests/test_rocm_windows_vram_7072.py b/studio/backend/tests/test_rocm_windows_vram_7072.pyindex a4f301325ef..5b46e2e5f92 100644--- a/studio/backend/tests/test_rocm_windows_vram_7072.py+++ b/studio/backend/tests/test_rocm_windows_vram_7072.py@@ -2514,3 +2514,38 @@ def test_an_igpu_enumerated_first_is_still_paired_by_luid(win_rocm, monkeypatch)assert devices[0]["used_gb"] == pytest.approx(1.5, abs = 0.01)assert devices[1]["used_gb"] == pytest.approx(4.28, abs = 0.01)assert aggregate == pytest.approx(5.78, abs = 0.01)+++ "gfx, name",+ [+ ("gfx1153", "AMD Radeon(TM) 820M Graphics"), # Krackan Point 2, Windows-supported+ ("gfx1036", "AMD Radeon(TM) Graphics"), # Raphael / Granite Ridge iGPU+ ("gfx1035", "AMD Radeon(TM) 680M"),+ ("gfx1033", "AMD Custom GPU 0405"), # Van Gogh+ ("gfx90c", "AMD Radeon(TM) Graphics"),+ ],+)+def test_every_llvm_apu_target_is_positively_unified_without_the_flag(win_rocm, gfx, name):+ """LLVM's AMDGPUUsage processor table types these APU only; a runtime that leaves is_integrated 0 must not+ make one read as a discrete card (the diffusion layer split would then hand it denoiser blocks)."""++ class _P:+ gcnArchName = f"{gfx}:xnack-"+ total_memory = int(PHOENIX_POOL)+ is_integrated = 0++ _P.name = name+ assert hw._rocm_props_unified_status(_P()) is True+ assert hw._rocm_props_are_positively_unified(_P()) is True+++def test_a_krackan_point_2_with_no_arch_attrs_or_flag_is_unified_by_name(win_rocm):+ """AMD SDK / Radeon wheels may leave every arch attribute and is_integrated unset: the name is all there is."""++ class _P:+ name = "AMD Radeon(TM) 820M Graphics"+ total_memory = int(PHOENIX_POOL)++ assert hw._rocm_props_unified_status(_P()) is True+ assert hw._rocm_props_are_positively_unified(_P()) is Truediff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 0d20775edff..e5d6235955d 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -2805,7 +2805,24 @@ def _hip_runtime_version() -> Optional[tuple[int, int]]:# APU architectures that are unified memory but sit outside the shared classifier's positive set, which names anything else from hipDeviceProp_t::integrated. clr never assigned deviceProps.integrated before rocm-6.2.0 and the legacy R0000 property ABI still does not, while paldevice.cpp inflates globalMemSize_ off settings().apuSystem_ in both cases -- harmless for the cap and the total, wrong for a NUMERATOR. gfx1103 Phoenix / Hawk Point never shipped as a discrete board, so naming it here can only upgrade a part whose props.total_memory really does span host memory.-_ROCM_UNNAMED_APU_ARCHES = frozenset({"gfx1103"})+# The rest of LLVM's AMDGPUUsage processor table rows typed APU are named here on the same footing (none ever shipped as a+# discrete board): gfx1153 Krackan Point 2 (Windows-supported), gfx1033 Van Gogh, gfx1035 Rembrandt, gfx1036 Raphael /+# Granite Ridge iGPU, gfx1013 Cyan Skillfish, gfx90c / gfx902 / gfx909 Renoir / Raven, gfx801 / gfx810 Carrizo / Stoney.+_ROCM_UNNAMED_APU_ARCHES = frozenset(+ {+ "gfx801",+ "gfx810",+ "gfx902",+ "gfx909",+ "gfx90c",+ "gfx1013",+ "gfx1033",+ "gfx1035",+ "gfx1036",+ "gfx1103",+ "gfx1153",+ }+)def _rocm_props_unified_status(props: Any) -> Optional[bool]: