Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: allow the multi-GPU layer split on discrete ROCm cards (#14479)

Daniel Han04d2beec9b68
diff --git a/studio/backend/core/inference/diffusion_layer_split_plan.py b/studio/backend/core/inference/diffusion_layer_split_plan.py
index 3a8656a119a..0c66d3ebbaa 100644
--- a/studio/backend/core/inference/diffusion_layer_split_plan.py
+++ b/studio/backend/core/inference/diffusion_layer_split_plan.py
@@ -7,8 +7,9 @@ Order of preference on a multi-card selection: companion split (diffusion_multig
one-card offload plan. A layer split is only considered when the one-card plan offloads the DENOISER itself and the
companion assignment found nothing; it is priced with the same per-card reserves as the companion split. The VAE and
every non-text-encoder companion stay on the primary (the decode and its tile verdict stay where today's guard reads
-them); the text encoders may move to any used card. One card, Keep multiple models loaded, ROCm and the kill switches
-keep today's plan unchanged.
+them); the text encoders may move to any used card. One card, Keep multiple models loaded and the kill switches keep
+today's plan unchanged. ROCm splits across discrete cards only (an APU's memory is the host's: a split across "cards"
+sharing one pool frees nothing), and a per-gfx-target attention kernel keeps blocks on cards of the primary's target.
The engine (block placement + device-crossing hooks) is a separate piece; this module only defines the interface it
must provide (``split_blocks``, ``engine_available``, ``place_layer_split``, ``undo_layer_split``) with stubs that keep
@@ -545,11 +546,67 @@ def _record_decline(pipe: Any, reason: str, logger: Any) -> None:
def _is_rocm() -> bool:
+ from .diffusion_multigpu import _rocm_torch
+ return _rocm_torch()
+
+
+UNIFIED_DECLINE = "unified memory"
+
+
+def _rocm_card_unified(ordinal: int) -> Optional[bool]:
+ """True for a ROCm APU (Strix Halo / Point, Phoenix: the GPU's memory is the host's), False for a discrete card,
+ None when unreadable. Studio's own classifier (the driver's integrated flag, then the gfx target)."""
try:
import torch
- return bool(getattr(torch.version, "hip", None))
+
+ from utils.hardware.hardware import _rocm_props_unified_status
+ return _rocm_props_unified_status(torch.cuda.get_device_properties(int(ordinal)))
except Exception: # noqa: BLE001
- return False
+ return None
+
+
+def _unified_reason(cards: Sequence[int]) -> str:
+ names = ", ".join(f"cuda:{int(o)}" for o in cards)
+ what = "is an integrated GPU (APU)" if len(cards) == 1 else "are integrated GPUs (APUs)"
+ return (
+ f"{UNIFIED_DECLINE}: {names} {what} sharing system memory, so cutting the denoiser across cards frees no "
+ "memory"
+ )
+
+
+def note_unified_primary(
+ pipe: Any,
+ selection: Any,
+ logger: Any = None,
+) -> None:
+ """``refine_plan_for_multigpu`` stopped on a unified-memory primary: on ROCm say why in the layer split status (a
+ multi-APU selection is otherwise silent). CUDA keeps today's silent no-op."""
+ if selection is None or layer_split_mode() == "off" or not _is_rocm():
+ return
+ _record_decline(pipe, _unified_reason((int(selection.primary),)), logger)
+
+
+def _rocm_discrete_secondaries(
+ pipe: Any, primary: int, secondaries: Sequence[int], logger: Any
+) -> Optional[tuple[int, ...]]:
+ """ROCm: the secondaries a split may use (discrete cards); None after recording a decline. A unified primary
+ declines (its blocks and the host share one pool); a unified secondary is dropped (its blocks would sit in host
+ memory). An unreadable card is kept: the per-card budgets below still price it."""
+ if _rocm_card_unified(primary) is True:
+ _record_decline(pipe, _unified_reason((int(primary),)), logger)
+ return None
+ unified = tuple(int(o) for o in secondaries if _rocm_card_unified(int(o)) is True)
+ kept = tuple(int(o) for o in secondaries if int(o) not in unified)
+ if not kept:
+ _record_decline(pipe, _unified_reason(unified), logger)
+ return None
+ if unified and logger is not None:
+ logger.info(
+ "diffusion.multigpu: layer split skips %s (%s)",
+ ", ".join(f"cuda:{o}" for o in unified),
+ UNIFIED_DECLINE,
+ )
+ return kept
def plan_offloads_denoiser(plan: Any) -> bool:
@@ -615,7 +672,10 @@ _PRIMARY_ARCH_BACKENDS = frozenset({"_flash_3_hub", "flash_4_hub", "sage", "sage
def _backend_cards(
- pipe: Any, primary: int, secondaries: Sequence[int]
+ pipe: Any,
+ primary: int,
+ secondaries: Sequence[int],
+ logger: Any = None,
) -> tuple[tuple[int, ...], Optional[str]]:
"""``(secondaries the engaged attention backend runs on, that backend)``; a card of unknown capability is kept."""
from . import diffusion_multigpu as mg
@@ -630,6 +690,8 @@ def _backend_cards(
backend = tag
break
cards = tuple(int(o) for o in secondaries)
+ if _is_rocm():
+ return _rocm_backend_cards(pipe, primary, cards, logger)
if backend is None:
return cards, None
need = mg._capability(primary)
@@ -638,6 +700,145 @@ def _backend_cards(
return tuple(o for o in cards if (mg._capability(o) or need) == need), backend
+# ROCm kernels built per gfx target: ROCm flash_attn (CK / Triton, #13792) and AITER.
+_ROCM_ARCH_BACKENDS = frozenset({"flash", "aiter"})
+
+
+def _rocm_gfx(ordinal: int) -> str:
+ try:
+ import torch
+
+ from .rocm_bf16 import _device_gfx_arch
+ return _device_gfx_arch(torch, int(ordinal))
+ except Exception: # noqa: BLE001
+ return ""
+
+
+def _denoiser_dtype(module: Any) -> Any:
+ try:
+ return next(module.parameters()).dtype
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _rocm_flash_runs_on(ordinal: int, dtype: Any) -> bool:
+ try:
+ from types import SimpleNamespace
+
+ from .diffusion_attention import _rocm_flash_attn_runs
+ return bool(
+ _rocm_flash_attn_runs(
+ SimpleNamespace(device = "cuda", torch_device = f"cuda:{int(ordinal)}", dtype = dtype)
+ )
+ )
+ except Exception: # noqa: BLE001
+ return False
+
+
+MATH_ONLY_DECLINE = "math-only attention"
+_PROBE_DTYPES = ("torch.float16", "torch.bfloat16", "torch.float32")
+
+
+def _rocm_attention_dtype(pipe: Any) -> Any:
+ """The compute dtype the SDPA probe should run at: the denoiser's, else the pipeline's; None (the probe's fp16
+ default) when only a quantized storage dtype is readable."""
+ from . import diffusion_multigpu as mg
+
+ for name in mg.DENOISER_ATTRS:
+ module = getattr(pipe, name, None)
+ if module is not None:
+ dtype = _denoiser_dtype(module)
+ if str(dtype) in _PROBE_DTYPES:
+ return dtype
+ dtype = getattr(pipe, "dtype", None)
+ return dtype if str(dtype) in _PROBE_DTYPES else None
+
+
+def _rocm_sdpa_math_only(ordinal: int, dtype: Any) -> bool:
+ """diffusion_attention's per-device SDPA probe: True only when math ran and every fused kernel refused on this
+ card at this dtype (an unanswered probe is False)."""
+ try:
+ from types import SimpleNamespace
+
+ from .diffusion_attention import sdpa_math_only
+ return bool(
+ sdpa_math_only(
+ SimpleNamespace(device = "cuda", torch_device = f"cuda:{int(ordinal)}", dtype = dtype)
+ )
+ )
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _rocm_drop_math_only(
+ pipe: Any,
+ primary: int,
+ cards: tuple[int, ...],
+ logger: Any = None,
+) -> tuple[int, ...]:
+ """Drop a secondary whose native SDPA is math-only when the primary's is not: the plan's activation figures were
+ priced for the primary's fused kernel, and math SDPA materialises the full score matrix on the card running the
+ block. A math-only primary priced that peak already, so cards matching it are kept."""
+ if not cards:
+ return cards
+ dtype = _rocm_attention_dtype(pipe)
+ if _rocm_sdpa_math_only(primary, dtype):
+ return cards
+ dropped = tuple(o for o in cards if _rocm_sdpa_math_only(o, dtype))
+ if dropped and logger is not None:
+ logger.info(
+ "diffusion.multigpu: layer split skips %s (%s at %s)",
+ ", ".join(f"cuda:{o}" for o in dropped),
+ MATH_ONLY_DECLINE,
+ dtype,
+ )
+ return tuple(o for o in cards if o not in dropped)
+
+
+def _math_only_reason(primary: int, cards: Sequence[int]) -> str:
+ names = ", ".join(f"cuda:{int(o)}" for o in cards)
+ has = "has" if len(cards) == 1 else "have"
+ return (
+ f"{MATH_ONLY_DECLINE}: {names} {has} no fused SDPA kernel at the model dtype (cuda:{int(primary)} does), so its "
+ "attention would materialise the full score matrix"
+ )
+
+
+def _rocm_backend_cards(
+ pipe: Any,
+ primary: int,
+ cards: tuple[int, ...],
+ logger: Any = None,
+) -> tuple[tuple[int, ...], Optional[str]]:
+ """ROCm: native SDPA runs on any gfx target (the load dtype is already matched by ``make_selection``), so every
+ card whose SDPA is not math-only where the primary's is fused is kept. A per-target kernel (flash_attn, AITER) the
+ primary verified keeps only cards of the primary's gfx target, and flash only where its own probe runs and matches
+ on that card (as the primary's did at load)."""
+ from . import diffusion_multigpu as mg
+
+ backend = module = None
+ with contextlib.suppress(Exception):
+ from .diffusion_attention import ATTENTION_BACKEND_ATTR
+ for name in mg.DENOISER_ATTRS:
+ candidate = getattr(pipe, name, None)
+ tag = (
+ getattr(candidate, ATTENTION_BACKEND_ATTR, None) if candidate is not None else None
+ )
+ if tag in _ROCM_ARCH_BACKENDS:
+ backend, module = tag, candidate
+ break
+ if backend is None:
+ return _rocm_drop_math_only(pipe, primary, cards, logger), None
+ need = _rocm_gfx(primary)
+ if not need:
+ return (), backend
+ kept = tuple(o for o in cards if _rocm_gfx(o) == need)
+ if backend == "flash":
+ dtype = _denoiser_dtype(module)
+ kept = tuple(o for o in kept if _rocm_flash_runs_on(o, dtype))
+ return kept, backend
+
+
def refine_plan_for_layer_split(
pipe: Any,
plan: Any,
@@ -658,19 +859,23 @@ def refine_plan_for_layer_split(
return plan
if not plan_offloads_denoiser(plan) and mode != "force":
return plan
+ candidates: Sequence[int] = selection.secondaries
if _is_rocm():
- _record_decline(pipe, "ROCm (layer split is CUDA only)", logger)
- return plan
+ candidates = _rocm_discrete_secondaries(pipe, int(selection.primary), candidates, logger)
+ if candidates is None:
+ return plan
layouts = _denoiser_layouts(pipe)
if layouts is None:
_record_decline(pipe, f"denoiser cannot be split ({_layout_refusal(pipe)})", logger)
return plan
limits: dict[int, int] = {int(selection.primary): int(primary_limit_mib)}
- secondaries, backend = _backend_cards(pipe, int(selection.primary), selection.secondaries)
+ secondaries, backend = _backend_cards(pipe, int(selection.primary), candidates, logger)
if not secondaries:
_record_decline(
pipe,
- f"attention backend {backend} was verified for cuda:{int(selection.primary)}'s architecture only",
+ _math_only_reason(int(selection.primary), candidates)
+ if backend is None and _is_rocm()
+ else f"attention backend {backend} was verified for cuda:{int(selection.primary)}'s architecture only",
logger,
)
return plan
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.py
index e0d0fb48ecd..418f92ee509 100644
--- a/studio/backend/core/inference/diffusion_multigpu.py
+++ b/studio/backend/core/inference/diffusion_multigpu.py
@@ -125,6 +125,15 @@ def make_selection(
pass
if not others:
return None
+ if _rocm_torch():
+ # ROCm's capability is the gfx target (gfx942 (9, 4)), which does not order; match the primary's load dtype
+ # instead (arch-pinned attention kernels are gated per card in diffusion_layer_split_plan).
+ others = tuple(o for o in others if _rocm_dtype_parity(int(primary), o))
+ return (
+ MultiGpuSelection(primary = int(primary), secondaries = others, automatic = automatic)
+ if others
+ else None
+ )
# The load dtype and the device-specific patches are resolved for the primary (bf16 only from Ampere, FP8 kernels
# from Ada): a card of a lower compute capability may not run them, so it takes no component.
need = _capability(int(primary))
@@ -173,6 +182,35 @@ def _capability(ordinal: int) -> Optional[tuple[int, int]]:
return None
+def _rocm_torch() -> bool:
+ """A ROCm torch build (AMD SDK / Radeon wheels included, which leave ``torch.version.hip`` unset)."""
+ try:
+ import torch
+
+ from .rocm_bf16 import is_rocm_torch
+ return is_rocm_torch(torch)
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _rocm_bf16(ordinal: int) -> Optional[bool]:
+ try:
+ import torch
+
+ from .rocm_bf16 import rocm_bf16_supported
+ return bool(rocm_bf16_supported(torch, int(ordinal)))
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _rocm_dtype_parity(primary: int, ordinal: int) -> bool:
+ """A ROCm card runs the dtype the load resolves for the primary: bf16 when the primary has native bf16 (RDNA1/2,
+ GCN and Vega emulate it), anything otherwise. An unreadable card is kept, like an unknown CUDA capability."""
+ if _rocm_bf16(primary) is not True:
+ return True
+ return _rocm_bf16(ordinal) is not False
+
+
@contextlib.contextmanager
def selection_scope(
selection: Optional[MultiGpuSelection],
@@ -383,12 +421,14 @@ def choose_component_assignment(
overhead_mib: int,
primary_limit_mib: int,
secondary_limit_mib: int,
+ move_denoisers: bool = True,
) -> Optional[tuple[str, ...]]:
"""Names to place on the secondary so each card holds its share resident, or None. ``sizes`` maps component name
-> (MiB, role) with role 'dit' / 'text_encoder' / 'other'. A card running a denoiser or the VAE (any 'other'
component) is priced with the runtime headroom; a card holding only text encoders with an encoder forward's room.
Tried in order: every companion; the text encoders alone; the second denoiser; the second denoiser plus the
- companions. The first denoiser always stays on the primary."""
+ companions. The first denoiser always stays on the primary; ``move_denoisers=False`` (the secondary cannot run the
+ denoiser's attention backend) keeps every denoiser there."""
dits = [n for n, (_m, r) in sizes.items() if r == "dit"]
if not dits:
return None
@@ -405,7 +445,7 @@ def choose_component_assignment(
candidates.append(tuple(companions))
if encoders and len(encoders) < len(companions):
candidates.append(tuple(encoders))
- if seconds:
+ if seconds and move_denoisers:
candidates.append(tuple(seconds))
if companions:
candidates.append(tuple(seconds + companions))
@@ -422,6 +462,32 @@ def choose_component_assignment(
return None
+def _secondary_runs_denoiser(
+ pipe: Any, selection: MultiGpuSelection, secondary: int, sizes: dict, logger: Any
+) -> bool:
+ """Whether a second denoiser may move wholesale to ``secondary``. ROCm admits mixed gfx targets (make_selection),
+ so the card must pass the layer split's per-card attention gate: an arch-pinned kernel (flash_attn, AITER) only on
+ the primary's gfx target, native SDPA not math-only beside a fused primary. CUDA's capability order already holds."""
+ if not _rocm_torch() or sum(1 for _m, role in sizes.values() if role == "dit") < 2:
+ return True
+ try:
+ from .diffusion_layer_split_plan import _rocm_backend_cards
+ kept, _backend = _rocm_backend_cards(
+ pipe, int(selection.primary), (int(secondary),), logger
+ )
+ except Exception: # noqa: BLE001 - unverifiable: keep the denoisers on the primary
+ return False
+ if int(secondary) in kept:
+ return True
+ if logger is not None:
+ logger.info(
+ "diffusion.multigpu: second denoiser stays on cuda:%s (attention backend not verified on cuda:%s)",
+ selection.primary,
+ secondary,
+ )
+ return False
+
+
def refine_plan_for_multigpu(
pipe: Any,
plan: Any,
@@ -451,6 +517,9 @@ def refine_plan_for_multigpu(
return plan
memory = plan.device_memory
if memory.is_unified or getattr(memory, "device", None) != "cuda":
+ if memory.is_unified:
+ from .diffusion_layer_split_plan import note_unified_primary
+ note_unified_primary(pipe, selection, logger)
return plan
budget = plan.estimates.get("safe_device_budget_mib")
if budget is None:
@@ -503,6 +572,7 @@ def refine_plan_for_multigpu(
overhead_mib = overhead,
primary_limit_mib = int(primary_limit),
secondary_limit_mib = int(sec_limit),
+ move_denoisers = _secondary_runs_denoiser(pipe, selection, secondary, sizes, logger),
)
if moved is None:
return layer_split_or_plan()
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 4288a6f51d3..2bbcc8108ee 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -1925,6 +1925,12 @@ def _ensure_mamba_ssm(
)
+# Integrated-only Radeon names for the other APU targets, as whole tokens (a substring would read "RX 7600M" as "600M"):
+# 820M Krackan Point 2 (gfx1153), 8040S Strix Halo (gfx1151), 780M / 760M / 740M Phoenix / Hawk Point (gfx1103),
+# 680M / 660M Rembrandt (gfx1035), 610M Raphael / Dragon Range / Mendocino (gfx1036 / gfx1037).
+_ROCM_APU_NAME_RE = re.compile(r"(?<![\w-])(?:820m|8040s|780m|760m|740m|680m|660m|610m)(?!\w)")
+
+
def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:
"""Classify a ROCm device as unified-memory (APU) or discrete, returning ``(gcn_arch,
is_unified)``. ``gcn_arch`` is the canonical arch string (e.g. ``"gfx1151"``) when a known
@@ -1970,6 +1976,7 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]:
or "8050s" in dev_lower
or "860m" in dev_lower
or "840m" in dev_lower
+ or _ROCM_APU_NAME_RE.search(dev_lower) is not None
)
return gcn_arch, is_unified
diff --git a/studio/backend/tests/test_diffusion_layer_split_order.py b/studio/backend/tests/test_diffusion_layer_split_order.py
index be875001f78..fd7abbf0351 100644
--- a/studio/backend/tests/test_diffusion_layer_split_order.py
+++ b/studio/backend/tests/test_diffusion_layer_split_order.py
@@ -30,6 +30,7 @@ def _env(monkeypatch):
monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)
monkeypatch.setattr(mg, "_capability", lambda o: None)
monkeypatch.setattr(lsp, "_is_rocm", lambda: False)
+ monkeypatch.setattr(mg, "_rocm_torch", lambda: False)
yield
diff --git a/studio/backend/tests/test_diffusion_layer_split_plan.py b/studio/backend/tests/test_diffusion_layer_split_plan.py
index a0d7c6bbe20..a47bd05928b 100644
--- a/studio/backend/tests/test_diffusion_layer_split_plan.py
+++ b/studio/backend/tests/test_diffusion_layer_split_plan.py
@@ -29,6 +29,7 @@ def _env(monkeypatch):
monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)
monkeypatch.setattr(mg, "_capability", lambda o: None)
monkeypatch.setattr(lsp, "_is_rocm", lambda: False)
+ monkeypatch.setattr(mg, "_rocm_torch", lambda: False)
yield
@@ -587,12 +588,16 @@ def test_cards_too_small_keep_the_offload_plan_and_record_why(monkeypatch, engin
assert mg.placement_status(pipe)["layer_split"] == status
-def test_rocm_explicit_modes_and_legacy_cpu_offload_keep_the_plan(monkeypatch, engine):
+def test_rocm_discrete_splits_and_explicit_modes_and_legacy_cpu_offload_keep_the_plan(
+ monkeypatch, engine
+):
_cards(monkeypatch)
plan = _plan()
- monkeypatch.setattr(lsp, "_is_rocm", lambda: True)
- assert _refine(plan)[0] is plan
+ _rocm_host(monkeypatch, {0: _RX7900, 1: _RX7900})
+ refined, pipe = _refine(plan)
+ assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)
monkeypatch.setattr(lsp, "_is_rocm", lambda: False)
+ monkeypatch.setattr(mg, "_rocm_torch", lambda: False)
balanced = _plan(requested_mode = "balanced")
assert _refine(balanced)[0] is balanced
legacy = _plan(explicit_offload = True)
@@ -945,3 +950,236 @@ def test_an_arch_pinned_attention_backend_keeps_blocks_off_other_architectures(m
) # runs on every admitted card
refined, _ = _refine(plan, pipe = pipe, ordinals = (0, 1))
assert refined.layer_split is not None and 1 in refined.layer_split.cards
+
+
+# ROCm: real classifiers (utils.hardware's unified test, rocm_bf16's gfx table) over fake device properties.
+_RX7900 = ("gfx1100", 0) # 7900 XTX, discrete
+_MI300 = ("gfx942", 0)
+_RX6900 = ("gfx1030", 0) # RDNA2: no native bf16
+_STRIX_HALO = ("gfx1151", 1) # APU: the driver's integrated flag set
+
+
+def _rocm_host(monkeypatch, cards):
+ from types import SimpleNamespace
+
+ monkeypatch.setattr(lsp, "_is_rocm", lambda: True)
+ monkeypatch.setattr(mg, "_rocm_torch", lambda: True)
+ monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda *a, **k: True)
+ monkeypatch.delenv("UNSLOTH_STUDIO_ROCM_BF16", raising = False)
+
+ def props(o = None):
+ arch, integrated = cards[int(o)]
+ return SimpleNamespace(
+ gcnArchName = f"{arch}:sramecc+:xnack-", is_integrated = integrated, name = "AMD Radeon"
+ )
+
+ monkeypatch.setattr(torch.cuda, "get_device_properties", props)
+ # the per-card SDPA probe spawns HIP children; every card has a fused kernel unless a test says otherwise
+ from core.inference import diffusion_attention as da
+
+ monkeypatch.setattr(da, "sdpa_math_only", lambda target: False)
+
+
+def test_rocm_two_discrete_cards_plan_the_same_split_as_cuda(monkeypatch, engine):
+ _cards(monkeypatch)
+ cuda, _ = _refine(_plan())
+ _rocm_host(monkeypatch, {0: _RX7900, 1: _RX7900})
+ rocm, pipe = _refine(_plan())
+ assert rocm.layer_split is not None
+ assert rocm.layer_split.ranges == cuda.layer_split.ranges
+ assert rocm.layer_split.card_required_mib == cuda.layer_split.card_required_mib
+ assert lsp.layer_split_status(pipe) is None # no decline recorded
+
+
+def test_rocm_two_apus_decline_with_the_unified_reason(monkeypatch, engine):
+ _cards(monkeypatch)
+ _rocm_host(monkeypatch, {0: _STRIX_HALO, 1: _STRIX_HALO})
+ plan = _plan()
+ refined, pipe = _refine(plan)
+ assert refined is plan and engine.placed == []
+ declined = lsp.layer_split_status(pipe)["declined"]
+ assert declined.startswith(lsp.UNIFIED_DECLINE) and "cuda:0" in declined
+
+
+def test_rocm_unified_primary_plan_records_why_and_cuda_stays_silent(monkeypatch, engine):
+ _cards(monkeypatch)
+ unified = dm.DeviceMemory(
+ backend = "cuda", device = "cuda", memory_kind = "unified_memory", free_mib = 90000, total_mib = 98000
+ )
+ plan = dataclasses.replace(_plan(), device_memory = unified)
+ monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "force")
+ refined, pipe = _refine(plan)
+ assert refined is plan and lsp.layer_split_status(pipe) is None # CUDA: unchanged
+ _rocm_host(monkeypatch, {0: _STRIX_HALO, 1: _STRIX_HALO})
+ refined, pipe = _refine(plan)
+ assert refined is plan and engine.placed == []
+ assert lsp.layer_split_status(pipe)["declined"].startswith(lsp.UNIFIED_DECLINE)
+
+
+def test_rocm_discrete_primary_skips_an_apu_secondary(monkeypatch, engine):
+ _cards(monkeypatch)
+ _rocm_host(monkeypatch, {0: _RX7900, 1: _STRIX_HALO, 2: _RX7900})
+ refined, _ = _refine(_plan(), ordinals = (0, 1, 2))
+ assert refined.layer_split is not None and 1 not in refined.layer_split.cards
+ assert 2 in refined.layer_split.cards
+ refined, pipe = _refine(_plan(), ordinals = (0, 1))
+ assert refined.layer_split is None
+ assert "cuda:1" in lsp.layer_split_status(pipe)["declined"]
+
+
+def test_rocm_kill_switch_keeps_the_plan_and_records_nothing(monkeypatch, engine):
+ _cards(monkeypatch)
+ _rocm_host(monkeypatch, {0: _RX7900, 1: _RX7900})
+ monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "0")
+ plan = _plan()
+ refined, pipe = _refine(plan)
+ assert refined is plan and engine.placed == [] and lsp.layer_split_status(pipe) is None
+
+
+def test_rocm_mixed_arch_selection_compares_dtype_not_gfx_order(monkeypatch):
+ # CUDA's capability order would drop MI300 (9, 4) under a 7900 XTX (11, 0) primary
+ _rocm_host(monkeypatch, {0: _RX7900, 1: _MI300, 2: _RX6900})
+ monkeypatch.setattr(mg, "_capability", lambda o: {0: (11, 0), 1: (9, 4), 2: (10, 3)}[int(o)])
+ assert mg.make_selection(0, [0, 1]) == mg.MultiGpuSelection(primary = 0, secondaries = (1,))
+ assert mg.make_selection(1, [1, 0]) == mg.MultiGpuSelection(primary = 1, secondaries = (0,))
+ # a bf16 primary keeps no RDNA2 card (it would emulate the primary's dtype); an fp16 primary keeps any
+ assert mg.make_selection(0, [0, 2]) is None
+ assert mg.make_selection(2, [2, 0, 1]) == mg.MultiGpuSelection(primary = 2, secondaries = (0, 1))
+
+
+def test_rocm_mixed_arch_native_sdpa_splits_and_flash_stays_on_the_primary_target(
+ monkeypatch, engine
+):
+ from core.inference.diffusion_attention import ATTENTION_BACKEND_ATTR
+
+ _cards(monkeypatch)
+ _rocm_host(monkeypatch, {0: _RX7900, 1: _MI300, 2: _RX7900})
+ pipe = _Pipe()
+ pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)
+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1))
+ assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)
+ probed = []
+ monkeypatch.setattr(
+ lsp, "_rocm_flash_runs_on", lambda o, dtype: probed.append((o, dtype)) or o != 2
+ )
+ for tag in ("flash", "aiter"):
+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, tag)
+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1))
+ assert refined.layer_split is None
+ assert f"attention backend {tag}" in lsp._DECLINES[pipe]
+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, "aiter")
+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1, 2))
+ assert refined.layer_split is not None and 1 not in refined.layer_split.cards
+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, "flash")
+ probed.clear()
+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1, 2))
+ assert probed == [
+ (2, torch.bfloat16)
+ ] # the same-target card is probed; the MI300 never reaches the probe
+ assert refined.layer_split is None # flash failed on cuda:2: no secondary left
+
+
+def test_windows_rocm_secondary_budget_uses_the_capped_free(monkeypatch):
+ """WDDM: mem_get_info reports free ~ total whatever is resident; the secondary is priced from what torch has not
+ reserved there (utils.hardware.trusted_mem_get_info)."""
+ import sys
+
+ from utils.hardware import hardware as hw
+
+ gib = 1 << 30
+ monkeypatch.setattr(sys, "platform", "win32")
+ monkeypatch.setattr(hw, "IS_ROCM", True)
+ monkeypatch.setattr(
+ torch.cuda, "mem_get_info", lambda o = None: (24 * gib - (64 << 20), 24 * gib)
+ )
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda o = None: 10 * gib)
+ memory = mg.secondary_device_memory(1)
+ assert memory.total_mib == 24 * 1024 and memory.free_mib == 14 * 1024
+ assert mg.secondary_resident_limit_mib(1, "auto") < 14 * 1024
+ monkeypatch.setattr(hw, "IS_ROCM", False)
+ assert mg.secondary_device_memory(1).free_mib == 24 * 1024 - 64
+
+
+def test_rocm_apu_without_the_integrated_flag_gets_no_blocks(monkeypatch, engine):
+ # Windows gfx1153 Krackan Point 2 on a runtime that leaves is_integrated 0: still an APU by its gfx target
+ _cards(monkeypatch)
+ _rocm_host(monkeypatch, {0: _RX7900, 1: ("gfx1153", 0), 2: _RX7900})
+ refined, _ = _refine(_plan(), ordinals = (0, 1, 2))
+ assert refined.layer_split is not None and 1 not in refined.layer_split.cards
+ refined, pipe = _refine(_plan(), ordinals = (0, 1))
+ assert refined.layer_split is None
+ assert lsp.layer_split_status(pipe)["declined"].startswith(lsp.UNIFIED_DECLINE)
+
+
+def test_rocm_math_only_secondary_gets_no_blocks_beside_a_fused_primary(monkeypatch, engine):
+ # mixed gfx, native SDPA: cuda:1 has no fused kernel at bf16, so math SDPA would hold the full score matrix there
+ _cards(monkeypatch)
+ _rocm_host(monkeypatch, {0: _RX7900, 1: _MI300, 2: _RX7900})
+ pipe = _Pipe()
+ pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)
+ from core.inference import diffusion_attention as da
+
+ probed = []
+ monkeypatch.setattr(
+ da,
+ "sdpa_math_only",
+ lambda t: probed.append((t.torch_device, t.dtype)) or t.torch_device == "cuda:1",
+ )
+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1, 2))
+ assert refined.layer_split is not None and 1 not in refined.layer_split.cards
+ assert 2 in refined.layer_split.cards
+ assert ("cuda:1", torch.bfloat16) in probed # probed per card at the denoiser's dtype
+ refined, pipe = _refine(_plan(), pipe = pipe, ordinals = (0, 1))
+ assert refined.layer_split is None
+ declined = lsp.layer_split_status(pipe)["declined"]
+ assert declined.startswith(lsp.MATH_ONLY_DECLINE) and "cuda:1" in declined
+ # a math-only primary already priced the quadratic peak: matching cards keep the split
+ monkeypatch.setattr(da, "sdpa_math_only", lambda t: True)
+ refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1))
+ assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)
+
+
+def test_cuda_split_never_runs_the_rocm_sdpa_probe(monkeypatch, engine):
+ _cards(monkeypatch)
+ from core.inference import diffusion_attention as da
+
+ monkeypatch.setattr(da, "sdpa_math_only", lambda t: pytest.fail("ROCm probe on CUDA"))
+ refined, _ = _refine(_plan())
+ assert refined.layer_split is not None
+
+
+_TWO_EXPERTS = { # only moving the second expert fits both cards
+ "transformer": (9000, "dit"),
+ "transformer_2": (9000, "dit"),
+ "text_encoder": (4000, "text_encoder"),
+ "vae": (500, "other"),
+}
+
+
+def test_rocm_second_expert_moves_only_to_a_card_that_runs_the_attention_backend(
+ monkeypatch, engine
+):
+ from core.inference import diffusion_attention as da
+ from core.inference.diffusion_attention import ATTENTION_BACKEND_ATTR
+
+ _cards(monkeypatch, sizes = _TWO_EXPERTS, sec_limit = 16000)
+ plan = _plan()
+ cuda, _ = _refine(plan)
+ # the component split moves the expert
+ assert "transformer_2" in (cuda.secondary_components or ())
+ _rocm_host(monkeypatch, {0: _RX7900, 1: _MI300})
+ pipe = _Pipe()
+ pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)
+ pipe.transformer_2 = torch.nn.Linear(2, 2).to(torch.bfloat16)
+ rocm, _ = _refine(_plan(), pipe = pipe)
+ assert "transformer_2" in (rocm.secondary_components or ()) # native SDPA, fused on both cards
+ for tag in ("flash", "aiter"): # pinned to gfx1100: the gfx942 card must not run the expert
+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, tag)
+ setattr(pipe.transformer_2, ATTENTION_BACKEND_ATTR, tag)
+ rocm, _ = _refine(_plan(), pipe = pipe)
+ assert "transformer_2" not in (rocm.secondary_components or ())
+ setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, None)
+ setattr(pipe.transformer_2, ATTENTION_BACKEND_ATTR, None)
+ monkeypatch.setattr(da, "sdpa_math_only", lambda t: t.torch_device == "cuda:1")
+ rocm, _ = _refine(_plan(), pipe = pipe)
+ assert "transformer_2" not in (rocm.secondary_components or ())
diff --git a/studio/backend/tests/test_diffusion_multigpu.py b/studio/backend/tests/test_diffusion_multigpu.py
index 47461fdcad7..31891fcc68c 100644
--- a/studio/backend/tests/test_diffusion_multigpu.py
+++ b/studio/backend/tests/test_diffusion_multigpu.py
@@ -39,6 +39,7 @@ def _env(monkeypatch):
# Keep make_selection's real-host filters from collapsing fake [0, 1] selections on a one-GPU host.
monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)
monkeypatch.setattr(mg, "_capability", lambda o: None)
+ monkeypatch.setattr(mg, "_rocm_torch", lambda: False)
yield
@@ -1846,6 +1847,8 @@ def test_two_discrete_rocm_cards_split_like_cuda(monkeypatch):
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
monkeypatch.setattr(mg, "_capability", lambda o: (11, 0)) # gfx1100, as ROCm reports it
+ monkeypatch.setattr(mg, "_rocm_torch", lambda: True)
+ monkeypatch.setattr(mg, "_rocm_bf16", lambda o: True)
monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
selection = mg.make_selection(0, [0, 1])
assert selection == mg.MultiGpuSelection(primary = 0, secondaries = (1,))
diff --git a/studio/backend/tests/test_rocm_oom_guard.py b/studio/backend/tests/test_rocm_oom_guard.py
index 8519d65a543..c2e1e12dd40 100644
--- a/studio/backend/tests/test_rocm_oom_guard.py
+++ b/studio/backend/tests/test_rocm_oom_guard.py
@@ -205,6 +205,19 @@ class TestDeviceNameFallback:
"AMD Radeon 860M Graphics",
"Radeon 840M",
"AMD Radeon 840M Graphics",
+ # gfx1153 Krackan Point 2 (Ryzen AI 5 330)
+ "AMD Radeon(TM) 820M Graphics",
+ # gfx1151 Strix Halo (Ryzen AI Max 380)
+ "AMD Radeon 8040S Graphics",
+ # gfx1103 Phoenix / Hawk Point
+ "AMD Radeon(TM) 780M",
+ "AMD Radeon 760M Graphics",
+ "AMD Radeon 740M",
+ # gfx1035 Rembrandt
+ "AMD Radeon 680M",
+ "AMD Radeon(TM) 660M",
+ # gfx1036 Raphael / Dragon Range, gfx1037 Mendocino
+ "AMD Radeon(TM) 610M",
# case variants
"RADEON 8060S GRAPHICS",
"radeon 8050s",
@@ -230,6 +243,12 @@ class TestDeviceNameFallback:
# Superficially similar substrings but discrete
"Radeon RX 580",
"Radeon VII",
+ # discrete mobile parts whose digits contain an APU name
+ "AMD Radeon RX 7600M XT",
+ "AMD Radeon RX 6800M",
+ "AMD Radeon RX 6650M",
+ "AMD Radeon Pro W6600M",
+ "AMD Radeon RX 7800M",
],
)
def test_discrete_not_misclassified(self, device_name: str) -> None:
diff --git a/studio/backend/tests/test_rocm_windows_vram_7072.py b/studio/backend/tests/test_rocm_windows_vram_7072.py
index a4f301325ef..5b46e2e5f92 100644
--- a/studio/backend/tests/test_rocm_windows_vram_7072.py
+++ b/studio/backend/tests/test_rocm_windows_vram_7072.py
@@ -2514,3 +2514,38 @@ def test_an_igpu_enumerated_first_is_still_paired_by_luid(win_rocm, monkeypatch)
assert devices[0]["used_gb"] == pytest.approx(1.5, abs = 0.01)
assert devices[1]["used_gb"] == pytest.approx(4.28, abs = 0.01)
assert aggregate == pytest.approx(5.78, abs = 0.01)
+
+
+ "gfx, name",
+ [
+ ("gfx1153", "AMD Radeon(TM) 820M Graphics"), # Krackan Point 2, Windows-supported
+ ("gfx1036", "AMD Radeon(TM) Graphics"), # Raphael / Granite Ridge iGPU
+ ("gfx1035", "AMD Radeon(TM) 680M"),
+ ("gfx1033", "AMD Custom GPU 0405"), # Van Gogh
+ ("gfx90c", "AMD Radeon(TM) Graphics"),
+ ],
+)
+def test_every_llvm_apu_target_is_positively_unified_without_the_flag(win_rocm, gfx, name):
+ """LLVM's AMDGPUUsage processor table types these APU only; a runtime that leaves is_integrated 0 must not
+ make one read as a discrete card (the diffusion layer split would then hand it denoiser blocks)."""
+
+ class _P:
+ gcnArchName = f"{gfx}:xnack-"
+ total_memory = int(PHOENIX_POOL)
+ is_integrated = 0
+
+ _P.name = name
+ assert hw._rocm_props_unified_status(_P()) is True
+ assert hw._rocm_props_are_positively_unified(_P()) is True
+
+
+def test_a_krackan_point_2_with_no_arch_attrs_or_flag_is_unified_by_name(win_rocm):
+ """AMD SDK / Radeon wheels may leave every arch attribute and is_integrated unset: the name is all there is."""
+
+ class _P:
+ name = "AMD Radeon(TM) 820M Graphics"
+ total_memory = int(PHOENIX_POOL)
+
+ assert hw._rocm_props_unified_status(_P()) is True
+ assert hw._rocm_props_are_positively_unified(_P()) is True
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 0d20775edff..e5d6235955d 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -2805,7 +2805,24 @@ def _hip_runtime_version() -> Optional[tuple[int, int]]:
# APU architectures that are unified memory but sit outside the shared classifier's positive set, which names anything else from hipDeviceProp_t::integrated. clr never assigned deviceProps.integrated before rocm-6.2.0 and the legacy R0000 property ABI still does not, while paldevice.cpp inflates globalMemSize_ off settings().apuSystem_ in both cases -- harmless for the cap and the total, wrong for a NUMERATOR. gfx1103 Phoenix / Hawk Point never shipped as a discrete board, so naming it here can only upgrade a part whose props.total_memory really does span host memory.
-_ROCM_UNNAMED_APU_ARCHES = frozenset({"gfx1103"})
+# The rest of LLVM's AMDGPUUsage processor table rows typed APU are named here on the same footing (none ever shipped as a
+# discrete board): gfx1153 Krackan Point 2 (Windows-supported), gfx1033 Van Gogh, gfx1035 Rembrandt, gfx1036 Raphael /
+# Granite Ridge iGPU, gfx1013 Cyan Skillfish, gfx90c / gfx902 / gfx909 Renoir / Raven, gfx801 / gfx810 Carrizo / Stoney.
+_ROCM_UNNAMED_APU_ARCHES = frozenset(
+ {
+ "gfx801",
+ "gfx810",
+ "gfx902",
+ "gfx909",
+ "gfx90c",
+ "gfx1013",
+ "gfx1033",
+ "gfx1035",
+ "gfx1036",
+ "gfx1103",
+ "gfx1153",
+ }
+)
def _rocm_props_unified_status(props: Any) -> Optional[bool]: