Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: probe the regional diffusion compile on ROCm instead of refusing it (#13774)

Daniel Han71cfcb974bf6
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 729bb65280a..1493085dc6a 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -189,8 +189,10 @@ from .diffusion_speed import (
apply_speed_optims,
engage_pinned_denoisers,
auto_dynamic_active,
+ _gguf_dequant_compilable,
compile_eligible,
compiled_shapes_are_static,
+ rocm_probe_load_scoped,
family_compiles_regionally,
fp16_compile_explicit_only,
fp16_unet_offloaded,
@@ -2049,7 +2051,12 @@ def _memory_request_forces_offload(memory_mode: Optional[str], cpu_offload: bool
def _pipeline_quant_uncompilable_reason(
- target: Any, fam: Any, speed_mode: Optional[str], *, model_kind: str
+ target: Any,
+ fam: Any,
+ speed_mode: Optional[str],
+ *,
+ model_kind: str,
+ probe: bool = False,
) -> Optional[str]:
"""Why a PIPELINE load must keep its dense weights rather than quantise them, or None.
@@ -2059,7 +2066,9 @@ def _pipeline_quant_uncompilable_reason(
bf16. GGUF is out of scope, substituting dense base weights and falling back to the packed file.
speed=off is absent on purpose: an engaged quant upgrades it to `default`, and an AUTO request
- under it is rewritten to off long before either caller."""
+ under it is rewritten to off long before either caller.
+
+ ``probe``: only the loader, which owns the evicted card, runs the allocating ROCm compile probe."""
if model_kind != "pipeline":
return None
# A string compare, not resolve_speed_mode: this runs on the route, where an unvalidated value must not raise.
@@ -2068,7 +2077,7 @@ def _pipeline_quant_uncompilable_reason(
"Speed is set to 'eager', and a quantised transformer that is not compiled runs far "
"slower than the bf16 weights it replaces. Pick a compiling speed mode to combine the two"
)
- if not compile_eligible(target, is_gguf = False, family = fam):
+ if not compile_eligible(target, is_gguf = False, family = fam, probe = probe):
return (
"this process cannot run a torch.compile (no Triton, TORCHDYNAMO_DISABLE, or a "
"family/device that does not compile), and a quantised transformer that is not "
@@ -2077,6 +2086,30 @@ def _pipeline_quant_uncompilable_reason(
return None
+def _compile_cache_wanted(
+ pipe: Any, target: Any, fam: Any, *, is_gguf: bool, speed_mode: str, offload_active: bool
+) -> bool:
+ """``compile_eligible`` for the compile-cache gate; on a dense ROCm load the allocating probe runs only once the
+ plan finds something in the model to compile."""
+ if is_gguf or getattr(target, "backend", None) != "rocm":
+ return compile_eligible(target, is_gguf = is_gguf, family = fam)
+ if not compile_eligible(target, is_gguf = False, family = fam, probe = False):
+ return False
+ try:
+ from .diffusion_autoopt import plan as plan_mod
+ step = plan_mod.build_plan(
+ pipe,
+ target,
+ family = fam,
+ backend = "image",
+ speed_mode = speed_mode,
+ offload_active = offload_active,
+ ).step("compiled")
+ except Exception: # noqa: BLE001 - no plan: the probing predicate decides as before
+ return compile_eligible(target, is_gguf = False, family = fam)
+ return bool(step is not None and step.attempt)
+
+
def _plan_proves_resident(plan: Any) -> bool:
"""Resident AND fits; the planner also stays resident when it cannot read the card."""
estimates = getattr(plan, "estimates", None) or {}
@@ -5931,6 +5964,7 @@ class DiffusionBackend:
return DiffusionBackend._union_over_cached_revs(base, _params, staged_dir) * 2
@_invalidates_gpu_memory("diffusion load")
+ @rocm_probe_load_scoped
@_account_owned_load
@_plans_at_requested_speed
@scoped_local_files_only
@@ -7547,11 +7581,6 @@ class DiffusionBackend:
_load_token,
)
- # The same helper the route preflight asked, so the two cannot disagree.
- pipeline_quant_uncompilable = _pipeline_quant_uncompilable_reason(
- target, fam, speed_mode, model_kind = kind
- )
-
# Quantise dense bf16 pipeline denoisers in place. The blocker excludes UNet and pre-quantised
# pipelines; offloaded plans stay dense unless whole-module.
if (
@@ -7561,6 +7590,15 @@ class DiffusionBackend:
and normalize_transformer_quant(transformer_quant) is not None
and (dense_transformer_supported(target) or native_scheme is not None)
):
+ # The same helper the route preflight asked; probed only for an engaged torchao request
+ # (a native scheme never reads it).
+ pipeline_quant_uncompilable = (
+ None
+ if native_scheme is not None
+ else _pipeline_quant_uncompilable_reason(
+ target, fam, speed_mode, model_kind = kind, probe = True
+ )
+ )
# from_pretrained widens a raw fp8/int8 checkpoint to bf16, erasing the one thing the blocker
# below reads, so recover it from the shard header (Ideogram's loader stamps its own).
# `fetch_base` too: a LOCAL diffusers directory stages nothing, so `_base_local_dir` is None
@@ -7799,7 +7837,8 @@ class DiffusionBackend:
and effective_speed == SPEED_OFF
and transformer_quant_engaged is None
and not comfy_compile
- and compile_eligible(target, is_gguf = False, family = fam)
+ # Non-probing: _engage_deferred_profile runs the ROCm probe if the 3rd image arrives.
+ and compile_eligible(target, is_gguf = False, family = fam, probe = False)
and not fp16_compile_explicit_only(target)
)
# Speed optims run BEFORE placement, so snapshot the global backend flags first for unload restore.
@@ -7974,7 +8013,37 @@ class DiffusionBackend:
# first compiled forward pays the 25-58s compile once.
if (
effective_speed in (SPEED_DEFAULT, SPEED_MAX)
- and compile_eligible(target, is_gguf = gguf_transformer, family = fam)
+ # ROCm GGUF default compiles only the dequant: gate on that predicate, never the regional probe.
+ # On max the dequant is the fallback when the regional compile is refused, so it keeps the cache.
+ and (
+ _gguf_dequant_compilable(
+ True,
+ getattr(target, "device", None) == "cuda",
+ bool(getattr(fam, "supports_torch_compile", True)),
+ target,
+ )
+ if gguf_transformer
+ and effective_speed == SPEED_DEFAULT
+ and getattr(target, "backend", None) == "rocm"
+ else _compile_cache_wanted(
+ pipe,
+ target,
+ fam,
+ is_gguf = gguf_transformer,
+ speed_mode = effective_speed,
+ offload_active = plan.offload_policy != OFFLOAD_NONE,
+ )
+ or (
+ gguf_transformer
+ and getattr(target, "backend", None) == "rocm"
+ and _gguf_dequant_compilable(
+ True,
+ getattr(target, "device", None) == "cuda",
+ bool(getattr(fam, "supports_torch_compile", True)),
+ target,
+ )
+ )
+ )
and not fp16_unet_offloaded(
target, pipe, offload_active = plan.offload_policy != OFFLOAD_NONE
)
@@ -10247,6 +10316,8 @@ class DiffusionBackend:
tc.update(tc_before)
raise
+ # Outside load_pipeline (3rd generation): its own probe scope, so one OOM is not re-probed by plan and apply.
+ @rocm_probe_load_scoped
def _engage_deferred_profile(self, state: _LoadState) -> None:
"""Engage the deferred `default` speed profile at the start of the 3rd generation this
session. The load left the pipe fully eager (bit-identical reference); by the 3rd image
@@ -10287,7 +10358,14 @@ class DiffusionBackend:
entry["value"] = attention_engaged or "native"
object.__setattr__(state, "resolved", {**state.resolved, "attention_backend": entry})
gguf_transformer = state.kind == "gguf" and state.transformer_quant is None
- if compile_eligible(target, is_gguf = gguf_transformer, family = state.family):
+ if _compile_cache_wanted(
+ state.pipe,
+ target,
+ state.family,
+ is_gguf = gguf_transformer,
+ speed_mode = SPEED_DEFAULT,
+ offload_active = state.offload_policy != OFFLOAD_NONE,
+ ):
compile_ctx = compile_cache.begin(
family = state.family.name,
# U-Net families (SDXL) carry the denoiser as pipe.unet.
diff --git a/studio/backend/core/inference/diffusion_autoopt/apply.py b/studio/backend/core/inference/diffusion_autoopt/apply.py
index b5e9535833e..13a53ebb0fb 100644
--- a/studio/backend/core/inference/diffusion_autoopt/apply.py
+++ b/studio/backend/core/inference/diffusion_autoopt/apply.py
@@ -316,9 +316,18 @@ def run_speed_layer(
if hook.pre is not None:
hook.pre(view, family)
family_for_apply = family
+ if (
+ plan is not None
+ and plan.compile_declined_by_model
+ and not is_gguf
+ and getattr(target, "backend", None) == "rocm"
+ ):
+ # The primitive probes the card before it looks at the model: pass the decline so a model with nothing to
+ # compile never pays the ROCm probe.
+ family_for_apply = _family_with(family, supports_torch_compile = False)
if plan is not None and plan.generic:
try:
- family_for_apply = _generic_family_view(view, family, plan, logger)
+ family_for_apply = _generic_family_view(view, family_for_apply, plan, logger)
except Exception as exc: # noqa: BLE001 - the generic trial is optional; the stock path still runs
if logger is not None:
logger.warning("diffusion.autoopt: generic path unavailable: %s", exc)
diff --git a/studio/backend/core/inference/diffusion_autoopt/plan.py b/studio/backend/core/inference/diffusion_autoopt/plan.py
index 012728a6049..e8b839f485a 100644
--- a/studio/backend/core/inference/diffusion_autoopt/plan.py
+++ b/studio/backend/core/inference/diffusion_autoopt/plan.py
@@ -68,6 +68,8 @@ class OptimizationPlan:
steps: list[PlannedStep] = field(default_factory = list)
profiles: list[ModelProfile] = field(default_factory = list)
generic: bool = False
+ # The compile was declined for the model itself (no candidate, every profile declines), not for the device.
+ compile_declined_by_model: bool = False
def step(self, name: str) -> Optional[PlannedStep]:
for s in self.steps:
@@ -102,8 +104,17 @@ def discovery_enabled() -> bool:
return (os.environ.get(DISCOVER_ENV) or "").strip().lower() not in ("0", "off", "false", "no")
-def compile_ineligible_reason(target: Any, family: Any) -> Optional[str]:
- """Why ``compile_eligible`` refuses, mirroring its checks in order; None when eligible."""
+ROCM_PROBE_FAILED = "ROCm regional compile probe failed on this install"
+
+
+def compile_ineligible_reason(
+ target: Any,
+ family: Any,
+ *,
+ probe: bool = True,
+) -> Optional[str]:
+ """Why ``compile_eligible`` refuses, mirroring its checks in order; None when eligible.
+ ``probe=False`` reads only a recorded ROCm probe result and never runs the allocating probe."""
from .. import diffusion_speed as ds # noqa: PLC0415
if not ds.torch_compile_runtime_available():
@@ -112,13 +123,14 @@ def compile_ineligible_reason(target: Any, family: Any) -> Optional[str]:
return "device does not support the default torch.compile profile"
if not bool(getattr(family, "supports_torch_compile", True)):
return "family opts out of torch.compile"
+ rocm_failed = ROCM_PROBE_FAILED
dtype = getattr(target, "dtype", None)
if ds._is_bfloat16(dtype):
- return None
+ return None if ds.rocm_compile_probe_passes(target, probe = probe) else rocm_failed
if bool(getattr(family, "fp16_incompatible", False)):
return "fp16-incompatible family stays eager outside bf16"
if ds._is_float16(dtype) and ds._fp16_compile_capable(target):
- return None
+ return None if ds.rocm_compile_probe_passes(target, probe = probe) else rocm_failed
return f"dtype {str(dtype).replace('torch.', '')} is not compiled on this device"
@@ -167,6 +179,7 @@ def build_plan(
on_cuda = getattr(target, "device", None) == "cuda"
compile_reason: Optional[str] = None
+ probe_pending = False
if mode == SPEED_EAGER:
compile_reason = "eager tier does not compile"
elif (
@@ -174,10 +187,13 @@ def build_plan(
and is_gguf
and on_cuda
and getattr(family, "supports_torch_compile", True)
+ and not ds.rocm_compile_switched_off(target)
):
compile_reason = "GGUF compiles the dequant op chain instead"
else:
- compile_reason = compile_ineligible_reason(target, family)
+ # The ROCm probe compiles a block on the card: run it only once the model itself can be compiled.
+ compile_reason = compile_ineligible_reason(target, family, probe = False)
+ probe_pending = compile_reason is None
if compile_reason is None and ds.fp16_unet_offloaded(
target, pipe, offload_active = offload_active
):
@@ -194,6 +210,8 @@ def build_plan(
]
if declines and all(declines):
compile_reason = declines[0]
+ if probe_pending and compile_reason is not None:
+ plan.compile_declined_by_model = True
add("compiled", compile_reason is None, compile_reason)
if generic and compile_reason is None:
# No measured data: a discovered block whose forward host-syncs would graph-break on every call.
@@ -205,10 +223,22 @@ def build_plan(
step = plan.step("compiled")
step.attempt = False
step.reason = f"{GRAPH_BREAK_DECLINE} ({p.class_name}: {suspects})"
+ step = plan.step("compiled")
+ if (
+ probe_pending
+ and step is not None
+ and step.attempt
+ and not ds.rocm_compile_probe_passes(target)
+ ):
+ step.attempt = False
+ step.reason = ROCM_PROBE_FAILED
+ rocm_off = ds.rocm_compile_switched_off(target)
add(
"compiled_dequant",
- bool(is_gguf and on_cuda),
- None if is_gguf else "not a GGUF transformer",
+ bool(is_gguf and on_cuda and not rocm_off),
+ "not a GGUF transformer"
+ if not is_gguf
+ else ("UNSLOTH_DIFFUSION_ROCM_COMPILE=0" if rocm_off else None),
)
add("tf32", mode == SPEED_MAX and on_cuda, None if mode == SPEED_MAX else "max tier only")
if not getattr(family, "cudnn_benchmark", True):
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.py
index b18208644ad..6da53836cc5 100644
--- a/studio/backend/core/inference/diffusion_device.py
+++ b/studio/backend/core/inference/diffusion_device.py
@@ -529,7 +529,11 @@ def diffusion_device_target_from_torch_device(
if device == "cuda":
try:
import torch
- is_rocm = bool(getattr(getattr(torch, "version", None), "hip", None))
+
+ from .rocm_bf16 import is_rocm_torch
+
+ # Tag-aware: AMD SDK / Radeon wheels leave torch.version.hip unset.
+ is_rocm = is_rocm_torch(torch)
except Exception:
is_rocm = False
return DiffusionDeviceTarget(
@@ -538,7 +542,7 @@ def diffusion_device_target_from_torch_device(
backend = "rocm" if is_rocm else "cuda",
vendor = "amd" if is_rocm else "nvidia",
supports_model_cpu_offload = True,
- supports_default_torch_compile = not is_rocm,
+ supports_default_torch_compile = (not is_rocm) or _rocm_compile_allowed(),
supports_pinned_transfer = True,
# An overriding caller's "cuda:1" is a device choice to keep, not one to drop back to ordinal 0.
ordinal = int(index) if index.isdigit() else None,
@@ -573,6 +577,15 @@ def float64_device(device: Any) -> Any:
return device if target.supports_float64 else "cpu"
+def _rocm_compile_allowed() -> bool:
+ """Kill switch only; the install probe runs at compile time (``diffusion_rocm_compile``)."""
+ try:
+ from .diffusion_rocm_compile import rocm_compile_allowed
+ return rocm_compile_allowed()
+ except Exception: # noqa: BLE001
+ return False
+
+
def _cuda_or_rocm_target(
torch: Any,
*,
@@ -607,7 +620,7 @@ def _cuda_or_rocm_target(
backend = "rocm" if is_rocm else "cuda",
vendor = "amd" if is_rocm else "nvidia",
supports_model_cpu_offload = True,
- supports_default_torch_compile = not is_rocm,
+ supports_default_torch_compile = (not is_rocm) or _rocm_compile_allowed(),
supports_pinned_transfer = True,
ordinal = ordinal,
)
diff --git a/studio/backend/core/inference/diffusion_rocm_compile.py b/studio/backend/core/inference/diffusion_rocm_compile.py
new file mode 100644
index 00000000000..4f7e2430c50
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_rocm_compile.py
@@ -0,0 +1,242 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Per-process probe: can this ROCm install regionally compile a transformer block?
+
+ROCm was refused because inductor's pooled buffers (``memory_planning``) are views AOTAutograd rejects; Studio now
+compiles with it off and no cudagraphs on ROCm. The probe compiles one small block as the default tier does and checks
+it against eager; any failure stays eager.
+
+``UNSLOTH_DIFFUSION_ROCM_COMPILE``: ``0`` never compiles on ROCm, ``1`` skips the probe, unset / ``auto`` probes.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import contextvars
+import functools
+import logging
+import os
+import threading
+import time
+from typing import Any, Callable, Iterator, Optional
+
+
+def _module_logger() -> Any:
+ # Studio's structured logger; stdlib logging alone is not routed at info level.
+ try:
+ from loggers import get_logger
+ return get_logger(__name__)
+ except Exception: # noqa: BLE001
+ return logging.getLogger(__name__)
+
+
+logger = _module_logger()
+
+ENV = "UNSLOTH_DIFFUSION_ROCM_COMPILE"
+
+_lock = threading.Lock()
+# Process-wide answers. Never an OOM: that says the card was busy, not that the compiler is broken.
+_result: dict[Any, tuple[bool, str]] = {}
+# The running load's own OOM answers, so one load never re-probes a busy card; another load starts empty.
+_load_oom: contextvars.ContextVar[Optional[dict[Any, tuple[bool, str]]]] = contextvars.ContextVar(
+ "rocm_probe_load_oom", default = None
+)
+
+
+def load_scope() -> Iterator[None]:
+ """One load: an OOM probe inside it is answered from memo for the rest of the load and nowhere else."""
+ token = _load_oom.set({})
+ try:
+ yield
+ finally:
+ _load_oom.reset(token)
+
+
+def load_scoped(fn: Callable[..., Any]) -> Callable[..., Any]:
+ @functools.wraps(fn)
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
+ with load_scope():
+ return fn(*args, **kwargs)
+
+ return wrapper
+
+
+def rocm_compile_setting() -> str:
+ raw = (os.environ.get(ENV) or "auto").strip().lower()
+ if raw in ("0", "off", "false", "no"):
+ return "off"
+ if raw in ("1", "on", "true", "yes", "force"):
+ return "on"
+ return "auto"
+
+
+def rocm_compile_allowed() -> bool:
+ return rocm_compile_setting() != "off"
+
+
+def rocm_block_compile_works(device: Any = None) -> bool:
+ return rocm_block_compile_probe(device)[0]
+
+
+def rocm_block_compile_known(device: Any = None) -> Optional[bool]:
+ """The probe's answer without running it (None = unprobed): preflight and status paths must not allocate."""
+ setting = rocm_compile_setting()
+ if setting != "auto":
+ return setting == "on"
+ # Lock-free: _lock is held for the whole multi-second probe, and a status read must not wait on it.
+ hit = _result.get(_index(device))
+ return None if hit is None else hit[0]
+
+
+def _index(device: Any) -> Any:
+ # A bare ordinal: torch.device(int) is not accepted by every supported torch.
+ if isinstance(device, int) and not isinstance(device, bool):
+ return device
+ try:
+ import torch
+ index = torch.cuda.current_device() if device is None else torch.device(device).index
+ return torch.cuda.current_device() if index is None else index
+ except Exception: # noqa: BLE001
+ return 0
+
+
+def rocm_block_compile_probe(device: Any = None) -> tuple[bool, str]:
+ setting = rocm_compile_setting()
+ if setting == "off":
+ return False, f"{ENV}=0"
+ if setting == "on":
+ return True, f"{ENV}=1 (probe skipped)"
+ try:
+ import torch
+ except Exception as exc: # noqa: BLE001
+ return False, f"torch unavailable: {exc}"
+ from .rocm_bf16 import is_rocm_torch
+
+ # Not torch.version.hip alone: AMD SDK / Radeon wheels leave it unset and carry the tag in __version__.
+ if not is_rocm_torch(torch):
+ return True, "not ROCm"
+ index = _index(device)
+ memo = _load_oom.get()
+ # Lock-free when this card already has an answer: only a load that must probe waits behind a running probe.
+ if memo is not None and index in memo:
+ return memo[index]
+ hit = _result.get(index)
+ if hit is not None:
+ return hit
+ with _lock:
+ # The load's own OOM first: a concurrent load's later success must not send this load into a compile.
+ if memo is not None and index in memo:
+ return memo[index]
+ if index in _result:
+ return _result[index]
+ ok, why = _probe(torch, index)
+ (logger.info if ok else logger.warning)(
+ f"diffusion.speed: ROCm regional compile probe on cuda:{index}: "
+ f"{'ok' if ok else 'failed, staying eager'} ({why})"
+ )
+ if not ok and "OutOfMemoryError" in why:
+ if memo is not None:
+ memo[index] = (ok, why)
+ else:
+ _result[index] = (ok, why)
+ return ok, why
+
+
+def _probe(
+ torch: Any,
+ index: int,
+ device: Any = None,
+) -> tuple[bool, str]:
+ start = time.perf_counter()
+ try:
+ import torch.nn.functional as F
+ from torch import nn
+
+ class _Block(nn.Module):
+ def __init__(
+ self,
+ dim: int = 128,
+ heads: int = 4,
+ ) -> None:
+ super().__init__()
+ self.heads = heads
+ self.norm1 = nn.LayerNorm(dim, elementwise_affine = False)
+ self.qkv = nn.Linear(dim, 3 * dim)
+ self.out = nn.Linear(dim, dim)
+ self.norm2 = nn.LayerNorm(dim)
+ self.mlp = nn.Sequential(
+ nn.Linear(dim, 4 * dim), nn.GELU(approximate = "tanh"), nn.Linear(4 * dim, dim)
+ )
+
+ def forward(self, x: Any, scale: Any) -> Any:
+ b, n, d = x.shape
+ h = self.norm1(x) * (1 + scale)
+ q, k, v = (
+ self.qkv(h).view(b, n, 3, self.heads, d // self.heads).permute(2, 0, 3, 1, 4)
+ )
+ a = F.scaled_dot_product_attention(q, k, v).transpose(1, 2).reshape(b, n, d)
+ x = x + self.out(a)
+ return x + self.mlp(self.norm2(x))
+
+ dev = torch.device("cuda", index) if device is None else device
+ dtype = torch.bfloat16
+ gen = torch.Generator(device = "cpu").manual_seed(0)
+ # Built on meta and filled from the private generator: the shared CPU RNG is never read or rewound, so a
+ # concurrent load or generation drawing from it is untouched.
+ with torch.device("meta"):
+ block = _Block()
+ block = block.to_empty(device = dev)
+ with torch.no_grad():
+ for name, param in block.named_parameters():
+ if param.dim() == 2:
+ bound = param.shape[1] ** -0.5
+ value = (torch.rand(param.shape, generator = gen) * 2 - 1) * bound
+ elif name.endswith("norm2.weight"):
+ value = torch.ones(param.shape)
+ else:
+ value = (torch.rand(param.shape, generator = gen) * 2 - 1) * 0.02
+ param.copy_(value)
+ block = block.to(dtype = dtype).eval()
+ inductor_cfg = torch._inductor.config
+ patches = {"memory_planning": False} if hasattr(inductor_cfg, "memory_planning") else {}
+ compiled = torch.compile(block, dynamic = True, fullgraph = True)
+ worst = 0.0
+ with (
+ torch.no_grad(),
+ inductor_cfg.patch(patches),
+ torch._dynamo.config.patch(suppress_errors = False),
+ ):
+ for n in (77, 160):
+ x = torch.randn(1, n, 128, generator = gen).to(device = dev, dtype = dtype)
+ s = torch.randn(1, 1, 128, generator = gen).to(device = dev, dtype = dtype) * 0.1
+ ref = block(x, s).float()
+ got = compiled(x, s).float()
+ if not bool(torch.isfinite(got).all()):
+ return False, "non-finite compiled output"
+ worst = max(worst, float((got - ref).abs().max() / ref.abs().max().clamp_min(1e-6)))
+ torch.cuda.synchronize(dev)
+ if worst > 0.05:
+ return False, f"compiled block disagrees with eager (rel max err {worst:.3g})"
+ return True, f"rel max err {worst:.3g}, {time.perf_counter() - start:.1f}s"
+ except Exception as exc: # noqa: BLE001
+ name = type(exc).__name__
+ try:
+ from .diffusion_batched import is_oom_error
+
+ # Classified before flattening: a compiler error wrapping an OOM, or a bare "HIP out of memory".
+ if is_oom_error(exc) and name != "OutOfMemoryError":
+ name = f"OutOfMemoryError ({name})"
+ except Exception: # noqa: BLE001, S110
+ pass
+ return False, f"{name}: {str(exc).splitlines()[0][:300] if str(exc) else ''}"
+
+
+def _reset_for_tests() -> None:
+ with _lock:
+ _result.clear()
+
+
+def probe_result(index: int = 0) -> Optional[tuple[bool, str]]:
+ return _result.get(index)
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 589ac667182..bafe9d2df6f 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -40,7 +40,7 @@ import sys
import threading
from contextlib import contextmanager
from functools import lru_cache
-from typing import Any, Iterator, Optional
+from typing import Any, Callable, Iterator, Optional
from . import diffusion_compile_config as compile_config
from .diffusion_bg_compile import eager_forced as _bg_eager_forced
@@ -267,8 +267,19 @@ def restore_backend_flags(state: Optional[dict]) -> None:
_set_knob(dynamo_cfg, _DYNAMO_MODULE, attr, key)
-def _gguf_dequant_compilable(is_gguf: bool, on_cuda: bool, family_allows_compile: bool) -> bool:
- return is_gguf and on_cuda and family_allows_compile and torch_compile_runtime_available()
+def _gguf_dequant_compilable(
+ is_gguf: bool,
+ on_cuda: bool,
+ family_allows_compile: bool,
+ target: Any = None,
+) -> bool:
+ return (
+ is_gguf
+ and on_cuda
+ and family_allows_compile
+ and not rocm_compile_switched_off(target)
+ and torch_compile_runtime_available()
+ )
def _pin_compile_defaults() -> None:
@@ -362,9 +373,17 @@ def torch_compile_runtime_available() -> bool:
return True
-def compile_eligible(target: Any, *, is_gguf: bool, family: Any) -> bool:
+def compile_eligible(
+ target: Any,
+ *,
+ is_gguf: bool,
+ family: Any,
+ probe: bool = True,
+) -> bool:
"""Whether the denoiser's repeated block should be regionally compiled: CUDA (incl. ROCm) bf16, or fp16 on NVIDIA
- sm_75+, on a compile-friendly family with inductor available. ``is_gguf`` is kept for compat only."""
+ sm_75+, on a compile-friendly family with inductor available. ``is_gguf`` is kept for compat only.
+ ``probe=False`` never runs the allocating ROCm probe (preflight / status, before eviction): an unprobed install
+ counts as eligible, a recorded failure does not."""
del is_gguf
if not torch_compile_runtime_available():
return False
@@ -374,12 +393,45 @@ def compile_eligible(target: Any, *, is_gguf: bool, family: Any) -> bool:
return False
dtype = getattr(target, "dtype", None)
if _is_bfloat16(dtype):
- return True
+ return rocm_compile_probe_passes(target, probe = probe)
# fp16-incompatible families run in fp32 (unmeasured under compile) or in fp16 behind diffusion_fp16_guard, which
# must stay eager: the int8 fused MLP installed for compile replaces the guard's FFN forward.
if bool(getattr(family, "fp16_incompatible", False)):
return False
- return _is_float16(dtype) and _fp16_compile_capable(target)
+ return (
+ _is_float16(dtype)
+ and _fp16_compile_capable(target)
+ and rocm_compile_probe_passes(target, probe = probe)
+ )
+
+
+def rocm_compile_probe_passes(target: Any, *, probe: bool = True) -> bool:
+ if getattr(target, "backend", None) != "rocm":
+ return True
+ try:
+ from . import diffusion_rocm_compile as rc
+ if not probe:
+ return rc.rocm_block_compile_known(getattr(target, "ordinal", None)) is not False
+ return rc.rocm_block_compile_works(getattr(target, "ordinal", None))
+ except Exception: # noqa: BLE001
+ return False
+
+
+def rocm_probe_load_scoped(fn: Callable[..., Any]) -> Callable[..., Any]:
+ """Wrap a load so a ROCm probe OOM is remembered for that load only (no re-probe within it, a retry in the next)."""
+ from .diffusion_rocm_compile import load_scoped
+ return load_scoped(fn)
+
+
+def rocm_compile_switched_off(target: Any) -> bool:
+ """``UNSLOTH_DIFFUSION_ROCM_COMPILE=0`` on a ROCm target: no compile at all, the GGUF dequant one included."""
+ if getattr(target, "backend", None) != "rocm":
+ return False
+ try:
+ from .diffusion_rocm_compile import rocm_compile_allowed
+ return not rocm_compile_allowed()
+ except Exception: # noqa: BLE001
+ return True
def fp16_compile_explicit_only(target: Any) -> bool:
@@ -551,7 +603,7 @@ def apply_speed_optims(
# of the repeated block. eager = no compile.
if mode == SPEED_DEFAULT:
# Asked directly: this arm never reaches compile_eligible(), unlike the dense arm below.
- if _gguf_dequant_compilable(is_gguf, on_cuda, family_allows_compile):
+ if _gguf_dequant_compilable(is_gguf, on_cuda, family_allows_compile, target):
applied["compiled_dequant"] = gguf_compile.install_compiled_dequant(logger)
elif compile_eligible(target, is_gguf = is_gguf, family = family) and not fp16_unet_offloaded(
target, pipe, offload_active = offload_active
@@ -585,7 +637,7 @@ def apply_speed_optims(
filter_reductions = filter_reductions,
)
if not applied["compiled"] and _gguf_dequant_compilable(
- is_gguf, on_cuda, family_allows_compile
+ is_gguf, on_cuda, family_allows_compile, target
):
# No block compile (ROCm, or its setup failed): keep default's dequant compile rather than run slower.
applied["compiled_dequant"] = gguf_compile.install_compiled_dequant(logger)
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 65771830a39..1a9b8814b6e 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -1029,7 +1029,7 @@ def dense_quant_host_capable(target: Any) -> bool:
# halves of that decision belong to the load, not to a row.
from .diffusion_speed import compile_eligible
- if not compile_eligible(target, is_gguf = False, family = None):
+ if not compile_eligible(target, is_gguf = False, family = None, probe = False):
return False
# Not redundant with the arch floor: `_scheme_supported` rejects every scheme when the torchao
# import fails, so an ABI skew would advertise a fast path every load then falls back from.
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 9394023be2d..52261593da7 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -145,6 +145,7 @@ from .diffusion_speed import (
SPEED_OFF,
apply_speed_optims,
arm_graphs_after_placement,
+ rocm_probe_load_scoped,
resolve_speed_mode,
restore_backend_flags,
settle_compile_fallback,
@@ -6457,6 +6458,7 @@ class VideoBackend:
# ── the load itself ──────────────────────────────────────────────────────
@_invalidates_gpu_memory("video load")
+ @rocm_probe_load_scoped
@scoped_local_files_only
def load_pipeline(
self,
diff --git a/studio/backend/tests/test_diffusion_device.py b/studio/backend/tests/test_diffusion_device.py
index d10450a615d..fb050b26b7a 100644
--- a/studio/backend/tests/test_diffusion_device.py
+++ b/studio/backend/tests/test_diffusion_device.py
@@ -196,7 +196,15 @@ def test_rocm_target(monkeypatch):
t = dd.resolve_diffusion_device_target()
assert (t.device, t.backend, t.vendor) == ("cuda", "rocm", "amd")
assert t.dtype == BF16
- assert t.supports_default_torch_compile is False # ROCm disables default compile
+ assert t.supports_default_torch_compile is True
+
+
+def test_rocm_target_compile_kill_switch(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_ROCM_COMPILE", "0")
+ torch = _make_torch(cuda_available = True, bf16_supported = True)
+ _install(monkeypatch, torch, studio_device = "cuda", is_rocm = True)
+ t = dd.resolve_diffusion_device_target()
+ assert t.backend == "rocm" and t.supports_default_torch_compile is False
def test_rocm_without_bf16_uses_fp16(monkeypatch):
@@ -313,6 +321,18 @@ def test_from_torch_device_cuda(monkeypatch):
assert t.is_cuda_torch_device
+def test_from_torch_device_tag_only_rocm(monkeypatch):
+ # AMD SDK / Radeon wheels: no torch.version.hip, ROCm only in __version__. Must stay ROCm so the
+ # probe and UNSLOTH_DIFFUSION_ROCM_COMPILE apply.
+ torch = _make_torch()
+ torch.__version__ = "2.9.0+rocm7.1.0"
+ monkeypatch.setitem(sys.modules, "torch", torch)
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_ROCM_COMPILE", "0")
+ t = dd.diffusion_device_target_from_torch_device("cuda:1", BF16)
+ assert (t.backend, t.vendor, t.ordinal) == ("rocm", "amd", 1)
+ assert t.supports_default_torch_compile is False
+
+
def test_from_torch_device_mps_and_cpu(monkeypatch):
torch = _make_torch()
monkeypatch.setitem(sys.modules, "torch", torch)
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index 6f6e75a5fe8..3d220e4626b 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -2570,3 +2570,564 @@ def test_int8_gemm_install_follows_the_denoiser_placement(
denoiser_offloaded = denoiser_offloaded,
)
assert seen == [expected]
+
+
+def test_compile_eligible_on_rocm_follows_the_probe(monkeypatch):
+ from core.inference import diffusion_rocm_compile as rc
+
+ monkeypatch.delenv("TORCHDYNAMO_DISABLE", raising = False)
+ ds_mod.torch_compile_runtime_available.cache_clear()
+ rocm = types.SimpleNamespace(
+ device = "cuda",
+ dtype = "bfloat16",
+ backend = "rocm",
+ ordinal = None,
+ supports_default_torch_compile = True,
+ )
+ monkeypatch.setattr(ds_mod, "_is_bfloat16", lambda d: d == "bfloat16")
+ monkeypatch.setattr(rc, "rocm_block_compile_works", lambda device = None: False)
+ assert ds_mod.compile_eligible(rocm, is_gguf = False, family = _family()) is False
+ monkeypatch.setattr(rc, "rocm_block_compile_works", lambda device = None: True)
+ assert ds_mod.compile_eligible(rocm, is_gguf = False, family = _family()) is True
+ monkeypatch.setattr(rc, "rocm_block_compile_works", lambda device = None: 1 / 0)
+ cuda = types.SimpleNamespace(
+ device = "cuda", dtype = "bfloat16", backend = "cuda", supports_default_torch_compile = True
+ )
+ assert ds_mod.compile_eligible(cuda, is_gguf = False, family = _family()) is True
+
+
+def test_rocm_compile_probe_settings(monkeypatch):
+ from core.inference import diffusion_rocm_compile as rc
+
+ rc._reset_for_tests()
+ monkeypatch.setenv(rc.ENV, "0")
+ assert rc.rocm_block_compile_probe() == (False, f"{rc.ENV}=0") and not rc.rocm_compile_allowed()
+ monkeypatch.setenv(rc.ENV, "1")
+ assert rc.rocm_block_compile_probe()[0] is True and rc.rocm_compile_allowed()
+
+
+def _rocm_bf16(dtype = "bfloat16"):
+ return types.SimpleNamespace(
+ device = "cuda",
+ dtype = dtype,
+ backend = "rocm",
+ ordinal = None,
+ supports_default_torch_compile = True,
+ )
+
+
+def _probe_must_not_run(monkeypatch):
+ from core.inference import diffusion_rocm_compile as rc
+
+ monkeypatch.delenv("TORCHDYNAMO_DISABLE", raising = False)
+ monkeypatch.delenv(rc.ENV, raising = False)
+ ds_mod.torch_compile_runtime_available.cache_clear()
+ monkeypatch.setattr(ds_mod, "_is_bfloat16", lambda d: d == "bfloat16")
+ monkeypatch.setattr(ds_mod, "_is_float16", lambda d: d == "float16")
+ calls = []
+ monkeypatch.setattr(
+ rc, "rocm_block_compile_works", lambda device = None: calls.append(device) or True
+ )
+ rc._reset_for_tests()
+ return rc, calls
+
+
+def test_rocm_preflight_never_runs_the_allocating_probe(monkeypatch):
+ # The route preflight and the polled status route run before eviction: they read the probe, never run it.
+ from core.inference import diffusion as dmod
+ from core.inference import diffusion_transformer_quant as tq
+
+ rc, calls = _probe_must_not_run(monkeypatch)
+ assert (
+ dmod._pipeline_quant_uncompilable_reason(
+ _rocm_bf16(), _family(), None, model_kind = "pipeline"
+ )
+ is None
+ )
+ assert ds_mod.compile_eligible(_rocm_bf16(), is_gguf = False, family = None, probe = False) is True
+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(tq, "torchao_unavailable_reason", lambda: "stop here")
+ assert tq.dense_quant_host_capable(_rocm_bf16()) is False
+ assert calls == []
+ # A recorded failure is honoured without re-probing; the loader (probe=True) runs it.
+ rc._result[rc._index(None)] = (False, "boom")
+ assert ds_mod.compile_eligible(_rocm_bf16(), is_gguf = False, family = None, probe = False) is False
+ rc._reset_for_tests()
+ assert (
+ dmod._pipeline_quant_uncompilable_reason(
+ _rocm_bf16(), _family(), None, model_kind = "pipeline", probe = True
+ )
+ is None
+ )
+ assert len(calls) == 1
+
+
+def test_rocm_probe_oom_is_held_for_the_load_and_retried_at_the_next(monkeypatch):
+ from core.inference import diffusion_rocm_compile as rc
+
+ monkeypatch.delenv(rc.ENV, raising = False)
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(torch.version, "hip", "6.4", raising = False)
+ monkeypatch.setattr(rc, "_index", lambda device: 0)
+ calls = []
+ answers = iter([(False, "OutOfMemoryError: HIP out of memory"), (True, "ok")])
+ monkeypatch.setattr(rc, "_probe", lambda torch, index: calls.append(1) or next(answers))
+ rc._reset_for_tests()
+ with rc.load_scope():
+ assert rc.rocm_block_compile_probe()[0] is False
+ # Same load: no second compile/OOM, and a non-probing read still sees "unprobed".
+ assert rc.rocm_block_compile_probe()[0] is False
+ assert rc.rocm_block_compile_known() is None
+ assert len(calls) == 1
+ with rc.load_scope():
+ assert rc.rocm_block_compile_probe()[0] is True
+ assert rc.rocm_block_compile_known() is True and len(calls) == 2
+ rc._reset_for_tests()
+
+
+def test_rocm_probe_oom_outside_a_load_is_never_cached(monkeypatch):
+ from core.inference import diffusion_rocm_compile as rc
+
+ monkeypatch.delenv(rc.ENV, raising = False)
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(torch.version, "hip", "6.4", raising = False)
+ monkeypatch.setattr(rc, "_index", lambda device: 0)
+ calls = []
+ monkeypatch.setattr(
+ rc, "_probe", lambda torch, index: calls.append(1) or (False, "OutOfMemoryError: x")
+ )
+ rc._reset_for_tests()
+ rc.rocm_block_compile_probe()
+ rc.rocm_block_compile_probe()
+ assert len(calls) == 2 and rc.probe_result(0) is None
+ rc._reset_for_tests()
+
+
+def test_rocm_known_read_does_not_wait_for_a_running_probe(monkeypatch):
+ import threading
+
+ from core.inference import diffusion_rocm_compile as rc
+
+ monkeypatch.delenv(rc.ENV, raising = False)
+ monkeypatch.setattr(rc, "_index", lambda device: 0)
+ rc._reset_for_tests()
+ out = []
+ with rc._lock: # a probe in progress holds this for seconds
+ t = threading.Thread(target = lambda: out.append(rc.rocm_block_compile_known()), daemon = True)
+ t.start()
+ t.join(2)
+ assert not t.is_alive() and out == [None]
+
+
+def test_rocm_fp16_never_pays_the_probe(monkeypatch):
+ from core.inference.diffusion_autoopt.plan import compile_ineligible_reason
+
+ _rc, calls = _probe_must_not_run(monkeypatch)
+ assert ds_mod.compile_eligible(_rocm_bf16("float16"), is_gguf = False, family = _family()) is False
+ assert compile_ineligible_reason(_rocm_bf16("float16"), _family()).startswith("dtype")
+ assert calls == []
+
+
+def test_rocm_kill_switch_also_stops_the_gguf_dequant_compile(monkeypatch):
+ from core.inference import diffusion_rocm_compile as rc
+
+ monkeypatch.delenv("TORCHDYNAMO_DISABLE", raising = False)
+ ds_mod.torch_compile_runtime_available.cache_clear()
+ cuda = types.SimpleNamespace(backend = "cuda")
+ monkeypatch.setenv(rc.ENV, "0")
+ assert ds_mod._gguf_dequant_compilable(True, True, True, _rocm_bf16()) is False
+ assert ds_mod._gguf_dequant_compilable(True, True, True, cuda) is True
+ monkeypatch.setenv(rc.ENV, "auto")
+ assert ds_mod._gguf_dequant_compilable(True, True, True, _rocm_bf16()) is True
+
+
+def test_loader_probes_rocm_only_for_an_engaged_pipeline_quant():
+ # speed=off with quant off must not pay the allocating probe: the loader's probe=True call sits inside the
+ # engaged-quant branch.
+ import ast
+ import inspect
+
+ from core.inference import diffusion as dmod
+
+ tree = ast.parse(inspect.getsource(dmod))
+ parents = {c: p for p in ast.walk(tree) for c in ast.iter_child_nodes(p)}
+ calls = [
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.Call)
+ and getattr(n.func, "id", None) == "_pipeline_quant_uncompilable_reason"
+ and any(k.arg == "probe" and getattr(k.value, "value", None) is True for k in n.keywords)
+ ]
+ assert len(calls) == 1
+ node, guards = calls[0], []
+ while node in parents:
+ child, node = node, parents[node]
+ if isinstance(node, ast.If) and child in node.body:
+ guards.append(ast.unparse(node.test))
+ assert any("normalize_transformer_quant(transformer_quant) is not None" in g for g in guards)
+
+
+def _diffusion_calls(name):
+ import ast
+ import inspect
+
+ from core.inference import diffusion as dmod
+
+ tree = ast.parse(inspect.getsource(dmod))
+ parents = {c: p for p in ast.walk(tree) for c in ast.iter_child_nodes(p)}
+ calls = [
+ n for n in ast.walk(tree) if isinstance(n, ast.Call) and getattr(n.func, "id", None) == name
+ ]
+ return calls, parents
+
+
+def test_deferred_speed_profile_arms_without_running_the_rocm_probe():
+ import ast
+
+ calls, parents = _diffusion_calls("compile_eligible")
+ deferred = []
+ for c in calls:
+ node = c
+ while node in parents and not isinstance(node, ast.Assign):
+ node = parents[node]
+ if isinstance(node, ast.Assign) and any(
+ getattr(t, "id", None) == "speed_deferred" for t in node.targets
+ ):
+ deferred.append(c)
+ assert deferred and all(
+ any(k.arg == "probe" and getattr(k.value, "value", None) is False for k in c.keywords)
+ for c in deferred
+ )
+
+
+def test_native_quant_never_runs_the_rocm_probe():
+ import ast
+
+ calls, parents = _diffusion_calls("_pipeline_quant_uncompilable_reason")
+ probing = [
+ c
+ for c in calls
+ if any(k.arg == "probe" and getattr(k.value, "value", None) is True for k in c.keywords)
+ ]
+ assert len(probing) == 1
+ node, guarded = probing[0], False
+ while node in parents:
+ child, node = node, parents[node]
+ if (
+ isinstance(node, ast.IfExp)
+ and child is node.orelse
+ and ast.unparse(node.test) == "native_scheme is not None"
+ ):
+ guarded = True
+ assert guarded
+
+
+def test_rocm_kill_switch_declines_the_planned_dequant_step(monkeypatch):
+ import torch
+
+ from core.inference import diffusion_rocm_compile as rc
+ from core.inference.diffusion_autoopt import plan as plan_mod
+
+ pipe = types.SimpleNamespace(transformer = None)
+ fam = types.SimpleNamespace(name = "z-image", supports_torch_compile = True)
+ rocm = types.SimpleNamespace(
+ device = "cuda", backend = "rocm", dtype = torch.bfloat16, supports_default_torch_compile = True
+ )
+ monkeypatch.setenv(rc.ENV, "0")
+ p = plan_mod.build_plan(
+ pipe, rocm, family = fam, backend = "image", speed_mode = "default", is_gguf = True
+ )
+ assert p.step("compiled_dequant").attempt is False
+ assert p.reason("compiled_dequant") == f"{rc.ENV}=0"
+ monkeypatch.setenv(rc.ENV, "1")
+ p = plan_mod.build_plan(
+ pipe, rocm, family = fam, backend = "image", speed_mode = "default", is_gguf = True
+ )
+ assert p.step("compiled_dequant").attempt is True
+
+
+def test_rocm_gguf_default_compile_cache_gate_uses_the_dequant_predicate():
+ # A regional probe failure recorded by an earlier dense load must not close the GGUF dequant cache context.
+ import ast
+
+ calls, parents = _diffusion_calls("_gguf_dequant_compilable")
+ gated = []
+ for c in calls:
+ node = c
+ while node in parents and not isinstance(node, ast.If):
+ node = parents[node]
+ if isinstance(node, ast.If) and "compile_cache.begin(" in ast.unparse(node.body[0]):
+ gated.append((c, parents[c]))
+ assert len(gated) == 2
+ c, ifexp = gated[0]
+ assert isinstance(ifexp, ast.IfExp) and ifexp.body is c
+ test = ast.unparse(ifexp.test)
+ assert "gguf_transformer" in test and "SPEED_DEFAULT" in test and "'rocm'" in test
+ # max: a refused regional compile falls back to the dequant, which keeps the cache context open.
+ assert isinstance(ifexp.orelse, ast.BoolOp) and isinstance(ifexp.orelse.op, ast.Or)
+ regional, fallback = ifexp.orelse.values
+ assert getattr(regional.func, "id", None) == "_compile_cache_wanted"
+ assert "'rocm'" in ast.unparse(fallback) and gated[1][0] in list(ast.walk(fallback))
+
+
+def test_rocm_probe_runs_on_wheels_without_a_hip_version(monkeypatch):
+ from core.inference import diffusion_rocm_compile as rc
+
+ monkeypatch.delenv(rc.ENV, raising = False)
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.setattr(torch, "__version__", "2.9.0+rocm7.1.0")
+ monkeypatch.setattr(rc, "_index", lambda device: 0)
+ monkeypatch.setattr(rc, "_probe", lambda torch, index: (False, "compiled block disagrees"))
+ rc._reset_for_tests()
+ assert rc.rocm_block_compile_probe() == (False, "compiled block disagrees")
+ rc._reset_for_tests()
+
+
+ "exc",
+ [
+ RuntimeError("HIP out of memory. Tried to allocate 2.00 MiB"),
+ type("OutOfMemoryError_", (RuntimeError,), {})("boom"),
+ ],
+)
+def test_rocm_probe_wrapped_oom_stays_transient(monkeypatch, exc):
+ from core.inference import diffusion_rocm_compile as rc
+
+ monkeypatch.delenv(rc.ENV, raising = False)
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(torch.version, "hip", "6.4", raising = False)
+ monkeypatch.setattr(rc, "_index", lambda device: 0)
+ wrapped = RuntimeError("backend='inductor' raised: compile failed")
+ wrapped.__cause__ = exc
+
+ class _Boom:
+ def __getattr__(self, name):
+ raise wrapped
+
+ monkeypatch.setattr(torch, "device", lambda *a, **k: _Boom().device)
+ rc._reset_for_tests()
+ ok, why = rc.rocm_block_compile_probe()
+ assert ok is False and "OutOfMemoryError" in why
+ assert rc.rocm_block_compile_known() is None
+ assert rc.probe_result(0) is None
+ rc._reset_for_tests()
+
+
+def test_rocm_probe_oom_of_one_load_is_never_reused_by_an_overlapping_load(monkeypatch):
+ # Overlapping image / video loads: an older load's OOM, whenever it lands, belongs to that load alone.
+ import threading
+
+ from core.inference import diffusion_rocm_compile as rc
+
+ monkeypatch.delenv(rc.ENV, raising = False)
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(torch.version, "hip", "6.4", raising = False)
+ monkeypatch.setattr(rc, "_index", lambda device: 0)
+ answers = iter([(False, "OutOfMemoryError: HIP out of memory"), (True, "ok")])
+ monkeypatch.setattr(rc, "_probe", lambda torch, index: next(answers))
+ rc._reset_for_tests()
+ newer_started, older_done = threading.Event(), threading.Event()
+ seen = {}
+
+ def _newer():
+ with rc.load_scope():
+ newer_started.set()
+ older_done.wait(5)
+ seen["newer"] = rc.rocm_block_compile_probe()[0]
+
+ t = threading.Thread(target = _newer)
+ t.start()
+ newer_started.wait(5)
+ with rc.load_scope(): # the older load reaches its probe only after the newer load began
+ seen["older"] = rc.rocm_block_compile_probe()[0]
+ older_done.set()
+ t.join(5)
+ assert seen == {"older": False, "newer": True}
+ rc._reset_for_tests()
+
+
+def test_rocm_probe_index_keeps_an_integer_ordinal(monkeypatch):
+ from core.inference import diffusion_rocm_compile as rc
+
+ torch = pytest.importorskip("torch")
+
+ def _old_device(*args, **kwargs): # torch 2.4: no torch.device(int)
+ if args and isinstance(args[0], int):
+ raise TypeError("device() received an invalid combination of arguments")
+ return types.SimpleNamespace(index = None)
+
+ monkeypatch.setattr(torch, "device", _old_device)
+ assert rc._index(1) == 1
+ assert rc._index(3) == 3
+
+
[email protected]("which", ["image", "video"])
[email protected]("token", [None, 7])
+def test_every_load_pipeline_call_runs_in_its_own_rocm_probe_scope(monkeypatch, which, token):
+ # Direct calls and begin_load workers (token passed) alike: the OOM memo is the load's own.
+ from core.inference import diffusion_rocm_compile as rc
+
+ if which == "image":
+ from core.inference import diffusion as mod
+ backend = mod.DiffusionBackend()
+ else:
+ from core.inference import video as mod
+ backend = mod.VideoBackend()
+ scopes = []
+
+ class _Stop(Exception):
+ pass
+
+ def _stop(*a, **k):
+ scopes.append(rc._load_oom.get())
+ raise _Stop
+
+ monkeypatch.setattr(backend, "validate_load_request", _stop)
+ if which == "image":
+ monkeypatch.setattr(backend, "_raise_if_load_cancelled", lambda token: None)
+ with pytest.raises(_Stop):
+ backend.load_pipeline(repo_id = "x/y", _load_token = token)
+ assert scopes == [{}] and rc._load_oom.get() is None
+
+
+def test_rocm_probe_leaves_the_process_rng_untouched():
+ from core.inference import diffusion_rocm_compile as rc
+
+ torch = pytest.importorskip("torch")
+ torch.manual_seed(1234)
+ before = torch.random.get_rng_state().clone()
+ rc._probe(torch, 0) # builds the block, then fails to reach a GPU on this host
+ assert torch.equal(torch.random.get_rng_state(), before)
+
+
+def test_deferred_profile_engagement_runs_in_its_own_rocm_probe_scope(monkeypatch):
+ # It engages at the 3rd generation, after load_pipeline's scope closed: one OOM must hold for the engagement.
+ from core.inference import diffusion as dmod
+ from core.inference import diffusion_rocm_compile as rc
+
+ backend = dmod.DiffusionBackend()
+ scopes = []
+
+ class _Stop(Exception):
+ pass
+
+ def _stop(state):
+ scopes.append(rc._load_oom.get())
+ raise _Stop
+
+ monkeypatch.setattr(backend, "_state_device_target", _stop)
+ state = types.SimpleNamespace(speed_deferred = True)
+ with pytest.raises(_Stop):
+ backend._engage_deferred_profile(state)
+ assert scopes == [{}] and rc._load_oom.get() is None
+
+
+def test_rocm_load_local_oom_wins_over_a_concurrent_global_success(monkeypatch):
+ from core.inference import diffusion_rocm_compile as rc
+
+ monkeypatch.delenv(rc.ENV, raising = False)
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(torch.version, "hip", "6.4", raising = False)
+ monkeypatch.setattr(rc, "_index", lambda device: 0)
+ monkeypatch.setattr(
+ rc, "_probe", lambda torch, index: (False, "OutOfMemoryError: HIP out of memory")
+ )
+ rc._reset_for_tests()
+ with rc.load_scope():
+ assert rc.rocm_block_compile_probe()[0] is False
+ rc._result[0] = (True, "ok") # another load on the same card probed successfully meanwhile
+ assert rc.rocm_block_compile_probe()[0] is False
+ assert rc.rocm_block_compile_probe()[0] is True
+ rc._reset_for_tests()
+
+
+def test_rocm_probe_waits_for_a_compilable_model(monkeypatch):
+ # No denoiser to compile: neither the plan nor the speed primitive it hands the family to may run the probe.
+ from core.inference.diffusion_autoopt import apply as apply_mod
+ from core.inference.diffusion_autoopt import plan as plan_mod
+
+ _rc, calls = _probe_must_not_run(monkeypatch)
+ pipe = types.SimpleNamespace(transformer = None)
+ fam = types.SimpleNamespace(name = "z-image", supports_torch_compile = True)
+ p = plan_mod.build_plan(pipe, _rocm_bf16(), family = fam, backend = "image", speed_mode = "default")
+ assert p.step("compiled").attempt is False
+ assert p.reason("compiled") == "no denoiser transformer or whole-compile U-Net"
+ assert p.compile_declined_by_model is True
+ seen = []
+
+ def _apply(family, logger):
+ seen.append(ds_mod.compile_eligible(_rocm_bf16(), is_gguf = False, family = family))
+ return {}
+
+ apply_mod.run_speed_layer(
+ pipe, _rocm_bf16(), family = fam, backend = "image", speed_mode = "default", apply = _apply
+ )
+ assert seen == [False] and calls == []
+
+
+def test_rocm_probe_still_runs_for_a_compilable_model(monkeypatch):
+ from core.inference.diffusion_autoopt import plan as plan_mod
+
+ _rc, calls = _probe_must_not_run(monkeypatch)
+ monkeypatch.setattr(plan_mod.targets, "denoiser_dits", lambda pipe: [object()])
+ fam = types.SimpleNamespace(name = "z-image", supports_torch_compile = True)
+ p = plan_mod.build_plan(
+ types.SimpleNamespace(), _rocm_bf16(), family = fam, backend = "image", speed_mode = "default"
+ )
+ assert p.compile_declined_by_model is False
+ assert len(calls) == 1
+
+
+def test_rocm_probe_answer_is_read_without_waiting_for_another_probe(monkeypatch):
+ import threading
+
+ from core.inference import diffusion_rocm_compile as rc
+
+ monkeypatch.delenv(rc.ENV, raising = False)
+ monkeypatch.setattr(rc, "_index", lambda device: 1)
+ monkeypatch.setattr("core.inference.rocm_bf16.is_rocm_torch", lambda torch: True)
+ rc._reset_for_tests()
+ rc._result[1] = (True, "cached")
+ out = []
+ with rc._lock: # a probe on another card holds this for seconds
+ t = threading.Thread(target = lambda: out.append(rc.rocm_block_compile_probe()), daemon = True)
+ t.start()
+ t.join(2)
+ assert not t.is_alive() and out == [(True, "cached")]
+ rc._reset_for_tests()
+
+
+def test_rocm_probe_never_rewinds_the_shared_rng(monkeypatch):
+ # A concurrent consumer of the default CPU generator must not see its draws undone by a probe snapshot/restore.
+ from core.inference import diffusion_rocm_compile as rc
+
+ torch = pytest.importorskip("torch")
+
+ def _no_fork(*args, **kwargs):
+ raise AssertionError("fork_rng snapshots and restores the shared generator")
+
+ monkeypatch.setattr(torch.random, "fork_rng", _no_fork)
+ monkeypatch.setattr(torch, "compile", lambda module, **kwargs: module)
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *args, **kwargs: None)
+ torch.manual_seed(1234)
+ before = torch.random.get_rng_state().clone()
+ ok, why = rc._probe(torch, 0, device = torch.device("cpu"))
+ assert ok, why
+ assert torch.equal(torch.random.get_rng_state(), before)
+
+
+def test_compile_cache_gate_skips_the_rocm_probe_for_a_model_with_nothing_to_compile(monkeypatch):
+ from core.inference import diffusion as dmod
+
+ _rc, calls = _probe_must_not_run(monkeypatch)
+ fam = types.SimpleNamespace(name = "z-image", supports_torch_compile = True)
+ pipe = types.SimpleNamespace(transformer = None)
+ wanted = dmod._compile_cache_wanted(
+ pipe, _rocm_bf16(), fam, is_gguf = False, speed_mode = "default", offload_active = False
+ )
+ assert wanted is False and calls == []
+ cuda = types.SimpleNamespace(**{**vars(_rocm_bf16()), "backend": "cuda"})
+ assert dmod._compile_cache_wanted(
+ pipe, cuda, fam, is_gguf = False, speed_mode = "default", offload_active = False
+ ) is ds_mod.compile_eligible(cuda, is_gguf = False, family = fam)