Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

8.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Per-process probe that a regional block compile matches eager on ROCm; failure stays eager; ROCm
compiles with memory_planning off and no cudagraphs; UNSLOTH_DIFFUSION_ROCM_COMPILE=0 off, 1
skips probe"""

from __future__ import annotations

import contextlib
import contextvars
import functools
import logging
import os
import threading
import time
from typing import Any, Callable, Iterator, Optional


def _module_logger() -> Any:
    # Studio's structured logger; stdlib logging alone is not routed at info level.
    try:
        from loggers import get_logger
        return get_logger(__name__)
    except Exception:  # noqa: BLE001
        return logging.getLogger(__name__)


logger = _module_logger()

ENV = "UNSLOTH_DIFFUSION_ROCM_COMPILE"

_lock = threading.Lock()
# Process-wide answers. Never an OOM: that says the card was busy, not that the compiler is broken.
_result: dict[Any, tuple[bool, str]] = {}
# The running load's own OOM answers, so one load never re-probes a busy card; another load starts empty.
_load_oom: contextvars.ContextVar[Optional[dict[Any, tuple[bool, str]]]] = contextvars.ContextVar(
    "rocm_probe_load_oom", default = None
)


@contextlib.contextmanager
def load_scope() -> Iterator[None]:
    """One load: an OOM probe inside it is answered from memo for the rest of the load and nowhere else."""
    token = _load_oom.set({})
    try:
        yield
    finally:
        _load_oom.reset(token)


def load_scoped(fn: Callable[..., Any]) -> Callable[..., Any]:
    @functools.wraps(fn)
    def wrapper(*args: Any, **kwargs: Any) -> Any:
        with load_scope():
            return fn(*args, **kwargs)

    return wrapper


def rocm_compile_setting() -> str:
    raw = (os.environ.get(ENV) or "auto").strip().lower()
    if raw in ("0", "off", "false", "no"):
        return "off"
    if raw in ("1", "on", "true", "yes", "force"):
        return "on"
    return "auto"


def rocm_compile_allowed() -> bool:
    return rocm_compile_setting() != "off"


def rocm_block_compile_works(device: Any = None) -> bool:
    return rocm_block_compile_probe(device)[0]


def rocm_block_compile_known(device: Any = None) -> Optional[bool]:
    """The probe's answer without running it (None = unprobed): preflight and status paths must not allocate."""
    setting = rocm_compile_setting()
    if setting != "auto":
        return setting == "on"
    # Lock-free: _lock is held for the whole multi-second probe, and a status read must not wait on it.
    hit = _result.get(_index(device))
    return None if hit is None else hit[0]


def _index(device: Any) -> Any:
    # A bare ordinal: torch.device(int) is not accepted by every supported torch.
    if isinstance(device, int) and not isinstance(device, bool):
        return device
    try:
        import torch
        index = torch.cuda.current_device() if device is None else torch.device(device).index
        return torch.cuda.current_device() if index is None else index
    except Exception:  # noqa: BLE001
        return 0


def rocm_block_compile_probe(device: Any = None) -> tuple[bool, str]:
    setting = rocm_compile_setting()
    if setting == "off":
        return False, f"{ENV}=0"
    if setting == "on":
        return True, f"{ENV}=1 (probe skipped)"
    try:
        import torch
    except Exception as exc:  # noqa: BLE001
        return False, f"torch unavailable: {exc}"
    from .rocm_bf16 import is_rocm_torch

    # Not torch.version.hip alone: AMD SDK / Radeon wheels leave it unset and carry the tag in __version__.
    if not is_rocm_torch(torch):
        return True, "not ROCm"
    index = _index(device)
    memo = _load_oom.get()
    # Lock-free when this card already has an answer: only a load that must probe waits behind a running probe.
    if memo is not None and index in memo:
        return memo[index]
    hit = _result.get(index)
    if hit is not None:
        return hit
    with _lock:
        # The load's own OOM first: a concurrent load's later success must not send this load into a compile.
        if memo is not None and index in memo:
            return memo[index]
        if index in _result:
            return _result[index]
        ok, why = _probe(torch, index)
        (logger.info if ok else logger.warning)(
            f"diffusion.speed: ROCm regional compile probe on cuda:{index}: "
            f"{'ok' if ok else 'failed, staying eager'} ({why})"
        )
        if not ok and "OutOfMemoryError" in why:
            if memo is not None:
                memo[index] = (ok, why)
        else:
            _result[index] = (ok, why)
        return ok, why


def _probe(
    torch: Any,
    index: int,
    device: Any = None,
) -> tuple[bool, str]:
    start = time.perf_counter()
    try:
        import torch.nn.functional as F
        from torch import nn

        class _Block(nn.Module):
            def __init__(
                self,
                dim: int = 128,
                heads: int = 4,
            ) -> None:
                super().__init__()
                self.heads = heads
                self.norm1 = nn.LayerNorm(dim, elementwise_affine = False)
                self.qkv = nn.Linear(dim, 3 * dim)
                self.out = nn.Linear(dim, dim)
                self.norm2 = nn.LayerNorm(dim)
                self.mlp = nn.Sequential(
                    nn.Linear(dim, 4 * dim), nn.GELU(approximate = "tanh"), nn.Linear(4 * dim, dim)
                )

            def forward(self, x: Any, scale: Any) -> Any:
                b, n, d = x.shape
                h = self.norm1(x) * (1 + scale)
                q, k, v = (
                    self.qkv(h).view(b, n, 3, self.heads, d // self.heads).permute(2, 0, 3, 1, 4)
                )
                a = F.scaled_dot_product_attention(q, k, v).transpose(1, 2).reshape(b, n, d)
                x = x + self.out(a)
                return x + self.mlp(self.norm2(x))

        dev = torch.device("cuda", index) if device is None else device
        dtype = torch.bfloat16
        gen = torch.Generator(device = "cpu").manual_seed(0)
        # Built on meta and filled from the private generator: the shared CPU RNG is never read or rewound, so a
        # concurrent load or generation drawing from it is untouched.
        with torch.device("meta"):
            block = _Block()
        block = block.to_empty(device = dev)
        with torch.no_grad():
            for name, param in block.named_parameters():
                if param.dim() == 2:
                    bound = param.shape[1] ** -0.5
                    value = (torch.rand(param.shape, generator = gen) * 2 - 1) * bound
                elif name.endswith("norm2.weight"):
                    value = torch.ones(param.shape)
                else:
                    value = (torch.rand(param.shape, generator = gen) * 2 - 1) * 0.02
                param.copy_(value)
        block = block.to(dtype = dtype).eval()
        inductor_cfg = torch._inductor.config
        patches = {"memory_planning": False} if hasattr(inductor_cfg, "memory_planning") else {}
        compiled = torch.compile(block, dynamic = True, fullgraph = True)
        worst = 0.0
        with (
            torch.no_grad(),
            inductor_cfg.patch(patches),
            torch._dynamo.config.patch(suppress_errors = False),
        ):
            for n in (77, 160):
                x = torch.randn(1, n, 128, generator = gen).to(device = dev, dtype = dtype)
                s = torch.randn(1, 1, 128, generator = gen).to(device = dev, dtype = dtype) * 0.1
                ref = block(x, s).float()
                got = compiled(x, s).float()
                if not bool(torch.isfinite(got).all()):
                    return False, "non-finite compiled output"
                worst = max(worst, float((got - ref).abs().max() / ref.abs().max().clamp_min(1e-6)))
        torch.cuda.synchronize(dev)
        if worst > 0.05:
            return False, f"compiled block disagrees with eager (rel max err {worst:.3g})"
        return True, f"rel max err {worst:.3g}, {time.perf_counter() - start:.1f}s"
    except Exception as exc:  # noqa: BLE001
        name = type(exc).__name__
        try:
            from .diffusion_batched import is_oom_error

            # Classified before flattening: a compiler error wrapping an OOM, or a bare "HIP out of memory".
            if is_oom_error(exc) and name != "OutOfMemoryError":
                name = f"OutOfMemoryError ({name})"
        except Exception:  # noqa: BLE001, S110
            pass
        return False, f"{name}: {str(exc).splitlines()[0][:300] if str(exc) else ''}"


def _reset_for_tests() -> None:
    with _lock:
        _result.clear()


def probe_result(index: int = 0) -> Optional[tuple[bool, str]]:
    return _result.get(index)