Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

12.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""ROCm TunableOp GEMM tables for diffusion denoisers on gfx1151 (hipBLASLt's default pick is 20-38% slow at DiT
shapes). Tuning costs 17-215 s per shape, so by default only READ the AMD CI tables in ``rocm_gemm_tables/`` whose
validators match exactly. ``UNSLOTH_DIFFUSION_ROCM_TUNABLEOP=tune`` also tunes on steady denoiser calls; ``=0`` off."""

from __future__ import annotations

import os
import re
import threading
import weakref
from pathlib import Path
from typing import Any, Optional

from .rocm_bf16 import is_rocm_torch

TUNABLEOP_ENV = "UNSLOTH_DIFFUSION_ROCM_TUNABLEOP"
# Measured archs only; widen with evidence.
TUNED_ARCHES = frozenset({"gfx1151"})
MAX_TUNING_MS = 30
MAX_TUNING_ITERS = 20
_HOOK_ATTR = "_unsloth_tunableop_hooks"
_OWNER_ATTR = "_unsloth_tunableop_owner"

_LOCK = threading.RLock()
_STATE: dict = {
    "active": False,
    "depth": 0,
    "written": 0,
    "file": None,
    "last_key": None,
    "tune": False,
    # Pipe tags using the table; the last release() restores prev_enabled.
    "owners": set(),
    "prev_enabled": False,
}


_TABLE_DIR = Path(__file__).resolve().parent / "rocm_gemm_tables"


def _env() -> str:
    return (os.environ.get(TUNABLEOP_ENV) or "").strip().lower()


def disabled() -> bool:
    return _env() in ("0", "off", "false", "no")


def tuning_requested() -> bool:
    return _env() == "tune"


def _validators(text: str) -> dict:
    out = {}
    for line in text.splitlines():
        parts = line.strip().split(",", 2)
        if len(parts) == 3 and parts[0] == "Validator":
            out[parts[1]] = parts[2]
    return out


def shipped_tables(arch: str, validators: dict) -> list:
    """Tables whose validators all match: solution indices are only valid for the exact build that measured them."""
    out = []
    want = {str(k): str(v) for k, v in validators.items()}
    for path in sorted(_TABLE_DIR.glob(f"{arch}_*.csv")):
        try:
            have = _validators(path.read_text(encoding = "utf-8"))
        except OSError:
            continue
        if have and have == want:
            out.append(path)
    return out


def _user_owns_tunableop() -> bool:
    return any(k.startswith("PYTORCH_TUNABLEOP_") for k in os.environ)


def _arch(device: Any = None) -> Optional[str]:
    try:
        import torch

        props = torch.cuda.get_device_properties(
            device if device is not None else torch.cuda.current_device()
        )
        name = str(getattr(props, "gcnArchName", "") or "")
        return name.split(":", 1)[0] or None
    except Exception:  # noqa: BLE001
        return None


def _device_count(torch: Any) -> int:
    try:
        return int(torch.cuda.device_count())
    except Exception:  # noqa: BLE001
        return 0


def eligible(device: Any = None) -> tuple:
    if disabled():
        return False, f"{TUNABLEOP_ENV}=0"
    if _user_owns_tunableop():
        return False, "PYTORCH_TUNABLEOP_* set by the user"
    try:
        import torch
    except Exception:  # noqa: BLE001
        return False, "no torch"
    if not is_rocm_torch(torch) or not torch.cuda.is_available():
        return False, "not ROCm"
    tunable = getattr(torch.cuda, "tunable", None)
    if tunable is None or not hasattr(tunable, "tuning_enable"):
        return False, "torch without TunableOp"
    arch = _arch(device)
    if arch not in TUNED_ARCHES:
        return False, f"arch {arch} not measured"
    # TunableOp is process-global and keyed on shape, not device: every visible card must share the arch.
    others = {_arch(i) for i in range(_device_count(torch))} - {arch}
    if others:
        return False, f"mixed archs {sorted(map(str, others))} beside {arch}"
    return True, arch


def results_file(arch: str) -> Path:
    import torch

    from .diffusion_compile_cache import cache_root

    # The HIP build string only labels the file (an f-string, as the ROCm gate guard expects); None on CUDA.
    hip = f"{getattr(torch.version, 'hip', None)}"
    tag = "_".join(re.sub(r"[^A-Za-z0-9.]+", "-", str(x)) for x in (arch, torch.__version__, hip))
    return cache_root() / "tunableop" / f"{tag}.csv"


def _write(tunable: Any) -> None:
    """Atomic write in torch's CSV layout; torch only writes at exit, which a killed server never reaches."""
    try:
        results = list(tunable.get_results() or ())
        validators = list(tunable.get_validators() or ())
    except Exception:  # noqa: BLE001
        return
    if len(results) <= _STATE["written"] or not _STATE["file"]:
        return
    lines = [f"Validator,{k},{v}" for k, v in validators]
    lines += [",".join(str(x) for x in row) for row in results]
    path = Path(_STATE["file"])
    tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
    try:
        tmp.write_text("\n".join(lines) + "\n", encoding = "utf-8")
        os.replace(tmp, path)
        _STATE["written"] = len(results)
    except OSError:
        try:
            tmp.unlink()
        except OSError:
            pass


def _call_key(args: Any, kwargs: Any) -> tuple:
    import torch

    key = []
    for v in list(args) + [kwargs[k] for k in sorted(kwargs)]:
        if isinstance(v, torch.Tensor):
            key.append(tuple(v.shape))
        elif isinstance(v, str):
            key.append(v)
    return tuple(key)


def _pre(_module: Any, args: Any, kwargs: Any) -> None:
    """Tune only on a call shaped like the previous one, so one-off shapes (first call, prefix prefill) never tune."""
    import torch

    key = _call_key(args, kwargs)
    with _LOCK:
        if not _STATE["active"]:
            return
        _STATE["depth"] += 1
        if _STATE["depth"] == 1:
            steady = key == _STATE.get("last_key") and bool(_STATE.get("tune"))
            _STATE["last_key"] = key
            _STATE["tuning"] = steady
            if steady:
                torch.cuda.tunable.tuning_enable(True)


def _post(_module: Any, _args: Any, _kwargs: Any, _out: Any) -> None:
    import torch
    with _LOCK:
        if not _STATE["active"] or _STATE["depth"] == 0:
            return
        _STATE["depth"] -= 1
        if _STATE["depth"] == 0 and _STATE.pop("tuning", False):
            torch.cuda.tunable.tuning_enable(False)
            _write(torch.cuda.tunable)


def _activate(arch: str, logger: Any = None) -> bool:
    import torch

    tunable = torch.cuda.tunable
    with _LOCK:
        if _STATE["active"]:
            return True
        tune = tuning_requested()
        path = results_file(arch)
        try:
            path.parent.mkdir(parents = True, exist_ok = True)
        except OSError:
            if tune:
                return False
        try:
            prev_enabled = bool(tunable.is_enabled())
        except Exception:  # noqa: BLE001
            prev_enabled = False
        tunable.tuning_enable(False)
        tunable.enable(True)
        _prime(torch)
        try:
            validators = dict(tunable.get_validators() or ())
        except Exception:  # noqa: BLE001
            validators = {}
        tables = shipped_tables(arch, validators)
        cached = path.exists() and _validators(_read(path)) == {
            str(k): str(v) for k, v in validators.items()
        }
        if not tune and not tables and not cached:
            tunable.enable(prev_enabled)
            return False
        tunable.set_filename(str(path), insert_device_ordinal = False)
        tunable.set_max_tuning_duration(MAX_TUNING_MS)
        tunable.set_max_tuning_iterations(MAX_TUNING_ITERS)
        for src in ([path] if cached else []) + tables:
            try:
                tunable.read_file(str(src))
            except Exception:  # noqa: BLE001 - a stale / foreign file: the default pick
                pass
        try:
            _STATE["written"] = len(tunable.get_results() or ())
        except Exception:  # noqa: BLE001
            _STATE["written"] = 0
        _STATE.update(
            active = True,
            depth = 0,
            file = str(path),
            last_key = None,
            tune = tune,
            prev_enabled = prev_enabled,
        )
    if logger is not None:
        logger.info(
            "diffusion.rocm_tunableop: %s GEMMs use the measured fastest solution for %d shapes (%s)%s",
            arch,
            _STATE["written"],
            ", ".join(p.name for p in tables) or "local cache",
            f"; tuning new denoiser shapes into {path}" if tune else "",
        )
    return True


def _prime(torch: Any) -> None:
    """torch registers validators on its first tunable GEMM: run a tiny one before comparing them."""
    try:
        a = torch.ones(8, 8, dtype = torch.bfloat16, device = "cuda")
        (a @ a).sum().item()
    except Exception:  # noqa: BLE001 - validators then stay partial and no table matches
        pass


def _read(path: Path) -> str:
    try:
        return path.read_text(encoding = "utf-8")
    except OSError:
        return ""


def _denoisers(pipe: Any) -> list:
    out = []
    for name in ("transformer", "transformer_2", "unet"):
        mod = getattr(pipe, name, None)
        if mod is not None and callable(getattr(mod, "register_forward_pre_hook", None)):
            out.append(mod)
    return out


def install_for_pipe(
    pipe: Any,
    device: Any = None,
    logger: Any = None,
) -> bool:
    ok, why = eligible(device)
    if not ok:
        if _STATE["active"]:  # a load on another arch must not inherit an earlier load's table
            uninstall(None)
        return False
    dits = _denoisers(pipe)
    if not dits:
        return False
    with (
        _LOCK
    ):  # activation and the claim are one step, so a concurrent last release cannot undo them
        if not _activate(why, logger):
            return False
        _claim(pipe)
    if not _STATE.get("tune"):
        return True
    for dit in dits:
        if dit.__dict__.get(_HOOK_ATTR):
            continue
        dit.__dict__[_HOOK_ATTR] = (
            dit.register_forward_pre_hook(_pre, with_kwargs = True),
            dit.register_forward_hook(_post, with_kwargs = True, always_call = True),
        )
    return True


def uninstall(pipe: Any = None) -> None:
    try:
        import torch
        tunable = torch.cuda.tunable
    except Exception:  # noqa: BLE001
        return
    with _LOCK:
        _restore(tunable)
    _remove_hooks(pipe)


def _restore(tunable: Any) -> None:
    # Caller holds _LOCK.
    if _STATE["active"]:
        _write(tunable)
        try:
            tunable.tuning_enable(False)
            tunable.enable(bool(_STATE.get("prev_enabled")))
        except Exception:  # noqa: BLE001
            pass
    _STATE["owners"].clear()
    _STATE.update(active = False, depth = 0)


def _remove_hooks(pipe: Any) -> None:
    for dit in _denoisers(pipe) if pipe is not None else ():
        for h in dit.__dict__.pop(_HOOK_ATTR, None) or ():
            try:
                h.remove()
            except Exception:  # noqa: BLE001
                pass


def _claim(pipe: Any) -> None:
    # Tag on the pipe so views over it (second expert, named DiT) share one owner.
    token = getattr(pipe, _OWNER_ATTR, None)
    if token is None:
        token = object()
        try:
            setattr(pipe, _OWNER_ATTR, token)
        except Exception:  # noqa: BLE001 - untaggable: never released, stays on as before
            return
        root = pipe
        while "_pipe" in getattr(
            root, "__dict__", {}
        ):  # a view: the claim lives on the pipe it wraps
            root = root.__dict__["_pipe"]
        try:
            # A load that fails or is cancelled drops the pipe without release(); collecting it ends the claim.
            weakref.finalize(root, _drop, token)
        except TypeError:
            pass
    with _LOCK:
        _STATE["owners"].add(token)


def _drop(token: object) -> None:
    with _LOCK:
        if token not in _STATE["owners"]:
            return
        _STATE["owners"].discard(token)
        if _STATE["active"] and not _STATE["owners"]:
            try:
                import torch
                _restore(torch.cuda.tunable)
            except Exception:  # noqa: BLE001
                pass


def release(pipe: Any) -> None:
    """Pipeline teardown: drop its claim; the last owner out restores the pre-activation TunableOp state."""
    if pipe is None:
        return
    token = getattr(pipe, _OWNER_ATTR, None)
    if token is None:
        return
    try:
        delattr(pipe, _OWNER_ATTR)
    except Exception:  # noqa: BLE001
        pass
    with (
        _LOCK
    ):  # the last-owner check and the restore must not interleave with another load's claim
        _STATE["owners"].discard(token)
        if _STATE["active"] and not _STATE["owners"]:
            try:
                import torch
                _restore(torch.cuda.tunable)
            except Exception:  # noqa: BLE001
                pass
    _remove_hooks(pipe)


def is_active() -> bool:
    return bool(_STATE["active"])