Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio ROCm: use the measured fastest GEMM solution table for diffusion denoisers on gfx1151 (#17299)

Daniel Hana784ee5b215e
diff --git a/pyproject.toml b/pyproject.toml
index 2c1b0beaa8e..dff6edda9e0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -84,6 +84,8 @@ studio = [
"backend/assets/docs_ui/*",
# Gate verdicts the auto ladder reads: if missing from the wheel, nvfp4 silently drops out.
"backend/core/inference/nvfp4_gate_record.json",
+ # ROCm GEMM tables: if missing from the wheel, gfx1151 silently falls back to the slower pick.
+ "backend/core/inference/rocm_gemm_tables/*.csv",
"backend/core/data_recipe/oxc-validator/*.json",
"backend/core/data_recipe/oxc-validator/*.mjs",
"backend/core/inference/bundled_skills/**/*.md",
diff --git a/studio/backend/core/inference/diffusion_rocm_tunableop.py b/studio/backend/core/inference/diffusion_rocm_tunableop.py
new file mode 100644
index 00000000000..5b4c2a1a75a
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_rocm_tunableop.py
@@ -0,0 +1,323 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""ROCm TunableOp GEMM tables for diffusion denoisers on gfx1151 (hipBLASLt's default pick is 20-38% slow at DiT
+shapes). Tuning costs 17-215 s per shape, so by default only READ the AMD CI tables in ``rocm_gemm_tables/`` whose
+validators match exactly. ``UNSLOTH_DIFFUSION_ROCM_TUNABLEOP=tune`` also tunes on steady denoiser calls; ``=0`` off."""
+
+from __future__ import annotations
+
+import os
+import re
+import threading
+from pathlib import Path
+from typing import Any, Optional
+
+from .rocm_bf16 import is_rocm_torch
+
+TUNABLEOP_ENV = "UNSLOTH_DIFFUSION_ROCM_TUNABLEOP"
+# Measured archs only; widen with evidence.
+TUNED_ARCHES = frozenset({"gfx1151"})
+MAX_TUNING_MS = 30
+MAX_TUNING_ITERS = 20
+_HOOK_ATTR = "_unsloth_tunableop_hooks"
+
+_LOCK = threading.Lock()
+_STATE: dict = {
+ "active": False,
+ "depth": 0,
+ "written": 0,
+ "file": None,
+ "last_key": None,
+ "tune": False,
+}
+
+
+_TABLE_DIR = Path(__file__).resolve().parent / "rocm_gemm_tables"
+
+
+def _env() -> str:
+ return (os.environ.get(TUNABLEOP_ENV) or "").strip().lower()
+
+
+def disabled() -> bool:
+ return _env() in ("0", "off", "false", "no")
+
+
+def tuning_requested() -> bool:
+ return _env() == "tune"
+
+
+def _validators(text: str) -> dict:
+ out = {}
+ for line in text.splitlines():
+ parts = line.strip().split(",", 2)
+ if len(parts) == 3 and parts[0] == "Validator":
+ out[parts[1]] = parts[2]
+ return out
+
+
+def shipped_tables(arch: str, validators: dict) -> list:
+ """Tables whose validators all match: solution indices are only valid for the exact build that measured them."""
+ out = []
+ want = {str(k): str(v) for k, v in validators.items()}
+ for path in sorted(_TABLE_DIR.glob(f"{arch}_*.csv")):
+ try:
+ have = _validators(path.read_text(encoding = "utf-8"))
+ except OSError:
+ continue
+ if have and have == want:
+ out.append(path)
+ return out
+
+
+def _user_owns_tunableop() -> bool:
+ return any(k.startswith("PYTORCH_TUNABLEOP_") for k in os.environ)
+
+
+def _arch(device: Any = None) -> Optional[str]:
+ try:
+ import torch
+
+ props = torch.cuda.get_device_properties(
+ device if device is not None else torch.cuda.current_device()
+ )
+ name = str(getattr(props, "gcnArchName", "") or "")
+ return name.split(":", 1)[0] or None
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _device_count(torch: Any) -> int:
+ try:
+ return int(torch.cuda.device_count())
+ except Exception: # noqa: BLE001
+ return 0
+
+
+def eligible(device: Any = None) -> tuple:
+ if disabled():
+ return False, f"{TUNABLEOP_ENV}=0"
+ if _user_owns_tunableop():
+ return False, "PYTORCH_TUNABLEOP_* set by the user"
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return False, "no torch"
+ if not is_rocm_torch(torch) or not torch.cuda.is_available():
+ return False, "not ROCm"
+ tunable = getattr(torch.cuda, "tunable", None)
+ if tunable is None or not hasattr(tunable, "tuning_enable"):
+ return False, "torch without TunableOp"
+ arch = _arch(device)
+ if arch not in TUNED_ARCHES:
+ return False, f"arch {arch} not measured"
+ # TunableOp is process-global and keyed on shape, not device: every visible card must share the arch.
+ others = {_arch(i) for i in range(_device_count(torch))} - {arch}
+ if others:
+ return False, f"mixed archs {sorted(map(str, others))} beside {arch}"
+ return True, arch
+
+
+def results_file(arch: str) -> Path:
+ import torch
+
+ from .diffusion_compile_cache import cache_root
+
+ tag = "_".join(
+ re.sub(r"[^A-Za-z0-9.]+", "-", str(x)) for x in (arch, torch.__version__, torch.version.hip)
+ )
+ return cache_root() / "tunableop" / f"{tag}.csv"
+
+
+def _write(tunable: Any) -> None:
+ """Atomic write in torch's CSV layout; torch only writes at exit, which a killed server never reaches."""
+ try:
+ results = list(tunable.get_results() or ())
+ validators = list(tunable.get_validators() or ())
+ except Exception: # noqa: BLE001
+ return
+ if len(results) <= _STATE["written"] or not _STATE["file"]:
+ return
+ lines = [f"Validator,{k},{v}" for k, v in validators]
+ lines += [",".join(str(x) for x in row) for row in results]
+ path = Path(_STATE["file"])
+ tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
+ try:
+ tmp.write_text("\n".join(lines) + "\n", encoding = "utf-8")
+ os.replace(tmp, path)
+ _STATE["written"] = len(results)
+ except OSError:
+ try:
+ tmp.unlink()
+ except OSError:
+ pass
+
+
+def _call_key(args: Any, kwargs: Any) -> tuple:
+ import torch
+
+ key = []
+ for v in list(args) + [kwargs[k] for k in sorted(kwargs)]:
+ if isinstance(v, torch.Tensor):
+ key.append(tuple(v.shape))
+ elif isinstance(v, str):
+ key.append(v)
+ return tuple(key)
+
+
+def _pre(_module: Any, args: Any, kwargs: Any) -> None:
+ """Tune only on a call shaped like the previous one, so one-off shapes (first call, prefix prefill) never tune."""
+ import torch
+
+ key = _call_key(args, kwargs)
+ with _LOCK:
+ if not _STATE["active"]:
+ return
+ _STATE["depth"] += 1
+ if _STATE["depth"] == 1:
+ steady = key == _STATE.get("last_key") and bool(_STATE.get("tune"))
+ _STATE["last_key"] = key
+ _STATE["tuning"] = steady
+ if steady:
+ torch.cuda.tunable.tuning_enable(True)
+
+
+def _post(_module: Any, _args: Any, _kwargs: Any, _out: Any) -> None:
+ import torch
+ with _LOCK:
+ if not _STATE["active"] or _STATE["depth"] == 0:
+ return
+ _STATE["depth"] -= 1
+ if _STATE["depth"] == 0 and _STATE.pop("tuning", False):
+ torch.cuda.tunable.tuning_enable(False)
+ _write(torch.cuda.tunable)
+
+
+def _activate(arch: str, logger: Any = None) -> bool:
+ import torch
+
+ tunable = torch.cuda.tunable
+ with _LOCK:
+ if _STATE["active"]:
+ return True
+ tune = tuning_requested()
+ path = results_file(arch)
+ try:
+ path.parent.mkdir(parents = True, exist_ok = True)
+ except OSError:
+ if tune:
+ return False
+ tunable.tuning_enable(False)
+ tunable.enable(True)
+ _prime(torch)
+ try:
+ validators = dict(tunable.get_validators() or ())
+ except Exception: # noqa: BLE001
+ validators = {}
+ tables = shipped_tables(arch, validators)
+ cached = path.exists() and _validators(_read(path)) == {
+ str(k): str(v) for k, v in validators.items()
+ }
+ if not tune and not tables and not cached:
+ tunable.enable(False)
+ return False
+ tunable.set_filename(str(path), insert_device_ordinal = False)
+ tunable.set_max_tuning_duration(MAX_TUNING_MS)
+ tunable.set_max_tuning_iterations(MAX_TUNING_ITERS)
+ for src in ([path] if cached else []) + tables:
+ try:
+ tunable.read_file(str(src))
+ except Exception: # noqa: BLE001 - a stale / foreign file: the default pick
+ pass
+ try:
+ _STATE["written"] = len(tunable.get_results() or ())
+ except Exception: # noqa: BLE001
+ _STATE["written"] = 0
+ _STATE.update(active = True, depth = 0, file = str(path), last_key = None, tune = tune)
+ if logger is not None:
+ logger.info(
+ "diffusion.rocm_tunableop: %s GEMMs use the measured fastest solution for %d shapes (%s)%s",
+ arch,
+ _STATE["written"],
+ ", ".join(p.name for p in tables) or "local cache",
+ f"; tuning new denoiser shapes into {path}" if tune else "",
+ )
+ return True
+
+
+def _prime(torch: Any) -> None:
+ """torch registers validators on its first tunable GEMM: run a tiny one before comparing them."""
+ try:
+ a = torch.ones(8, 8, dtype = torch.bfloat16, device = "cuda")
+ (a @ a).sum().item()
+ except Exception: # noqa: BLE001 - validators then stay partial and no table matches
+ pass
+
+
+def _read(path: Path) -> str:
+ try:
+ return path.read_text(encoding = "utf-8")
+ except OSError:
+ return ""
+
+
+def _denoisers(pipe: Any) -> list:
+ out = []
+ for name in ("transformer", "transformer_2", "unet"):
+ mod = getattr(pipe, name, None)
+ if mod is not None and callable(getattr(mod, "register_forward_pre_hook", None)):
+ out.append(mod)
+ return out
+
+
+def install_for_pipe(
+ pipe: Any,
+ device: Any = None,
+ logger: Any = None,
+) -> bool:
+ ok, why = eligible(device)
+ if not ok:
+ if _STATE["active"]: # a load on another arch must not inherit an earlier load's table
+ uninstall(None)
+ return False
+ dits = _denoisers(pipe)
+ if not dits or not _activate(why, logger):
+ return False
+ if not _STATE.get("tune"):
+ return True
+ for dit in dits:
+ if dit.__dict__.get(_HOOK_ATTR):
+ continue
+ dit.__dict__[_HOOK_ATTR] = (
+ dit.register_forward_pre_hook(_pre, with_kwargs = True),
+ dit.register_forward_hook(_post, with_kwargs = True, always_call = True),
+ )
+ return True
+
+
+def uninstall(pipe: Any = None) -> None:
+ try:
+ import torch
+ tunable = torch.cuda.tunable
+ except Exception: # noqa: BLE001
+ return
+ with _LOCK:
+ if _STATE["active"]:
+ _write(tunable)
+ try:
+ tunable.tuning_enable(False)
+ tunable.enable(False)
+ except Exception: # noqa: BLE001
+ pass
+ _STATE.update(active = False, depth = 0)
+ for dit in _denoisers(pipe) if pipe is not None else ():
+ for h in dit.__dict__.pop(_HOOK_ATTR, None) or ():
+ try:
+ h.remove()
+ except Exception: # noqa: BLE001
+ pass
+
+
+def is_active() -> bool:
+ return bool(_STATE["active"])
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 69fc6446bbd..c21571f2118 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -515,6 +515,15 @@ def apply_speed_optims(
_warn(logger, "vae attention chunks", exc)
# Stability, not speed, so also before the `off` return.
_enable_miopen_immediate(logger)
+ if getattr(target, "backend", None) == "rocm":
+ try:
+ from . import diffusion_rocm_tunableop as rocm_tunableop
+ if mode == SPEED_OFF:
+ rocm_tunableop.uninstall(pipe)
+ else:
+ rocm_tunableop.install_for_pipe(pipe, getattr(target, "device", None), logger)
+ except Exception as exc: # noqa: BLE001 - keep torch's default GEMM pick
+ _warn(logger, "rocm tunableop", exc)
# TF32 (max) and cudnn.benchmark (any non-off CUDA load) are process-global; the caller restores them so a later
# `off` load never inherits them.
if mode == SPEED_OFF:
diff --git a/studio/backend/core/inference/rocm_gemm_tables/gfx1151_torch2.11.0_hip713_qwenimage21_flux3klein_1024.csv b/studio/backend/core/inference/rocm_gemm_tables/gfx1151_torch2.11.0_hip713_qwenimage21_flux3klein_1024.csv
new file mode 100644
index 00000000000..3ba0307305f
--- /dev/null
+++ b/studio/backend/core/inference/rocm_gemm_tables/gfx1151_torch2.11.0_hip713_qwenimage21_flux3klein_1024.csv
@@ -0,0 +1,30 @@
+Validator,PT_VERSION,2.11.0
+Validator,HIP_VERSION,713
+Validator,HIPBLASLT_VERSION,100300-02d4f738fc
+Validator,GCN_ARCH_NAME,gfx1151
+Validator,ROCBLAS_VERSION,5.4.0.02d4f738fc
+GemmTunableOp_BFloat16_TN,tn_6144_1_3072_ld_3072_3072_6144,Gemm_Hipblaslt_667,0.11484694927930833
+GemmTunableOp_BFloat16_TN,tn_3072_4176_9216_ld_9216_9216_3072,Gemm_Hipblaslt_629,7.338281750679016
+GemmTunableOp_BFloat16_TN,tn_18432_4176_3072_ld_3072_3072_18432,Gemm_Hipblaslt_629,14.003121376037598
+GemmTunableOp_BFloat16_TN,tn_3072_4176_3072_ld_3072_3072_3072,Gemm_Hipblaslt_629,2.2569542114551253
+GemmTunableOp_BFloat16_TN,tn_18432_4096_3072_ld_3072_3072_18432,Gemm_Hipblaslt_629,13.642011642456055
+GemmTunableOp_BFloat16_TN,tn_3072_4096_3072_ld_3072_3072_3072,Gemm_Hipblaslt_629,2.213201541167039
+GemmTunableOp_BFloat16_TN,tn_128_4096_3072_ld_3072_3072_128,Gemm_Hipblaslt_557,0.10843500010669231
+GemmTunableOp_BFloat16_TN,tn_3072_80_9216_ld_9216_9216_3072,Gemm_Hipblaslt_565,0.31924929767847054
+GemmTunableOp_BFloat16_TN,tn_18432_80_3072_ld_3072_3072_18432,Gemm_Hipblaslt_663,0.649364399909973
+GemmTunableOp_BFloat16_TN,tn_3072_4096_9216_ld_9216_9216_3072,Gemm_Hipblaslt_629,7.2732932567596436
+GemmTunableOp_BFloat16_TN,tn_64_4096_4096_ld_4096_4096_64,Gemm_Hipblaslt_605,0.08287774957716466
+GemmTunableOp_BFloat16_TN,tn_24576_4096_4096_ld_4096_4096_24576,Gemm_Hipblaslt_547,24.405466079711914
+GemmTunableOp_BFloat16_TN,tn_4096_4096_4096_ld_4096_4096_4096,Gemm_Hipblaslt_547,4.392675797144571
+GemmTunableOp_BFloat16_TN,tn_3072_1_256_ld_256_256_3072,Gemm_Hipblaslt_649,0.01066969996318221
+GemmTunableOp_BFloat16_TN,tn_3072_80_20480_ld_20480_20480_3072,Gemm_Hipblaslt_669,0.9796434998512268
+GemmTunableOp_BFloat16_TN,tn_9216_1_3072_ld_3072_3072_9216,Gemm_Hipblaslt_667,0.2518795467913151
+GemmTunableOp_BFloat16_TN,tn_16384_2_4096_ld_4096_4096_16384,Gemm_Hipblaslt_664,0.6665960043668747
+GemmTunableOp_BFloat16_TN,tn_4096_2_4096_ld_4096_4096_4096,Gemm_Hipblaslt_667,0.05601640045642853
+GemmTunableOp_BFloat16_TN,tn_4096_4096_12288_ld_12288_12288_4096,Gemm_Hipblaslt_547,14.220678806304932
+GemmTunableOp_BFloat16_TN,tn_12288_4096_4096_ld_4096_4096_12288,Gemm_Hipblaslt_547,12.396608352661133
+GemmTunableOp_BFloat16_TN,tn_4096_2_256_ld_256_256_4096,Gemm_Hipblaslt_652,0.012021700059995056
+GemmTunableOp_BFloat16_TN,tn_3072_1_3072_ld_3072_3072_3072,Gemm_Hipblaslt_666,0.03424699939787389
+GemmTunableOp_BFloat16_TN,tn_3072_80_3072_ld_3072_3072_3072,Gemm_Hipblaslt_615,0.07518800012767314
+GemmTunableOp_BFloat16_TT,tt_3072_4096_128_ld_128_4096_3072,Gemm_Rocblas_-1162085804,0.1158729501068592
+GemmTunableOp_BFloat16_TT,tt_4096_4096_64_ld_64_4096_4096,Gemm_Rocblas_-1162085812,0.10524164959788321
diff --git a/studio/backend/tests/test_diffusion_rocm_tunableop.py b/studio/backend/tests/test_diffusion_rocm_tunableop.py
new file mode 100644
index 00000000000..34de0b36a8b
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_rocm_tunableop.py
@@ -0,0 +1,308 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""ROCm TunableOp scoping for diffusion denoisers (a fake torch.cuda.tunable; no GPU)."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+from torch import nn # noqa: E402
+
+from core.inference import diffusion_rocm_tunableop as rt # noqa: E402
+
+
+class FakeTunable:
+ def __init__(self):
+ self.enabled = False
+ self.tuning = False
+ self.calls = []
+ self.results = {}
+ self.filename = None
+
+ def enable(self, v = True):
+ self.enabled = bool(v)
+ self.calls.append(("enable", bool(v)))
+
+ def tuning_enable(self, v = True):
+ self.tuning = bool(v)
+ self.calls.append(("tuning", bool(v)))
+
+ def set_filename(
+ self,
+ name,
+ insert_device_ordinal = False,
+ ):
+ self.filename = name
+
+ def set_max_tuning_duration(self, ms):
+ self.calls.append(("duration", ms))
+
+ def set_max_tuning_iterations(self, n):
+ self.calls.append(("iters", n))
+
+ def read_file(self, name = None):
+ self.calls.append(("read", name))
+ return True
+
+ def get_results(self):
+ return tuple(
+ ("GemmTunableOp_BFloat16_TN", k, "Gemm_Hipblaslt_1", v) for k, v in self.results.items()
+ )
+
+ def get_validators(self):
+ return (("PT_VERSION", "2.11.0"), ("GCN_ARCH_NAME", "gfx1151"))
+
+
+class Denoiser(nn.Module):
+ def __init__(self, fake):
+ super().__init__()
+ self.lin = nn.Linear(4, 4)
+ self.fake = fake
+ self.seen = []
+
+ def forward(self, x, **_kw):
+ self.seen.append(self.fake.tuning)
+ self.fake.results[f"shape{len(self.fake.results)}"] = 1.0
+ return self.lin(x)
+
+
+def fake(monkeypatch, tmp_path):
+ f = FakeTunable()
+ for name in (
+ "enable",
+ "tuning_enable",
+ "set_filename",
+ "set_max_tuning_duration",
+ "set_max_tuning_iterations",
+ "read_file",
+ "get_results",
+ "get_validators",
+ ):
+ monkeypatch.setattr(torch.cuda.tunable, name, getattr(f, name))
+ monkeypatch.setattr(rt, "is_rocm_torch", lambda _t: True)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(rt, "_arch", lambda device = None: "gfx1151")
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_COMPILE_CACHE_DIR", str(tmp_path))
+ monkeypatch.setenv(rt.TUNABLEOP_ENV, "tune")
+ monkeypatch.setattr(rt, "_TABLE_DIR", tmp_path / "tables")
+ monkeypatch.setattr(rt, "_prime", lambda _torch: None)
+ for k in [k for k in __import__("os").environ if k.startswith("PYTORCH_TUNABLEOP_")]:
+ monkeypatch.delenv(k)
+ rt._STATE.update(active = False, depth = 0, written = 0, file = None, tune = False)
+ yield f
+ rt._STATE.update(active = False, depth = 0, written = 0, file = None, tune = False)
+
+
+def _pipe(fake):
+ return types.SimpleNamespace(transformer = Denoiser(fake), vae = nn.Linear(4, 4))
+
+
+def test_tunes_only_inside_the_denoiser_and_persists(fake, tmp_path):
+ pipe = _pipe(fake)
+ assert rt.install_for_pipe(pipe)
+ assert fake.enabled and not fake.tuning
+ assert fake.filename.startswith(str(tmp_path / "tunableop"))
+ assert ("duration", rt.MAX_TUNING_MS) in fake.calls
+ x = torch.randn(2, 4)
+ pipe.transformer(x) # a render's first call: lookups only
+ assert pipe.transformer.seen == [False]
+ pipe.transformer(x) # the next step, same shapes: tunes
+ assert pipe.transformer.seen == [False, True]
+ assert not fake.tuning
+ from pathlib import Path
+
+ text = Path(fake.filename).read_text(encoding = "utf-8").splitlines()
+ assert text[0] == "Validator,PT_VERSION,2.11.0"
+ assert text[-1] == "GemmTunableOp_BFloat16_TN,shape1,Gemm_Hipblaslt_1,1.0"
+ mtime = Path(fake.filename).stat().st_mtime_ns
+ pipe.transformer.forward = lambda x: pipe.transformer.lin(x) # nothing new tuned: no rewrite
+ pipe.transformer(torch.randn(2, 4))
+ assert Path(fake.filename).stat().st_mtime_ns == mtime
+
+
+def test_nested_calls_keep_tuning_on_until_the_outer_returns(fake):
+ pipe = _pipe(fake)
+ rt.install_for_pipe(pipe)
+ rt._pre(None, (torch.zeros(1, 4),), {})
+ rt._post(None, (), {}, None)
+ rt._pre(None, (torch.zeros(1, 4),), {}) # steady: tuning on
+ pipe.transformer(torch.randn(1, 4))
+ assert fake.tuning
+ rt._post(None, (), {}, None)
+ assert not fake.tuning
+
+
+def test_exception_in_the_forward_turns_tuning_off(fake):
+ pipe = _pipe(fake)
+ rt.install_for_pipe(pipe)
+
+ def boom(x):
+ raise RuntimeError("x")
+
+ pipe.transformer.forward = boom
+ for _ in range(2): # the second call is steady, so tuning is on when it raises
+ with pytest.raises(RuntimeError):
+ pipe.transformer(torch.randn(1, 4))
+ assert not fake.tuning and rt._STATE["depth"] == 0
+
+
[email protected]("why", ["kill", "user", "arch", "not_rocm"])
+def test_declines(fake, monkeypatch, why):
+ if why == "kill":
+ monkeypatch.setenv(rt.TUNABLEOP_ENV, "0")
+ elif why == "user":
+ monkeypatch.setenv("PYTORCH_TUNABLEOP_ENABLED", "1")
+ elif why == "arch":
+ monkeypatch.setattr(rt, "_arch", lambda device = None: "gfx1100")
+ else:
+ monkeypatch.setattr(rt, "is_rocm_torch", lambda _t: False)
+ pipe = _pipe(fake)
+ assert not rt.install_for_pipe(pipe)
+ assert not fake.enabled
+ assert not pipe.transformer._forward_pre_hooks
+
+
+def test_uninstall_restores_the_default_pick(fake):
+ pipe = _pipe(fake)
+ rt.install_for_pipe(pipe)
+ rt.uninstall(pipe)
+ assert not fake.enabled and not fake.tuning
+ assert not pipe.transformer._forward_pre_hooks and not pipe.transformer._forward_hooks
+ assert not rt.is_active()
+
+
+def test_results_file_keys_on_arch_torch_and_hip(fake, tmp_path):
+ p = rt.results_file("gfx1151")
+ assert p.parent == tmp_path / "tunableop"
+ assert p.name.startswith("gfx1151_") and p.suffix == ".csv"
+
+
+def test_speed_layer_installs_on_rocm_tiers_and_removes_on_off(monkeypatch):
+ from core.inference import diffusion_speed as ds
+
+ seen = []
+ monkeypatch.setattr(
+ rt, "install_for_pipe", lambda pipe, device = None, logger = None: seen.append("install")
+ )
+ monkeypatch.setattr(rt, "uninstall", lambda pipe = None: seen.append("uninstall"))
+ pipe = types.SimpleNamespace(transformer = nn.Linear(2, 2), vae = None)
+ rocm = types.SimpleNamespace(backend = "rocm", device = "cuda", dtype = torch.bfloat16)
+ ds.apply_speed_optims(pipe, rocm, is_gguf = False, family = None, speed_mode = "off")
+ assert seen == ["uninstall"]
+ cuda = types.SimpleNamespace(backend = "cuda", device = "cuda", dtype = torch.bfloat16)
+ ds.apply_speed_optims(pipe, cuda, is_gguf = False, family = None, speed_mode = "off")
+ assert seen == ["uninstall"]
+
+
+def test_prefill_and_new_prompt_shapes_do_not_tune(fake):
+ pipe = _pipe(fake)
+ rt.install_for_pipe(pipe)
+ lat = torch.randn(1, 4)
+ for text in (5, 9):
+ pipe.transformer(
+ lat, encoder_hidden_states = torch.zeros(1, text, 4), kv_cache_mode = "extract"
+ )
+ pipe.transformer(lat, encoder_hidden_states = torch.zeros(1, text, 4), kv_cache_mode = "cached")
+ pipe.transformer(lat, encoder_hidden_states = torch.zeros(1, text, 4), kv_cache_mode = "cached")
+ assert pipe.transformer.seen == [False, False, True, False, False, True]
+
+
+def _table(tmp_path, validators):
+ d = tmp_path / "tables"
+ d.mkdir(exist_ok = True)
+ p = d / "gfx1151_test.csv"
+ lines = [f"Validator,{k},{v}" for k, v in validators]
+ lines.append(
+ "GemmTunableOp_BFloat16_TN,tn_4096_4096_4096_ld_4096_4096_4096,Gemm_Hipblaslt_547,3.8"
+ )
+ p.write_text("\n".join(lines) + "\n", encoding = "utf-8")
+ return p
+
+
+def test_default_reads_a_matching_table_and_never_tunes(fake, monkeypatch, tmp_path):
+ monkeypatch.delenv(rt.TUNABLEOP_ENV)
+ table = _table(tmp_path, fake.get_validators())
+ pipe = _pipe(fake)
+ assert rt.install_for_pipe(pipe)
+ assert fake.enabled and ("read", str(table)) in fake.calls
+ assert not pipe.transformer._forward_pre_hooks
+ for _ in range(3):
+ pipe.transformer(torch.randn(2, 4))
+ assert pipe.transformer.seen == [False, False, False]
+ assert ("tuning", True) not in fake.calls
+
+
[email protected]("case", ["no_table", "other_build"])
+def test_default_without_a_matching_table_leaves_tunableop_off(fake, monkeypatch, tmp_path, case):
+ monkeypatch.delenv(rt.TUNABLEOP_ENV)
+ if case == "other_build":
+ _table(tmp_path, (("PT_VERSION", "2.12.0"), ("GCN_ARCH_NAME", "gfx1151")))
+ pipe = _pipe(fake)
+ assert not rt.install_for_pipe(pipe)
+ assert not fake.enabled and fake.calls[-1] == ("enable", False)
+ assert not any(c[0] == "read" for c in fake.calls)
+ assert not rt.is_active()
+
+
+def test_table_with_a_missing_validator_does_not_match(fake, tmp_path):
+ _table(tmp_path, (("PT_VERSION", "2.11.0"),))
+ assert rt.shipped_tables("gfx1151", dict(fake.get_validators())) == []
+ assert rt.shipped_tables("gfx1100", dict(fake.get_validators())) == []
+
+
+def test_shipped_tables_are_torch_result_files_with_full_validators():
+ from pathlib import Path
+
+ tables = sorted(Path(rt.__file__).resolve().parent.joinpath("rocm_gemm_tables").glob("*.csv"))
+ assert tables
+ for path in tables:
+ text = path.read_text(encoding = "utf-8")
+ validators = rt._validators(text)
+ assert set(validators) >= {
+ "PT_VERSION",
+ "HIP_VERSION",
+ "HIPBLASLT_VERSION",
+ "ROCBLAS_VERSION",
+ "GCN_ARCH_NAME",
+ }
+ assert path.name.startswith(validators["GCN_ARCH_NAME"] + "_")
+ assert validators["GCN_ARCH_NAME"] in rt.TUNED_ARCHES
+ rows = [
+ line.split(",")
+ for line in text.splitlines()
+ if line and not line.startswith("Validator,")
+ ]
+ assert rows and all(len(r) == 4 and r[0].startswith("GemmTunableOp_") for r in rows)
+ assert rt.shipped_tables(validators["GCN_ARCH_NAME"], validators) == [path]
+
+
+def test_shipped_tables_are_package_data():
+ from pathlib import Path
+
+ root = Path(rt.__file__).resolve().parents[4]
+ text = (root / "pyproject.toml").read_text(encoding = "utf-8")
+ assert '"backend/core/inference/rocm_gemm_tables/*.csv"' in text
+
+
+def test_mixed_arch_hosts_decline(fake, monkeypatch):
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ monkeypatch.setattr(rt, "_arch", lambda device = None: "gfx1100" if device == 1 else "gfx1151")
+ ok, why = rt.eligible(0)
+ assert not ok and "mixed archs" in why
+ assert not rt.install_for_pipe(_pipe(fake), 0)
+ assert not fake.enabled
+
+
+def test_a_load_on_another_arch_drops_the_earlier_table(fake, monkeypatch, tmp_path):
+ monkeypatch.delenv(rt.TUNABLEOP_ENV)
+ _table(tmp_path, fake.get_validators())
+ assert rt.install_for_pipe(_pipe(fake))
+ assert fake.enabled and rt.is_active()
+ monkeypatch.setattr(rt, "_arch", lambda device = None: "gfx1201")
+ assert not rt.install_for_pipe(_pipe(fake))
+ assert not fake.enabled and not rt.is_active()