Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio ROCm: use the measured fastest GEMM solution table for diffusion denoisers on gfx1151 (#17299)
Daniel Hana784ee5b215e
diff --git a/pyproject.toml b/pyproject.tomlindex 2c1b0beaa8e..dff6edda9e0 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -84,6 +84,8 @@ studio = ["backend/assets/docs_ui/*",# Gate verdicts the auto ladder reads: if missing from the wheel, nvfp4 silently drops out."backend/core/inference/nvfp4_gate_record.json",+ # ROCm GEMM tables: if missing from the wheel, gfx1151 silently falls back to the slower pick.+ "backend/core/inference/rocm_gemm_tables/*.csv","backend/core/data_recipe/oxc-validator/*.json","backend/core/data_recipe/oxc-validator/*.mjs","backend/core/inference/bundled_skills/**/*.md",diff --git a/studio/backend/core/inference/diffusion_rocm_tunableop.py b/studio/backend/core/inference/diffusion_rocm_tunableop.pynew file mode 100644index 00000000000..5b4c2a1a75a--- /dev/null+++ b/studio/backend/core/inference/diffusion_rocm_tunableop.py@@ -0,0 +1,323 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""ROCm TunableOp GEMM tables for diffusion denoisers on gfx1151 (hipBLASLt's default pick is 20-38% slow at DiT+shapes). Tuning costs 17-215 s per shape, so by default only READ the AMD CI tables in ``rocm_gemm_tables/`` whose+validators match exactly. ``UNSLOTH_DIFFUSION_ROCM_TUNABLEOP=tune`` also tunes on steady denoiser calls; ``=0`` off."""++from __future__ import annotations++import os+import re+import threading+from pathlib import Path+from typing import Any, Optional++from .rocm_bf16 import is_rocm_torch++TUNABLEOP_ENV = "UNSLOTH_DIFFUSION_ROCM_TUNABLEOP"+# Measured archs only; widen with evidence.+TUNED_ARCHES = frozenset({"gfx1151"})+MAX_TUNING_MS = 30+MAX_TUNING_ITERS = 20+_HOOK_ATTR = "_unsloth_tunableop_hooks"++_LOCK = threading.Lock()+_STATE: dict = {+ "active": False,+ "depth": 0,+ "written": 0,+ "file": None,+ "last_key": None,+ "tune": False,+}+++_TABLE_DIR = Path(__file__).resolve().parent / "rocm_gemm_tables"+++def _env() -> str:+ return (os.environ.get(TUNABLEOP_ENV) or "").strip().lower()+++def disabled() -> bool:+ return _env() in ("0", "off", "false", "no")+++def tuning_requested() -> bool:+ return _env() == "tune"+++def _validators(text: str) -> dict:+ out = {}+ for line in text.splitlines():+ parts = line.strip().split(",", 2)+ if len(parts) == 3 and parts[0] == "Validator":+ out[parts[1]] = parts[2]+ return out+++def shipped_tables(arch: str, validators: dict) -> list:+ """Tables whose validators all match: solution indices are only valid for the exact build that measured them."""+ out = []+ want = {str(k): str(v) for k, v in validators.items()}+ for path in sorted(_TABLE_DIR.glob(f"{arch}_*.csv")):+ try:+ have = _validators(path.read_text(encoding = "utf-8"))+ except OSError:+ continue+ if have and have == want:+ out.append(path)+ return out+++def _user_owns_tunableop() -> bool:+ return any(k.startswith("PYTORCH_TUNABLEOP_") for k in os.environ)+++def _arch(device: Any = None) -> Optional[str]:+ try:+ import torch++ props = torch.cuda.get_device_properties(+ device if device is not None else torch.cuda.current_device()+ )+ name = str(getattr(props, "gcnArchName", "") or "")+ return name.split(":", 1)[0] or None+ except Exception: # noqa: BLE001+ return None+++def _device_count(torch: Any) -> int:+ try:+ return int(torch.cuda.device_count())+ except Exception: # noqa: BLE001+ return 0+++def eligible(device: Any = None) -> tuple:+ if disabled():+ return False, f"{TUNABLEOP_ENV}=0"+ if _user_owns_tunableop():+ return False, "PYTORCH_TUNABLEOP_* set by the user"+ try:+ import torch+ except Exception: # noqa: BLE001+ return False, "no torch"+ if not is_rocm_torch(torch) or not torch.cuda.is_available():+ return False, "not ROCm"+ tunable = getattr(torch.cuda, "tunable", None)+ if tunable is None or not hasattr(tunable, "tuning_enable"):+ return False, "torch without TunableOp"+ arch = _arch(device)+ if arch not in TUNED_ARCHES:+ return False, f"arch {arch} not measured"+ # TunableOp is process-global and keyed on shape, not device: every visible card must share the arch.+ others = {_arch(i) for i in range(_device_count(torch))} - {arch}+ if others:+ return False, f"mixed archs {sorted(map(str, others))} beside {arch}"+ return True, arch+++def results_file(arch: str) -> Path:+ import torch++ from .diffusion_compile_cache import cache_root++ tag = "_".join(+ re.sub(r"[^A-Za-z0-9.]+", "-", str(x)) for x in (arch, torch.__version__, torch.version.hip)+ )+ return cache_root() / "tunableop" / f"{tag}.csv"+++def _write(tunable: Any) -> None:+ """Atomic write in torch's CSV layout; torch only writes at exit, which a killed server never reaches."""+ try:+ results = list(tunable.get_results() or ())+ validators = list(tunable.get_validators() or ())+ except Exception: # noqa: BLE001+ return+ if len(results) <= _STATE["written"] or not _STATE["file"]:+ return+ lines = [f"Validator,{k},{v}" for k, v in validators]+ lines += [",".join(str(x) for x in row) for row in results]+ path = Path(_STATE["file"])+ tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")+ try:+ tmp.write_text("\n".join(lines) + "\n", encoding = "utf-8")+ os.replace(tmp, path)+ _STATE["written"] = len(results)+ except OSError:+ try:+ tmp.unlink()+ except OSError:+ pass+++def _call_key(args: Any, kwargs: Any) -> tuple:+ import torch++ key = []+ for v in list(args) + [kwargs[k] for k in sorted(kwargs)]:+ if isinstance(v, torch.Tensor):+ key.append(tuple(v.shape))+ elif isinstance(v, str):+ key.append(v)+ return tuple(key)+++def _pre(_module: Any, args: Any, kwargs: Any) -> None:+ """Tune only on a call shaped like the previous one, so one-off shapes (first call, prefix prefill) never tune."""+ import torch++ key = _call_key(args, kwargs)+ with _LOCK:+ if not _STATE["active"]:+ return+ _STATE["depth"] += 1+ if _STATE["depth"] == 1:+ steady = key == _STATE.get("last_key") and bool(_STATE.get("tune"))+ _STATE["last_key"] = key+ _STATE["tuning"] = steady+ if steady:+ torch.cuda.tunable.tuning_enable(True)+++def _post(_module: Any, _args: Any, _kwargs: Any, _out: Any) -> None:+ import torch+ with _LOCK:+ if not _STATE["active"] or _STATE["depth"] == 0:+ return+ _STATE["depth"] -= 1+ if _STATE["depth"] == 0 and _STATE.pop("tuning", False):+ torch.cuda.tunable.tuning_enable(False)+ _write(torch.cuda.tunable)+++def _activate(arch: str, logger: Any = None) -> bool:+ import torch++ tunable = torch.cuda.tunable+ with _LOCK:+ if _STATE["active"]:+ return True+ tune = tuning_requested()+ path = results_file(arch)+ try:+ path.parent.mkdir(parents = True, exist_ok = True)+ except OSError:+ if tune:+ return False+ tunable.tuning_enable(False)+ tunable.enable(True)+ _prime(torch)+ try:+ validators = dict(tunable.get_validators() or ())+ except Exception: # noqa: BLE001+ validators = {}+ tables = shipped_tables(arch, validators)+ cached = path.exists() and _validators(_read(path)) == {+ str(k): str(v) for k, v in validators.items()+ }+ if not tune and not tables and not cached:+ tunable.enable(False)+ return False+ tunable.set_filename(str(path), insert_device_ordinal = False)+ tunable.set_max_tuning_duration(MAX_TUNING_MS)+ tunable.set_max_tuning_iterations(MAX_TUNING_ITERS)+ for src in ([path] if cached else []) + tables:+ try:+ tunable.read_file(str(src))+ except Exception: # noqa: BLE001 - a stale / foreign file: the default pick+ pass+ try:+ _STATE["written"] = len(tunable.get_results() or ())+ except Exception: # noqa: BLE001+ _STATE["written"] = 0+ _STATE.update(active = True, depth = 0, file = str(path), last_key = None, tune = tune)+ if logger is not None:+ logger.info(+ "diffusion.rocm_tunableop: %s GEMMs use the measured fastest solution for %d shapes (%s)%s",+ arch,+ _STATE["written"],+ ", ".join(p.name for p in tables) or "local cache",+ f"; tuning new denoiser shapes into {path}" if tune else "",+ )+ return True+++def _prime(torch: Any) -> None:+ """torch registers validators on its first tunable GEMM: run a tiny one before comparing them."""+ try:+ a = torch.ones(8, 8, dtype = torch.bfloat16, device = "cuda")+ (a @ a).sum().item()+ except Exception: # noqa: BLE001 - validators then stay partial and no table matches+ pass+++def _read(path: Path) -> str:+ try:+ return path.read_text(encoding = "utf-8")+ except OSError:+ return ""+++def _denoisers(pipe: Any) -> list:+ out = []+ for name in ("transformer", "transformer_2", "unet"):+ mod = getattr(pipe, name, None)+ if mod is not None and callable(getattr(mod, "register_forward_pre_hook", None)):+ out.append(mod)+ return out+++def install_for_pipe(+ pipe: Any,+ device: Any = None,+ logger: Any = None,+) -> bool:+ ok, why = eligible(device)+ if not ok:+ if _STATE["active"]: # a load on another arch must not inherit an earlier load's table+ uninstall(None)+ return False+ dits = _denoisers(pipe)+ if not dits or not _activate(why, logger):+ return False+ if not _STATE.get("tune"):+ return True+ for dit in dits:+ if dit.__dict__.get(_HOOK_ATTR):+ continue+ dit.__dict__[_HOOK_ATTR] = (+ dit.register_forward_pre_hook(_pre, with_kwargs = True),+ dit.register_forward_hook(_post, with_kwargs = True, always_call = True),+ )+ return True+++def uninstall(pipe: Any = None) -> None:+ try:+ import torch+ tunable = torch.cuda.tunable+ except Exception: # noqa: BLE001+ return+ with _LOCK:+ if _STATE["active"]:+ _write(tunable)+ try:+ tunable.tuning_enable(False)+ tunable.enable(False)+ except Exception: # noqa: BLE001+ pass+ _STATE.update(active = False, depth = 0)+ for dit in _denoisers(pipe) if pipe is not None else ():+ for h in dit.__dict__.pop(_HOOK_ATTR, None) or ():+ try:+ h.remove()+ except Exception: # noqa: BLE001+ pass+++def is_active() -> bool:+ return bool(_STATE["active"])diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 69fc6446bbd..c21571f2118 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -515,6 +515,15 @@ def apply_speed_optims(_warn(logger, "vae attention chunks", exc)# Stability, not speed, so also before the `off` return._enable_miopen_immediate(logger)+ if getattr(target, "backend", None) == "rocm":+ try:+ from . import diffusion_rocm_tunableop as rocm_tunableop+ if mode == SPEED_OFF:+ rocm_tunableop.uninstall(pipe)+ else:+ rocm_tunableop.install_for_pipe(pipe, getattr(target, "device", None), logger)+ except Exception as exc: # noqa: BLE001 - keep torch's default GEMM pick+ _warn(logger, "rocm tunableop", exc)# TF32 (max) and cudnn.benchmark (any non-off CUDA load) are process-global; the caller restores them so a later# `off` load never inherits them.if mode == SPEED_OFF:diff --git a/studio/backend/core/inference/rocm_gemm_tables/gfx1151_torch2.11.0_hip713_qwenimage21_flux3klein_1024.csv b/studio/backend/core/inference/rocm_gemm_tables/gfx1151_torch2.11.0_hip713_qwenimage21_flux3klein_1024.csvnew file mode 100644index 00000000000..3ba0307305f--- /dev/null+++ b/studio/backend/core/inference/rocm_gemm_tables/gfx1151_torch2.11.0_hip713_qwenimage21_flux3klein_1024.csv@@ -0,0 +1,30 @@+Validator,PT_VERSION,2.11.0+Validator,HIP_VERSION,713+Validator,HIPBLASLT_VERSION,100300-02d4f738fc+Validator,GCN_ARCH_NAME,gfx1151+Validator,ROCBLAS_VERSION,5.4.0.02d4f738fc+GemmTunableOp_BFloat16_TN,tn_6144_1_3072_ld_3072_3072_6144,Gemm_Hipblaslt_667,0.11484694927930833+GemmTunableOp_BFloat16_TN,tn_3072_4176_9216_ld_9216_9216_3072,Gemm_Hipblaslt_629,7.338281750679016+GemmTunableOp_BFloat16_TN,tn_18432_4176_3072_ld_3072_3072_18432,Gemm_Hipblaslt_629,14.003121376037598+GemmTunableOp_BFloat16_TN,tn_3072_4176_3072_ld_3072_3072_3072,Gemm_Hipblaslt_629,2.2569542114551253+GemmTunableOp_BFloat16_TN,tn_18432_4096_3072_ld_3072_3072_18432,Gemm_Hipblaslt_629,13.642011642456055+GemmTunableOp_BFloat16_TN,tn_3072_4096_3072_ld_3072_3072_3072,Gemm_Hipblaslt_629,2.213201541167039+GemmTunableOp_BFloat16_TN,tn_128_4096_3072_ld_3072_3072_128,Gemm_Hipblaslt_557,0.10843500010669231+GemmTunableOp_BFloat16_TN,tn_3072_80_9216_ld_9216_9216_3072,Gemm_Hipblaslt_565,0.31924929767847054+GemmTunableOp_BFloat16_TN,tn_18432_80_3072_ld_3072_3072_18432,Gemm_Hipblaslt_663,0.649364399909973+GemmTunableOp_BFloat16_TN,tn_3072_4096_9216_ld_9216_9216_3072,Gemm_Hipblaslt_629,7.2732932567596436+GemmTunableOp_BFloat16_TN,tn_64_4096_4096_ld_4096_4096_64,Gemm_Hipblaslt_605,0.08287774957716466+GemmTunableOp_BFloat16_TN,tn_24576_4096_4096_ld_4096_4096_24576,Gemm_Hipblaslt_547,24.405466079711914+GemmTunableOp_BFloat16_TN,tn_4096_4096_4096_ld_4096_4096_4096,Gemm_Hipblaslt_547,4.392675797144571+GemmTunableOp_BFloat16_TN,tn_3072_1_256_ld_256_256_3072,Gemm_Hipblaslt_649,0.01066969996318221+GemmTunableOp_BFloat16_TN,tn_3072_80_20480_ld_20480_20480_3072,Gemm_Hipblaslt_669,0.9796434998512268+GemmTunableOp_BFloat16_TN,tn_9216_1_3072_ld_3072_3072_9216,Gemm_Hipblaslt_667,0.2518795467913151+GemmTunableOp_BFloat16_TN,tn_16384_2_4096_ld_4096_4096_16384,Gemm_Hipblaslt_664,0.6665960043668747+GemmTunableOp_BFloat16_TN,tn_4096_2_4096_ld_4096_4096_4096,Gemm_Hipblaslt_667,0.05601640045642853+GemmTunableOp_BFloat16_TN,tn_4096_4096_12288_ld_12288_12288_4096,Gemm_Hipblaslt_547,14.220678806304932+GemmTunableOp_BFloat16_TN,tn_12288_4096_4096_ld_4096_4096_12288,Gemm_Hipblaslt_547,12.396608352661133+GemmTunableOp_BFloat16_TN,tn_4096_2_256_ld_256_256_4096,Gemm_Hipblaslt_652,0.012021700059995056+GemmTunableOp_BFloat16_TN,tn_3072_1_3072_ld_3072_3072_3072,Gemm_Hipblaslt_666,0.03424699939787389+GemmTunableOp_BFloat16_TN,tn_3072_80_3072_ld_3072_3072_3072,Gemm_Hipblaslt_615,0.07518800012767314+GemmTunableOp_BFloat16_TT,tt_3072_4096_128_ld_128_4096_3072,Gemm_Rocblas_-1162085804,0.1158729501068592+GemmTunableOp_BFloat16_TT,tt_4096_4096_64_ld_64_4096_4096,Gemm_Rocblas_-1162085812,0.10524164959788321diff --git a/studio/backend/tests/test_diffusion_rocm_tunableop.py b/studio/backend/tests/test_diffusion_rocm_tunableop.pynew file mode 100644index 00000000000..34de0b36a8b--- /dev/null+++ b/studio/backend/tests/test_diffusion_rocm_tunableop.py@@ -0,0 +1,308 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""ROCm TunableOp scoping for diffusion denoisers (a fake torch.cuda.tunable; no GPU)."""++from __future__ import annotations++import types++import pytest++torch = pytest.importorskip("torch")+from torch import nn # noqa: E402++from core.inference import diffusion_rocm_tunableop as rt # noqa: E402+++class FakeTunable:+ def __init__(self):+ self.enabled = False+ self.tuning = False+ self.calls = []+ self.results = {}+ self.filename = None++ def enable(self, v = True):+ self.enabled = bool(v)+ self.calls.append(("enable", bool(v)))++ def tuning_enable(self, v = True):+ self.tuning = bool(v)+ self.calls.append(("tuning", bool(v)))++ def set_filename(+ self,+ name,+ insert_device_ordinal = False,+ ):+ self.filename = name++ def set_max_tuning_duration(self, ms):+ self.calls.append(("duration", ms))++ def set_max_tuning_iterations(self, n):+ self.calls.append(("iters", n))++ def read_file(self, name = None):+ self.calls.append(("read", name))+ return True++ def get_results(self):+ return tuple(+ ("GemmTunableOp_BFloat16_TN", k, "Gemm_Hipblaslt_1", v) for k, v in self.results.items()+ )++ def get_validators(self):+ return (("PT_VERSION", "2.11.0"), ("GCN_ARCH_NAME", "gfx1151"))+++class Denoiser(nn.Module):+ def __init__(self, fake):+ super().__init__()+ self.lin = nn.Linear(4, 4)+ self.fake = fake+ self.seen = []++ def forward(self, x, **_kw):+ self.seen.append(self.fake.tuning)+ self.fake.results[f"shape{len(self.fake.results)}"] = 1.0+ return self.lin(x)+++def fake(monkeypatch, tmp_path):+ f = FakeTunable()+ for name in (+ "enable",+ "tuning_enable",+ "set_filename",+ "set_max_tuning_duration",+ "set_max_tuning_iterations",+ "read_file",+ "get_results",+ "get_validators",+ ):+ monkeypatch.setattr(torch.cuda.tunable, name, getattr(f, name))+ monkeypatch.setattr(rt, "is_rocm_torch", lambda _t: True)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(rt, "_arch", lambda device = None: "gfx1151")+ monkeypatch.setenv("UNSLOTH_DIFFUSION_COMPILE_CACHE_DIR", str(tmp_path))+ monkeypatch.setenv(rt.TUNABLEOP_ENV, "tune")+ monkeypatch.setattr(rt, "_TABLE_DIR", tmp_path / "tables")+ monkeypatch.setattr(rt, "_prime", lambda _torch: None)+ for k in [k for k in __import__("os").environ if k.startswith("PYTORCH_TUNABLEOP_")]:+ monkeypatch.delenv(k)+ rt._STATE.update(active = False, depth = 0, written = 0, file = None, tune = False)+ yield f+ rt._STATE.update(active = False, depth = 0, written = 0, file = None, tune = False)+++def _pipe(fake):+ return types.SimpleNamespace(transformer = Denoiser(fake), vae = nn.Linear(4, 4))+++def test_tunes_only_inside_the_denoiser_and_persists(fake, tmp_path):+ pipe = _pipe(fake)+ assert rt.install_for_pipe(pipe)+ assert fake.enabled and not fake.tuning+ assert fake.filename.startswith(str(tmp_path / "tunableop"))+ assert ("duration", rt.MAX_TUNING_MS) in fake.calls+ x = torch.randn(2, 4)+ pipe.transformer(x) # a render's first call: lookups only+ assert pipe.transformer.seen == [False]+ pipe.transformer(x) # the next step, same shapes: tunes+ assert pipe.transformer.seen == [False, True]+ assert not fake.tuning+ from pathlib import Path++ text = Path(fake.filename).read_text(encoding = "utf-8").splitlines()+ assert text[0] == "Validator,PT_VERSION,2.11.0"+ assert text[-1] == "GemmTunableOp_BFloat16_TN,shape1,Gemm_Hipblaslt_1,1.0"+ mtime = Path(fake.filename).stat().st_mtime_ns+ pipe.transformer.forward = lambda x: pipe.transformer.lin(x) # nothing new tuned: no rewrite+ pipe.transformer(torch.randn(2, 4))+ assert Path(fake.filename).stat().st_mtime_ns == mtime+++def test_nested_calls_keep_tuning_on_until_the_outer_returns(fake):+ pipe = _pipe(fake)+ rt.install_for_pipe(pipe)+ rt._pre(None, (torch.zeros(1, 4),), {})+ rt._post(None, (), {}, None)+ rt._pre(None, (torch.zeros(1, 4),), {}) # steady: tuning on+ pipe.transformer(torch.randn(1, 4))+ assert fake.tuning+ rt._post(None, (), {}, None)+ assert not fake.tuning+++def test_exception_in_the_forward_turns_tuning_off(fake):+ pipe = _pipe(fake)+ rt.install_for_pipe(pipe)++ def boom(x):+ raise RuntimeError("x")++ pipe.transformer.forward = boom+ for _ in range(2): # the second call is steady, so tuning is on when it raises+ with pytest.raises(RuntimeError):+ pipe.transformer(torch.randn(1, 4))+ assert not fake.tuning and rt._STATE["depth"] == 0++[email protected]("why", ["kill", "user", "arch", "not_rocm"])+def test_declines(fake, monkeypatch, why):+ if why == "kill":+ monkeypatch.setenv(rt.TUNABLEOP_ENV, "0")+ elif why == "user":+ monkeypatch.setenv("PYTORCH_TUNABLEOP_ENABLED", "1")+ elif why == "arch":+ monkeypatch.setattr(rt, "_arch", lambda device = None: "gfx1100")+ else:+ monkeypatch.setattr(rt, "is_rocm_torch", lambda _t: False)+ pipe = _pipe(fake)+ assert not rt.install_for_pipe(pipe)+ assert not fake.enabled+ assert not pipe.transformer._forward_pre_hooks+++def test_uninstall_restores_the_default_pick(fake):+ pipe = _pipe(fake)+ rt.install_for_pipe(pipe)+ rt.uninstall(pipe)+ assert not fake.enabled and not fake.tuning+ assert not pipe.transformer._forward_pre_hooks and not pipe.transformer._forward_hooks+ assert not rt.is_active()+++def test_results_file_keys_on_arch_torch_and_hip(fake, tmp_path):+ p = rt.results_file("gfx1151")+ assert p.parent == tmp_path / "tunableop"+ assert p.name.startswith("gfx1151_") and p.suffix == ".csv"+++def test_speed_layer_installs_on_rocm_tiers_and_removes_on_off(monkeypatch):+ from core.inference import diffusion_speed as ds++ seen = []+ monkeypatch.setattr(+ rt, "install_for_pipe", lambda pipe, device = None, logger = None: seen.append("install")+ )+ monkeypatch.setattr(rt, "uninstall", lambda pipe = None: seen.append("uninstall"))+ pipe = types.SimpleNamespace(transformer = nn.Linear(2, 2), vae = None)+ rocm = types.SimpleNamespace(backend = "rocm", device = "cuda", dtype = torch.bfloat16)+ ds.apply_speed_optims(pipe, rocm, is_gguf = False, family = None, speed_mode = "off")+ assert seen == ["uninstall"]+ cuda = types.SimpleNamespace(backend = "cuda", device = "cuda", dtype = torch.bfloat16)+ ds.apply_speed_optims(pipe, cuda, is_gguf = False, family = None, speed_mode = "off")+ assert seen == ["uninstall"]+++def test_prefill_and_new_prompt_shapes_do_not_tune(fake):+ pipe = _pipe(fake)+ rt.install_for_pipe(pipe)+ lat = torch.randn(1, 4)+ for text in (5, 9):+ pipe.transformer(+ lat, encoder_hidden_states = torch.zeros(1, text, 4), kv_cache_mode = "extract"+ )+ pipe.transformer(lat, encoder_hidden_states = torch.zeros(1, text, 4), kv_cache_mode = "cached")+ pipe.transformer(lat, encoder_hidden_states = torch.zeros(1, text, 4), kv_cache_mode = "cached")+ assert pipe.transformer.seen == [False, False, True, False, False, True]+++def _table(tmp_path, validators):+ d = tmp_path / "tables"+ d.mkdir(exist_ok = True)+ p = d / "gfx1151_test.csv"+ lines = [f"Validator,{k},{v}" for k, v in validators]+ lines.append(+ "GemmTunableOp_BFloat16_TN,tn_4096_4096_4096_ld_4096_4096_4096,Gemm_Hipblaslt_547,3.8"+ )+ p.write_text("\n".join(lines) + "\n", encoding = "utf-8")+ return p+++def test_default_reads_a_matching_table_and_never_tunes(fake, monkeypatch, tmp_path):+ monkeypatch.delenv(rt.TUNABLEOP_ENV)+ table = _table(tmp_path, fake.get_validators())+ pipe = _pipe(fake)+ assert rt.install_for_pipe(pipe)+ assert fake.enabled and ("read", str(table)) in fake.calls+ assert not pipe.transformer._forward_pre_hooks+ for _ in range(3):+ pipe.transformer(torch.randn(2, 4))+ assert pipe.transformer.seen == [False, False, False]+ assert ("tuning", True) not in fake.calls++[email protected]("case", ["no_table", "other_build"])+def test_default_without_a_matching_table_leaves_tunableop_off(fake, monkeypatch, tmp_path, case):+ monkeypatch.delenv(rt.TUNABLEOP_ENV)+ if case == "other_build":+ _table(tmp_path, (("PT_VERSION", "2.12.0"), ("GCN_ARCH_NAME", "gfx1151")))+ pipe = _pipe(fake)+ assert not rt.install_for_pipe(pipe)+ assert not fake.enabled and fake.calls[-1] == ("enable", False)+ assert not any(c[0] == "read" for c in fake.calls)+ assert not rt.is_active()+++def test_table_with_a_missing_validator_does_not_match(fake, tmp_path):+ _table(tmp_path, (("PT_VERSION", "2.11.0"),))+ assert rt.shipped_tables("gfx1151", dict(fake.get_validators())) == []+ assert rt.shipped_tables("gfx1100", dict(fake.get_validators())) == []+++def test_shipped_tables_are_torch_result_files_with_full_validators():+ from pathlib import Path++ tables = sorted(Path(rt.__file__).resolve().parent.joinpath("rocm_gemm_tables").glob("*.csv"))+ assert tables+ for path in tables:+ text = path.read_text(encoding = "utf-8")+ validators = rt._validators(text)+ assert set(validators) >= {+ "PT_VERSION",+ "HIP_VERSION",+ "HIPBLASLT_VERSION",+ "ROCBLAS_VERSION",+ "GCN_ARCH_NAME",+ }+ assert path.name.startswith(validators["GCN_ARCH_NAME"] + "_")+ assert validators["GCN_ARCH_NAME"] in rt.TUNED_ARCHES+ rows = [+ line.split(",")+ for line in text.splitlines()+ if line and not line.startswith("Validator,")+ ]+ assert rows and all(len(r) == 4 and r[0].startswith("GemmTunableOp_") for r in rows)+ assert rt.shipped_tables(validators["GCN_ARCH_NAME"], validators) == [path]+++def test_shipped_tables_are_package_data():+ from pathlib import Path++ root = Path(rt.__file__).resolve().parents[4]+ text = (root / "pyproject.toml").read_text(encoding = "utf-8")+ assert '"backend/core/inference/rocm_gemm_tables/*.csv"' in text+++def test_mixed_arch_hosts_decline(fake, monkeypatch):+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)+ monkeypatch.setattr(rt, "_arch", lambda device = None: "gfx1100" if device == 1 else "gfx1151")+ ok, why = rt.eligible(0)+ assert not ok and "mixed archs" in why+ assert not rt.install_for_pipe(_pipe(fake), 0)+ assert not fake.enabled+++def test_a_load_on_another_arch_drops_the_earlier_table(fake, monkeypatch, tmp_path):+ monkeypatch.delenv(rt.TUNABLEOP_ENV)+ _table(tmp_path, fake.get_validators())+ assert rt.install_for_pipe(_pipe(fake))+ assert fake.enabled and rt.is_active()+ monkeypatch.setattr(rt, "_arch", lambda device = None: "gfx1201")+ assert not rt.install_for_pipe(_pipe(fake))+ assert not fake.enabled and not rt.is_active()