Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio NVFP4: tune mm_fp4 once per M bucket and keep the tactics on disk (#17467)

Daniel Han116a9551890c
diff --git a/studio/backend/core/inference/diffusion_nvfp4_dispatch.py b/studio/backend/core/inference/diffusion_nvfp4_dispatch.py
index 7c96235c65d..7a969ccf8ee 100644
--- a/studio/backend/core/inference/diffusion_nvfp4_dispatch.py
+++ b/studio/backend/core/inference/diffusion_nvfp4_dispatch.py
@@ -6,6 +6,7 @@ all-or-nothing try, plus per-device verify()"""
from __future__ import annotations
+import dataclasses
import os
import threading
import weakref
@@ -26,6 +27,13 @@ _VERIFIED: dict[int, tuple] = {}
_QUANT_FN: dict[int, tuple] = {}
_GEMM_PLAN: dict = {}
_TRANSPOSED: dict = {}
+# device index -> (runner, workspace, tactic count): everything in a plan except the tactic.
+_RUNNERS: dict[int, tuple] = {}
+# One tune at a time: profiling launches must not overlap, and 0.6.6's tuning mode is process-global.
+_TUNE_LOCK = threading.Lock()
+_BUCKET_CONFIGS: dict = {}
+# Own op name: a stock mm_fp4 entry (measured at some other M) must never answer a bucket.
+BUCKET_OP = "fp4_gemm.studio_m_bucket"
def fast_dispatch_env() -> str:
@@ -254,23 +262,164 @@ def gemm_plan(
*,
force: bool = False,
):
- """``(runner, tactic, workspace)``, or None (use ``mm_fp4``), also for a COLD key under capture."""
+ """``(runner, tactic, workspace)``, or None (use ``mm_fp4``), also for a COLD key under capture.
+ Keyed by M bucket; ``force`` (verify) keeps the exact-M AutoTuner lookup and caches nothing."""
+ from . import diffusion_nvfp4_tactics as tactics
from .diffusion_nvfp4_ops import _device_index, _is_capturing
device = xq.device
- key = (xq.shape[0], xq.shape[1], n, backend, _device_index(device))
+ bucketed = tactics.bucketed_enabled() and not force
+ m = int(xq.shape[0])
+ key = (tactics.m_bucket(m) if bucketed else m, xq.shape[1], n, backend, _device_index(device))
got = _GEMM_PLAN.get(key)
if got is not None:
return got
- if backend != "cutlass" or not (force or enabled(device)) or _is_capturing():
+ if backend != "cutlass" or not (force or enabled(device)):
return None
+ if _is_capturing():
+ # Never a build or a tune here; a bucket already tuned (this process or on disk) is fine.
+ return _warm_plan(key, device) if bucketed else None
+ operands = [xq, wq_t, x_sf, w_sf_t, alpha, out]
try:
- return _build_plan(key, device, [xq, wq_t, x_sf, w_sf_t, alpha, out])
+ if bucketed:
+ return _build_bucket_plan(key, device, operands)
+ return _build_plan(key, device, operands, store = not force)
except Exception: # noqa: BLE001 - no plan means the public API
return None
-def _build_plan(key: tuple, device: Any, operands: list):
+def _runner(device: Any, index: int) -> tuple:
+ got = _RUNNERS.get(index)
+ if got is not None:
+ return got
+ import torch
+
+ from flashinfer.gemm.gemm_base import (
+ DEFAULT_WORKSPACE_SIZE,
+ _get_cache_buf,
+ get_cutlass_fp4_gemm_module,
+ )
+ from flashinfer.utils import get_compute_capability
+
+ with torch.cuda.device(device):
+ workspace = _get_cache_buf("mm_fp4_workspace", DEFAULT_WORKSPACE_SIZE, device)
+ major, minor = get_compute_capability(device)
+ runner = get_cutlass_fp4_gemm_module(major, minor).cutlass_fp4_gemm_runner()
+ try:
+ # Static per module (an index into the cutlass config table); it ignores both arguments.
+ count = len(runner.get_valid_tactics(None, None))
+ except Exception: # noqa: BLE001 - unknown table size: tactics stay in memory only
+ count = None
+ with _LOCK:
+ return _RUNNERS.setdefault(index, (runner, workspace, count))
+
+
+def _warm_plan(key: tuple, device: Any):
+ """A plan from cached parts only (no allocation, no profiling), for use inside a capture."""
+ from . import diffusion_nvfp4_tactics as tactics
+
+ bucket, half_k, n, backend, index = key
+ parts = _RUNNERS.get(index)
+ if parts is None:
+ return None
+ runner, workspace, count = parts
+ try:
+ env = tactics.device_environment(device, count, backend)
+ tactic = tactics.lookup(env, half_k * 2, n, bucket)
+ except Exception: # noqa: BLE001 - a cold bucket under capture takes the public API
+ return None
+ if tactic is None:
+ return None
+ plan = (runner, tactic, workspace)
+ with _LOCK:
+ return _GEMM_PLAN.setdefault(key, plan)
+
+
+def _build_bucket_plan(key: tuple, device: Any, operands: list):
+ from . import diffusion_nvfp4_tactics as tactics
+
+ bucket, half_k, n, backend, index = key
+ runner, workspace, count = _runner(device, index)
+ env = tactics.device_environment(device, count, backend)
+ k = half_k * 2
+ tactic = tactics.lookup(env, k, n, bucket)
+ if tactic is None:
+ with _TUNE_LOCK:
+ tactic = tactics.lookup(env, k, n, bucket)
+ if tactic is None:
+ try:
+ tuned = _tune_bucket(device, runner, workspace, operands, bucket)
+ except Exception: # noqa: BLE001 - the default tactic, this process only
+ tuned = None
+ # -1 is FlashInfer's "nothing ran" (every tactic failed, or an OOM): never persisted.
+ persist = tuned is not None and tuned >= 0
+ tactic = tactics.record(
+ env, k, n, bucket, -1 if tuned is None else tuned, persist = persist
+ )
+ plan = (runner, tactic, workspace)
+ with _LOCK:
+ return _GEMM_PLAN.setdefault(key, plan)
+
+
+def _constant_bucket(bucket: int, _m: int) -> int:
+ return bucket
+
+
+def bucket_config(base: Any, bucket: int):
+ """``base`` profiling exactly one M (``bucket``) and keying every M to it; memoised per bucket so
+ FlashInfer's nearest-profile cache sees a stable spec."""
+ cache_key = (id(base), int(bucket))
+ got = _BUCKET_CONFIGS.get(cache_key)
+ if got is not None and got[0] is base:
+ return got[1]
+ spec = base.dynamic_tensor_specs[0]
+ pinned = dataclasses.replace(
+ spec,
+ gen_tuning_buckets = (int(bucket),),
+ map_to_tuning_buckets = partial(_constant_bucket, int(bucket)),
+ )
+ config = dataclasses.replace(base, dynamic_tensor_specs = (pinned,))
+ with _LOCK:
+ return _BUCKET_CONFIGS.setdefault(cache_key, (base, config))[1]
+
+
+def _tune_bucket(device: Any, runner: Any, workspace: Any, operands: list, bucket: int) -> int:
+ """FlashInfer's own profiler (its timing, failure handling and blocklist), at M = ``bucket`` only."""
+ import time
+
+ import flashinfer
+ import torch
+
+ from flashinfer.autotuner import AutoTuner
+ from flashinfer.gemm.gemm_base import _MM_FP4_TUNING_CONFIG_128x4
+
+ config = bucket_config(_MM_FP4_TUNING_CONFIG_128x4, bucket)
+ xq, wq_t, x_sf, w_sf_t, alpha, out = operands
+ inputs = [xq, wq_t, x_sf, w_sf_t, alpha, torch.bfloat16, out, 16, True, workspace]
+ start = time.perf_counter()
+ with torch.cuda.device(device):
+ with flashinfer.autotune(True):
+ _, tactic = AutoTuner.get().choose_one(BUCKET_OP, [runner], config, inputs)
+ from .diffusion_nvfp4_tactics import logger
+
+ logger.info(
+ "nvfp4.tactics: tuned mm_fp4 K=%d N=%d at M bucket %d -> tactic %s (%.2f s)",
+ int(xq.shape[1]) * 2,
+ int(out.shape[1]),
+ int(bucket),
+ tactic,
+ time.perf_counter() - start,
+ )
+ return int(tactic)
+
+
+def _build_plan(
+ key: tuple,
+ device: Any,
+ operands: list,
+ *,
+ store: bool = True,
+):
import torch
from flashinfer.autotuner import AutoTuner
@@ -293,6 +442,8 @@ def _build_plan(key: tuple, device: Any, operands: list):
"fp4_gemm", [runner], _MM_FP4_TUNING_CONFIG_128x4, inputs
)
plan = (runner, tactic, workspace)
+ if not store:
+ return plan
with _LOCK:
return _GEMM_PLAN.setdefault(key, plan)
@@ -313,13 +464,16 @@ def reset() -> None:
_QUANT_FN.clear()
_GEMM_PLAN.clear()
_TRANSPOSED.clear()
+ _RUNNERS.clear()
def describe() -> dict:
+ from . import diffusion_nvfp4_tactics as tactics
return {
"available": available()[0],
"reason": available()[1],
"verified_devices": sorted(index for index, rec in _VERIFIED.items() if rec[0]),
"plans": len(_GEMM_PLAN),
"transposed": len(_TRANSPOSED),
+ "tactics": tactics.describe(),
}
diff --git a/studio/backend/core/inference/diffusion_nvfp4_linear.py b/studio/backend/core/inference/diffusion_nvfp4_linear.py
index 9de5f8b4b15..6e5e4c2d115 100644
--- a/studio/backend/core/inference/diffusion_nvfp4_linear.py
+++ b/studio/backend/core/inference/diffusion_nvfp4_linear.py
@@ -312,7 +312,7 @@ def _prewarm_shapes(modules, shapes, *, logger, tuned_box) -> None:
m = int(m)
if m <= 0:
continue
- key = (m, module.in_features, module.out_features)
+ key = _prewarm_key(m, module.in_features, module.out_features)
if key in _TUNED_SHAPES:
continue
device = module.wq.device
@@ -332,6 +332,16 @@ def _prewarm_shapes(modules, shapes, *, logger, tuned_box) -> None:
module._tuned = True
+def _prewarm_key(m: int, k: int, n: int) -> tuple:
+ """Legacy: the exact shape, which also stands in for the forward's claim. Bucketed: the forward
+ claims (or the plan tunes) its own bucket, so this only skips repeating the same warm-up."""
+ from .diffusion_nvfp4_tactics import bucketed_enabled
+
+ if bucketed_enabled():
+ return ("prewarm", int(m), int(k), int(n))
+ return (int(m), int(k), int(n))
+
+
def _iter_linears(transformer: Any):
"""Every Linear-like leaf, by feature counts (a converted layer has no ``weight``)."""
for name, module in transformer.named_modules():
diff --git a/studio/backend/core/inference/diffusion_nvfp4_ops.py b/studio/backend/core/inference/diffusion_nvfp4_ops.py
index 70294984674..eff6159f70f 100644
--- a/studio/backend/core/inference/diffusion_nvfp4_ops.py
+++ b/studio/backend/core/inference/diffusion_nvfp4_ops.py
@@ -121,6 +121,7 @@ def _mm_impl(xq: Any, wq: Any, x_sf: Any, w_sf: Any, alpha: Any, n: int, backend
import torch
from . import diffusion_nvfp4_dispatch as dispatch
+ from . import diffusion_nvfp4_tactics as tactics
with _device_guard(xq):
m = xq.shape[0]
@@ -129,7 +130,11 @@ def _mm_impl(xq: Any, wq: Any, x_sf: Any, w_sf: Any, alpha: Any, n: int, backend
else:
out = torch.empty(m, n, device = xq.device, dtype = torch.bfloat16)
_fire_barrier(xq.device)
- if _claim_first_call_tune(m, xq.shape[1] * 2, n):
+ fast = dispatch.enabled(xq.device)
+ # Bucketed fast path: the plan tunes its own M bucket once (and reads it back from disk).
+ if not (fast and tactics.bucketed_enabled()) and _claim_first_call_tune(
+ m, xq.shape[1] * 2, n
+ ):
# Before the dispatch plan, which caches whatever tactic FlashInfer holds on its first build.
try:
with flashinfer.autotune(True):
@@ -138,7 +143,7 @@ def _mm_impl(xq: Any, wq: Any, x_sf: Any, w_sf: Any, alpha: Any, n: int, backend
)
except Exception: # noqa: BLE001 - claimed, so it is not retried; run it untuned
pass
- if dispatch.enabled(xq.device):
+ if fast:
wq_t, w_sf_t = dispatch.transposed(wq), dispatch.transposed(w_sf)
plan = dispatch.gemm_plan(xq, wq_t, x_sf, w_sf_t, alpha, out, n, backend)
if plan is not None:
@@ -164,12 +169,33 @@ def _mm_impl(xq: Any, wq: Any, x_sf: Any, w_sf: Any, alpha: Any, n: int, backend
)
+def _flashinfer_bucket(m: int) -> int:
+ """The M FlashInfer's own mm_fp4 tuning keys ``m`` under (exact ``m`` when it cannot be read)."""
+ try:
+ from flashinfer.gemm.gemm_base import _MM_FP4_TUNING_CONFIG_128x4
+ return int(
+ _MM_FP4_TUNING_CONFIG_128x4.dynamic_tensor_specs[0].map_to_tuning_buckets(int(m))
+ )
+ except Exception: # noqa: BLE001 - an unreadable mapper only costs a retune per length
+ return int(m)
+
+
+def _first_tune_key(m: int, k: int, n: int) -> tuple:
+ """Public-API tuning covers FlashInfer's whole bucket, so a new length in a tuned bucket is no
+ new tune; ``UNSLOTH_NVFP4_TACTIC_CACHE=0`` restores one claim per exact length."""
+ from .diffusion_nvfp4_tactics import bucketed_enabled
+
+ if bucketed_enabled():
+ return ("flashinfer_bucket", _flashinfer_bucket(m), int(k), int(n))
+ return (int(m), int(k), int(n))
+
+
def _claim_first_call_tune(m: int, k: int, n: int) -> bool:
"""Claims the one autotune for a shape, never inside a CUDA graph capture, where profiling is
illegal."""
from .diffusion_nvfp4_linear import _TUNED_SHAPES
- key = (int(m), int(k), int(n))
+ key = _first_tune_key(m, k, n)
if key in _TUNED_SHAPES or _is_capturing():
return False
_TUNED_SHAPES.add(key)
diff --git a/studio/backend/core/inference/diffusion_nvfp4_tactics.py b/studio/backend/core/inference/diffusion_nvfp4_tactics.py
new file mode 100644
index 00000000000..a5da9145496
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_nvfp4_tactics.py
@@ -0,0 +1,305 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""mm_fp4 tactics per (GEMM shape, M bucket): each bucket profiled once at its own M, reused by every M in it,
+persisted per GPU, CUDA and FlashInfer build (a tactic indexes a static, shape-independent config table)."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# auto (default): bucketed + on disk; memory: bucketed, never written; 0: the old per-length tuning.
+NVFP4_TACTIC_CACHE_ENV = "UNSLOTH_NVFP4_TACTIC_CACHE"
+NVFP4_TACTIC_CACHE_DIR_ENV = "UNSLOTH_NVFP4_TACTIC_CACHE_DIR"
+MODE_AUTO = "auto"
+MODE_MEMORY = "memory"
+MODE_OFF = "0"
+
+SCHEMA = 1
+_OP = "mm_fp4"
+
+_LOCK = threading.Lock()
+# env hash -> {entry key -> tactic}
+_MEMORY: dict[str, dict[str, int]] = {}
+_LOADED: set = set()
+_IDENTITY: dict[tuple, tuple] = {}
+_STATS = {"tuned": 0, "disk_entries_loaded": 0, "memory_hits": 0, "persist_failures": 0}
+
+
+def mode() -> str:
+ raw = (os.environ.get(NVFP4_TACTIC_CACHE_ENV) or "").strip().lower()
+ if raw in ("0", "off", "false", "no", "legacy"):
+ return MODE_OFF
+ if raw == MODE_MEMORY:
+ return MODE_MEMORY
+ return MODE_AUTO
+
+
+def bucketed_enabled() -> bool:
+ return mode() != MODE_OFF
+
+
+def persistent_enabled() -> bool:
+ return mode() == MODE_AUTO
+
+
+def m_bucket(m: int) -> int:
+ """Largest bucket <= ``m``: powers of two, plus 1.5x each power from 768 up (spacing <= 1.5x)."""
+ m = int(m)
+ if m <= 1:
+ return 1
+ p = 1 << (m.bit_length() - 1)
+ if p >= 512 and m >= p + (p >> 1):
+ return p + (p >> 1)
+ return p
+
+
+def cache_dir() -> Optional[Path]:
+ raw = os.environ.get(NVFP4_TACTIC_CACHE_DIR_ENV)
+ if raw:
+ return Path(raw)
+ try:
+ from utils.paths.storage_roots import cache_root
+ except ImportError:
+ return None
+ return cache_root() / "nvfp4_tactics"
+
+
+def environment(
+ *,
+ flashinfer_version: str,
+ capability: tuple,
+ device_name: str,
+ sm_count: Any,
+ cuda: Any,
+ tactic_count: Any,
+ backend: str = "cutlass",
+) -> tuple[str, dict]:
+ """``(hash, meta)``: everything a stored tactic index depends on. Any change is a different file."""
+ meta = {
+ "schema": SCHEMA,
+ "op": _OP,
+ "backend": str(backend),
+ "sf_layout": "128x4",
+ "out_dtype": "bfloat16",
+ "flashinfer": str(flashinfer_version),
+ "capability": [int(c) for c in capability],
+ "device": str(device_name),
+ "sm_count": None if sm_count is None else int(sm_count),
+ "cuda": None if cuda is None else str(cuda),
+ "tactic_count": None if tactic_count is None else int(tactic_count),
+ }
+ digest = hashlib.sha256(json.dumps(meta, sort_keys = True).encode("utf-8")).hexdigest()[:16]
+ return digest, meta
+
+
+def device_environment(
+ device: Any,
+ tactic_count: Any,
+ backend: str = "cutlass",
+) -> tuple[str, dict]:
+ """``environment`` for a CUDA device, memoised per (device index, tactic count, backend)."""
+ from .diffusion_nvfp4_ops import _device_index
+
+ index = _device_index(device)
+ key = (index, tactic_count, backend)
+ got = _IDENTITY.get(key)
+ if got is not None:
+ return got
+ import torch
+
+ import flashinfer
+
+ props = torch.cuda.get_device_properties(index)
+ got = environment(
+ flashinfer_version = str(getattr(flashinfer, "__version__", "unknown")),
+ capability = tuple(torch.cuda.get_device_capability(index)),
+ device_name = str(getattr(props, "name", "")) or torch.cuda.get_device_name(index),
+ sm_count = getattr(props, "multi_processor_count", None),
+ cuda = getattr(torch.version, "cuda", None),
+ tactic_count = tactic_count,
+ backend = backend,
+ )
+ with _LOCK:
+ return _IDENTITY.setdefault(key, got)
+
+
+def _entry_key(k: int, n: int, bucket: int) -> str:
+ return f"{int(k)}x{int(n)}@{int(bucket)}"
+
+
+def _path_for(env_hash: str) -> Optional[Path]:
+ root = cache_dir()
+ return None if root is None else root / f"{_OP}_{env_hash}.json"
+
+
+def _valid_tactic(value: Any, meta: dict) -> bool:
+ if isinstance(value, bool) or not isinstance(value, int):
+ return False
+ count = meta.get("tactic_count")
+ if count is None:
+ return value >= -1
+ return -1 <= value < int(count)
+
+
+def _read_file(path: Path, meta: dict) -> dict[str, int]:
+ """The valid entries of ``path``; a missing, foreign or corrupt file reads as empty, never raises."""
+ try:
+ with open(path, "r", encoding = "utf-8") as fh:
+ data = json.load(fh)
+ except FileNotFoundError:
+ return {}
+ except Exception as exc: # noqa: BLE001 - a corrupt file costs a retune, never a load
+ logger.debug(
+ "nvfp4.tactics: ignoring unreadable %s (%s: %s)", path, type(exc).__name__, exc
+ )
+ return {}
+ if not isinstance(data, dict) or data.get("schema") != SCHEMA or data.get("meta") != meta:
+ return {}
+ entries = data.get("entries")
+ if not isinstance(entries, dict):
+ return {}
+ return {str(k): int(v) for k, v in entries.items() if _valid_tactic(v, meta)}
+
+
+class _FileLock:
+ """Advisory cross-process lock where the OS has one (FlashInfer itself is Linux only)."""
+
+ def __init__(self, path: Path):
+ self.path = path.with_name(path.name + ".lock")
+ self.fh = None
+
+ def __enter__(self):
+ try:
+ import fcntl
+ self.fh = open(self.path, "a+", encoding = "utf-8")
+ fcntl.flock(self.fh.fileno(), fcntl.LOCK_EX)
+ except Exception: # noqa: BLE001 - no lock means last writer wins, still atomic per file
+ if self.fh is not None:
+ self.fh.close()
+ self.fh = None
+ return self
+
+ def __exit__(self, *exc):
+ if self.fh is not None:
+ try:
+ import fcntl
+ fcntl.flock(self.fh.fileno(), fcntl.LOCK_UN)
+ except Exception: # noqa: BLE001
+ pass
+ self.fh.close()
+ self.fh = None
+ return False
+
+
+def _persist(path: Path, meta: dict, update: dict[str, int]) -> bool:
+ """Merge ``update`` into the file and publish it with an atomic rename."""
+ try:
+ path.parent.mkdir(parents = True, exist_ok = True)
+ with _FileLock(path):
+ entries = _read_file(path, meta)
+ entries.update(update)
+ tmp = path.with_name(f".{path.name}.{os.getpid()}.{threading.get_ident()}.tmp")
+ with open(tmp, "w", encoding = "utf-8") as fh:
+ json.dump(
+ {"schema": SCHEMA, "meta": meta, "entries": dict(sorted(entries.items()))},
+ fh,
+ indent = 1,
+ sort_keys = True,
+ )
+ os.replace(tmp, path)
+ return True
+ except Exception as exc: # noqa: BLE001 - persistence is an optimisation
+ _STATS["persist_failures"] += 1
+ logger.debug("nvfp4.tactics: could not write %s (%s: %s)", path, type(exc).__name__, exc)
+ return False
+
+
+def _disk_ok(meta: dict) -> bool:
+ """On disk only when the table size is known: an index is only checkable against it."""
+ return persistent_enabled() and meta.get("tactic_count") is not None
+
+
+def _ensure_loaded(env_hash: str, meta: dict) -> dict[str, int]:
+ table = _MEMORY.get(env_hash)
+ if table is not None and env_hash in _LOADED:
+ return table
+ with _LOCK:
+ table = _MEMORY.setdefault(env_hash, {})
+ if env_hash in _LOADED:
+ return table
+ if _disk_ok(meta):
+ path = _path_for(env_hash)
+ if path is not None:
+ on_disk = _read_file(path, meta)
+ for key, value in on_disk.items():
+ table.setdefault(key, value)
+ _STATS["disk_entries_loaded"] += len(on_disk)
+ _LOADED.add(env_hash)
+ return table
+
+
+def lookup(env: tuple[str, dict], k: int, n: int, bucket: int) -> Optional[int]:
+ """The stored tactic for this GEMM shape and bucket (memory, then disk once per environment), or None."""
+ env_hash, meta = env
+ table = _ensure_loaded(env_hash, meta)
+ got = table.get(_entry_key(k, n, bucket))
+ if got is not None:
+ _STATS["memory_hits"] += 1
+ return got
+
+
+def record(
+ env: tuple[str, dict],
+ k: int,
+ n: int,
+ bucket: int,
+ tactic: int,
+ *,
+ persist: bool = True,
+) -> int:
+ """Keep a tuned tactic (first writer wins in this process) and publish it; returns the kept one."""
+ env_hash, meta = env
+ if not _valid_tactic(tactic, meta):
+ raise ValueError(
+ f"tactic {tactic!r} is outside this build's table ({meta.get('tactic_count')})"
+ )
+ table = _ensure_loaded(env_hash, meta)
+ key = _entry_key(k, n, bucket)
+ with _LOCK:
+ fresh = key not in table
+ kept = table.setdefault(key, int(tactic))
+ if fresh:
+ _STATS["tuned"] += 1
+ if fresh and persist and _disk_ok(meta):
+ path = _path_for(env_hash)
+ if path is not None:
+ _persist(path, meta, {key: kept})
+ return kept
+
+
+def forget_memory() -> None:
+ """Drop the in-process tables (tests; the disk is the source of truth across processes)."""
+ with _LOCK:
+ _MEMORY.clear()
+ _LOADED.clear()
+ _IDENTITY.clear()
+ for key in _STATS:
+ _STATS[key] = 0
+
+
+def describe() -> dict:
+ return {
+ "mode": mode(),
+ "dir": None if cache_dir() is None else str(cache_dir()),
+ "entries": sum(len(t) for t in _MEMORY.values()),
+ **_STATS,
+ }
diff --git a/studio/backend/tests/test_diffusion_nvfp4_backend.py b/studio/backend/tests/test_diffusion_nvfp4_backend.py
index 44743ab537e..8fbb27a5119 100644
--- a/studio/backend/tests/test_diffusion_nvfp4_backend.py
+++ b/studio/backend/tests/test_diffusion_nvfp4_backend.py
@@ -429,6 +429,9 @@ def test_an_unprewarmed_gemm_shape_autotunes_on_its_first_eager_call_only(monkey
import sys
import types
+ # The per-length legacy mode; the bucketed default is covered in test_diffusion_nvfp4_tactics.py.
+ monkeypatch.setenv("UNSLOTH_NVFP4_TACTIC_CACHE", "0")
+
torch = pytest.importorskip("torch")
from core.inference import diffusion_nvfp4_linear as nl
@@ -493,6 +496,9 @@ def test_the_first_call_tune_runs_before_the_cached_dispatch_plan_is_built(monke
import sys
import types
+ # Legacy mode only: the bucketed plan tunes its own bucket (test_diffusion_nvfp4_tactics.py).
+ monkeypatch.setenv("UNSLOTH_NVFP4_TACTIC_CACHE", "0")
+
torch = pytest.importorskip("torch")
from core.inference import diffusion_nvfp4_dispatch as dispatch
from core.inference import diffusion_nvfp4_linear as nl
diff --git a/studio/backend/tests/test_diffusion_nvfp4_dispatch.py b/studio/backend/tests/test_diffusion_nvfp4_dispatch.py
index 25528b6eea9..00abd23c98a 100644
--- a/studio/backend/tests/test_diffusion_nvfp4_dispatch.py
+++ b/studio/backend/tests/test_diffusion_nvfp4_dispatch.py
@@ -344,7 +344,11 @@ def test_the_real_flashinfer_verifies_bit_identical():
@pytest.mark.parametrize("m,k,n", CUDA_SHAPES)
-def test_the_cached_dispatch_matches_the_public_api_exactly(m, k, n):
+def test_the_cached_dispatch_matches_the_public_api_exactly(m, k, n, monkeypatch):
+ # Exact-length mode: both arms then read the tactic from FlashInfer's own AutoTuner, so the bits
+ # must agree. The bucketed default picks its own (equally valid) tactic per bucket; it is covered
+ # in test_diffusion_nvfp4_tactics.py.
+ monkeypatch.setenv("UNSLOTH_NVFP4_TACTIC_CACHE", "0")
torch = _cuda_or_skip()
import flashinfer
diff --git a/studio/backend/tests/test_diffusion_nvfp4_tactics.py b/studio/backend/tests/test_diffusion_nvfp4_tactics.py
new file mode 100644
index 00000000000..562ffa0acc9
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_nvfp4_tactics.py
@@ -0,0 +1,606 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Bucketed, persisted mm_fp4 tactics: a new prompt length never retunes, a warm disk never tunes."""
+
+from __future__ import annotations
+
+import contextlib
+import dataclasses
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+import types
+from pathlib import Path
+from typing import Any, Callable
+
+import pytest
+
+from core.inference import diffusion_nvfp4_dispatch as dispatch
+from core.inference import diffusion_nvfp4_linear as nl
+from core.inference import diffusion_nvfp4_ops as ops
+from core.inference import diffusion_nvfp4_tactics as tactics
+
+torch = pytest.importorskip("torch")
+
+_BACKEND_DIR = Path(__file__).resolve().parents[1]
+
+
[email protected](slots = True, unsafe_hash = True)
+class _Spec:
+ input_idx: tuple
+ dim_idx: tuple
+ gen_tuning_buckets: Any
+ map_to_tuning_buckets: Callable
+
+
[email protected](kw_only = True, unsafe_hash = True)
+class _Config:
+ dynamic_tensor_specs: tuple
+ constraint_specs: tuple = ()
+ use_cuda_graph: bool = True
+
+
+def _pow2_floor(x: int) -> int:
+ return 1 << (max(int(x), 1).bit_length() - 1)
+
+
+def _pow2_buckets(m: int) -> tuple:
+ top, out = _pow2_floor(m), []
+ while top >= 1:
+ out.append(top)
+ top //= 2
+ return tuple(out)
+
+
+# FlashInfer 0.6.6's mm_fp4 config shape: all powers of two up to M, keyed by the floor power of two.
+_BASE_CONFIG = _Config(dynamic_tensor_specs = (_Spec((0,), (0,), _pow2_buckets, _pow2_floor),))
+
+_GPUS = {
+ 0: ((10, 0), "NVIDIA B200", 148),
+ 1: ((10, 0), "NVIDIA B200", 148),
+ 2: ((12, 0), "NVIDIA RTX PRO 6000 Blackwell", 188),
+}
+
+
+class _Device:
+ type = "cuda"
+
+ def __init__(self, index: int):
+ self.index = index
+
+ def __repr__(self): # pragma: no cover - debug aid
+ return f"cuda:{self.index}"
+
+
+class _T:
+ """Just enough tensor for the plan: a shape and a device."""
+
+ def __init__(self, shape, device):
+ self.shape = tuple(shape)
+ self.device = device
+
+
+class _Runner:
+ def __init__(self, count: int):
+ self.count = count
+ self.runs: list = []
+
+ def get_valid_tactics(self, inputs, profile):
+ return list(range(self.count))
+
+ def __call__(
+ self,
+ inputs,
+ tactic = -1,
+ ):
+ self.runs.append((inputs[0].shape[0], tactic))
+ return inputs[6]
+
+
+class _Fake:
+ """Records every profiling request FlashInfer would see."""
+
+ def __init__(
+ self,
+ *,
+ version = "0.7.3",
+ tactic_count = 32,
+ ):
+ self.version = version
+ self.runner = _Runner(tactic_count)
+ self.tunes: list = []
+ self.public_tunes: list = []
+ self.tuning = False
+ self.delay = 0.0
+ self.result: Callable = (
+ lambda k, n, bucket: (k // 16 + n // 16 + bucket) % self.runner.count
+ )
+
+ def install(self, monkeypatch):
+ fake = self
+
+ @contextlib.contextmanager
+ def autotune(flag = True, **_kw):
+ old = fake.tuning
+ fake.tuning = bool(flag)
+ try:
+ yield
+ finally:
+ fake.tuning = old
+
+ class _AutoTuner:
+ @staticmethod
+ def get():
+ return _AutoTuner
+
+ @staticmethod
+ def choose_one(op, runners, config, inputs):
+ spec = config.dynamic_tensor_specs[0]
+ m, half_k, n = inputs[0].shape[0], inputs[0].shape[1], inputs[6].shape[1]
+ if not fake.tuning:
+ return runners[0], -1
+ buckets = spec.gen_tuning_buckets
+ buckets = tuple(buckets) if isinstance(buckets, tuple) else tuple(buckets(m))
+ fake.tunes.append(
+ {
+ "op": op,
+ "m": m,
+ "k": half_k * 2,
+ "n": n,
+ "profiled": buckets,
+ "key_m": spec.map_to_tuning_buckets(m),
+ }
+ )
+ if fake.delay:
+ time.sleep(fake.delay)
+ return runners[0], fake.result(half_k * 2, n, buckets[0])
+
+ def mm_fp4(
+ a,
+ *args,
+ out = None,
+ **kwargs,
+ ):
+ if fake.tuning:
+ fake.public_tunes.append(a.shape[0])
+ return out
+
+ root = types.ModuleType("flashinfer")
+ root.__version__ = self.version
+ root.__path__ = []
+ root.autotune = autotune
+ root.mm_fp4 = mm_fp4
+ gemm = types.ModuleType("flashinfer.gemm")
+ gemm.__path__ = []
+ base = types.ModuleType("flashinfer.gemm.gemm_base")
+ base.DEFAULT_WORKSPACE_SIZE = 1 << 20
+ base._MM_FP4_TUNING_CONFIG_128x4 = _BASE_CONFIG
+ base._get_cache_buf = lambda name, size, device: ("workspace", device.index)
+ base.get_cutlass_fp4_gemm_module = lambda major, minor: types.SimpleNamespace(
+ cutlass_fp4_gemm_runner = lambda: fake.runner
+ )
+ autotuner = types.ModuleType("flashinfer.autotuner")
+ autotuner.AutoTuner = _AutoTuner
+ utils = types.ModuleType("flashinfer.utils")
+ utils.get_compute_capability = lambda device: _GPUS[device.index][0]
+ for name, module in {
+ "flashinfer": root,
+ "flashinfer.gemm": gemm,
+ "flashinfer.gemm.gemm_base": base,
+ "flashinfer.autotuner": autotuner,
+ "flashinfer.utils": utils,
+ }.items():
+ monkeypatch.setitem(sys.modules, name, module)
+ return self
+
+
+def fake(monkeypatch, tmp_path):
+ monkeypatch.setenv(tactics.NVFP4_TACTIC_CACHE_DIR_ENV, str(tmp_path / "tactics"))
+ monkeypatch.delenv(tactics.NVFP4_TACTIC_CACHE_ENV, raising = False)
+ monkeypatch.setattr(torch.cuda, "device", lambda device: contextlib.nullcontext())
+ monkeypatch.setattr(
+ torch.cuda,
+ "get_device_properties",
+ lambda index: types.SimpleNamespace(
+ name = _GPUS[index][1], multi_processor_count = _GPUS[index][2]
+ ),
+ )
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index: _GPUS[index][0])
+ monkeypatch.setattr(ops, "_is_capturing", lambda: False)
+ monkeypatch.setattr(dispatch, "enabled", lambda device: True)
+ _fresh_process()
+ yield _Fake().install(monkeypatch)
+ _fresh_process()
+
+
+def _fresh_process() -> None:
+ """Everything a restart forgets; only the cache directory survives."""
+ dispatch.reset()
+ tactics.forget_memory()
+ nl.reset_tuned_shapes()
+ dispatch._BUCKET_CONFIGS.clear()
+
+
+def _gemm(
+ m: int,
+ k: int,
+ n: int,
+ device: int = 0,
+):
+ dev = _Device(device)
+ return dispatch.gemm_plan(
+ _T((m, k // 2), dev),
+ _T((k // 2, n), dev),
+ _T((m, k // 16), dev),
+ _T((k // 16, n), dev),
+ _T((1,), dev),
+ _T((m, n), dev),
+ n,
+ "cutlass",
+ )
+
+
+# A FLUX-like DiT: joint img / txt streams, a fused single stream on the packed sequence, modulation.
+_STREAM = ((3072, 9216), (3072, 3072), (3072, 12288), (12288, 3072))
+_SINGLE = ((3072, 21504), (15360, 3072))
+_MOD = ((3072, 18432),)
+
+
+def _render(
+ height: int,
+ width: int,
+ text_tokens: int,
+ batch: int = 1,
+ device: int = 0,
+) -> set:
+ img = (height // 16) * (width // 16)
+ seen = set()
+ for m, shapes in (
+ (img * batch, _STREAM),
+ (text_tokens * batch, _STREAM),
+ ((img + text_tokens) * batch, _SINGLE),
+ (batch, _MOD),
+ ):
+ for k, n in shapes:
+ plan = _gemm(m, k, n, device)
+ assert plan is not None
+ seen.add((m, k, n))
+ return seen
+
+
+_RENDERS = [
+ (h, w, text, batch)
+ for (h, w) in ((512, 512), (768, 768), (1024, 1024), (1024, 768), (1344, 768))
+ for text in (17, 77, 93, 154, 230, 301, 377, 512)
+ for batch in (1, 2) # 2 = CFG batched
+]
+
+
+def test_the_bucket_ladder_is_a_floor_with_bounded_spacing():
+ last = 0
+ for m in range(1, 70000):
+ b = tactics.m_bucket(m)
+ assert 1 <= b <= m
+ assert b >= last # monotone
+ last = b
+ assert m < 2 * b
+ if m >= 512:
+ assert m < 1.5 * b
+ assert [
+ tactics.m_bucket(m) for m in (1, 2, 3, 255, 256, 511, 512, 767, 768, 1023, 4608, 9216)
+ ] == [
+ 1,
+ 2,
+ 2,
+ 128,
+ 256,
+ 256,
+ 512,
+ 512,
+ 768,
+ 768,
+ 4096,
+ 8192,
+ ]
+ # 1..128k tokens (up to long video) is 26 buckets in all.
+ assert len({tactics.m_bucket(m) for m in range(1, 131073)}) == 26
+
+
+def test_many_renders_with_many_lengths_tune_only_the_bucket_set(fake):
+ exact: set = set()
+ for h, w, text, batch in _RENDERS:
+ exact |= _render(h, w, text, batch)
+ expected = {(k, n, tactics.m_bucket(m)) for (m, k, n) in exact}
+ got = [(t["k"], t["n"], t["profiled"][0]) for t in fake.tunes]
+
+ assert sorted(got) == sorted(expected) # one tune per (shape, bucket), never a repeat
+ assert len(got) == len(set(got))
+ assert all(len(t["profiled"]) == 1 for t in fake.tunes) # one M profiled per tune, not a ladder
+ assert all(t["profiled"][0] == t["key_m"] == tactics.m_bucket(t["m"]) for t in fake.tunes)
+ assert all(t["op"] == dispatch.BUCKET_OP for t in fake.tunes)
+ assert fake.public_tunes == []
+ # The old path tuned once per exact length; this workload has far more lengths than buckets.
+ assert len(exact) > 3 * len(expected)
+
+
+def test_a_new_prompt_length_in_a_tuned_bucket_is_never_a_new_tune(fake):
+ _render(1024, 1024, 77)
+ before = len(fake.tunes)
+ for text in range(78, 512):
+ # img + text stays in [4096, 6144) and the text stream in its bucket's range.
+ if tactics.m_bucket(text) == tactics.m_bucket(77):
+ _render(1024, 1024, text)
+ assert len(fake.tunes) == before
+
+
+def test_every_length_in_a_bucket_runs_the_buckets_tactic(fake):
+ plans = {id(_gemm(m, 3072, 3072)) for m in range(4096, 6144, 37)}
+ assert len(plans) == 1
+ assert len(fake.tunes) == 1
+ tactic = _gemm(5000, 3072, 3072)[1]
+ assert tactic == fake.result(3072, 3072, 4096)
+
+
+def test_a_warm_disk_cache_means_zero_tunes_after_a_restart(fake, tmp_path):
+ for h, w, text, batch in _RENDERS[:12]:
+ _render(h, w, text, batch)
+ tuned = {
+ (t["k"], t["n"], t["profiled"][0]): fake.result(t["k"], t["n"], t["profiled"][0])
+ for t in fake.tunes
+ }
+ assert tuned
+
+ _fresh_process()
+ fake.tunes.clear()
+ for h, w, text, batch in _RENDERS[:12]:
+ _render(h, w, text, batch)
+ assert fake.tunes == []
+ for (k, n, bucket), tactic in tuned.items():
+ assert _gemm(bucket, k, n)[1] == tactic
+ assert tactics.describe()["disk_entries_loaded"] == len(tuned)
+ files = list((tmp_path / "tactics").glob("mm_fp4_*.json"))
+ assert len(files) == 1
+ data = json.loads(files[0].read_text(encoding = "utf-8"))
+ assert data["meta"]["device"] == "NVIDIA B200" and data["meta"]["tactic_count"] == 32
+ assert len(data["entries"]) == len(tuned)
+
+
+ "change",
+ ["flashinfer_version", "arch", "device_name", "tactic_table"],
+)
+def test_the_disk_cache_invalidates_on_any_identity_change(fake, monkeypatch, change):
+ _gemm(4608, 3072, 3072)
+ assert len(fake.tunes) == 1
+ _fresh_process()
+ if change == "flashinfer_version":
+ sys.modules["flashinfer"].__version__ = "0.6.6"
+ elif change == "arch":
+ monkeypatch.setitem(_GPUS, 0, ((10, 3), "NVIDIA B200", 148))
+ elif change == "device_name":
+ monkeypatch.setitem(_GPUS, 0, ((10, 0), "NVIDIA B300", 148))
+ else:
+ fake.runner.count = 40
+ _gemm(4608, 3072, 3072)
+ assert len(fake.tunes) == 2
+
+
+def test_a_new_weight_shape_is_its_own_tune(fake):
+ _gemm(4608, 3072, 3072)
+ _gemm(4608, 3072, 12288)
+ _gemm(4608, 12288, 3072)
+ _gemm(4700, 3072, 3072)
+ assert [(t["k"], t["n"]) for t in fake.tunes] == [(3072, 3072), (3072, 12288), (12288, 3072)]
+
+
+def test_a_corrupt_or_foreign_or_out_of_range_file_costs_a_retune_never_a_raise(fake, tmp_path):
+ _gemm(4608, 3072, 3072)
+ (path,) = (tmp_path / "tactics").glob("mm_fp4_*.json")
+
+ path.write_text("{not json", encoding = "utf-8")
+ _fresh_process()
+ _gemm(4608, 3072, 3072)
+ assert len(fake.tunes) == 2
+
+ data = json.loads(path.read_text(encoding = "utf-8"))
+ data["entries"] = {key: 999 for key in data["entries"]}
+ path.write_text(json.dumps(data), encoding = "utf-8")
+ _fresh_process()
+ _gemm(4608, 3072, 3072)
+ assert len(fake.tunes) == 3
+
+ data = json.loads(path.read_text(encoding = "utf-8"))
+ data["meta"]["device"] = "another card"
+ path.write_text(json.dumps(data), encoding = "utf-8")
+ _fresh_process()
+ _gemm(4608, 3072, 3072)
+ assert len(fake.tunes) == 4
+
+
+def test_concurrent_callers_on_one_bucket_tune_it_exactly_once(fake):
+ fake.delay = 0.05
+ barrier = threading.Barrier(8)
+ plans: list = []
+ errors: list = []
+
+ def _worker(i):
+ try:
+ barrier.wait()
+ plans.append(_gemm(4096 + 97 * i, 3072, 3072, device = i % 2))
+ except Exception as exc: # pragma: no cover - reported below
+ errors.append(exc)
+
+ threads = [threading.Thread(target = _worker, args = (i,)) for i in range(8)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ assert not errors
+ # Devices 0 and 1 are the same card model, so they share the one tuned entry.
+ assert len(fake.tunes) == 1
+ assert len({plan[1] for plan in plans}) == 1
+
+
+def test_concurrent_processes_merge_their_entries(tmp_path):
+ root = tmp_path / "shared"
+ script = (
+ "import sys\n"
+ "from core.inference import diffusion_nvfp4_tactics as t\n"
+ "env = t.environment(flashinfer_version='0.7.3', capability=(10, 0), device_name='B200',"
+ " sm_count=148, cuda='13.0', tactic_count=32)\n"
+ "w = int(sys.argv[1])\n"
+ "for i in range(20):\n"
+ " t.record(env, 3072, 3072 + 32 * i, 1 << w, (w + i) % 32)\n"
+ )
+ env = dict(os.environ, **{tactics.NVFP4_TACTIC_CACHE_DIR_ENV: str(root)})
+ env.pop(tactics.NVFP4_TACTIC_CACHE_ENV, None)
+ procs = [
+ subprocess.Popen([sys.executable, "-c", script, str(w)], cwd = str(_BACKEND_DIR), env = env)
+ for w in range(6)
+ ]
+ assert [p.wait(timeout = 120) for p in procs] == [0] * 6
+ (path,) = root.glob("mm_fp4_*.json")
+ entries = json.loads(path.read_text(encoding = "utf-8"))["entries"]
+ assert len(entries) == 6 * 20
+ assert not list(root.glob(".*.tmp"))
+
+
+def test_different_cards_in_one_process_keep_separate_tables(fake):
+ _gemm(4608, 3072, 3072, device = 0)
+ _gemm(4608, 3072, 3072, device = 1) # same model as 0: shared
+ _gemm(4608, 3072, 3072, device = 2) # sm_120: its own tune
+ assert len(fake.tunes) == 2
+
+
+def test_a_cold_bucket_under_capture_neither_builds_nor_tunes(fake, monkeypatch):
+ monkeypatch.setattr(ops, "_is_capturing", lambda: True)
+ assert _gemm(4608, 3072, 3072) is None
+ assert fake.tunes == []
+
+
+def test_a_bucket_tuned_on_disk_serves_a_capture_without_any_tune(fake, monkeypatch):
+ _gemm(4608, 3072, 3072) # tunes + persists
+ _fresh_process()
+ _gemm(1024, 3072, 3072) # another bucket: builds the runner, tunes 1024
+ assert len(fake.tunes) == 2
+ monkeypatch.setattr(ops, "_is_capturing", lambda: True)
+ plan = _gemm(5000, 3072, 3072) # 4096 bucket: on disk, plan from cached parts
+ assert plan is not None and plan[1] == fake.result(3072, 3072, 4096)
+ assert len(fake.tunes) == 2
+
+
+def test_memory_mode_never_writes(fake, monkeypatch, tmp_path):
+ monkeypatch.setenv(tactics.NVFP4_TACTIC_CACHE_ENV, "memory")
+ _gemm(4608, 3072, 3072)
+ _gemm(4700, 3072, 3072)
+ assert len(fake.tunes) == 1
+ assert not (tmp_path / "tactics").exists()
+
+
+def test_the_kill_switch_restores_the_exact_length_plans(fake, monkeypatch):
+ monkeypatch.setenv(tactics.NVFP4_TACTIC_CACHE_ENV, "0")
+ _gemm(4608, 3072, 3072)
+ _gemm(4700, 3072, 3072)
+ assert fake.tunes == [] # legacy plans only look up FlashInfer's own cache
+ assert (4608, 1536, 3072, "cutlass", 0) in dispatch._GEMM_PLAN
+ assert (4700, 1536, 3072, "cutlass", 0) in dispatch._GEMM_PLAN
+
+
+def test_a_failed_tune_is_never_persisted(fake, tmp_path):
+ fake.result = lambda k, n, bucket: -1 # every tactic failed (or an OOM) inside FlashInfer
+ plan = _gemm(4608, 3072, 3072)
+ assert plan[1] == -1
+ assert not list((tmp_path / "tactics").glob("mm_fp4_*.json"))
+ _gemm(4700, 3072, 3072)
+ assert len(fake.tunes) == 1 # not retried in this process either
+
+
+def test_a_tune_that_raises_falls_back_once_and_is_not_persisted(fake, tmp_path):
+ def _boom(*_a, **_kw):
+ raise RuntimeError("profiling failed")
+
+ fake.result = _boom
+ assert _gemm(4608, 3072, 3072)[1] == -1
+ assert _gemm(4700, 3072, 3072)[1] == -1
+ assert len(fake.tunes) == 1
+ assert not list((tmp_path / "tactics").glob("mm_fp4_*.json"))
+
+
+def test_verify_keeps_the_autotuner_lookup_and_caches_nothing(fake):
+ dev = _Device(0)
+ plan = dispatch.gemm_plan(
+ _T((256, 256), dev), None, None, None, None, _T((256, 128), dev), 128, "cutlass", force = True
+ )
+ assert plan is not None and plan[1] == -1
+ assert fake.tunes == [] and dispatch._GEMM_PLAN == {}
+
+
+def test_the_bucket_config_profiles_one_m_and_keys_every_m_to_it():
+ config = dispatch.bucket_config(_BASE_CONFIG, 4096)
+ assert dispatch.bucket_config(_BASE_CONFIG, 4096) is config # stable for FlashInfer's LRU
+ spec = config.dynamic_tensor_specs[0]
+ assert spec.gen_tuning_buckets == (4096,)
+ assert {spec.map_to_tuning_buckets(m) for m in (1, 4096, 5000, 6143)} == {4096}
+ assert spec.input_idx == (0,) and spec.dim_idx == (0,)
+ assert (
+ config.use_cuda_graph is True and config.constraint_specs == _BASE_CONFIG.constraint_specs
+ )
+ assert _BASE_CONFIG.dynamic_tensor_specs[0].gen_tuning_buckets is _pow2_buckets # untouched
+
+
+def _mm_through_the_op(monkeypatch, m: int, k: int, n: int):
+ monkeypatch.setattr(ops, "_device_guard", lambda t: contextlib.nullcontext())
+ monkeypatch.setattr(ops, "_fire_barrier", lambda device: None)
+ monkeypatch.setattr(ops, "_device_index", lambda device: 0)
+ monkeypatch.setattr(dispatch, "transposed", lambda t: t)
+ xq = torch.zeros(m, k // 2, dtype = torch.uint8)
+ wq = torch.zeros(n, k // 2, dtype = torch.uint8)
+ return ops._mm_impl(xq, wq, None, wq, None, n, "cutlass")
+
+
+def test_the_op_takes_the_bucketed_plan_and_never_the_public_tune(fake, monkeypatch):
+ monkeypatch.setattr(
+ tactics,
+ "device_environment",
+ lambda device, count, backend = "cutlass": tactics.environment(
+ flashinfer_version = "0.7.3",
+ capability = (10, 0),
+ device_name = "NVIDIA B200",
+ sm_count = 148,
+ cuda = "13.0",
+ tactic_count = count,
+ ),
+ )
+ monkeypatch.setattr(dispatch, "_runner", lambda device, index: (fake.runner, "ws", 32))
+ for m in (4100, 4608, 5000, 6000):
+ _mm_through_the_op(monkeypatch, m, 64, 64)
+ assert fake.public_tunes == []
+ assert len(fake.tunes) == 1
+ assert [tactic for _, tactic in fake.runner.runs] == [fake.result(64, 64, 4096)] * 4
+
+
+def test_the_public_path_claims_flashinfers_own_bucket_not_the_exact_length(fake, monkeypatch):
+ monkeypatch.setattr(dispatch, "enabled", lambda device: False)
+ for m in (4100, 4608, 5000, 8191, 8192, 9000):
+ _mm_through_the_op(monkeypatch, m, 64, 64)
+ # 0.6.6-shaped config: floor power of two, so 4100..8191 share one tune and 8192.. the next.
+ assert fake.public_tunes == [4100, 8192]
+
+
+def test_the_public_path_under_the_kill_switch_claims_every_length(fake, monkeypatch):
+ monkeypatch.setenv(tactics.NVFP4_TACTIC_CACHE_ENV, "0")
+ monkeypatch.setattr(dispatch, "enabled", lambda device: False)
+ for m in (4100, 4608, 4608):
+ _mm_through_the_op(monkeypatch, m, 64, 64)
+ assert fake.public_tunes == [4100, 4608]
+
+
+def test_describe_reports_the_tactic_cache(fake):
+ _gemm(4608, 3072, 3072)
+ record = dispatch.describe()["tactics"]
+ assert record["mode"] == "auto" and record["tuned"] == 1 and record["entries"] == 1