Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE fallback kernel configs: fit the device shared memory for float32 operands (#14576)

Daniel Han039181491734
diff --git a/tests/test_moe_fallback_smem.py b/tests/test_moe_fallback_smem.py
new file mode 100644
index 00000000000..7e5abf5d79e
--- /dev/null
+++ b/tests/test_moe_fallback_smem.py
@@ -0,0 +1,333 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Fallback MoE configs fit shared memory for float32 X / dY (default dX needed 163,840 B > L4's 101,376 B).
+
+Byte counts are Triton metadata.shared cross-compiled on CPU (Triton 3.6-3.8)."""
+
+import importlib
+import inspect
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+
+torch = pytest.importorskip("torch")
+triton = pytest.importorskip("triton")
+pytest.importorskip("pandas")
+
+MOE_DIR = ROOT / "unsloth" / "kernels" / "moe"
+
+GPUS = {
+ "T4": ((7, 5), 65536),
+ "L4": ((8, 9), 101376),
+ "RTX4090": ((8, 9), 101376),
+ "A10": ((8, 6), 101376),
+ "A100": ((8, 0), 166912),
+ "H100": ((9, 0), 232448),
+ "B200": ((10, 0), 232448),
+ "RTX5090": ((12, 0), 101376),
+}
+F32, BF16, F16 = torch.float32, torch.bfloat16, torch.float16
+DTYPES = [
+ (BF16, None, BF16),
+ (F16, None, F16),
+ (F32, BF16, BF16),
+ (F32, F16, F16),
+ (F32, None, F32),
+ (F32, F32, F32),
+]
+# (hidden, moe_intermediate)
+SHAPES = [(2048, 768), (2048, 1408), (1024, 352), (768, 2048)]
+OLD_HEURISTIC = ((64, 128, 128, 8, 3), (64, 64, 256, 8, 4), (64, 64, 256, 8, 4))
+OLD_DEFAULT = ((128, 128, 64, 8, 3),) * 3
+
+# Main's float32-activation configs: (capability major, kind, activation, weight, (M, N, K, num_stages)) -> bytes.
+MAIN_FLOAT32_COMPILED = {
+ (8, "dX", "fp32", "bf16", (128, 128, 64, 3)): 163840, # default dX, the L4 crash
+ (8, "fwd", "fp32", "bf16", (64, 128, 128, 3)): 131072, # heuristic forward
+ (8, "dX", "fp32", "bf16", (64, 64, 256, 4)): 147456, # heuristic dX
+ (8, "dX", "fp32", "fp32", (64, 64, 256, 4)): 245760,
+ (8, "fwd", "fp32", "fp32", (64, 128, 128, 3)): 196608,
+ (9, "dX", "fp32", "fp32", (64, 64, 256, 4)): 327680, # over the H100 limit too
+ (9, "fwd", "fp32", "fp32", (64, 128, 128, 3)): 294912,
+}
+
+
+def ac(monkeypatch):
+ for name, path in (
+ ("unsloth", ROOT / "unsloth"),
+ ("unsloth.kernels", ROOT / "unsloth" / "kernels"),
+ ("unsloth.kernels.moe", MOE_DIR),
+ ):
+ pkg = types.ModuleType(name)
+ pkg.__path__ = [str(path)]
+ monkeypatch.setitem(sys.modules, name, pkg)
+ for name in list(sys.modules):
+ if name.startswith("unsloth.kernels.moe."):
+ monkeypatch.delitem(sys.modules, name)
+ module = importlib.import_module("unsloth.kernels.moe.autotune_cache")
+ yield module
+ for name in list(sys.modules):
+ if name.startswith("unsloth.kernels.moe."):
+ sys.modules.pop(name, None)
+
+
+def _device(ac, monkeypatch, capability, smem):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: capability)
+ monkeypatch.setattr(ac, "_device_smem_per_block", lambda: smem)
+
+
+def _gemm_dims(hidden, inter):
+ return [(2 * inter, hidden), (hidden, inter)]
+
+
+def _tiles(configs):
+ return tuple(
+ (c.BLOCK_SIZE_M, c.BLOCK_SIZE_N, c.BLOCK_SIZE_K, c.num_warps, c.num_stages) for c in configs
+ )
+
+
+def _pick(ac, which, n, k, act, weight):
+ fn = ac._get_heuristic_configs if which == "heuristic" else ac._get_default_configs
+ return fn(n_dim = n, k_dim = k, dtype = act, weight_dtype = weight)
+
+
[email protected]("gpu", list(GPUS))
[email protected]("act,weight,seen", DTYPES)
[email protected]("which", ["heuristic", "default"])
+def test_picked_configs_fit_the_device(ac, monkeypatch, gpu, act, weight, seen, which):
+ capability, smem = GPUS[gpu]
+ _device(ac, monkeypatch, capability, smem)
+ for hidden, inter in SHAPES:
+ for n, k in _gemm_dims(hidden, inter):
+ for kind, c in zip(("fwd", "dX", "dW"), _pick(ac, which, n, k, act, weight)):
+ need = ac._estimate_smem(kind, c, act.itemsize, seen.itemsize, capability)
+ assert need <= smem, (gpu, which, kind, act, seen, n, k, _tiles([c]))
+ assert n % c.BLOCK_SIZE_N == 0 and k % c.BLOCK_SIZE_K == 0
+ assert min(c.BLOCK_SIZE_M, c.BLOCK_SIZE_N, c.BLOCK_SIZE_K) >= 16
+ key = (
+ capability[0] if capability[0] in (8, 9) else None,
+ kind,
+ "fp32" if act == F32 else "16",
+ "fp32" if seen == F32 else ("bf16" if seen == BF16 else "16"),
+ (c.BLOCK_SIZE_M, c.BLOCK_SIZE_N, c.BLOCK_SIZE_K, c.num_stages),
+ )
+ compiled = MAIN_FLOAT32_COMPILED.get(key)
+ assert compiled is None or compiled <= smem, (gpu, key, compiled)
+
+
[email protected]("gpu", list(GPUS))
[email protected]("act", [BF16, F16])
[email protected]("which", ["heuristic", "default"])
+def test_16bit_configs_unchanged(ac, monkeypatch, gpu, act, which):
+ capability, smem = GPUS[gpu]
+ for hidden, inter in SHAPES:
+ for n, k in _gemm_dims(hidden, inter):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ unfitted = _tiles(_pick(ac, which, n, k, None, None))
+ _device(ac, monkeypatch, capability, smem)
+ expected = list(unfitted)
+ # main's only smem step-down: the 4-stage heuristic dX (122,880 B) to 3 stages on sm_8x / sm_12x.
+ if which == "heuristic" and smem < 122880 and capability[0] >= 8:
+ if unfitted[1] == (64, 64, 256, 8, 4):
+ expected[1] = (64, 64, 256, 8, 3)
+ assert _tiles(_pick(ac, which, n, k, act, None)) == tuple(expected), (gpu, n, k)
+ assert _tiles(_pick(ac, which, n, k, None, None)) == tuple(expected), (gpu, n, k)
+
+
+def test_l4_float32_default_dx_no_longer_over_limit(ac, monkeypatch):
+ # The configuration that crashed: autotune failed, float32 dY, bf16 W, L4.
+ _device(ac, monkeypatch, (8, 9), 101376)
+ _, dx, _ = _pick(ac, "default", 1536, 2048, F32, BF16)
+ assert (dx.BLOCK_SIZE_M, dx.BLOCK_SIZE_N, dx.BLOCK_SIZE_K, dx.num_stages) != (128, 128, 64, 3)
+ assert ac._estimate_smem("dX", dx, 4, 2, (8, 9)) <= 101376
+ # Unknown weight dtype: sized for float32 W, which also fits bf16 W.
+ _, dx32, _ = _pick(ac, "default", 1536, 2048, F32, None)
+ assert ac._estimate_smem("dX", dx32, 4, 4, (8, 9)) <= 101376
+
+
+def test_unknown_smem_or_no_cuda_keeps_configs(ac, monkeypatch):
+ _device(ac, monkeypatch, (8, 9), None)
+ assert _tiles(_pick(ac, "heuristic", 1536, 2048, F32, BF16)) == OLD_HEURISTIC
+ assert _tiles(_pick(ac, "default", 1536, 2048, F32, BF16)) == OLD_DEFAULT
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ assert _tiles(_pick(ac, "default", 1536, 2048, F32, BF16)) == OLD_DEFAULT
+
+
[email protected]("env", ["1", "0"])
+def test_get_or_autotune_passes_dtypes(ac, monkeypatch, env):
+ _device(ac, monkeypatch, (8, 9), 101376)
+ monkeypatch.setenv("UNSLOTH_MOE_DISABLE_AUTOTUNE", env)
+ monkeypatch.setattr(ac, "load_cached_config", lambda key: None)
+ monkeypatch.setattr(ac, "save_cached_config", lambda *a, **k: None)
+ seen = {}
+
+ def tune(*args):
+ seen["args"] = args
+ raise RuntimeError("tuning failed")
+
+ monkeypatch.setattr(ac, "_run_moe_autotuning", tune)
+ ac.clear_cache()
+ configs = ac.get_or_autotune_moe_kernels(
+ num_experts = 8,
+ hidden_dim = 2048,
+ intermediate_dim = 768,
+ top_k = 2,
+ dtype = F32,
+ weight_dtype = BF16,
+ )
+ for kind, c in zip(("fwd", "dX", "dW"), configs):
+ assert ac._estimate_smem(kind, c, 4, 2, (8, 9)) <= 101376
+ if env == "0":
+ assert seen["args"][4] == F32 and seen["args"][-1] == BF16
+
+
+def test_cache_key_unchanged_without_a_distinct_weight_dtype(ac):
+ base = ac._get_cache_key(8, 2048, 768, 2, BF16, (9, 0), 8192)
+ assert ac._get_cache_key(8, 2048, 768, 2, BF16, (9, 0), 8192, None) == base
+ assert ac._get_cache_key(8, 2048, 768, 2, BF16, (9, 0), 8192, BF16) == base
+ f32 = ac._get_cache_key(8, 2048, 768, 2, F32, (9, 0), 8192)
+ assert ac._get_cache_key(8, 2048, 768, 2, F32, (9, 0), 8192, BF16) != f32
+
+
+# Recompile a sample (CPU only, needs Triton's CUDA backend + ptxas).
+
+
+def _compile_shared(fn, ptr_types, consts, arch, num_warps, num_stages):
+ from triton.backends.compiler import GPUTarget
+ from triton.compiler import ASTSource
+
+ params = list(inspect.signature(fn.fn).parameters)
+ signature, constexprs = {}, {}
+ for i, p in enumerate(params):
+ if p in ptr_types:
+ signature[p] = ptr_types[p]
+ elif p in ("NUM_TOKENS", "NUM_SMS"):
+ signature[p] = "i32"
+ else:
+ signature[p] = "constexpr"
+ constexprs[(i,)] = consts.get(p, inspect.signature(fn.fn).parameters[p].default)
+ # A real launch specializes 16 B aligned pointers; without it cp.async is not used.
+ attrs = {(i,): [["tt.divisibility", 16]] for i, p in enumerate(params) if p in ptr_types}
+ kernel = triton.compile(
+ ASTSource(fn, signature, constexprs, attrs),
+ target = GPUTarget("cuda", arch, 32),
+ options = dict(num_warps = num_warps, num_stages = num_stages),
+ )
+ return kernel.metadata.shared
+
+
+SAMPLE_GPUS = ["L4", "A100", "H100", "B200", "RTX5090"]
+
+
[email protected]("gpu", SAMPLE_GPUS)
+def test_estimate_matches_compiled_kernels(ac, monkeypatch, gpu):
+ try:
+ from triton.backends.compiler import GPUTarget # noqa: F401
+ from triton.compiler import ASTSource # noqa: F401
+ from unsloth.kernels.moe.grouped_gemm.kernels import backward, forward
+ except Exception as e:
+ pytest.skip(f"Triton CUDA cross compile unavailable: {e}")
+ capability, smem = GPUS[gpu]
+ arch = capability[0] * 10 + capability[1]
+ _device(ac, monkeypatch, capability, smem)
+ names = {F32: "*fp32", BF16: "*bf16", F16: "*fp16"}
+ n, k = 1536, 2048 # Qwen3-30B-A3B gate_up
+ try:
+ _compile_shared(
+ forward._grouped_gemm_forward_kernel,
+ dict(
+ x_ptr = "*bf16",
+ w_ptr = "*bf16",
+ y_ptr = "*bf16",
+ m_sizes_ptr = "*i32",
+ gather_indices_ptr = "*i64",
+ topk_weights_ptr = "*bf16",
+ ),
+ dict(
+ NUM_EXPERTS = 8,
+ TOPK = 2,
+ N = n,
+ K = k,
+ BLOCK_SIZE_M = 32,
+ BLOCK_SIZE_N = 32,
+ BLOCK_SIZE_K = 32,
+ PERMUTE_X = True,
+ PERMUTE_Y = False,
+ ),
+ arch,
+ 4,
+ 2,
+ )
+ except Exception as e:
+ pytest.skip(f"Triton cannot cross compile on this host: {type(e).__name__}: {e}")
+ for act, weight, seen in [(BF16, None, BF16), (F32, BF16, BF16), (F32, None, F32)]:
+ for which in ("heuristic", "default"):
+ for kind, c in zip(("fwd", "dX", "dW"), _pick(ac, which, n, k, act, weight)):
+ consts = dict(
+ NUM_EXPERTS = 8,
+ TOPK = 2,
+ N = n,
+ K = k,
+ BLOCK_SIZE_M = c.BLOCK_SIZE_M,
+ BLOCK_SIZE_N = c.BLOCK_SIZE_N,
+ BLOCK_SIZE_K = c.BLOCK_SIZE_K,
+ PERMUTE_X = True,
+ PERMUTE_Y = False,
+ )
+ a, w = names[act], names[seen]
+ if kind == "fwd":
+ compiled = _compile_shared(
+ forward._grouped_gemm_forward_kernel,
+ dict(
+ x_ptr = a,
+ w_ptr = w,
+ y_ptr = a,
+ m_sizes_ptr = "*i32",
+ gather_indices_ptr = "*i64",
+ topk_weights_ptr = a,
+ ),
+ consts,
+ arch,
+ c.num_warps,
+ c.num_stages,
+ )
+ elif kind == "dX":
+ compiled = _compile_shared(
+ backward._grouped_gemm_dX_kernel,
+ dict(
+ dY_ptr = a,
+ w_ptr = w,
+ dX_ptr = a,
+ gather_indices_ptr = "*i64",
+ m_sizes_ptr = "*i32",
+ ),
+ consts,
+ arch,
+ c.num_warps,
+ c.num_stages,
+ )
+ else:
+ compiled = _compile_shared(
+ backward._grouped_gemm_dW_per_expert_kernel,
+ dict(
+ x_ptr = a,
+ dY_ptr = a,
+ dW_ptr = w,
+ m_sizes_ptr = "*i32",
+ gather_indices_ptr = "*i64",
+ ),
+ dict(consts, EXPERTS_POW2 = 8),
+ arch,
+ c.num_warps,
+ 1 if act == F32 else c.num_stages,
+ )
+ estimate = ac._estimate_smem(kind, c, act.itemsize, seen.itemsize, capability)
+ assert compiled <= smem, (gpu, which, kind, act, seen, _tiles([c]), compiled)
+ assert compiled <= estimate, (gpu, which, kind, act, seen, _tiles([c]), compiled)
diff --git a/tests/test_moe_fallback_tile_dims.py b/tests/test_moe_fallback_tile_dims.py
index 7e817b20b38..141334ba37b 100644
--- a/tests/test_moe_fallback_tile_dims.py
+++ b/tests/test_moe_fallback_tile_dims.py
@@ -182,13 +182,18 @@ def test_heuristic_shrunk_dx_is_not_stepped(ac, monkeypatch):
assert (dx.BLOCK_SIZE_K, dx.num_stages) == (128, 4)
[email protected]("smem", [None, 65536])
-def test_heuristic_unknown_or_tiny_smem_keeps_config(ac, monkeypatch, smem):
- # Unknown limit, or one the step-down does not fit either (81,920 B): keep today's config.
- _device(ac, monkeypatch, (8, 9), smem)
+def test_heuristic_unknown_smem_keeps_config(ac, monkeypatch):
+ _device(ac, monkeypatch, (8, 9), None)
assert _tiles(ac._get_heuristic_configs(n_dim = 1536, k_dim = 2048)) == OLD_HEURISTIC
+def test_heuristic_tiny_smem_steps_down_until_it_fits(ac, monkeypatch):
+ _device(ac, monkeypatch, (8, 9), 65536)
+ configs = ac._get_heuristic_configs(n_dim = 1536, k_dim = 2048)
+ for kind, c in zip(("fwd", "dX", "dW"), configs):
+ assert ac._estimate_smem(kind, c, 2, 2, (8, 9)) <= 65536
+
+
def test_default_configs_not_stepped(ac, monkeypatch):
_device(ac, monkeypatch, (8, 9), 101376)
assert _tiles(ac._get_default_configs(n_dim = 1536, k_dim = 2048)) == OLD_DEFAULT
diff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.py
index 95f2360337c..e12a4030a7a 100644
--- a/unsloth/kernels/moe/autotune_cache.py
+++ b/unsloth/kernels/moe/autotune_cache.py
@@ -38,6 +38,7 @@ def _get_cache_key(
dtype: torch.dtype,
device_capability: Tuple[int, int],
seq_len: int = 8192, # Default sequence length for tuning
+ weight_dtype: Optional[torch.dtype] = None,
) -> str:
"""Generate a unique cache key based on model configuration."""
key_data = {
@@ -49,6 +50,8 @@ def _get_cache_key(
"device_capability": device_capability,
"seq_len": seq_len,
}
+ if weight_dtype is not None and weight_dtype != dtype:
+ key_data["weight_dtype"] = str(weight_dtype)
key_str = json.dumps(key_data, sort_keys = True)
return hashlib.md5(key_str.encode()).hexdigest()
@@ -133,6 +136,7 @@ def get_or_autotune_moe_kernels(
dtype: torch.dtype,
force_autotune: bool = False,
seq_len: int = 8192,
+ weight_dtype: Optional[torch.dtype] = None,
) -> Tuple[Any, Any, Any]:
"""
Get cached kernel configurations or run auto-tuning.
@@ -142,7 +146,8 @@ def get_or_autotune_moe_kernels(
hidden_dim: Hidden dimension of the model
intermediate_dim: Intermediate dimension for MoE MLP
top_k: Number of experts to route to
- dtype: Data type for computation
+ dtype: Data type of the activations (X and dY)
+ weight_dtype: Data type of the expert weights, if different from dtype
force_autotune: Force re-running autotuning even if cache exists
seq_len: Sequence length to use for tuning benchmarks
@@ -158,6 +163,7 @@ def get_or_autotune_moe_kernels(
dtype,
device_capability,
seq_len,
+ weight_dtype,
)
# Env override to disable autotuning
@@ -165,7 +171,9 @@ def get_or_autotune_moe_kernels(
logger.info(
f"UNSLOTH_MOE_DISABLE_AUTOTUNE=1: Using Heuristic (Safe) MoE kernel configs for SM{device_capability[0]}{device_capability[1]}"
)
- return _get_heuristic_configs(n_dim = 2 * intermediate_dim, k_dim = hidden_dim)
+ return _get_heuristic_configs(
+ n_dim = 2 * intermediate_dim, k_dim = hidden_dim, dtype = dtype, weight_dtype = weight_dtype
+ )
if not force_autotune and cache_key in _kernel_config_cache:
logger.info(f"Using in-memory cached MoE kernel configs: {cache_key}")
return _kernel_config_cache[cache_key]
@@ -201,7 +209,7 @@ def get_or_autotune_moe_kernels(
try:
configs = _run_moe_autotuning(
- num_experts, hidden_dim, intermediate_dim, top_k, dtype, seq_len
+ num_experts, hidden_dim, intermediate_dim, top_k, dtype, seq_len, weight_dtype
)
_kernel_config_cache[cache_key] = configs
@@ -230,7 +238,9 @@ def get_or_autotune_moe_kernels(
"Unsloth: Your Triton version might be incompatible with TMA features. Falling back to default configs."
)
logger.info("Falling back to default kernel configurations")
- return _get_default_configs(n_dim = 2 * intermediate_dim, k_dim = hidden_dim)
+ return _get_default_configs(
+ n_dim = 2 * intermediate_dim, k_dim = hidden_dim, dtype = dtype, weight_dtype = weight_dtype
+ )
def _run_moe_autotuning(
@@ -240,6 +250,7 @@ def _run_moe_autotuning(
top_k: int,
dtype: torch.dtype,
seq_len: int,
+ weight_dtype: Optional[torch.dtype] = None,
) -> Tuple[Any, Any, Any]:
"""Run the actual auto-tuning for MoE kernels."""
@@ -251,10 +262,11 @@ def _run_moe_autotuning(
hidden_states = torch.randn(num_tokens, hidden_dim, device = device, dtype = dtype)
gate_up_weights = torch.randn(
- num_experts, 2 * intermediate_dim, hidden_dim, device = device, dtype = dtype
- )
- down_weights = torch.randn(
- num_experts, hidden_dim, intermediate_dim, device = device, dtype = dtype
+ num_experts,
+ 2 * intermediate_dim,
+ hidden_dim,
+ device = device,
+ dtype = dtype if weight_dtype is None else weight_dtype,
)
# Dummy routing data
@@ -410,14 +422,6 @@ def _fit_configs_to_dims(
return configs
-# Triton metadata.shared (16-bit) of the 4-stage 64x64x256 dX: 122,880 B, over the 101,376 B of L4 / RTX PRO 6000;
-# 3 stages need 81,920 B. sm_75 needs 65,536 B (no cp.async multi-buffering), so it is excluded.
-# kind -> {(BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages): (bytes, fitting num_stages, its bytes)}
-_HEURISTIC_SMEM_STEP_DOWN = {
- "dX": {(64, 64, 256, 4): (122880, 3, 81920)},
-}
-
-
def _device_smem_per_block() -> Optional[int]:
# Read per current device: the launching card on a mixed host.
try:
@@ -435,30 +439,116 @@ def _device_smem_per_block() -> Optional[int]:
return None
-def _fit_heuristic_configs_to_smem(configs: Tuple[Any, Any, Any]) -> Tuple[Any, Any, Any]:
+def _itemsize(dtype: Optional[torch.dtype], default: int = 2) -> int:
+ return default if dtype is None else dtype.itemsize
+
+
+def _estimate_smem(
+ kind: str, config: Any, act_bytes: int, weight_bytes: int, capability: Tuple[int, int]
+) -> int:
+ """Shared memory (B) per launch; matches compiled metadata.shared (Triton 3.6-3.8, sm_75..120) or exceeds it by <= 1 KiB."""
+ M, N, K = config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K
+ stages = max(int(config.num_stages), 1)
+ if kind == "fwd":
+ a, b, a_cast = M * K * act_bytes, N * K * weight_bytes, act_bytes != weight_bytes
+ elif kind == "dX":
+ a, b, a_cast = M * N * act_bytes, N * K * weight_bytes, act_bytes != weight_bytes
+ else:
+ # The launcher runs float32 X single stage.
+ a, b, a_cast = M * N * act_bytes, M * K * act_bytes, False
+ if act_bytes >= 4:
+ stages = 1
+ major = capability[0]
+ if major < 8:
+ # No cp.async multi-buffering; 16-bit dot operands add a 2x larger-tile scratch.
+ dot_bytes = act_bytes if kind == "dW" else weight_bytes
+ if dot_bytes >= 4:
+ return a + b
+ return max(a + b, 2 * max(a * dot_bytes // act_bytes, b))
+ hopper_like = major in (9, 10, 11)
+ if kind == "dW":
+ if stages > 4:
+ buffers = stages if hopper_like else stages - 1
+ else:
+ buffers = min(stages, 2) if hopper_like else 1
+ return buffers * (a + b) + (2048 if hopper_like else 1024)
+ if hopper_like:
+ # A cast operand keeps num_stages - 1 buffers plus one copy in the dot dtype.
+ if a_cast:
+ return max(stages - 1, 1) * a + stages * b + a * weight_bytes // act_bytes + 64
+ return stages * (a + b) + 64
+ return max(stages - 1, 1) * (a + b)
+
+
+_LOOP_DIM = {"fwd": "BLOCK_SIZE_K", "dX": "BLOCK_SIZE_N", "dW": "BLOCK_SIZE_M"}
+_MIN_SMEM_BLOCK = 16
+
+
+def _fit_config_to_smem(
+ kind: str,
+ config: Any,
+ act_bytes: int,
+ weight_bytes: int,
+ capability: Tuple[int, int],
+ smem: int,
+) -> Any:
+ # Halving a power-of-two tile keeps it dividing N / K.
+ def fits():
+ return _estimate_smem(kind, config, act_bytes, weight_bytes, capability) <= smem
+
+ before = (config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K, config.num_stages)
+ loop_dim = _LOOP_DIM[kind]
+ while not fits():
+ if config.num_stages > 3:
+ config.num_stages -= 1
+ elif getattr(config, loop_dim) > 32:
+ setattr(config, loop_dim, getattr(config, loop_dim) // 2)
+ elif config.num_stages > 2:
+ config.num_stages -= 1
+ else:
+ name = max(
+ ("BLOCK_SIZE_M", "BLOCK_SIZE_N", "BLOCK_SIZE_K"), key = lambda n: getattr(config, n)
+ )
+ if getattr(config, name) > _MIN_SMEM_BLOCK:
+ setattr(config, name, getattr(config, name) // 2)
+ elif config.num_stages > 1:
+ config.num_stages = 1
+ else:
+ break
+ after = (config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K, config.num_stages)
+ if after != before:
+ logger.info(
+ f"MoE {kind} config (M, N, K, stages) {before} -> {after} to fit {smem} B shared memory"
+ )
+ return config
+
+
+def _fit_configs_to_smem(
+ configs: Tuple[Any, Any, Any],
+ dtype: Optional[torch.dtype] = None,
+ weight_dtype: Optional[torch.dtype] = None,
+) -> Tuple[Any, Any, Any]:
try:
if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
return configs
- if torch.cuda.get_device_capability()[0] < 8:
- return configs
+ capability = tuple(torch.cuda.get_device_capability())
except Exception:
return configs
smem = _device_smem_per_block()
if smem is None:
return configs
+ act_bytes = _itemsize(dtype)
+ weight_bytes = _itemsize(weight_dtype, act_bytes)
for kind, config in zip(("fwd", "dX", "dW"), configs):
- key = (config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K, config.num_stages)
- entry = _HEURISTIC_SMEM_STEP_DOWN.get(kind, {}).get(key)
- if entry is not None and entry[2] <= smem < entry[0]:
- config.num_stages = entry[1]
- logger.info(
- f"MoE heuristic {kind} config: num_stages {key[3]} -> {entry[1]} to fit {smem} B shared memory"
- )
+ _fit_config_to_smem(kind, config, act_bytes, weight_bytes, capability, smem)
return configs
def _get_heuristic_configs(
- n_dim: Optional[int] = None, k_dim: Optional[int] = None
+ n_dim: Optional[int] = None,
+ k_dim: Optional[int] = None,
+ dtype: Optional[torch.dtype] = None,
+ weight_dtype: Optional[torch.dtype] = None,
) -> Tuple[Any, Any, Any]:
"""
Get 'Safe Heuristic' kernel configurations.
@@ -511,11 +601,14 @@ def _get_heuristic_configs(
)
configs = _fit_configs_to_dims((config_fwd, config_bwd_dx, config_bwd_dw), n_dim, k_dim)
- return _fit_heuristic_configs_to_smem(configs)
+ return _fit_configs_to_smem(configs, dtype, weight_dtype)
def _get_default_configs(
- n_dim: Optional[int] = None, k_dim: Optional[int] = None
+ n_dim: Optional[int] = None,
+ k_dim: Optional[int] = None,
+ dtype: Optional[torch.dtype] = None,
+ weight_dtype: Optional[torch.dtype] = None,
) -> Tuple[Any, Any, Any]:
"""Get default kernel configurations as fallback."""
from .grouped_gemm.kernels.tuning import (
@@ -559,7 +652,8 @@ def _get_default_configs(
use_tma_store = False,
)
- return _fit_configs_to_dims((config_fwd, config_bwd_dx, config_bwd_dw), n_dim, k_dim)
+ configs = _fit_configs_to_dims((config_fwd, config_bwd_dx, config_bwd_dw), n_dim, k_dim)
+ return _fit_configs_to_smem(configs, dtype, weight_dtype)
def clear_cache() -> None: