Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MoE fallback kernel configs: fit the device shared memory for float32 operands (#14576)
Daniel Han039181491734
diff --git a/tests/test_moe_fallback_smem.py b/tests/test_moe_fallback_smem.pynew file mode 100644index 00000000000..7e5abf5d79e--- /dev/null+++ b/tests/test_moe_fallback_smem.py@@ -0,0 +1,333 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Fallback MoE configs fit shared memory for float32 X / dY (default dX needed 163,840 B > L4's 101,376 B).++Byte counts are Triton metadata.shared cross-compiled on CPU (Triton 3.6-3.8)."""++import importlib+import inspect+import sys+import types+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]++torch = pytest.importorskip("torch")+triton = pytest.importorskip("triton")+pytest.importorskip("pandas")++MOE_DIR = ROOT / "unsloth" / "kernels" / "moe"++GPUS = {+ "T4": ((7, 5), 65536),+ "L4": ((8, 9), 101376),+ "RTX4090": ((8, 9), 101376),+ "A10": ((8, 6), 101376),+ "A100": ((8, 0), 166912),+ "H100": ((9, 0), 232448),+ "B200": ((10, 0), 232448),+ "RTX5090": ((12, 0), 101376),+}+F32, BF16, F16 = torch.float32, torch.bfloat16, torch.float16+DTYPES = [+ (BF16, None, BF16),+ (F16, None, F16),+ (F32, BF16, BF16),+ (F32, F16, F16),+ (F32, None, F32),+ (F32, F32, F32),+]+# (hidden, moe_intermediate)+SHAPES = [(2048, 768), (2048, 1408), (1024, 352), (768, 2048)]+OLD_HEURISTIC = ((64, 128, 128, 8, 3), (64, 64, 256, 8, 4), (64, 64, 256, 8, 4))+OLD_DEFAULT = ((128, 128, 64, 8, 3),) * 3++# Main's float32-activation configs: (capability major, kind, activation, weight, (M, N, K, num_stages)) -> bytes.+MAIN_FLOAT32_COMPILED = {+ (8, "dX", "fp32", "bf16", (128, 128, 64, 3)): 163840, # default dX, the L4 crash+ (8, "fwd", "fp32", "bf16", (64, 128, 128, 3)): 131072, # heuristic forward+ (8, "dX", "fp32", "bf16", (64, 64, 256, 4)): 147456, # heuristic dX+ (8, "dX", "fp32", "fp32", (64, 64, 256, 4)): 245760,+ (8, "fwd", "fp32", "fp32", (64, 128, 128, 3)): 196608,+ (9, "dX", "fp32", "fp32", (64, 64, 256, 4)): 327680, # over the H100 limit too+ (9, "fwd", "fp32", "fp32", (64, 128, 128, 3)): 294912,+}+++def ac(monkeypatch):+ for name, path in (+ ("unsloth", ROOT / "unsloth"),+ ("unsloth.kernels", ROOT / "unsloth" / "kernels"),+ ("unsloth.kernels.moe", MOE_DIR),+ ):+ pkg = types.ModuleType(name)+ pkg.__path__ = [str(path)]+ monkeypatch.setitem(sys.modules, name, pkg)+ for name in list(sys.modules):+ if name.startswith("unsloth.kernels.moe."):+ monkeypatch.delitem(sys.modules, name)+ module = importlib.import_module("unsloth.kernels.moe.autotune_cache")+ yield module+ for name in list(sys.modules):+ if name.startswith("unsloth.kernels.moe."):+ sys.modules.pop(name, None)+++def _device(ac, monkeypatch, capability, smem):+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: capability)+ monkeypatch.setattr(ac, "_device_smem_per_block", lambda: smem)+++def _gemm_dims(hidden, inter):+ return [(2 * inter, hidden), (hidden, inter)]+++def _tiles(configs):+ return tuple(+ (c.BLOCK_SIZE_M, c.BLOCK_SIZE_N, c.BLOCK_SIZE_K, c.num_warps, c.num_stages) for c in configs+ )+++def _pick(ac, which, n, k, act, weight):+ fn = ac._get_heuristic_configs if which == "heuristic" else ac._get_default_configs+ return fn(n_dim = n, k_dim = k, dtype = act, weight_dtype = weight)++[email protected]("gpu", list(GPUS))[email protected]("act,weight,seen", DTYPES)[email protected]("which", ["heuristic", "default"])+def test_picked_configs_fit_the_device(ac, monkeypatch, gpu, act, weight, seen, which):+ capability, smem = GPUS[gpu]+ _device(ac, monkeypatch, capability, smem)+ for hidden, inter in SHAPES:+ for n, k in _gemm_dims(hidden, inter):+ for kind, c in zip(("fwd", "dX", "dW"), _pick(ac, which, n, k, act, weight)):+ need = ac._estimate_smem(kind, c, act.itemsize, seen.itemsize, capability)+ assert need <= smem, (gpu, which, kind, act, seen, n, k, _tiles([c]))+ assert n % c.BLOCK_SIZE_N == 0 and k % c.BLOCK_SIZE_K == 0+ assert min(c.BLOCK_SIZE_M, c.BLOCK_SIZE_N, c.BLOCK_SIZE_K) >= 16+ key = (+ capability[0] if capability[0] in (8, 9) else None,+ kind,+ "fp32" if act == F32 else "16",+ "fp32" if seen == F32 else ("bf16" if seen == BF16 else "16"),+ (c.BLOCK_SIZE_M, c.BLOCK_SIZE_N, c.BLOCK_SIZE_K, c.num_stages),+ )+ compiled = MAIN_FLOAT32_COMPILED.get(key)+ assert compiled is None or compiled <= smem, (gpu, key, compiled)++[email protected]("gpu", list(GPUS))[email protected]("act", [BF16, F16])[email protected]("which", ["heuristic", "default"])+def test_16bit_configs_unchanged(ac, monkeypatch, gpu, act, which):+ capability, smem = GPUS[gpu]+ for hidden, inter in SHAPES:+ for n, k in _gemm_dims(hidden, inter):+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ unfitted = _tiles(_pick(ac, which, n, k, None, None))+ _device(ac, monkeypatch, capability, smem)+ expected = list(unfitted)+ # main's only smem step-down: the 4-stage heuristic dX (122,880 B) to 3 stages on sm_8x / sm_12x.+ if which == "heuristic" and smem < 122880 and capability[0] >= 8:+ if unfitted[1] == (64, 64, 256, 8, 4):+ expected[1] = (64, 64, 256, 8, 3)+ assert _tiles(_pick(ac, which, n, k, act, None)) == tuple(expected), (gpu, n, k)+ assert _tiles(_pick(ac, which, n, k, None, None)) == tuple(expected), (gpu, n, k)+++def test_l4_float32_default_dx_no_longer_over_limit(ac, monkeypatch):+ # The configuration that crashed: autotune failed, float32 dY, bf16 W, L4.+ _device(ac, monkeypatch, (8, 9), 101376)+ _, dx, _ = _pick(ac, "default", 1536, 2048, F32, BF16)+ assert (dx.BLOCK_SIZE_M, dx.BLOCK_SIZE_N, dx.BLOCK_SIZE_K, dx.num_stages) != (128, 128, 64, 3)+ assert ac._estimate_smem("dX", dx, 4, 2, (8, 9)) <= 101376+ # Unknown weight dtype: sized for float32 W, which also fits bf16 W.+ _, dx32, _ = _pick(ac, "default", 1536, 2048, F32, None)+ assert ac._estimate_smem("dX", dx32, 4, 4, (8, 9)) <= 101376+++def test_unknown_smem_or_no_cuda_keeps_configs(ac, monkeypatch):+ _device(ac, monkeypatch, (8, 9), None)+ assert _tiles(_pick(ac, "heuristic", 1536, 2048, F32, BF16)) == OLD_HEURISTIC+ assert _tiles(_pick(ac, "default", 1536, 2048, F32, BF16)) == OLD_DEFAULT+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ assert _tiles(_pick(ac, "default", 1536, 2048, F32, BF16)) == OLD_DEFAULT++[email protected]("env", ["1", "0"])+def test_get_or_autotune_passes_dtypes(ac, monkeypatch, env):+ _device(ac, monkeypatch, (8, 9), 101376)+ monkeypatch.setenv("UNSLOTH_MOE_DISABLE_AUTOTUNE", env)+ monkeypatch.setattr(ac, "load_cached_config", lambda key: None)+ monkeypatch.setattr(ac, "save_cached_config", lambda *a, **k: None)+ seen = {}++ def tune(*args):+ seen["args"] = args+ raise RuntimeError("tuning failed")++ monkeypatch.setattr(ac, "_run_moe_autotuning", tune)+ ac.clear_cache()+ configs = ac.get_or_autotune_moe_kernels(+ num_experts = 8,+ hidden_dim = 2048,+ intermediate_dim = 768,+ top_k = 2,+ dtype = F32,+ weight_dtype = BF16,+ )+ for kind, c in zip(("fwd", "dX", "dW"), configs):+ assert ac._estimate_smem(kind, c, 4, 2, (8, 9)) <= 101376+ if env == "0":+ assert seen["args"][4] == F32 and seen["args"][-1] == BF16+++def test_cache_key_unchanged_without_a_distinct_weight_dtype(ac):+ base = ac._get_cache_key(8, 2048, 768, 2, BF16, (9, 0), 8192)+ assert ac._get_cache_key(8, 2048, 768, 2, BF16, (9, 0), 8192, None) == base+ assert ac._get_cache_key(8, 2048, 768, 2, BF16, (9, 0), 8192, BF16) == base+ f32 = ac._get_cache_key(8, 2048, 768, 2, F32, (9, 0), 8192)+ assert ac._get_cache_key(8, 2048, 768, 2, F32, (9, 0), 8192, BF16) != f32+++# Recompile a sample (CPU only, needs Triton's CUDA backend + ptxas).+++def _compile_shared(fn, ptr_types, consts, arch, num_warps, num_stages):+ from triton.backends.compiler import GPUTarget+ from triton.compiler import ASTSource++ params = list(inspect.signature(fn.fn).parameters)+ signature, constexprs = {}, {}+ for i, p in enumerate(params):+ if p in ptr_types:+ signature[p] = ptr_types[p]+ elif p in ("NUM_TOKENS", "NUM_SMS"):+ signature[p] = "i32"+ else:+ signature[p] = "constexpr"+ constexprs[(i,)] = consts.get(p, inspect.signature(fn.fn).parameters[p].default)+ # A real launch specializes 16 B aligned pointers; without it cp.async is not used.+ attrs = {(i,): [["tt.divisibility", 16]] for i, p in enumerate(params) if p in ptr_types}+ kernel = triton.compile(+ ASTSource(fn, signature, constexprs, attrs),+ target = GPUTarget("cuda", arch, 32),+ options = dict(num_warps = num_warps, num_stages = num_stages),+ )+ return kernel.metadata.shared+++SAMPLE_GPUS = ["L4", "A100", "H100", "B200", "RTX5090"]++[email protected]("gpu", SAMPLE_GPUS)+def test_estimate_matches_compiled_kernels(ac, monkeypatch, gpu):+ try:+ from triton.backends.compiler import GPUTarget # noqa: F401+ from triton.compiler import ASTSource # noqa: F401+ from unsloth.kernels.moe.grouped_gemm.kernels import backward, forward+ except Exception as e:+ pytest.skip(f"Triton CUDA cross compile unavailable: {e}")+ capability, smem = GPUS[gpu]+ arch = capability[0] * 10 + capability[1]+ _device(ac, monkeypatch, capability, smem)+ names = {F32: "*fp32", BF16: "*bf16", F16: "*fp16"}+ n, k = 1536, 2048 # Qwen3-30B-A3B gate_up+ try:+ _compile_shared(+ forward._grouped_gemm_forward_kernel,+ dict(+ x_ptr = "*bf16",+ w_ptr = "*bf16",+ y_ptr = "*bf16",+ m_sizes_ptr = "*i32",+ gather_indices_ptr = "*i64",+ topk_weights_ptr = "*bf16",+ ),+ dict(+ NUM_EXPERTS = 8,+ TOPK = 2,+ N = n,+ K = k,+ BLOCK_SIZE_M = 32,+ BLOCK_SIZE_N = 32,+ BLOCK_SIZE_K = 32,+ PERMUTE_X = True,+ PERMUTE_Y = False,+ ),+ arch,+ 4,+ 2,+ )+ except Exception as e:+ pytest.skip(f"Triton cannot cross compile on this host: {type(e).__name__}: {e}")+ for act, weight, seen in [(BF16, None, BF16), (F32, BF16, BF16), (F32, None, F32)]:+ for which in ("heuristic", "default"):+ for kind, c in zip(("fwd", "dX", "dW"), _pick(ac, which, n, k, act, weight)):+ consts = dict(+ NUM_EXPERTS = 8,+ TOPK = 2,+ N = n,+ K = k,+ BLOCK_SIZE_M = c.BLOCK_SIZE_M,+ BLOCK_SIZE_N = c.BLOCK_SIZE_N,+ BLOCK_SIZE_K = c.BLOCK_SIZE_K,+ PERMUTE_X = True,+ PERMUTE_Y = False,+ )+ a, w = names[act], names[seen]+ if kind == "fwd":+ compiled = _compile_shared(+ forward._grouped_gemm_forward_kernel,+ dict(+ x_ptr = a,+ w_ptr = w,+ y_ptr = a,+ m_sizes_ptr = "*i32",+ gather_indices_ptr = "*i64",+ topk_weights_ptr = a,+ ),+ consts,+ arch,+ c.num_warps,+ c.num_stages,+ )+ elif kind == "dX":+ compiled = _compile_shared(+ backward._grouped_gemm_dX_kernel,+ dict(+ dY_ptr = a,+ w_ptr = w,+ dX_ptr = a,+ gather_indices_ptr = "*i64",+ m_sizes_ptr = "*i32",+ ),+ consts,+ arch,+ c.num_warps,+ c.num_stages,+ )+ else:+ compiled = _compile_shared(+ backward._grouped_gemm_dW_per_expert_kernel,+ dict(+ x_ptr = a,+ dY_ptr = a,+ dW_ptr = w,+ m_sizes_ptr = "*i32",+ gather_indices_ptr = "*i64",+ ),+ dict(consts, EXPERTS_POW2 = 8),+ arch,+ c.num_warps,+ 1 if act == F32 else c.num_stages,+ )+ estimate = ac._estimate_smem(kind, c, act.itemsize, seen.itemsize, capability)+ assert compiled <= smem, (gpu, which, kind, act, seen, _tiles([c]), compiled)+ assert compiled <= estimate, (gpu, which, kind, act, seen, _tiles([c]), compiled)diff --git a/tests/test_moe_fallback_tile_dims.py b/tests/test_moe_fallback_tile_dims.pyindex 7e817b20b38..141334ba37b 100644--- a/tests/test_moe_fallback_tile_dims.py+++ b/tests/test_moe_fallback_tile_dims.py@@ -182,13 +182,18 @@ def test_heuristic_shrunk_dx_is_not_stepped(ac, monkeypatch):assert (dx.BLOCK_SIZE_K, dx.num_stages) == (128, 4)[email protected]("smem", [None, 65536])-def test_heuristic_unknown_or_tiny_smem_keeps_config(ac, monkeypatch, smem):- # Unknown limit, or one the step-down does not fit either (81,920 B): keep today's config.- _device(ac, monkeypatch, (8, 9), smem)+def test_heuristic_unknown_smem_keeps_config(ac, monkeypatch):+ _device(ac, monkeypatch, (8, 9), None)assert _tiles(ac._get_heuristic_configs(n_dim = 1536, k_dim = 2048)) == OLD_HEURISTIC+def test_heuristic_tiny_smem_steps_down_until_it_fits(ac, monkeypatch):+ _device(ac, monkeypatch, (8, 9), 65536)+ configs = ac._get_heuristic_configs(n_dim = 1536, k_dim = 2048)+ for kind, c in zip(("fwd", "dX", "dW"), configs):+ assert ac._estimate_smem(kind, c, 2, 2, (8, 9)) <= 65536++def test_default_configs_not_stepped(ac, monkeypatch):_device(ac, monkeypatch, (8, 9), 101376)assert _tiles(ac._get_default_configs(n_dim = 1536, k_dim = 2048)) == OLD_DEFAULTdiff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.pyindex 95f2360337c..e12a4030a7a 100644--- a/unsloth/kernels/moe/autotune_cache.py+++ b/unsloth/kernels/moe/autotune_cache.py@@ -38,6 +38,7 @@ def _get_cache_key(dtype: torch.dtype,device_capability: Tuple[int, int],seq_len: int = 8192, # Default sequence length for tuning+ weight_dtype: Optional[torch.dtype] = None,) -> str:"""Generate a unique cache key based on model configuration."""key_data = {@@ -49,6 +50,8 @@ def _get_cache_key("device_capability": device_capability,"seq_len": seq_len,}+ if weight_dtype is not None and weight_dtype != dtype:+ key_data["weight_dtype"] = str(weight_dtype)key_str = json.dumps(key_data, sort_keys = True)return hashlib.md5(key_str.encode()).hexdigest()@@ -133,6 +136,7 @@ def get_or_autotune_moe_kernels(dtype: torch.dtype,force_autotune: bool = False,seq_len: int = 8192,+ weight_dtype: Optional[torch.dtype] = None,) -> Tuple[Any, Any, Any]:"""Get cached kernel configurations or run auto-tuning.@@ -142,7 +146,8 @@ def get_or_autotune_moe_kernels(hidden_dim: Hidden dimension of the modelintermediate_dim: Intermediate dimension for MoE MLPtop_k: Number of experts to route to- dtype: Data type for computation+ dtype: Data type of the activations (X and dY)+ weight_dtype: Data type of the expert weights, if different from dtypeforce_autotune: Force re-running autotuning even if cache existsseq_len: Sequence length to use for tuning benchmarks@@ -158,6 +163,7 @@ def get_or_autotune_moe_kernels(dtype,device_capability,seq_len,+ weight_dtype,)# Env override to disable autotuning@@ -165,7 +171,9 @@ def get_or_autotune_moe_kernels(logger.info(f"UNSLOTH_MOE_DISABLE_AUTOTUNE=1: Using Heuristic (Safe) MoE kernel configs for SM{device_capability[0]}{device_capability[1]}")- return _get_heuristic_configs(n_dim = 2 * intermediate_dim, k_dim = hidden_dim)+ return _get_heuristic_configs(+ n_dim = 2 * intermediate_dim, k_dim = hidden_dim, dtype = dtype, weight_dtype = weight_dtype+ )if not force_autotune and cache_key in _kernel_config_cache:logger.info(f"Using in-memory cached MoE kernel configs: {cache_key}")return _kernel_config_cache[cache_key]@@ -201,7 +209,7 @@ def get_or_autotune_moe_kernels(try:configs = _run_moe_autotuning(- num_experts, hidden_dim, intermediate_dim, top_k, dtype, seq_len+ num_experts, hidden_dim, intermediate_dim, top_k, dtype, seq_len, weight_dtype)_kernel_config_cache[cache_key] = configs@@ -230,7 +238,9 @@ def get_or_autotune_moe_kernels("Unsloth: Your Triton version might be incompatible with TMA features. Falling back to default configs.")logger.info("Falling back to default kernel configurations")- return _get_default_configs(n_dim = 2 * intermediate_dim, k_dim = hidden_dim)+ return _get_default_configs(+ n_dim = 2 * intermediate_dim, k_dim = hidden_dim, dtype = dtype, weight_dtype = weight_dtype+ )def _run_moe_autotuning(@@ -240,6 +250,7 @@ def _run_moe_autotuning(top_k: int,dtype: torch.dtype,seq_len: int,+ weight_dtype: Optional[torch.dtype] = None,) -> Tuple[Any, Any, Any]:"""Run the actual auto-tuning for MoE kernels."""@@ -251,10 +262,11 @@ def _run_moe_autotuning(hidden_states = torch.randn(num_tokens, hidden_dim, device = device, dtype = dtype)gate_up_weights = torch.randn(- num_experts, 2 * intermediate_dim, hidden_dim, device = device, dtype = dtype- )- down_weights = torch.randn(- num_experts, hidden_dim, intermediate_dim, device = device, dtype = dtype+ num_experts,+ 2 * intermediate_dim,+ hidden_dim,+ device = device,+ dtype = dtype if weight_dtype is None else weight_dtype,)# Dummy routing data@@ -410,14 +422,6 @@ def _fit_configs_to_dims(return configs-# Triton metadata.shared (16-bit) of the 4-stage 64x64x256 dX: 122,880 B, over the 101,376 B of L4 / RTX PRO 6000;-# 3 stages need 81,920 B. sm_75 needs 65,536 B (no cp.async multi-buffering), so it is excluded.-# kind -> {(BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages): (bytes, fitting num_stages, its bytes)}-_HEURISTIC_SMEM_STEP_DOWN = {- "dX": {(64, 64, 256, 4): (122880, 3, 81920)},-}--def _device_smem_per_block() -> Optional[int]:# Read per current device: the launching card on a mixed host.try:@@ -435,30 +439,116 @@ def _device_smem_per_block() -> Optional[int]:return None-def _fit_heuristic_configs_to_smem(configs: Tuple[Any, Any, Any]) -> Tuple[Any, Any, Any]:+def _itemsize(dtype: Optional[torch.dtype], default: int = 2) -> int:+ return default if dtype is None else dtype.itemsize+++def _estimate_smem(+ kind: str, config: Any, act_bytes: int, weight_bytes: int, capability: Tuple[int, int]+) -> int:+ """Shared memory (B) per launch; matches compiled metadata.shared (Triton 3.6-3.8, sm_75..120) or exceeds it by <= 1 KiB."""+ M, N, K = config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K+ stages = max(int(config.num_stages), 1)+ if kind == "fwd":+ a, b, a_cast = M * K * act_bytes, N * K * weight_bytes, act_bytes != weight_bytes+ elif kind == "dX":+ a, b, a_cast = M * N * act_bytes, N * K * weight_bytes, act_bytes != weight_bytes+ else:+ # The launcher runs float32 X single stage.+ a, b, a_cast = M * N * act_bytes, M * K * act_bytes, False+ if act_bytes >= 4:+ stages = 1+ major = capability[0]+ if major < 8:+ # No cp.async multi-buffering; 16-bit dot operands add a 2x larger-tile scratch.+ dot_bytes = act_bytes if kind == "dW" else weight_bytes+ if dot_bytes >= 4:+ return a + b+ return max(a + b, 2 * max(a * dot_bytes // act_bytes, b))+ hopper_like = major in (9, 10, 11)+ if kind == "dW":+ if stages > 4:+ buffers = stages if hopper_like else stages - 1+ else:+ buffers = min(stages, 2) if hopper_like else 1+ return buffers * (a + b) + (2048 if hopper_like else 1024)+ if hopper_like:+ # A cast operand keeps num_stages - 1 buffers plus one copy in the dot dtype.+ if a_cast:+ return max(stages - 1, 1) * a + stages * b + a * weight_bytes // act_bytes + 64+ return stages * (a + b) + 64+ return max(stages - 1, 1) * (a + b)+++_LOOP_DIM = {"fwd": "BLOCK_SIZE_K", "dX": "BLOCK_SIZE_N", "dW": "BLOCK_SIZE_M"}+_MIN_SMEM_BLOCK = 16+++def _fit_config_to_smem(+ kind: str,+ config: Any,+ act_bytes: int,+ weight_bytes: int,+ capability: Tuple[int, int],+ smem: int,+) -> Any:+ # Halving a power-of-two tile keeps it dividing N / K.+ def fits():+ return _estimate_smem(kind, config, act_bytes, weight_bytes, capability) <= smem++ before = (config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K, config.num_stages)+ loop_dim = _LOOP_DIM[kind]+ while not fits():+ if config.num_stages > 3:+ config.num_stages -= 1+ elif getattr(config, loop_dim) > 32:+ setattr(config, loop_dim, getattr(config, loop_dim) // 2)+ elif config.num_stages > 2:+ config.num_stages -= 1+ else:+ name = max(+ ("BLOCK_SIZE_M", "BLOCK_SIZE_N", "BLOCK_SIZE_K"), key = lambda n: getattr(config, n)+ )+ if getattr(config, name) > _MIN_SMEM_BLOCK:+ setattr(config, name, getattr(config, name) // 2)+ elif config.num_stages > 1:+ config.num_stages = 1+ else:+ break+ after = (config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K, config.num_stages)+ if after != before:+ logger.info(+ f"MoE {kind} config (M, N, K, stages) {before} -> {after} to fit {smem} B shared memory"+ )+ return config+++def _fit_configs_to_smem(+ configs: Tuple[Any, Any, Any],+ dtype: Optional[torch.dtype] = None,+ weight_dtype: Optional[torch.dtype] = None,+) -> Tuple[Any, Any, Any]:try:if not torch.cuda.is_available() or getattr(torch.version, "hip", None):return configs- if torch.cuda.get_device_capability()[0] < 8:- return configs+ capability = tuple(torch.cuda.get_device_capability())except Exception:return configssmem = _device_smem_per_block()if smem is None:return configs+ act_bytes = _itemsize(dtype)+ weight_bytes = _itemsize(weight_dtype, act_bytes)for kind, config in zip(("fwd", "dX", "dW"), configs):- key = (config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K, config.num_stages)- entry = _HEURISTIC_SMEM_STEP_DOWN.get(kind, {}).get(key)- if entry is not None and entry[2] <= smem < entry[0]:- config.num_stages = entry[1]- logger.info(- f"MoE heuristic {kind} config: num_stages {key[3]} -> {entry[1]} to fit {smem} B shared memory"- )+ _fit_config_to_smem(kind, config, act_bytes, weight_bytes, capability, smem)return configsdef _get_heuristic_configs(- n_dim: Optional[int] = None, k_dim: Optional[int] = None+ n_dim: Optional[int] = None,+ k_dim: Optional[int] = None,+ dtype: Optional[torch.dtype] = None,+ weight_dtype: Optional[torch.dtype] = None,) -> Tuple[Any, Any, Any]:"""Get 'Safe Heuristic' kernel configurations.@@ -511,11 +601,14 @@ def _get_heuristic_configs()configs = _fit_configs_to_dims((config_fwd, config_bwd_dx, config_bwd_dw), n_dim, k_dim)- return _fit_heuristic_configs_to_smem(configs)+ return _fit_configs_to_smem(configs, dtype, weight_dtype)def _get_default_configs(- n_dim: Optional[int] = None, k_dim: Optional[int] = None+ n_dim: Optional[int] = None,+ k_dim: Optional[int] = None,+ dtype: Optional[torch.dtype] = None,+ weight_dtype: Optional[torch.dtype] = None,) -> Tuple[Any, Any, Any]:"""Get default kernel configurations as fallback."""from .grouped_gemm.kernels.tuning import (@@ -559,7 +652,8 @@ def _get_default_configs(use_tma_store = False,)- return _fit_configs_to_dims((config_fwd, config_bwd_dx, config_bwd_dw), n_dim, k_dim)+ configs = _fit_configs_to_dims((config_fwd, config_bwd_dx, config_bwd_dw), n_dim, k_dim)+ return _fit_configs_to_smem(configs, dtype, weight_dtype)def clear_cache() -> None: