Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MoE autotune cache: pick kernel configs for the operands' GPU (#14751)

Daniel Han08a5ae752610
diff --git a/tests/test_grouped_gemm_mixed_tma_device.py b/tests/test_grouped_gemm_mixed_tma_device.py
index c8ab81ceca0..fa885a0d3c9 100644
--- a/tests/test_grouped_gemm_mixed_tma_device.py
+++ b/tests/test_grouped_gemm_mixed_tma_device.py
@@ -46,6 +46,7 @@ class _Kernel:
def __getitem__(self, grid):
def launch(**kwargs):
self.launches.append(self.guard_stack[-1] if self.guard_stack else None)
+ self.last_kwargs = kwargs
return types.SimpleNamespace(name = "k", n_regs = 0, n_spills = 0, metadata = {})
return launch
@@ -377,3 +378,83 @@ def test_sm_count_is_read_once_per_card(monkeypatch):
assert module._num_sms(torch.device("cuda", 1)) == 4
assert module._num_sms(torch.device("cuda", 0)) == 4
assert sorted(reads) == [0, 1]
+
+
+
+def _real_autotune_cache():
+ spec = importlib.util.spec_from_file_location(
+ "_unsloth_ac_spoof", ROOT / "unsloth" / "kernels" / "moe" / "autotune_cache.py"
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+SMEM = {H100: 232448, A100: 166912, L4: 101376}
+TILES = ("BLOCK_SIZE_M", "BLOCK_SIZE_N", "BLOCK_SIZE_K", "num_stages")
+
+
+def _launch_tiles(monkeypatch, caps, operand_index, M, N, K, stages, dtype = torch.bfloat16):
+ """Run the three non-autotuned impls on cuda:operand_index (current: cuda:0); return each launch's tiles."""
+ module, _ = _load(monkeypatch, caps)
+ real = torch.cuda.get_device_properties
+
+ def props(device = None):
+ p = real(device)
+ index = 0 if device is None else getattr(device, "index", device)
+ p.shared_memory_per_block_optin = SMEM[caps[index]]
+ return p
+
+ monkeypatch.setattr(torch.cuda, "get_device_properties", props)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(
+ torch.cuda,
+ "get_device_capability",
+ lambda device = None: caps[0 if device is None else getattr(device, "index", device)],
+ )
+ ac = _real_autotune_cache()
+ monkeypatch.setattr(module, "_autotune_cache", lambda: ac)
+ E, T, Nd, Kd = 2, 8, 512, 512
+ tiles = dict(BLOCK_SIZE_M = M, BLOCK_SIZE_N = N, BLOCK_SIZE_K = K, num_stages = stages, num_warps = 8)
+ out = {}
+ with FakeTensorMode():
+ dev = torch.device("cuda", operand_index)
+ X = torch.empty(T, Kd, device = dev, dtype = dtype)
+ W = torch.empty(E, Nd, Kd, device = dev, dtype = torch.bfloat16)
+ dY = torch.empty(T, Nd, device = dev, dtype = dtype)
+ m_sizes = torch.empty(E, device = dev, dtype = torch.int32)
+ common = dict(m_sizes = m_sizes, gather_indices = None, topk = 1, autotune = False, **tiles)
+ kernels = sys.modules[f"{PKG}.kernels.forward"], sys.modules[f"{PKG}.kernels.backward"]
+ module._grouped_gemm_forward_impl(X = X, W = W, **common)
+ out["fwd"] = kernels[0]._grouped_gemm_forward_kernel.last_kwargs
+ module._grouped_gemm_dX_impl(dY = dY, W = W, **common)
+ out["dX"] = kernels[1]._grouped_gemm_dX_kernel.last_kwargs
+ module._grouped_gemm_dW_impl(X = X, dY = dY, **common)
+ per_expert = module._use_per_expert_dW_kernel()
+ dw_kernel = kernels[1]._grouped_gemm_dW_per_expert_kernel if per_expert else kernels[1]._grouped_gemm_dW_kernel
+ out["dW"] = dw_kernel.last_kwargs
+ return ac, {k: tuple(v[t] for t in TILES) for k, v in out.items()}
+
+
+def test_caller_tiles_fitted_for_the_current_card_are_refit_for_the_operand_card(monkeypatch):
+ # Released unsloth_zoo passes no device: an H100-fitted bf16 dX (64, 64, 256, 4) needs 122,880 B,
+ # over the L4's 101,376 B. The launch on the L4 operand shrinks it to fit; the H100 copy is untouched.
+ ac, got = _launch_tiles(monkeypatch, [H100, L4], 1, 64, 64, 256, 4)
+ assert ac._estimate_smem("dX", types.SimpleNamespace(**dict(zip(TILES, (64, 64, 256, 4)))), 2, 2, L4) > SMEM[L4]
+ assert got["dX"] == (64, 64, 256, 3)
+ for kind, tiles in got.items():
+ config = types.SimpleNamespace(**dict(zip(TILES, tiles)))
+ assert ac._estimate_smem(kind, config, 2, 2, L4) <= SMEM[L4]
+
+
+ "caps, operand_index",
+ [
+ pytest.param([L4], 0, id = "one-gpu"),
+ pytest.param([H100, H100], 1, id = "same-model"),
+ pytest.param([H100, L4], 0, id = "operand-on-current"),
+ ],
+)
+def test_tiles_unchanged_on_one_gpu_or_the_same_model(monkeypatch, caps, operand_index):
+ _, got = _launch_tiles(monkeypatch, caps, operand_index, 64, 64, 256, 4)
+ assert got == {"fwd": (64, 64, 256, 4), "dX": (64, 64, 256, 4), "dW": (64, 64, 256, 4)}
diff --git a/tests/test_moe_autotune_operand_device.py b/tests/test_moe_autotune_operand_device.py
new file mode 100644
index 00000000000..25fa946fca1
--- /dev/null
+++ b/tests/test_moe_autotune_operand_device.py
@@ -0,0 +1,178 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""MoE kernel configs follow the operands' GPU, not the current one (H100 + L4 in one process).
+
+The launcher enters X.device, so configs picked from the current card's capability, shared memory
+or disk cache entry could exceed the operand card's limit. CPU only: device properties are mocked."""
+
+import importlib
+import json
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("triton")
+
+MOE_DIR = ROOT / "unsloth" / "kernels" / "moe"
+
+# index -> (capability, shared_memory_per_block_optin)
+H100, L4 = ((9, 0), 232448), ((8, 9), 101376)
+F32, BF16, F16 = torch.float32, torch.bfloat16, torch.float16
+DTYPES = [(BF16, None), (F16, None), (F32, BF16), (F32, F16)]
+SHAPES = [(2048, 768), (2048, 1408), (1024, 352), (768, 2048)]
+KINDS = ("fwd", "dX", "dW")
+
+
+def ac(monkeypatch, tmp_path):
+ for name, path in (
+ ("unsloth", ROOT / "unsloth"),
+ ("unsloth.kernels", ROOT / "unsloth" / "kernels"),
+ ("unsloth.kernels.moe", MOE_DIR),
+ ):
+ pkg = types.ModuleType(name)
+ pkg.__path__ = [str(path)]
+ monkeypatch.setitem(sys.modules, name, pkg)
+ for name in list(sys.modules):
+ if name.startswith("unsloth.kernels.moe."):
+ monkeypatch.delitem(sys.modules, name)
+ monkeypatch.setenv("HOME", str(tmp_path))
+ monkeypatch.delenv("UNSLOTH_MOE_DISABLE_AUTOTUNE", raising = False)
+ module = importlib.import_module("unsloth.kernels.moe.autotune_cache")
+ module.clear_cache()
+ yield module
+ for name in list(sys.modules):
+ if name.startswith("unsloth.kernels.moe."):
+ sys.modules.pop(name, None)
+
+
+def _gpus(monkeypatch, cards, current = 0):
+ def index(device = None):
+ if device is None:
+ return current
+ if isinstance(device, int):
+ return device
+ device = torch.device(device)
+ return current if device.index is None else device.index
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: len(cards))
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: current)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda d = None: cards[index(d)][0])
+ monkeypatch.setattr(
+ torch.cuda,
+ "get_device_properties",
+ lambda d = None: types.SimpleNamespace(shared_memory_per_block_optin = cards[index(d)][1]),
+ )
+
+
+def _smem(ac, configs, act, weight, card):
+ a, w = act.itemsize, (weight or act).itemsize
+ return [ac._estimate_smem(k, c, a, w, card[0]) for k, c in zip(KINDS, configs)]
+
+
+def _dicts(configs):
+ return [c.__dict__.copy() for c in configs]
+
+
[email protected]("which", ["heuristic", "default"])
[email protected]("act,weight", DTYPES)
[email protected]("hidden,inter", SHAPES)
+def test_fallback_configs_fit_the_operand_card(ac, monkeypatch, which, act, weight, hidden, inter):
+ _gpus(monkeypatch, [H100, L4], current = 0)
+ fn = ac._get_heuristic_configs if which == "heuristic" else ac._get_default_configs
+ for n, k in ((2 * inter, hidden), (hidden, inter)):
+ configs = fn(n_dim = n, k_dim = k, dtype = act, weight_dtype = weight, device = torch.device("cuda", 1))
+ assert max(_smem(ac, configs, act, weight, L4)) <= L4[1]
+
+
+def test_heuristic_env_path_uses_the_operand_card(ac, monkeypatch):
+ # Current H100, operand L4: the H100 fit left the bf16 heuristic dX at 122,880 B (> 101,376 B).
+ _gpus(monkeypatch, [H100, L4], current = 0)
+ monkeypatch.setenv("UNSLOTH_MOE_DISABLE_AUTOTUNE", "1")
+ dims = dict(num_experts = 8, hidden_dim = 2048, intermediate_dim = 768, top_k = 2)
+ for act, weight in ((BF16, None), (F32, BF16)):
+ configs = ac.get_or_autotune_moe_kernels(
+ dtype = act, weight_dtype = weight, device = torch.device("cuda", 1), **dims
+ )
+ assert max(_smem(ac, configs, act, weight, L4)) <= L4[1]
+
+
+def _big_configs():
+ from unsloth.kernels.moe.grouped_gemm.kernels.tuning import (
+ KernelConfigBackward_dW,
+ KernelConfigBackward_dX,
+ KernelConfigForward,
+ )
+
+ return (
+ KernelConfigForward(BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 128, BLOCK_SIZE_K = 128, num_warps = 8, num_stages = 3),
+ KernelConfigBackward_dX(BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 256, num_warps = 8, num_stages = 4),
+ KernelConfigBackward_dW(BLOCK_SIZE_M = 128, BLOCK_SIZE_N = 256, BLOCK_SIZE_K = 256, num_warps = 8, num_stages = 4),
+ )
+
+
+def test_cache_lookup_and_tuning_follow_the_operand_card(ac, monkeypatch):
+ _gpus(monkeypatch, [H100, L4], current = 0)
+ dims = dict(num_experts = 8, hidden_dim = 2048, intermediate_dim = 768, top_k = 2)
+ h100_key = ac._get_cache_key(8, 2048, 768, 2, BF16, H100[0], 8192, None)
+ ac.save_cached_config(h100_key, *_big_configs())
+
+ tuned_on = []
+
+ def tune(*args, device = None, **kwargs):
+ tuned_on.append(device)
+ return ac._get_heuristic_configs(n_dim = 1536, k_dim = 2048, dtype = BF16, device = device)
+
+ monkeypatch.setattr(ac, "_run_moe_autotuning", tune)
+ operand = torch.device("cuda", 1)
+ configs = ac.get_or_autotune_moe_kernels(dtype = BF16, device = operand, **dims)
+ # The H100 entry is not reused for the L4 operand; tuning runs on (and is saved for) the L4.
+ assert tuned_on == [operand]
+ assert max(_smem(ac, configs, BF16, None, L4)) <= L4[1]
+ l4_key = ac._get_cache_key(8, 2048, 768, 2, BF16, L4[0], 8192, None)
+ saved = json.loads(Path(ac._get_cache_file_path(l4_key)).read_text())
+ assert tuple(saved["device_capability"]) == L4[0]
+ # Both entries survive: the capability check reads the operand card, not the current one.
+ assert ac.load_cached_config(l4_key, device = operand) is not None
+ assert ac.load_cached_config(h100_key, device = torch.device("cuda", 0)) is not None
+
+
+def test_disk_hit_refits_to_the_operand_card(ac, monkeypatch):
+ _gpus(monkeypatch, [H100, L4], current = 0)
+ operand = torch.device("cuda", 1)
+ l4_key = ac._get_cache_key(8, 2048, 768, 2, BF16, L4[0], 8192, None)
+ ac.save_cached_config(l4_key, *_big_configs(), device = operand)
+ monkeypatch.setattr(ac, "_run_moe_autotuning", lambda *a, **k: pytest.fail("cache miss"))
+ configs = ac.get_or_autotune_moe_kernels(
+ num_experts = 8, hidden_dim = 2048, intermediate_dim = 768, top_k = 2, dtype = BF16, device = operand
+ )
+ assert max(_smem(ac, configs, BF16, None, L4)) <= L4[1]
+
+
+def test_smem_limit_reads_the_operand_index(ac, monkeypatch):
+ _gpus(monkeypatch, [H100, L4], current = 0)
+ assert ac._device_smem_per_block() == H100[1]
+ assert ac._device_smem_per_block(torch.device("cuda", 1)) == L4[1]
+ assert ac._device_smem_per_block(1) == L4[1]
+ assert ac._device_smem_per_block(torch.device("cuda")) == H100[1]
+
+
[email protected]("card", [((7, 5), 65536), L4, ((8, 0), 166912), H100, ((12, 0), 101376)])
+def test_single_gpu_configs_unchanged(ac, monkeypatch, card):
+ # One card: passing its device must pick exactly what the device-less call picked.
+ _gpus(monkeypatch, [card], current = 0)
+ for fn in (ac._get_heuristic_configs, ac._get_default_configs):
+ for act, weight in DTYPES:
+ for hidden, inter in SHAPES:
+ kw = dict(n_dim = 2 * inter, k_dim = hidden, dtype = act, weight_dtype = weight)
+ base = _dicts(fn(**kw))
+ assert _dicts(fn(device = torch.device("cuda", 0), **kw)) == base
+ assert _dicts(fn(device = torch.device("cuda"), **kw)) == base
+ assert _dicts(fn(device = 0, **kw)) == base
diff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.py
index 6fb9f289cb2..be20a9321f0 100644
--- a/unsloth/kernels/moe/autotune_cache.py
+++ b/unsloth/kernels/moe/autotune_cache.py
@@ -68,7 +68,12 @@ def _get_cache_file_path(cache_key: str) -> str:
return os.path.join(cache_dir, f"{cache_key}.json")
-def load_cached_config(cache_key: str) -> Optional[Dict[str, Any]]:
+def _device_kwargs(device) -> Dict[str, Any]:
+ # device=None keeps the current-device calls (and their callers' signatures) exactly as before.
+ return {} if device is None else {"device": device}
+
+
+def load_cached_config(cache_key: str, device = None) -> Optional[Dict[str, Any]]:
"""Load cached kernel configuration from disk."""
cache_file = _get_cache_file_path(cache_key)
if not os.path.exists(cache_file):
@@ -79,7 +84,7 @@ def load_cached_config(cache_key: str) -> Optional[Dict[str, Any]]:
cached_data = json.load(f)
# Invalidate if device capability changed. JSON stores the tuple as a list, so compare as tuples.
- current_device_capability = tuple(torch.cuda.get_device_capability())
+ current_device_capability = tuple(torch.cuda.get_device_capability(device))
cached_device_capability = cached_data.get("device_capability")
if not isinstance(cached_device_capability, (list, tuple)) or (
tuple(cached_device_capability) != current_device_capability
@@ -111,13 +116,14 @@ def save_cached_config(
config_bwd_dx: Any,
config_bwd_dw: Any,
metadata: Dict[str, Any] = None,
+ device = None,
) -> None:
"""Save kernel configuration to disk cache."""
cache_file = _get_cache_file_path(cache_key)
cache_data = {
"timestamp": time.time(),
- "device_capability": torch.cuda.get_device_capability(),
+ "device_capability": torch.cuda.get_device_capability(device),
"schema_version": _CACHE_SCHEMA_VERSION,
"triton_version": getattr(triton, "__version__", None),
"config_fwd": config_fwd.__dict__ if hasattr(config_fwd, "__dict__") else str(config_fwd),
@@ -150,6 +156,7 @@ def get_or_autotune_moe_kernels(
force_autotune: bool = False,
seq_len: int = 8192,
weight_dtype: Optional[torch.dtype] = None,
+ device = None,
) -> Tuple[Any, Any, Any]:
"""
Get cached kernel configurations or run auto-tuning.
@@ -163,11 +170,15 @@ def get_or_autotune_moe_kernels(
weight_dtype: Data type of the expert weights, if different from dtype
force_autotune: Force re-running autotuning even if cache exists
seq_len: Sequence length to use for tuning benchmarks
+ device: The operands' CUDA device (default: current). On a mixed-GPU host the configs
+ must fit the card the kernels launch on, so capability, shared memory, the cache key
+ and tuning all follow it.
Returns:
Tuple of (config_fwd, config_bwd_dx, config_bwd_dw)
"""
- device_capability = torch.cuda.get_device_capability()
+ device_capability = torch.cuda.get_device_capability(device)
+ dev_kw = _device_kwargs(device)
cache_key = _get_cache_key(
num_experts,
hidden_dim,
@@ -185,14 +196,18 @@ def get_or_autotune_moe_kernels(
f"UNSLOTH_MOE_DISABLE_AUTOTUNE=1: Using Heuristic (Safe) MoE kernel configs for SM{device_capability[0]}{device_capability[1]}"
)
return _get_heuristic_configs(
- n_dim = 2 * intermediate_dim, k_dim = hidden_dim, dtype = dtype, weight_dtype = weight_dtype
+ n_dim = 2 * intermediate_dim,
+ k_dim = hidden_dim,
+ dtype = dtype,
+ weight_dtype = weight_dtype,
+ **dev_kw,
)
if not force_autotune and cache_key in _kernel_config_cache:
logger.info(f"Using in-memory cached MoE kernel configs: {cache_key}")
return _kernel_config_cache[cache_key]
if not force_autotune:
- cached_data = load_cached_config(cache_key)
+ cached_data = load_cached_config(cache_key, **dev_kw)
if cached_data is not None:
try:
from .grouped_gemm.kernels.tuning import (
@@ -209,7 +224,7 @@ def get_or_autotune_moe_kernels(
configs = _fit_configs_to_dims(
(config_fwd, config_bwd_dx, config_bwd_dw), 2 * intermediate_dim, hidden_dim
)
- configs = _fit_configs_to_smem(configs, dtype, weight_dtype)
+ configs = _fit_configs_to_smem(configs, dtype, weight_dtype, **dev_kw)
_kernel_config_cache[cache_key] = configs
return configs
except Exception as e:
@@ -226,7 +241,7 @@ def get_or_autotune_moe_kernels(
try:
configs = _run_moe_autotuning(
- num_experts, hidden_dim, intermediate_dim, top_k, dtype, seq_len, weight_dtype
+ num_experts, hidden_dim, intermediate_dim, top_k, dtype, seq_len, weight_dtype, **dev_kw
)
_kernel_config_cache[cache_key] = configs
@@ -243,6 +258,7 @@ def get_or_autotune_moe_kernels(
"hidden_dim": hidden_dim,
"intermediate_dim": intermediate_dim,
},
+ **dev_kw,
)
logger.info(f"MoE kernel auto-tuning completed: {cache_key}")
@@ -256,7 +272,11 @@ def get_or_autotune_moe_kernels(
)
logger.info("Falling back to default kernel configurations")
return _get_default_configs(
- n_dim = 2 * intermediate_dim, k_dim = hidden_dim, dtype = dtype, weight_dtype = weight_dtype
+ n_dim = 2 * intermediate_dim,
+ k_dim = hidden_dim,
+ dtype = dtype,
+ weight_dtype = weight_dtype,
+ **dev_kw,
)
@@ -268,10 +288,29 @@ def _run_moe_autotuning(
dtype: torch.dtype,
seq_len: int,
weight_dtype: Optional[torch.dtype] = None,
+ device = None,
) -> Tuple[Any, Any, Any]:
- """Run the actual auto-tuning for MoE kernels."""
+ """Run the actual auto-tuning for MoE kernels on `device` (default: current)."""
+ if device is None:
+ return _run_moe_autotuning_on(
+ "cuda", num_experts, hidden_dim, intermediate_dim, top_k, dtype, weight_dtype
+ )
+ # Tune on the operands' card: the tensors and the benchmarks both belong there.
+ with torch.cuda.device(device):
+ return _run_moe_autotuning_on(
+ device, num_experts, hidden_dim, intermediate_dim, top_k, dtype, weight_dtype
+ )
- device = "cuda"
+
+def _run_moe_autotuning_on(
+ device,
+ num_experts: int,
+ hidden_dim: int,
+ intermediate_dim: int,
+ top_k: int,
+ dtype: torch.dtype,
+ weight_dtype: Optional[torch.dtype] = None,
+) -> Tuple[Any, Any, Any]:
# Fixed token count avoids OOMs and a seq_len dependency, so the passed seq_len is ignored.
num_tokens = 4096
total_tokens = num_tokens * top_k
@@ -439,10 +478,10 @@ def _fit_configs_to_dims(
return configs
-def _device_smem_per_block() -> Optional[int]:
- # Read per current device: the launching card on a mixed host.
+def _device_smem_per_block(device = None) -> Optional[int]:
+ # Read per launching card: the operands' device, else the current device.
try:
- index = torch.cuda.current_device()
+ index = torch.cuda.current_device() if device is None else _cuda_index(device)
except Exception:
return None
try:
@@ -456,6 +495,13 @@ def _device_smem_per_block() -> Optional[int]:
return None
+def _cuda_index(device) -> int:
+ if isinstance(device, int):
+ return device
+ device = torch.device(device)
+ return torch.cuda.current_device() if device.index is None else device.index
+
+
def _itemsize(dtype: Optional[torch.dtype], default: int = 2) -> int:
return default if dtype is None else dtype.itemsize
@@ -544,14 +590,15 @@ def _fit_configs_to_smem(
configs: Tuple[Any, Any, Any],
dtype: Optional[torch.dtype] = None,
weight_dtype: Optional[torch.dtype] = None,
+ device = None,
) -> Tuple[Any, Any, Any]:
try:
if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
return configs
- capability = tuple(torch.cuda.get_device_capability())
+ capability = tuple(torch.cuda.get_device_capability(device))
except Exception:
return configs
- smem = _device_smem_per_block()
+ smem = _device_smem_per_block(**_device_kwargs(device))
if smem is None:
return configs
act_bytes = _itemsize(dtype)
@@ -566,6 +613,7 @@ def _get_heuristic_configs(
k_dim: Optional[int] = None,
dtype: Optional[torch.dtype] = None,
weight_dtype: Optional[torch.dtype] = None,
+ device = None,
) -> Tuple[Any, Any, Any]:
"""
Get 'Safe Heuristic' kernel configurations.
@@ -618,7 +666,7 @@ def _get_heuristic_configs(
)
configs = _fit_configs_to_dims((config_fwd, config_bwd_dx, config_bwd_dw), n_dim, k_dim)
- return _fit_configs_to_smem(configs, dtype, weight_dtype)
+ return _fit_configs_to_smem(configs, dtype, weight_dtype, **_device_kwargs(device))
def _get_default_configs(
@@ -626,6 +674,7 @@ def _get_default_configs(
k_dim: Optional[int] = None,
dtype: Optional[torch.dtype] = None,
weight_dtype: Optional[torch.dtype] = None,
+ device = None,
) -> Tuple[Any, Any, Any]:
"""Get default kernel configurations as fallback."""
from .grouped_gemm.kernels.tuning import (
@@ -670,7 +719,7 @@ def _get_default_configs(
)
configs = _fit_configs_to_dims((config_fwd, config_bwd_dx, config_bwd_dw), n_dim, k_dim)
- return _fit_configs_to_smem(configs, dtype, weight_dtype)
+ return _fit_configs_to_smem(configs, dtype, weight_dtype, **_device_kwargs(device))
def clear_cache() -> None:
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index fbedaf0d507..7dd35b673dc 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -102,6 +102,62 @@ def _num_sms(device):
return count
+_OPERAND_TILE_FITS = {}
+
+
+def _autotune_cache():
+ from .. import autotune_cache
+
+ return autotune_cache
+
+
+def _fit_tiles_to_operand_device(kind, operand, weight_dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages):
+ """Caller-supplied tiles were fitted to the current card's shared memory when the caller passed no device
+ (unsloth_zoo <= 2026.10.7). If the operands sit on a different GPU model, shrink a copy to fit their card.
+ One GPU, or the same model as the current card: returned unchanged."""
+ tiles = (BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages)
+ device = operand.device
+ try:
+ if device.type != "cuda" or DEVICE_TYPE != "cuda" or torch.cuda.device_count() <= 1:
+ return tiles
+ current = torch.cuda.current_device()
+ index = _device_index(device)
+ if index == current or _device_model(device) == _device_model(torch.device("cuda", current)):
+ return tiles
+ except Exception:
+ return tiles
+ key = (kind, index, operand.dtype, weight_dtype, tiles)
+ fitted = _OPERAND_TILE_FITS.get(key)
+ if fitted is None:
+ fitted = tiles
+ try:
+ from types import SimpleNamespace
+
+ autotune_cache = _autotune_cache()
+
+ smem = autotune_cache._device_smem_per_block(device)
+ if smem is not None:
+ config = SimpleNamespace(
+ BLOCK_SIZE_M = BLOCK_SIZE_M,
+ BLOCK_SIZE_N = BLOCK_SIZE_N,
+ BLOCK_SIZE_K = BLOCK_SIZE_K,
+ num_stages = num_stages,
+ )
+ autotune_cache._fit_config_to_smem(
+ kind,
+ config,
+ operand.dtype.itemsize,
+ weight_dtype.itemsize,
+ tuple(torch.cuda.get_device_capability(device)),
+ smem,
+ )
+ fitted = (config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K, config.num_stages)
+ except Exception:
+ fitted = tiles
+ _OPERAND_TILE_FITS[key] = fitted
+ return fitted
+
+
def _autotuner_for(kernel, device):
"""Triton's autotune key omits the device, so on several GPUs each card runs its own copy. Copies on
the same GPU model share one results cache, so identical cards reuse each other's tuning; a run's
@@ -287,7 +343,9 @@ def _grouped_gemm_forward_impl(
return (NUM_SMS,)
if not autotune:
- pass
+ BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages = _fit_tiles_to_operand_device(
+ "fwd", X, W.dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages
+ )
if debug:
print(
@@ -434,7 +492,9 @@ def _grouped_gemm_dX_impl(
return (NUM_SMS,)
if not autotune:
- pass
+ BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages = _fit_tiles_to_operand_device(
+ "dX", dY, W.dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages
+ )
if debug:
print(
@@ -639,7 +699,9 @@ def _launch_grouped_gemm_dW(
use_tma_store = False
if not autotune:
- pass
+ BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages = _fit_tiles_to_operand_device(
+ "dW", X, X.dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages
+ )
def grid(META):
if per_expert: