Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

RMSNorm / LayerNorm: do not hash a symbolic width in the launch settings cache (#14727)

Daniel Handa007294dcc5
diff --git a/tests/test_norm_settings_symbolic_width.py b/tests/test_norm_settings_symbolic_width.py
new file mode 100644
index 00000000000..a90103efab5
--- /dev/null
+++ b/tests/test_norm_settings_symbolic_width.py
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The per-width launch settings cache must not hash a symbolic width: a traced norm op's fake run
+under dynamic=True passes one, outside Dynamo (is_compiling() is False there)."""
+
+import pytest
+import torch
+
+pytest.importorskip("triton")
+
+from torch._dynamo.source import ConstantSource
+from torch.fx.experimental.symbolic_shapes import ShapeEnv
+
+from unsloth.kernels import layernorm, rms_layernorm
+from unsloth.kernels import utils as kernel_utils
+from unsloth.kernels.utils import calculate_settings
+
+
+def _symbolic(n):
+ env = ShapeEnv()
+ return env.create_symintnode(env.create_symbol(n, ConstantSource("n_cols")), hint = n)
+
+
[email protected]("module", [rms_layernorm, layernorm], ids = ["rms", "layernorm"])
[email protected]("n", [64, 4096])
+def test_a_symbolic_width_skips_the_cache(module, n):
+ width = _symbolic(n)
+ with pytest.raises(TypeError):
+ hash(width)
+ assert tuple(module._settings(width)) == tuple(calculate_settings(n))
+
+
[email protected]("module", [rms_layernorm, layernorm], ids = ["rms", "layernorm"])
[email protected]("dw", [False, True], ids = ["fwd", "dw"])
+def test_a_symbolic_width_reads_the_cuda_warps_table(module, dw, monkeypatch):
+ # On CUDA the warps come from a (capability, width) table, which must not hash the width either.
+ table = module._DW_NUM_WARPS if dw else module._NUM_WARPS
+ (capability, n), warps = next(iter(table.items()))
+ monkeypatch.setattr(kernel_utils, "_NORM_ARCH_WARPS", True)
+ monkeypatch.setattr(kernel_utils, "_cuda_capability", lambda index: capability)
+ monkeypatch.setattr(torch.version, "hip", None)
+ device = torch.device("cuda", 0)
+ expected = module._arch_settings(n, device, dw)
+ assert expected == (calculate_settings(n)[0], warps)
+ assert tuple(module._settings(_symbolic(n), device, dw)) == expected
+ # A width the table does not list keeps calculate_settings' warps.
+ assert tuple(module._settings(_symbolic(n + 1), device, dw)) == tuple(calculate_settings(n + 1))
+
+
[email protected]("module", [rms_layernorm, layernorm], ids = ["rms", "layernorm"])
+def test_an_int_width_still_uses_the_cache(module):
+ module._cached_arch_settings.cache_clear()
+ for _ in range(3):
+ assert module._settings(256) == calculate_settings(256)
+ info = module._cached_arch_settings.cache_info()
+ assert info.misses == 1 and info.hits == 2
diff --git a/tests/test_rms_rope_compile.py b/tests/test_rms_rope_compile.py
index 05ea43839d1..e0b0e24e968 100644
--- a/tests/test_rms_rope_compile.py
+++ b/tests/test_rms_rope_compile.py
@@ -267,6 +267,23 @@ def test_bf16_without_native_support_stays_untraced(monkeypatch):
assert _graph_breaks() > 0
[email protected]("gemma", [False, True], ids = ["llama", "gemma"])
[email protected]("dim", [64, 4096])
+def test_rms_layernorm_dynamic_shapes(dim, gemma):
+ """dynamic=True gives the op's fake run a symbolic width, which the per-width settings cache
+ cannot hash; a compiled norm must still trace in one graph and match eager bytes."""
+ if not TRACEABLE:
+ pytest.skip("this torch has no torch.library.triton_op")
+ torch._dynamo.reset()
+ compiled = torch.compile(
+ lambda n, x, gm: fast_rms_layernorm(n, x, gm), fullgraph = True, dynamic = True
+ )
+ for seq in (16, 24, 40):
+ shape = (2, seq, dim)
+ eager = _rms_case(fast_rms_layernorm, shape, CDTYPE, gemma, False)
+ _assert_rms_matches(eager, _rms_case(compiled, shape, CDTYPE, gemma, False))
+
+
def test_rope_dynamic_shapes():
"""dynamic=True makes the launch grid SymInts (divmod rejected them)."""
if not TRACEABLE:
diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py
index adc07c89b45..6942271b3b8 100644
--- a/unsloth/kernels/layernorm.py
+++ b/unsloth/kernels/layernorm.py
@@ -247,8 +247,8 @@ def _settings(
device = None,
dw = False,
):
- # Cached per (width, device) for eager calls (host bound at small shapes); tracing calls it directly.
- if torch.compiler.is_compiling():
+ # Cached per (width, device) for eager calls; tracing and symbolic (unhashable) widths skip it.
+ if torch.compiler.is_compiling() or type(n_cols) is not int:
return _arch_settings(n_cols, device, dw)
return _cached_arch_settings(n_cols, device, dw)
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index a7aba6d0f44..0d7868a06d8 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -547,8 +547,8 @@ def _settings(
device = None,
dw = False,
):
- # Cached per (width, device) for eager calls (host bound at small shapes); tracing calls it directly.
- if torch.compiler.is_compiling():
+ # Cached per (width, device) for eager calls; tracing and symbolic (unhashable) widths skip it.
+ if torch.compiler.is_compiling() or type(n_cols) is not int:
return _arch_settings(n_cols, device, dw)
return _cached_arch_settings(n_cols, device, dw)
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 190ad29dfa7..945f8de125e 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -165,6 +165,12 @@ def norm_num_warps(table, n_cols, num_warps, device):
capability = tuple(torch.cuda.get_device_capability(index))
else:
capability = _cuda_capability(index)
+ if type(n_cols) is not int:
+ # A traced op's fake run passes a symbolic width, which cannot be hashed: match by equality.
+ for (table_capability, width), warps in table.items():
+ if table_capability == capability and width == n_cols:
+ return warps
+ return num_warps
return table.get((capability, n_cols), num_warps)