Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
RMSNorm / LayerNorm: do not hash a symbolic width in the launch settings cache (#14727)
Daniel Handa007294dcc5
diff --git a/tests/test_norm_settings_symbolic_width.py b/tests/test_norm_settings_symbolic_width.pynew file mode 100644index 00000000000..a90103efab5--- /dev/null+++ b/tests/test_norm_settings_symbolic_width.py@@ -0,0 +1,57 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The per-width launch settings cache must not hash a symbolic width: a traced norm op's fake run+under dynamic=True passes one, outside Dynamo (is_compiling() is False there)."""++import pytest+import torch++pytest.importorskip("triton")++from torch._dynamo.source import ConstantSource+from torch.fx.experimental.symbolic_shapes import ShapeEnv++from unsloth.kernels import layernorm, rms_layernorm+from unsloth.kernels import utils as kernel_utils+from unsloth.kernels.utils import calculate_settings+++def _symbolic(n):+ env = ShapeEnv()+ return env.create_symintnode(env.create_symbol(n, ConstantSource("n_cols")), hint = n)++[email protected]("module", [rms_layernorm, layernorm], ids = ["rms", "layernorm"])[email protected]("n", [64, 4096])+def test_a_symbolic_width_skips_the_cache(module, n):+ width = _symbolic(n)+ with pytest.raises(TypeError):+ hash(width)+ assert tuple(module._settings(width)) == tuple(calculate_settings(n))++[email protected]("module", [rms_layernorm, layernorm], ids = ["rms", "layernorm"])[email protected]("dw", [False, True], ids = ["fwd", "dw"])+def test_a_symbolic_width_reads_the_cuda_warps_table(module, dw, monkeypatch):+ # On CUDA the warps come from a (capability, width) table, which must not hash the width either.+ table = module._DW_NUM_WARPS if dw else module._NUM_WARPS+ (capability, n), warps = next(iter(table.items()))+ monkeypatch.setattr(kernel_utils, "_NORM_ARCH_WARPS", True)+ monkeypatch.setattr(kernel_utils, "_cuda_capability", lambda index: capability)+ monkeypatch.setattr(torch.version, "hip", None)+ device = torch.device("cuda", 0)+ expected = module._arch_settings(n, device, dw)+ assert expected == (calculate_settings(n)[0], warps)+ assert tuple(module._settings(_symbolic(n), device, dw)) == expected+ # A width the table does not list keeps calculate_settings' warps.+ assert tuple(module._settings(_symbolic(n + 1), device, dw)) == tuple(calculate_settings(n + 1))++[email protected]("module", [rms_layernorm, layernorm], ids = ["rms", "layernorm"])+def test_an_int_width_still_uses_the_cache(module):+ module._cached_arch_settings.cache_clear()+ for _ in range(3):+ assert module._settings(256) == calculate_settings(256)+ info = module._cached_arch_settings.cache_info()+ assert info.misses == 1 and info.hits == 2diff --git a/tests/test_rms_rope_compile.py b/tests/test_rms_rope_compile.pyindex 05ea43839d1..e0b0e24e968 100644--- a/tests/test_rms_rope_compile.py+++ b/tests/test_rms_rope_compile.py@@ -267,6 +267,23 @@ def test_bf16_without_native_support_stays_untraced(monkeypatch):assert _graph_breaks() > 0[email protected]("gemma", [False, True], ids = ["llama", "gemma"])[email protected]("dim", [64, 4096])+def test_rms_layernorm_dynamic_shapes(dim, gemma):+ """dynamic=True gives the op's fake run a symbolic width, which the per-width settings cache+ cannot hash; a compiled norm must still trace in one graph and match eager bytes."""+ if not TRACEABLE:+ pytest.skip("this torch has no torch.library.triton_op")+ torch._dynamo.reset()+ compiled = torch.compile(+ lambda n, x, gm: fast_rms_layernorm(n, x, gm), fullgraph = True, dynamic = True+ )+ for seq in (16, 24, 40):+ shape = (2, seq, dim)+ eager = _rms_case(fast_rms_layernorm, shape, CDTYPE, gemma, False)+ _assert_rms_matches(eager, _rms_case(compiled, shape, CDTYPE, gemma, False))++def test_rope_dynamic_shapes():"""dynamic=True makes the launch grid SymInts (divmod rejected them)."""if not TRACEABLE:diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.pyindex adc07c89b45..6942271b3b8 100644--- a/unsloth/kernels/layernorm.py+++ b/unsloth/kernels/layernorm.py@@ -247,8 +247,8 @@ def _settings(device = None,dw = False,):- # Cached per (width, device) for eager calls (host bound at small shapes); tracing calls it directly.- if torch.compiler.is_compiling():+ # Cached per (width, device) for eager calls; tracing and symbolic (unhashable) widths skip it.+ if torch.compiler.is_compiling() or type(n_cols) is not int:return _arch_settings(n_cols, device, dw)return _cached_arch_settings(n_cols, device, dw)diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex a7aba6d0f44..0d7868a06d8 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -547,8 +547,8 @@ def _settings(device = None,dw = False,):- # Cached per (width, device) for eager calls (host bound at small shapes); tracing calls it directly.- if torch.compiler.is_compiling():+ # Cached per (width, device) for eager calls; tracing and symbolic (unhashable) widths skip it.+ if torch.compiler.is_compiling() or type(n_cols) is not int:return _arch_settings(n_cols, device, dw)return _cached_arch_settings(n_cols, device, dw)diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 190ad29dfa7..945f8de125e 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -165,6 +165,12 @@ def norm_num_warps(table, n_cols, num_warps, device):capability = tuple(torch.cuda.get_device_capability(index))else:capability = _cuda_capability(index)+ if type(n_cols) is not int:+ # A traced op's fake run passes a symbolic width, which cannot be hashed: match by equality.+ for (table_capability, width), warps in table.items():+ if table_capability == capability and width == n_cols:+ return warps+ return num_warpsreturn table.get((capability, n_cols), num_warps)