Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

2.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

"""UNSLOTH_HIGH_PRECISION_LAYERNORM must upcast every norm in a block, not only names ending in norm."""

import torch
import torch.nn as nn

from unsloth.models.vision import _NORM_MODULE_TYPES


def _selected(model):
    """Mirror of the loader's selector in unsloth/models/vision.py."""
    out = []
    for name, module in model.named_modules():
        if (
            name.endswith(("norm", "norm1", "norm2", "norm3", "norm4"))
            or "layernorm" in name
            or "layer_norm" in name
            or isinstance(module, _NORM_MODULE_TYPES)
        ) and hasattr(module, "weight"):
            out.append(name)
    return set(out)


class _VisionEmbedderLike(nn.Module):
    def __init__(self):
        super().__init__()
        self.patch_ln1 = nn.LayerNorm(8)
        self.patch_ln2 = nn.LayerNorm(8)
        self.pos_norm = nn.LayerNorm(8)
        self.proj = nn.Linear(8, 8)


def test_sibling_norms_are_all_selected():
    got = _selected(_VisionEmbedderLike())
    assert got == {"patch_ln1", "patch_ln2", "pos_norm"}


def test_non_norm_modules_are_not_selected():
    assert "proj" not in _selected(_VisionEmbedderLike())


def test_custom_rmsnorm_still_matched_by_name():
    """Model-specific RMSNorm classes are not torch.nn types, so the name rule must still match them."""

    class RMSNorm(nn.Module):
        def __init__(self):
            super().__init__()
            self.weight = nn.Parameter(torch.ones(8))

    class Block(nn.Module):
        def __init__(self):
            super().__init__()
            self.input_layernorm = RMSNorm()
            self.q_norm = RMSNorm()

    assert _selected(Block()) == {"input_layernorm", "q_norm"}


def test_norm_types_is_non_empty_and_typed():
    assert _NORM_MODULE_TYPES
    assert nn.LayerNorm in _NORM_MODULE_TYPES
    assert all(isinstance(t, type) for t in _NORM_MODULE_TYPES)


def test_weightless_norm_is_skipped():
    """`.to(float32)` on a norm with no weight is a no-op, and named_modules
    exposes such norms, so the hasattr guard must stay."""

    class Weightless(nn.LayerNorm):
        def __init__(self):
            super().__init__(8, elementwise_affine = False)

    class Block(nn.Module):
        def __init__(self):
            super().__init__()
            self.v_norm = Weightless()

    m = Block()
    assert m.v_norm.weight is None
    assert "v_norm" in _selected(m)  # selected, but .to() is harmless