Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

11.9 KB
Raw Blame
# Unsloth Zoo - Utilities for Unsloth
# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

"""rl.py picks float16 autocast when neither flag is set, which overflows float32 models to NaN."""

import ast
import types
from pathlib import Path

import pytest
import torch

REPO_ROOT = Path(__file__).resolve().parents[1]
RL_PY = REPO_ROOT / "unsloth" / "models" / "rl.py"


def _mixed_precision_source() -> str:
    """Extract the `mixed_precision = (...)` string literal from rl.py."""
    src = RL_PY.read_text(encoding = "utf-8")
    tree = ast.parse(src)
    for node in ast.walk(tree):
        if not isinstance(node, ast.Assign):
            continue
        targets = [t.id for t in node.targets if isinstance(t, ast.Name)]
        if "mixed_precision" not in targets:
            continue
        if isinstance(node.value, (ast.Constant, ast.JoinedStr, ast.BinOp)):
            pass
        try:
            return ast.literal_eval(node.value)
        except ValueError:
            continue
    raise AssertionError("mixed_precision block not found in rl.py")


MP_SRC = _mixed_precision_source()


def _get_dtype(dtype):
    """Stand-in for unsloth_zoo.utils._get_dtype: accept a dtype or its name."""
    if isinstance(dtype, torch.dtype):
        return dtype
    return getattr(torch, str(dtype).replace("torch.", ""))


class _Args:
    def __init__(
        self,
        fp16 = False,
        bf16 = False,
    ):
        self.fp16 = fp16
        self.bf16 = bf16


def _run(
    model_dtype,
    bf16_supported,
    fp16 = False,
    bf16 = False,
    force_float32 = "0",
    full_finetuning = "0",
    mixed_precision = "float32",
    user_float32 = None,
    device_type = "cuda",
):
    """Execute the block and report what it decided."""
    config = types.SimpleNamespace(dtype = model_dtype, torch_dtype = model_dtype)
    # from_pretrained records this only for an explicit dtype = torch.float32.
    model = types.SimpleNamespace(
        config = config,
        _unsloth_user_float32 = (
            (model_dtype is torch.float32) if user_float32 is None else user_float32 == "1"
        ),
    )
    args = _Args(fp16 = fp16, bf16 = bf16)
    env = {
        "UNSLOTH_FORCE_FLOAT32": force_float32,
        "UNSLOTH_ENABLE_FULL_FINETUNING": full_finetuning,
        "UNSLOTH_MIXED_PRECISION": mixed_precision,
    }
    fake_os = types.SimpleNamespace(environ = env)

    ns = {
        "torch": torch,
        "os": fake_os,
        "args": args,
        "model": model,
        "print": lambda *a, **k: None,
    }
    # The block falls back to torch.cuda.is_bf16_supported; make both probes agree.
    real_cuda = torch.cuda
    torch.cuda = types.SimpleNamespace(is_bf16_supported = lambda: bf16_supported)
    import sys

    # Stub the parent too: a raising unsloth_zoo __init__ would route through the torch.cuda fallback.
    mod = types.ModuleType("unsloth_zoo.device_type")
    mod.device_is_bf16_supported = lambda: bf16_supported
    mod.DEVICE_TYPE = device_type
    utils = types.ModuleType("unsloth_zoo.utils")
    utils._get_dtype = _get_dtype
    parent = types.ModuleType("unsloth_zoo")
    parent.__path__ = []
    parent.device_type = mod
    parent.utils = utils
    names = ("unsloth_zoo", "unsloth_zoo.device_type", "unsloth_zoo.utils")
    saved = {k: sys.modules.get(k) for k in names}
    sys.modules["unsloth_zoo"] = parent
    sys.modules["unsloth_zoo.device_type"] = mod
    sys.modules["unsloth_zoo.utils"] = utils
    try:
        exec(MP_SRC, ns)
    finally:
        torch.cuda = real_cuda
        for k, v in saved.items():
            if v is None:
                sys.modules.pop(k, None)
            else:
                sys.modules[k] = v
    # The fallback would mask a broken branch, so prove the stub was used.
    assert ns["_bf16_supported"] is mod.device_is_bf16_supported
    return args, env


def test_float32_model_on_t4_stays_float32():
    args, env = _run(torch.float32, bf16_supported = False)
    assert args.fp16 is False, "float32 model must not get float16 autocast"
    assert args.bf16 is False
    assert env["ACCELERATE_MIXED_PRECISION"] == "no"


def test_float32_full_finetuning_on_t4_stays_float32():
    args, env = _run(torch.float32, bf16_supported = False, full_finetuning = "1")
    assert (args.fp16, args.bf16) == (False, False)
    assert env["ACCELERATE_MIXED_PRECISION"] == "no"


def test_float32_model_on_bf16_gpu_still_autocasts():
    # bf16 shares float32's exponent range, so this stays safe and cheap.
    args, env = _run(torch.float32, bf16_supported = True)
    assert args.bf16 is True and args.fp16 is False
    assert env["ACCELERATE_MIXED_PRECISION"] == "bf16"


def test_float16_model_on_t4_still_gets_fp16_autocast():
    args, env = _run(torch.float16, bf16_supported = False)
    assert args.fp16 is True and args.bf16 is False
    assert env["ACCELERATE_MIXED_PRECISION"] == "fp16"


def test_bfloat16_model_on_bf16_gpu_unchanged():
    args, env = _run(torch.bfloat16, bf16_supported = True)
    assert args.bf16 is True and args.fp16 is False
    assert env["ACCELERATE_MIXED_PRECISION"] == "bf16"


def test_explicit_fp16_on_a_float32_model_is_obeyed():
    args, env = _run(torch.float32, bf16_supported = False, fp16 = True)
    assert args.fp16 is True
    assert env["ACCELERATE_MIXED_PRECISION"] == "fp16"


def test_explicit_bf16_on_a_float32_model_is_obeyed():
    args, env = _run(torch.float32, bf16_supported = True, bf16 = True)
    assert args.bf16 is True
    assert env["ACCELERATE_MIXED_PRECISION"] == "bf16"


def test_force_float32_models_take_the_earlier_branch():
    # force_float32 wins before the new branch, so Gemma3 / gpt-oss on a T4 are unchanged.
    args, env = _run(torch.float32, bf16_supported = False, force_float32 = "1")
    assert (args.fp16, args.bf16) == (False, False)
    assert env["ACCELERATE_MIXED_PRECISION"] == "no"


def test_force_float32_full_finetuning_on_bf16_gpu_keeps_bf16_autocast():
    args, env = _run(torch.float32, bf16_supported = True, force_float32 = "1", full_finetuning = "1")
    assert args.bf16 is True and args.fp16 is False
    assert env["ACCELERATE_MIXED_PRECISION"] == "bf16"


def test_bfloat16_mixed_precision_mode_unchanged():
    # UNSLOTH_MIXED_PRECISION = bfloat16 does no autocasting at all.
    args, env = _run(torch.bfloat16, bf16_supported = True, mixed_precision = "bfloat16")
    assert (args.fp16, args.bf16) == (False, False)
    assert env["ACCELERATE_MIXED_PRECISION"] == "no"


def test_upcast_float32_on_a_v100_still_gets_fp16_autocast():
    """An upcast to float32 keeps fp16 autocast; only an explicit float32 load dtype opts out."""
    args, env = _run(torch.float32, bf16_supported = False, full_finetuning = "1", user_float32 = "0")
    assert (args.fp16, args.bf16) == (True, False)
    assert env["ACCELERATE_MIXED_PRECISION"] == "fp16"


def test_loaders_record_the_explicit_request():
    """Every public entry point, since only the outermost one sees the
    argument as the caller wrote it."""
    for rel in ("unsloth/models/loader.py", "unsloth/models/vision.py"):
        src = (REPO_ROOT / rel).read_text(encoding = "utf-8")
        assert "_requested_float32(dtype)" in src, rel
        assert "_mark_requested_float32(" in src, rel


def test_the_legacy_language_model_path_records_it_too():
    """llama, mistral, gemma, gemma2, qwen2 and qwen3 LoRA/QLoRA loads go
    through dispatch_model.from_pretrained, which is neither of the two loaders
    that used to record this. Those are most of the notebooks."""
    src = (REPO_ROOT / "unsloth" / "models" / "loader.py").read_text(encoding = "utf-8")
    tree = ast.parse(src)
    cls = next(
        n for n in ast.walk(tree) if isinstance(n, ast.ClassDef) and n.name == "FastLanguageModel"
    )
    fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")
    body = ast.unparse(fn)
    assert "_requested_float32(dtype)" in body
    # Every exit, including the FastModel hand-offs, or it records a dtype derived from 4bit compute.
    returns = [
        ast.unparse(n) for n in ast.walk(fn) if isinstance(n, ast.Return) and n.value is not None
    ]
    assert returns, "expected the loader to return a model"
    for statement in returns:
        assert "_mark_requested_float32(" in statement, statement


def test_the_text_diffusion_path_records_it_too():
    """_dispatch_diffusion skips the end-of-load stamp, so it must record the float32 request itself."""
    src = (REPO_ROOT / "unsloth" / "models" / "loader.py").read_text(encoding = "utf-8")
    tree = ast.parse(src)
    fn = next(
        n
        for n in ast.walk(tree)
        if isinstance(n, ast.FunctionDef) and n.name == "_dispatch_diffusion"
    )
    returns = [ast.unparse(n) for n in ast.walk(fn) if isinstance(n, ast.Return)]
    assert returns, "expected the diffusion dispatch to return a model"
    for statement in returns:
        assert "_mark_requested_float32(model, user_float32)" in statement, statement


def test_the_direct_diffusion_loader_records_it_too():
    """FastDiffusionModel.from_pretrained called directly never passes through _dispatch_diffusion, so on a
    GPU without bf16 a float32 request trained under fp16 mixed precision with a loss scaler."""
    src = (REPO_ROOT / "unsloth" / "models" / "diffusion.py").read_text(encoding = "utf-8")
    tree = ast.parse(src)
    cls = next(
        n for n in ast.walk(tree) if isinstance(n, ast.ClassDef) and n.name == "FastDiffusionModel"
    )
    fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")
    first = ast.unparse(fn.body[0])
    assert first == "user_float32 = _requested_float32(dtype)", first
    returns = [ast.unparse(n) for n in ast.walk(fn) if isinstance(n, ast.Return)]
    assert returns, "expected the diffusion loader to return a model"
    for statement in returns:
        assert "_mark_requested_float32(model, user_float32)" in statement, statement


def test_the_request_is_read_from_the_model_not_the_environment():
    """The request is stored on each model, since a process-global would describe whichever model
    loaded last and train the first with the second's precision."""
    assert "_unsloth_user_float32" in MP_SRC
    assert "UNSLOTH_USER_FLOAT32" not in MP_SRC


def test_a_model_without_the_marker_keeps_the_old_behaviour():
    """Anything the loaders did not touch must not opt into the new branch."""
    args, _ = _run(torch.float32, bf16_supported = False, user_float32 = "0")
    assert (args.fp16, args.bf16) == (True, False)


def test_block_still_compiles():
    compile(MP_SRC, "mixed_precision", "exec")


if __name__ == "__main__":
    raise SystemExit(pytest.main([__file__, "-q"]))


def test_explicit_bf16_model_on_emulated_xpu_autocasts_bf16():
    """XPU keeps an explicit bf16 load (emulated); fp16 autocast over bf16 weights would overflow."""
    args, env = _run(torch.bfloat16, bf16_supported = False, device_type = "xpu")
    assert (args.bf16, args.fp16) == (True, False)
    assert env["ACCELERATE_MIXED_PRECISION"] == "bf16"
    # A default XPU load there is float16 and keeps float16 autocast; CUDA is unchanged.
    args, _ = _run(torch.float16, bf16_supported = False, device_type = "xpu")
    assert (args.bf16, args.fp16) == (False, True)
    args, _ = _run(torch.bfloat16, bf16_supported = False, device_type = "cuda")
    assert (args.bf16, args.fp16) == (False, True)