Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

16.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

import copy
import importlib.util
import sys
import gc
import weakref
from pathlib import Path

import pytest
import torch
from real_accelerator import has_real_cuda


@pytest.fixture(scope = "module")
def api():
    path = Path(__file__).resolve().parents[1] / "unsloth/optimizers/bf16_adamw.py"
    spec = importlib.util.spec_from_file_location("_test_bf16_adamw", path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = module
    spec.loader.exec_module(module)
    return module


def test_prepare_preserves_identity_ties_and_frozen_parameters(api):
    model = torch.nn.Module()
    model.first = torch.nn.Linear(4, 4)
    model.second = torch.nn.Linear(4, 4, bias = False)
    model.second.weight = model.first.weight
    model.frozen = torch.nn.Parameter(torch.arange(4.0), requires_grad = False)
    model.register_buffer("statistics", torch.ones(4))
    before = {name: id(p) for name, p in model.named_parameters()}
    frozen = model.frozen.detach().clone()
    api.prepare_bf16_training(model)
    assert {name: id(p) for name, p in model.named_parameters()} == before
    assert model.first.weight is model.second.weight
    assert model.first.weight.dtype == model.first.bias.dtype == torch.bfloat16
    assert model.first.weight.requires_grad and model.first.bias.requires_grad
    assert model.frozen.dtype == torch.float32 and not model.frozen.requires_grad
    assert torch.equal(model.frozen, frozen)
    assert model.statistics.dtype == torch.float32
    api.prepare_bf16_training(model)
    assert {name: id(p) for name, p in model.named_parameters()} == before


@pytest.mark.parametrize("invalid", ["dtype", "gradient", "layout"])
def test_prepare_preflight_does_not_partially_cast(api, invalid):
    model = torch.nn.Module()
    model.first = torch.nn.Parameter(torch.ones(4))
    model.last = torch.nn.Parameter(
        torch.ones(4, dtype = torch.float64 if invalid == "dtype" else torch.float32)
    )
    if invalid == "layout":
        model.last = torch.nn.Parameter(torch.ones(4, 4).T)
    if invalid == "gradient":
        model.last.grad = torch.ones_like(model.last)
    before = [(id(p), p.dtype, p.detach().clone()) for p in model.parameters()]
    with pytest.raises((ValueError, RuntimeError, TypeError)):
        api.prepare_bf16_training(model)
    for param, (identity, dtype, value) in zip(model.parameters(), before):
        assert id(param) == identity and param.dtype == dtype
        assert torch.equal(param, value)


def test_moments_match_independent_fp32_adamw(api):
    p = torch.nn.Parameter(torch.linspace(-2, 2, 65, dtype = torch.bfloat16))
    reference = torch.nn.Parameter(p.detach().float())
    kwargs = dict(lr = 0.003, betas = (0.7, 0.95), eps = 1e-6, weight_decay = 0.03)
    optimizer = api.AdamWBF16([p], seed = 57, **kwargs)
    control = torch.optim.AdamW([reference], foreach = False, fused = False, **kwargs)
    for step in range(1, 5):
        gradient = (torch.arange(65).float().sin() * step / 7).bfloat16()
        p.grad = gradient.clone()
        reference.grad = gradient.float()
        optimizer.step()
        control.step()
        for key in ("exp_avg", "exp_avg_sq"):
            assert optimizer.state[p][key].dtype == torch.float32
            torch.testing.assert_close(
                optimizer.state[p][key], control.state[reference][key], atol = 2e-8, rtol = 2e-6
            )
        assert int(optimizer.state[p]["step"]) == step


def test_sub_ulp_update_moves_correct_fraction_and_preserves_rng(api):
    p = torch.nn.Parameter(torch.ones(65536, dtype = torch.bfloat16))
    optimizer = api.AdamWBF16([p], lr = 2**-10, betas = (0.0, 0.0), eps = 0.0, weight_decay = 0.0, seed = 71)
    p.grad = torch.ones_like(p)
    global_rng = torch.random.get_rng_state().clone()
    optimizer.step()
    assert torch.equal(torch.random.get_rng_state(), global_rng)
    lower = torch.tensor(1 - 2**-8, dtype = torch.bfloat16)
    assert torch.all((p == 1) | (p == lower))
    moved = (p == lower).sum().item()
    assert abs(moved - p.numel() / 4) < 6 * (p.numel() * 0.25 * 0.75) ** 0.5
    assert abs(p.float().mean().item() - (1 - 2**-10)) < 3e-5
    mask = (p != 1).clone()
    with torch.no_grad():
        p.fill_(1)
    optimizer.step()
    assert 0.3 < float((mask != (p != 1)).float().mean()) < 0.45


def test_checkpoint_restores_fp32_moments_and_next_random_update(api):
    p = torch.nn.Parameter(torch.linspace(0.5, 1.5, 4096, dtype = torch.bfloat16))
    optimizer = api.AdamWBF16([p], lr = 0.0009, weight_decay = 0.01, seed = 81)
    for step in range(3):
        p.grad = (torch.arange(p.numel()).float().cos() + step / 10).bfloat16()
        optimizer.step()
    state = copy.deepcopy(optimizer.state_dict())
    restored_p = torch.nn.Parameter(p.detach().clone())
    restored = api.AdamWBF16([restored_p], lr = 0.2, seed = 999)
    restored.load_state_dict(state)
    for key in ("exp_avg", "exp_avg_sq"):
        assert restored.state[restored_p][key].dtype == torch.float32
        assert torch.equal(restored.state[restored_p][key], optimizer.state[p][key])
    for step in range(3):
        gradient = (torch.arange(p.numel()).float().sin() + step / 20).bfloat16()
        p.grad = gradient.clone()
        restored_p.grad = gradient.clone()
        optimizer.step()
        restored.step()
        assert torch.equal(p, restored_p)
        for key in ("step", "exp_avg", "exp_avg_sq"):
            assert torch.equal(
                torch.as_tensor(optimizer.state[p][key]),
                torch.as_tensor(restored.state[restored_p][key]),
            )


def test_none_gradient_is_skipped_and_groups_keep_own_steps(api):
    p = torch.nn.Parameter(torch.ones(64, dtype = torch.bfloat16))
    q = torch.nn.Parameter(torch.full((64,), 2.0, dtype = torch.bfloat16))
    optimizer = api.AdamWBF16([p], lr = 0.01, weight_decay = 0.1)
    optimizer.add_param_group({"params": [q], "lr": 0.0})
    p.grad = torch.ones_like(p)
    optimizer.step()
    if optimizer.state.get(q):
        assert int(optimizer.state[q]["step"]) == 0
        assert torch.count_nonzero(optimizer.state[q]["exp_avg"]) == 0
        assert torch.count_nonzero(optimizer.state[q]["exp_avg_sq"]) == 0
    assert torch.all(q == 2)
    p_before = p.detach().clone()
    p.grad = None
    q.grad = torch.ones_like(q)
    optimizer.step()
    assert torch.equal(p, p_before)
    assert int(optimizer.state[p]["step"]) == 1
    assert int(optimizer.state[q]["step"]) == 1
    assert torch.all(q == 2)
    optimizer.param_groups[1]["lr"] = 0.03
    optimizer.step()
    assert torch.all(q < 2)
    assert int(optimizer.state[q]["step"]) == 2


def _tensor_bytes(value):
    if isinstance(value, torch.Tensor):
        return value.numel() * value.element_size()
    if isinstance(value, dict):
        return sum(_tensor_bytes(v) for v in value.values())
    if isinstance(value, (list, tuple)):
        return sum(_tensor_bytes(v) for v in value)
    return 0


def test_persistent_storage_has_no_master_weight(api):
    p = torch.nn.Parameter(torch.ones(4096, dtype = torch.bfloat16))
    optimizer = api.AdamWBF16([p])
    p.sum().backward()
    optimizer.step()
    storage = p.numel() * p.element_size() + p.grad.numel() * p.grad.element_size()
    storage += _tensor_bytes(optimizer.state_dict())
    assert p.grad.dtype == torch.bfloat16
    assert storage <= 12 * p.numel() + 8192
    for key, value in optimizer.state[p].items():
        if isinstance(value, torch.Tensor) and value.numel() == p.numel():
            assert key in ("exp_avg", "exp_avg_sq")


def test_optimizer_does_not_retain_gradients(api):
    p = torch.nn.Parameter(torch.ones(32, dtype = torch.bfloat16))
    optimizer = api.AdamWBF16([p])
    p.sum().backward()
    gradient = weakref.ref(p.grad)
    optimizer.step()
    optimizer.zero_grad(set_to_none = True)
    gc.collect()
    assert gradient() is None


def test_frozen_parameters_have_no_moments_and_can_be_unfrozen(api):
    frozen = torch.nn.Parameter(torch.ones(65536), requires_grad = False)
    packed = torch.nn.Parameter(torch.ones(65536, dtype = torch.uint8), requires_grad = False)
    p = torch.nn.Parameter(torch.ones(32, dtype = torch.bfloat16))
    optimizer = api.AdamWBF16([frozen, packed, p])
    assert not optimizer.state
    assert frozen not in optimizer.state and packed not in optimizer.state
    p.sum().backward()
    optimizer.step()
    assert frozen not in optimizer.state and packed not in optimizer.state
    frozen.data = frozen.data.bfloat16()
    frozen.requires_grad_(True)
    frozen.grad = torch.ones_like(frozen)
    optimizer.step()
    assert int(optimizer.state[frozen]["step"]) == 1
    assert torch.any(frozen != 1)
    assert packed not in optimizer.state


@pytest.mark.parametrize("dtype", [torch.float16, torch.float64, torch.complex64])
def test_unsupported_parameter_dtype_is_rejected(api, dtype):
    p = torch.nn.Parameter(torch.ones(8, dtype = dtype))
    with pytest.raises((ValueError, RuntimeError, TypeError)):
        optimizer = api.AdamWBF16([p])
        p.grad = torch.ones_like(p)
        optimizer.step()


def test_sparse_gradient_is_rejected_without_updating_parameter(api):
    p = torch.nn.Parameter(torch.ones(8, dtype = torch.bfloat16))
    optimizer = api.AdamWBF16([p])
    p.grad = torch.sparse_coo_tensor(
        [[1, 3]], torch.ones(2, dtype = p.dtype), (8,), check_invariants = True
    )
    before = p.detach().clone()
    with pytest.raises((ValueError, RuntimeError, TypeError)):
        optimizer.step()
    assert torch.equal(p, before)


def _cuda_pair(api):
    initial = torch.linspace(0.5, 1.5, 4096, device = "cuda", dtype = torch.bfloat16)
    params = [torch.nn.Parameter(initial.clone()) for _ in range(2)]
    optimizers = [api.AdamWBF16([p], lr = 0.0009, seed = 73) for p in params]
    for p in params:
        p.grad = torch.ones_like(p)
    return params, optimizers


@pytest.mark.gpu
@pytest.mark.skipif(not has_real_cuda(), reason = "requires real CUDA")
def test_gpu_stochastic_update_has_expected_mean_and_preserves_rng(api):
    p = torch.nn.Parameter(torch.ones(65536, device = "cuda", dtype = torch.bfloat16))
    optimizer = api.AdamWBF16([p], lr = 2**-10, betas = (0.0, 0.0), eps = 0.0, weight_decay = 0.0)
    p.grad = torch.ones_like(p)
    rng = torch.cuda.get_rng_state().clone()
    optimizer.step()
    assert torch.equal(torch.cuda.get_rng_state(), rng)
    lower = 1 - 2**-8
    assert torch.all((p == 1) | (p == lower))
    moved = int((p == lower).sum())
    assert abs(moved - p.numel() / 4) < 6 * (p.numel() * 0.25 * 0.75) ** 0.5
    assert abs(float(p.float().mean()) - (1 - 2**-10)) < 3e-5
    mask = (p != 1).clone()
    with torch.no_grad():
        p.fill_(1)
    optimizer.step()
    assert 0.3 < float((mask != (p != 1)).float().mean()) < 0.45


@pytest.mark.gpu
@pytest.mark.skipif(not has_real_cuda(), reason = "requires real CUDA")
def test_gpu_moments_match_independent_fp32_adamw(api):
    p = torch.nn.Parameter(torch.ones(65, device = "cuda", dtype = torch.bfloat16))
    reference = torch.nn.Parameter(p.detach().float().cpu())
    kwargs = dict(lr = 0.003, betas = (0.7, 0.95), eps = 1e-6, weight_decay = 0.03)
    optimizer = api.AdamWBF16([p], **kwargs)
    control = torch.optim.AdamW([reference], **kwargs)
    for step in range(1, 5):
        gradient = (torch.arange(65).float().sin() * step / 7).bfloat16()
        p.grad = gradient.to("cuda")
        reference.grad = gradient.float()
        optimizer.step()
        control.step()
        for key in ("exp_avg", "exp_avg_sq"):
            torch.testing.assert_close(
                optimizer.state[p][key].cpu(), control.state[reference][key], atol = 2e-8, rtol = 2e-6
            )


def _assert_optimizer_equal(params, optimizers):
    assert torch.equal(*params)
    for key in ("exp_avg", "exp_avg_sq", "step"):
        assert torch.equal(optimizers[0].state[params[0]][key], optimizers[1].state[params[1]][key])


@pytest.mark.gpu
@pytest.mark.skipif(not has_real_cuda(), reason = "requires real CUDA")
def test_compiled_optimizer_matches_eager_without_graph_breaks(api):
    from torch._dynamo.utils import counters

    params, optimizers = _cuda_pair(api)
    torch._dynamo.reset()
    counters.clear()
    compiled = torch.compile(optimizers[1].step, fullgraph = True)
    try:
        for index in range(4):
            gradient = (torch.arange(4096, device = "cuda").float().sin() + index / 10).bfloat16()
            for p in params:
                p.grad.copy_(gradient)
            optimizers[0].step()
            compiled()
            _assert_optimizer_equal(params, optimizers)
        assert int(optimizers[1].state[params[1]]["step"]) == 4
        assert sum(counters["graph_break"].values()) == 0
        assert counters["stats"]["unique_graphs"] > 0
    finally:
        torch._dynamo.reset()


@pytest.mark.gpu
@pytest.mark.skipif(not has_real_cuda(), reason = "requires real CUDA")
def test_cuda_graph_replay_advances_stochastic_updates(api):
    params, optimizers = _cuda_pair(api)
    stream = torch.cuda.Stream()
    stream.wait_stream(torch.cuda.current_stream())
    with torch.cuda.stream(stream):
        for optimizer in optimizers:
            optimizer.step()
    torch.cuda.current_stream().wait_stream(stream)
    torch.cuda.synchronize()
    graph = torch.cuda.CUDAGraph()
    with torch.cuda.graph(graph):
        optimizers[1].step()
    for _ in range(5):
        optimizers[0].step()
        graph.replay()
        _assert_optimizer_equal(params, optimizers)
    assert int(optimizers[1].state[params[1]]["step"]) == 6


@pytest.mark.gpu
@pytest.mark.skipif(not has_real_cuda(), reason = "requires real CUDA")
def test_batched_updates_match_separate_groups_and_new_gradients(api):
    shapes = [0, 1, 17, 255, 256, 257, 1025, 4096, 8193, 33, 65537] * 7
    params = [
        torch.nn.Parameter(torch.ones(n, device = "cuda", dtype = torch.bfloat16)) for n in shapes
    ]
    reference = [torch.nn.Parameter(p.detach().clone()) for p in params]
    optimizer = api.AdamWBF16(params, lr = 0.0001)
    control = api.AdamWBF16([{"params": [p]} for p in reference], lr = 0.0001)
    for step in range(4):
        for i, (p, q) in enumerate(zip(params, reference)):
            gradient = (torch.arange(p.numel(), device = "cuda").float().sin() + step / 10).bfloat16()
            p.grad = None if i == step else gradient.clone()
            q.grad = None if i == step else gradient.clone()
        optimizer.step()
        control.step()
        assert all(torch.equal(p, q) for p, q in zip(params, reference))
        for p, q in zip(params, reference):
            assert bool(optimizer.state.get(p)) == bool(control.state.get(q))
            if not optimizer.state.get(p):
                continue
            for key in ("step", "exp_avg", "exp_avg_sq"):
                assert torch.equal(optimizer.state[p][key], control.state[q][key])


@pytest.mark.parametrize("bad", ["short_moment", "wide_step", "empty_seed"])
def test_load_state_dict_rejects_state_the_kernel_would_overrun(api, bad):
    # The CUDA kernel masks moment reads / writes by the parameter's size and loads step / seed as scalars.
    p = torch.nn.Parameter(torch.ones(8, 4, dtype = torch.bfloat16))
    opt = api.AdamWBF16([p], lr = 1e-3)
    p.grad = torch.ones_like(p)
    opt.step()
    state = opt.state_dict()
    entry = state["state"][0]
    if bad == "short_moment":
        entry["exp_avg"] = torch.zeros(3)
    elif bad == "wide_step":
        entry["step"] = torch.zeros(4, dtype = torch.int64)
    else:
        entry["sr_seed"] = torch.zeros(0, dtype = torch.int64)
    fresh = torch.nn.Parameter(torch.ones(8, 4, dtype = torch.bfloat16))
    with pytest.raises(ValueError):
        api.AdamWBF16([fresh], lr = 1e-3).load_state_dict(state)


def test_load_state_dict_round_trip_still_loads(api):
    p = torch.nn.Parameter(torch.ones(8, 4, dtype = torch.bfloat16))
    opt = api.AdamWBF16([p], lr = 1e-3)
    p.grad = torch.ones_like(p)
    opt.step()
    fresh = torch.nn.Parameter(torch.ones(8, 4, dtype = torch.bfloat16))
    other = api.AdamWBF16([fresh], lr = 1e-3)
    other.load_state_dict(opt.state_dict())
    assert torch.equal(other.state[fresh]["exp_avg"], opt.state[p]["exp_avg"])