Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

21.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.


import functools
import multiprocessing
import os

if multiprocessing.parent_process() is not None or getattr(
    multiprocessing.current_process(), "_inheriting", False
):
    os.environ["CUDA_VISIBLE_DEVICES"] = ""  # Gloo ranks must stay on CPU even on GPU hosts.

import sys  # noqa: E402
import tempfile  # noqa: E402

import pytest  # noqa: E402
import torch  # noqa: E402

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from test_pp_native_gloo import _clip_fp64, ppn  # noqa: E402
from test_tp_native_oracle import tpn  # noqa: E402

LLAMA = "hf-internal-testing/tiny-random-LlamaForCausalLM"
QWEN3 = "trl-internal-testing/tiny-Qwen3ForCausalLM"
GEMMA3 = "hf-internal-testing/tiny-random-Gemma3ForCausalLM"

ARCHS = {
    "llama": (LLAMA, {}),
    "qwen3": (QWEN3, {}),
    "gemma3": (GEMMA3, {"num_attention_heads": 4, "num_key_value_heads": 2}),
}
_LOCAL_CONFIG = False
N_LAYERS = 6
STEPS = 3
# Gemma3 reduction-order noise needs a wider bound than Llama/Qwen3.

STEP0_NORM_REL = {"llama": 1e-5, "qwen3": 1e-5, "gemma3": 3e-4}
# CPU ISA and TP summation order move near-zero Gemma3 gradients; keep other models strict.


GRAD_ATOL_REL = {"llama": 2e-5, "qwen3": 2e-5, "gemma3": 1e-3}
GRAD_L2_REL = {"llama": 5e-5, "qwen3": 5e-5, "gemma3": 5e-4}


def _native():
    module = sys.modules.get("unsloth.models.fsdp2_native")
    if module is None:
        from test_fsdp2_native_units import _load_native
        module = _load_native()
    return module


def build(
    repo,
    overrides = None,
    seed = 0,
):
    from peft import LoraConfig, get_peft_model
    from transformers import AutoConfig, AutoModelForCausalLM

    probe = AutoConfig.from_pretrained(repo, local_files_only = _LOCAL_CONFIG)
    values = probe.to_dict()
    values["num_hidden_layers"] = N_LAYERS
    values.pop("layer_types", None)
    config = type(probe).from_dict(values)
    text = config.get_text_config()
    for key, value in (overrides or {}).items():
        setattr(text, key, value)
    for key in ("attention_dropout", "hidden_dropout", "dropout"):
        if isinstance(getattr(text, key, None), float):
            setattr(text, key, 0.0)
    config._attn_implementation = "eager"
    torch.manual_seed(seed)
    model = AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)
    with torch.no_grad():
        for name, p in model.named_parameters():
            if "norm" in name and p.ndim == 1:
                p.add_(torch.randn_like(p) * 0.1)
    torch.manual_seed(seed + 1)
    lora = LoraConfig(
        r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False, lora_dropout = 0.0
    )
    return get_peft_model(model, lora).train()


def global_batch(
    model,
    bsz,
    seq = 8,
    seed = 1,
):
    vocab = ppn._text_config(model).vocab_size
    g = torch.Generator().manual_seed(seed)
    ids = torch.randint(0, vocab, (bsz, seq), generator = g)
    labels = ids.clone()
    labels[0, :3] = -100  # Uneven token counts exercise global normalization.
    labels[-1, -2:] = -100
    labels[1, -1] = -100
    return ids, labels


def _reference(model, ids, labels, steps):
    params = [p for p in model.parameters() if p.requires_grad]
    opt = torch.optim.AdamW(params, lr = 1e-2, foreach = False)
    losses, norms, grads = [], [], None
    for step in range(steps):
        out = model(input_ids = ids, labels = labels, use_cache = False)
        out.loss.backward()
        if step == 0:
            grads = {
                n: p.grad.detach().clone()
                for n, p in model.named_parameters()
                if p.grad is not None
            }
        norms.append(_clip_fp64(params, 1.0))
        opt.step()
        opt.zero_grad(set_to_none = True)
        losses.append(float(out.loss))
    return {"losses": losses, "norms": norms, "grads": grads}


def _trainable_param_groups(model, trainable = True):
    from torch.distributed.fsdp import FSDPModule

    groups = []
    for m in model.modules():
        if isinstance(m, FSDPModule):
            state = m._get_fsdp_state()
            found = getattr(state, "_fsdp_param_groups", None)
            if found is None:
                group = getattr(state, "_fsdp_param_group", None)
                found = [group] if group is not None else []
            for group in found:
                if not trainable or any(fp.sharded_param.requires_grad for fp in group.fsdp_params):
                    groups.append(group)
    return groups


def _count_reduces():
    import torch.distributed.fsdp._fully_shard._fsdp_param_group as pg

    calls = [0]
    original = pg.foreach_reduce

    def counted(*args, **kwargs):
        calls[0] += 1
        return original(*args, **kwargs)

    pg.foreach_reduce = counted
    return calls


def _raised(fn):
    try:
        fn()
    except (ValueError, RuntimeError) as exc:
        return str(exc)
    return None


def _case_compose(
    rank,
    world,
    tmp,
    arch,
    dims,
    schedule = "gpipe",
    n_mb = 2,
    fit_layout = None,
    rows = 4,
):
    """Split over pp before applying orthogonal tp/dp meshes; each dp rank gets its batch slice."""
    import torch.distributed as dist
    from safetensors.torch import load_file
    from torch.distributed.device_mesh import init_device_mesh

    native = _native()
    repo, overrides = ARCHS[arch]
    names = tuple(dims)
    mesh = init_device_mesh("cpu", tuple(dims.values()), mesh_dim_names = names)
    dp = dims.get("dp", 1)
    dp_rank = mesh["dp"].get_local_rank() if "dp" in dims else 0
    ref = build(repo, overrides)
    ids, labels = global_batch(ref, rows * dp)
    ref_res = _reference(ref, ids, labels, STEPS)
    if rank == 0:
        ref.save_pretrained(os.path.join(tmp, "ref"))
    mine = slice(rows * dp_rank, rows * (dp_rank + 1))
    model = build(repo, overrides)
    pp = ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = n_mb, schedule = schedule)
    refusals = {}
    if world != dims["pp"]:
        refusals["step_before_all_dims"] = _raised(lambda: pp.step(ids[mine], labels[mine]))
    if "tp" in dims:
        tpn.apply_tensor_parallel(model, mesh["tp"])
    if "dp" in dims:
        refusals["shard_model_pp_mesh"] = _raised(
            lambda: native.shard_model(
                model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["pp"])
            )
        )
        refusals["shard_model_no_mesh"] = _raised(
            lambda: native.shard_model(
                model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32)
            )
        )
        refusals["shard_model_inner_level"] = _raised(
            lambda: native.shard_model(
                model.base_model,
                native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"]),
            )
        )
        refusals["shard_model_cpu_offload"] = _raised(
            lambda: native.shard_model(
                model,
                native.FSDP2Config(
                    device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"], cpu_offload = True
                ),
            )
        )
        refusals["fsdp_after_refusals"] = sum(
            type(m).__name__.startswith("FSDP") for m in model.modules()
        )
        native.shard_model(
            model,
            native.FSDP2Config(
                device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"], fit_layout = fit_layout
            ),
        )
    from torch.distributed.fsdp import FSDPModule

    layouts = tpn._param_layouts(model)
    groups = _trainable_param_groups(model)
    calls = _count_reduces()
    params = pp.parameters()
    opt = torch.optim.AdamW(params, lr = 1e-2, foreach = False)
    losses, norms, reduces, grads = [], [], [], None
    for step in range(STEPS):
        before = calls[0]
        loss = pp.step(ids[mine], labels[mine])
        reduces.append(calls[0] - before)
        if step == 0:
            grads = {
                n: tpn._full(p.grad, layout = layouts.get(id(p))).detach().clone()
                for n, p in model.named_parameters()
                if p.grad is not None
            }
        norms.append(float(pp.clip_grad_norm_(1.0)))
        opt.step()
        opt.zero_grad(set_to_none = True)
        losses.append(float(loss))
    pp.save_adapter(os.path.join(tmp, "pp"))
    dist.barrier()
    placements = {
        n: tuple(str(pl) for pl in p.placements)
        for n, p in model.named_parameters()
        if p.requires_grad and tpn._is_dtensor(p)
    }
    return {
        "ref": ref_res,
        "losses": losses,
        "norms": norms,
        "grads": grads,
        "owned": {n for n, p in model.named_parameters() if p.requires_grad},
        "reduces": reduces,
        "groups": len(groups),
        "reshard_after_forward": [
            g.post_forward_mesh_info is not None
            for g in _trainable_param_groups(model, trainable = False)
        ],
        "stage_root_fsdp": isinstance(pp.module, FSDPModule),
        "placements": placements,
        "refusals": refusals,
        "stage": pp.stage_index,
        "saved_ref": load_file(os.path.join(tmp, "ref", "adapter_model.safetensors"))
        if rank == 0
        else None,
        "saved_pp": load_file(os.path.join(tmp, "pp", "adapter_model.safetensors"))
        if rank == 0
        else None,
    }


def _case_scaler(rank, world, tmp):
    from torch.distributed.device_mesh import init_device_mesh

    native = _native()
    mesh = init_device_mesh("cpu", (2, 2), mesh_dim_names = ("pp", "dp"))
    model = build(LLAMA)
    ids, labels = global_batch(model, 8)
    dp_rank = mesh["dp"].get_local_rank()
    pp = ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = 2)
    native.shard_model(
        model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"])
    )
    scaler = torch.amp.GradScaler("cpu", init_scale = 1024.0)
    opt = torch.optim.SGD(pp.parameters(), lr = 1.0)
    pp.step(
        ids[4 * dp_rank : 4 * dp_rank + 4], labels[4 * dp_rank : 4 * dp_rank + 4], scaler = scaler
    )
    if rank == world - 1:
        # Inject before PP agreement; DTensor gradients already reduce infs over DP during unscale.
        unscale_grads = scaler._unscale_grads_

        def unscale_with_inf(*args, **kwargs):
            found = unscale_grads(*args, **kwargs)
            for t in found.values():
                t.fill_(1.0)
            return found

        scaler._unscale_grads_ = unscale_with_inf
    before = [tpn._local(p).detach().clone() for p in pp.parameters()]
    pp.unscale_(scaler, opt)
    scaler.step(opt)
    scaler.update()
    after = [tpn._local(p).detach().clone() for p in pp.parameters()]
    return {
        "unchanged": all(torch.equal(a, b) for a, b in zip(before, after)),
        "scale": scaler.get_scale(),
    }


def _case_order_refusals(rank, world, tmp):
    from torch.distributed.device_mesh import init_device_mesh

    native = _native()
    mesh = init_device_mesh("cpu", (2, 2), mesh_dim_names = ("pp", "dp"))
    out = {}
    model = build(LLAMA)
    tpn.apply_tensor_parallel(model, mesh["dp"])
    layers = len(model.get_base_model().model.layers)
    out["tp_first"] = _raised(lambda: ppn.apply_pipeline_parallel(model, mesh["pp"]))
    out["tp_first_unchanged"] = not ppn.is_pipeline_parallel(model) and layers == N_LAYERS
    model = build(LLAMA)
    native.shard_model(
        model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"])
    )
    out["fsdp_first"] = _raised(lambda: ppn.apply_pipeline_parallel(model, mesh["pp"]))
    out["fsdp_first_unchanged"] = not ppn.is_pipeline_parallel(model)

    model = build(LLAMA)
    out["mesh_2d"] = _raised(lambda: ppn.apply_pipeline_parallel(model, mesh))

    pp = ppn.apply_pipeline_parallel(model, mesh["pp"])
    native.shard_model(
        model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"])
    )
    out["tp_after_fsdp"] = _raised(lambda: tpn.apply_tensor_parallel(model, mesh["dp"]))
    out["shard_twice_is_noop"] = native.shard_model(model) is model
    out["composed"] = pp.composed
    return out


def _case_pp_tp_mesh_refusal(rank, world, tmp):
    from torch.distributed.device_mesh import init_device_mesh

    mesh = init_device_mesh("cpu", (2, 2), mesh_dim_names = ("pp", "tp"))
    model = build(LLAMA)
    ppn.apply_pipeline_parallel(model, mesh["pp"])
    err = _raised(lambda: tpn.apply_tensor_parallel(model, mesh["pp"]))
    return {"err": err, "tp": tpn.is_tensor_parallel(model)}


def _case_fp16_replicas(rank, world, tmp):
    from torch.distributed.device_mesh import init_device_mesh

    native = _native()
    mesh = init_device_mesh("cpu", (2, 2), mesh_dim_names = ("pp", "dp"))
    model = build(LLAMA)
    norm = model.get_base_model().model.norm.weight
    norm.data = norm.data.half()
    norm.requires_grad_(True)
    ids, labels = global_batch(model, 8)
    model(input_ids = ids, labels = labels, use_cache = False).loss.backward()
    expected = norm.grad.detach().float().clone()
    model.zero_grad(set_to_none = True)
    pp = ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = 2)
    native.shard_model(
        model,
        native.FSDP2Config(
            device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"], ignored_params = {norm}
        ),
    )
    dp_rank = mesh["dp"].get_local_rank()
    grads, pending = [], []
    for _ in range(2):
        pp.step(ids[4 * dp_rank : 4 * dp_rank + 4], labels[4 * dp_rank : 4 * dp_rank + 4])
        if pp.is_last:
            grads.append(None if norm.grad is None else norm.grad.detach().float().clone())
            pending.append(bool(model.__dict__["_unsloth_replicated_grad_state"].get("pending")))
        model.zero_grad(set_to_none = True)
    return {"last": pp.is_last, "grads": grads, "expected": expected, "pending": pending}


CASES = {
    "compose": _case_compose,
    "scaler": _case_scaler,
    "order": _case_order_refusals,
    "pp_tp_mesh": _case_pp_tp_mesh_refusal,
    "fp16_replicas": _case_fp16_replicas,
}


def _free_port():
    from tests._rendezvous_port import free_port
    return free_port()


def _worker(rank, world, port, case, tmp, kwargs):
    global _LOCAL_CONFIG
    _LOCAL_CONFIG = True
    os.environ.update(
        MASTER_ADDR = "127.0.0.1",
        MASTER_PORT = str(port),
        RANK = str(rank),
        WORLD_SIZE = str(world),
        LOCAL_RANK = str(rank),
        LOCAL_WORLD_SIZE = str(world),
    )
    import datetime

    import torch.distributed as dist

    torch.set_num_threads(1)
    dist.init_process_group(
        "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 240)
    )
    try:
        res = CASES[case](rank, world, tmp, **kwargs)
        torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
    finally:
        dist.destroy_process_group()


@functools.lru_cache(maxsize = 1)
def _prefetch_configs():
    from transformers import AutoConfig
    for repo, _ in ARCHS.values():
        AutoConfig.from_pretrained(repo)


def _spawn(
    case,
    world = 4,
    **kwargs,
):
    try:
        ppn._check_torch_pipelining()
    except RuntimeError as exc:
        pytest.skip(str(exc))
    native = _native()
    if not native._supports_ignored_params(native._fsdp_api()[2]):
        pytest.skip("native FSDP2 composition needs fully_shard(ignored_params) (torch 2.7+)")
    _prefetch_configs()
    ctx = multiprocessing.get_context("spawn")
    port = _free_port()
    with tempfile.TemporaryDirectory() as tmp:
        procs = [
            ctx.Process(target = _worker, args = (r, world, port, case, tmp, kwargs))
            for r in range(world)
        ]
        for p in procs:
            p.start()
        for p in procs:
            p.join(900)
        codes = [p.exitcode for p in procs]
        assert codes == [0] * world, f"rank exit codes {codes}"
        return [
            torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
            for r in range(world)
        ]


def _check(
    results,
    dims,
    n_mb = 2,
    norm_rel = 1e-5,
    grad_rel = 2e-5,
    l2_rel = 5e-5,
):
    ref = results[0]["ref"]
    union = set()
    for rank, res in enumerate(results):
        assert res["losses"] == pytest.approx(ref["losses"], rel = 1e-5, abs = 1e-6), (
            rank,
            res["losses"],
            ref["losses"],
        )
        assert res["norms"][0] == pytest.approx(ref["norms"][0], rel = norm_rel), (
            rank,
            res["norms"],
            ref["norms"],
        )
        # AdamW updates amplify reduction-order noise in near-zero gradients.

        assert res["norms"] == pytest.approx(ref["norms"], rel = 2e-3), (
            rank,
            res["norms"],
            ref["norms"],
        )
        assert set(res["grads"]) == res["owned"], sorted(set(res["grads"]) ^ res["owned"])[:6]
        num = den = 0.0
        for name, grad in res["grads"].items():
            want = ref["grads"][name]

            atol = max(1e-6, grad_rel * float(want.abs().max()))
            torch.testing.assert_close(
                grad, want, rtol = 1e-4, atol = atol, msg = lambda m, name = name: f"{name}: {m}"
            )
            num += float((grad.double() - want.double()).square().sum())
            den += float(want.double().square().sum())
        assert (num / den) ** 0.5 < l2_rel, (rank, (num / den) ** 0.5)
        union |= set(res["grads"])
        refusals = res["refusals"]
        if len(results) != dims["pp"]:
            assert "Apply the other mesh dimensions" in (
                refusals["step_before_all_dims"] or ""
            ), refusals
        if "dp" in dims:
            assert res["stage_root_fsdp"]

            assert res["placements"] and all(
                any("S(" in pl or "Shard" in pl for pl in pls) for pls in res["placements"].values()
            ), res["placements"]
            assert res["groups"] >= 1 and res["reduces"] == [res["groups"]] * STEPS, (
                res["reduces"],
                res["groups"],
            )
            assert res["reshard_after_forward"] and not any(res["reshard_after_forward"]), res[
                "reshard_after_forward"
            ]
            assert "meeting only at this rank" in (refusals["shard_model_pp_mesh"] or ""), refusals
            assert "needs FSDP2Config(mesh = mesh['dp'])" in (
                refusals["shard_model_no_mesh"] or ""
            ), refusals
            assert "not a level inside it" in (refusals["shard_model_inner_level"] or ""), refusals
            assert "cpu_offload is not supported on a pipeline parallel model" in (
                refusals["shard_model_cpu_offload"] or ""
            ), refusals
            assert refusals["fsdp_after_refusals"] == 0, refusals
            if dims["dp"] == 1:
                continue
        else:
            assert res["reduces"] == [0] * STEPS
        if "tp" in dims:
            assert res["placements"], "no tensor parallel DTensors"
    assert union == set(ref["grads"]), sorted(union ^ set(ref["grads"]))[:6]
    saved_ref, saved_pp = results[0]["saved_ref"], results[0]["saved_pp"]
    assert set(saved_pp) == set(saved_ref), sorted(set(saved_pp) ^ set(saved_ref))[:6]
    for key, value in saved_ref.items():
        assert saved_pp[key].shape == value.shape, key

        torch.testing.assert_close(saved_pp[key], value, rtol = 1e-3, atol = 2e-3, msg = key)


@pytest.mark.parametrize("arch", sorted(ARCHS))
def test_pp2_x_fsdp2_dp2_matches_one_process(arch):
    dims = {"pp": 2, "dp": 2}
    _check(
        _spawn("compose", arch = arch, dims = dims),
        dims,
        norm_rel = STEP0_NORM_REL[arch],
        grad_rel = GRAD_ATOL_REL[arch],
        l2_rel = GRAD_L2_REL[arch],
    )


def test_pp2_x_fsdp2_dp2_1f1b_sharded_layout_matches_one_process():
    dims = {"pp": 2, "dp": 2}
    _check(
        _spawn("compose", arch = "llama", dims = dims, schedule = "1f1b", n_mb = 4, fit_layout = False),
        dims,
    )


@pytest.mark.parametrize("arch", sorted(ARCHS))
def test_pp2_x_tp2_matches_one_process(arch):
    dims = {"pp": 2, "tp": 2}
    _check(
        _spawn("compose", arch = arch, dims = dims),
        dims,
        norm_rel = STEP0_NORM_REL[arch],
        grad_rel = GRAD_ATOL_REL[arch],
        l2_rel = GRAD_L2_REL[arch],
    )


@pytest.mark.parametrize("other", ["dp", "tp"])
def test_pp2_x_size_one_dp_or_tp_matches_one_process(other):
    dims = {"pp": 2, other: 1}
    _check(_spawn("compose", world = 2, arch = "llama", dims = dims), dims)


def test_pp2_x_dp2_x_tp2_matches_one_process():
    dims = {"pp": 2, "dp": 2, "tp": 2}
    _check(_spawn("compose", world = 8, arch = "llama", dims = dims), dims)


def test_pp2_x_dp2_grad_scaler_skip_is_global():
    results = _spawn("scaler")
    for res in results:
        assert res["unchanged"], res
        assert res["scale"] == 512.0, res


def test_pp2_x_fsdp2_dp2_consumes_fp16_replicated_gradients():
    for res in _spawn("fp16_replicas"):
        if not res["last"]:
            continue
        assert res["pending"] == [False, False], res
        assert res["expected"].norm() > 0
        for grad in res["grads"]:
            assert grad is not None, res
            torch.testing.assert_close(grad, res["expected"], rtol = 1e-2, atol = 1e-4)


def test_pp_composition_order_and_meshes_are_refused_before_any_change():
    for res in _spawn("order"):
        assert "must come before apply_tensor_parallel" in (res["tp_first"] or ""), res
        assert res["tp_first_unchanged"], res
        assert "must come before FSDP2" in (res["fsdp_first"] or ""), res
        assert res["fsdp_first_unchanged"], res
        assert "1-D pp mesh" in (res["mesh_2d"] or ""), res
        assert res["tp_after_fsdp"], res
        assert res["shard_twice_is_noop"], res
        assert res["composed"], res
    for res in _spawn("pp_tp_mesh"):
        assert "does not compose with Unsloth pipeline parallelism on these meshes" in (
            res["err"] or ""
        ), res
        assert not res["tp"], res