Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

37.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

"""UNSLOTH_PP_SIZE (models/pp_env.py): pipeline parallel from the environment inside the HF Trainer, on a gloo
CPU world (pp 2, pp 2 x tp 2, pp 2 x dp 2 DDP-style / native FSDP2), vs the same script in one process.

The workers set the torchrun environment and let tp_env create the process group and the mesh; the loader /
get_peft_model hooks wrap AutoModelForCausalLM / PEFT the way they wrap FastLanguageModel / FastModel."""

import multiprocessing
import os

if multiprocessing.parent_process() is not None or getattr(
    multiprocessing.current_process(), "_inheriting", False
):
    os.environ["CUDA_VISIBLE_DEVICES"] = ""  # spawned gloo ranks: CPU only, also on GPU hosts

import sys  # noqa: E402
import tempfile  # noqa: E402

import pytest  # noqa: E402
import torch  # noqa: E402

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from test_tp_native_gloo import (  # noqa: E402
    _Rows,
    _free_port,
    _import_checkout_unsloth,
    _raised,
    _save_checkpoint,
    _trainer_args,
)

_ADAM_EPS = 1e-4  # as test_tp_env_gloo: fp32 reduction-order noise stays below 1e-6 over 3 steps
_ENV_KEYS = (
    "UNSLOTH_TP_SIZE",
    "UNSLOTH_PP_SIZE",
    "UNSLOTH_PP_MICROBATCHES",
    "UNSLOTH_PP_SCHEDULE",
    "UNSLOTH_PP_KEEP_REPLICA",
    "UNSLOTH_FSDP2",
    "UNSLOTH_FSDP2_BACKEND",
    "ACCELERATE_USE_FSDP",
)
N_LAYERS = 4


def _checkpoint(tmp, arch = "llama"):
    overrides = {"num_hidden_layers": N_LAYERS}
    if (
        arch == "gemma3"
    ):  # sliding / full layers on both stages, window shorter than the 9-token rows
        overrides.update(
            layer_types = ["sliding_attention", "full_attention"] * (N_LAYERS // 2), sliding_window = 4
        )
    return _save_checkpoint(arch, os.path.join(tmp, "ckpt"), overrides)


def _case_units(rank, world, tmp):
    """Layout parsing, load / Trainer refusals (no process group)."""
    error = _import_checkout_unsloth()
    if error is not None:
        return {"skip": error}
    from unsloth.models import pp_env, tp_env

    out = {}
    # TrainingArguments set up accelerate's state from WORLD_SIZE: build them before the layouts below set it.
    args = _trainer_args(tmp, 2)
    best = _trainer_args(
        tmp, 2, load_best_model_at_end = True, eval_strategy = "steps", save_strategy = "steps"
    )
    smoothed = _trainer_args(tmp, 2, label_smoothing_factor = 0.3)
    try:
        from trl import SFTConfig, SFTTrainer
        packed = SFTConfig(output_dir = tmp, packing = True, report_to = [], use_cpu = True)
        plain_sft = SFTConfig(output_dir = tmp, report_to = [], use_cpu = True)
    except ImportError:
        SFTTrainer = None

    def layout(**env):
        for key in _ENV_KEYS + ("WORLD_SIZE",):
            os.environ.pop(key, None)
        os.environ.update({k: str(v) for k, v in env.items()})
        tp_env._reset_env_layout()
        try:
            got = tp_env.env_layout()
        except Exception as e:
            return f"{type(e).__name__}: {e}"
        return (
            None
            if got is None
            else (
                got.pp,
                got.dp,
                got.tp,
                got.world,
                got.fsdp2,
                got.microbatches,
                got.schedule,
                repr(got),
            )
        )

    out["pp2"] = layout(UNSLOTH_PP_SIZE = 2, WORLD_SIZE = 2)
    out["pp2_dp2"] = layout(UNSLOTH_PP_SIZE = 2, WORLD_SIZE = 4)
    out["pp2_fsdp2"] = layout(UNSLOTH_PP_SIZE = 2, UNSLOTH_FSDP2 = 1, WORLD_SIZE = 4)
    out["pp2_tp2"] = layout(UNSLOTH_PP_SIZE = 2, UNSLOTH_TP_SIZE = 2, WORLD_SIZE = 4)
    out["pp2_tp2_dp2"] = layout(UNSLOTH_PP_SIZE = 2, UNSLOTH_TP_SIZE = 2, WORLD_SIZE = 8)
    out["pp_one"] = layout(UNSLOTH_PP_SIZE = 1, WORLD_SIZE = 4)
    out["no_divide"] = layout(UNSLOTH_PP_SIZE = 3, WORLD_SIZE = 4)
    out["no_divide_tp"] = layout(UNSLOTH_PP_SIZE = 2, UNSLOTH_TP_SIZE = 2, WORLD_SIZE = 6)
    out["zero"] = layout(UNSLOTH_PP_SIZE = 0, WORLD_SIZE = 2)
    out["text"] = layout(UNSLOTH_PP_SIZE = "two", WORLD_SIZE = 2)
    out["mb"] = layout(UNSLOTH_PP_SIZE = 2, UNSLOTH_PP_MICROBATCHES = 3, WORLD_SIZE = 2)
    out["mb_text"] = layout(UNSLOTH_PP_SIZE = 2, UNSLOTH_PP_MICROBATCHES = "x", WORLD_SIZE = 2)
    out["schedule"] = layout(UNSLOTH_PP_SIZE = 2, UNSLOTH_PP_SCHEDULE = "1F1B", WORLD_SIZE = 2)
    out["schedule_bad"] = layout(UNSLOTH_PP_SIZE = 2, UNSLOTH_PP_SCHEDULE = "zb", WORLD_SIZE = 2)
    out["schedule_mb"] = layout(
        UNSLOTH_PP_SIZE = 4, UNSLOTH_PP_SCHEDULE = "1f1b", UNSLOTH_PP_MICROBATCHES = 2, WORLD_SIZE = 4
    )
    out["accelerate_fsdp"] = layout(
        UNSLOTH_PP_SIZE = 2, UNSLOTH_FSDP2 = 1, UNSLOTH_FSDP2_BACKEND = "accelerate", WORLD_SIZE = 4
    )

    calls = []

    def loader(
        model_name,
        load_in_4bit = True,
        fast_inference = False,
        full_finetuning = False,
        **kw,
    ):
        calls.append(model_name)
        return "loaded"

    layout(UNSLOTH_PP_SIZE = 2, WORLD_SIZE = 2)
    wrapped = tp_env.tensor_parallel_env_load(loader)
    out["refuse_vllm"] = _raised(lambda: wrapped("m", fast_inference = True))
    out["refuse_full_ft"] = _raised(lambda: wrapped("m", False, full_finetuning = True))
    out["refused_before_load"] = calls == []

    # Trainer refusals (classes / arguments), before anything is split.
    import transformers

    lay = tp_env.env_layout()

    class DPOTrainer(transformers.Trainer):
        pass

    DPOTrainer.__module__ = "trl.trainer.dpo_trainer"

    class UnslothGRPOTrainer(transformers.Trainer):
        pass

    class MyTrainer(transformers.Trainer):
        def compute_loss(self, *args, **kwargs):
            return None

    class PlainSubclass(transformers.Trainer):
        def _get_train_sampler(self, *args, **kwargs):
            return None

    out["dpo"] = _raised(lambda: pp_env.pp_trainer_refusals(DPOTrainer, args, lay))
    out["grpo"] = _raised(lambda: pp_env.pp_trainer_refusals(UnslothGRPOTrainer, args, lay))
    out["custom_loss"] = _raised(lambda: pp_env.pp_trainer_refusals(MyTrainer, args, lay))
    out["plain"] = _raised(lambda: pp_env.pp_trainer_refusals(PlainSubclass, args, lay))
    out["trainer"] = _raised(lambda: pp_env.pp_trainer_refusals(transformers.Trainer, args, lay))
    out["best"] = _raised(lambda: pp_env.pp_trainer_refusals(transformers.Trainer, best, lay))
    # Failing before: label smoothing and compute_loss_func were accepted and silently skipped.
    out["smoothing"] = _raised(
        lambda: pp_env.pp_trainer_refusals(transformers.Trainer, smoothed, lay)
    )
    out["loss_func"] = _raised(
        lambda: pp_env.pp_trainer_refusals(
            transformers.Trainer, args, lay, compute_loss_func = lambda *a, **k: None
        )
    )
    if SFTTrainer is not None:
        out["packing"] = _raised(lambda: pp_env.pp_trainer_refusals(SFTTrainer, packed, lay))
        out["sft"] = _raised(lambda: pp_env.pp_trainer_refusals(SFTTrainer, plain_sft, lay))
    else:
        out["packing"] = out["sft"] = "no trl"
    # A model the split refuses is refused at from_pretrained, naming the variable: a mixture-of-experts
    # family whose router auxiliary loss pipeline parallel does not carry (Mixtral). Qwen3-MoE, whose loss it
    # carries between stages (tests/test_pp_moe_aux_gloo.py), loads.
    from transformers import AutoConfig, AutoModelForCausalLM

    config = transformers.MixtralConfig(
        vocab_size = 128,
        hidden_size = 64,
        intermediate_size = 96,
        num_hidden_layers = 2,
        num_attention_heads = 4,
        num_key_value_heads = 2,
        head_dim = 16,
        num_local_experts = 4,
        num_experts_per_tok = 2,
    )
    moe = tp_env.tensor_parallel_env_load(
        lambda name, load_in_4bit = True: AutoModelForCausalLM.from_config(config)
    )
    qwen3_moe_config = AutoConfig.from_pretrained("trl-internal-testing/tiny-Qwen3MoeForCausalLM")
    qwen3_moe = tp_env.tensor_parallel_env_load(
        lambda name, load_in_4bit = True: AutoModelForCausalLM.from_config(qwen3_moe_config)
    )
    os.environ.update(
        MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(_free_port()), RANK = "0", LOCAL_RANK = "0"
    )
    layout(UNSLOTH_PP_SIZE = 1, WORLD_SIZE = 1)  # (no world for the mesh here: refusal path only)
    os.environ["UNSLOTH_PP_SIZE"] = "2"
    tp_env._reset_env_layout()
    tp_env._LAYOUT = tp_env.TPEnvLayout(1, 1, 2, False, 2)
    tp_env._LAYOUT._mesh = object()  # the refusal comes from the model, not the mesh
    out["moe"] = _raised(lambda: moe("m", load_in_4bit = False))
    out["qwen3_moe"] = _raised(lambda: qwen3_moe("m", load_in_4bit = False))
    # Failing before: under pp x tp the tensor split in Trainer.__init__ replaced the parameters a prebuilt
    # optimizer held, so training stepped the old ones (no update, no error).
    from unsloth.models import _utils as unsloth_utils

    tp_env._LAYOUT = tp_env.TPEnvLayout(2, 1, 4, False, 2)
    trainer_like = object.__new__(transformers.Trainer)
    prebuilt = torch.optim.SGD([torch.nn.Parameter(torch.zeros(1))], lr = 0.1)
    out["prebuilt_optimizer"] = _raised(
        lambda: unsloth_utils._pipeline_parallel_env_refusals(
            trainer_like,
            transformers.Trainer.__init__,
            (),
            {"args": args, "optimizers": (prebuilt, None)},
        )
    )
    out["prebuilt_param_groups"] = _raised(
        lambda: unsloth_utils._pipeline_parallel_env_refusals(
            trainer_like,
            transformers.Trainer.__init__,
            (),
            {
                "args": args,
                "optimizer_cls_and_kwargs": (
                    torch.optim.SGD,
                    {"params": prebuilt.param_groups, "lr": 0.1},
                ),
            },
        )
    )
    out["no_prebuilt_optimizer"] = _raised(
        lambda: unsloth_utils._pipeline_parallel_env_refusals(
            trainer_like, transformers.Trainer.__init__, (), {"args": args}
        )
    )
    # Hidden states sent between stages: float32 for a model Unsloth trains in float32 (Gemma 3 on a T4).
    import types

    from unsloth.models import pp_native

    parts = types.SimpleNamespace(
        embed = types.SimpleNamespace(weight = torch.zeros(1, dtype = torch.float16))
    )
    plain, forced = torch.nn.Linear(1, 1), torch.nn.Linear(1, 1)
    forced._unsloth_forced_float32 = True
    out["act_dtype"] = (
        pp_native._default_activation_dtype(plain, parts),
        pp_native._default_activation_dtype(forced, parts),
    )

    # Later stages' embedding stub returns zeros in the dtype the real (here upcasting) embedding returns.
    class Upcast(torch.nn.Embedding):
        def forward(self, ids):
            return super().forward(ids).float()

    stub = pp_native._PPEmbeddingStub(Upcast(10, 4).half())
    out["stub_dtype"] = stub(torch.zeros(2, 3, dtype = torch.long)).dtype
    return out


def _case_env(
    rank,
    world,
    tmp,
    ckpt,
    bs,
    steps = 3,
    evaluate = True,
    post = True,
    optim = "adamw_torch",
    ga = 2,
    eval_bs = None,
    noisy = False,
):
    error = _import_checkout_unsloth()
    if error is not None:
        return {"skip": error}
    import torch.distributed as dist
    import transformers
    from peft import LoraConfig, get_peft_model
    from safetensors.torch import load_file
    from transformers import AutoModelForCausalLM

    from unsloth.models import pp_env, pp_native, tp_env, tp_native
    from unsloth.models._utils import patch_gradient_accumulation_fix

    patch_gradient_accumulation_fix(transformers.Trainer)
    layout = tp_env.env_layout()
    seen = []

    class SequentialTrainer(transformers.Trainer):
        def _get_train_sampler(self, *args, **kwargs):
            return torch.utils.data.SequentialSampler(self.train_dataset)

    def collate(rows):
        batch = {
            "input_ids": torch.stack([r["input_ids"] for r in rows]),
            "labels": torch.stack([r["labels"] for r in rows]).clone(),
        }
        # Uneven token counts per micro-batch / replica, the same per row however the rows are batched.
        batch["labels"][batch["labels"] % 5 == 0] = -100
        if noisy:
            # A stochastic collator whose RNG the stages advance differently (as dropout over different layers).
            if rank % 2 == 1:
                torch.rand(7)
            swap = torch.rand(batch["input_ids"].shape) < 0.3
            batch["input_ids"] = torch.where(
                swap, torch.randint_like(batch["input_ids"], 1, 50), batch["input_ids"]
            )
        seen.append(batch["input_ids"].sum().item())
        return transformers.BatchEncoding(
            batch
        )  # what tokenizer-based collators return (a Mapping)

    def load(
        model_name,
        load_in_4bit = True,
        **kwargs,
    ):  # FastLanguageModel.from_pretrained's flags
        return AutoModelForCausalLM.from_pretrained(
            model_name, torch_dtype = torch.float32, attn_implementation = "eager"
        )

    def peft(model):
        torch.manual_seed(1)
        config = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False)
        return get_peft_model(model, config)

    load = tp_env.tensor_parallel_env_load(load)
    peft = tp_env.tensor_parallel_env_peft(peft)
    model = peft(load(ckpt, load_in_4bit = False))
    pre_split = not pp_native.is_pipeline_parallel(model)
    dp = layout.dp if layout is not None else 1
    g = torch.Generator().manual_seed(1)
    ids = torch.randint(0, 1000, (steps * ga * bs * max(dp, 1), 9), generator = g)
    data = _Rows(list(ids))
    eval_data = _Rows(list(torch.randint(0, 1000, (4 * bs * max(dp, 1), 9), generator = g)))
    args = _trainer_args(
        os.path.join(tmp, "out"),
        bs,
        optim = optim,
        max_steps = steps,
        adam_epsilon = _ADAM_EPS,
        gradient_accumulation_steps = ga,
        per_device_eval_batch_size = eval_bs or bs,
    )
    trainer = SequentialTrainer(model = model, args = args, train_dataset = data, data_collator = collate)
    stage_ids = []
    split = pp_native.pipeline_stage_of(trainer.model)
    if split is not None and not split.is_first:
        # Later stages run the model on the batch's real token ids (not zeros), as one GPU does.
        def record(module, args, kwargs):
            if torch.is_grad_enabled() and kwargs.get("input_ids") is not None:
                stage_ids.append(int(kwargs["input_ids"].sum()))

        split.parts.inner.register_forward_pre_hook(record, with_kwargs = True)
    trainer.train()
    logs = [h for h in trainer.state.log_history if "loss" in h]
    res = {
        "steps": [h["step"] for h in logs],
        "losses": [h["loss"] for h in logs],
        "grad_norms": [h.get("grad_norm") for h in logs],
        "seen": list(seen),
        "stage_ids": list(stage_ids),  # the training run above (not the later cells)
        "pre_split": pre_split,
        "layout": None if layout is None else repr(layout),
        "is_pp": pp_native.is_pipeline_parallel(trainer.model),
        "wrapped": type(trainer.model_wrapped).__name__,
    }
    stage = pp_native.pipeline_stage_of(trainer.model)
    if stage is not None:
        res["stage"] = stage.stage_index
        # Every stage seeds its train sampler per dp replica (LengthGroupedSampler would draw from the global RNG).
        res["sampler_hooked"] = "_get_train_sampler" in trainer.__dict__
        res["owned"] = sorted(n for n, p in trainer.model.named_parameters() if p.requires_grad)
        res["kept_on_cpu"] = (
            all(p.device.type == "cpu" for _, _, m, _ in stage._kept for p in m.parameters())
            if stage._kept and stage.tp_mesh is None
            else None
        )
    if evaluate:
        res["eval_loss"] = trainer.evaluate(eval_data)["eval_loss"]
    save_dir = os.path.join(tmp, "adapter")
    if stage is not None:
        pp_env.pp_save_adapter(trainer.model, (save_dir,), {})
    elif rank == 0:
        trainer.model.save_pretrained(save_dir)
    if dist.is_initialized():
        dist.barrier()
    res["adapter"] = (
        load_file(os.path.join(save_dir, "adapter_model.safetensors")) if rank == 0 else None
    )
    trainer.save_model(os.path.join(tmp, "trainer_save"))
    if dist.is_initialized():
        dist.barrier()
    res["trainer_adapter"] = (
        load_file(os.path.join(tmp, "trainer_save", "adapter_model.safetensors"))
        if rank == 0
        else None
    )
    if post:
        prompt = ids[:1, :4]
        refusal = pp_env.replica_refusal(stage, "generate") if stage is not None else None
        if refusal is None:
            trainer.model.eval()
            out = trainer.model.generate(
                input_ids = prompt, max_new_tokens = 4, do_sample = False, pad_token_id = 0
            )
            res["generated"] = out.tolist()
            if stage is not None:
                res["replica_active"] = stage.replica is not None
                # Failing before: the other stages' modules came back in training mode (dropout on).
                res["replica_training"] = sorted(
                    n for n, m in trainer.model.named_modules() if m.training
                )[:3]
                # An export: the whole one-GPU state dict, built on rank 0 only (the others wait).
                full = pp_env.pp_collective(
                    trainer.model,
                    "export",
                    lambda: {k: v.detach().clone() for k, v in trainer.model.state_dict().items()},
                )
                res["export"] = full if rank == 0 else None
                # The other ranks return rank 0's result (as tp_replica's collective saves do).
                res["export_other"] = sorted(full) if rank != 0 else None
                # Failing before: a checkpoint written with the replica active recorded every stage's params.
                trainer._save_optimizer_and_scheduler(os.path.join(tmp, "ckpt_with_replica"))
                res["released_by_save"] = stage.replica is None
                trainer.model.train()
                pp_env.release_replica(trainer.model)
                res["released"] = stage.replica is None and all(
                    not isinstance(layer, torch.nn.Module)
                    or type(layer).__name__ == "_PPPassThrough"
                    for i, layer in enumerate(stage.parts.layers)
                    if not (stage.start <= i < stage.end)
                )
                # A training step after generate: the pipeline split is back.
                trainer.args.max_steps = steps + 1
                res["after_generate"] = _raised(lambda: trainer.train(resume_from_checkpoint = None))
            else:
                res["export"] = {
                    k: v.detach().clone() for k, v in trainer.model.state_dict().items()
                }
        else:
            res["generate_refused"] = _raised(
                lambda: trainer.model.generate(
                    input_ids = prompt, max_new_tokens = 4, do_sample = False, pad_token_id = 0
                )
            )
            res["export_refused"] = _raised(
                lambda: pp_env.pp_collective(trainer.model, "export", lambda: None)
            )
    if dist.is_initialized():
        dist.barrier()
    return res


def _case_resume(
    rank,
    world,
    tmp,
    ckpt,
    bs = 2,
):
    """save_strategy = "steps" checkpoints (the 1-GPU adapter + per-rank optimizer state) and an exact resume."""
    error = _import_checkout_unsloth()
    if error is not None:
        return {"skip": error}
    import transformers
    from peft import LoraConfig, get_peft_model
    from transformers import AutoModelForCausalLM

    from unsloth.models import tp_env
    from unsloth.models._utils import patch_gradient_accumulation_fix

    patch_gradient_accumulation_fix(transformers.Trainer)
    layout = tp_env.env_layout()
    dp = layout.dp if layout is not None else 1

    class SequentialTrainer(transformers.Trainer):
        def _get_train_sampler(self, *args, **kwargs):
            return torch.utils.data.SequentialSampler(self.train_dataset)

    load = tp_env.tensor_parallel_env_load(
        lambda name, load_in_4bit = True: AutoModelForCausalLM.from_pretrained(
            name, torch_dtype = torch.float32, attn_implementation = "eager"
        )
    )

    def fresh():
        torch.manual_seed(1)
        config = LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear", init_lora_weights = False)
        return get_peft_model(load(ckpt, load_in_4bit = False), config)

    g = torch.Generator().manual_seed(1)
    data = _Rows(list(torch.randint(0, 1000, (4 * 2 * bs * dp, 9), generator = g)))

    def run(out):
        args = _trainer_args(
            os.path.join(tmp, out),
            bs,
            max_steps = 4,
            adam_epsilon = _ADAM_EPS,
            save_strategy = "steps",
            save_steps = 2,
        )
        return SequentialTrainer(model = fresh(), args = args, train_dataset = data)

    def summary(trainer):
        logs = [h for h in trainer.state.log_history if "loss" in h]
        return {
            "steps": [h["step"] for h in logs],
            "losses": [h["loss"] for h in logs],
            "norms": [h["grad_norm"] for h in logs],
        }

    a = run("a")
    a.train()
    ckpt2 = os.path.join(tmp, "a", "checkpoint-2")
    files = sorted(os.listdir(ckpt2)) + sorted(os.listdir(os.path.join(ckpt2, "unsloth_tp")))
    b = run("b")
    b.train(resume_from_checkpoint = ckpt2)
    state_a = {n: p.detach().clone() for n, p in a.model.named_parameters() if p.requires_grad}
    state_b = {n: p.detach().clone() for n, p in b.model.named_parameters() if p.requires_grad}
    return {
        "a": summary(a),
        "b": summary(b),
        "files": files,
        "same": all(torch.equal(state_a[k], state_b[k]) for k in state_a),
    }


def _case_scaler_clip(
    rank,
    world,
    tmp,
    wrapper_unscale = False,
):
    """float16: the Trainer clip under pp unscales the gradients once, for the torch optimizer the
    AcceleratedOptimizer hands the GradScaler (a second unscale inside scaler.step would shrink the update)."""
    error = _import_checkout_unsloth()
    if error is not None:
        return {"skip": error}
    import types

    from accelerate.optimizer import AcceleratedOptimizer
    from torch.distributed.device_mesh import init_device_mesh
    from test_pp_fsdp2_gloo import LLAMA, build, global_batch

    from unsloth.models import pp_env, pp_native

    import torch.distributed as dist

    if wrapper_unscale:  # the old lookup: unscale the AcceleratedOptimizer itself
        pp_env._unwrap_optimizer = lambda opt: opt

    dist.init_process_group("gloo")
    mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("pp",))
    model = build(LLAMA)
    stage = pp_native.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = 2)
    ids, labels = global_batch(model, 4)
    from accelerate import Accelerator

    Accelerator(cpu = True)  # the state AcceleratedOptimizer reads
    scaler = torch.amp.GradScaler("cpu", init_scale = 1024.0)
    inner = torch.optim.SGD(stage.parameters(), lr = 1.0)
    optimizer = AcceleratedOptimizer(inner, device_placement = False, scaler = scaler)
    accelerator = types.SimpleNamespace(scaler = scaler, _optimizers = [optimizer])
    stage.step(ids, labels, scaler = scaler)
    norm = pp_env._pp_clip(accelerator, stage)(model.parameters(), 1e9)
    grads = [p.grad.detach().clone() for p in stage.parameters()]
    before = [p.detach().clone() for p in stage.parameters()]
    optimizer.step()
    deltas = [(b - p.detach()) for b, p in zip(before, stage.parameters())]
    return {
        "norm": float(norm),
        "max_rel": max(
            float((d - g).abs().max() / g.abs().max().clamp_min(1e-30))
            for d, g in zip(deltas, grads)
        ),
        "scale": scaler.get_scale(),
    }


CASES = {
    "units": _case_units,
    "env": _case_env,
    "resume": _case_resume,
    "scaler_clip": _case_scaler_clip,
}


def _worker(rank, world, port, case, tmp, env, kwargs):
    os.environ.update(
        MASTER_ADDR = "127.0.0.1",
        MASTER_PORT = str(port),
        RANK = str(rank),
        WORLD_SIZE = str(world),
        LOCAL_RANK = str(rank),
        LOCAL_WORLD_SIZE = str(world),
    )
    for key in _ENV_KEYS:
        os.environ.pop(key, None)
    os.environ.update(env)
    torch.set_num_threads(2)
    import faulthandler

    faulthandler.dump_traceback_later(
        max(int(os.environ.get("PP_TEST_TIMEOUT", 600)) - 30, 30), exit = True
    )
    import torch.distributed as dist

    try:
        res = CASES[case](rank, world, tmp, **kwargs)
        torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
    finally:
        if dist.is_initialized():
            dist.destroy_process_group()


def _spawn(
    case,
    world = 1,
    env = None,
    **kwargs,
):
    ctx = multiprocessing.get_context("spawn")
    port = _free_port()
    with tempfile.TemporaryDirectory() as tmp:
        if case in ("env", "resume"):
            kwargs["ckpt"] = _checkpoint(tmp, kwargs.pop("arch", "llama"))
        procs = [
            ctx.Process(target = _worker, args = (r, world, port, case, tmp, env or {}, kwargs))
            for r in range(world)
        ]
        for p in procs:
            p.start()
        for p in procs:
            p.join(int(os.environ.get("PP_TEST_TIMEOUT", 600)))
        codes = [p.exitcode for p in procs]
        for p in procs:
            if p.is_alive():
                p.kill()
        assert codes == [0] * world, f"rank exit codes {codes}"
        results = [
            torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
            for r in range(world)
        ]
    if "skip" in results[0]:
        pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")
    return results


def _pipelining_available():
    try:
        import inspect

        from torch.distributed.pipelining import ScheduleGPipe
    except ImportError:
        return False
    return "return_outputs" in inspect.signature(ScheduleGPipe.step).parameters


needs_pipelining = pytest.mark.skipif(
    not _pipelining_available(),
    reason = "torch.distributed.pipelining with step(return_outputs) (torch >= 2.10)",
)


def _close(
    got,
    want,
    rel = 1e-5,
    what = "",
):
    assert got["steps"] == want["steps"], (what, got["steps"], want["steps"])
    assert got["losses"] == pytest.approx(want["losses"], rel = rel), (
        what,
        got["losses"],
        want["losses"],
    )
    # Step 1 is tight; later steps carry the one-process reference's own run-to-run spread on this host
    # (3e-5 absolute at a 2.3 grad norm between two identical reference runs).
    assert got["grad_norms"][:1] == pytest.approx(want["grad_norms"][:1], rel = rel), (
        what,
        got["grad_norms"],
    )
    assert got["grad_norms"] == pytest.approx(want["grad_norms"], rel = 1e-4), (
        what,
        got["grad_norms"],
        want["grad_norms"],
    )
    if "eval_loss" in want:
        assert got["eval_loss"] == pytest.approx(want["eval_loss"], rel = rel), (
            what,
            got["eval_loss"],
            want["eval_loss"],
        )


def _same_tensors(
    got,
    want,
    rel = 1e-4,
    atol = 1e-5,
    what = "",
):
    assert set(got) == set(want) and want, (what, sorted(set(got) ^ set(want))[:6])
    for key, value in want.items():
        torch.testing.assert_close(
            got[key], value, rtol = rel, atol = atol, msg = lambda m: f"{what} {key}: {m}"
        )


def _ref(bs, **kwargs):
    return _spawn("env", 1, bs = bs, **kwargs)[0]


@needs_pipelining
def test_pp_env_layout_and_refusals():
    (res,) = _spawn("units")
    assert res["pp2"][:5] == (2, 1, 1, 2, False) and res["pp2"][7] == "pp 2", res["pp2"]
    assert res["pp2_dp2"][:5] == (2, 2, 1, 4, False) and res["pp2_dp2"][7] == "pp 2 x dp 2 (DDP)"
    assert res["pp2_fsdp2"][:5] == (2, 2, 1, 4, True) and "native FSDP2" in res["pp2_fsdp2"][7]
    assert res["pp2_tp2"][:5] == (2, 1, 2, 4, False) and res["pp2_tp2"][7] == "pp 2 x tp 2"
    assert res["pp2_tp2_dp2"][:5] == (2, 2, 2, 8, False)
    assert res["pp_one"] is None
    assert "UNSLOTH_PP_SIZE=3" in res["no_divide"] and "WORLD_SIZE=4" in res["no_divide"], res[
        "no_divide"
    ]
    assert "UNSLOTH_PP_SIZE x UNSLOTH_TP_SIZE = 4" in res["no_divide_tp"], res["no_divide_tp"]
    assert "must be >= 1" in res["zero"] and "must be an integer" in res["text"]
    assert res["mb"][5] == 3 and "UNSLOTH_PP_MICROBATCHES" in res["mb_text"]
    assert res["schedule"][6] == "1f1b" and "UNSLOTH_PP_SCHEDULE" in res["schedule_bad"]
    assert "UNSLOTH_PP_MICROBATCHES >= UNSLOTH_PP_SIZE" in res["schedule_mb"], res["schedule_mb"]
    assert (
        "native FSDP2 only" in res["accelerate_fsdp"]
        and "UNSLOTH_PP_SIZE=2" in res["accelerate_fsdp"]
    )
    assert (
        "vLLM" in res["refuse_vllm"]
        and "UNSLOTH_PP_SIZE=2 (pipeline parallel)" in res["refuse_vllm"]
    )
    assert "full finetuning" in res["refuse_full_ft"] and res["refused_before_load"]
    for key in ("dpo", "grpo"):
        assert (
            "Trainer / SFTTrainer only" in (res[key] or "") and "UNSLOTH_PP_SIZE=2" in res[key]
        ), res[key]
    assert "overrides compute_loss" in (res["custom_loss"] or ""), res["custom_loss"]
    assert res["plain"] is None and res["trainer"] is None, (res["plain"], res["trainer"])
    assert "load_best_model_at_end" in (res["best"] or ""), res["best"]
    assert "label_smoothing_factor" in (res["smoothing"] or ""), res["smoothing"]
    assert "compute_loss_func" in (res["loss_func"] or ""), res["loss_func"]
    assert "optimizers = (optimizer" in (res["prebuilt_optimizer"] or ""), res["prebuilt_optimizer"]
    assert res["no_prebuilt_optimizer"] is None, res["no_prebuilt_optimizer"]
    # Failing before: parameter groups passed through optimizer_cls_and_kwargs went around the guard.
    assert "optimizers = (optimizer" in (res["prebuilt_param_groups"] or ""), res[
        "prebuilt_param_groups"
    ]
    if res["sft"] != "no trl":
        assert res["sft"] is None, res["sft"]
        assert "packing" in (res["packing"] or ""), res["packing"]
    assert (
        "mixture-of-experts" in (res["moe"] or "") and "UNSLOTH_PP_SIZE=2: unset it" in res["moe"]
    ), res["moe"]
    assert res["qwen3_moe"] is None, res["qwen3_moe"]
    assert res["act_dtype"] == (torch.float16, torch.float32), res["act_dtype"]
    assert res["stub_dtype"] == torch.float32, res["stub_dtype"]


@needs_pipelining
@pytest.mark.parametrize(
    "world,env,ref_bs",
    [
        (2, {"UNSLOTH_PP_SIZE": "2"}, 2),
        (4, {"UNSLOTH_PP_SIZE": "2", "UNSLOTH_TP_SIZE": "2"}, 2),
        (4, {"UNSLOTH_PP_SIZE": "2"}, 4),
        (4, {"UNSLOTH_PP_SIZE": "2", "UNSLOTH_FSDP2": "1"}, 4),
    ],
    ids = ["pp2", "pp2_x_tp2", "pp2_x_ddp2", "pp2_x_fsdp2_dp2"],
)
def test_pp_env_trainer_matches_one_process(world, env, ref_bs):
    ref = _ref(ref_bs, eval_bs = 2)  # eval batches as each dp replica's
    results = _spawn("env", world, env, bs = 2)
    stages = set()
    for rank, res in enumerate(results):
        assert res["is_pp"] and res["pre_split"], res  # whole until the Trainer
        if res["stage"] == 1:
            assert res["stage_ids"] and sum(res["stage_ids"]) == sum(res["seen"]), (
                res["stage_ids"],
                res["seen"],
            )
        assert "DistributedDataParallel" not in res["wrapped"]
        assert res["sampler_hooked"], res
        _close(res, ref, what = rank)
        stages.add(res["stage"])
    assert stages == {0, 1}
    owned = [set(r["owned"]) for r in results if r["stage"] == 0][0] | [
        set(r["owned"]) for r in results if r["stage"] == 1
    ][0]
    assert not (
        [set(r["owned"]) for r in results if r["stage"] == 0][0]
        & [set(r["owned"]) for r in results if r["stage"] == 1][0]
    )
    assert owned, owned
    _same_tensors(results[0]["adapter"], ref["adapter"], what = "save_pretrained")
    _same_tensors(results[0]["trainer_adapter"], ref["adapter"], what = "Trainer.save_model")
    dp = 2 if world == 4 and "UNSLOTH_TP_SIZE" not in env else 1
    by_replica = {}
    for rank, res in enumerate(results):
        by_replica.setdefault(rank % dp if dp > 1 else 0, []).append(res["seen"])
    for replica, seen in by_replica.items():
        assert all(s == seen[0] for s in seen), (
            replica,
            seen,
        )  # every stage of a replica: same rows
    if dp > 1:
        assert by_replica[0][0] != by_replica[1][0]
    replica_ok = "UNSLOTH_TP_SIZE" not in env and "UNSLOTH_FSDP2" not in env
    for rank, res in enumerate(results):
        if replica_ok:
            assert res["kept_on_cpu"] is True, res["kept_on_cpu"]
            assert res["generated"] == ref["generated"], (rank, res["generated"], ref["generated"])
            assert res["replica_active"] and res["released"], res
            assert res["replica_training"] == [], res["replica_training"]
            assert res["released_by_save"], res
            assert res["after_generate"] is None, res["after_generate"]
            if rank == 0:
                _same_tensors(res["export"], ref["export"], what = "export")
            else:
                assert res["export_other"] == sorted(ref["export"]), res["export_other"]
        else:
            assert "not supported yet on a pipeline parallel x" in (
                res["generate_refused"] or ""
            ), res
            assert "UNSLOTH_PP_SIZE=2" in res["generate_refused"], res["generate_refused"]
            assert res["export_refused"] == res["generate_refused"].replace(
                "`generate`", "`export`"
            )


@needs_pipelining
@pytest.mark.parametrize(
    "ga,bs,mb,schedule,arch",
    [
        (1, 4, None, None, "llama"),
        (4, 2, None, None, "llama"),
        (2, 4, "1", None, "llama"),
        (2, 1, None, "1f1b", "llama"),
        (1, 4, None, None, "gemma3"),
    ],
    ids = ["ga1_bs4", "ga4_bs2", "mb1", "bs1_1f1b_padded", "gemma3_ga1_bs4"],
)
def test_pp_env_gradient_accumulation_and_microbatches(ga, bs, mb, schedule, arch):
    if mb == "1" and tuple(int(x) for x in torch.__version__.split("+")[0].split(".")[:2]) < (
        2,
        13,
    ):
        # torch 2.10's gpipe needs micro-batches >= stages; Unsloth raises a ValueError naming both counts.
        pytest.skip("gpipe with 1 micro-batch over 2 stages needs torch >= 2.13")
    # bs1_1f1b_padded: 1F1B needs 2 micro-batches, so the 1-row batch gets a fully masked filler row.
    # gemma3_ga1_bs4: the Gemma3_(270M) notebook's shape (scaled tied embedding, sliding layers).
    ref = _ref(bs, ga = ga, evaluate = False, post = False, arch = arch)
    env = {"UNSLOTH_PP_SIZE": "2"}
    if mb:
        env["UNSLOTH_PP_MICROBATCHES"] = mb
    if schedule:
        env["UNSLOTH_PP_SCHEDULE"] = schedule
    for rank, res in enumerate(
        _spawn("env", 2, env, bs = bs, ga = ga, evaluate = False, post = False, arch = arch)
    ):
        _close(res, ref, what = rank)
        # _close already pins every loss and grad norm. The trained adapter after several steps carries
        # more of the reduction-order noise: GitHub runners put gemma3_ga1_bs4's lora_A 1.6e-5 from the
        # reference on two PRs (1 of 64 elements), against at most 3.7e-6 here.
        _same_tensors(res["adapter"] or ref["adapter"], ref["adapter"], atol = 5e-5)


@needs_pipelining
def test_pp_env_stages_train_on_the_first_stage_batch():
    # Failing before: each stage collated its own batch, so with a stochastic collator the last stage scored labels
    # for tokens the first stage never embedded.
    results = _spawn(
        "env", 2, {"UNSLOTH_PP_SIZE": "2"}, bs = 2, noisy = True, evaluate = False, post = False
    )
    first, last = sorted(results, key = lambda r: r["stage"])
    assert first["seen"] != last["seen"]  # the collators did diverge
    assert last["stage_ids"] and sum(last["stage_ids"]) == sum(first["seen"]), (
        last["stage_ids"],
        first["seen"],
    )


@needs_pipelining
@pytest.mark.parametrize("world", [2, 4], ids = ["pp2", "pp2_x_ddp2"])
def test_pp_env_checkpoint_resume_equals_uninterrupted(world):
    for rank, res in enumerate(_spawn("resume", world, {"UNSLOTH_PP_SIZE": "2"})):
        a, b = res["a"], res["b"]
        assert a["steps"] == [1, 2, 3, 4] and b["steps"] == [1, 2, 3, 4], (a, b)
        assert (
            "adapter_model.safetensors" in res["files"] and f"rank{rank:05d}.pt" in res["files"]
        ), res["files"]
        assert b["losses"][2:] == a["losses"][2:] and b["norms"][2:] == a["norms"][2:], (rank, a, b)
        assert res["same"], rank


@needs_pipelining
def test_pp_env_fp16_scaler_unscales_once():
    for res in _spawn("scaler_clip", 2):
        assert (
            res["norm"] > 0 and res["max_rel"] < 1e-3
        ), res  # update == -lr x the unscaled gradient
    for res in _spawn("scaler_clip", 2, wrapper_unscale = True):
        assert (
            res["max_rel"] > 0.9
        ), res  # unscaled twice: the update is 1 / 1024 of it (the bug this pins)


@needs_pipelining
def test_pp_env_tensor_peers_train_on_one_batch():
    # Failing before: only the pipeline group took the first stage's batch, so under pp=2,tp=2 a stochastic
    # collator left tensor parallel peers [0, 1] / [2, 3] reducing activations of different examples.
    results = _spawn(
        "env",
        4,
        {"UNSLOTH_PP_SIZE": "2", "UNSLOTH_TP_SIZE": "2"},
        bs = 2,
        noisy = True,
        evaluate = False,
        post = False,
    )
    assert results[0]["seen"] != results[1]["seen"]  # the tensor peers' collators did diverge
    first = [r for r in results if r["stage"] == 0]
    last = [
        r for r in results if r["stage"] == 1
    ]  # the scoring stage records the ids it trained on
    sums = [sum(r["stage_ids"]) for r in last]
    assert len(last) == 2 and all(r["stage_ids"] for r in last), sums
    assert sums == [sum(first[0]["seen"])] * 2, (sums, [sum(r["seen"]) for r in first])