Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

63.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

"""Context parallel (Ulysses) composed with tensor / pipeline parallel through SFTTrainer, on a gloo CPU world, vs
the same script in one process: losses, per-parameter gradients of the first optimizer step and the adapters
after the run.

CPU ranks cannot run Unsloth's fast Llama attention (it needs CUDA), so the attention is the Hugging Face one, which
the production support check accepts and the trainer routes through generic Ulysses (ALL_ATTENTION_FUNCTIONS). The
env layout, the mesh, tensor parallel, the CP sharding, the token count and the gradient reduction are all the
production path; nothing is patched open."""

import multiprocessing
import os

if os.environ.get("UNSLOTH_TEST_RANKS_ON_GPU") != "1" and (
    multiprocessing.parent_process() is not None
    or getattr(multiprocessing.current_process(), "_inheriting", False)
):
    os.environ["CUDA_VISIBLE_DEVICES"] = ""  # spawned gloo ranks: CPU only, also on GPU hosts

import dataclasses  # noqa: E402
import sys  # noqa: E402
import tempfile  # noqa: E402
import types  # noqa: E402

import pytest  # noqa: E402
import torch  # noqa: E402

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from test_tp_native_gloo import (  # noqa: E402
    _free_port,
    _import_checkout_unsloth,
    _raised,
    _save_checkpoint,
)

_ENV_KEYS = (
    "UNSLOTH_PARALLEL",
    "UNSLOTH_TP_SIZE",
    "UNSLOTH_PP_SIZE",
    "UNSLOTH_CP_SIZE",
    "UNSLOTH_CP_BACKEND",
    "UNSLOTH_PP_MICROBATCHES",
    "UNSLOTH_PP_SCHEDULE",
    "UNSLOTH_FSDP2",
    "UNSLOTH_FSDP2_BACKEND",
    "ACCELERATE_USE_FSDP",
)
_ADAM_EPS = 1e-4  # as test_tp_env_gloo
SEQ = 16
N_LAYERS = 4


def _checkpoint(tmp, arch = "llama"):
    from transformers import AutoTokenizer

    from test_tp_native_oracle import LLAMA, QWEN3, QWEN3_MOE, build

    if arch == "qwen3_5":
        # Hybrid: Gated DeltaNet (linear attention) layers beside attention, 4 heads / 2 kv heads, 4 value heads
        # over 2 key heads in the linear layers, so Ulysses cp 2 splits both kinds. No hub weights: random init.
        import transformers

        tok = AutoTokenizer.from_pretrained(QWEN3)
        config = transformers.Qwen3_5TextConfig(
            vocab_size = len(tok),
            hidden_size = 32,
            intermediate_size = 64,
            num_hidden_layers = N_LAYERS,
            layer_types = ["linear_attention", "full_attention"] * (N_LAYERS // 2),
            num_attention_heads = 4,
            num_key_value_heads = 2,
            head_dim = 8,
            linear_num_key_heads = 2,
            linear_num_value_heads = 4,
            linear_key_head_dim = 8,
            linear_value_head_dim = 8,
            linear_conv_kernel_dim = 4,
            attention_dropout = 0.0,
            tie_word_embeddings = False,
        )
        torch.manual_seed(0)
        model = transformers.Qwen3_5ForCausalLM(config)
        model.generation_config.pad_token_id = None
        path = os.path.join(tmp, "ckpt")
        model.save_pretrained(path, safe_serialization = True)
        tok.save_pretrained(path)
        return path
    if arch in ("qwen3_moe", "qwen3", "qwen3_moe_wide"):
        # qwen3_moe: 4 heads / 2 kv heads, 4 experts (2 per token), router aux loss off as shipped.
        # qwen3_moe_wide: hidden and expert widths of 32, so Unsloth's Triton grouped GEMM takes the experts (it
        # needs multiples of 16; the shipped hidden size 8 sends unsloth_triton to torch._grouped_mm).
        repo = QWEN3 if arch == "qwen3" else QWEN3_MOE
        overrides = {"num_hidden_layers": 2}
        if arch == "qwen3_moe_wide":
            overrides.update(hidden_size = 32, moe_intermediate_size = 32)
        path = os.path.join(tmp, "ckpt")
        model = build(repo, overrides, lora = False)
        model.generation_config.pad_token_id = None
        model.save_pretrained(path, safe_serialization = True)
        AutoTokenizer.from_pretrained(repo).save_pretrained(path)
        return path
    # 4 heads / 4 kv heads: tp 2 leaves 2 local heads, which Ulysses splits over cp 2.
    path = _save_checkpoint("llama", os.path.join(tmp, "ckpt"), {"num_hidden_layers": N_LAYERS})
    AutoTokenizer.from_pretrained(LLAMA).save_pretrained(path)
    return path


# Packed rows, one per micro-batch, each with other documents (compiled layers must follow every row): "padfree"
# fills the row, "pack" leaves a one-token tail that context parallel pads to the cp multiple.
_PACKED_DOCS = {
    "padfree": ([5, 7, 4], [3, 9, 4], [8, 2, 6], [4, 4, 8]),
    "pack": ([5, 7, 3], [3, 9, 3], [8, 2, 5], [6, 6, 3]),
}


def _packed_row(batch, kind, row):
    """One packed row as Unsloth's packing / padding-free collators hand it on: the global document lengths,
    positions restarting per document, and no target across a document start."""
    lengths = _PACKED_DOCS[kind][row % len(_PACKED_DOCS[kind])]
    n = sum(lengths)
    input_ids, labels = batch["input_ids"][:, :n], batch["labels"][:, :n].clone()
    starts = torch.tensor([0, *lengths[:-1]]).cumsum(0)
    labels[:, starts] = -100
    position_ids = torch.cat([torch.arange(k) for k in lengths])[None]
    return {
        "input_ids": input_ids,
        "labels": labels,
        "position_ids": position_ids,
        "packed_seq_lengths": torch.tensor(lengths, dtype = torch.int32),
    }


def _packed_oracle(model, batch):
    """The one-process reference itself keeps documents apart: each document's logits in the packed row equal
    the same document alone (eager, so no compiled graph is added to the reference's counters)."""
    ids, lengths = batch["input_ids"], batch["packed_seq_lengths"].tolist()
    was = model.training
    model.eval()
    with torch.no_grad(), torch.compiler.set_stance("force_eager"):
        full = model(
            input_ids = ids,
            position_ids = batch["position_ids"],
            packed_seq_lengths = batch["packed_seq_lengths"],
            use_cache = False,
        ).logits[0]
        worst, start = 0.0, 0
        for n in lengths:
            alone = model(input_ids = ids[:, start : start + n], use_cache = False).logits[0]
            worst = max(
                worst, float((full[start : start + n] - alone).abs().max() / alone.abs().max())
            )
            start += n
    model.train(was)
    return worst


def _full(t):
    from torch.distributed.tensor import DTensor
    return (t.full_tensor() if isinstance(t, DTensor) else t).detach().float().cpu().clone()


def _block_compile_on_cpu(model):
    """Unsloth's block compile (one torch.compile region per decoder layer) on CPU ranks: its CUDA gate opened
    and the aot_eager backend instead of inductor. Graph breaks and recompiles are Dynamo's, so they count the
    same; the generated kernels are not under test here."""
    from unsloth.models import _block_compile as bc

    real_compile = torch.compile

    def compile_aot_eager(fn, **kwargs):
        kwargs.pop("options", None)
        kwargs["backend"] = "aot_eager"
        return real_compile(fn, **kwargs)

    available = torch.cuda.is_available
    bc.torch.cuda.is_available = lambda: True
    bc.torch.compile = compile_aot_eager
    try:
        return bc.patch_block_compile(model)
    finally:
        bc.torch.cuda.is_available = available
        bc.torch.compile = real_compile


def _install_unsloth_checkpoint(model, gc, stats):
    """Unsloth's checkpoint on CPU: "unsloth" = path A (UnslothCheckpointFunction, never offloads on CPU),
    "unsloth_nonre" = path B (vision.py _nonre_checkpoint) with CPU device seams: bookkeeping real, copies not."""
    import contextlib

    import torch.utils.checkpoint as torch_checkpoint
    import transformers.modeling_utils as hf_modeling_utils
    import unsloth_zoo.gradient_checkpointing as G

    original_forward = G.UnslothCheckpointFunction.forward

    def counted_forward(ctx, run_function, preserve_rng_state, *args):
        stats["checkpoint_calls"] += 1
        return original_forward(ctx, run_function, preserve_rng_state, *args)

    G.UnslothCheckpointFunction.forward = staticmethod(counted_forward)
    if gc == "unsloth":
        # No CUDA: no pinned host buffers and no double buffer (this worker process only).
        os.environ["UNSLOTH_DISABLE_PINNED_MEMORY"] = "1"
        os.environ["UNSLOTH_DISABLE_DOUBLE_BUFFER"] = "1"
        G.patch_unsloth_smart_gradient_checkpointing(dtype = torch.float32)
        reentrant = True
    else:
        from unsloth.models import vision

        G.unpatch_unsloth_smart_gradient_checkpointing()

        class _Stream:
            def wait_event(self, event):
                pass

            def wait_stream(self, other):
                pass

        class _Event:
            def record(self, stream = None):
                pass

            def query(self):
                return True

        compute = _Stream()

        class CpuSeamOffloader(G.NonReentrantActivationOffloader):
            def _on_host(self, x):
                return x.numel() == 0  # torch's checkpoint sentinel

            def _streamable(self, x):
                return not G._is_compiling()

            def _new_stream(self, device_index):
                return _Stream()

            def _current_stream(self, device_index):
                return compute

            def _stream_ctx(self, stream):
                return contextlib.nullcontext()

            def _new_event(self):
                return _Event()

            def _record_stream(self, x, stream):
                pass

            def _host_copy(
                self,
                x,
                non_blocking = True,
            ):
                stats["offloaded"] += 1
                return x.detach().clone()

            def _device_copy(self, host, device):
                stats["restored"] += 1
                return host.clone()

            def _empty_on_device(self, host, device):
                return torch.empty_like(host)

        G._reserve_pinned = lambda nbytes: True
        G._release_pinned = lambda nbytes: None
        G.PINNED_MEMORY_AVAILABLE = True
        G._new_activation_offloader = lambda: CpuSeamOffloader()
        original = getattr(
            torch_checkpoint.checkpoint, "_unsloth_original", torch_checkpoint.checkpoint
        )
        vision._register_gc_offload_root(model, True)

        def _nonre_checkpoint(function, *args, **kwargs):
            stats["checkpoint_calls"] += 1
            kwargs["use_reentrant"] = False
            with vision._offload_activation_hooks(vision._gc_checkpoint_owner(function)):
                return original(function, *args, **kwargs)

        _nonre_checkpoint._unsloth_original = original
        _nonre_checkpoint._unsloth_offloads = True
        torch_checkpoint.checkpoint = _nonre_checkpoint
        hf_modeling_utils.checkpoint = _nonre_checkpoint
        reentrant = False
    model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": reentrant})
    model.enable_input_require_grads()


def _selected_moe_backend():
    try:
        from unsloth_zoo.temporary_patches import moe_utils
        return moe_utils.select_moe_backend()
    except Exception as e:
        return f"unknown: {e!r}"[:200]


def _count_moe_backend_calls():
    """Engagement for GPU MoE cells: calls into each Unsloth Zoo expert backend and into torch._grouped_mm."""
    import collections

    calls = collections.Counter()
    try:
        from unsloth_zoo.temporary_patches import moe_utils
    except Exception:
        return calls
    modules = [moe_utils]
    try:
        cached = moe_utils._load_cached_moe_utils_module()
        if cached is not None and cached is not moe_utils:
            modules.append(cached)
    except Exception:
        pass
    try:
        from unsloth_zoo.temporary_patches import moe_utils_bnb4bit
        modules.append(moe_utils_bnb4bit)
    except Exception:
        pass

    def counted(name, fn):
        def wrapper(*a, **k):
            calls[name] += 1
            return fn(*a, **k)

        wrapper.__wrapped__ = fn
        return wrapper

    for module in modules:
        for name in (
            "forward_native_grouped_mm",
            "forward_triton_grouped_gemm",
            "forward_native_moe_loop",
            "forward_moe_backend_bnb4bit",
            "_triton_grouped_mm",
            "_manual_grouped_mm",
        ):
            if callable(getattr(module, name, None)):
                setattr(module, name, counted(name, getattr(module, name)))
    try:
        from unsloth.kernels.moe.grouped_gemm import interface
        interface.grouped_gemm = counted("unsloth grouped_gemm (Triton)", interface.grouped_gemm)
    except Exception:
        pass
    if hasattr(torch, "_grouped_mm"):
        torch._grouped_mm = counted("torch._grouped_mm", torch._grouped_mm)
    return calls


def _case_train(
    rank,
    world,
    tmp,
    ckpt,
    bs,
    cp = 1,
    steps = 2,
    ga = 2,
    sft_cp = True,
    keep = False,
    compile = False,
    arch = "llama",
    router_aux = False,
    attn = "sdpa",
    ungate = False,
    mode = "lora",
    trl_aux = False,
    aux_coef = None,
    zoo_compile = False,
    device = "cpu",
    bypass_pre = False,
    bypass_post = False,
    args_coef = "unset",
    probe_refusal = False,
    gc = "none",
    packed = None,
    eval_only = False,
):
    error = _import_checkout_unsloth()
    if error is not None:
        return {"skip": error}
    import torch.distributed as dist
    import transformers
    import trl
    from peft import LoraConfig, get_peft_model
    from transformers import AutoModelForCausalLM, AutoTokenizer

    from unsloth import context_parallel as cpm
    from unsloth.models import pp_native, tp_env, tp_native
    from unsloth.models._utils import patch_gradient_accumulation_fix

    patch_gradient_accumulation_fix(transformers.Trainer)
    cpm.patch_sft_trainer()
    if device == "cpu":
        cpm.DEVICE_TYPE_TORCH = "cpu"
    else:  # NCCL ranks under torchrun (scripts/local_correctness.py): one GPU each
        if torch.cuda.device_count() < world:
            # Correctness only: every rank shares the one card (a one-GPU cloud VM). NCCL >= 2.32 takes that with
            # NCCL_MULTI_RANK_GPU_ENABLE=1 (the driver probes it), else gloo on CUDA tensors.
            os.environ["LOCAL_RANK"], os.environ["LOCAL_WORLD_SIZE"] = "0", "1"
            import torch.distributed as dist
            if world > 1 and not dist.is_initialized():
                dist.init_process_group(os.environ.get("UNSLOTH_TEST_SHARED_GPU_BACKEND", "gloo"))
        torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", "0")))
        # fp32 like the CPU reference: no TF32 matmuls / convolutions (Unsloth may have allowed them).
        torch.backends.cuda.matmul.allow_tf32 = False
        torch.backends.cudnn.allow_tf32 = False
        torch.set_float32_matmul_precision("highest")
        # The math SDPA kernel: deterministic fp32 (the fused ones differ in the last bits per head count).
        torch.backends.cuda.enable_flash_sdp(False)
        torch.backends.cuda.enable_mem_efficient_sdp(False)
        torch.backends.cuda.enable_cudnn_sdp(False)
        torch.use_deterministic_algorithms(True, warn_only = True)
    if bypass_pre or bypass_post:
        # Evidence runs only: switch off the aux-loss guards to show what each one catches.
        from unsloth import parallel_capabilities
        if bypass_pre:
            parallel_capabilities._aux_loss_on = lambda *a, **k: False
        if bypass_post:
            cpm._refuse_router_aux_loss_after_init = lambda *a, **k: None
    if ungate:
        # Measure a cell the registry refuses for lack of evidence (the evidence run itself).
        from unsloth import parallel_capabilities
        parallel_capabilities._gated = lambda query: False
    from unsloth.utils import attention_dispatch

    attention_dispatch.HAS_XFORMERS = (
        False  # torch SDPA on every rank (fp32: flash / xformers take 16-bit)
    )
    attention_dispatch.HAS_FLASH_ATTENTION = False
    layout = tp_env.env_layout()

    def load(
        model_name,
        load_in_4bit = True,
        **kwargs,
    ):
        extra = {}
        if mode == "qlora":
            from transformers import BitsAndBytesConfig
            extra = dict(
                quantization_config = BitsAndBytesConfig(
                    load_in_4bit = True,
                    bnb_4bit_quant_type = "nf4",
                    bnb_4bit_compute_dtype = torch.float32,
                ),
                device_map = {"": torch.cuda.current_device()},
            )
        return AutoModelForCausalLM.from_pretrained(
            model_name, torch_dtype = torch.float32, attn_implementation = attn, **extra
        )

    def peft(model):
        torch.manual_seed(1)
        config = LoraConfig(
            r = 4,
            lora_alpha = 8,
            target_modules = "all-linear",
            init_lora_weights = False,
            lora_dropout = 0.0,
        )
        return get_peft_model(model, config)

    if zoo_compile:
        # FastModel's path: Zoo's compiled CausalLM forward delegates to a module-level rewrite.
        from unsloth_zoo.compiler import unsloth_compile_transformers
        unsloth_compile_transformers(model_type = arch)
    load = tp_env.tensor_parallel_env_load(load)
    peft = tp_env.tensor_parallel_env_peft(peft)
    model = load(ckpt, load_in_4bit = False)
    cpu_chunked_ce = False
    if eval_only:
        from unsloth_zoo.cpu.ops import patch_chunked_causal_lm_loss
        cpu_chunked_ce = patch_chunked_causal_lm_loss(model)
    gdn_calls = []
    if arch == "qwen3_5":
        # The torch Gated DeltaNet kernels (fla / Triton need CUDA), and a counter on Unsloth's GDN exchange.
        from test_ulysses_hybrid import _force_torch_kernels

        from unsloth.utils import ulysses_hybrid as gdn

        _force_torch_kernels(model)
        original_gdn = gdn.ulysses_gated_delta_forward

        def counted_gdn(*a, **k):
            gdn_calls.append(1)
            return original_gdn(*a, **k)

        gdn.ulysses_gated_delta_forward = counted_gdn
    gc_stats = {"checkpoint_calls": 0, "offloaded": 0, "restored": 0}
    if gc != "none":
        _install_unsloth_checkpoint(model, gc, gc_stats)
    if mode in ("lora", "qlora"):
        model = peft(model)
    else:
        model.train()
    count = device != "cpu" or os.environ.get("UNSLOTH_TEST_COUNT_MOE") == "1"
    moe_calls = _count_moe_backend_calls() if count else None
    if router_aux:
        model.config.output_router_logits = True
    if aux_coef is not None:
        # TRL < 1.6 has no SFTConfig.router_aux_loss_coef: set the model's own
        for m in [model, *model.modules()]:
            config = getattr(m, "config", None)
            if config is not None and hasattr(config, "router_aux_loss_coef"):
                config.router_aux_loss_coef = aux_coef
            if "router_aux_loss_coef" in vars(m):
                m.router_aux_loss_coef = aux_coef
    compiled = _block_compile_on_cpu(model) if compile else []

    # Data parallel replicas read different rows; cp / tp / pp peers share theirs.
    model_ranks = cp * (1 if layout is None else layout.tp * layout.pp)
    data_dp = world // model_ranks
    g = torch.Generator().manual_seed(1)
    ids = torch.randint(0, 1000, (steps * ga * bs * data_dp, SEQ), generator = g)
    import datasets

    data = datasets.Dataset.from_dict({"input_ids": ids.tolist(), "row": list(range(len(ids)))})
    rows_read = []  # dataset row ids per micro-batch, in order: parity needs the same rows on both sides

    def collate(rows):
        rows_read.append([int(r["row"]) for r in rows])
        input_ids = torch.tensor([r["input_ids"] for r in rows])
        batch = {"input_ids": input_ids, "labels": input_ids.clone()}
        batch["labels"][
            batch["labels"] % 5 == 0
        ] = -100  # uneven token counts per row and per shard
        if eval_only:
            batch["attention_mask"] = torch.ones_like(input_ids)
        if packed:
            batch.update(_packed_row(batch, packed, int(rows[0]["row"])))
        return batch

    class SequentialSFT(trl.SFTTrainer):
        def _get_train_sampler(self, *args, **kwargs):
            return torch.utils.data.SequentialSampler(self.train_dataset)

    grads, query = {}, None

    class Grabber(transformers.TrainerCallback):
        def on_pre_optimizer_step(
            self,
            args,
            state,
            control,
            model = None,
            **kwargs,
        ):
            if grads:
                return
            target = trainer.model
            for name, p in target.named_parameters():
                if p.requires_grad and p.grad is not None:
                    grads[name] = _full(p.grad)

    args = trl.SFTConfig(
        output_dir = os.path.join(tmp, "out"),
        per_device_train_batch_size = bs,
        gradient_accumulation_steps = ga,
        max_steps = steps,
        learning_rate = 1e-2,
        optim = "adamw_torch",
        adam_epsilon = _ADAM_EPS,
        lr_scheduler_type = "constant",
        warmup_steps = 0,
        weight_decay = 0.0,
        max_grad_norm = 1.0,
        logging_steps = 1,
        save_strategy = "no",
        report_to = [],
        use_cpu = device == "cpu",
        seed = 0,
        dataloader_drop_last = True,
        remove_unused_columns = False,
        disable_tqdm = True,
        dataset_kwargs = {"skip_prepare_dataset": True},
        gradient_checkpointing = False,
    )
    if eval_only:
        args.loss_type = "nll"
    if packed:
        # What the registry reads for a packed run (query_from): SFTConfig.padding_free; the rows come from the
        # collator above, so TRL's own padding-free collator is not involved.
        args.padding_free = True
    if device != "cpu" and torch.cuda.device_count() < world:
        args.ddp_backend = os.environ.get("UNSLOTH_TEST_SHARED_GPU_BACKEND", "gloo")
    if hasattr(args, "router_aux_loss_coef"):
        # TRL turns the MoE load balancing loss on from the architecture's coefficient unless this is 0.0
        # (it then sets output_router_logits on the config itself): aux off unless the case asks for it.
        args.router_aux_loss_coef = aux_coef if (router_aux or trl_aux) else 0.0
        if args_coef != "unset":
            args.router_aux_loss_coef = args_coef
    if sft_cp:
        args.context_parallel_size = cp
        args.context_parallel_backend = "ulysses"
    if device != "cpu":
        # Pure fp32 like the CPU reference. SFTConfig defaults to bf16 = True, and Unsloth's trainer turns
        # bf16 = fp16 = False on a float32 model into bf16 autocast on a bf16 GPU, so both flags alone are not
        # enough: the stamp below is the per-model "train this one in float32" switch the trainer honours.
        args.bf16 = args.fp16 = False
        model._unsloth_forced_float32 = True
    if packed:
        from unsloth import parallel_capabilities as caps

        # The registry's view of the run as configured, before the CPU route's normalize_cpu_training_arguments
        # (unsloth_zoo/cpu/trainer.py) turns padding_free off on CPU ranks inside the trainer's __init__.
        layout_now = types.SimpleNamespace(
            dp = 1, tp = 1, pp = 1, cp = cp, fsdp2 = False, cp_backend = "ulysses"
        )
        query = dataclasses.asdict(
            caps.query_from(layout_now, SequentialSFT, model, args, cp = cp, cp_backend = "ulysses")
        )
    tokenizer = AutoTokenizer.from_pretrained(ckpt)
    if probe_refusal:
        before = {n: (p.detach().clone(), p.device) for n, p in model.named_parameters()}

        def untouched():
            now = dict(model.named_parameters())
            return now.keys() == before.keys() and all(
                now[n].device == d and now[n].shape == t.shape and torch.equal(now[n].detach(), t)
                for n, (t, d) in before.items()
            )

        phase = "init"
        try:
            trainer = SequentialSFT(
                model = model,
                args = args,
                train_dataset = data,
                data_collator = collate,
                processing_class = tokenizer,
            )
            phase = "train"
            trainer.train()
        except Exception as e:
            return {
                "error": f"{type(e).__name__}: {e}",
                "phase": phase,
                "split": pp_native.pipeline_stage_of(model) is not None,
                "untouched": untouched(),
            }
        logs = [h for h in trainer.state.log_history if "loss" in h]
        stage = pp_native.pipeline_stage_of(trainer.model)
        return {
            "phase": None,
            "losses": [h["loss"] for h in logs],
            "stage": None if stage is None else int(stage.stage_index),
            "output_router_logits": bool(
                getattr(trainer.model.config, "output_router_logits", False)
            ),
        }
    trainer = SequentialSFT(
        model = model,
        args = args,
        train_dataset = data,
        data_collator = collate,
        processing_class = tokenizer,
        callbacks = [Grabber()],
    )
    if eval_only:
        trainer.args.per_device_eval_batch_size = bs
        metrics = trainer.evaluate(eval_dataset = data)
        stage = pp_native.pipeline_stage_of(trainer.model)
        return {
            "eval_loss": metrics["eval_loss"],
            "rows_read": rows_read,
            "stage": None if stage is None else int(stage.stage_index),
            "cpu_compute_loss": bool(getattr(type(trainer).compute_loss, "_unsloth_cpu", False)),
            "cpu_chunked_ce": cpu_chunked_ce,
        }
    if device != "cpu":
        assert trainer.accelerator.mixed_precision == "no", trainer.accelerator.mixed_precision
    manager = getattr(trainer, "_context_parallel_manager", None)
    seen_lengths, ulysses_calls = set(), []
    embed = trainer.model.get_input_embeddings()
    embed.register_forward_pre_hook(lambda m, a: seen_lengths.add(int(a[0].shape[-1])))
    autocast_seen = (
        set()
    )  # autocast state inside every training forward (pure fp32 cells: {False} only)
    autocast_device = "cpu" if device == "cpu" else "cuda"
    embed.register_forward_pre_hook(
        lambda m, a: autocast_seen.add(bool(torch.is_autocast_enabled(autocast_device)))
    )
    from unsloth.utils import ulysses_hybrid

    original_ulysses = ulysses_hybrid._ulysses_hf_attention

    def counted(*a, **k):
        ulysses_calls.append(1)
        return original_ulysses(*a, **k)

    if not compile:
        # A Python counter inside a compiled region would be guarded on (a recompile per call).
        ulysses_hybrid._ulysses_hf_attention = counted
    from unsloth.utils import compile_counters

    packed_seen, oracle, exchanges = [], None, None
    if packed:
        # Eager, outside the compiled layers: the global document lengths of every micro-batch at its shard.
        original_shard = cpm.ContextParallelManager._ulysses_shard

        def recorded_shard(self, inputs):
            packed_seen.append(
                [int(x) for x in torch.as_tensor(inputs["packed_seq_lengths"]).tolist()]
            )
            return original_shard(self, inputs)

        cpm.ContextParallelManager._ulysses_shard = recorded_shard
        if world == 1:
            oracle = _packed_oracle(trainer.model, collate([data[0]]))
            rows_read.clear()  # the oracle's row is not a training micro-batch
    torch._dynamo.reset()
    before = compile_counters.snapshot()
    if packed:
        # Inside the compiled layers no Python counter runs: the profiler sees each collective they issue.
        from torch.profiler import ProfilerActivity, profile
        with profile(activities = [ProfilerActivity.CPU]) as prof:
            trainer.train()
        exchanges = sum(1 for e in prof.events() if e.name == "gloo:all_to_all")
    else:
        trainer.train()
    compile_used = compile_counters.since(before)
    if packed:
        # torch >= 2.13 counts data-dependent breaks (the old cu_seqlens tail check) under "unimplemented".
        compile_used["unimplemented"] = int(
            sum(torch._dynamo.utils.counters.get("unimplemented", {}).values())
        )
    if device != "cpu":
        assert autocast_seen == {False}, ("a GPU cell ran under autocast", autocast_seen)
    logs = [h for h in trainer.state.log_history if "loss" in h]
    is_tp = tp_native.is_tensor_parallel(trainer.model)
    if is_tp:
        state = tp_native.tp_full_state_dict(trainer.model, adapters_only = True)
    else:
        state = {
            k: _full(v)
            for k, v in trainer.model.state_dict().items()
            if "lora_" in k or mode == "full"
        }
    stage = pp_native.pipeline_stage_of(trainer.model)
    res = {
        "layout": None if layout is None else repr(layout),
        "losses": [h["loss"] for h in logs],
        "grad_norms": [h.get("grad_norm") for h in logs],
        "grads": {k.replace("module.", "").replace("_orig_mod.", ""): v for k, v in grads.items()},
        "state": {
            k.replace("module.", "").replace("_orig_mod.", ""): _full(v)
            for k, v in state.items()
            if "lora_" in k or mode == "full"
        },
        "cp_group": None
        if manager is None
        else sorted(int(r) for r in manager.mesh.mesh.flatten().tolist()),
        "is_tp": is_tp,
        "stage": None if stage is None else int(stage.stage_index),
        "seen_lengths": sorted(seen_lengths),
        "autocast_seen": sorted(autocast_seen),
        "ulysses_calls": None if compile else len(ulysses_calls),
        "cp_backend": None if manager is None else manager.backend,
        "epoch": float(trainer.state.epoch or 0.0),
        "rows_read": rows_read,
        "dataset_rows": len(data),
        "ga": ga,
        "compiled_classes": [getattr(c, "__name__", c) for c in compiled],
        "native_fsdp2": __import__("unsloth.models.fsdp2_native", fromlist = ["x"]).is_native_fsdp2(
            trainer.model
        ),
        "banner": trainer.__dict__.get("_unsloth_parallel_announced"),
        "output_router_logits": bool(getattr(trainer.model.config, "output_router_logits", False)),
        "compile": compile_used,
        "moe_calls": None if moe_calls is None else dict(moe_calls),
        "moe_backend": _selected_moe_backend(),
        "device_name": torch.cuda.get_device_name() if device != "cpu" else "cpu",
        "dist_backend": dist.get_backend() if dist.is_initialized() else None,
        "gc": dict(gc_stats),
        "memory": __import__("unsloth.parallel_capabilities", fromlist = ["x"])._memory(
            trainer.model, args
        ),
        "packed_seen": packed_seen,
        "oracle": oracle,
        "query": query,
        "exchanges": exchanges,
        "n_layers": int(trainer.model.config.num_hidden_layers),
        "gdn_calls": len(gdn_calls),
    }
    if dist.is_initialized():
        dist.barrier()
    return res


CASES = {"train": _case_train}


def _worker(rank, world, port, case, tmp, env, kwargs):
    os.environ.update(
        MASTER_ADDR = "127.0.0.1",
        MASTER_PORT = str(port),
        RANK = str(rank),
        WORLD_SIZE = str(world),
        LOCAL_RANK = str(rank),
        LOCAL_WORLD_SIZE = str(world),
    )
    for key in _ENV_KEYS:
        os.environ.pop(key, None)
    os.environ.update(env)
    torch.set_num_threads(1)
    import torch.distributed as dist

    try:
        try:
            res = CASES[case](rank, world, tmp, **kwargs)
        except Exception as e:
            import traceback
            res = {"error": f"{type(e).__name__}: {e}", "trace": traceback.format_exc()}
        torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
    finally:
        if dist.is_initialized():
            try:
                dist.destroy_process_group()
            except Exception:
                pass


def _spawn(
    case,
    world = 1,
    env = None,
    **kwargs,
):
    ctx = multiprocessing.get_context("spawn")
    port = _free_port()
    with tempfile.TemporaryDirectory() as tmp:
        kwargs["ckpt"] = _checkpoint(tmp, kwargs.get("arch", "llama"))
        procs = [
            ctx.Process(target = _worker, args = (r, world, port, case, tmp, env or {}, kwargs))
            for r in range(world)
        ]
        for p in procs:
            p.start()
        for p in procs:
            p.join(900)
        for p in procs:
            if p.is_alive():
                p.kill()
        results = []
        for r in range(world):
            path = os.path.join(tmp, f"result_{r}.pt")
            results.append(
                torch.load(path, weights_only = False)
                if os.path.exists(path)
                else {"error": f"rank {r} exit code {procs[r].exitcode}"}
            )
    if "skip" in results[0]:
        pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")
    return results


_REF = {}


def _ref(
    bs,
    compile = False,
    arch = "llama",
    **kwargs,
):
    key = (bs, compile, arch, tuple(sorted(kwargs.items())))
    if key not in _REF:
        (_REF[key],) = _spawn("train", 1, bs = bs, compile = compile, arch = arch, **kwargs)
        assert "error" not in _REF[key], _REF[key]
    return _REF[key]


def _rows_per_step(res):
    """The dataset rows each optimizer step trained on (one rank's view)."""
    assert "error" not in res, ("a worker failed", res.get("error"), res.get("trace"))
    ga, read = res["ga"], res["rows_read"]
    return [
        sorted(r for mb in read[i : i + ga] for r in mb)
        for i in range(0, len(read) - len(read) % ga, ga)
    ]


# torch < 2.13 refuses fewer micro-batches than stages in every single-stage schedule (PipelineScheduleSingle);
# 2.13 keeps the check in Schedule1F1B only.
_ONE_MICROBATCH_REFUSED = "pipeline schedule cannot run 1 micro-batches over 2 stages"


def _skip_if_one_microbatch_refused(results):
    from unsloth.models import pp_env

    errors = [res.get("error") or "" for res in results]
    # Where GPipe pads a single micro-batch up to the stage count, a batch of one splits too and is refused.
    padded = pp_env._gpipe_needs_stage_count() and all(
        "splits each Trainer batch into 2" in e for e in errors
    )
    if errors and (padded or all(_ONE_MICROBATCH_REFUSED in e for e in errors)):
        import torch as _torch
        pytest.skip(
            f"torch {_torch.__version__}'s GPipe schedule refuses 1 micro-batch over 2 stages (torch < 2.13), and "
            "the pipeline takes the router aux loss over one micro-batch only: " + errors[0][:200]
        )


def _same_rows(
    results,
    want,
    what = "",
):
    """Every step of the composed run trains on exactly the rows one process trains on (data parallel replicas
    split them, cp / tp / pp peers share theirs), and no side read a row twice (no second epoch)."""
    for rank, res in enumerate([want, *results]):
        assert "error" not in res, (
            what,
            "reference" if rank == 0 else f"rank {rank - 1}",
            res.get("error"),
            res.get("trace"),
        )
    expected = _rows_per_step(want)
    flat = [r for step in expected for r in step]
    assert len(flat) == len(set(flat)), (what, "one process read a row twice", want["rows_read"])
    union = [set() for _ in expected]
    for res in results:
        for i, step in enumerate(_rows_per_step(res)[: len(expected)]):
            union[i] |= set(step)
    assert [sorted(u) for u in union] == expected, (
        what,
        "the composed run read other rows",
        union,
        expected,
    )


def _close(
    got,
    want,
    rel = 1e-4,
    what = "",
    state_atol = 1e-6,
):
    assert "error" not in got, (what, got.get("error"), got.get("trace"))
    # The run must read every row once and no row twice: too few rows for the data parallel replicas wrap the
    # dataloader into a second epoch, and the later steps then train on other rows than one process.
    for side in (got, want):
        if "epoch" in side:
            assert side["epoch"] <= 1.0 + 1e-6, (what, "the dataloader wrapped", side["epoch"])
    assert got["losses"] == pytest.approx(want["losses"], rel = rel), (
        what,
        got["losses"],
        want["losses"],
    )
    assert got["grad_norms"] == pytest.approx(want["grad_norms"], rel = rel), (
        what,
        got["grad_norms"],
        want["grad_norms"],
    )
    for kind in ("grads", "state"):
        assert set(got[kind]) <= set(want[kind]) and got[kind], (
            what,
            kind,
            sorted(set(got[kind]) - set(want[kind])),
        )
        for key, value in got[kind].items():
            atol = state_atol if kind == "state" else 1e-6
            torch.testing.assert_close(
                value, want[kind][key], rtol = rel * 10, atol = atol, msg = f"{what} {kind} {key}"
            )


def test_cp2_alone_matches_one_process():
    """The harness: Ulysses cp 2 through SFTTrainer reproduces one process (losses, grads, adapters)."""
    ref = _ref(2)
    assert ref["seen_lengths"] == [SEQ] and ref["ulysses_calls"] == 0
    for rank, res in enumerate(_spawn("train", 2, bs = 2, cp = 2)):
        assert "error" not in res, (rank, res.get("error"), res.get("trace"))
        assert res["cp_group"] == [0, 1]
        assert res["seen_lengths"] == [SEQ // 2] and res["ulysses_calls"] >= N_LAYERS
        _close(res, ref, what = rank)


def test_cp2_with_ddp_dp2_matches_one_process():
    """Plain DDP x cp 2 on 4 ranks: cp peers adjacent, dp replicas averaged, same as one process on the batch."""
    ref = _ref(4)
    for rank, res in enumerate(_spawn("train", 4, bs = 2, cp = 2)):
        assert "error" not in res, (rank, res.get("error"), res.get("trace"))
        assert res["cp_group"] == [rank - rank % 2, rank - rank % 2 + 1], (rank, res["cp_group"])
        assert not res["native_fsdp2"] and not res["is_tp"]
        assert res["seen_lengths"] == [SEQ // 2] and res["ulysses_calls"] >= N_LAYERS
        _close(res, ref, what = ("ddp", rank))


@pytest.mark.parametrize("env", [{"UNSLOTH_PP_SIZE": "2"}], ids = ["pp2"])
def test_cp_with_pipeline_parallel_env_is_refused_before_the_split(env):
    for res in _spawn("train", 4, env, bs = 2, cp = 2):
        assert "does not compose with" in (res.get("error") or ""), (
            res.get("error") or res["losses"]
        )
        assert "pipeline parallel (UNSLOTH_PP_SIZE=2)" in res["error"]


def test_cp_from_the_config_beside_a_tp_launch_is_refused_with_the_layout_hint():
    """SFTConfig(context_parallel_size = 2) under UNSLOTH_TP_SIZE=2: the layout was read without a cp axis."""
    for res in _spawn("train", 4, {"UNSLOTH_TP_SIZE": "2"}, bs = 2, cp = 2):
        error = res.get("error") or ""
        assert "does not compose with" in error and "UNSLOTH_PARALLEL" in error, (
            error or res["losses"]
        )


@pytest.mark.parametrize(
    "world, flag, bs, ref_bs, layout",
    [
        (4, "tp=2,cp=2", 2, 2, "cp 2 x tp 2"),
        (8, "tp=2,cp=2", 2, 4, "dp 2 (DDP) x cp 2 x tp 2"),
        (8, "dp=2,tp=2,cp=2,fsdp=1", 2, 4, "dp 2 (native FSDP2) x cp 2 x tp 2"),
        (
            8,
            "dp=2,tp=2,pp=1,cp=2,fsdp=1,cp_backend=ulysses",
            2,
            4,
            "dp 2 (native FSDP2) x cp 2 x tp 2",
        ),
    ],
    ids = ["cp2_tp2", "dp2_cp2_tp2", "fsdp2_cp2_tp2", "launcher_form"],
)
def test_cp_x_tp_from_unsloth_parallel_matches_one_process(world, flag, bs, ref_bs, layout):
    ref = _ref(ref_bs)
    env = {"UNSLOTH_PARALLEL": flag}
    if flag.startswith("dp=2,tp=2,pp=1"):
        # What the notebooks launcher exports: every axis, plus agreeing single-axis variables.
        env.update(UNSLOTH_FSDP2 = "1", UNSLOTH_TP_SIZE = "2", UNSLOTH_CP_SIZE = "2", UNSLOTH_PP_SIZE = "1")
    results = _spawn("train", world, env, bs = bs, cp = 2, sft_cp = False)
    for rank, res in enumerate(results):
        assert "error" not in res, (rank, res.get("trace"))
        assert res["layout"] == layout and res["is_tp"], res["layout"]
        # cp peers are 2 apart (tp innermost), never tp peers.
        tp_index = rank % 2
        base = rank - rank % 4
        assert res["cp_group"] == [base + tp_index, base + tp_index + 2], (rank, res["cp_group"])
        # Engaged: 'auto' picked Ulysses, each rank embedded SEQ / cp tokens, every layer's attention exchanged.
        assert res["cp_backend"] == "ulysses" and res["seen_lengths"] == [SEQ // 2], res[
            "seen_lengths"
        ]
        assert res["ulysses_calls"] >= N_LAYERS, res["ulysses_calls"]
        _close(res, ref, what = (flag, rank))


@pytest.mark.parametrize("flag", ["pp=2,cp=2", "pp=2,tp=2,cp=2"])
def test_cp_x_pp_is_refused_by_the_registry_before_the_split(flag):
    world = 8 if "tp" in flag else 4
    for res in _spawn("train", world, {"UNSLOTH_PARALLEL": flag}, bs = 2, cp = 2, sft_cp = False):
        error = res.get("error") or ""
        assert "cp x pp" in error and "Instead:" in error, error or res["losses"]
        assert res.get("stage") is None


# The registry's BLOCK_COMPILE_TORCH (checked in test_parallel_env.py), read here without importing unsloth: below it
# these cells run eager (torch 2.10 recompiles more than one process).
_BLOCK_COMPILE_TORCH = (2, 13)
needs_block_compile_torch = pytest.mark.skipif(
    tuple(int(x) for x in torch.__version__.split("+")[0].split(".")[:2]) < _BLOCK_COMPILE_TORCH,
    reason = "block compile cells need torch >= 2.13 (the registry runs them eager below)",
)


@needs_block_compile_torch
@pytest.mark.parametrize(
    "world, flag, ref_bs",
    [(4, "tp=2,cp=2", 2), (8, "dp=2,tp=2,cp=2,fsdp=1", 4)],
    ids = ["cp2_tp2", "fsdp2_cp2_tp2"],
)
def test_block_compile_composes_like_one_process(world, flag, ref_bs):
    """Unsloth's block compile under cp x tp: the same numbers as one eager process, every decoder layer compiled,
    and no more graph breaks / recompiles than the same compiled script in one process (Dynamo's counters)."""
    from unsloth.utils.compile_counters import not_above

    eager = _ref(ref_bs)
    single = _ref(ref_bs, compile = True)
    assert (
        single["compiled_classes"] == ["LlamaDecoderLayer"]
        and single["compile"]["unique_graphs"] > 0
    )
    _close(single, eager, what = "one process compiled")
    for rank, res in enumerate(
        _spawn("train", world, {"UNSLOTH_PARALLEL": flag}, bs = 2, cp = 2, sft_cp = False, compile = True)
    ):
        assert "error" not in res, (rank, res.get("trace"))
        assert (
            res["compiled_classes"] == ["LlamaDecoderLayer"] and res["compile"]["unique_graphs"] > 0
        )
        assert res["cp_backend"] == "ulysses" and res["seen_lengths"] == [SEQ // 2], res[
            "seen_lengths"
        ]
        assert not not_above(res["compile"], single["compile"]), (
            rank,
            res["compile"],
            single["compile"],
        )
        _close(res, eager, what = (flag, "compiled", rank))


@needs_block_compile_torch
@pytest.mark.parametrize("ga", [1, 2])
@pytest.mark.parametrize("packed", ["pack", "padfree"])
@pytest.mark.parametrize("arch", ["qwen3", "llama"])
def test_block_compile_packed_rows_under_cp_match_one_process(arch, packed, ga):
    """Unsloth's block compile x packed / padding-free rows x cp 2 Ulysses (the registry's packed compiled
    cell): the same numbers as one eager process; the registry sees block compile, packing and cp Ulysses;
    every micro-batch's documents reach the exchange; the compiled layers issue the all-to-alls; no more graph
    breaks (also data-dependent ones) / recompiles / graphs than the same compiled script in one process."""
    from unsloth.utils.compile_counters import not_above

    kw = dict(arch = arch, packed = packed, ga = ga)
    eager = _ref(1, **kw)
    single = _ref(1, compile = True, **kw)
    assert single["compiled_classes"] and single["compile"]["unique_graphs"] > 0, single["compile"]
    assert single["oracle"] < 1e-4, (
        "the one-process reference attends across documents",
        single["oracle"],
    )
    _close(single, eager, what = "one process compiled")
    micro = len(single["rows_read"])
    want_docs = [list(_PACKED_DOCS[packed][r[0] % 4]) for r in single["rows_read"]]
    results = _spawn("train", 2, bs = 1, cp = 2, compile = True, **kw)
    _same_rows(results, single, what = kw)
    for rank, res in enumerate(results):
        what = (kw, rank)
        assert res["compiled_classes"] == single["compiled_classes"], what
        query = res["query"]
        assert (
            query["compile"],
            query["packing"],
            query["cp"],
            query["cp_backend"],
            query["family"],
        ) == (
            "block",
            True,
            2,
            "ulysses",
            arch,
        ), (what, query)
        # The run as configured reaches the registry's packed compiled cell, whose evidence is this test.
        from unsloth import parallel_capabilities as caps

        status, cell = caps.cell_status(caps.Query(**query))
        assert (
            status == "supported" and "test_block_compile_packed_rows_under_cp" in cell.compile
        ), (what, cell)
        assert res["cp_backend"] == "ulysses" and res["seen_lengths"] == [SEQ // 2], (
            what,
            res["seen_lengths"],
        )
        assert res["packed_seen"] == want_docs, (what, res["packed_seen"], want_docs)
        # At least one exchange per layer per micro-batch forward and backward (Ulysses swaps q / k / v and back).
        assert res["exchanges"] >= 2 * res["n_layers"] * micro, (what, res["exchanges"])
        assert not not_above(res["compile"], single["compile"]), (
            what,
            res["compile"],
            single["compile"],
        )
        for key in ("unimplemented", "unique_graphs"):
            assert res["compile"][key] <= single["compile"][key], (
                what,
                key,
                res["compile"],
                single["compile"],
            )
        # Adam (eps 1e-4) amplifies cp's fp32 reduction order on near-zero adapter grads: the eager cp arm moves
        # the adapters by the same 1.8e-5 (qwen3 padding-free), so the band is the eager composition's.
        _close(res, eager, what = what, state_atol = 5e-5)


@pytest.mark.parametrize(
    "env, sft_cp",
    [({"UNSLOTH_FSDP2": "1"}, True), ({"UNSLOTH_PARALLEL": "cp=2,fsdp=1"}, False)],
    ids = ["fsdp2_env_cp_config", "unsloth_parallel"],
)
def test_cp_with_native_fsdp2_matches_one_process(env, sft_cp):
    """Native FSDP2 (UNSLOTH_FSDP2=1) never reads context_parallel_size: it must shard over the whole
    (dp_shard, cp) world so cp peers' gradients are reduced together, and the loss is normalised over dp x cp."""
    ref = _ref(4)
    for rank, res in enumerate(_spawn("train", 4, env, bs = 2, cp = 2, sft_cp = sft_cp)):
        assert "error" not in res, (rank, res.get("trace"))
        assert res["native_fsdp2"] and res["cp_group"] == [rank - rank % 2, rank - rank % 2 + 1]
        assert res["seen_lengths"] == [SEQ // 2] and res["ulysses_calls"] >= N_LAYERS
        assert "FSDP2 shards and reduces over dp_shard x cp" in (res["banner"] or ""), res["banner"]
        _close(res, ref, what = (env, rank))


def _needs_qwen3_5():
    import transformers
    if not hasattr(transformers, "Qwen3_5TextConfig"):
        pytest.skip(f"transformers {transformers.__version__} has no Qwen3.5")


def test_hybrid_qwen3_5_under_cp_matches_one_process():
    """Qwen3.5 hybrid (Gated DeltaNet + attention) under Ulysses cp 2: both layer kinds exchange, one process's
    numbers."""
    _needs_qwen3_5()
    ref = _ref(2, arch = "qwen3_5")
    assert ref["gdn_calls"] == 0 and ref["ulysses_calls"] == 0
    for rank, res in enumerate(_spawn("train", 2, bs = 2, cp = 2, arch = "qwen3_5")):
        assert "error" not in res, (rank, res.get("error"), res.get("trace"))
        assert res["cp_group"] == [0, 1] and res["seen_lengths"] == [SEQ // 2]
        # Every forward of a training step: N_LAYERS // 2 linear attention and as many attention layers.
        assert res["gdn_calls"] >= N_LAYERS // 2 and res["ulysses_calls"] >= N_LAYERS // 2, res
        _close(res, ref, what = ("qwen3_5 cp2", rank))


@pytest.mark.parametrize(
    "env, sft_cp",
    [({"UNSLOTH_FSDP2": "1"}, True), ({"UNSLOTH_PARALLEL": "cp=2,fsdp=1"}, False)],
    ids = ["fsdp2_env_cp_config", "unsloth_parallel"],
)
def test_hybrid_qwen3_5_cp_with_native_fsdp2_matches_one_process(env, sft_cp):
    """The registry's hybrid cp x native FSDP2 cell: strict mode (unverified layouts refused) trains it, FSDP2
    shards over (dp_shard, cp), the Gated DeltaNet and attention exchanges run, one process's numbers."""
    _needs_qwen3_5()
    ref = _ref(4, arch = "qwen3_5")
    env = {**env, "UNSLOTH_PARALLEL_STRICT": "1"}
    for rank, res in enumerate(_spawn("train", 4, env, bs = 2, cp = 2, sft_cp = sft_cp, arch = "qwen3_5")):
        assert "error" not in res, (rank, res.get("error"), res.get("trace"))
        assert res["native_fsdp2"] and res["cp_group"] == [rank - rank % 2, rank - rank % 2 + 1]
        assert res["seen_lengths"] == [SEQ // 2]
        assert res["gdn_calls"] >= N_LAYERS // 2 and res["ulysses_calls"] >= N_LAYERS // 2, res
        assert "FSDP2 shards and reduces over dp_shard x cp" in (res["banner"] or ""), res["banner"]
        _close(res, ref, what = ("qwen3_5", env, rank), state_atol = 1e-5)


def test_dense_qwen3_under_cp_matches_one_process():
    """The control for the MoE cell below: the same family without experts trains like one process."""
    ref = _ref(2, arch = "qwen3")
    for rank, res in enumerate(_spawn("train", 2, bs = 2, cp = 2, arch = "qwen3")):
        assert "error" not in res, (rank, res.get("trace"))
        assert res["seen_lengths"] == [SEQ // 2] and res["ulysses_calls"] >= 2
        _close(res, ref, what = ("qwen3", rank))


@pytest.mark.parametrize(
    "mode, world, env, cp",
    [
        ("lora", 2, {}, 1),
        ("lora", 2, {"UNSLOTH_FSDP2": "1"}, 1),
        ("lora", 2, {}, 2),
        ("lora", 4, {"UNSLOTH_FSDP2": "1"}, 2),
        ("full", 2, {"UNSLOTH_FSDP2": "1"}, 1),
        ("full", 2, {}, 2),
    ],
    ids = ["lora-ddp", "lora-fsdp2", "lora-cp2", "lora-fsdp2_cp2", "full-fsdp2", "full-cp2"],
)
def test_moe_under_data_and_context_parallel_matches_one_process(mode, world, env, cp):
    """Tiny Qwen3-MoE (4 experts, 2 per token) with the router aux loss off (router_aux_loss_coef = 0.0: TRL
    would otherwise turn it on from the architecture's coefficient): LoRA on attention, router and the experts
    (ParamWrapper), or full finetuning, under DDP / native FSDP2 / Ulysses cp, trains like one process."""
    ref = _ref(2 * world // cp, arch = "qwen3_moe", mode = mode)
    assert not ref["output_router_logits"]
    results = _spawn("train", world, env, bs = 2, cp = cp, arch = "qwen3_moe", mode = mode)
    _same_rows(results, ref, what = (mode, world, env, cp))
    for rank, res in enumerate(results):
        assert "error" not in res, (rank, res.get("trace"))
        assert not res["output_router_logits"]
        experts = [k for k in res["grads"] if ".experts." in k]
        assert experts and all(
            res["grads"][k].abs().sum() > 0 for k in experts
        ), experts  # routed experts train
        if mode == "lora":
            assert any("lora_" in k for k in experts), experts
        if cp > 1:
            assert res["seen_lengths"] == [SEQ // cp] and res["ulysses_calls"] >= 2
        # Full finetuning also steps the 151936-row embedding: Adam (lr 1e-2, eps 1e-4) turns 1e-8 gradient noise
        # on a nearly unused row into a 2e-6 weight difference after two steps (losses agree to 1e-7).
        _close(
            res, ref, what = (mode, world, env, cp, rank), state_atol = 1e-5 if mode == "full" else 1e-6
        )


@pytest.mark.parametrize("env", [{}, {"UNSLOTH_FSDP2": "1"}], ids = ["ddp", "fsdp2"])
def test_moe_with_router_aux_loss_under_ddp_matches_per_micro_batch(env):
    """The load balancing loss is per micro-batch (a product of means). Data parallel keeps each micro-batch on
    one rank, so dp 2 x accumulation 2 equals one process accumulating the same 4 micro-batches."""
    (ref,) = _spawn("train", 1, bs = 2, ga = 4, arch = "qwen3_moe", router_aux = True)
    assert ref["output_router_logits"]
    results = _spawn("train", 2, env, bs = 2, ga = 2, arch = "qwen3_moe", router_aux = True)
    _same_rows(results, ref, what = env)
    for rank, res in enumerate(results):
        assert "error" not in res, (rank, res.get("trace"))
        assert res["output_router_logits"]
        _close(res, ref, what = (env, rank))


@pytest.mark.parametrize("zoo_compile", [False, True], ids = ["forward_install", "zoo_compiled"])
@pytest.mark.parametrize("aux", [True, False], ids = ["aux_on", "aux_off"])
@pytest.mark.parametrize("ga", [1, 2, 4])
def test_pp_moe_router_aux_loss_with_accumulation_matches_one_process(
    ga, aux, zoo_compile, tmp_path
):
    """PP=2 Qwen3-MoE GA 1/2/4 matches one process: the aux term is token-share weighted (full = ~ga x larger)."""
    kw = dict(bs = 2, ga = ga, steps = 3, arch = "qwen3_moe", router_aux = aux, aux_coef = 0.5 if aux else None)
    base = {}
    if zoo_compile:
        kw["zoo_compile"] = True
        base = {
            "UNSLOTH_FUSED_FORWARD": "0",
            "UNSLOTH_COMPILE_DISABLE": "1",
            "UNSLOTH_COMPILE_LOCATION": str(tmp_path / "unsloth_compiled_cache"),
        }
    (ref,) = _spawn("train", 1, base, **kw)
    assert ref["output_router_logits"] is aux, ref
    env = {
        "UNSLOTH_PP_SIZE": "2",
        "UNSLOTH_PP_MICROBATCHES": "1",
        "UNSLOTH_PP_SCHEDULE": "gpipe",
        **base,
    }
    results = _spawn("train", 2, env, **kw)
    _skip_if_one_microbatch_refused(results)
    _same_rows(results, ref, what = (ga, aux))
    assert sorted(res.get("stage") for res in results) == [0, 1], results
    for rank, res in enumerate(results):
        # transformers 4.x drifts ~1.4e-4 / ~5e-6 at GA 4 even aux off; the guarded error is 13-42% of the loss
        _close(res, ref, rel = 5e-4, state_atol = 1e-5, what = (ga, aux, rank))


@pytest.mark.parametrize(
    "arch,aux",
    [("llama", False), ("qwen3_moe", False), ("qwen3_moe", True)],
    ids = ["dense", "moe_aux_off", "moe_aux_on"],
)
def test_cpu_pipeline_eval_matches_one_process(monkeypatch, arch, aux):
    monkeypatch.setenv("UNSLOTH_FORCE_CPU", "1")
    kw = dict(
        bs = 2,
        ga = 1,
        steps = 3,
        arch = arch,
        eval_only = True,
        router_aux = aux,
        aux_coef = 0.5 if aux else None,
    )
    (ref,) = _spawn("train", 1, **kw)
    assert "error" not in ref, ref
    results = _spawn(
        "train",
        2,
        {"UNSLOTH_PP_SIZE": "2", "UNSLOTH_PP_MICROBATCHES": "1", "UNSLOTH_PP_SCHEDULE": "gpipe"},
        **kw,
    )
    _skip_if_one_microbatch_refused(results)
    assert sorted(res.get("stage", -1) for res in results) == [0, 1], results
    for res in [ref, *results]:
        assert "error" not in res, res
        assert res["cpu_compute_loss"]
        assert res["cpu_chunked_ce"] is (arch == "llama")
        assert res["rows_read"] == ref["rows_read"]
        assert res["eval_loss"] == pytest.approx(ref["eval_loss"], rel = 1e-5)


_PP_AUX_SPLIT = "load-balancing loss per micro-batch"


@pytest.mark.parametrize(
    "how",
    [
        dict(router_aux = True, aux_coef = 0.5),
        dict(trl_aux = True, aux_coef = 0.5),
    ],
    ids = ["config_flag", "trl_coef"],
)
@pytest.mark.parametrize("env", [{}, {"UNSLOTH_PP_SCHEDULE": "1f1b"}], ids = ["default", "1f1b"])
def test_pp_moe_router_aux_loss_split_over_micro_batches_is_refused_at_trainer_init(how, env):
    env = {"UNSLOTH_PP_SIZE": "2", **env}
    for rank, res in enumerate(
        _spawn("train", 2, env, bs = 2, ga = 1, steps = 1, arch = "qwen3_moe", probe_refusal = True, **how)
    ):
        assert _PP_AUX_SPLIT in (res.get("error") or ""), (rank, res)
        assert res["phase"] == "init", (rank, res)
        assert not res["split"] and res["untouched"], (rank, res)


@pytest.mark.parametrize(
    "how, env",
    [
        (dict(), {}),
        (dict(router_aux = True, args_coef = 0.0), {}),
        (
            dict(router_aux = True, aux_coef = 0.5, ga = 2),
            {"UNSLOTH_PP_MICROBATCHES": "1", "UNSLOTH_PP_SCHEDULE": "gpipe"},
        ),
        (dict(router_aux = True, aux_coef = 0.5, bs = 1), {"UNSLOTH_PP_SCHEDULE": "gpipe"}),
    ],
    ids = ["aux_off", "config_flag_coef0", "aux_on_one_microbatch", "aux_on_batch_of_one"],
)
def test_pp_moe_router_aux_loss_allowed_layouts_still_train(how, env):
    kw = dict(bs = 2, ga = 1, steps = 2, arch = "qwen3_moe", probe_refusal = True)
    kw.update(how)
    results = _spawn("train", 2, {"UNSLOTH_PP_SIZE": "2", **env}, **kw)
    _skip_if_one_microbatch_refused(results)
    assert sorted(res.get("stage") for res in results) == [0, 1], results
    for rank, res in enumerate(results):
        assert res["phase"] is None and "error" not in res, (rank, res)
        assert len(res["losses"]) == 2 and all(l == l for l in res["losses"]), (rank, res)


# LoRA on the experts is eager only under cp (the registry's qwen3_moe x lora+experts x cp row): its per instance
# ParametrizedQwen3MoeTopKRouter type guards recompile more than one process once the router capture hooks are
# installed before the first block compiles.
@needs_block_compile_torch
@pytest.mark.parametrize("mode", ["full"])
def test_moe_block_compile_under_cp_is_compile_clean(mode):
    from unsloth.utils.compile_counters import not_above

    atol = 1e-5 if mode == "full" else 1e-6  # the embedding row note in the parity test above
    ref = _ref(2, arch = "qwen3_moe", mode = mode)
    single = _ref(2, compile = True, arch = "qwen3_moe", mode = mode)
    assert single["compiled_classes"] and single["compile"]["unique_graphs"] > 0
    _close(single, ref, what = "qwen3_moe one process compiled", state_atol = atol)
    results = _spawn("train", 2, bs = 2, cp = 2, arch = "qwen3_moe", mode = mode, compile = True)
    _same_rows(results, ref, what = ("compiled", mode))
    for rank, res in enumerate(results):
        assert "error" not in res, (rank, res.get("trace"))
        assert res["compiled_classes"] and res["seen_lengths"] == [SEQ // 2]
        assert not not_above(res["compile"], single["compile"]), (res["compile"], single["compile"])
        _close(res, ref, what = ("qwen3_moe cp2 compiled", mode, rank), state_atol = atol)


@pytest.mark.parametrize(
    "config_flag", [True, False], ids = ["output_router_logits", "trl_default_coef"]
)
def test_moe_with_router_aux_loss_under_cp_is_refused(config_flag):
    """Refused before the split, also when only TRL's default router_aux_loss_coef would turn the loss on."""
    for res in _spawn(
        "train", 2, bs = 2, cp = 2, arch = "qwen3_moe", router_aux = config_flag, trl_aux = True
    ):
        error = res.get("error") or ""
        assert "MoE x pp / cp" in error and "router_aux_loss_coef = 0.0" in error, (
            error or res["losses"]
        )


# Unsloth's checkpoint (paths A and B) in the gated cells above, against one process without checkpointing.
_UNSLOTH_GC = pytest.mark.parametrize("gc", ["unsloth", "unsloth_nonre"])


def _checkpoint_engaged(res, gc):
    assert res["memory"] == "unsloth_offload", res["memory"]
    assert res["gc"]["checkpoint_calls"] > 0, res["gc"]
    if gc == "unsloth_nonre":
        assert res["gc"]["offloaded"] > 0 and res["gc"]["restored"] == res["gc"]["offloaded"], res[
            "gc"
        ]


@_UNSLOTH_GC
@pytest.mark.parametrize(
    "world, flag, ref_bs, layout",
    [
        (4, "tp=2,cp=2", 2, "cp 2 x tp 2"),
        (8, "tp=2,cp=2", 4, "dp 2 (DDP) x cp 2 x tp 2"),
        (8, "dp=2,tp=2,cp=2,fsdp=1", 4, "dp 2 (native FSDP2) x cp 2 x tp 2"),
    ],
    ids = ["cp2_tp2", "dp2_cp2_tp2", "fsdp2_cp2_tp2"],
)
def test_cp_x_tp_with_unsloth_checkpointing_matches_one_process(world, flag, ref_bs, layout, gc):
    ref = _ref(ref_bs)
    results = _spawn("train", world, {"UNSLOTH_PARALLEL": flag}, bs = 2, cp = 2, sft_cp = False, gc = gc)
    for rank, res in enumerate(results):
        assert "error" not in res, (rank, res.get("error"), res.get("trace"))
        assert res["layout"] == layout and res["is_tp"], res["layout"]
        assert res["cp_backend"] == "ulysses" and res["seen_lengths"] == [SEQ // 2]
        assert res["ulysses_calls"] >= N_LAYERS, res["ulysses_calls"]
        _checkpoint_engaged(res, gc)
        _close(res, ref, what = (flag, gc, rank))


@needs_block_compile_torch
@_UNSLOTH_GC
@pytest.mark.parametrize(
    "world, flag, ref_bs",
    [(4, "tp=2,cp=2", 2), (8, "dp=2,tp=2,cp=2,fsdp=1", 4)],
    ids = ["cp2_tp2", "fsdp2_cp2_tp2"],
)
def test_block_compile_with_unsloth_checkpointing_composes_like_one_process(
    world, flag, ref_bs, gc
):
    from unsloth.utils.compile_counters import not_above

    eager = _ref(ref_bs)
    # The baseline runs under 1-rank DDP, which refuses reentrant checkpointing ("marked ready twice"), so
    # path A is compared to the compiled run without checkpointing.
    single = _ref(ref_bs, compile = True, gc = "none" if gc == "unsloth" else gc)
    assert single["compile"]["unique_graphs"] > 0
    if gc != "unsloth":
        _checkpoint_engaged(single, gc)
    _close(single, eager, what = ("one process compiled", gc))
    results = _spawn(
        "train", world, {"UNSLOTH_PARALLEL": flag}, bs = 2, cp = 2, sft_cp = False, compile = True, gc = gc
    )
    for rank, res in enumerate(results):
        assert "error" not in res, (rank, res.get("error"), res.get("trace"))
        assert res["compiled_classes"] == ["LlamaDecoderLayer"] and res["seen_lengths"] == [
            SEQ // 2
        ]
        _checkpoint_engaged(res, gc)
        assert not not_above(res["compile"], single["compile"]), (
            rank,
            res["compile"],
            single["compile"],
        )
        _close(res, eager, what = (flag, "compiled", gc, rank))


@_UNSLOTH_GC
@pytest.mark.parametrize(
    "env, sft_cp",
    [({"UNSLOTH_FSDP2": "1"}, True), ({"UNSLOTH_PARALLEL": "cp=2,fsdp=1"}, False)],
    ids = ["fsdp2_env_cp_config", "unsloth_parallel"],
)
def test_cp_with_native_fsdp2_and_unsloth_checkpointing_matches_one_process(env, sft_cp, gc):
    ref = _ref(4)
    for rank, res in enumerate(_spawn("train", 4, env, bs = 2, cp = 2, sft_cp = sft_cp, gc = gc)):
        assert "error" not in res, (rank, res.get("error"), res.get("trace"))
        assert (
            res["native_fsdp2"]
            and res["seen_lengths"] == [SEQ // 2]
            and res["ulysses_calls"] >= N_LAYERS
        )
        _checkpoint_engaged(res, gc)
        _close(res, ref, what = (env, gc, rank))


@_UNSLOTH_GC
@pytest.mark.parametrize(
    "mode, world, env",
    [("lora", 2, {}), ("lora", 4, {"UNSLOTH_FSDP2": "1"}), ("full", 2, {})],
    ids = ["lora-cp2", "lora-fsdp2_cp2", "full-cp2"],
)
def test_moe_under_cp_with_unsloth_checkpointing_matches_one_process(mode, world, env, gc):
    ref = _ref(2 * world // 2, arch = "qwen3_moe", mode = mode)
    results = _spawn("train", world, env, bs = 2, cp = 2, arch = "qwen3_moe", mode = mode, gc = gc)
    _same_rows(results, ref, what = (mode, world, env, gc))
    for rank, res in enumerate(results):
        assert "error" not in res, (rank, res.get("error"), res.get("trace"))
        assert not res["output_router_logits"]
        experts = [k for k in res["grads"] if ".experts." in k]
        assert experts and all(res["grads"][k].abs().sum() > 0 for k in experts), experts
        assert res["seen_lengths"] == [SEQ // 2] and res["ulysses_calls"] >= 2
        _checkpoint_engaged(res, gc)
        _close(
            res, ref, what = (mode, world, env, gc, rank), state_atol = 1e-5 if mode == "full" else 1e-6
        )


@needs_block_compile_torch
@_UNSLOTH_GC
def test_moe_block_compile_under_cp_with_unsloth_checkpointing_is_compile_clean(gc):
    from unsloth.utils.compile_counters import not_above

    ref = _ref(2, arch = "qwen3_moe", mode = "full")
    single = _ref(2, compile = True, arch = "qwen3_moe", mode = "full", gc = gc)
    _checkpoint_engaged(single, gc)
    _close(single, ref, what = ("qwen3_moe one process compiled", gc), state_atol = 1e-5)
    results = _spawn("train", 2, bs = 2, cp = 2, arch = "qwen3_moe", mode = "full", compile = True, gc = gc)
    _same_rows(results, ref, what = ("compiled", gc))
    for rank, res in enumerate(results):
        assert "error" not in res, (rank, res.get("error"), res.get("trace"))
        assert res["compiled_classes"] and res["seen_lengths"] == [SEQ // 2]
        _checkpoint_engaged(res, gc)
        assert not not_above(res["compile"], single["compile"]), (res["compile"], single["compile"])
        _close(res, ref, what = ("qwen3_moe cp2 compiled", gc, rank), state_atol = 1e-5)