Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

26.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

"""DeepSeek Ulysses context parallelism against independent fp64 single-process references, on CPU gloo
ranks: V2 / V3 MLA (YaRN softmax scale, narrower value heads) on main's generic route, V3.2's global top-k
indexer, V4 sliding / CSA / HCA sink attention at release compression ratios, sdpa / eager, HF checkpointing, right-padded batches, a rank with no
labels, a label across the shard seam, CP = 2 / 4 and DP 2 x CP 2, LoRA (also on the fused experts), one AdamW step, and negative
controls that must fail."""

import math
import os
import tempfile
import types

import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
import torch.nn.functional as F

import unsloth
from unsloth import context_parallel as cp
from unsloth.utils import attention_dispatch as ad
from unsloth.utils import ulysses_deepseek as ud
from unsloth.utils import ulysses_hybrid as uh

transformers = pytest.importorskip("transformers")
# Each arm must import the unsloth tree that holds this test file.
_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
assert os.path.abspath(unsloth.__file__).startswith(_ROOT + os.sep), unsloth.__file__
if not all(
    hasattr(transformers, n) for n in ("DeepseekV3Config", "DeepseekV32Config", "DeepseekV4Config")
):
    pytest.skip("transformers without DeepSeek V3 / V3.2 / V4", allow_module_level = True)

VOCAB = 96
# DeepSeek-V3's released YaRN parameters (factor 40, mscale 1.0): softmax scale = mscale^2 / sqrt(d).
YARN = {
    "rope_type": "yarn",
    "rope_theta": 10000.0,
    "factor": 40.0,
    "beta_fast": 32,
    "beta_slow": 1,
    "mscale": 1.0,
    "mscale_all_dim": 1.0,
    "original_max_position_embeddings": 64,
}


def _yarn_scale(head_dim):
    mscale = 0.1 * 1.0 * math.log(40.0) + 1.0
    return head_dim**-0.5 * mscale * mscale


def _named(name):
    return lambda detail: f"{name}: {detail}"


def _spawn(fn, world, *args):
    if "fork" not in mp.get_all_start_methods():
        pytest.skip("needs the fork start method")
    with tempfile.TemporaryDirectory() as d:
        mp.start_processes(
            _entry,
            args = (world, os.path.join(d, "init"), fn, args),
            nprocs = world,
            join = True,
            start_method = "fork",
        )


def _entry(rank, world, init_file, fn, args):
    torch.set_num_threads(1)
    dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = world)
    try:
        fn(rank, world, *args)
    finally:
        dist.destroy_process_group()


def _cp_groups(world, cp_size):
    """Contiguous CP groups (rank // cp_size = DP replica); every rank creates every group."""
    mine = None
    for start in range(0, world, cp_size):
        ranks = list(range(start, start + cp_size))
        group = dist.new_group(ranks)
        if dist.get_rank() in ranks:
            mine = group
    return mine


def _manager(cp_size, group):
    manager = object.__new__(cp.ContextParallelManager)
    manager.size = cp_size
    manager.backend = "ulysses"
    local = dist.get_rank(group)
    manager.mesh = types.SimpleNamespace(get_group = lambda: group, get_local_rank = lambda: local)
    return manager


# 1. Model-level parity: V2 / V3 (main's generic route), V3.2 (indexer), V4 (sliding / CSA / HCA).


def _mla(**extra):
    return dict(
        vocab_size = VOCAB,
        hidden_size = 64,
        intermediate_size = 96,
        moe_intermediate_size = 32,
        num_hidden_layers = 2,
        num_attention_heads = 4,
        num_key_value_heads = 4,
        kv_lora_rank = 16,
        q_lora_rank = 24,
        qk_rope_head_dim = 8,
        v_head_dim = 16,
        qk_nope_head_dim = 16,
        n_routed_experts = 4,
        n_shared_experts = 1,
        num_experts_per_tok = 2,
        first_k_dense_replace = 1,
        n_group = 1,
        topk_group = 1,
        rope_parameters = YARN,
        **extra,
    )


def _build(kind, v4_topk):
    t = transformers
    if kind == "v2":
        return t.DeepseekV2ForCausalLM(t.DeepseekV2Config(**_mla()))
    if kind == "v3":
        return t.DeepseekV3ForCausalLM(t.DeepseekV3Config(**_mla()))
    if kind == "v32":
        # 8 indexer heads keep exact zero-score ties (all ReLUs zero) improbable; top-6 of up to 24 keys.
        config = t.DeepseekV32Config(**_mla(index_topk = 6, index_head_dim = 16, index_n_heads = 8))
        return t.DeepseekV32ForCausalLM(config)
    # Release ratios: CSA 4, HCA 128, sliding window 128 (DeepSeek-V4 config defaults).
    config = t.DeepseekV4Config(
        vocab_size = VOCAB,
        hidden_size = 64,
        intermediate_size = 64,
        moe_intermediate_size = 32,
        num_hidden_layers = 3,
        num_attention_heads = 4,
        num_key_value_heads = 1,
        head_dim = 32,
        rope_head_dim = 16,
        q_lora_rank = 32,
        o_lora_rank = 16,
        o_groups = 2,
        num_local_experts = 4,
        num_experts_per_tok = 2,
        sliding_window = 128,
        # 32 indexer heads: an all-zero ReLU score (a tie whose top-k pick depends on the row
        # length, also without CP) is improbable; the release uses 64.
        index_n_heads = 32,
        index_head_dim = 16,
        index_topk = v4_topk,
        layer_types = [
            "sliding_attention",
            "compressed_sparse_attention",
            "heavily_compressed_attention",
        ],
        mlp_layer_types = ["hash_moe", "moe", "moe"],
        compress_rates = {"compressed_sparse_attention": 4, "heavily_compressed_attention": 128},
    )
    model = t.DeepseekV4ForCausalLM(config)
    with torch.no_grad():
        g = torch.Generator().manual_seed(0)
        for module in model.modules():
            if hasattr(module, "sinks"):
                module.sinks.copy_(0.5 * torch.randn(module.sinks.shape, generator = g))
            if hasattr(module, "position_bias"):
                module.position_bias.copy_(
                    0.3 * torch.randn(module.position_bias.shape, generator = g)
                )
            if hasattr(module, "tid2eid"):
                ids = torch.arange(VOCAB)
                module.tid2eid[:, 0] = ids % 4
                module.tid2eid[:, 1] = (ids + 1) % 4
    return model


def _model(
    kind,
    v4_topk = 16,
    lora = False,
    attn = None,
    checkpoint = False,
):
    torch.manual_seed(0)
    model = _build(kind, v4_topk).double().train()
    if attn is None:
        attn = "eager" if kind in ("v32", "v4") else "sdpa"
    model.config._attn_implementation = attn
    model.config.use_cache = False
    for module in model.modules():
        if hasattr(module, "set_experts_implementation"):
            module.set_experts_implementation("eager")
    if checkpoint:
        # HF gradient checkpointing: every decoder layer (and its CP forward) is recomputed in backward.
        model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False})
    if lora:
        from peft import LoraConfig, get_peft_model

        targets = (
            ["q_b_proj", "kv_proj", "o_b_proj"]
            if kind == "v4"
            else ["q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"]
        )
        extra = {}
        if lora == "experts":
            # LoRA on the routed experts too: the fused 3D expert weights.
            extra["target_parameters"] = ["experts.gate_up_proj", "experts.down_proj"]
        model = get_peft_model(
            model, LoraConfig(r = 4, lora_alpha = 8, target_modules = targets, **extra)
        )
        g = torch.Generator().manual_seed(5)
        with torch.no_grad():
            for name, p in model.named_parameters():
                if "lora_B" in name:  # nonzero B: the adapter changes the forward
                    p.copy_(0.05 * torch.randn(p.shape, generator = g, dtype = p.dtype))
    return model


def _batch(seq, lengths, labels_scheme, cp_size):
    g = torch.Generator().manual_seed(1)
    ids = torch.randint(1, VOCAB, (len(lengths), seq), generator = g)
    mask = torch.zeros_like(ids)
    for row, n in enumerate(lengths):
        mask[row, :n] = 1
    ids = ids * mask  # right padding with pad id 0
    labels = ids.masked_fill(mask == 0, -100)
    if labels_scheme == "late_rank_empty":
        # The last CP rank of the padded row gets no labels; the label predicting the token right
        # after the first seam is kept so a label crosses the seam.
        padded = seq + (-seq) % cp_size
        start = padded - padded // cp_size
        labels[:, start:] = -100
        labels[:, padded // cp_size] = ids[:, padded // cp_size].masked_fill(
            mask[:, padded // cp_size] == 0, -100
        )
    return ids, mask, labels


def _sum_ce(logits, shift_labels):
    return F.cross_entropy(
        logits.flatten(0, 1), shift_labels.flatten(), ignore_index = -100, reduction = "sum"
    )


def _controls(model, kind, control, monkey):
    if control is None:
        return
    if control == "shard_local":
        for module in model.modules():
            if getattr(module, "_unsloth_cp_orig_forward", None) is not None:
                module.forward = module._unsloth_cp_orig_forward
    elif control == "default_scale":
        for module in model.modules():
            if hasattr(module, "qk_head_dim") and hasattr(module, "scaling"):
                module.scaling = module.qk_head_dim**-0.5
    elif control == "dropped_window":
        real = ud._sink_attention
        monkey(ud, "_sink_attention", lambda q, kv, bb, s, w, *a: real(q, kv, bb, s, None, *a))
    elif control == "wrong_offset":
        real = ud.ulysses_deepseek_v4_forward

        def shard_positions(
            module,
            *args,
            position_ids = None,
            group = None,
            **kw,
        ):
            return real(
                module, *args, position_ids = position_ids - position_ids[:, :1], group = group, **kw
            )

        monkey(ud, "ulysses_deepseek_v4_forward", shard_positions)
    elif control == "local_topk":
        real = ud._v32_global_topk

        def local_topk(indexer, hidden, q_resid, pe, group, module):
            # Shard-local selection: each rank scores only its own keys, then indices are offset.
            local = real(indexer, hidden, q_resid, pe, _SOLO[0], module)
            local = local + dist.get_rank(group) * hidden.shape[1]
            parts = [torch.empty_like(local) for _ in range(dist.get_world_size(group))]
            dist.all_gather(parts, local.contiguous(), group = group)
            return torch.cat(parts, dim = 1)

        monkey(ud, "_v32_global_topk", local_topk)
    else:
        raise ValueError(control)


_SOLO = [None]


def _parity_worker(rank, world, case):
    ad.HAS_FLASH_ATTENTION = False
    ad.HAS_XFORMERS = False
    if case.get("chunk"):
        os.environ["UNSLOTH_CP_ATTN_CHUNK"] = str(case["chunk"])
    kind, cp_size = case["kind"], case["cp"]
    dp = world // cp_size
    group = _cp_groups(world, cp_size)
    solos = [dist.new_group([r]) for r in range(world)]
    _SOLO[0] = solos[rank]
    patched = []

    def monkey(obj, name, value):
        patched.append((obj, name, getattr(obj, name)))
        setattr(obj, name, value)

    try:
        _parity_body(rank, world, case, kind, cp_size, dp, group, monkey)
    finally:
        for obj, name, value in reversed(patched):
            setattr(obj, name, value)


def _parity_body(rank, world, case, kind, cp_size, dp, group, monkey):
    import copy

    model = _model(
        kind,
        case.get("v4_topk", 16),
        case.get("lora", False),
        case.get("attn"),
        case.get("checkpoint", False),
    )
    if kind in ("v2", "v3"):
        # The MLA cases exercise what the generic route must carry: a non-default softmax scale and value
        # heads narrower than query / key heads.
        attn = next(m for n, m in model.named_modules() if n.endswith("self_attn"))
        assert abs(attn.scaling * attn.qk_head_dim**0.5 - 1) > 0.1, attn.scaling
        assert attn.v_head_dim < attn.qk_head_dim
        if kind == "v3":
            assert abs(attn.scaling - _yarn_scale(attn.qk_head_dim)) < 1e-12, attn.scaling
    reference = copy.deepcopy(model)
    seq, lengths = case["seq"], case["lengths"]
    ids, mask, labels = _batch(seq, lengths, case.get("labels", "all"), cp_size)
    shift = F.pad(labels, (0, 1), value = -100)[:, 1:]
    count = shift.ne(-100).sum()
    assert count > 0

    # transformers runs the MoE router, the V3.2 / V4 indexer scores and V4's compressor gate softmax
    # in fp32, so even the reference moves at ~1e-8 with the batch shape.
    tol = dict(rtol = 1e-6, atol = 1e-6)
    ref_logits = reference(input_ids = ids, attention_mask = mask, use_cache = False).logits
    (_sum_ce(ref_logits, shift) / count).backward()
    # Valid tokens must not depend on the padding: each row alone, unpadded.
    with torch.no_grad():
        for row, n in enumerate(lengths):
            alone = reference(input_ids = ids[row : row + 1, :n], use_cache = False).logits
            torch.testing.assert_close(
                ref_logits[row, :n].detach(), alone[0], msg = _named("ref pad"), **tol
            )

    rows = list(range(rank // cp_size, len(lengths), dp))
    manager = _manager(cp_size, group)
    manager.attach_attention_hooks(model)
    _controls(model, kind, case.get("control"), monkey)
    calls = []
    # Engagement: the CP path runs once per layer (twice under checkpointing), no silent fallback.
    owner, fn_name = (
        (ud, f"ulysses_deepseek_{kind}_forward")
        if kind in ("v32", "v4")
        else (uh, "_ulysses_hf_attention")
    )
    real_fn = getattr(owner, fn_name)

    def counted(*args, **kwargs):
        calls.append(1)
        return real_fn(*args, **kwargs)

    monkey(owner, fn_name, counted)
    inputs = {
        "input_ids": ids[rows].clone(),
        "attention_mask": mask[rows].clone(),
        "labels": labels[rows].clone(),
    }
    with manager.apply(inputs):
        out = model(
            input_ids = inputs["input_ids"],
            attention_mask = inputs["attention_mask"],
            position_ids = inputs["position_ids"],
            use_cache = False,
        )
        local_ce = _sum_ce(out.logits, inputs["shift_labels"])
        (local_ce / count).backward()
    if case.get("control") is None:
        runs = 2 if case.get("checkpoint") else 1
        layers = model.config.num_hidden_layers
        assert len(calls) == runs * layers, (len(calls), runs, layers)
    if case.get("labels") == "late_rank_empty" and dist.get_rank(group) == cp_size - 1:
        assert inputs["shift_labels"].ne(-100).sum() == 0
    logits = torch.cat(dist_all_gather(out.logits.detach(), group), dim = 1)
    for i, row in enumerate(rows):
        n = lengths[row]
        torch.testing.assert_close(
            logits[i, :n], ref_logits[row, :n].detach(), msg = _named("logits"), **tol
        )
        with torch.no_grad():
            alone = reference(input_ids = ids[row : row + 1, :n], use_cache = False).logits
        torch.testing.assert_close(logits[i, :n], alone[0], msg = _named("unpadded"), **tol)
    total = local_ce.detach().clone()
    dist.all_reduce(total)
    torch.testing.assert_close(total / count, _sum_ce(ref_logits.detach(), shift) / count, **tol)
    # Same fp32 ops; chunked attention also accumulates full-sequence gradients in fp32.
    gtol = dict(rtol = 1e-5, atol = 1e-8)

    # Gradients: None stays None, zero stays zero, values match; then one AdamW step.
    ref_params = dict(reference.named_parameters())
    for name, param in model.named_parameters():
        has = torch.tensor([0.0 if param.grad is None else 1.0])
        dist.all_reduce(has)
        grad = torch.zeros_like(param) if param.grad is None else param.grad
        dist.all_reduce(grad)
        ref_grad = ref_params[name].grad
        if ref_grad is None:
            assert has.item() == 0, f"{name}: reference grad is None, CP produced one"
            param.grad = None
            continue
        assert has.item() > 0, f"{name}: CP grad is None, reference has one"
        torch.testing.assert_close(grad, ref_grad, msg = _named(name), **gtol)
        param.grad = grad
    # Adam's first step is lr * g / (|g| + eps): with eps = 1e-8 the fp32 noise of tiny gradients
    # (V4's Sinkhorn hyper-connections run in fp32) is amplified to ~lr. eps = 1e-3 bounds that at
    # ~1e-8 while weight decay on a zero-instead-of-None gradient (lr * wd * |p| ~ 1e-4) still fails.
    lr = dict(lr = 1e-2, weight_decay = 0.1, eps = 1e-3)
    atol = dict(rtol = 1e-6, atol = 1e-7)
    torch.optim.AdamW([p for p in model.parameters() if p.requires_grad], **lr).step()
    torch.optim.AdamW([p for p in reference.parameters() if p.requires_grad], **lr).step()
    for name, param in model.named_parameters():
        torch.testing.assert_close(param, ref_params[name], msg = _named(f"adamw {name}"), **atol)


def dist_all_gather(t, group):
    parts = [torch.empty_like(t) for _ in range(dist.get_world_size(group))]
    dist.all_gather(parts, t.contiguous(), group = group)
    return parts


# V3.2: 23 tokens pads to 24 under CP; row 1 holds 17 valid tokens, so CP ranks see unequal counts.
V32 = dict(kind = "v32", seq = 23, lengths = [23, 17])
# V4 at release ratios: 397 tokens -> 3 full HCA blocks (128) + partial, ~99 CSA blocks, window 128;
# the CP = 2 seam (199) sits inside HCA block 1, the CP = 4 seams (100, 200, 300) inside blocks 0-2.
V4 = dict(kind = "v4", seq = 397, lengths = [397, 290])
V3 = dict(kind = "v3", seq = 23, lengths = [23, 17])
V2 = dict(V3, kind = "v2")

CASES = {
    "v2_cp2": dict(V2, cp = 2, world = 2),
    "v2_cp4": dict(V2, cp = 4, world = 4),
    "v2_dp2cp2": dict(V2, cp = 2, world = 4),
    "v2_cp2_eager_checkpoint": dict(V2, cp = 2, world = 2, attn = "eager", checkpoint = True),
    "v2_cp2_lora": dict(V2, cp = 2, world = 2, lora = True),
    "v2_cp2_lora_experts": dict(V2, cp = 2, world = 2, lora = "experts"),
    "v3_cp2": dict(V3, cp = 2, world = 2),
    "v3_cp4": dict(V3, cp = 4, world = 4),
    "v3_dp2cp2": dict(V3, cp = 2, world = 4),
    "v3_cp2_eager_checkpoint": dict(V3, cp = 2, world = 2, attn = "eager", checkpoint = True),
    "v3_cp2_lora": dict(V3, cp = 2, world = 2, lora = True),
    "v3_cp2_lora_experts": dict(V3, cp = 2, world = 2, lora = "experts"),
    "v32_cp2": dict(V32, cp = 2, world = 2),
    "v32_cp2_chunk1": dict(V32, cp = 2, world = 2, chunk = 1),
    "v32_cp2_chunk5": dict(V32, cp = 2, world = 2, chunk = 5),
    "v32_cp4_late_rank_empty": dict(V32, cp = 4, world = 4, labels = "late_rank_empty"),
    "v32_dp2cp2": dict(V32, cp = 2, world = 4),
    "v32_cp2_lora": dict(V32, cp = 2, world = 2, lora = True),
    "v32_cp2_lora_experts": dict(V32, cp = 2, world = 2, lora = "experts"),
    "v32_cp2_sdpa_checkpoint": dict(V32, cp = 2, world = 2, attn = "sdpa", checkpoint = True),
    "v4_cp2": dict(V4, cp = 2, world = 2),
    "v4_cp2_topk512": dict(V4, cp = 2, world = 2, v4_topk = 512),
    "v4_cp2_chunk1": dict(V4, cp = 2, world = 2, chunk = 1),
    "v4_cp2_chunk37": dict(V4, cp = 2, world = 2, chunk = 37),
    "v4_cp4_late_rank_empty": dict(V4, cp = 4, world = 4, labels = "late_rank_empty", chunk = 37),
    "v4_dp2cp2": dict(V4, cp = 2, world = 4, chunk = 64),
    "v4_cp2_lora": dict(V4, cp = 2, world = 2, lora = True, chunk = 50),
    "v4_cp2_lora_experts": dict(V4, cp = 2, world = 2, lora = "experts", chunk = 50),
    "v4_cp2_checkpoint": dict(V4, cp = 2, world = 2, checkpoint = True, chunk = 64),
}

CONTROLS = {
    "v32_local_topk": dict(V32, cp = 2, world = 2, control = "local_topk"),
    "v32_default_scale": dict(V32, cp = 2, world = 2, control = "default_scale"),
    "v2_default_scale": dict(V2, cp = 2, world = 2, control = "default_scale"),
    "v3_default_scale": dict(V3, cp = 2, world = 2, control = "default_scale"),
    "v4_shard_local": dict(V4, cp = 2, world = 2, control = "shard_local"),
    "v4_dropped_window": dict(V4, cp = 2, world = 2, control = "dropped_window"),
    "v4_wrong_offset": dict(V4, cp = 2, world = 2, control = "wrong_offset"),
}


@pytest.mark.parametrize("name", list(CASES))
def test_deepseek_cp_matches_fp64_reference(name):
    case = CASES[name]
    _spawn(_parity_worker, case["world"], case)


@pytest.mark.parametrize("name", list(CONTROLS))
def test_deepseek_cp_negative_control_fails(name):
    case = CONTROLS[name]
    with pytest.raises(Exception, match = "AssertionError"):
        _spawn(_parity_worker, case["world"], case)


# 2. V3.2 indexer: global causal top-k vs transformers' full-sequence indexer.


def _topk_worker(rank, world, seq):
    model = _model("v32")
    attn = model.model.layers[0].self_attn
    modeling = __import__(type(attn).__module__, fromlist = ["x"])
    g = torch.Generator().manual_seed(7)
    hidden = torch.randn(2, seq, 64, generator = g, dtype = torch.float64)
    pos = torch.arange(seq)[None].expand(2, -1)
    pe = model.model.rotary_emb(hidden, pos)
    q_resid = attn.q_a_layernorm(attn.q_a_proj(hidden)).detach()
    causal = torch.full((2, seq, seq), torch.finfo(torch.float64).min, dtype = torch.float64).triu(1)
    ref = attn.indexer(hidden, q_resid, pe, causal, pos)

    # Reference scores, to prove the compared rows are not tied at the top-k boundary.
    idx = attn.indexer
    q = idx.wq_b(q_resid).view(2, seq, idx.n_heads, idx.head_dim)
    k = idx.k_norm(idx.wk(hidden)).unsqueeze(2)
    r = idx.qk_rope_head_dim
    qr, kr = modeling.apply_rotary_pos_emb(q[..., :r], k[..., :r], *pe, unsqueeze_dim = 2)
    q = torch.cat([qr, q[..., r:]], -1).float()
    k = torch.cat([kr, k[..., r:]], -1).squeeze(2).float()
    scores = F.relu(torch.matmul(q, k.transpose(-1, -2).unsqueeze(1)) * idx.softmax_scale)
    w = idx.weights_proj(hidden).float() * idx.n_heads**-0.5
    s = torch.matmul(w.unsqueeze(-2), scores).squeeze(-2)

    local = seq // world
    sl = slice(rank * local, (rank + 1) * local)
    pe_local = tuple(t[:, sl] for t in pe)
    got = ud._v32_global_topk(
        idx, hidden[:, sl], q_resid[:, sl], pe_local, dist.group.WORLD, modeling
    )
    assert got.shape == ref.shape
    topk = idx.index_topk
    short = 0
    for b in range(2):
        for t in range(seq):
            want = {int(i) for i in ref[b, t] if int(i) <= t}
            have = {int(i) for i in got[b, t] if int(i) <= t}
            if t + 1 <= topk:
                short += 1
                assert want == have == set(range(t + 1)), (b, t)
                continue
            row = s[b, t, : t + 1].sort(descending = True).values
            assert row[topk - 1] - row[topk] > 1e-6, f"tie at ({b}, {t}): pick another seed"
            assert want == have, (b, t, want, have)
    assert short == 2 * topk  # early queries with fewer causal keys than top-k were exercised


@pytest.mark.parametrize("world", [2, 4])
def test_v32_global_topk_matches_full_sequence_indexer(world):
    _spawn(_topk_worker, world, 24)


# 3. Chunked sink attention custom backward vs an independent fp64 reference.


def _reference_sink(q, kv, block_bias, sinks, window, scaling):
    rows, seq = q.shape[2], q.shape[2]
    pos = torch.arange(seq)
    diff = pos[:, None] - pos[None, :]
    keep = (diff >= 0) & (diff < window)
    logits = torch.matmul(q, kv.transpose(-1, -2)) * scaling
    dense = logits[..., :seq].masked_fill(~keep, float("-inf"))
    full = torch.cat([dense, logits[..., seq:] + block_bias], -1)
    full = torch.cat([full, sinks.view(1, -1, 1, 1).expand(q.shape[0], -1, rows, 1)], -1)
    probs = torch.softmax(full, -1)[..., :-1]
    return torch.matmul(probs, kv).transpose(1, 2)


def _sink_case(chunk, monkeypatch):
    monkeypatch.setenv("UNSLOTH_CP_ATTN_CHUNK", chunk)
    g = torch.Generator().manual_seed(0)
    seq, blocks, window, scaling = 40, 5, 9, 0.21
    q = torch.randn(2, 3, seq, 8, generator = g, dtype = torch.float64)
    kv = torch.randn(2, 1, seq + blocks, 8, generator = g, dtype = torch.float64)
    sinks = torch.randn(3, generator = g, dtype = torch.float64)
    # Block w is visible to query t once (t + 1) // 8 > w (HCA rule), plus one indexer-dropped block.
    t = torch.arange(seq)[:, None]
    bias = torch.where(torch.arange(blocks)[None] < (t + 1) // 8, 0.0, float("-inf"))
    bias = bias.to(torch.float64).expand(2, 1, seq, blocks).clone()
    bias[1, :, 30:, 1] = float("-inf")
    grad_out = torch.randn(2, seq, 3, 8, generator = g, dtype = torch.float64)
    a = [x.clone().requires_grad_() for x in (q, kv, sinks)]
    b = [x.clone().requires_grad_() for x in (q, kv, sinks)]
    out = ud._sink_attention(a[0], a[1], bias, a[2], window, scaling, 0.0, True)
    ref = _reference_sink(b[0], b[1], bias, b[2], window, scaling)
    torch.testing.assert_close(out, ref, rtol = 1e-12, atol = 1e-12)
    (out * grad_out).sum().backward()
    (ref * grad_out).sum().backward()
    # q's rows are written once; kv and sinks accumulate over chunks in fp32 by design.
    exact = dict(rtol = 1e-12, atol = 1e-12)
    tol = exact if int(chunk) >= seq else dict(rtol = 1e-6, atol = 1e-7)
    for name, x, y in zip(("q", "kv", "sinks"), a, b):
        torch.testing.assert_close(
            x.grad, y.grad, msg = _named(name), **(exact if name == "q" else tol)
        )


# 1 and 7 run the chunked autograd op (7 leaves a partial last chunk); 40 = the sequence and 4096
# take the plain autograd path.
@pytest.mark.parametrize("chunk", ["1", "7", "40", "4096"])
def test_chunked_sink_attention_backward_matches_reference(chunk, monkeypatch):
    _sink_case(chunk, monkeypatch)


@pytest.mark.parametrize("chunk", ["1", "7"])
def test_chunked_sink_attention_dropping_a_chunk_is_wrong(chunk, monkeypatch):
    real = ud._recompute_rows

    def drop_first(chunk_size, full_seq, run, grad_out, *args):
        grad_out = grad_out.clone()
        grad_out[:, :chunk_size] = 0
        return real(chunk_size, full_seq, run, grad_out, *args)

    monkeypatch.setattr(ud, "_recompute_rows", drop_first)
    with pytest.raises(AssertionError):
        _sink_case(chunk, monkeypatch)


# 4. The own-mask exemption covers only layer types and the window.


@pytest.mark.parametrize("kind", ["v32", "v4"])
def test_own_mask_exemption_is_narrow(kind):
    config = _model(kind).config
    # generic = True is what _check_cp_model passes for them (their attention calls the registry).
    for generic in (False, True):
        config = _model(kind).config
        assert cp._unsupported_attention(config, "ulysses", generic = generic) is None
        assert cp._unsupported_attention(config, "ring", generic = generic) is not None
        config.attn_logit_softcapping = 30.0
        assert (
            cp._unsupported_attention(config, "ulysses", generic = generic)
            == "attention logit softcapping"
        )
        config.attn_logit_softcapping = None
        config.attention_chunk_size = 64
        assert cp._unsupported_attention(config, "ulysses", generic = generic) == "chunked attention"
        config.attention_chunk_size = None
        config.layer_types = ["chunked_attention"] * config.num_hidden_layers
        assert cp._unsupported_attention(config, "ulysses", generic = generic) is not None