Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

12.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.

"""Single-token decode RoPE (llama path) and q/k norm + RoPE (qwen3 path) in one launch vs the
unfused decode op sequence they replace, both against an fp64 reference, plus the fallbacks."""

import pytest
import torch
from real_accelerator import has_real_cuda  # tests/_shared, on sys.path via tests/conftest.py

if not has_real_cuda():
    pytest.skip("needs a CUDA device", allow_module_level = True)

pytest.importorskip("triton")

import unsloth  # noqa: F401
from unsloth.kernels import qk_norm_rope as QK
from unsloth.kernels import rope_embedding as RE

SHAPES = [(1, 16, 8, 128), (4, 16, 8, 128), (8, 32, 8, 64), (3, 14, 2, 64), (5, 8, 8, 256)]


class _Norm(torch.nn.Module):
    def __init__(
        self,
        dim,
        dtype,
        eps = 1e-6,
    ):
        super().__init__()
        w = 1 + 0.1 * torch.randn(dim, device = "cuda", dtype = dtype)
        self.weight = torch.nn.Parameter(w, requires_grad = False)
        self.variance_epsilon = eps


def _table(positions, dim, dtype):
    inv = 1.0 / (1e6 ** (torch.arange(0, dim, 2, device = "cuda").float() / dim))
    f = torch.outer(torch.arange(positions, device = "cuda").float(), inv)
    e = torch.cat((f, f), -1)
    return e.cos().to(dtype), e.sin().to(dtype)


def _unfused_norm(norm, X):
    XX = X.to(torch.float32)
    var = XX.square().mean(-1, keepdim = True)
    var += norm.variance_epsilon
    XX *= var.rsqrt_()
    X = XX.to(X.dtype)
    X *= norm.weight
    return X


def _unfused_rope(Qn, Kn, cos, sin, pos):
    # The decode block of LlamaAttention_fast_forward_inference, in place.
    cos = cos[pos].unsqueeze(1).to(Qn.dtype)
    sin = sin[pos].unsqueeze(1).to(Qn.dtype)
    h = Qn.shape[-1] // 2
    for X in (Qn, Kn):
        RH = torch.empty_like(X)
        RH[..., :h] = X[..., h:]
        RH[..., h:] = X[..., :h]
        RH[..., :h].neg_()
        X *= cos
        X.addcmul_(RH, sin)
    return Qn, Kn


def _oracle_rope(X, cos, sin, pos):
    c, s = cos[pos].unsqueeze(1).double(), sin[pos].unsqueeze(1).double()
    h = X.shape[-1] // 2
    return X * c + torch.cat((-X[..., h:], X[..., :h]), -1) * s


def _oracle_norm(norm, X):
    X = X.double()
    return (
        X
        * torch.rsqrt(X.square().mean(-1, keepdim = True) + norm.variance_epsilon)
        * norm.weight.double()
    )


def _err(a, b):
    return (a.double() - b).abs().mean().item()


@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("shape", SHAPES)
def test_decode_rope_inplace_matches_unfused(dtype, shape):
    torch.manual_seed(0)
    bsz, nq, nkv, d = shape
    cos, sin = _table(4096, d, dtype)
    pos = torch.randint(0, 4000, (bsz, 1), device = "cuda")
    qbuf = 3 * torch.randn(bsz, 1, nq * d, device = "cuda", dtype = dtype)
    kbuf = 3 * torch.randn(bsz, 1, nkv * d, device = "cuda", dtype = dtype)
    view = lambda b, n: b.view(bsz, 1, n, d).transpose(1, 2)  # noqa: E731
    Qo, Ko = (
        _oracle_rope(view(qbuf, nq).double(), cos, sin, pos),
        _oracle_rope(view(kbuf, nkv).double(), cos, sin, pos),
    )
    Qu, Ku = _unfused_rope(view(qbuf, nq).clone(), view(kbuf, nkv).clone(), cos, sin, pos)
    qf, kf = qbuf.clone(), kbuf.clone()
    assert RE.decode_rope_inplace(view(qf, nq), view(kf, nkv), cos, sin, pos)
    Qf, Kf = view(qf, nq), view(kf, nkv)
    ulp = torch.finfo(dtype).eps * max(Qo.abs().max().item(), Ko.abs().max().item())
    assert (Qf - Qu).abs().max().item() <= 2 * ulp and (Kf - Ku).abs().max().item() <= 2 * ulp
    # mean error vs fp64 does not grow (fewer roundings, not more)
    assert _err(Qf, Qo) <= _err(Qu, Qo) * 1.05 and _err(Kf, Ko) <= _err(Ku, Ko) * 1.05


@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("shape", SHAPES)
def test_decode_qk_norm_rope_matches_unfused(dtype, shape):
    torch.manual_seed(0)
    bsz, nq, nkv, d = shape
    cos, sin = _table(4096, d, dtype)
    pos = torch.randint(0, 4000, (bsz, 1), device = "cuda")
    Q = 3 * torch.randn(bsz, 1, nq, d, device = "cuda", dtype = dtype)
    K = 3 * torch.randn(bsz, 1, nkv, d, device = "cuda", dtype = dtype)
    qn, kn = _Norm(d, dtype), _Norm(d, dtype)
    Qo = _oracle_rope(_oracle_norm(qn, Q).transpose(1, 2), cos, sin, pos)
    Ko = _oracle_rope(_oracle_norm(kn, K).transpose(1, 2), cos, sin, pos)
    Qu, Ku = _unfused_rope(
        _unfused_norm(qn, Q.clone()).transpose(1, 2),
        _unfused_norm(kn, K.clone()).transpose(1, 2),
        cos,
        sin,
        pos,
    )
    out = QK.decode_qk_norm_rope(qn, kn, Q, K, cos, sin, pos)
    assert out is not None
    Qf, Kf = out
    assert Qf.shape == Qu.shape and Kf.shape == Ku.shape
    ulp = torch.finfo(dtype).eps * max(Qo.abs().max().item(), Ko.abs().max().item())
    assert (Qf - Qu).abs().max().item() <= 2 * ulp and (Kf - Ku).abs().max().item() <= 2 * ulp
    assert _err(Qf, Qo) <= _err(Qu, Qo) * 1.05 and _err(Kf, Ko) <= _err(Ku, Ko) * 1.05


def test_decode_fallbacks():
    cos, sin = _table(64, 128, torch.bfloat16)
    q = torch.randn(2, 4, 1, 128, device = "cuda", dtype = torch.bfloat16)
    k = torch.randn(2, 2, 1, 128, device = "cuda", dtype = torch.bfloat16)
    p = torch.zeros(2, 1, dtype = torch.long, device = "cuda")
    before = q.clone()
    # a float32 table: the unfused path multiplies in Q's dtype, so it keeps that path
    assert not RE.decode_rope_inplace(q, k, cos.float(), sin.float(), p)
    assert torch.equal(q, before)
    assert not RE.decode_rope_inplace(
        torch.randn(2, 4, 3, 128, device = "cuda", dtype = torch.bfloat16), k, cos, sin, p
    )
    assert not RE.decode_rope_inplace(q.cpu(), k.cpu(), cos.cpu(), sin.cpu(), p.cpu())
    n32 = _Norm(128, torch.float32)
    assert (
        QK.decode_qk_norm_rope(n32, n32, q.transpose(1, 2), k.transpose(1, 2), cos, sin, p) is None
    )


def test_decode_kill_switch(monkeypatch):
    cos, sin = _table(64, 128, torch.bfloat16)
    q = torch.randn(2, 4, 1, 128, device = "cuda", dtype = torch.bfloat16)
    k = torch.randn(2, 2, 1, 128, device = "cuda", dtype = torch.bfloat16)
    p = torch.zeros(2, 1, dtype = torch.long, device = "cuda")
    n = _Norm(128, torch.bfloat16)
    monkeypatch.setattr(RE, "DECODE_FUSED_ROPE", False)
    monkeypatch.setattr(QK, "DECODE_FUSED_ROPE", False)
    assert not RE.decode_rope_inplace(q, k, cos, sin, p)
    assert QK.decode_qk_norm_rope(n, n, q.transpose(1, 2), k.transpose(1, 2), cos, sin, p) is None


def test_decode_strided_int32_positions():
    # int32 last-column slices keep their row stride; the kernels index positions densely.
    torch.manual_seed(0)
    dtype, d = torch.bfloat16, 128
    cos, sin = _table(64, d, dtype)
    pos = torch.tensor([[0, 1, 2], [2, 3, 4], [9, 10, 11]], device = "cuda", dtype = torch.int32)[
        :, -1:
    ]
    assert pos.stride(0) == 3
    Q = 3 * torch.randn(3, 4, 1, d, device = "cuda", dtype = dtype)
    K = 3 * torch.randn(3, 2, 1, d, device = "cuda", dtype = dtype)
    Qu, Ku = _unfused_rope(Q.clone(), K.clone(), cos, sin, pos.long())
    Qf, Kf = Q.clone(), K.clone()
    assert RE.decode_rope_inplace(Qf, Kf, cos, sin, pos)
    assert torch.allclose(Qf.float(), Qu.float(), atol = 0.05) and torch.allclose(
        Kf.float(), Ku.float(), atol = 0.05
    )
    n = _Norm(d, dtype)
    Qs, Ks = Q.transpose(1, 2).contiguous(), K.transpose(1, 2).contiguous()
    Qu, Ku = _unfused_rope(
        _unfused_norm(n, Qs.clone()).transpose(1, 2),
        _unfused_norm(n, Ks.clone()).transpose(1, 2),
        cos,
        sin,
        pos.long(),
    )
    out = QK.decode_qk_norm_rope(n, n, Qs, Ks, cos, sin, pos)
    assert out is not None
    assert torch.allclose(out[0].float(), Qu.float(), atol = 0.05) and torch.allclose(
        out[1].float(), Ku.float(), atol = 0.05
    )


def test_decode_split_sync_and_mixed_devices(monkeypatch):
    # Split models: the decode helpers never drain the stream on the host.
    cos, sin = _table(64, 128, torch.bfloat16)
    q = torch.randn(2, 4, 1, 128, device = "cuda", dtype = torch.bfloat16)
    k = torch.randn(2, 2, 1, 128, device = "cuda", dtype = torch.bfloat16)
    p = torch.zeros(2, 1, dtype = torch.long, device = "cuda")
    n = _Norm(128, torch.bfloat16)
    calls = []
    monkeypatch.setattr(RE, "_sync_if_split", lambda X: calls.append(X.device))
    monkeypatch.setattr(QK, "_sync_if_split", lambda X: calls.append(X.device))
    assert RE.decode_rope_inplace(q, k, cos, sin, p)
    assert (
        QK.decode_qk_norm_rope(n, n, q.transpose(1, 2), k.transpose(1, 2), cos, sin, p) is not None
    )
    assert len(calls) == 0
    # any operand off Q's device keeps the unfused path, with nothing written
    before_q, before_k = q.clone(), k.clone()
    assert not RE.decode_rope_inplace(q, k, cos, sin.cpu(), p)
    assert torch.equal(q, before_q) and torch.equal(k, before_k)
    nc = _Norm(128, torch.bfloat16)
    nc.weight.data = nc.weight.data.cpu()
    qs, ks = q.transpose(1, 2), k.transpose(1, 2)
    assert QK.decode_qk_norm_rope(n, nc, qs, ks, cos, sin, p) is None
    assert QK.decode_qk_norm_rope(n, n, qs, ks.cpu(), cos, sin, p) is None
    assert QK.decode_qk_norm_rope(n, n, qs, ks, cos, sin.cpu(), p) is None
    assert len(calls) == 0


@pytest.mark.skipif(
    not has_real_cuda() or torch.cuda.device_count() < 2, reason = "needs two real GPUs"
)
def test_decode_rope_on_another_device_orders_with_the_copy():
    # Rotate on GPU 1 with GPU 0 current, then copy to GPU 0: the copy must see rotated values.
    cos, sin = _table(4096, 128, torch.bfloat16)
    cos, sin = cos.to("cuda:1"), sin.to("cuda:1")
    p = torch.randint(0, 4000, (4, 1), device = "cuda:1")
    for _ in range(3):
        q = 3 * torch.randn(4, 32, 1, 128, device = "cuda:1", dtype = torch.bfloat16)
        k = 3 * torch.randn(4, 8, 1, 128, device = "cuda:1", dtype = torch.bfloat16)
        # Same kernel, same inputs, rotated with its own device current: bitwise what the copy must
        # see (a racing copy sees unrotated values). Fused rounds once and the unfused ops several
        # times, so versus unfused only the 2 ulp bound of test_decode_rope_inplace_matches_unfused
        # holds (0.0625 differences at |x| >= 8 fail a fixed atol = 0.05).
        qr, kr = q.clone(), k.clone()
        with torch.cuda.device(1):
            assert RE.decode_rope_inplace(qr, kr, cos, sin, p)
        qu, ku = _unfused_rope(q.clone(), k.clone(), cos, sin, p)
        with torch.cuda.device(0):
            assert RE.decode_rope_inplace(q, k, cos, sin, p)
            q0, k0 = q.to("cuda:0"), k.to("cuda:0")
        assert torch.equal(q0.cpu(), qr.cpu()) and torch.equal(k0.cpu(), kr.cpu())
        ulp = torch.finfo(q.dtype).eps * max(qu.abs().max().item(), ku.abs().max().item())
        assert (q0.cpu().float() - qu.cpu().float()).abs().max().item() <= 2 * ulp
        assert (k0.cpu().float() - ku.cpu().float()).abs().max().item() <= 2 * ulp


@pytest.mark.gpu
def test_decode_rope_past_int32_cache_offsets_matches_the_reference():
    head_dim = 128
    pos = 2**31 // head_dim + 3
    need = 2 * (pos + 1) * head_dim * 2 + (1 << 30)
    if torch.cuda.mem_get_info()[0] < need:
        pytest.skip("needs about 9 GiB of free GPU memory for two 4 GiB tables")
    cos = torch.empty(pos + 1, head_dim, device = "cuda", dtype = torch.float16)
    sin = torch.empty_like(cos)
    half = head_dim // 2
    cos[pos, :half].uniform_(-1, 1)
    sin[pos, :half].uniform_(-1, 1)
    cos[pos, half:] = cos[pos, :half]
    sin[pos, half:] = sin[pos, :half]
    torch.manual_seed(0)
    Q = torch.randn(2, 8, 1, head_dim, device = "cuda", dtype = torch.float16)
    K = torch.randn(2, 2, 1, head_dim, device = "cuda", dtype = torch.float16)

    def reference(x):
        x = x.double()
        rot = torch.cat([-x[..., half:], x[..., :half]], -1)
        return x * cos[pos].double() + rot * sin[pos].double()

    expected_q, expected_k = reference(Q), reference(K)
    position_ids = torch.full((2, 1), pos, device = "cuda")
    assert RE.decode_rope_inplace(Q, K, cos, sin, position_ids)
    torch.cuda.synchronize()
    torch.testing.assert_close(Q.double(), expected_q, rtol = 2e-3, atol = 2e-3)
    torch.testing.assert_close(K.double(), expected_k, rtol = 2e-3, atol = 2e-3)