Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

9.4 KB
Raw Blame
"""TileLang C target: original limitations, fp32 CPU execution, and CUDA parity.

Run: python -m pytest tests/test_fast_cpu.py -q -s
CPU checks need torch, tilelang and a C++ compiler; CUDA parity skips without CUDA.
"""
import os, sys
import pytest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
torch = pytest.importorskip("torch")
tilelang = pytest.importorskip("tilelang")
from laya import tl_kernels as K  # noqa: E402


@pytest.fixture(params=[torch.float32, torch.bfloat16, torch.float16], ids=["cpu", "bf16", "fp16"])
def dtype(request):
    if request.param != torch.float32 and not torch.cuda.is_available():
        pytest.skip("GPU parity needs CUDA")
    torch.manual_seed(1234)
    return request.param


def rand(*shape, dtype=torch.float32, scale=1):
    return (torch.randn(*shape) * scale).to(dtype).float()


def check(name, actual, expected, tol):
    delta = (actual.float() - expected.float()).abs().max().item()
    print("%s max_abs=%.9g tolerance=%g" % (name, delta, tol))
    assert torch.isfinite(actual).all()
    assert delta <= tol


def gpu(dtype, factory, args, tensors, floating, **kwargs):
    dt = {torch.bfloat16: "bfloat16", torch.float16: "float16"}[dtype]
    values = [x.cuda().to(dtype) if i in floating else x.cuda() for i, x in enumerate(tensors)]
    factory(*args, dtype=dt, **kwargs)(*values)
    return [x.cpu() for x in values]


@pytest.mark.parametrize("bias,act", [(False, "none"), (True, "gelu"), (True, "relu")])
def test_gemm(dtype, bias, act):
    M, N, D = 17, 70, 67
    a, w, b = rand(M, D, dtype=dtype), rand(N, D, dtype=dtype, scale=0.02), rand(N)
    out = torch.empty(M, N)
    opts = dict(bias=bias, act=act)
    K.compile_cpu(K.gemm_kernel, N, D, **opts)(a, w, b, out)
    ref = a @ w.T + (b if bias else 0)
    ref = {"none": ref, "gelu": torch.nn.functional.gelu(ref), "relu": torch.relu(ref)}[act]
    check("gemm/cpu-reference", out, ref, 2e-5)
    if dtype != torch.float32:
        other = gpu(dtype, K.gemm_kernel, (N,D), [a,w,b,torch.empty_like(out)], (0,1,3), **opts)[-1]
        check("gemm/cpu-gpu", out, other, 0.05)


def test_geglu(dtype):
    M, F, D = 17, 70, 67
    a, w = rand(M,D,dtype=dtype), rand(2*F,D,dtype=dtype,scale=0.02)
    out = torch.empty(M,F)
    K.compile_cpu(K.gemm_geglu_kernel,F,D)(a,w,out)
    x = a @ w.T
    check("geglu/cpu-reference", out, torch.nn.functional.gelu(x[:,:F]) * x[:,F:], 2e-5)
    if dtype != torch.float32:
        other = gpu(dtype,K.gemm_geglu_kernel,(F,D),[a,w,torch.empty_like(out)],(0,1,2))[-1]
        check("geglu/cpu-gpu", out, other, 0.05)


@pytest.mark.parametrize("residual,bias", [(False,False),(False,True),(True,False),(True,True)])
def test_layernorm(dtype, residual, bias):
    M, D = 17, 128
    x, r = rand(M,D,scale=3000), rand(M,D,dtype=dtype,scale=50)
    w, b = torch.rand(D)+0.5, rand(D)
    updated, out = x.clone(), torch.empty_like(x)
    opts = dict(residual=residual,bias=bias)
    K.compile_cpu(K.add_ln_kernel,D,**opts)(updated,r,w,b,out)
    refx = x + r if residual else x
    assert torch.equal(updated,refx)
    check("ln/cpu-reference",out,torch.nn.functional.layer_norm(refx,(D,),w,b if bias else None,1e-5),2e-5)
    if dtype != torch.float32:
        other = gpu(dtype,K.add_ln_kernel,(D,),[x.clone(),r,w,b,torch.empty_like(out)],(1,4),**opts)
        assert torch.equal(updated,other[0])
        check("ln/cpu-gpu",out,other[-1],0.05)


def test_rope(dtype):
    M, L, H, D = 35, 7, 2, 64
    qkv = rand(M,3*H*D,dtype=dtype)
    angles = rand(L,D//2)
    cos, sin = angles.cos(), angles.sin()
    out = qkv.clone()
    K.compile_cpu(K.rope_kernel,H,D)(out,cos,sin)
    ref = qkv.reshape(M,3,H,D).clone()
    x0, x1 = ref[:,:2,:,:D//2].clone(),ref[:,:2,:,D//2:].clone()
    cs,sn = [v[torch.arange(M)%L,None,None,:] for v in (cos,sin)]
    ref[:,:2,:,:D//2],ref[:,:2,:,D//2:] = x0*cs-x1*sn,x1*cs+x0*sn
    check("rope/cpu-reference",out,ref.reshape_as(out),2e-6)
    assert torch.equal(out[:,2*H*D:],qkv[:,2*H*D:])
    if dtype != torch.float32:
        other = gpu(dtype,K.rope_kernel,(H,D),[qkv.clone(),cos,sin],(0,))[0]
        check("rope/cpu-gpu",out,other,0.05)


@pytest.mark.parametrize("window,dynamic", [(0,False),(0,True),(17,False),(17,True)])
def test_attention(dtype, window, dynamic):
    B,L,H,D = 3, 80, 2, 32
    qkv = rand(B,L,3,H,D,dtype=dtype)
    lens = torch.tensor([L,39,0],dtype=torch.int32)
    out = torch.empty(B,L,H*D)
    args = (None if dynamic else B,None if dynamic else L,H,D)
    K.compile_cpu(K.attn_kernel,*args,window=window)(qkv,lens,out)
    q,k,v = [qkv[:,:,i].transpose(1,2) for i in range(3)]
    idx = torch.arange(L)
    mask = (idx[None,:]<lens[:,None])[:,None,None,:].expand(B,1,L,L)
    if window:
        mask = mask & ((idx[:,None]-idx[None,:]).abs()<=window)[None,None]
    ref = torch.nn.functional.scaled_dot_product_attention(q,k,v,attn_mask=mask).transpose(1,2).reshape_as(out)
    valid = idx[None,:]<lens[:,None]
    assert torch.isfinite(out).all()
    assert torch.count_nonzero(out[2]) == 0
    check("attn/cpu-reference",out[valid],ref[valid],2e-5)
    if dtype != torch.float32:
        other = gpu(dtype,K.attn_kernel,args,[qkv,lens,torch.empty_like(out)],(0,2),window=window)[-1]
        assert torch.isfinite(other).all()
        check("attn/cpu-gpu",out[valid],other[valid],0.02)


# These are the first failures of the original GPU specialization, not a claim
# that the math is impossible on CPU. Fail on changed diagnostics after upgrades.
PROBES = [
    (K.gemm_kernel,(128,64),"The layout for fragment C_l can not be inferred correctly."),
    (K.gemm_geglu_kernel,(64,64),"CPU fill only supports local and global buffers, but got dst scope `local.fragment`."),
    (K.add_ln_kernel,(128,),"CPU reduce only supports local src and local/local.var dst buffers, got src scope `local.fragment` and dst scope `local.fragment`."),
    (K.rope_kernel,(2,64),"Cannot convert type bfloat16 to C type"),
    (K.attn_kernel,(1,64,2,64),"The layout for fragment s_c can not be inferred correctly."),
]


@pytest.mark.parametrize("factory,args,message",PROBES,ids=[p[0].func.__name__ for p in PROBES])
@pytest.mark.parametrize("dtype",K.DTYPES)
def test_original_cpu_failure(factory,args,message,dtype,capfd):
    if tilelang.__version__ != "0.1.14":
        pytest.skip("diagnostics pinned to TileLang 0.1.14; re-probe on upgrade")
    if factory is K.rope_kernel and dtype == "float16":
        with pytest.raises(RuntimeError,match="Compilation Failed!"):
            tilelang.compile(factory.get_tir(*args,dtype=dtype),target="c",pass_configs=K.FAST)
        captured = capfd.readouterr()
        # Pin the type mismatch, not the full compiler sentence: the wording differs
        # between runs of the same compiler ("no matching function for call to
        # 'vec_type<float, 4>::vec_type(half4&)'" vs "no known conversion for argument 1
        # from 'vec_type<float, 4>' to 'vec_type<half_float::half, 4>&&'").
        assert "vec_type<float, 4>" in captured.err
        assert ("no matching function for call to" in captured.err
                or "no known conversion for argument 1" in captured.err)
        print("rope_kernel float16: vec_type<float, 4> / half4 conversion is unavailable")
    else:
        with pytest.raises(tilelang.tvm.error.InternalError) as exc:
            tilelang.compile(factory.get_tir(*args,dtype=dtype),target="c",pass_configs=K.FAST)
        assert message in str(exc.value)
        print(factory.func.__name__ + " " + dtype + ": " + message)


@pytest.mark.parametrize("factory,args,message",PROBES,ids=[p[0].func.__name__ for p in PROBES])
def test_bf16_after_removing_fragments(factory,args,message):
    if tilelang.__version__ != "0.1.14":
        pytest.skip("diagnostics pinned to TileLang 0.1.14; re-probe on upgrade")
    with pytest.raises(tilelang.tvm.error.InternalError) as exc:
        tilelang.compile(factory.get_tir(*args,cpu=True),target="c",
                         pass_configs={tilelang.PassConfigKey.TIR_DISABLE_VECTORIZE: True})
    assert "Cannot convert type bfloat16 to C type" in str(exc.value)


def test_gpu_still_rejects_fp32():
    with pytest.raises(ValueError):
        K.gemm_kernel.get_tir(128,64,dtype="float32")


@pytest.mark.parametrize("operation",["fill","sum","max"])
def test_fragment_operations_without_bf16(operation):
    # Isolate GEGLU's fill and LN/attention's reductions from GEMM layout and dtype.
    if tilelang.__version__ != "0.1.14":
        pytest.skip("diagnostics pinned to TileLang 0.1.14; re-probe on upgrade")
    import tilelang.language as T

    @T.prim_func
    def probe(X: T.Tensor((4,128),"float32"),Y: T.Tensor((4,),"float32")):
        with T.Kernel(1,threads=32):
            x = T.alloc_fragment((4,128),"float32")
            y = T.alloc_fragment((4,),"float32")
            for i,j in T.Parallel(4,128):
                x[i,j] = X[i,j]
            if operation == "fill":
                T.fill(y,0)
            elif operation == "sum":
                T.reduce_sum(x,y,dim=1)
            else:
                T.reduce_max(x,y,dim=1)
            for i in T.Parallel(4):
                Y[i] = y[i]

    message = ("CPU fill only supports local and global buffers, but got dst scope `local.fragment`."
               if operation == "fill" else
               "CPU reduce only supports local src and local/local.var dst buffers, got src scope `local.fragment` and dst scope `local.fragment`.")
    with pytest.raises(tilelang.tvm.error.InternalError) as exc:
        tilelang.compile(probe,target="c")
    assert message in str(exc.value)
    print(operation + ": " + message)