Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

166 KB
Raw Blame
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import weakref
from collections import OrderedDict
from dataclasses import dataclass
from typing import Optional
import torch

# Intel GPU compiler (IGC) environment variable: ensure implicit local IDs are
# not removed by the compiler even when unused. This guarantees consistent
# behavior across driver upgrades and prevents performance regressions.
# Reference: https://github.com/intel/intel-graphics-compiler/issues/412
os.environ.setdefault("IGC_RemoveUnusedIdImplicitLocalIDs", "0")

# Tensor layout constants passed to native C++ backends.
# 0 = HND ([B, H, S, D]), 1 = NHD ([B, S, H, D]).
LAYOUT_HND = 0
LAYOUT_NHD = 1


class ARK_DT:
    float64 = 64
    float32 = 32
    float16 = 16
    bfloat16 = 65552
    int2 = 258
    int3 = 259
    int4 = 260
    int5 = 261
    int6 = 262
    int7 = 263
    int8 = 264
    int32 = 288
    float8_e4m3 = 8
    float8_e5m2 = 65544
    float8_e8m0 = 196616
    undef = 0


def cvt_dtype(dtype):
    if dtype == torch.float32:
        return ARK_DT.float32
    if dtype == torch.float16:
        return ARK_DT.float16
    if dtype == torch.bfloat16:
        return ARK_DT.bfloat16
    if dtype == torch.float8_e4m3fn:
        return ARK_DT.float8_e4m3
    if dtype == torch.float8_e5m2:
        return ARK_DT.float8_e5m2
    if dtype == torch.int8:
        return ARK_DT.int8
    if dtype == torch.int32:
        return ARK_DT.int32
    return ARK_DT.undef


def cvtstr_dtype(dtype):
    if dtype == "fp32":
        return ARK_DT.float32
    if dtype == "fp16":
        return ARK_DT.float16
    if dtype == "bf16":
        return ARK_DT.bfloat16
    if dtype == "fp8_e4m3":
        return ARK_DT.float8_e4m3
    if dtype == "fp8_e5m2":
        return ARK_DT.float8_e5m2
    if dtype == "fp8_e8m0":
        return ARK_DT.float8_e8m0
    if dtype == "int8":
        return ARK_DT.int8
    if dtype == "int4":
        return ARK_DT.int4
    if dtype == "int2":
        return ARK_DT.int2
    if dtype == "int3":
        return ARK_DT.int3
    if dtype == "int5":
        return ARK_DT.int5
    if dtype == "int6":
        return ARK_DT.int6
    if dtype == "int7":
        return ARK_DT.int7
    if dtype == "int32":
        return ARK_DT.int32
    return ARK_DT.undef


def get_stream(A: torch.Tensor) -> int:
    if A.device.type == "cpu":
        return 0
    if A.device.type == "xpu":
        return torch.xpu.current_stream().sycl_queue


def _normalize_tensor_layout(tensor_layout: str) -> str:
    layout = tensor_layout.upper()
    if layout not in ("HND", "NHD"):
        raise ValueError(f"tensor_layout must be either 'HND' or 'NHD', got {tensor_layout!r}")
    return layout


def _attention_shape(tensor: torch.Tensor, tensor_layout: str) -> tuple[int, int, int, int]:
    layout = _normalize_tensor_layout(tensor_layout)
    if layout == "HND":
        batch, num_heads, seq_len, head_dim = tensor.shape
    else:
        batch, seq_len, num_heads, head_dim = tensor.shape
    return batch, num_heads, seq_len, head_dim


def _attention_strides_qko(tensor: torch.Tensor, tensor_layout: str) -> tuple[int, int, int, int]:
    layout = _normalize_tensor_layout(tensor_layout)
    if layout == "HND":
        batch_stride, head_stride, seq_stride, dim_stride = tensor.stride()
    else:
        batch_stride, seq_stride, head_stride, dim_stride = tensor.stride()
    return seq_stride, dim_stride, head_stride, batch_stride


def _attention_strides_v(tensor: torch.Tensor, tensor_layout: str) -> tuple[int, int, int, int]:
    layout = _normalize_tensor_layout(tensor_layout)
    if layout == "HND":
        batch_stride, head_stride, seq_stride, dim_stride = tensor.stride()
    else:
        batch_stride, seq_stride, head_stride, dim_stride = tensor.stride()
    return dim_stride, seq_stride, head_stride, batch_stride


def _validate_canonical_strides(tensor: torch.Tensor, name: str, tensor_layout: str) -> None:
    """Verify that *batched* 4-D tensor strides match the canonical packed layout.

    C++ backends compute strides from layout + shape alone (no stride params),
    so the input tensors must have the exact canonical strides.

    ``tensor.stride()`` returns strides in dimension order:

        HND [B, H, S, D]:  stride=(H*S*D, S*D, D, 1)
        NHD [B, S, H, D]:  stride=(S*H*D, H*D, D, 1)
    """
    layout = _normalize_tensor_layout(tensor_layout)
    if layout == "HND":
        B, H, S, D = tensor.shape
        expected = (H * S * D, S * D, D, 1)
    else:
        B, S, H, D = tensor.shape
        expected = (S * H * D, H * D, D, 1)
    actual = tensor.stride()
    if actual != expected:
        raise ValueError(
            f"{name} strides {actual} do not match canonical {layout} layout {expected}. "
            f"Non-contiguous inputs are not supported; call .contiguous() first."
        )


def _validate_attention_tensor(
    tensor: torch.Tensor,
    name: str,
    tensor_layout: str,
    *,
    expected_dtype: torch.dtype | None = None,
) -> tuple[int, int, int, int]:
    if tensor.ndim != 4:
        raise ValueError(f"{name} must be a 4D tensor")
    if expected_dtype is not None and tensor.dtype != expected_dtype:
        raise ValueError(f"{name} must have dtype {expected_dtype}, got {tensor.dtype}")

    qko_strides = _attention_strides_qko(tensor, tensor_layout)
    if qko_strides[1] != 1:
        raise ValueError(f"{name} must be contiguous along the head-dim axis; got stride {qko_strides[1]}")
    if any(stride <= 0 for stride in qko_strides):
        raise ValueError(f"{name} must have positive non-zero strides, got {tensor.stride()}")

    return _attention_shape(tensor, tensor_layout)


def _empty_attention_output(
    batch: int,
    num_heads: int,
    seq_len: int,
    head_dim: int,
    *,
    dtype: torch.dtype,
    device: torch.device,
    tensor_layout: str,
) -> torch.Tensor:
    layout = _normalize_tensor_layout(tensor_layout)
    shape = (batch, num_heads, seq_len, head_dim) if layout == "HND" else (batch, seq_len, num_heads, head_dim)
    return torch.empty(shape, device=device, dtype=dtype)


@dataclass
class _CpuPackedKVCacheEntry:
    descriptor: object
    cache_k: torch.Tensor
    cache_v: torch.Tensor
    key_ref: weakref.ReferenceType[torch.Tensor]
    value_ref: weakref.ReferenceType[torch.Tensor]
    seq_len: int
    key_version: int
    value_version: int


_CPU_PUBLIC_PACKED_KV_CACHE_MAX = 8
_CPU_PUBLIC_PACKED_KV_CACHE: "OrderedDict[tuple, _CpuPackedKVCacheEntry]" = OrderedDict()


def _cpu_public_packed_kv_available() -> bool:
    return (
        cpu_lib is not None
        and hasattr(cpu_lib, "ark_cpu_bestla_sdpa_packed_desc")
        and hasattr(cpu_lib, "ark_cpu_update_packed_k_desc")
        and hasattr(cpu_lib, "ark_cpu_update_packed_v_desc")
    )


def _cpu_public_packed_kv_cache_key(key: torch.Tensor, value: torch.Tensor, tensor_layout: str) -> tuple:
    batch, num_heads_kv, _, head_dim = _attention_shape(key, tensor_layout)
    return (
        key.device.type,
        key.device.index,
        key.dtype,
        value.dtype,
        key.data_ptr(),
        value.data_ptr(),
        key.stride(),
        value.stride(),
        batch,
        num_heads_kv,
        head_dim,
        _normalize_tensor_layout(tensor_layout),
    )


def _attention_seq_slice(tensor: torch.Tensor, tensor_layout: str, start: int, end: int) -> torch.Tensor:
    layout = _normalize_tensor_layout(tensor_layout)
    if layout == "HND":
        return tensor[:, :, start:end, :]
    return tensor[:, start:end, :, :]


def _cpu_public_get_packed_kv_entry(
    key: torch.Tensor,
    value: torch.Tensor,
    *,
    tensor_layout: str,
) -> tuple[_CpuPackedKVCacheEntry, int]:
    layout = _normalize_tensor_layout(tensor_layout)
    batch, num_heads_kv, seq_len_kv, head_dim = _attention_shape(key, layout)
    cache_key = _cpu_public_packed_kv_cache_key(key, value, layout)
    try:
        key_version = int(key._version)
        value_version = int(value._version)
        use_version = True
    except RuntimeError:
        # Inference tensors do not track version counter; always repack.
        key_version = -1
        value_version = -1
        use_version = False
    entry = _CPU_PUBLIC_PACKED_KV_CACHE.get(cache_key)

    if entry is not None and (entry.key_ref() is not key or entry.value_ref() is not value):
        entry = None

    if entry is None or seq_len_kv > int(entry.descriptor.logical_capacity):
        descriptor = ark_cpu_packed_kv_descriptor(batch, num_heads_kv, seq_len_kv, head_dim, dtype=key.dtype)
        cache_k, cache_v = ark_cpu_packed_kv_alloc_from_descriptor(descriptor, dtype=key.dtype, device=key.device)
        entry = _CpuPackedKVCacheEntry(
            descriptor,
            cache_k,
            cache_v,
            weakref.ref(key),
            weakref.ref(value),
            0,
            -1,
            -1,
        )
        _CPU_PUBLIC_PACKED_KV_CACHE[cache_key] = entry
        weakref.finalize(key, lambda ck=cache_key: _CPU_PUBLIC_PACKED_KV_CACHE.pop(ck, None))
    else:
        _CPU_PUBLIC_PACKED_KV_CACHE.move_to_end(cache_key)

    if len(_CPU_PUBLIC_PACKED_KV_CACHE) > _CPU_PUBLIC_PACKED_KV_CACHE_MAX:
        _CPU_PUBLIC_PACKED_KV_CACHE.popitem(last=False)

    if use_version and entry.key_version == key_version and entry.value_version == value_version:
        if seq_len_kv > entry.seq_len:
            ark_cpu_update_packed_kv_from_descriptor(
                entry.descriptor,
                entry.cache_k,
                entry.cache_v,
                _attention_seq_slice(key, layout, entry.seq_len, seq_len_kv),
                _attention_seq_slice(value, layout, entry.seq_len, seq_len_kv),
                entry.seq_len,
                tensor_layout=layout,
                no_zeroing=False,
            )
            entry.seq_len = seq_len_kv
        return entry, seq_len_kv

    ark_cpu_update_packed_kv_from_descriptor(
        entry.descriptor,
        entry.cache_k,
        entry.cache_v,
        key,
        value,
        0,
        tensor_layout=layout,
        no_zeroing=False,
    )
    entry.seq_len = seq_len_kv
    entry.key_version = key_version
    entry.value_version = value_version
    return entry, seq_len_kv


def _cpu_public_mixed_sdpa_packed(
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    *,
    is_causal: bool,
    scale: float | None,
    tensor_layout: str,
) -> torch.Tensor:
    # Defect-2 mitigation: for fp16 K/V on prefill (sl_q > 1, where the GEMM
    # dominates), convert to bf16 so the packed-KV cache stores NTILE48 ROWPACK2
    # data.  The AMX-BF16 stable wrapper then runs with MTILE=16 / NTILE=48 vs
    # the AVX2 path's MTILE=4 / NTILE=24.  PyTorch's dtype conversion is SIMD.
    # For decode (sl_q == 1) the conversion overhead dominates; keep fp16 packed.
    # Only active when the BF16 route is compiled in (requires AVX512F).
    if key.dtype == torch.float16 and _attention_shape(query, tensor_layout)[2] > 1:
        if getattr(cpu_lib, "ARK_CPU_SDPA_BUILD_HAS_BF16_ROUTE", False):
            key = key.to(dtype=torch.bfloat16)
            value = value.to(dtype=torch.bfloat16)

    entry, seq_len_kv = _cpu_public_get_packed_kv_entry(key, value, tensor_layout=tensor_layout)
    return ark_cpu_bestla_sdpa_packed_from_descriptor(
        entry.descriptor,
        query,
        entry.cache_k,
        entry.cache_v,
        seq_len_kv,
        is_causal=is_causal,
        scale=scale,
        tensor_layout=tensor_layout,
    )


def _validate_attention_mask(
    attn_mask: torch.Tensor | None,
    *,
    batch: int,
    seq_len_q: int,
    seq_len_kv: int,
    device: torch.device,
) -> None:
    if attn_mask is None:
        return
    if attn_mask.device != device:
        raise ValueError("attn_mask must be on the same device as Q")
    if not attn_mask.is_contiguous():
        raise ValueError("attn_mask must be contiguous")
    if attn_mask.dtype != torch.float32:
        raise ValueError(f"attn_mask must be float32 (additive bias), got {attn_mask.dtype}")
    expected_mask_shape = (batch, 1, seq_len_q, seq_len_kv)
    if attn_mask.shape != expected_mask_shape:
        raise ValueError(f"attn_mask shape must be {expected_mask_shape}, got {tuple(attn_mask.shape)}")


def _validate_no_dropout(dropout_p: float, api_name: str) -> None:
    if dropout_p != 0.0:
        raise NotImplementedError(f"{api_name}: dropout_p must be 0.0 (got {dropout_p}); dropout is not supported")


def _validate_head_ratio(num_heads_q: int, num_heads_kv: int) -> None:
    if num_heads_kv <= 0:
        raise ValueError("num_heads_kv must be greater than 0")
    if num_heads_q % num_heads_kv != 0:
        raise ValueError(
            f"num_heads_q ({num_heads_q}) must be divisible by num_heads_kv ({num_heads_kv}) for MQA/GQA attention"
        )


def _validate_attention_geometry(
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    tensor_layout: str,
    *,
    key_dtype: torch.dtype | None = None,
    value_dtype: torch.dtype | None = None,
) -> tuple[int, int, int, int, int, int]:
    if key.device != query.device or value.device != query.device:
        raise ValueError(f"Q/K/V must be on the same device, got Q={query.device}, K={key.device}, V={value.device}")
    B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)
    Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout, expected_dtype=key_dtype)
    Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout, expected_dtype=value_dtype)

    if Bk != B or Bv != B:
        raise ValueError("Batch size mismatch between Q/K/V")
    if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
        raise ValueError("K/V shape mismatch")
    if Dk != D:
        raise ValueError("Head dim mismatch between Q and K/V")
    _validate_head_ratio(Hq, Hkv)
    return B, Hq, Hkv, Sq, Skv, D


def _contiguous_hnd_qko_strides(num_heads: int, seq_len: int, head_dim: int) -> tuple[int, int, int, int]:
    return head_dim, 1, seq_len * head_dim, num_heads * seq_len * head_dim


def _contiguous_hnd_v_strides(num_heads: int, seq_len: int, head_dim: int) -> tuple[int, int, int, int]:
    return 1, head_dim, seq_len * head_dim, num_heads * seq_len * head_dim


def _torch_dtype_from_ark_dtype(dtype: int) -> torch.dtype:
    if dtype == ARK_DT.float16:
        return torch.float16
    if dtype == ARK_DT.bfloat16:
        return torch.bfloat16
    if dtype == ARK_DT.float32:
        return torch.float32
    raise ValueError(f"Unsupported ARK dtype code: {dtype}")


cpu_lib = None
xpu_lib = None


def get_lib(A: torch.Tensor):
    lib = None
    if A.device.type == "xpu":
        lib = xpu_lib
    if A.device.type == "cpu":
        lib = cpu_lib
    if lib is None:
        raise NotImplementedError(f"Current device {A.device} is not supported")
    return lib


# A: mxk,  B: nxk, bias: n or [1, n]
def matmul_sycl_tla(A: torch.Tensor, B: torch.Tensor, bias: torch.Tensor | None = None):
    if A.device.type != "xpu" or B.device.type != "xpu":
        raise NotImplementedError("matmul_sycl_tla is only supported on XPU")
    if A.ndim != 2 or B.ndim != 2:
        raise ValueError("A and B must be 2D tensors")
    if A.device != B.device:
        raise ValueError("A and B must be on the same device")
    if A.dtype not in (torch.float32, torch.float16, torch.bfloat16):
        raise ValueError("matmul_sycl_tla only supports torch.float32, torch.float16 and torch.bfloat16")
    if B.dtype != A.dtype:
        raise ValueError("A and B must have the same dtype")

    m, k = A.shape
    n, kb = B.shape
    if k != kb:
        raise ValueError(f"Shape mismatch: A.shape={tuple(A.shape)}, B.shape={tuple(B.shape)}")

    lib = get_lib(A)
    if lib is None or not hasattr(lib, "matmul_sycl_tla"):
        raise NotImplementedError("Current XPU build does not expose matmul_sycl_tla")

    A_arg = A.contiguous()
    B_arg = B.contiguous()
    C = torch.empty(m, n, dtype=A.dtype, device=A.device)

    bias_ptr = 0
    bias_arg = None
    if bias is not None and bias.numel() > 0:
        bias_arg = bias.to(dtype=C.dtype, device=A.device).contiguous().view(-1)
        if bias_arg.numel() != n:
            raise ValueError(f"bias must have {n} elements, got {bias_arg.numel()}")
        bias_ptr = bias_arg.data_ptr()

    stream = get_stream(A_arg)
    lib.matmul_sycl_tla(
        stream,
        m,
        n,
        k,
        A_arg.data_ptr(),
        cvt_dtype(A_arg.dtype),
        B_arg.data_ptr(),
        cvt_dtype(B_arg.dtype),
        C.data_ptr(),
        cvt_dtype(C.dtype),
        bias_ptr,
        True,
    )
    return C


# A: mxk,  B: nxk, bias: n
def matmul(A: torch.Tensor, B: torch.Tensor, bias: torch.Tensor):
    m = A.shape[0]
    n = B.shape[0]
    k = B.shape[1]
    lib = get_lib(A)
    ctype = A.dtype
    if A.device.type == "cpu":
        ctype = torch.float32
    C = torch.zeros(m, n, dtype=ctype, device=A.device)
    stream = get_stream(A)
    lib.matmul(
        stream,
        m,
        n,
        k,
        A.contiguous().data_ptr(),
        cvt_dtype(A.dtype),
        B.contiguous().data_ptr(),
        cvt_dtype(B.dtype),
        C.contiguous().data_ptr(),
        cvt_dtype(C.dtype),
        bias.to(C.dtype).contiguous().data_ptr(),
        True,
    )
    return C


# A: mxk:s8,  B: nxk:s8, return: mxn:s32
def igemm_s8s8s32(A: torch.Tensor, B: torch.Tensor):
    m = A.shape[0]
    n = B.shape[0]
    k = B.shape[1]
    lib = get_lib(A)
    if lib is None:
        raise NotImplementedError(f"Current device {A.device} is not supported")
    C = torch.zeros(m, n, dtype=torch.int32, device=A.device)
    stream = get_stream(A)
    lib.matmul(
        stream,
        m,
        n,
        k,
        A.contiguous().data_ptr(),
        cvt_dtype(A.dtype),
        B.contiguous().data_ptr(),
        cvt_dtype(B.dtype),
        C.contiguous().data_ptr(),
        cvt_dtype(C.dtype),
        0,
        True,
    )
    return C


# A: mxk:DT,  B: nxk:s8, scaleB: n:DT
# return: mxn:DT
def woqgemm_s8(A: torch.Tensor, B: torch.Tensor, scaleB: torch.Tensor, bias: torch.Tensor | None = None):
    m = A.shape[0]
    n = B.shape[0]
    k = B.shape[1]
    lib = get_lib(A)

    C = torch.zeros(m, n, dtype=A.dtype, device=A.device)
    stream = get_stream(A)
    lib.woqgemm_s8(
        stream,
        m,
        n,
        k,
        A.contiguous().data_ptr(),
        cvt_dtype(A.dtype),
        B.contiguous().data_ptr(),
        C.contiguous().data_ptr(),
        0 if bias is None or bias.numel() == 0 else bias.contiguous().data_ptr(),
        True,
        scaleB.contiguous().data_ptr(),
    )
    return C


# A: mxk:DT,  B: BS:s8, bias: n:DT
# return: C: mxn:DT
def woqgemm(
    A: torch.Tensor,
    B: torch.Tensor,
    bias: torch.Tensor,
    n,
    k,
    groupsize,
    compute_type,
    weight_type,
    scale_type,
    asym,
    out: torch.Tensor | None = None,
):
    _validate_packed_blob(B, n, k, groupsize, compute_type, weight_type, scale_type, asym)
    m = A.shape[0]
    lib = get_lib(A)
    ct = cvtstr_dtype(compute_type)
    wt = cvtstr_dtype(weight_type)
    st = cvtstr_dtype(scale_type)
    if out is None:
        C = torch.zeros(m, n, dtype=A.dtype, device=A.device)
    else:
        if out.shape != (m, n):
            raise ValueError(f"out must have shape {(m, n)}, got {tuple(out.shape)}")
        if out.dtype != A.dtype:
            raise ValueError(f"out dtype must be {A.dtype}, got {out.dtype}")
        if out.device != A.device:
            raise ValueError(f"out device must be {A.device}, got {out.device}")
        if not out.is_contiguous():
            raise ValueError("out must be contiguous")
        C = out
    stream = get_stream(A)
    lib.woqgemm(
        stream,
        m,
        n,
        k,
        A.contiguous().data_ptr(),
        cvt_dtype(A.dtype),
        B.contiguous().data_ptr(),
        C.data_ptr(),
        bias.contiguous().data_ptr(),
        groupsize,
        ct,
        wt,
        st,
        asym,
        B.numel(),
    )
    return C


def _validate_packed_blob(
    blob: torch.Tensor,
    n: int,
    k: int,
    groupsize: int,
    compute_type: str,
    weight_type: str,
    scale_type: str,
    asym: bool,
) -> None:
    """Validate a packed-weight blob before passing it to native code.

    Raises ``TypeError`` or ``ValueError`` if any check fails, preventing
    out-of-bounds memory access in the native ``unpack_weight`` /
    ``woqgemm`` path when the blob is malformed or the parameters are
    inconsistent with the blob contents.
    """
    if not isinstance(blob, torch.Tensor):
        raise TypeError(f"blob must be a torch.Tensor, got {type(blob).__name__}")

    if blob.device.type not in ("cpu", "xpu"):
        raise ValueError(f"blob must reside on cpu or xpu, got {blob.device.type}")

    if blob.dtype != torch.int8:
        raise ValueError(f"blob must have dtype torch.int8, got {blob.dtype}")

    if blob.dim() != 1:
        raise ValueError(f"blob must be a 1-D tensor, got {blob.dim()}-D")

    if not isinstance(n, (int,)) or n <= 0:
        raise ValueError(f"n must be a positive integer, got {n!r}")
    if not isinstance(k, (int,)) or k <= 0:
        raise ValueError(f"k must be a positive integer, got {k!r}")
    if not isinstance(groupsize, (int,)) or groupsize <= 0:
        raise ValueError(f"groupsize must be a positive integer, got {groupsize!r}")
    if not isinstance(asym, bool):
        raise TypeError(f"asym must be a bool, got {type(asym).__name__}")

    valid_types = {
        "fp32",
        "fp16",
        "bf16",
        "int8",
        "int4",
        "int2",
        "int3",
        "int5",
        "int6",
        "int7",
        "fp8_e4m3",
        "fp8_e5m2",
        "fp8_e8m0",
    }
    valid_compute_types = valid_types | {"auto"}
    if compute_type not in valid_compute_types:
        raise ValueError(f"compute_type must be one of {valid_compute_types}, got {compute_type!r}")
    if weight_type not in valid_types:
        raise ValueError(f"weight_type must be one of {valid_types}, got {weight_type!r}")
    if scale_type not in valid_types:
        raise ValueError(f"scale_type must be one of {valid_types}, got {scale_type!r}")


# QB: k*n:int8,  scaleB: k/blocksize*n:DT
# return: blob:BS:int8
def _repack_quantized_weight_core(
    QB: torch.Tensor,
    scaleB: torch.Tensor,
    zp: torch.Tensor,
    groupsize,
    compute_type,
    weight_type,
    scale_type,
    asym,
):
    if not isinstance(QB, torch.Tensor) or QB.dim() != 2:
        raise ValueError(f"QB must be a 2-D tensor, got shape {QB.shape!r}")
    if not isinstance(scaleB, torch.Tensor) or scaleB.dim() != 2:
        raise ValueError(f"scaleB must be a 2-D tensor, got shape {scaleB.shape!r}")
    if not isinstance(zp, torch.Tensor):
        raise TypeError(f"zp must be a torch.Tensor, got {type(zp).__name__}")
    k = QB.shape[0]
    n = QB.shape[1]
    lib = get_lib(QB)
    stream = get_stream(QB)
    ct = cvtstr_dtype(compute_type)
    wt = cvtstr_dtype(weight_type)
    st = cvtstr_dtype(scale_type)
    BS = lib.packed_weight_size(stream, n, k, groupsize, ct, wt, st, asym)
    blob = torch.zeros(BS, dtype=torch.int8, device=QB.device)
    lib.repack_quantized_weight(
        stream,
        QB.contiguous().data_ptr(),
        zp.contiguous().data_ptr(),
        scaleB.contiguous().data_ptr(),
        blob.data_ptr(),
        n,
        k,
        groupsize,
        ct,
        wt,
        st,
        asym,
    )
    return blob


# QB: blob:BS:int8
# return: out:nxk:out_dtype
def _unpack_weight_core(
    blob: torch.Tensor,
    out_dtype: torch.dtype,
    n,
    k,
    groupsize,
    compute_type,
    weight_type,
    scale_type,
    asym,
):
    _validate_packed_blob(blob, n, k, groupsize, compute_type, weight_type, scale_type, asym)
    lib = get_lib(blob)
    stream = get_stream(blob)
    ct = cvtstr_dtype(compute_type)
    wt = cvtstr_dtype(weight_type)

    st = cvtstr_dtype(scale_type)
    oshape = (n, k) if blob.device.type == "xpu" else (k, n)
    out = torch.zeros(oshape, dtype=out_dtype, device=blob.device)
    lib.unpack_weight(
        stream,
        blob.data_ptr(),
        out.data_ptr(),
        cvt_dtype(out_dtype),
        n,
        k,
        groupsize,
        ct,
        wt,
        st,
        asym,
        blob.numel(),
    )
    if blob.device.type == "cpu":
        return out.T
    return out


def sdpa(
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    attn_mask: torch.Tensor | None = None,
    dropout_p: float = 0.0,
    is_causal: bool = False,
    scale: float | None = None,
    tensor_layout: str = "HND",
    return_lse: bool = False,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
    """Scaled dot-product attention (SDPA) prefill+decode.

    Supported tensor layouts:
    - HND: [B, H, N, D]
    - NHD: [B, N, H, D]

    Args:
    - attn_mask: Additive float32 attention bias with shape [B, 1, Sq, Skv].
    - dropout_p: Must be 0.0; dropout is not supported.
    - is_causal: Apply the standard causal mask.
    - scale: Softmax scale. Uses 1 / sqrt(D) when None.
    - tensor_layout: Layout of Q/K/V/O tensors.
    - return_lse: If True, returns (O, LSE) where LSE[b, h, q] = log(sum_j exp(score_{b,h,q,j})).

    Returns:
    - O: same layout as the input tensors.
    - (O, LSE): if return_lse is True.
    """
    if query.device.type not in ("cpu", "xpu"):
        raise NotImplementedError(f"sdpa is not supported on {query.device.type}")
    supported_dtypes = (
        (torch.float32, torch.float16, torch.bfloat16)
        if query.device.type == "cpu"
        else (
            torch.float16,
            torch.bfloat16,
        )
    )
    if query.dtype not in supported_dtypes:
        raise ValueError(f"Q dtype {query.dtype} is unsupported on {query.device.type}")

    # CPU BestLA mixed precision: F32 query with F16/BF16 K/V produces an F32
    # output. These are the only two cross-dtype combinations wired today; every
    # other combination still requires K/V to match Q. Homogeneous fp16/bf16 is
    # NOT a mixed combination and is unaffected by this branch.
    mixed_kv = (
        query.device.type == "cpu"
        and query.dtype == torch.float32
        and key.dtype == value.dtype
        and key.dtype in (torch.float16, torch.bfloat16)
    )
    if not mixed_kv and (key.dtype != query.dtype or value.dtype != query.dtype):
        raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")

    B, Hq, Hkv, Sq, Skv, D = _validate_attention_geometry(
        query, key, value, tensor_layout, key_dtype=key.dtype, value_dtype=value.dtype
    )
    # The SYCL-TLA (XPU) flash-attention kernels are only compiled for a fixed
    # set of head dimensions. The CPU kernel supports arbitrary head_dim.
    if query.device.type == "xpu" and D not in (64, 128, 96, 192):
        raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128, 96, 192")

    _validate_no_dropout(dropout_p, "sdpa")
    _validate_attention_mask(attn_mask, batch=B, seq_len_q=Sq, seq_len_kv=Skv, device=query.device)

    lib = get_lib(query)
    stream = get_stream(query)

    _validate_canonical_strides(query, "Q", tensor_layout)
    _validate_canonical_strides(key, "K", tensor_layout)
    _validate_canonical_strides(value, "V", tensor_layout)

    # Mixed precision (F32 Q + F16/BF16 K/V) accumulates in and emits F32; the
    # homogeneous path keeps the operand dtype.
    out_dtype = torch.float32 if mixed_kv else value.dtype
    O = _empty_attention_output(
        B,
        Hq,
        Sq,
        D,
        dtype=out_dtype,
        device=query.device,
        tensor_layout=tensor_layout,
    )
    LSE = torch.empty(B, Hq, Sq, dtype=torch.float32, device=query.device) if return_lse else None

    if query.device.type == "cpu" and return_lse:
        raise NotImplementedError("return_lse is not supported on CPU")

    q_strides = _attention_strides_qko(query, tensor_layout)
    k_strides = _attention_strides_qko(key, tensor_layout)
    v_strides = _attention_strides_v(value, tensor_layout)
    o_strides = _attention_strides_qko(O, tensor_layout)

    if query.device.type == "cpu":
        if mixed_kv and attn_mask is None and _cpu_public_packed_kv_available():
            try:
                return _cpu_public_mixed_sdpa_packed(
                    query,
                    key,
                    value,
                    is_causal=bool(is_causal),
                    scale=scale,
                    tensor_layout=tensor_layout,
                )
            except RuntimeError:
                pass  # packed path unavailable at runtime; fall through to lib.sdpa()
        lib.sdpa(
            stream,
            query.data_ptr(),
            key.data_ptr(),
            value.data_ptr(),
            O.data_ptr(),
            attn_mask.data_ptr() if attn_mask is not None else 0,
            *q_strides,
            *k_strides,
            *v_strides,
            *o_strides,
            cvt_dtype(query.dtype),
            cvt_dtype(key.dtype),
            cvt_dtype(O.dtype),
            B,
            Hq,
            Hkv,
            Sq,
            Skv,
            D,
            float(scale) if scale is not None else 1.0 / (D**0.5),
            bool(is_causal),
        )
        return O

    layout_code = LAYOUT_HND if _normalize_tensor_layout(tensor_layout) == "HND" else LAYOUT_NHD
    lib.sdpa(
        stream,
        query.data_ptr(),
        key.data_ptr(),
        value.data_ptr(),
        O.data_ptr(),
        attn_mask.data_ptr() if attn_mask is not None else 0,
        cvt_dtype(query.dtype),
        cvt_dtype(key.dtype),
        cvt_dtype(O.dtype),
        B,
        Hq,
        Hkv,
        Sq,
        Skv,
        D,
        float(scale) if scale is not None else 1.0 / (D**0.5),
        bool(is_causal),
        layout_code,
        LSE.data_ptr() if LSE is not None else 0,
    )

    if return_lse:
        return O, LSE
    return O


def _xpu_capturing() -> bool:
    """True if the current XPU stream is inside a torch.xpu graph capture.

    Used to skip host-side device->host syncs (.item() / .cpu()) that are
    illegal while a SyclTensor command graph is being recorded.
    """
    try:
        return bool(torch.xpu.is_current_stream_capturing())
    except Exception:
        return False


def sdpa_varlen(
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    cu_seqlens_q: torch.Tensor,
    cu_seqlens_k: torch.Tensor,
    max_seqlen_q: int,
    max_seqlen_k: int,
    attn_mask: torch.Tensor | None = None,
    dropout_p: float = 0.0,
    is_causal: bool = False,
    scale: float | None = None,
    return_lse: bool = False,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
    """Scaled dot-product attention (SDPA) prefill+decode with variable-length sequences.

    Q, K, V use a flat 3-D layout where tokens from all sequences in the batch
    are concatenated along dim-0 (the *total* dimension).  Per-sequence
    boundaries are given by ``cu_seqlens_q`` / ``cu_seqlens_k``.

    Shapes:
        Q:   [total_q, num_heads_q, head_dim]
        K:   [total_kv, num_heads_kv, head_dim]
        V:   [total_kv, num_heads_kv, head_dim]
        O:   [total_q, num_heads_q, head_dim]

    where ``total_q = cu_seqlens_q[-1]`` and ``total_kv = cu_seqlens_k[-1]``.

    Args:
        cu_seqlens_q: Cumulative sequence lengths for Q  [batch_size + 1] int32.
        cu_seqlens_k: Cumulative sequence lengths for K/V  [batch_size + 1] int32.
        max_seqlen_q: Maximum Q sequence length in the batch.
        max_seqlen_k: Maximum K/V sequence length in the batch.
        scale: Softmax scale.  Uses ``1 / sqrt(head_dim)`` when ``None``.

    Returns:
        O: Flat output with shape [total_q, num_heads_q, head_dim].
    """
    if query.device.type != "xpu":
        raise NotImplementedError("sdpa_varlen is only supported on XPU")

    if query.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")
    if key.dtype != query.dtype or value.dtype != query.dtype:
        raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")

    if query.ndim != 3 or key.ndim != 3 or value.ndim != 3:
        raise ValueError(f"Q/K/V must be 3-D for varlen; got Q={query.ndim}D, K={key.ndim}D, V={value.ndim}D")

    if cu_seqlens_q.dtype not in (torch.int32, torch.int64):
        raise ValueError(f"cu_seqlens_q must be int32 or int64, got {cu_seqlens_q.dtype}")
    if cu_seqlens_k.dtype not in (torch.int32, torch.int64):
        raise ValueError(f"cu_seqlens_k must be int32 or int64, got {cu_seqlens_k.dtype}")
    if cu_seqlens_q.dim() != 1 or cu_seqlens_k.dim() != 1:
        raise ValueError("cu_seqlens_q and cu_seqlens_k must be 1-D")
    if cu_seqlens_q.device.type != "xpu" or cu_seqlens_k.device.type != "xpu":
        raise ValueError("cu_seqlens_q and cu_seqlens_k must be on XPU")

    batch = cu_seqlens_q.shape[0] - 1
    if cu_seqlens_k.shape[0] - 1 != batch:
        raise ValueError("cu_seqlens_q and cu_seqlens_k must describe the same batch size")

    total_q, Hq, D = query.shape
    total_kv, Hkv, Dk = key.shape
    total_kv_v, Hkv2, Dv = value.shape

    if not _xpu_capturing():
        # Cross-check flat layout against the cumulative sequence-length
        # boundaries.  These .item() calls force a device->host sync, which is
        # illegal while a command graph is being captured, so the check is
        # skipped during capture (the caller must keep buffer geometry stable).
        if total_q != int(cu_seqlens_q[-1].item()):
            raise ValueError(f"Q dim-0 ({total_q}) != cu_seqlens_q[-1]")
        if total_kv != int(cu_seqlens_k[-1].item()):
            raise ValueError(f"K dim-0 ({total_kv}) != cu_seqlens_k[-1]")
        if total_kv_v != int(cu_seqlens_k[-1].item()):
            raise ValueError(f"V dim-0 ({total_kv_v}) != cu_seqlens_k[-1]")
    if Hkv != Hkv2 or Dk != Dv:
        raise ValueError("K/V shape mismatch")
    if Dk != D:
        raise ValueError("Head dim mismatch between Q and K/V")
    if D not in (64, 128, 96, 192):
        raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128, 96, 192")

    if dropout_p != 0.0:
        raise NotImplementedError(f"dropout_p must be 0.0 (got {dropout_p}); dropout is not supported")

    if attn_mask is not None:
        raise NotImplementedError("attn_mask is not yet supported in sdpa_varlen")

    if Hq % Hkv != 0:
        raise ValueError(f"num_heads_q ({Hq}) must be divisible by num_heads_kv ({Hkv})")

    # Validate strides: the dim axis must be contiguous.
    # Flat layout [total, H, D]: dim-stride (stride(2)) must be 1.
    if query.stride(2) != 1:
        raise ValueError(f"Q must be contiguous along head-dim axis; got stride {query.stride()}")
    if key.stride(2) != 1:
        raise ValueError(f"K must be contiguous along head-dim axis; got stride {key.stride()}")
    if value.stride(2) != 1:
        raise ValueError(f"V must be contiguous along head-dim axis; got stride {value.stride()}")

    lib = get_lib(query)
    stream = get_stream(query)
    cu_seqlens_q_i32 = cu_seqlens_q.contiguous().to(torch.int32)
    cu_seqlens_k_i32 = cu_seqlens_k.contiguous().to(torch.int32)

    O = torch.empty(total_q, Hq, D, dtype=value.dtype, device=query.device)

    if return_lse:
        # .item() is a device->host sync; illegal during graph capture.
        max_q = max_seqlen_q if _xpu_capturing() else int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())
        if max_seqlen_q < max_q:
            raise ValueError(f"max_seqlen_q ({max_seqlen_q}) < max sequence length in cu_seqlens_q ({max_q})")
        LSE = torch.full(
            (batch, Hq, max_seqlen_q),
            float("-inf"),
            dtype=torch.float32,
            device=query.device,
        )
    else:
        LSE = None
    lib.sdpa_varlen(
        stream,
        query.data_ptr(),
        key.data_ptr(),
        value.data_ptr(),
        O.data_ptr(),
        attn_mask.data_ptr() if attn_mask is not None else 0,
        cvt_dtype(query.dtype),
        cvt_dtype(key.dtype),
        cvt_dtype(O.dtype),
        batch,
        Hq,
        Hkv,
        total_q,
        total_kv,
        max_seqlen_q,
        max_seqlen_k,
        D,
        float(scale) if scale is not None else 1.0 / (D**0.5),
        bool(is_causal),
        cu_seqlens_q_i32.data_ptr(),
        cu_seqlens_k_i32.data_ptr(),
        LAYOUT_HND,  # unused by the native varlen path (always flat 3-D)
        LSE.data_ptr() if LSE is not None else 0,
    )

    if return_lse:
        return O, LSE
    return O


def debug_cpu_sdpa_route(
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    *,
    attn_mask: torch.Tensor | None = None,
    is_causal: bool = False,
    scale: float | None = None,
    tensor_layout: str = "HND",
) -> int:
    """Return the resolved internal CPU SDPA route for tests/debugging."""
    if query.device.type != "cpu":
        raise NotImplementedError("debug_cpu_sdpa_route is only supported on CPU")
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_debug_resolve_sdpa_route"):
        raise NotImplementedError("ARK CPU debug route resolver is not available")

    mixed_kv = (
        query.dtype == torch.float32 and key.dtype == value.dtype and key.dtype in (torch.float16, torch.bfloat16)
    )
    if not mixed_kv and (key.dtype != query.dtype or value.dtype != query.dtype):
        raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")
    B, Hq, Hkv, Sq, Skv, D = _validate_attention_geometry(
        query, key, value, tensor_layout, key_dtype=key.dtype, value_dtype=value.dtype
    )
    _validate_attention_mask(attn_mask, batch=B, seq_len_q=Sq, seq_len_kv=Skv, device=query.device)

    out_dtype = torch.float32 if mixed_kv else value.dtype
    O = _empty_attention_output(B, Hq, Sq, D, dtype=out_dtype, device=query.device, tensor_layout=tensor_layout)
    q_strides = _attention_strides_qko(query, tensor_layout)
    k_strides = _attention_strides_qko(key, tensor_layout)
    v_strides = _attention_strides_v(value, tensor_layout)
    o_strides = _attention_strides_qko(O, tensor_layout)
    return cpu_lib.ark_cpu_debug_resolve_sdpa_route(
        query.data_ptr(),
        key.data_ptr(),
        value.data_ptr(),
        O.data_ptr(),
        attn_mask.data_ptr() if attn_mask is not None else 0,
        *q_strides,
        *k_strides,
        *v_strides,
        *o_strides,
        cvt_dtype(query.dtype),
        cvt_dtype(key.dtype),
        cvt_dtype(O.dtype),
        B,
        Hq,
        Hkv,
        Sq,
        Skv,
        D,
        float(scale) if scale is not None else 1.0 / (D**0.5),
        bool(is_causal),
    )


def sage(
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    attn_mask: torch.Tensor | None = None,
    dropout_p: float = 0.0,
    is_causal: bool = False,
    scale: float | None = None,
    enable_gqa: bool = False,
    quant_block_size: int = 64,
    qscale: torch.Tensor = None,
    kscale: torch.Tensor = None,
    tensor_layout: str = "HND",
) -> torch.Tensor:
    """SAGE attention prefill+decode.

    Supported tensor layouts:
    - HND: [B, H, N, D]
    - NHD: [B, N, H, D]

    Args:
    - scale: Attention scale. Uses 1 / sqrt(D) when None.
    - quant_block_size: Block size for qscale and kscale.
    - tensor_layout: Layout of Q/K/V/O tensors.

    Returns:
    - O: same layout as the input tensors.
    """
    if query.device.type != "xpu":
        raise NotImplementedError("sage is only supported on XPU")
    if query.dtype != torch.int8 or key.dtype != torch.int8:
        raise ValueError(f"sage expects int8 Q/K tensors, got Q={query.dtype}, K={key.dtype}")
    if value.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(f"sage expects fp16/bf16 V tensors, got V={value.dtype}")
    if qscale is None or kscale is None:
        raise ValueError("qscale and kscale must be provided for sage")

    B, Hq, Hkv, Sq, Skv, D = _validate_attention_geometry(
        query, key, value, tensor_layout, key_dtype=torch.int8, value_dtype=value.dtype
    )
    if D not in (64, 128):
        raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
    _validate_no_dropout(dropout_p, "sage")
    _validate_attention_mask(attn_mask, batch=B, seq_len_q=Sq, seq_len_kv=Skv, device=query.device)

    lib = get_lib(query)
    stream = get_stream(query)

    _validate_canonical_strides(query, "Q", tensor_layout)
    _validate_canonical_strides(key, "K", tensor_layout)
    _validate_canonical_strides(value, "V", tensor_layout)

    O = _empty_attention_output(
        B,
        Hq,
        Sq,
        D,
        dtype=value.dtype,
        device=query.device,
        tensor_layout=tensor_layout,
    )
    layout_code = LAYOUT_HND if _normalize_tensor_layout(tensor_layout) == "HND" else LAYOUT_NHD
    lib.sage(
        stream,
        query.data_ptr(),
        key.data_ptr(),
        value.data_ptr(),
        O.data_ptr(),
        attn_mask.data_ptr() if attn_mask is not None else 0,
        quant_block_size,
        qscale.data_ptr() if qscale is not None else 0,
        kscale.data_ptr() if kscale is not None else 0,
        cvt_dtype(query.dtype),
        cvt_dtype(key.dtype),
        cvt_dtype(O.dtype),
        B,
        Hq,
        Hkv,
        Sq,
        Skv,
        D,
        float(scale) if scale is not None else 1.0 / (D**0.5),
        bool(is_causal),
        layout_code,
    )
    return O


def sage_pvi8(
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    attn_mask: torch.Tensor | None = None,
    dropout_p: float = 0.0,
    is_causal: bool = False,
    scale: float | None = None,
    enable_gqa: bool = False,
    quant_block_size: int = 64,
    qscale: torch.Tensor = None,
    kscale: torch.Tensor = None,
    vscale: torch.Tensor = None,
    out_dtype: torch.dtype = torch.float16,
    tensor_layout: str = "HND",
) -> torch.Tensor:
    """Low-level SAGE attention with pre-quantized INT8 Q/K/V and PV int8.

    Expects contiguous layouts:
    - query: [B, Hq, Sq, D] int8
    - key: [B, Hkv, Skv, D] int8
    - value: [B, Hkv, Skv, D] int8
    - qscale: [B, Hq, ceil(Sq / quant_block_size), 1] float32
    - kscale: [B, Hkv, ceil(Skv / quant_block_size), 1] float32
    - vscale: [B, Hkv, ceil(Skv / quant_block_size), D] float32

    Returns:
    - O: [B, Hq, Sq, D] float16
    """
    if query.device.type != "xpu":
        raise NotImplementedError("sage_pvi8 is only supported on XPU")
    if query.dtype != torch.int8 or key.dtype != torch.int8 or value.dtype != torch.int8:
        raise ValueError(f"Q/K/V must be int8, got Q={query.dtype}, K={key.dtype}, V={value.dtype}")
    if out_dtype != torch.float16:
        raise ValueError(f"sage_pvi8 output must be float16, got {out_dtype}")
    if qscale is None or kscale is None or vscale is None:
        raise ValueError("qscale, kscale and vscale must be provided for sage_pvi8")

    B, Hq, Hkv, Sq, Skv, D = _validate_attention_geometry(
        query, key, value, tensor_layout, key_dtype=torch.int8, value_dtype=torch.int8
    )
    if D not in (64, 128):
        raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
    _validate_no_dropout(dropout_p, "sage_pvi8")
    _validate_attention_mask(attn_mask, batch=B, seq_len_q=Sq, seq_len_kv=Skv, device=query.device)

    q_blocks = (Sq + quant_block_size - 1) // quant_block_size
    kv_blocks = (Skv + quant_block_size - 1) // quant_block_size
    if qscale.numel() != B * Hq * q_blocks:
        raise ValueError(
            f"qscale must have {B * Hq * q_blocks} elements for shape [B, Hq, ceil(Sq/block), 1], got {qscale.numel()}"
        )
    if kscale.numel() != B * Hkv * kv_blocks:
        raise ValueError(
            f"kscale must have {B * Hkv * kv_blocks} elements for shape [B, Hkv, ceil(Skv/block), 1], got {kscale.numel()}"
        )
    if vscale.numel() != B * Hkv * kv_blocks * D:
        raise ValueError(
            f"vscale must have {B * Hkv * kv_blocks * D} elements for shape [B, Hkv, ceil(Skv/block), D], got {vscale.numel()}"
        )

    lib = get_lib(query)
    stream = get_stream(query)

    _validate_canonical_strides(query, "Q", tensor_layout)
    _validate_canonical_strides(key, "K", tensor_layout)
    _validate_canonical_strides(value, "V", tensor_layout)

    O = _empty_attention_output(
        B,
        Hq,
        Sq,
        D,
        dtype=out_dtype,
        device=query.device,
        tensor_layout=tensor_layout,
    )
    layout_code = LAYOUT_HND if _normalize_tensor_layout(tensor_layout) == "HND" else LAYOUT_NHD
    lib.sage_pvi8(
        stream,
        query.data_ptr(),
        key.data_ptr(),
        value.data_ptr(),
        O.data_ptr(),
        attn_mask.data_ptr() if attn_mask is not None else 0,
        quant_block_size,
        qscale.data_ptr(),
        kscale.data_ptr(),
        vscale.data_ptr(),
        cvt_dtype(query.dtype),
        cvt_dtype(key.dtype),
        cvt_dtype(O.dtype),
        B,
        Hq,
        Hkv,
        Sq,
        Skv,
        D,
        float(scale) if scale is not None else 1.0 / (D**0.5),
        bool(is_causal),
        layout_code,
    )
    return O


def sagev1(
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    attn_mask: torch.Tensor | None = None,
    dropout_p: float = 0.0,
    is_causal: bool = False,
    scale: float | None = None,
    enable_gqa: bool = False,
    quant_block_size: int = 64,
    tensor_layout: str = "HND",
    return_lse: bool = False,
    smooth_k: bool = True,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
    """SAGE v1 attention prefill+decode.

    Supported tensor layouts:
    - HND: [B, H, N, D]
    - NHD: [B, N, H, D]

    Args:
    - scale: Attention scale. Uses 1 / sqrt(D) when None.
    - quant_block_size: Quantization block size used by the kernel.
    - tensor_layout: Layout of Q/K/V/O tensors.
    - return_lse: If True, returns (O, LSE) where LSE[b, h, q] = log(sum_j exp(score_{b,h,q,j})).
    - smooth_k: Whether to smooth the key by subtracting the sequence mean
      before INT8 quantization (handled by the C++ kernel). Default: True.

    Returns:
    - O: same layout as the input tensors.
    - (O, LSE): if return_lse is True.
    """
    if quant_block_size <= 0:
        return sdpa(
            query=query,
            key=key,
            value=value,
            attn_mask=attn_mask,
            dropout_p=dropout_p,
            is_causal=is_causal,
            scale=scale,
            tensor_layout=tensor_layout,
            return_lse=return_lse,
        )
    if query.device.type != "xpu":
        raise NotImplementedError("sagev1 is only supported on XPU")
    if query.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")
    if key.dtype != query.dtype or value.dtype != query.dtype:
        raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")

    B, Hq, Hkv, Sq, Skv, D = _validate_attention_geometry(
        query, key, value, tensor_layout, key_dtype=query.dtype, value_dtype=query.dtype
    )
    if D not in (64, 128):
        raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
    _validate_no_dropout(dropout_p, "sagev1")
    _validate_attention_mask(attn_mask, batch=B, seq_len_q=Sq, seq_len_kv=Skv, device=query.device)

    lib = get_lib(query)
    stream = get_stream(query)

    _validate_canonical_strides(query, "Q", tensor_layout)
    _validate_canonical_strides(key, "K", tensor_layout)
    _validate_canonical_strides(value, "V", tensor_layout)

    O = _empty_attention_output(
        B,
        Hq,
        Sq,
        D,
        dtype=value.dtype,
        device=query.device,
        tensor_layout=tensor_layout,
    )

    LSE = torch.empty(B, Hq, Sq, dtype=torch.float32, device=query.device) if return_lse else None

    layout_code = LAYOUT_HND if _normalize_tensor_layout(tensor_layout) == "HND" else LAYOUT_NHD
    lib.sagev1(
        stream,
        query.data_ptr(),
        key.data_ptr(),
        value.data_ptr(),
        O.data_ptr(),
        attn_mask.data_ptr() if attn_mask is not None else 0,
        quant_block_size,
        cvt_dtype(query.dtype),
        cvt_dtype(key.dtype),
        cvt_dtype(value.dtype),
        cvt_dtype(O.dtype),
        B,
        Hq,
        Hkv,
        Sq,
        Skv,
        D,
        float(scale) if scale is not None else 1.0 / (D**0.5),
        bool(is_causal),
        layout_code,
        bool(smooth_k),
        LSE.data_ptr() if LSE is not None else 0,
    )

    if return_lse:
        return O, LSE
    return O


def sagev1_pvi8(
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    attn_mask: torch.Tensor | None = None,
    dropout_p: float = 0.0,
    is_causal: bool = False,
    scale: float | None = None,
    enable_gqa: bool = False,
    quant_block_size: int = 64,
    tensor_layout: str = "HND",
    return_lse: bool = False,
    smooth_k: bool = True,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
    """SAGE v1 attention with PV int8 path.

    Expects FP16 Q/K/V input and quantizes Q/K/V internally before calling
    the PV int8 kernel.
    """
    if quant_block_size <= 0:
        return sdpa(
            query=query,
            key=key,
            value=value,
            attn_mask=attn_mask,
            dropout_p=dropout_p,
            is_causal=is_causal,
            scale=scale,
            tensor_layout=tensor_layout,
            return_lse=return_lse,
        )
    if query.device.type != "xpu":
        raise NotImplementedError("sagev1_pvi8 is only supported on XPU")
    if query.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")
    if key.dtype != query.dtype or value.dtype != query.dtype:
        raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")

    B, Hq, Hkv, Sq, Skv, D = _validate_attention_geometry(
        query, key, value, tensor_layout, key_dtype=query.dtype, value_dtype=query.dtype
    )
    if D not in (64, 128):
        raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
    _validate_no_dropout(dropout_p, "sagev1_pvi8")
    _validate_attention_mask(attn_mask, batch=B, seq_len_q=Sq, seq_len_kv=Skv, device=query.device)

    lib = get_lib(query)
    stream = get_stream(query)

    _validate_canonical_strides(query, "Q", tensor_layout)
    _validate_canonical_strides(key, "K", tensor_layout)
    _validate_canonical_strides(value, "V", tensor_layout)

    O = _empty_attention_output(
        B,
        Hq,
        Sq,
        D,
        dtype=value.dtype,
        device=query.device,
        tensor_layout=tensor_layout,
    )

    LSE = torch.empty(B, Hq, Sq, dtype=torch.float32, device=query.device) if return_lse else None

    layout_code = LAYOUT_HND if _normalize_tensor_layout(tensor_layout) == "HND" else LAYOUT_NHD
    lib.sagev1_pvi8(
        stream,
        query.data_ptr(),
        key.data_ptr(),
        value.data_ptr(),
        O.data_ptr(),
        attn_mask.data_ptr() if attn_mask is not None else 0,
        quant_block_size,
        cvt_dtype(query.dtype),
        cvt_dtype(key.dtype),
        cvt_dtype(value.dtype),
        cvt_dtype(O.dtype),
        B,
        Hq,
        Hkv,
        Sq,
        Skv,
        D,
        float(scale) if scale is not None else 1.0 / (D**0.5),
        bool(is_causal),
        layout_code,
        bool(smooth_k),
        LSE.data_ptr() if LSE is not None else 0,
    )

    if return_lse:
        return O, LSE
    return O


def ark_cpu_kv_cache_alloc(
    batch: int,
    num_heads_kv: int,
    capacity: int,
    head_dim: int,
    *,
    dtype: torch.dtype = torch.float32,
    device: torch.device | str = "cpu",
) -> tuple[torch.Tensor, torch.Tensor]:
    """Allocate an ARK CPU KV cache in internal HND layout: [B, Hkv, capacity, D]."""
    device = torch.device(device)
    if device.type != "cpu":
        raise ValueError("ark_cpu_kv_cache_alloc only supports CPU tensors")
    if dtype not in (torch.float32, torch.float16, torch.bfloat16):
        raise ValueError(f"Unsupported KV cache dtype: {dtype}")
    shape = (batch, num_heads_kv, capacity, head_dim)
    return torch.empty(shape, device=device, dtype=dtype), torch.empty(shape, device=device, dtype=dtype)


def ark_cpu_kv_update(
    key_cache: torch.Tensor,
    value_cache: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    start_pos: int,
    *,
    tensor_layout: str = "HND",
) -> tuple[torch.Tensor, torch.Tensor]:
    """Append K/V tensors to an ARK CPU KV cache allocated by ``ark_cpu_kv_cache_alloc``."""
    if (
        key_cache.device.type != "cpu"
        or value_cache.device.type != "cpu"
        or key.device.type != "cpu"
        or value.device.type != "cpu"
    ):
        raise ValueError("ark_cpu_kv_update only supports CPU tensors")
    if key_cache.dtype != value_cache.dtype or key.dtype != key_cache.dtype or value.dtype != key_cache.dtype:
        raise ValueError("K/V cache and source tensors must have the same dtype")
    if key_cache.ndim != 4 or value_cache.shape != key_cache.shape:
        raise ValueError("K/V caches must be 4D tensors with identical shape")
    if not key_cache.is_contiguous() or not value_cache.is_contiguous():
        raise ValueError("K/V caches must be contiguous")

    batch, num_heads_kv, capacity, head_dim = key_cache.shape
    Bk, Hkv, append_len, Dk = _validate_attention_tensor(key, "K", tensor_layout, expected_dtype=key_cache.dtype)
    Bv, Hkv2, append_len_v, Dv = _validate_attention_tensor(value, "V", tensor_layout, expected_dtype=key_cache.dtype)
    if (Bk, Bv) != (batch, batch) or Hkv != num_heads_kv or Hkv2 != num_heads_kv:
        raise ValueError("K/V source batch or head count does not match cache")
    if append_len_v != append_len or Dk != head_dim or Dv != head_dim:
        raise ValueError("K/V source shape does not match cache")
    if start_pos < 0 or start_pos + append_len > capacity:
        raise ValueError("KV append range exceeds cache capacity")
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_kv_update"):
        raise NotImplementedError("ARK CPU KV cache update kernel is not available")

    k_strides = _attention_strides_qko(key, tensor_layout)
    v_strides = _attention_strides_v(value, tensor_layout)
    cpu_lib.ark_cpu_kv_update(
        key_cache.data_ptr(),
        value_cache.data_ptr(),
        key.data_ptr(),
        value.data_ptr(),
        *k_strides,
        *v_strides,
        cvt_dtype(key_cache.dtype),
        batch,
        num_heads_kv,
        append_len,
        head_dim,
        capacity,
        int(start_pos),
    )
    return key_cache, value_cache


# -----------------------------------------------------------------------------
# Internal/experimental CPU mixed-route lifecycle helpers.
#
# These APIs exist to manage backend state (packed descriptors/caches/rope/packed
# forwards). They are intentionally outside the public sdpa() contract, which
# remains the standard SDPA surface.
# -----------------------------------------------------------------------------
@dataclass(frozen=True)
class ArkCpuPackedKVHandle:
    """Internal/experimental handle for the packed BestLA CPU KV-cache path."""

    descriptor: object
    dtype: torch.dtype

    @classmethod
    def create(
        cls,
        batch: int,
        num_heads_kv: int,
        capacity: int,
        head_dim: int,
        *,
        dtype: torch.dtype = torch.float16,
    ) -> "ArkCpuPackedKVHandle":
        return cls(ark_cpu_packed_kv_descriptor(batch, num_heads_kv, capacity, head_dim, dtype=dtype), dtype)

    def info(self) -> dict:
        return ark_cpu_packed_kv_info(descriptor=self.descriptor)

    def alloc(self, *, device: str = "cpu") -> tuple[torch.Tensor, torch.Tensor]:
        return ark_cpu_packed_kv_alloc_from_descriptor(self.descriptor, dtype=self.dtype, device=device)

    def update(
        self,
        cache_k: torch.Tensor,
        cache_v: torch.Tensor,
        key: torch.Tensor,
        value: torch.Tensor,
        start_pos: int,
        *,
        tensor_layout: str = "HND",
        no_zeroing: bool = False,
    ) -> None:
        return ark_cpu_update_packed_kv_from_descriptor(
            self.descriptor, cache_k, cache_v, key, value, start_pos, tensor_layout=tensor_layout, no_zeroing=no_zeroing
        )

    def copy(
        self,
        dst_cache_k: torch.Tensor,
        dst_cache_v: torch.Tensor,
        src_cache_k: torch.Tensor,
        src_cache_v: torch.Tensor,
        seq_off: int,
        seq_size: int,
        *,
        no_zeroing: bool = False,
    ) -> None:
        return ark_cpu_copy_packed_kv_from_descriptor(
            self.descriptor,
            dst_cache_k,
            dst_cache_v,
            src_cache_k,
            src_cache_v,
            seq_off,
            seq_size,
            no_zeroing=no_zeroing,
        )

    def shift_k(self, cache_k: torch.Tensor, cossin: torch.Tensor, *, seq_keep: int) -> None:
        return ark_cpu_shift_packed_k_from_descriptor(self.descriptor, cache_k, cossin, seq_keep=seq_keep)

    def forward(
        self,
        query: torch.Tensor,
        cache_k: torch.Tensor,
        cache_v: torch.Tensor,
        seq_len_kv: int,
        num_heads_kv: int | None = None,
        *,
        is_causal: bool = False,
        scale: float | None = None,
        tensor_layout: str = "HND",
    ) -> torch.Tensor:
        del num_heads_kv
        return ark_cpu_bestla_sdpa_packed_from_descriptor(
            self.descriptor,
            query,
            cache_k,
            cache_v,
            seq_len_kv,
            is_causal=is_causal,
            scale=scale,
            tensor_layout=tensor_layout,
        )


def ark_cpu_packed_kv_descriptor(
    batch: int,
    num_heads_kv: int,
    capacity: int,
    head_dim: int,
    *,
    dtype: torch.dtype = torch.float16,
):
    """Create an internal/experimental packed-KV descriptor for repeated CPU BestLA cache operations."""
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_packed_kv_descriptor"):
        raise NotImplementedError("ARK CPU packed KV descriptor is not available (requires BestLA CPU extension build)")
    return cpu_lib.ark_cpu_packed_kv_descriptor(batch, num_heads_kv, capacity, head_dim, cvt_dtype(dtype))


def ark_cpu_packed_kv_alloc_from_descriptor(
    descriptor,
    *,
    dtype: torch.dtype | None = None,
    device: str = "cpu",
) -> tuple[torch.Tensor, torch.Tensor]:
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_packed_kv_elems_desc"):
        raise NotImplementedError(
            "ARK CPU packed KV descriptor allocation is not available (requires BestLA CPU extension build)"
        )
    desc_info = ark_cpu_packed_kv_info(descriptor=descriptor)
    desc_dtype = _torch_dtype_from_ark_dtype(int(desc_info["dtype"]))
    alloc_dtype = dtype if dtype is not None else desc_dtype
    if alloc_dtype != desc_dtype:
        raise ValueError(f"Descriptor dtype {desc_dtype} does not match requested allocation dtype {alloc_dtype}")
    k_elems, v_elems = cpu_lib.ark_cpu_packed_kv_elems_desc(descriptor)
    return (
        torch.zeros(k_elems, dtype=alloc_dtype, device=device),
        torch.zeros(v_elems, dtype=alloc_dtype, device=device),
    )


def ark_cpu_packed_kv_alloc(
    batch: int,
    num_heads_kv: int,
    capacity: int,
    head_dim: int,
    *,
    dtype: torch.dtype = torch.float16,
    device: str = "cpu",
) -> tuple:
    """Allocate internal/experimental 1-D packed K/V tensors for the BestLA decode path.

    Returns (cache_k, cache_v) as 1-D tensors of the requested dtype.  The packed
    geometry is NTILE24_ROWPACK1 for fp16, NTILE48_ROWPACK2 for bf16, matching the
    layout expected by ark_cpu_update_packed_k/v and ark_cpu_bestla_sdpa_packed.

    Both tensors are zero-initialized (unwritten packed slots read as zero).
    """
    descriptor = ark_cpu_packed_kv_descriptor(batch, num_heads_kv, capacity, head_dim, dtype=dtype)
    return ark_cpu_packed_kv_alloc_from_descriptor(descriptor, dtype=dtype, device=device)


def ark_cpu_packed_kv_info(
    batch: int | None = None,
    num_heads_kv: int | None = None,
    capacity: int | None = None,
    head_dim: int | None = None,
    *,
    dtype: torch.dtype = torch.float16,
    descriptor=None,
) -> dict:
    """Return the internal/experimental packed-KV descriptor used by the CPU BestLA path."""
    if descriptor is not None:
        if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_packed_kv_info_desc"):
            raise NotImplementedError(
                "ARK CPU packed KV descriptor query is not available (requires BestLA CPU extension build)"
            )
        return dict(cpu_lib.ark_cpu_packed_kv_info_desc(descriptor))
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_packed_kv_info"):
        raise NotImplementedError("ARK CPU packed KV info query is not available (requires BestLA CPU extension build)")
    if batch is None or num_heads_kv is None or capacity is None or head_dim is None:
        raise ValueError("batch, num_heads_kv, capacity, and head_dim are required when descriptor is not provided")
    return dict(cpu_lib.ark_cpu_packed_kv_info(batch, num_heads_kv, capacity, head_dim, cvt_dtype(dtype)))


def ark_cpu_update_packed_kv_from_descriptor(
    descriptor,
    cache_k: torch.Tensor,
    cache_v: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    start_pos: int,
    *,
    tensor_layout: str = "HND",
    no_zeroing: bool = False,
) -> None:
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_update_packed_k_desc"):
        raise NotImplementedError(
            "ARK CPU packed KV descriptor update is not available (requires BestLA CPU extension build)"
        )
    batch, num_heads_kv, append_len, head_dim = _attention_shape(key, tensor_layout)
    if (
        batch != int(descriptor.batch_size)
        or num_heads_kv != int(descriptor.heads_kv)
        or head_dim != int(descriptor.head_dim)
    ):
        raise ValueError("K descriptor shape does not match the key/value tensors")
    if start_pos < 0 or start_pos + append_len > int(descriptor.logical_capacity):
        raise ValueError("KV append range exceeds packed descriptor capacity")
    k_strides = _attention_strides_qko(key, tensor_layout)
    v_strides = _attention_strides_v(value, tensor_layout)
    cpu_lib.ark_cpu_update_packed_k_desc(
        cache_k.data_ptr(), key.data_ptr(), *k_strides, descriptor, append_len, int(start_pos), bool(no_zeroing)
    )
    cpu_lib.ark_cpu_update_packed_v_desc(
        cache_v.data_ptr(), value.data_ptr(), *v_strides, descriptor, append_len, int(start_pos), bool(no_zeroing)
    )


def ark_cpu_update_packed_kv(
    cache_k: torch.Tensor,
    cache_v: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    start_pos: int,
    capacity: int,
    *,
    tensor_layout: str = "HND",
    no_zeroing: bool = False,
) -> None:
    """Append raw K/V tokens at [start_pos, start_pos+append_len) into packed caches.

    cache_k and cache_v must have been allocated by ark_cpu_packed_kv_alloc with
    the same (batch, num_heads_kv, capacity, head_dim, dtype).  key and value are
    raw HND/NHD tensors; tensor_layout selects the stride convention.  capacity
    must match the value passed to ark_cpu_packed_kv_alloc.  The update is in-place.
    """
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_update_packed_k"):
        raise NotImplementedError("ARK CPU packed KV update is not available (requires BestLA CPU extension build)")
    kv_dtype = cvt_dtype(key.dtype)
    batch, num_heads_kv, append_len, head_dim = _attention_shape(key, tensor_layout)
    k_strides = _attention_strides_qko(key, tensor_layout)
    v_strides = _attention_strides_v(value, tensor_layout)
    cpu_lib.ark_cpu_update_packed_k(
        cache_k.data_ptr(),
        key.data_ptr(),
        *k_strides,
        kv_dtype,
        batch,
        num_heads_kv,
        append_len,
        head_dim,
        capacity,
        int(start_pos),
        bool(no_zeroing),
    )
    cpu_lib.ark_cpu_update_packed_v(
        cache_v.data_ptr(),
        value.data_ptr(),
        *v_strides,
        kv_dtype,
        batch,
        num_heads_kv,
        append_len,
        head_dim,
        capacity,
        int(start_pos),
        bool(no_zeroing),
    )


def ark_cpu_copy_packed_kv(
    dst_cache_k: torch.Tensor,
    dst_cache_v: torch.Tensor,
    src_cache_k: torch.Tensor,
    src_cache_v: torch.Tensor,
    seq_off: int,
    seq_size: int,
    *,
    batch: int,
    num_heads_kv: int,
    capacity: int,
    head_dim: int,
    dtype: torch.dtype,
    no_zeroing: bool = False,
) -> None:
    """Copy a logical window from one packed KV cache to another."""
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_copy_packed_k"):
        raise NotImplementedError("ARK CPU packed KV copy is not available (requires BestLA CPU extension build)")
    kv_dtype = cvt_dtype(dtype)
    cpu_lib.ark_cpu_copy_packed_k(
        dst_cache_k.data_ptr(),
        src_cache_k.data_ptr(),
        kv_dtype,
        batch,
        num_heads_kv,
        capacity,
        head_dim,
        int(seq_off),
        int(seq_size),
        bool(no_zeroing),
    )
    cpu_lib.ark_cpu_copy_packed_v(
        dst_cache_v.data_ptr(),
        src_cache_v.data_ptr(),
        kv_dtype,
        batch,
        num_heads_kv,
        capacity,
        head_dim,
        int(seq_off),
        int(seq_size),
        bool(no_zeroing),
    )


def ark_cpu_copy_packed_kv_from_descriptor(
    descriptor,
    dst_cache_k: torch.Tensor,
    dst_cache_v: torch.Tensor,
    src_cache_k: torch.Tensor,
    src_cache_v: torch.Tensor,
    seq_off: int,
    seq_size: int,
    *,
    no_zeroing: bool = False,
) -> None:
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_copy_packed_k_desc"):
        raise NotImplementedError(
            "ARK CPU packed KV descriptor copy is not available (requires BestLA CPU extension build)"
        )
    cpu_lib.ark_cpu_copy_packed_k_desc(
        dst_cache_k.data_ptr(), src_cache_k.data_ptr(), descriptor, int(seq_off), int(seq_size), bool(no_zeroing)
    )
    cpu_lib.ark_cpu_copy_packed_v_desc(
        dst_cache_v.data_ptr(), src_cache_v.data_ptr(), descriptor, int(seq_off), int(seq_size), bool(no_zeroing)
    )


def ark_cpu_shift_packed_k(
    cache_k: torch.Tensor,
    cossin: torch.Tensor,
    *,
    batch: int,
    num_heads_kv: int,
    capacity: int,
    head_dim: int,
    dtype: torch.dtype,
    seq_keep: int,
) -> None:
    """Apply packed-K shift-RoPE in-place on the BF16 packed cache path."""
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_shift_packed_k"):
        raise NotImplementedError("ARK CPU packed K shift-RoPE is not available (requires BestLA CPU extension build)")
    if cossin.dtype != torch.float16:
        raise ValueError(f"cossin must be float16, got {cossin.dtype}")
    cpu_lib.ark_cpu_shift_packed_k(
        cache_k.data_ptr(),
        cossin.data_ptr(),
        cvt_dtype(dtype),
        batch,
        num_heads_kv,
        capacity,
        head_dim,
        int(seq_keep),
    )


def ark_cpu_shift_packed_k_from_descriptor(
    descriptor,
    cache_k: torch.Tensor,
    cossin: torch.Tensor,
    *,
    seq_keep: int,
) -> None:
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_shift_packed_k_desc"):
        raise NotImplementedError(
            "ARK CPU packed K descriptor shift-RoPE is not available (requires BestLA CPU extension build)"
        )
    if cossin.dtype != torch.float16:
        raise ValueError(f"cossin must be float16, got {cossin.dtype}")
    cpu_lib.ark_cpu_shift_packed_k_desc(cache_k.data_ptr(), cossin.data_ptr(), descriptor, int(seq_keep))


def ark_cpu_bestla_sdpa_packed(
    query: torch.Tensor,
    cache_k: torch.Tensor,
    cache_v: torch.Tensor,
    seq_len_kv: int,
    capacity: int,
    num_heads_kv: int,
    *,
    is_causal: bool = False,
    scale: float | None = None,
    tensor_layout: str = "HND",
) -> torch.Tensor:
    """Internal/experimental BestLA mixed-precision SDPA over a packed K/V cache.

    query must be float32; cache_k/cache_v must be float16 or bfloat16 (produced
    by ark_cpu_packed_kv_alloc + ark_cpu_update_packed_kv).  seq_len_kv is the
    current valid sequence length in the cache (<= capacity).  capacity and
    num_heads_kv must match the values used at allocation time.

    This helper is outside the standard public sdpa() contract and exists for the
    internal mixed-route / packed-cache feature surface.
    """
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_bestla_sdpa_packed"):
        raise NotImplementedError("ARK CPU packed BestLA SDPA is not available (requires BestLA CPU extension build)")

    kv_dtype = cvt_dtype(cache_k.dtype)
    batch, num_heads_q, seq_len_q, head_dim = _attention_shape(query, tensor_layout)
    sm_scale = scale if scale is not None else (head_dim**-0.5)
    output = _empty_attention_output(
        batch, num_heads_q, seq_len_q, head_dim, dtype=query.dtype, device=query.device, tensor_layout=tensor_layout
    )
    q_strides = _attention_strides_qko(query, tensor_layout)
    o_strides = _attention_strides_qko(output, tensor_layout)
    cpu_lib.ark_cpu_bestla_sdpa_packed(
        query.data_ptr(),
        cache_k.data_ptr(),
        cache_v.data_ptr(),
        output.data_ptr(),
        *q_strides,
        *o_strides,
        cvt_dtype(query.dtype),
        kv_dtype,
        batch,
        num_heads_q,
        num_heads_kv,
        seq_len_q,
        seq_len_kv,
        capacity,
        head_dim,
        float(sm_scale),
        is_causal,
    )
    return output


def ark_cpu_bestla_sdpa_packed_from_descriptor(
    descriptor,
    query: torch.Tensor,
    cache_k: torch.Tensor,
    cache_v: torch.Tensor,
    seq_len_kv: int,
    *,
    is_causal: bool = False,
    scale: float | None = None,
    tensor_layout: str = "HND",
) -> torch.Tensor:
    """Descriptor-based internal/experimental packed BestLA SDPA forward."""
    if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_bestla_sdpa_packed_desc"):
        raise NotImplementedError(
            "ARK CPU packed BestLA SDPA descriptor path is not available (requires BestLA CPU extension build)"
        )
    batch, num_heads_q, seq_len_q, head_dim = _attention_shape(query, tensor_layout)
    if batch != int(descriptor.batch_size) or head_dim != int(descriptor.head_dim):
        raise ValueError("Query shape does not match the packed KV descriptor")
    sm_scale = scale if scale is not None else (head_dim**-0.5)
    output = _empty_attention_output(
        batch, num_heads_q, seq_len_q, head_dim, dtype=query.dtype, device=query.device, tensor_layout=tensor_layout
    )
    q_strides = _attention_strides_qko(query, tensor_layout)
    o_strides = _attention_strides_qko(output, tensor_layout)
    cpu_lib.ark_cpu_bestla_sdpa_packed_desc(
        query.data_ptr(),
        cache_k.data_ptr(),
        cache_v.data_ptr(),
        output.data_ptr(),
        *q_strides,
        *o_strides,
        cvt_dtype(query.dtype),
        descriptor,
        num_heads_q,
        seq_len_q,
        seq_len_kv,
        float(sm_scale),
        bool(is_causal),
    )
    return output


def sageattn(
    q: torch.Tensor,
    k: torch.Tensor,
    v: torch.Tensor,
    tensor_layout: str = "HND",
    is_causal: bool = False,
    sm_scale: float | None = None,
    return_lse: bool = False,
    kernel: str = "v1_pvhalf",
    **kwargs,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
    """SAGE attention dispatcher.

    Signature mirrors ``sageattention.sageattn``.

    Args:
    - q, k, v: Query/Key/Value tensors. Layout selected by ``tensor_layout``.
    - tensor_layout: "HND" or "NHD".
    - is_causal: Whether to apply causal mask.
    - sm_scale: Softmax scale. Uses ``1 / sqrt(head_dim)`` when None.
    - return_lse: If True, returns (O, LSE) tuple.
    - kernel: Which SAGE variant to dispatch to.
        - "v1_pvhalf" (default): PV in half precision (calls ``sagev1``).
        - "v1_pvi8": PV in INT8 precision (calls ``sagev1_pvi8``).
    - kwargs: Forwarded to the underlying kernel (e.g. ``attn_mask``,
      ``dropout_p``, ``enable_gqa``, ``quant_block_size``).

    Returns:
    - O: same layout as the input tensors.
    - (O, LSE): if return_lse is True.
    """
    if kernel == "v1_pvhalf":
        impl = sagev1
    elif kernel == "v1_pvi8":
        impl = sagev1_pvi8
    else:
        raise ValueError(f"Unsupported sageattn kernel={kernel!r}; supported: 'v1_pvhalf', 'v1_pvi8'")

    return impl(
        query=q,
        key=k,
        value=v,
        is_causal=is_causal,
        scale=sm_scale,
        tensor_layout=tensor_layout,
        return_lse=return_lse,
        **kwargs,
    )


from .sparse_attention import (
    _block_map_lut_torch,
    _build_block_causal_mask,
    _build_sparge_preprocess_context,
    _finalize_sparge_preprocess_outputs,
    _from_hnd,
    _sparge_preprocess_topk_torch,
    _sequence_mean_native_layout,
    _slice_sequence_native_layout,
    _to_hnd,
    block_sparse_sdpa,
    sage_sparse,
    sparge_block_map_to_mask,
    sparge_preprocess_topk,
    sparge_sage2_attn_meansim_topk_xpu,
    sparge_sage2_attn_meansim_topk_xpu_sdpa,
)


def sageattn_varlen(
    q: torch.Tensor,
    k: torch.Tensor,
    v: torch.Tensor,
    cu_seqlens_q: torch.Tensor,
    cu_seqlens_k: torch.Tensor,
    max_seqlen_q: int,
    max_seqlen_k: int,
    is_causal: bool = False,
    sm_scale: float | None = None,
    kernel: str = "v1_pvhalf",
    quant_block_size: int = 64,
    return_lse: bool = False,
    smooth_k: bool = True,
    **kwargs,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
    """SAGE attention with variable-length sequences (no padding).

    Q/K/V are flat 3-D tensors: [total_tokens, num_heads, head_dim].
    Per-sequence boundaries given by ``cu_seqlens_q`` / ``cu_seqlens_k``.

    Internally quantizes Q/K (and V for pvi8) to INT8, then dispatches to
    the SAGE varlen kernel.

    Args:
        q:   [total_q, num_heads_q, head_dim] float16/bfloat16
        k:   [total_kv, num_heads_kv, head_dim] float16/bfloat16
        v:   [total_kv, num_heads_kv, head_dim] float16/bfloat16
        cu_seqlens_q: Cumulative sequence lengths for Q [batch+1] int32.
        cu_seqlens_k: Cumulative sequence lengths for K/V [batch+1] int32.
        max_seqlen_q: Maximum Q sequence length.
        max_seqlen_k: Maximum K/V sequence length.
        is_causal: Apply causal mask.
        sm_scale: Softmax scale. Uses 1/sqrt(D) when None.
        kernel: "v1_pvhalf" (PV half) or "v1_pvi8" (PV int8).
        quant_block_size: Block size for INT8 quantization (default 64).
        smooth_k: Whether to smooth the key by subtracting the sequence mean
            before INT8 quantization (handled by the C++ kernel). Default: True.
        **kwargs: Forwarded (attn_mask, dropout_p etc. not yet supported).

    Returns:
        O: [total_q, num_heads_q, head_dim] float16/bfloat16
    """
    if q.device.type != "xpu":
        raise NotImplementedError("sageattn_varlen is only supported on XPU")

    if q.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(f"Q must be float16 or bfloat16, got {q.dtype}")

    if q.ndim != 3 or k.ndim != 3 or v.ndim != 3:
        raise ValueError(f"Q/K/V must be 3-D for varlen; got Q={q.ndim}D, K={k.ndim}D, V={v.ndim}D")

    if cu_seqlens_q.dtype not in (torch.int32, torch.int64):
        raise ValueError(f"cu_seqlens_q must be int32 or int64, got {cu_seqlens_q.dtype}")
    if cu_seqlens_k.dtype not in (torch.int32, torch.int64):
        raise ValueError(f"cu_seqlens_k must be int32 or int64, got {cu_seqlens_k.dtype}")
    if cu_seqlens_q.dim() != 1 or cu_seqlens_k.dim() != 1:
        raise ValueError("cu_seqlens_q and cu_seqlens_k must be 1-D")
    if cu_seqlens_q.device.type != "xpu" or cu_seqlens_k.device.type != "xpu":
        raise ValueError("cu_seqlens_q and cu_seqlens_k must be on XPU")

    batch = cu_seqlens_q.shape[0] - 1
    if cu_seqlens_k.shape[0] - 1 != batch:
        raise ValueError("cu_seqlens_q and cu_seqlens_k must describe the same batch size")

    total_q, Hq, D = q.shape
    total_kv, Hkv, Dk = k.shape
    total_kv_v, Hkv2, Dv = v.shape

    if total_q != cu_seqlens_q[-1].item():
        raise ValueError(f"Q dim-0 ({total_q}) != cu_seqlens_q[-1] ({cu_seqlens_q[-1].item()})")
    if total_kv != cu_seqlens_k[-1].item():
        raise ValueError(f"K dim-0 ({total_kv}) != cu_seqlens_k[-1] ({cu_seqlens_k[-1].item()})")
    if total_kv_v != cu_seqlens_k[-1].item():
        raise ValueError(f"V dim-0 ({total_kv_v}) != cu_seqlens_k[-1] ({cu_seqlens_k[-1].item()})")
    if Hkv != Hkv2 or Dk != Dv:
        raise ValueError("K/V shape mismatch")
    if Dk != D:
        raise ValueError("Head dim mismatch between Q and K/V")
    if D not in (64, 128):
        raise ValueError(f"Unsupported head_dim={D}; only 64, 128 supported for SAGE")

    if kernel not in ("v1_pvhalf", "v1_pvi8"):
        raise ValueError(f"Unsupported kernel={kernel!r}; supported: 'v1_pvhalf', 'v1_pvi8'")

    if quant_block_size <= 0:
        quant_block_size = 1

    use_int8_pv = 1 if kernel == "v1_pvi8" else 0

    # Validate contiguous along head-dim
    if q.stride(2) != 1:
        raise ValueError(f"Q must be contiguous along head-dim axis; got stride {q.stride()}")
    if k.stride(2) != 1:
        raise ValueError(f"K must be contiguous along head-dim axis; got stride {k.stride()}")
    if v.stride(2) != 1:
        raise ValueError(f"V must be contiguous along head-dim axis; got stride {v.stride()}")

    lib = get_lib(q)
    stream = get_stream(q)
    cu_seqlens_q_i32 = cu_seqlens_q.contiguous().to(torch.int32)
    cu_seqlens_k_i32 = cu_seqlens_k.contiguous().to(torch.int32)

    O = torch.empty(total_q, Hq, D, dtype=v.dtype, device=q.device)

    if return_lse:
        # .item() is a device->host sync; illegal during graph capture.
        max_q = max_seqlen_q if _xpu_capturing() else int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())
        if max_seqlen_q < max_q:
            raise ValueError(f"max_seqlen_q ({max_seqlen_q}) < max sequence length in cu_seqlens_q ({max_q})")
        LSE = torch.full(
            (batch, Hq, max_seqlen_q),
            float("-inf"),
            dtype=torch.float32,
            device=q.device,
        )
    else:
        LSE = None
    lib.sagev1_varlen(
        stream,
        q.data_ptr(),
        k.data_ptr(),
        v.data_ptr(),
        O.data_ptr(),
        0,  # mask
        quant_block_size,
        cvt_dtype(q.dtype),
        cvt_dtype(k.dtype),
        cvt_dtype(v.dtype),
        cvt_dtype(O.dtype),
        batch,
        Hq,
        Hkv,
        total_q,
        total_kv,
        max_seqlen_q,
        max_seqlen_k,
        D,
        sm_scale if sm_scale is not None else 1.0 / (D**0.5),
        bool(is_causal),
        cu_seqlens_q_i32.data_ptr(),
        cu_seqlens_k_i32.data_ptr(),
        use_int8_pv,
        bool(smooth_k),
        LSE.data_ptr() if LSE is not None else 0,
    )

    if return_lse:
        return O, LSE
    return O


def sage_dynquant(
    query: torch.Tensor,
    key: torch.Tensor,
    value: torch.Tensor,
    attn_mask: torch.Tensor | None = None,
    dropout_p: float = 0.0,
    is_causal: bool = False,
    scale: float | None = None,
    enable_gqa: bool = False,
    quant_block_size: int = 64,
    tensor_layout: str = "HND",
) -> torch.Tensor:
    """SAGE Attention with dynamic INT8 block-wise quantization of Q/K.

    Takes FP16 Q, K, V inputs. Quantizes Q and K to INT8 per-block
    using a fused SYCL kernel, then calls SAGE V1 with INT8 data.
    API is like SDPA but with an extra quant_block_size parameter.

    Args:
        query: [B, Hq, Sq, D] float16
        key:   [B, Hkv, Skv, D] float16
        value: [B, Hkv, Skv, D] float16
        quant_block_size: Number of tokens sharing one INT8 scale.
            E.g. 64 means 64 consecutive tokens share one absmax.
            0 means per-token (block_size=1).

    Returns:
        O: [B, Hq, Sq, D] float16
    """
    if query.device.type != "xpu":
        raise NotImplementedError("sage_dynquant is only supported on XPU")

    if query.dtype != torch.float16:
        raise ValueError(f"sage_dynquant currently supports only float16 Q/K/V tensors, got {query.dtype}")
    if key.dtype != query.dtype or value.dtype != query.dtype:
        raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")

    B, Hq, Hkv, Sq, Skv, D = _validate_attention_geometry(
        query, key, value, tensor_layout, key_dtype=query.dtype, value_dtype=query.dtype
    )
    if D not in (64, 128):
        raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
    _validate_no_dropout(dropout_p, "sage_dynquant")
    _validate_attention_mask(attn_mask, batch=B, seq_len_q=Sq, seq_len_kv=Skv, device=query.device)

    # block_size=0 means per-token
    block_size = quant_block_size if quant_block_size > 0 else 1

    # SAGE V1 kernel uses K-tile size=32; quant_block_size must be 1 or >=32
    if block_size != 1 and block_size < 32:
        raise ValueError(
            f"quant_block_size={block_size} is not supported. "
            f"Must be 1 (per-token) or >= 32 (e.g. 32, 64, 128, 256)."
        )

    lib = get_lib(query)
    stream = get_stream(query)
    q_blocks = (Sq + block_size - 1) // block_size
    kv_blocks = (Skv + block_size - 1) // block_size
    q_i8 = torch.empty_like(query, dtype=torch.int8)
    q_scale = torch.empty((B, Hq, q_blocks, 1), dtype=torch.float32, device=query.device)
    q_strides = _attention_strides_qko(query, tensor_layout)
    lib.sage_dynamic_quant_layout(
        stream,
        query.data_ptr(),
        0,
        q_i8.data_ptr(),
        q_scale.data_ptr(),
        B,
        Hq,
        Sq,
        D,
        block_size,
        *q_strides,
    )

    k_i8 = torch.empty_like(key, dtype=torch.int8)
    k_scale = torch.empty((B, Hkv, kv_blocks, 1), dtype=torch.float32, device=key.device)
    k_strides = _attention_strides_qko(key, tensor_layout)
    lib.sage_dynamic_quant_layout(
        stream,
        key.data_ptr(),
        0,
        k_i8.data_ptr(),
        k_scale.data_ptr(),
        B,
        Hkv,
        Skv,
        D,
        block_size,
        *k_strides,
    )

    # Call SAGE v1 with matching quant_block_size
    return sage(
        q_i8,
        k_i8,
        value,
        attn_mask=attn_mask,
        dropout_p=dropout_p,
        is_causal=is_causal,
        scale=scale,
        enable_gqa=enable_gqa,
        quant_block_size=block_size,
        qscale=q_scale,
        kscale=k_scale,
        tensor_layout=tensor_layout,
    )


def moe_gemm_decode(
    activations: torch.Tensor,
    weights: torch.Tensor,
    num_tokens_per_expert: torch.Tensor,
    *,
    scales: torch.Tensor | None = None,
    zeros: torch.Tensor | None = None,
    weight_bits: int = 4,
    group_size: int = 128,
    asym: bool = False,
) -> torch.Tensor:
    """MoE GEMV optimized for the decode phase.

    Each expert typically processes only 1-2 tokens (top-k routing with
    small batch). Activations must already be gathered/sorted by expert
    (same convention as ``moe_gemm``).

    Args:
        activations: ``[total_tokens, K]`` in fp16 or bf16.
        weights: 3-D tensor ``[E, N, K_packed]``. The accepted layouts are:

            * Unquantized (``weight_bits=16``): ``torch.float16`` / ``torch.bfloat16``
              matching the activations dtype, ``K_packed == K``.
            * Int8 (``weight_bits=8``): ``torch.uint8``, ``K_packed == K``.
              Sym (``asym=False``) reinterprets each byte as signed int8;
              asym (``asym=True``) treats each byte as ``uint8`` with a
              per-group zero-point.
            * Int4 (``weight_bits=4``): ``torch.uint8`` packed,
              ``K_packed == K // 2`` (two 4-bit values per byte; low nibble
              at the lower K index).
            * Int2 (``weight_bits=2``): ``torch.uint8`` packed,
              ``K_packed == K // 4`` (four 2-bit values per byte; field j at
              K index ``4*i + j`` occupies bits 2j and 2j+1 of byte i).
            * FP8 (``torch.float8_e4m3fn`` / ``torch.float8_e5m2``):
              ``K_packed == K``. ``weight_bits`` is ignored; ``asym`` must
              be ``False`` (no zero-points for FP8).
        num_tokens_per_expert: ``[E]`` int32. Sum must equal
            ``activations.shape[0]``; this is a caller contract. It is checked
            eagerly only when the tensor lives on the host, or when
            ``ARK_MOE_VALIDATE_ROUTING`` is set -- summing a device tensor
            costs a blocking device-to-host sync on the decode hot path.
        scales: ``[E, N, K // group_size]`` in activations dtype. Required
            for all quantized paths (int8/int4/int2/fp8); must be ``None``
            for unquantized weights.
        zeros: ``[E, N, K // group_size]`` in activations dtype. Required
            when ``asym=True`` (int8/int4/int2 only); otherwise ``None``.
        weight_bits: 2, 4, 8, or 16. Ignored when ``weights`` is an FP8
            tensor (the FP8 sub-format is taken from ``weights.dtype``).
        group_size: group along K for quantized weights (default 128).
        asym: if ``True``, weights use unsigned encoding and ``zeros`` must
            be provided. Not supported for FP8.

    Returns:
        outputs: ``[total_tokens, N]`` in the same dtype as activations.
    """
    activations, weights, scales, zeros, num_tokens_per_expert, weight_dtype, total_tokens, N, K, num_experts = (
        _validate_moe_quant_args(
            activations,
            weights,
            num_tokens_per_expert,
            scales=scales,
            zeros=zeros,
            weight_bits=weight_bits,
            group_size=group_size,
            asym=asym,
            api_name="moe_gemm_decode",
        )
    )

    lib = get_lib(activations)
    stream = get_stream(activations)
    outputs = torch.empty((total_tokens, N), device=activations.device, dtype=activations.dtype)
    # Scratch buffer mapping each token to its expert id; filled on-device
    # inside the kernel wrapper so we avoid host-device sync.
    expert_id_per_token = torch.empty((total_tokens,), device=activations.device, dtype=torch.int32)

    scales_ptr = scales.data_ptr() if scales is not None else 0
    zeros_ptr = zeros.data_ptr() if zeros is not None else 0

    lib.moe_gemm_decode(
        stream,
        activations.data_ptr(),
        weights.data_ptr(),
        scales_ptr,
        zeros_ptr,
        outputs.data_ptr(),
        expert_id_per_token.data_ptr(),
        cvt_dtype(activations.dtype),
        weight_dtype,
        N,
        K,
        group_size,
        num_tokens_per_expert.data_ptr(),
        num_experts,
        total_tokens,
        bool(asym),
    )
    return outputs


def moe_decode_release_scratch() -> None:
    """Release the device scratch buffers held by the int4 decode fallbacks.

    :func:`moe_gemm_decode` serves its int4 weight-repack and activation-sum
    buffers from grow-on-demand per-queue slabs that are kept for the lifetime
    of the process so the decode hot path never allocates. Call this to hand
    that memory back, or to drop a repack cached via
    ``ARK_MOE_DECODE_INT4_REPACK_CACHE=1`` before the underlying weight tensor
    is freed. A no-op when the XPU extension is not loaded.
    """
    lib = xpu_lib
    if lib is None or not hasattr(lib, "moe_decode_release_scratch"):
        return
    lib.moe_decode_release_scratch()


def moe_routing_validation_enabled() -> bool:
    """Whether ``num_tokens_per_expert`` is checked against ``total_tokens``.

    The check needs the *sum* of the routing table, which for a table that
    already lives on the device costs a reduction kernel plus a blocking
    device-to-host copy -- a full pipeline flush on every call. Decode issues
    one call per generated token, so that sync lands directly in the
    token-latency path (and inside the timed region of the decode benchmarks),
    where it is worth tens of microseconds against kernels that take ~150us.

    So the check runs unconditionally for host-side (CPU) routing tables, where
    it is free, and is skipped for device tables unless
    ``ARK_MOE_VALIDATE_ROUTING`` is set to a truthy value. The C++ side does not
    need the host value: it consumes the device pointer directly and derives
    ``expert_id_per_token`` on-device, clamped to ``num_experts - 1``.

    Truthy values (case-insensitive): anything other than "0", "false", "off",
    "no". Unset means disabled (no sync).
    """
    env = os.environ.get("ARK_MOE_VALIDATE_ROUTING")
    if env is None:
        return False
    return env.strip().lower() not in ("0", "false", "off", "no")


def _check_routing_total(num_tokens_per_expert: torch.Tensor, total_tokens: int) -> None:
    """Check ``sum(num_tokens_per_expert) == total_tokens`` without a device sync.

    See :func:`moe_routing_validation_enabled` for when the check is skipped.
    """
    if num_tokens_per_expert.device.type != "cpu" and not moe_routing_validation_enabled():
        return
    expected_total = int(num_tokens_per_expert.sum().item())
    if expected_total != total_tokens:
        raise ValueError(f"Sum of num_tokens_per_expert ({expected_total}) != total_tokens ({total_tokens})")


def _validate_moe_quant_args(
    activations: torch.Tensor,
    weights: torch.Tensor,
    num_tokens_per_expert: torch.Tensor,
    *,
    scales: torch.Tensor | None,
    zeros: torch.Tensor | None,
    weight_bits: int,
    group_size: int,
    asym: bool,
    api_name: str,
):
    """Shared validation/normalisation for quantized MoE entry points.

    Returns a tuple of normalised tensors and dtype/shape metadata used by the
    kernel-call site:
        ``(activations, weights, scales, zeros, num_tokens_per_expert,
           weight_dtype, total_tokens, N, K, num_experts)``.

    The caller owns the contract that ``num_tokens_per_expert`` sums to
    ``activations.shape[0]``; see :func:`moe_routing_validation_enabled` for how
    that is (or is not) enforced.
    """
    if activations.device.type != "xpu":
        raise NotImplementedError(f"{api_name} is only supported on XPU")

    if activations.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(f"activations must be fp16/bf16, got {activations.dtype}")

    if activations.ndim != 2:
        raise ValueError("activations must be 2D [total_tokens, K]")
    if weights.ndim != 3:
        raise ValueError("weights must be 3D [E, N, K_packed]")

    if not activations.is_contiguous():
        activations = activations.contiguous()
    if not weights.is_contiguous():
        weights = weights.contiguous()

    if num_tokens_per_expert.dtype != torch.int32:
        num_tokens_per_expert = num_tokens_per_expert.to(torch.int32)
    if not num_tokens_per_expert.is_contiguous():
        num_tokens_per_expert = num_tokens_per_expert.contiguous()

    total_tokens, K = activations.shape
    num_experts = weights.shape[0]
    N = weights.shape[1]

    if num_tokens_per_expert.shape[0] != num_experts:
        raise ValueError(f"num_tokens_per_expert length {num_tokens_per_expert.shape[0]} != num_experts {num_experts}")

    # Detect FP8 weight dtype first (overrides weight_bits).
    is_fp8 = weights.dtype in (torch.float8_e4m3fn, torch.float8_e5m2)

    if is_fp8:
        if asym:
            raise ValueError("FP8 weights do not support asym=True")
        if weights.shape[2] != K:
            raise ValueError(f"FP8 weights K dim {weights.shape[2]} != activations K {K}")
        if scales is None:
            raise ValueError("scales is required for FP8 weights")
        if scales.dtype != activations.dtype:
            raise ValueError("scales dtype must match activations dtype")
        if K % group_size != 0:
            raise ValueError("K must be a multiple of group_size")
        expected_scale_shape = (num_experts, N, K // group_size)
        if tuple(scales.shape) != expected_scale_shape:
            raise ValueError(f"scales shape {tuple(scales.shape)} != expected {expected_scale_shape}")
        if zeros is not None:
            raise ValueError("zeros must be None for FP8 weights")
        weight_dtype = ARK_DT.float8_e4m3 if weights.dtype == torch.float8_e4m3fn else ARK_DT.float8_e5m2
        if not scales.is_contiguous():
            scales = scales.contiguous()
    elif weight_bits == 16:
        if weights.dtype != activations.dtype:
            raise ValueError("Unquantized weights must match activations dtype")
        if weights.shape[2] != K:
            raise ValueError(f"Unquantized weights K dim {weights.shape[2]} != activations K {K}")
        weight_dtype = cvt_dtype(activations.dtype)
        if scales is not None or zeros is not None:
            raise ValueError("scales/zeros must be None when weight_bits=16")
    elif weight_bits in (8, 4, 2):
        if weights.dtype != torch.uint8:
            raise ValueError(f"Int{weight_bits} packed weights must be torch.uint8")
        if weight_bits == 8:
            k_packed_expected = K
            k_div = 1
        elif weight_bits == 4:
            k_packed_expected = K // 2
            k_div = 2
        else:  # weight_bits == 2
            k_packed_expected = K // 4
            k_div = 4
        if K % k_div != 0:
            raise ValueError(f"K must be a multiple of {k_div} for weight_bits={weight_bits}")
        if weights.shape[2] != k_packed_expected:
            raise ValueError(
                f"Int{weight_bits} packed weights last dim {weights.shape[2]} must equal K/{k_div} "
                f"({k_packed_expected})"
            )
        if scales is None:
            raise ValueError(f"scales is required for int{weight_bits} weights")
        if scales.dtype != activations.dtype:
            raise ValueError("scales dtype must match activations dtype")
        if K % group_size != 0:
            raise ValueError("K must be a multiple of group_size")
        # Group_size constraints per dtype.
        if weight_bits == 4 and (group_size & 1) != 0:
            raise ValueError("group_size must be even for int4 weights")
        if weight_bits == 2 and (group_size & 3) != 0:
            raise ValueError("group_size must be a multiple of 4 for int2 weights")
        expected_scale_shape = (num_experts, N, K // group_size)
        if tuple(scales.shape) != expected_scale_shape:
            raise ValueError(f"scales shape {tuple(scales.shape)} != expected {expected_scale_shape}")
        if asym:
            if zeros is None:
                raise ValueError("zeros is required when asym=True")
            if zeros.dtype != activations.dtype:
                raise ValueError("zeros dtype must match activations dtype")
            if tuple(zeros.shape) != expected_scale_shape:
                raise ValueError(f"zeros shape {tuple(zeros.shape)} != expected {expected_scale_shape}")
        else:
            if zeros is not None:
                raise ValueError("zeros must be None when asym=False")
        weight_dtype = {8: ARK_DT.int8, 4: ARK_DT.int4, 2: ARK_DT.int2}[weight_bits]
        if not scales.is_contiguous():
            scales = scales.contiguous()
        if asym and not zeros.is_contiguous():
            zeros = zeros.contiguous()
    else:
        raise ValueError(f"Unsupported weight_bits={weight_bits} (supported: 2, 4, 8, 16)")

    if N % 16 != 0:
        raise ValueError(f"N must be a multiple of 16 (got {N})")

    _check_routing_total(num_tokens_per_expert, total_tokens)

    return (activations, weights, scales, zeros, num_tokens_per_expert, weight_dtype, total_tokens, N, K, num_experts)


@dataclass(eq=False)
class MoeSymmetricGemm:
    """Prepared symmetric quantized MoE GEMM runtime state.

    This is an internal fast-path contract for callers that already validated
    fixed layer state during model load. Dynamic per-call tensors must still
    match the prepared shape/dtype contract.
    """

    lib: object
    weights: torch.Tensor
    scales: torch.Tensor
    prefill_workspace: torch.Tensor
    weight_bits: int
    group_size: int
    act_dtype: int
    weight_dtype: int
    N: int
    K: int
    num_experts: int
    weights_ptr: int
    scales_ptr: int
    zeros_ptr: int = 0
    decode_expert_id_per_token: torch.Tensor | None = None

    @classmethod
    def prepare(
        cls,
        weights: torch.Tensor,
        scales: torch.Tensor | None,
        *,
        weight_bits: int = 4,
        group_size: int = 128,
        activation_dtype: torch.dtype | None = None,
        max_decode_tokens: int = 0,
    ) -> "MoeSymmetricGemm":
        """Prepare the symmetric quantized MoE GEMM fast path.

        Call this once after weights/scales are finalized, e.g. from
        ``process_weights_after_loading``. It performs the fixed validation
        from ``moe_gemm_decode``/``moe_gemm_prefill`` and caches metadata and
        pointers that do not depend on the current token count.

        The returned object is for symmetric quantized weights only:
        ``zeros=None`` and ``asym=False``. Dynamic inputs passed to the
        prepared call sites must be contiguous ``[total_tokens, K]`` fp16/bf16
        tensors and an int32 routing table of length ``num_experts``.
        """
        if weights.device.type != "xpu":
            raise NotImplementedError("MoeSymmetricGemm.prepare is only supported on XPU")
        if weights.ndim != 3:
            raise ValueError("weights must be 3D [E, N, K_packed]")
        if weight_bits not in (2, 4, 8):
            raise ValueError(f"MoeSymmetricGemm.prepare supports weight_bits 2, 4, or 8, got {weight_bits}")
        if weights.dtype != torch.uint8:
            raise ValueError(f"Int{weight_bits} packed weights must be torch.uint8")
        if group_size <= 0:
            raise ValueError(f"group_size must be positive, got {group_size}")
        if max_decode_tokens < 0:
            raise ValueError(f"max_decode_tokens must be non-negative, got {max_decode_tokens}")

        if not weights.is_contiguous():
            weights = weights.contiguous()

        num_experts, N, K_packed = weights.shape
        pack_factor = 8 // weight_bits
        K = K_packed * pack_factor

        if N % 16 != 0:
            raise ValueError(f"N must be a multiple of 16 (got {N})")
        if K % group_size != 0:
            raise ValueError("K must be a multiple of group_size")
        if weight_bits == 4 and (group_size & 1) != 0:
            raise ValueError("group_size must be even for int4 weights")
        if weight_bits == 2 and (group_size & 3) != 0:
            raise ValueError("group_size must be a multiple of 4 for int2 weights")

        if scales is None:
            raise ValueError(f"scales is required for int{weight_bits} weights")
        if scales.device != weights.device:
            raise ValueError(f"scales must be on the same device as weights, got {scales.device} vs {weights.device}")

        if activation_dtype is None:
            activation_dtype = scales.dtype
        if activation_dtype not in (torch.float16, torch.bfloat16):
            raise ValueError(f"activation_dtype must be fp16/bf16, got {activation_dtype}")
        if scales.dtype != activation_dtype:
            raise ValueError("scales dtype must match activation_dtype")

        expected_scale_shape = (num_experts, N, K // group_size)
        if tuple(scales.shape) != expected_scale_shape:
            raise ValueError(f"scales shape {tuple(scales.shape)} != expected {expected_scale_shape}")
        if not scales.is_contiguous():
            scales = scales.contiguous()

        prefill_workspace = _get_moe_prefill_workspace(weights.device, activation_dtype, num_experts, K, N)
        decode_scratch = (
            torch.empty((max_decode_tokens,), device=weights.device, dtype=torch.int32)
            if max_decode_tokens > 0
            else None
        )

        return cls(
            lib=get_lib(weights),
            weights=weights,
            scales=scales,
            prefill_workspace=prefill_workspace,
            weight_bits=weight_bits,
            group_size=group_size,
            act_dtype=cvt_dtype(activation_dtype),
            weight_dtype={8: ARK_DT.int8, 4: ARK_DT.int4, 2: ARK_DT.int2}[weight_bits],
            N=N,
            K=K,
            num_experts=num_experts,
            weights_ptr=weights.data_ptr(),
            scales_ptr=scales.data_ptr(),
            zeros_ptr=0,
            decode_expert_id_per_token=decode_scratch,
        )

    def get_decode_scratch(
        self,
        total_tokens: int,
        device: torch.device,
    ) -> torch.Tensor:
        scratch = self.decode_expert_id_per_token
        if scratch is None or scratch.device != device or scratch.shape[0] < total_tokens:
            scratch = torch.empty((total_tokens,), device=device, dtype=torch.int32)
            self.decode_expert_id_per_token = scratch
        return scratch[:total_tokens]

    def decode(
        self,
        activations: torch.Tensor,
        num_tokens_per_expert: torch.Tensor,
        *,
        outputs: torch.Tensor | None = None,
        expert_id_per_token: torch.Tensor | None = None,
    ) -> torch.Tensor:
        total_tokens = int(activations.shape[0])
        if outputs is None:
            outputs = torch.empty((total_tokens, self.N), device=activations.device, dtype=activations.dtype)
        else:
            outputs = outputs[:total_tokens]
        if expert_id_per_token is None:
            expert_id_per_token = self.get_decode_scratch(total_tokens, activations.device)
        else:
            expert_id_per_token = expert_id_per_token[:total_tokens]

        stream = get_stream(activations)
        self.lib.moe_gemm_decode(
            stream,
            activations.data_ptr(),
            self.weights_ptr,
            self.scales_ptr,
            self.zeros_ptr,
            outputs.data_ptr(),
            expert_id_per_token.data_ptr(),
            self.act_dtype,
            self.weight_dtype,
            self.N,
            self.K,
            self.group_size,
            num_tokens_per_expert.data_ptr(),
            self.num_experts,
            total_tokens,
            False,
        )
        return outputs

    def prefill(
        self,
        activations: torch.Tensor,
        num_tokens_per_expert: torch.Tensor,
        *,
        outputs: torch.Tensor | None = None,
        workspace: torch.Tensor | None = None,
    ) -> torch.Tensor:
        total_tokens = int(activations.shape[0])
        if outputs is None:
            outputs = torch.empty((total_tokens, self.N), device=activations.device, dtype=activations.dtype)
        else:
            outputs = outputs[:total_tokens]
        if workspace is None:
            workspace = self.prefill_workspace

        stream = get_stream(activations)
        self.lib.moe_gemm_prefill(
            stream,
            activations.data_ptr(),
            self.weights_ptr,
            self.scales_ptr,
            self.zeros_ptr,
            outputs.data_ptr(),
            workspace.data_ptr(),
            self.act_dtype,
            self.weight_dtype,
            self.N,
            self.K,
            self.group_size,
            num_tokens_per_expert.data_ptr(),
            self.num_experts,
            total_tokens,
            False,
        )
        return outputs

    def moe(
        self,
        activations: torch.Tensor,
        num_tokens_per_expert: torch.Tensor,
        *,
        phase: str = "auto",
        decode_threshold: int | None = None,
        outputs: torch.Tensor | None = None,
        expert_id_per_token: torch.Tensor | None = None,
        workspace: torch.Tensor | None = None,
    ) -> torch.Tensor:
        if phase not in _MOE_VALID_PHASES:
            raise ValueError(f"phase must be one of {_MOE_VALID_PHASES}, got {phase!r}")

        if phase == "auto":
            threshold = _moe_auto_decode_max_total_tokens() if decode_threshold is None else int(decode_threshold)
            phase = "decode" if int(activations.shape[0]) <= threshold else "prefill"

        if phase == "decode":
            return self.decode(
                activations,
                num_tokens_per_expert,
                outputs=outputs,
                expert_id_per_token=expert_id_per_token,
            )
        return self.prefill(
            activations,
            num_tokens_per_expert,
            outputs=outputs,
            workspace=workspace,
        )

    def apply(
        self,
        activations: torch.Tensor,
        num_tokens_per_expert: torch.Tensor,
        *,
        phase: str = "auto",
        decode_threshold: int | None = None,
        outputs: torch.Tensor | None = None,
        expert_id_per_token: torch.Tensor | None = None,
        workspace: torch.Tensor | None = None,
    ) -> torch.Tensor:
        return self.moe(
            activations,
            num_tokens_per_expert,
            phase=phase,
            decode_threshold=decode_threshold,
            outputs=outputs,
            expert_id_per_token=expert_id_per_token,
            workspace=workspace,
        )


def moe_gemm(
    activations: torch.Tensor,
    weights: torch.Tensor,
    num_tokens_per_expert: torch.Tensor,
    *,
    scales: torch.Tensor | None = None,
) -> torch.Tensor:
    """MOE GEMM (Mixture of Experts Grouped GEMM).

    Computes grouped GEMM for MOE layers where different experts process
    different numbers of tokens.

    Expects contiguous layouts:
    - activations: [total_tokens, K] (BF16/FP16)
    - weights: [num_experts, K, N] (BF16/FP16, Row major)
    - num_tokens_per_expert: [num_experts] (int32)
    - scales (optional): [num_experts, N] or None

    Returns:
    - outputs: [total_tokens, N] (same dtype as activations)
    """
    if activations.device.type != "xpu":
        raise NotImplementedError("moe_gemm is only supported on XPU")

    if activations.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(f"activations must be fp16/bf16, got {activations.dtype}")
    if weights.dtype != activations.dtype:
        raise ValueError("weights dtype must match activations dtype")

    if activations.ndim != 2 or weights.ndim != 3:
        raise ValueError("activations must be 2D [total_tokens, K], weights must be 3D [num_experts, K, N]")

    if not activations.is_contiguous() or not weights.is_contiguous():
        raise ValueError("activations and weights must be contiguous")

    if num_tokens_per_expert.dtype != torch.int32:
        num_tokens_per_expert = num_tokens_per_expert.to(torch.int32)

    if not num_tokens_per_expert.is_contiguous():
        num_tokens_per_expert = num_tokens_per_expert.contiguous()

    total_tokens, K = activations.shape
    num_experts, K_w, N = weights.shape  # weights are [num_experts, K, N]

    if K != K_w:
        raise ValueError(f"K dimension mismatch: activations K={K}, weights K={K_w}")

    if num_tokens_per_expert.shape[0] != num_experts:
        raise ValueError(f"num_tokens_per_expert length {num_tokens_per_expert.shape[0]} != num_experts {num_experts}")

    # Validate total tokens
    _check_routing_total(num_tokens_per_expert, total_tokens)

    lib = get_lib(activations)
    stream = get_stream(activations)
    outputs = torch.empty((total_tokens, N), device=activations.device, dtype=activations.dtype)

    scales_ptr = scales.data_ptr() if scales is not None else 0

    lib.moe_gemm(
        stream,
        activations.data_ptr(),
        weights.data_ptr(),
        scales_ptr,
        outputs.data_ptr(),
        cvt_dtype(activations.dtype),
        N,
        K,
        num_tokens_per_expert.data_ptr(),
        num_experts,
    )
    return outputs


def _dpas_fp8_prefill_enabled() -> bool:
    """Return True unless ``ARK_MOE_PREFILL_DPAS_FP8`` is explicitly falsy.

    Mirrors the C++-side default used by the Variant B dispatcher in
    `sycl_tla_moe_mixed.hpp` (default ON). This helper is exposed so
    Python callers/tests can query the effective state.

    Falsy values (case-insensitive): "0", "false", "off", "no".
    Any other value (including unset) is treated as enabled.
    """
    env = os.environ.get("ARK_MOE_PREFILL_DPAS_FP8")
    if env is None:
        return True
    return env.strip().lower() not in ("0", "false", "off", "no")


def _moe_gemm_prefill_fp8_pertensor(
    activations: torch.Tensor,
    weights: torch.Tensor,
    num_tokens_per_expert: torch.Tensor,
    *,
    scales: torch.Tensor,
) -> torch.Tensor:
    """Variant A per-tensor FP8 DPAS grouped GEMM.

    Weights: ``[E, K, N]`` row-major FP8 (``float8_e4m3fn`` / ``float8_e5m2``).
    Scales:  ``[E]`` FP32, one per-tensor scale per expert.
    Activations / outputs: ``[total_tokens, K]`` / ``[total_tokens, N]`` in
    F16 or BF16.

    STATUS: NEEDS-HARDWARE-VALIDATION.
    """
    if scales is None:
        raise ValueError("moe_gemm_prefill(scale_scheme='per_tensor'): scales is required")
    if weights.dtype not in (torch.float8_e4m3fn, torch.float8_e5m2):
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): weights must be FP8 "
            f"(float8_e4m3fn or float8_e5m2), got {weights.dtype}"
        )
    if activations.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): activations must be F16 or BF16, " f"got {activations.dtype}"
        )
    if weights.ndim != 3:
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): weights must be 3-D [E, K, N], "
            f"got shape {tuple(weights.shape)}"
        )
    num_experts, K, N = weights.shape
    if activations.ndim != 2 or activations.shape[1] != K:
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): activations must be [total_tokens, K] "
            f"with K={K}, got shape {tuple(activations.shape)}"
        )
    if num_tokens_per_expert.ndim != 1 or int(num_tokens_per_expert.shape[0]) != num_experts:
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): num_tokens_per_expert must be [E] "
            f"with E={num_experts}, got shape {tuple(num_tokens_per_expert.shape)}"
        )
    if scales.dtype != torch.float32 or scales.shape != (num_experts,):
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): scales must be FP32 with shape [E] "
            f"(E={num_experts}), got dtype={scales.dtype} shape={tuple(scales.shape)}"
        )

    total_tokens = int(activations.shape[0])
    lib = get_lib(activations)
    if not hasattr(lib, "moe_gemm_prefill_fp8_dpas"):
        raise RuntimeError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): the C++ backend was built without the "
            "`moe_gemm_prefill_fp8_dpas` symbol. Rebuild auto_round_extension with sycl-tla "
            "support (needs Intel BMG/PVC + cutlass-sycl)."
        )
    stream = get_stream(activations)
    outputs = torch.empty((total_tokens, N), device=activations.device, dtype=activations.dtype)

    lib.moe_gemm_prefill_fp8_dpas(
        stream,
        activations.data_ptr(),
        weights.data_ptr(),
        scales.data_ptr(),
        outputs.data_ptr(),
        cvt_dtype(activations.dtype),
        cvt_dtype(weights.dtype),
        N,
        K,
        num_tokens_per_expert.data_ptr(),
        num_experts,
        total_tokens,
    )
    return outputs


def _moe_gemm_prefill_int_pertensor(
    activations: torch.Tensor,
    weights: torch.Tensor,
    num_tokens_per_expert: torch.Tensor,
    *,
    scales: torch.Tensor,
) -> torch.Tensor:
    """Variant A per-tensor INT8 DPAS grouped GEMM.

    Storage-only INT8 sibling of :func:`_moe_gemm_prefill_fp8_pertensor`:
    weights are held as one signed byte per element and the DPAS atom
    still runs on the activation dtype after an in-register
    ``int8 -> ElementA`` upcast in the kernel mainloop. Numerically this
    is equivalent to dequantizing to bf16/fp16 and calling the W16A16
    grouped GEMM, but the mainloop avoids materialising the
    ``[E, K, N]`` bf16/fp16 workspace.

    Weights: ``[E, K, N]`` row-major ``torch.int8`` (sym per-tensor,
    matches the vllm-xpu-kernels FP8 layout modulo dtype).
    Scales:  ``[E]`` FP32, one per-tensor scale per expert.
    Activations / outputs: ``[total_tokens, K]`` / ``[total_tokens, N]`` in
    F16 or BF16.

    STATUS: NEEDS-HARDWARE-VALIDATION.
    """
    if scales is None:
        raise ValueError("moe_gemm_prefill(scale_scheme='per_tensor'): scales is required")
    if weights.dtype != torch.int8:
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): weights must be torch.int8, " f"got {weights.dtype}"
        )
    if activations.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): activations must be F16 or BF16, " f"got {activations.dtype}"
        )
    if weights.ndim != 3:
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): weights must be 3-D [E, K, N], "
            f"got shape {tuple(weights.shape)}"
        )
    num_experts, K, N = weights.shape
    if activations.ndim != 2 or activations.shape[1] != K:
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): activations must be [total_tokens, K] "
            f"with K={K}, got shape {tuple(activations.shape)}"
        )
    if num_tokens_per_expert.ndim != 1 or int(num_tokens_per_expert.shape[0]) != num_experts:
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): num_tokens_per_expert must be [E] "
            f"with E={num_experts}, got shape {tuple(num_tokens_per_expert.shape)}"
        )
    if scales.dtype != torch.float32 or scales.shape != (num_experts,):
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): scales must be FP32 with shape [E] "
            f"(E={num_experts}), got dtype={scales.dtype} shape={tuple(scales.shape)}"
        )

    total_tokens = int(activations.shape[0])
    lib = get_lib(activations)
    if not hasattr(lib, "moe_gemm_prefill_int_dpas"):
        raise RuntimeError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): the C++ backend was built without the "
            "`moe_gemm_prefill_int_dpas` symbol. Rebuild auto_round_extension with sycl-tla "
            "support (needs Intel BMG/PVC + cutlass-sycl)."
        )
    stream = get_stream(activations)
    outputs = torch.empty((total_tokens, N), device=activations.device, dtype=activations.dtype)

    lib.moe_gemm_prefill_int_dpas(
        stream,
        activations.data_ptr(),
        weights.data_ptr(),
        scales.data_ptr(),
        outputs.data_ptr(),
        cvt_dtype(activations.dtype),
        cvt_dtype(weights.dtype),
        N,
        K,
        num_tokens_per_expert.data_ptr(),
        num_experts,
        total_tokens,
    )
    return outputs


def moe_gemm_prefill(
    activations: torch.Tensor,
    weights: torch.Tensor,
    num_tokens_per_expert: torch.Tensor,
    *,
    scales: torch.Tensor | None = None,
    zeros: torch.Tensor | None = None,
    weight_bits: int = 4,
    group_size: int = 128,
    asym: bool = False,
    scale_scheme: str | None = None,
) -> torch.Tensor:
    """MoE Grouped GEMM optimized for the prefill phase, supporting all weight
    encodings of ``moe_gemm_decode`` (FP16/BF16, INT8 sym/asym, INT4 sym/asym,
    INT2 sym/asym, FP8 E4M3/E5M2).

    The argument shapes/dtypes match :func:`moe_gemm_decode` exactly so the same
    quantized weights/scales/zeros tensors can be re-used between prefill and
    decode without reshaping. Internally, for the quantized paths the kernel
    materialises a ``[E, K, N]`` ``act_dtype`` temporary via an on-device
    dequantization kernel and then dispatches to the existing CUTLASS-SYCL
    Grouped GEMM (``moe_gemm``). Numerical results are bit-identical to
    ``moe_gemm`` applied to the same dequantized weights.

    Args:
        activations: ``[total_tokens, K]`` in fp16 or bf16.
        weights: 3-D tensor; same layout/dtype contract as
            :func:`moe_gemm_decode`. Quantized layouts are ``[E, N, K_packed]``;
            the unquantized fast path (``weight_bits=16``) accepts
            ``[E, N, K]`` -- callers providing already-``[E, K, N]`` weights
            (as ``moe_gemm`` requires) should call ``moe_gemm`` directly.
        num_tokens_per_expert: ``[E]`` int32. Sum must equal
            ``activations.shape[0]`` (see :func:`moe_gemm_decode`).
        scales: ``[E, N, K // group_size]`` in activations dtype. Required for
            quantized paths; ignored (must be ``None``) for unquantized.
        zeros: ``[E, N, K // group_size]`` in activations dtype, required when
            ``asym=True`` (int8/int4/int2 only).
        weight_bits: 2, 4, 8, or 16. Ignored for FP8 weights.
        group_size: group along K for quantized weights (default 128).
        asym: if ``True``, weights use unsigned encoding; ``zeros`` required.
        scale_scheme: optional string selecting the per-tensor scale layout.
            Default (``None``) uses the standard ``[E, N, K // group_size]``
            per-K-group act-dtype scales -- the drop-in scheme that
            auto-round's calibration pipeline produces. When set to
            ``"per_tensor"``, routes to the Variant A per-tensor DPAS
            entry point. In this mode ``weights`` must be ``[E, K, N]``
            row-major (either FP8 -- ``float8_e4m3fn`` / ``float8_e5m2`` --
            or ``torch.int8`` sym) and ``scales`` must be ``[E]`` FP32
            (one per-expert scalar). No dequant workspace is allocated
            on this path.

    Returns:
        outputs: ``[total_tokens, N]`` in the same dtype as activations.
    """
    # ------------------------------------------------------------------
    # Variant A: FP8 / INT8 per-tensor DPAS grouped GEMM.
    #
    # Distinct entry point (not multiplexed through the C++ dispatcher).
    # Weights are `[E, K, N]` row-major (vllm-xpu-kernels convention);
    # scales are `[E]` FP32 (one per-tensor scale per expert). No workspace
    # is allocated -- the fused kernel writes only the final output row.
    #
    # Weight dtype selects the backend:
    #   * FP8  (float8_e4m3fn / float8_e5m2)  -> `moe_gemm_prefill_fp8_dpas`
    #   * INT8 (torch.int8, sym per-tensor)   -> `moe_gemm_prefill_int_dpas`
    #
    # STATUS: NEEDS-HARDWARE-VALIDATION. See
    # `sycl_tla_moe_prefill_{fp8,int}_dpas.hpp` for the ports' provenance
    # & the on-hardware TODOs.
    # ------------------------------------------------------------------
    if scale_scheme is not None:
        if scale_scheme != "per_tensor":
            raise ValueError(f"moe_gemm_prefill: unknown scale_scheme={scale_scheme!r}; expected 'per_tensor' or None")
        if weights.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
            return _moe_gemm_prefill_fp8_pertensor(activations, weights, num_tokens_per_expert, scales=scales)
        if weights.dtype == torch.int8:
            return _moe_gemm_prefill_int_pertensor(activations, weights, num_tokens_per_expert, scales=scales)
        raise ValueError(
            "moe_gemm_prefill(scale_scheme='per_tensor'): weights must be FP8 "
            "(float8_e4m3fn / float8_e5m2) or INT8 (torch.int8), "
            f"got {weights.dtype}"
        )

    activations, weights, scales, zeros, num_tokens_per_expert, weight_dtype, total_tokens, N, K, num_experts = (
        _validate_moe_quant_args(
            activations,
            weights,
            num_tokens_per_expert,
            scales=scales,
            zeros=zeros,
            weight_bits=weight_bits,
            group_size=group_size,
            asym=asym,
            api_name="moe_gemm_prefill",
        )
    )

    lib = get_lib(activations)
    stream = get_stream(activations)
    outputs = torch.empty((total_tokens, N), device=activations.device, dtype=activations.dtype)

    # Native FP8 fused GEMM (Variant A): fp8 weight is upcast to bf16/fp16
    # in registers inside the GEMM kernel and the per-K-group scale is
    # folded into the accumulator. Opt-in via `ARK_MOE_PREFILL_NATIVE_FP8=1`;
    # dispatch is decided entirely inside the C++ `moe_gemm_prefill`
    # dispatcher (which additionally enforces shape preconditions:
    # N % 16 == 0, K % 32 == 0, K % group_size == 0, group_size % 32 == 0).
    # If any precondition fails, or the C++-side env cache disagrees with
    # the Python-side view, the dispatcher silently falls through to the
    # generic dequant path -- which requires the `[E, K, N]` workspace we
    # allocate below. See the `else` branch for details.

    # Quantized paths need an [E, K, N] act-dtype scratch buffer that the
    # on-device dequant kernel fills before the inner Grouped GEMM consumes
    # it. The unquantized fast path forwards directly through `moe_gemm` and
    # doesn't need scratch (passing 0 is safe -- the C++ wrapper short-circuits
    # before touching the workspace pointer in that case). We allocate the
    # workspace from PyTorch's caching allocator so repeated calls reuse the
    # same memory.
    is_unquantized = (weight_bits == 16) and (weights.dtype == activations.dtype)
    if is_unquantized:
        # `moe_gemm` requires `[E, K, N]` row-major weights; the decode-style
        # `[E, N, K]` weight shape coming through this validator can be
        # transposed into a temporary contiguous `[E, K, N]` view. The
        # workspace serves the same role as the dequant scratch so the
        # on-device path stays uniform.
        dequant_workspace = weights.transpose(1, 2).contiguous()
        weights_ptr = dequant_workspace.data_ptr()
        workspace_ptr = dequant_workspace.data_ptr()
    else:
        # Reuse a persistent `[E, K, N]` workspace across calls with the same
        # (device, dtype, E, K, N). For real MoE prefill workloads the same
        # shape is dispatched on every iteration; allocating a fresh
        # `E*K*N*sizeof(act)` tensor each call adds non-trivial caching-
        # allocator overhead (and, on the small shapes, dominates the
        # quantized GEMM cost). The workspace is kept alive by the cache so
        # we hand the data_ptr() to the kernel without taking a new ref.
        #
        # We allocate the workspace unconditionally for all quantized paths,
        # including native FP8. The native FP8 launcher fuses GEMM+scale and
        # never reads the workspace, so the allocation is pure insurance: the
        # C++ dispatcher may silently fall through to the generic dequant
        # path if any of its own preconditions disagree with the caller's
        # opt-in -- e.g. `moe_prefill_native_fp8_enabled()` on
        # the C++ side caches its env value on first call (so runtime env
        # toggling won't propagate), the shape preconditions
        # (`N % 16`, `K % 32`, `K % group_size`, `group_size % 32`) may not
        # hold, or the act dtype may not be F16/BF16. Without a workspace the
        # fall-through would hit the generic null-pointer check in
        # `sycl_tla_moe_mixed.hpp` and raise. Since the workspace lives in
        # the module-level cache, allocation happens once per shape and adds
        # no per-call overhead when the native path is taken.
        dequant_workspace = _get_moe_prefill_workspace(activations.device, activations.dtype, num_experts, K, N)
        weights_ptr = weights.data_ptr()
        workspace_ptr = dequant_workspace.data_ptr()

    scales_ptr = scales.data_ptr() if scales is not None else 0
    zeros_ptr = zeros.data_ptr() if zeros is not None else 0

    lib.moe_gemm_prefill(
        stream,
        activations.data_ptr(),
        weights_ptr,
        scales_ptr,
        zeros_ptr,
        outputs.data_ptr(),
        workspace_ptr,
        cvt_dtype(activations.dtype),
        weight_dtype,
        N,
        K,
        group_size,
        num_tokens_per_expert.data_ptr(),
        num_experts,
        total_tokens,
        bool(asym),
    )
    # The inner CUTLASS-SYCL `moe_gemm` calls `event.wait()` before returning
    # (see `moe_detail::moe_gemm_launcher` in `sycl_tla_moe.hpp`), so by the
    # time `lib.moe_gemm_prefill` returns the device has already consumed the
    # workspace. For the unquantized fast path the workspace is a per-call
    # transposed copy of `weights` -- drop it now. For the quantized paths
    # the workspace lives in the module-level cache (`_get_moe_prefill_workspace`)
    # and is intentionally retained for reuse on the next call. The native
    # fp8 path allocates no workspace at all, so there is nothing to drop.
    if is_unquantized:
        del dequant_workspace
    return outputs


# ---------------------------------------------------------------------------
# `moe_gemm_prefill` dequant-workspace cache.
#
# The Stage-1 quantized prefill kernel dequantises weights into an
# `[E, K, N]` act-dtype scratch buffer before dispatching to the existing
# CUTLASS-SYCL grouped GEMM. In real model usage the same `(E, K, N, dtype)`
# tuple is hit on every prefill step, so allocating a fresh
# `E * K * N * sizeof(act_dtype)` tensor per call adds caching-allocator
# overhead that is significant on the small/medium shapes.
#
# We cache one tensor per `(device, dtype, E, K, N)` key. The cache holds
# references that keep the tensors alive across calls; callers can clear it
# explicitly via `clear_moe_prefill_workspace_cache()` if they need to
# release the memory (e.g., before allocating large buffers for a different
# subsystem).
# ---------------------------------------------------------------------------

_MOE_PREFILL_WORKSPACE_CACHE: "dict[tuple, torch.Tensor]" = {}


def _get_moe_prefill_workspace(device: torch.device, dtype: torch.dtype, E: int, K: int, N: int) -> torch.Tensor:
    """Return a persistent `[E, K, N]` workspace tensor for the prefill kernel.

    The tensor is allocated lazily on first use and retained in a module-level
    cache so subsequent calls with the same `(device, dtype, E, K, N)` reuse
    the same memory. Returned tensors are contiguous and uninitialised; the
    kernel writes every element before reading.
    """
    # `device` may be a `torch.device` or a string; normalise so the cache key
    # is hashable and identifies the exact device (including ordinal).
    if not isinstance(device, torch.device):
        device = torch.device(device)
    key = (device.type, device.index, dtype, int(E), int(K), int(N))
    ws = _MOE_PREFILL_WORKSPACE_CACHE.get(key)
    if ws is None:
        ws = torch.empty((E, K, N), device=device, dtype=dtype)
        _MOE_PREFILL_WORKSPACE_CACHE[key] = ws
    return ws


def clear_moe_prefill_workspace_cache() -> None:
    """Release all cached `moe_gemm_prefill` dequant-workspace tensors."""
    _MOE_PREFILL_WORKSPACE_CACHE.clear()


# ---------------------------------------------------------------------------
# W4A8 MoE: int4 weights, int8 DPAS compute, dynamically quantized int8
# activations.
#
# The int4 weights are converted ONCE into int8 using ARK's `AUTO_S8` re-scale
# rule (see `sycl_tla_moe_w4a8.hpp`):
#
#     sxt[e][n][j] = max_{g in block j} |s[e][n][g]| * 8 / 127
#     w8[e][n][k]  = round(w4[e][n][k] * s[e][n][k // group_size] / sxt[...])
#
# With the default block (`rescale_group_size=-1`, one scale per output
# channel) the GEMM runs a single full-K `s8 x s8 -> s32` accumulation with one
# scalar multiply in the epilogue -- the shape the DPAS pipeline is happiest
# with. Coarser-than-group blocks are what makes this profitable: an int4
# group=32 checkpoint would otherwise force an accumulator fold every 32 K
# elements.
#
# `moe_w4a8` keeps the conversion result in a module-level cache keyed on the
# source tensor identity plus the layout / mutation state and the resolved
# AUTO_S8 block, so repeated forward passes over the same expert weights pay
# for it only once. Callers that manage their own storage should use
# `moe_w4a8_prepack` + `moe_gemm_w4a8` directly.
#
# Cache entries are `(weights_s8, wscales, block, weights, scales)`: the source
# tensors are pinned in the entry because the address half of the key is
# pointer identity, and a freed-then-reallocated buffer can otherwise land on
# the same address. Every MoE layer's expert weights share shape / dtype /
# group_size, so no other key component would catch such a collision.
# ---------------------------------------------------------------------------

_MOE_W4A8_PREPACK_CACHE: "dict[tuple, tuple]" = {}
_MOE_W4A8_MAX_ACCUM_K = (2**31 - 1) // (127 * 127)


def moe_w4a8_rescale_block_size(K: int, group_size: int, rescale_group_size: int = -1) -> int:
    """Return the effective AUTO_S8 re-scale block size used by the W4A8 path.

    ``rescale_group_size <= 0`` (the ``group=-1`` spelling) selects one scale
    per output channel, i.e. a block spanning the whole K axis. Any value that
    is not a multiple of both ``group_size`` and 64, or that does not divide
    ``K``, also falls back to ``K``. ``ARK_MOE_W4A8_AUTO_S8`` overrides the
    argument.

    The result is the divisor of the ``wscales`` last dimension
    (``K // block``), so callers must use it to size that tensor.
    """
    lib = xpu_lib
    if lib is not None and hasattr(lib, "moe_w4a8_rescale_block_size"):
        return int(lib.moe_w4a8_rescale_block_size(int(K), int(group_size), int(rescale_group_size)))
    # Pure-Python mirror of `ark::moe_w4a8::moe_w4a8_rescale_block_size` so the
    # shape math is available without a loaded extension (tests, docs, CPU).
    env = os.environ.get("ARK_MOE_W4A8_AUTO_S8")
    value = int(rescale_group_size)
    if env is not None:
        try:
            value = int(env.strip())
        except ValueError:
            pass
    if K <= 0:
        return K
    if value <= 0 or value >= K:
        return K
    if group_size > 0 and (value < group_size or value % group_size != 0):
        return K
    if K % value != 0 or value % 64 != 0:
        return K
    return value


def _synchronize_xpu_device(device: torch.device) -> None:
    if device.type != "xpu":
        return
    if device.index is None:
        torch.xpu.synchronize()
    else:
        torch.xpu.synchronize(device.index)


def _validate_moe_w4a8_k(K: int, api_name: str) -> None:
    if K > _MOE_W4A8_MAX_ACCUM_K:
        raise ValueError(
            f"{api_name}: K must be <= {_MOE_W4A8_MAX_ACCUM_K} to avoid int32 accumulator overflow (got {K})"
        )


def _require_same_device(tensor: torch.Tensor, reference: torch.Tensor, tensor_name: str, api_name: str) -> None:
    if tensor.device != reference.device:
        raise ValueError(f"{api_name}: {tensor_name} must be on {reference.device}, got {tensor.device}")


def clear_moe_w4a8_prepack_cache() -> None:
    """Release every int8 weight/scale pair cached by :func:`moe_w4a8`.

    Synchronizes the cached XPU devices first so queued kernels cannot outlive
    the cached tensors they still read.
    """
    synced = set()
    for entry in _MOE_W4A8_PREPACK_CACHE.values():
        device = entry[0].device
        if device.type != "xpu":
            continue
        key = (device.type, device.index)
        if key in synced:
            continue
        _synchronize_xpu_device(device)
        synced.add(key)
    _MOE_W4A8_PREPACK_CACHE.clear()


def _validate_moe_w4a8_shape(N: int, K: int, group_size: int, api_name: str) -> None:
    if N % 16 != 0:
        raise ValueError(f"{api_name}: N must be a multiple of 16 (got {N})")
    if K % 64 != 0:
        raise ValueError(f"{api_name}: K must be a multiple of 64 (got {K})")
    _validate_moe_w4a8_k(K, api_name)
    if group_size <= 0 or group_size % 8 != 0:
        raise ValueError(f"{api_name}: group_size must be a positive multiple of 8 (got {group_size})")
    if K % group_size != 0:
        raise ValueError(f"{api_name}: K must be a multiple of group_size")


def moe_w4a8_prepack(
    weights: torch.Tensor,
    scales: torch.Tensor,
    *,
    group_size: int = 128,
    rescale_group_size: int = -1,
) -> "tuple[torch.Tensor, torch.Tensor, int]":
    """Convert packed int4-sym MoE weights into the W4A8 int8 representation.

    Applies ARK's ``AUTO_S8`` re-scale on device, producing weights the int8
    DPAS grouped GEMM / GEMV can consume without any per-K-group fold.

    Args:
        weights: ``[E, N, K // 2]`` ``torch.uint8``, two signed nibbles per
            byte (auto-round's int4-sym packing: K index ``2i`` in the low
            nibble, ``2i + 1`` in the high nibble).
        scales: ``[E, N, K // group_size]`` in fp16 or bf16.
        group_size: quantization group along K of the int4 weights.
        rescale_group_size: AUTO_S8 block size; ``-1`` (default) means one
            scale per output channel -- the fastest configuration.

    Returns:
        ``(weights_s8, wscales, rescale_block_size)`` where ``weights_s8`` is
        ``[E, N, K]`` ``torch.int8``, ``wscales`` is
        ``[E, N, K // rescale_block_size]`` ``torch.float32``, and
        ``rescale_block_size`` is the resolved block size (pass it to
        :func:`moe_gemm_w4a8`).
    """
    if weights.device.type != "xpu":
        raise NotImplementedError("moe_w4a8_prepack is only supported on XPU")
    if weights.dtype != torch.uint8:
        raise ValueError(f"moe_w4a8_prepack: weights must be torch.uint8, got {weights.dtype}")
    if weights.ndim != 3:
        raise ValueError("moe_w4a8_prepack: weights must be 3D [E, N, K // 2]")
    if scales.ndim != 3:
        raise ValueError("moe_w4a8_prepack: scales must be 3D [E, N, K // group_size]")
    if scales.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(f"moe_w4a8_prepack: scales must be fp16/bf16, got {scales.dtype}")
    if scales.device != weights.device:
        raise ValueError("moe_w4a8_prepack: weights and scales must be on the same device")

    E, N, k_packed = weights.shape
    K = k_packed * 2
    _validate_moe_w4a8_shape(N, K, group_size, "moe_w4a8_prepack")
    expected_scale_shape = (E, N, K // group_size)
    if tuple(scales.shape) != expected_scale_shape:
        raise ValueError(f"moe_w4a8_prepack: scales shape {tuple(scales.shape)} != expected {expected_scale_shape}")

    weights = weights.contiguous()
    scales = scales.contiguous()

    block = moe_w4a8_rescale_block_size(K, group_size, rescale_group_size)
    nblk = K // block

    weights_s8 = torch.empty((E, N, K), device=weights.device, dtype=torch.int8)
    wscales = torch.empty((E, N, nblk), device=weights.device, dtype=torch.float32)

    lib = get_lib(weights)
    stream = get_stream(weights)
    lib.moe_w4a8_prepack(
        stream,
        weights.data_ptr(),
        scales.data_ptr(),
        weights_s8.data_ptr(),
        wscales.data_ptr(),
        cvt_dtype(scales.dtype),
        E,
        N,
        K,
        group_size,
        rescale_group_size,
    )
    return weights_s8, wscales, block


def moe_gemm_w4a8(
    activations: torch.Tensor,
    weights_s8: torch.Tensor,
    wscales: torch.Tensor,
    num_tokens_per_expert: torch.Tensor,
    *,
    rescale_block_size: int | None = None,
    phase: str = "auto",
    activation_scale: torch.Tensor | None = None,
    out_dtype: torch.dtype | None = None,
    row_to_token: torch.Tensor | None = None,
    routing_weights: torch.Tensor | None = None,
    output_rows: int | None = None,
) -> torch.Tensor:
    """W4A8 MoE GEMM: int4 weights (pre-converted to int8), int8 compute.

    Activations are dynamically quantized to int8 per token inside the kernel
    (per-row absmax), then multiplied against the ``AUTO_S8`` int8 weights with
    the ``s8 x s8 -> s32`` DPAS atom. The output is
    ``acc * act_scale[token] * weight_scale[n, block]``.

    Two optional contracts cut DRAM traffic by moving work across the call
    boundary; both default off and change nothing when unused.

    *Pre-quantized activations.* Pass an ``int8`` ``activations`` together with
    ``activation_scale`` and the in-call quantization pass is skipped: the
    kernel then never reads the 16-bit activations, never writes the int8 copy
    and never reads it back (``4 * T * K`` bytes, 27% of the qwen3
    down-projection's traffic). The producer of the activations -- typically
    the SiLU/gate elementwise kernel, which already writes ``[T, K]`` once --
    can emit that int8 and its per-row scale directly.

    *Fused top-k reduction.* Pass ``row_to_token``, ``routing_weights`` and
    ``output_rows`` and the epilogue scales each routed row by its routing
    weight and accumulates it into ``out[row_to_token[row]]`` of a
    ``[output_rows, N]`` fp32 tensor, instead of writing the ``[T, N]``
    unreduced result for the caller to reduce afterwards. That replaces
    ``T * N`` written + ``T * N`` read + ``batch * N`` written with a
    ``batch * N`` read-modify-write, and removes the caller's reduction kernel.
    The combiner is a device-scope fp32 atomic add, so the summation order is
    not deterministic and the result is *not* bit-identical to reducing the
    unfused output. Prefill only.

    Args:
        activations: ``[total_tokens, K]`` fp16/bf16, rows sorted by expert, or
            ``[total_tokens, K]`` int8 when ``activation_scale`` is given.
        weights_s8: ``[E, N, K]`` ``torch.int8`` from :func:`moe_w4a8_prepack`.
        wscales: ``[E, N, K // rescale_block_size]`` fp32 from
            :func:`moe_w4a8_prepack`.
        num_tokens_per_expert: ``[E]`` int32; sum must equal ``total_tokens``.
        rescale_block_size: block size the scales were built with. Defaults to
            ``K // wscales.shape[2]``.
        phase: ``"auto"`` (GEMV for small batches, grouped GEMM otherwise),
            ``"decode"`` (force GEMV) or ``"prefill"`` (force grouped GEMM).
        activation_scale: ``[total_tokens]`` fp32 per-row scales
            (``absmax / 127``) for pre-quantized int8 activations.
        out_dtype: output dtype for pre-quantized activations, which carry no
            floating dtype of their own. Defaults to ``torch.bfloat16``.
        row_to_token: ``[total_tokens]`` int32, routed row -> model token.
        routing_weights: ``[total_tokens]`` fp32, the top-k weight of each row.
        output_rows: number of model tokens, i.e. rows of the fused output.

    Returns:
        ``[total_tokens, N]`` in the activations dtype, or ``[output_rows, N]``
        fp32 when the fused reduction is used.
    """
    if phase not in _MOE_VALID_PHASES:
        raise ValueError(f"phase must be one of {_MOE_VALID_PHASES}, got {phase!r}")
    if activations.device.type != "xpu":
        raise NotImplementedError("moe_gemm_w4a8 is only supported on XPU")

    prequantized = activation_scale is not None or activations.dtype == torch.int8
    if prequantized:
        if activations.dtype != torch.int8:
            raise ValueError(f"pre-quantized activations must be int8, got {activations.dtype}")
        if activation_scale is None:
            raise ValueError("int8 activations require activation_scale")
        if activation_scale.dtype != torch.float32:
            raise ValueError(f"activation_scale must be fp32, got {activation_scale.dtype}")
        if activation_scale.numel() != activations.shape[0]:
            raise ValueError(
                f"activation_scale has {activation_scale.numel()} entries, expected {activations.shape[0]}"
            )
        act_dtype = torch.bfloat16 if out_dtype is None else out_dtype
        if act_dtype not in (torch.float16, torch.bfloat16):
            raise ValueError(f"out_dtype must be fp16/bf16, got {act_dtype}")
    elif activations.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(f"activations must be fp16/bf16, got {activations.dtype}")
    else:
        act_dtype = activations.dtype
    if activations.ndim != 2:
        raise ValueError("activations must be 2D [total_tokens, K]")
    if weights_s8.ndim != 3 or weights_s8.dtype != torch.int8:
        raise ValueError("weights_s8 must be a 3D torch.int8 tensor [E, N, K]")
    if wscales.ndim != 3 or wscales.dtype != torch.float32:
        raise ValueError("wscales must be a 3D torch.float32 tensor [E, N, K // block]")
    _require_same_device(weights_s8, activations, "weights_s8", "moe_gemm_w4a8")
    _require_same_device(wscales, activations, "wscales", "moe_gemm_w4a8")
    _require_same_device(num_tokens_per_expert, activations, "num_tokens_per_expert", "moe_gemm_w4a8")
    if prequantized:
        _require_same_device(activation_scale, activations, "activation_scale", "moe_gemm_w4a8")

    activations = activations.contiguous()
    weights_s8 = weights_s8.contiguous()
    wscales = wscales.contiguous()
    if prequantized:
        activation_scale = activation_scale.contiguous()

    total_tokens, K = activations.shape
    num_experts, N, weight_K = weights_s8.shape
    if weight_K != K:
        raise ValueError(f"weights_s8 K dim {weight_K} != activations K {K}")
    if wscales.shape[0] != num_experts or wscales.shape[1] != N:
        raise ValueError(f"wscales shape {tuple(wscales.shape)} incompatible with weights {tuple(weights_s8.shape)}")
    if N % 16 != 0:
        raise ValueError(f"N must be a multiple of 16 (got {N})")
    if K % 64 != 0:
        raise ValueError(f"K must be a multiple of 64 (got {K})")
    _validate_moe_w4a8_k(K, "moe_gemm_w4a8")

    nblk = wscales.shape[2]
    if nblk <= 0 or K % nblk != 0:
        raise ValueError(f"wscales last dim {nblk} must divide K ({K})")
    block = K // nblk if rescale_block_size is None else int(rescale_block_size)
    if block * nblk != K:
        raise ValueError(f"rescale_block_size {block} is inconsistent with wscales last dim {nblk} and K {K}")

    if num_tokens_per_expert.dtype != torch.int32:
        num_tokens_per_expert = num_tokens_per_expert.to(torch.int32)
    if not num_tokens_per_expert.is_contiguous():
        num_tokens_per_expert = num_tokens_per_expert.contiguous()
    if num_tokens_per_expert.shape[0] != num_experts:
        raise ValueError(f"num_tokens_per_expert length {num_tokens_per_expert.shape[0]} != num_experts {num_experts}")
    _check_routing_total(num_tokens_per_expert, total_tokens)

    fused = row_to_token is not None or routing_weights is not None or output_rows is not None
    if fused:
        if row_to_token is None or routing_weights is None or output_rows is None:
            raise ValueError("the fused top-k reduction needs row_to_token, routing_weights and output_rows")
        if phase != "prefill":
            raise ValueError("the fused top-k reduction is prefill-only; pass phase='prefill'")
        if int(output_rows) <= 0:
            raise ValueError(f"output_rows must be positive (got {output_rows})")
        _require_same_device(row_to_token, activations, "row_to_token", "moe_gemm_w4a8")
        _require_same_device(routing_weights, activations, "routing_weights", "moe_gemm_w4a8")
        if row_to_token.dtype != torch.int32:
            row_to_token = row_to_token.to(torch.int32)
        if routing_weights.dtype != torch.float32:
            routing_weights = routing_weights.to(torch.float32)
        row_to_token = row_to_token.contiguous()
        routing_weights = routing_weights.contiguous()
        if row_to_token.numel() != total_tokens or routing_weights.numel() != total_tokens:
            raise ValueError(
                f"row_to_token / routing_weights must have {total_tokens} entries, got "
                f"{row_to_token.numel()} / {routing_weights.numel()}"
            )
        # The epilogue accumulates into this buffer with atomics, so it starts
        # at zero and is fp32 regardless of the activation dtype.
        outputs = torch.zeros((int(output_rows), N), device=activations.device, dtype=torch.float32)
    else:
        outputs = torch.empty((total_tokens, N), device=activations.device, dtype=act_dtype)
    if total_tokens == 0:
        return outputs

    lib = get_lib(activations)
    stream = get_stream(activations)
    lib.moe_gemm_w4a8(
        stream,
        0 if prequantized else activations.data_ptr(),
        weights_s8.data_ptr(),
        wscales.data_ptr(),
        0 if fused else outputs.data_ptr(),
        cvt_dtype(act_dtype),
        N,
        K,
        block,
        num_tokens_per_expert.data_ptr(),
        num_experts,
        total_tokens,
        _MOE_VALID_PHASES.index(phase),
        activations.data_ptr() if prequantized else 0,
        activation_scale.data_ptr() if prequantized else 0,
        row_to_token.data_ptr() if fused else 0,
        routing_weights.data_ptr() if fused else 0,
        outputs.data_ptr() if fused else 0,
        int(output_rows) if fused else 0,
    )
    return outputs


def moe_w4a8_quant_act(activations: torch.Tensor) -> "tuple[torch.Tensor, torch.Tensor]":
    """Quantize activations per token the way :func:`moe_gemm_w4a8` does.

    This is the call's own quantization pass, on its own. Two uses:

    * **Pre-quantization.** The returned pair is exactly what
      ``moe_gemm_w4a8(..., activation_scale=...)`` accepts, so a caller that
      routes the same rows through several experts, or that can produce int8
      upstream, can pay for the pass once instead of once per call.
    * **Measurement.** The pass had no standalone entry point, so its cost
      could only be priced by differencing a 16-bit-input call against a
      pre-quantized one -- two GEMM timings subtracted, with the noise of both
      landing on a number a fraction of their size.

    Args:
        activations: ``[T, K]`` fp16 or bf16 on XPU, ``K % 64 == 0``.

    Returns:
        ``(qact, ascale)``: ``[T, K]`` ``torch.int8`` and ``[T]``
        ``torch.float32`` row scales, ``absmax / 127`` (a row that is entirely
        zero gets scale 0 and quantizes to zeros).
    """
    if activations.device.type != "xpu":
        raise NotImplementedError("moe_w4a8_quant_act is only supported on XPU")
    if activations.dtype not in (torch.float16, torch.bfloat16):
        raise ValueError(f"moe_w4a8_quant_act: activations must be fp16/bf16, got {activations.dtype}")
    if activations.ndim != 2:
        raise ValueError("moe_w4a8_quant_act: activations must be 2D [T, K]")

    total_tokens, K = activations.shape
    if K % 64 != 0:
        raise ValueError(f"moe_w4a8_quant_act: K must be a multiple of 64 (got {K})")

    activations = activations.contiguous()
    qact = torch.empty((total_tokens, K), device=activations.device, dtype=torch.int8)
    ascale = torch.empty((total_tokens,), device=activations.device, dtype=torch.float32)
    if total_tokens == 0:
        return qact, ascale

    lib = get_lib(activations)
    lib.moe_w4a8_quant_act(
        get_stream(activations),
        activations.data_ptr(),
        qact.data_ptr(),
        ascale.data_ptr(),
        cvt_dtype(activations.dtype),
        total_tokens,
        K,
    )
    return qact, ascale


def moe_w4a8(
    activations: torch.Tensor,
    weights: torch.Tensor,
    num_tokens_per_expert: torch.Tensor,
    *,
    scales: torch.Tensor | None = None,
    group_size: int = 128,
    rescale_group_size: int = -1,
    phase: str = "auto",
    cache_prepack: bool = True,
) -> torch.Tensor:
    """W4A8 MoE from auto-round's packed int4-sym weights (prefill + decode).

    Convenience wrapper that runs :func:`moe_w4a8_prepack` (cached on the
    source tensor identity plus layout / mutation state and the resolved
    AUTO_S8 block) and then :func:`moe_gemm_w4a8`. It is a drop-in replacement
    for :func:`moe` on int4-sym weights, trading a small amount of extra
    quantization error on the activations for the int8 DPAS throughput.

    Args:
        activations: ``[total_tokens, K]`` fp16/bf16, rows sorted by expert.
        weights: ``[E, N, K // 2]`` ``torch.uint8`` packed int4-sym weights.
        num_tokens_per_expert: ``[E]`` int32.
        scales: ``[E, N, K // group_size]`` in the activations dtype.
        group_size: quantization group along K (default 128).
        rescale_group_size: AUTO_S8 block size, ``-1`` = per output channel.
        phase: ``"auto"``, ``"decode"`` or ``"prefill"``.
        cache_prepack: keep the converted int8 weights in a module-level cache
            keyed on source identity, layout, mutation version and the resolved
            block size. The cache entry holds strong references to ``weights`` /
            ``scales`` so their addresses cannot be recycled by another tensor
            while the entry lives (see :data:`_MOE_W4A8_PREPACK_CACHE`). Set to
            ``False`` for one-shot use so the (large) int8 copy is released as
            soon as the queued work completes.

    Returns:
        ``[total_tokens, N]`` in the activations dtype.
    """
    if scales is None:
        raise ValueError("moe_w4a8: scales is required for int4 weights")
    if scales.dtype != activations.dtype:
        raise ValueError("moe_w4a8: scales dtype must match activations dtype")

    key = None
    entry = None
    block = None
    if cache_prepack:
        if weights.ndim == 3:
            block = moe_w4a8_rescale_block_size(weights.shape[2] * 2, group_size, rescale_group_size)
        device = weights.device
        key = (
            device.type,
            device.index,
            weights.data_ptr(),
            scales.data_ptr(),
            tuple(weights.shape),
            tuple(scales.shape),
            tuple(weights.stride()),
            tuple(scales.stride()),
            int(weights._version),
            int(scales._version),
            int(group_size),
            int(block) if block is not None else None,
            str(scales.dtype),
        )
        entry = _MOE_W4A8_PREPACK_CACHE.get(key)
    if entry is None:
        weights_s8, wscales, block = moe_w4a8_prepack(
            weights,
            scales,
            group_size=group_size,
            rescale_group_size=rescale_group_size,
        )
        # Pin the source tensors in the entry: the address half of the key is
        # pointer identity, and a freed-then-reallocated buffer can land on the
        # same address (every MoE layer's expert weights share shape/dtype/
        # group_size, so nothing else in the key would discriminate). Holding a
        # reference keeps the allocator from handing the address to a different
        # tensor for as long as the entry is cached.
        entry = (weights_s8, wscales, block, weights, scales)
        if cache_prepack:
            _MOE_W4A8_PREPACK_CACHE[key] = entry

    weights_s8, wscales, block = entry[0], entry[1], entry[2]
    outputs = moe_gemm_w4a8(
        activations,
        weights_s8,
        wscales,
        num_tokens_per_expert,
        rescale_block_size=block,
        phase=phase,
    )
    if not cache_prepack:
        _synchronize_xpu_device(activations.device)
    return outputs


# ---------------------------------------------------------------------------
# Native FP8 prefill opt-in.
#
# When `ARK_MOE_PREFILL_NATIVE_FP8` is truthy, `moe_gemm_prefill` skips the
# `[E, K, N]` bf16/fp16 workspace and dispatches to the fused native-fp8
# kernel in `sycl_tla_moe_prefill_fp8_native.hpp`. The kernel loads fp8
# weight bytes and the per-K-group scale directly, upcasts to
# ``act_dtype`` in registers, folds the scale into the accumulator, and
# writes only the final output row -- no `[E, K, N]` global-memory round-
# trip.
#
# The C++ dispatcher performs an additional shape check
# (``N % 16 == 0`` and ``K % 32 == 0`` and ``K % group_size == 0`` and
# ``group_size % 32 == 0``) and silently falls back to the dequant path
# for shapes the native tile can't cover, so it is safe to leave this
# flag enabled globally.
#
# Truthy values (case-insensitive): "1", "true", "on", "yes".
# ---------------------------------------------------------------------------


def _native_fp8_prefill_enabled() -> bool:
    """Return True iff ``ARK_MOE_PREFILL_NATIVE_FP8`` is truthy."""
    env = os.environ.get("ARK_MOE_PREFILL_NATIVE_FP8")
    if env is None:
        return False
    return env.strip().lower() in ("1", "true", "on", "yes")


# ---------------------------------------------------------------------------
# Unified MoE entry point
#
# `moe_gemm_decode` and `moe_gemm_prefill` accept identical argument shapes
# and dtypes -- the only difference is which underlying SYCL kernel is
# launched (a GEMV variant tuned for smaller total-token workloads vs. a
# Grouped GEMM variant tuned for larger total-token workloads). Model code
# that runs through both
# regimes (prefill of a prompt, then autoregressive decode) traditionally
# has to keep two call sites and branch on phase. `moe(...)` collapses that
# into a single API and auto-selects the right kernel from the token
# distribution.
#
# Callers that already know the phase (e.g., a model's generation loop knows
# whether it's in prefill or decode) should pass it via the `phase` argument
# to bypass the auto-dispatch heuristic entirely.
# ---------------------------------------------------------------------------

# Default total-token threshold used by `phase="auto"`: dispatch to decode
# when `activations.shape[0] <= threshold`, otherwise prefill. This threshold
# is hardware-dependent and can be overridden via
# `ARK_MOE_AUTO_DECODE_MAX_TOKENS`.
#
# The cutoff used to be 32, a deliberately conservative value picked while the
# decode GEMV was still the bottleneck: back then only the tiny single-/few-
# stream case (every expert well under one DPAS tile row) was worth keeping off
# the prefill grouped-GEMM. The decode GEMV has since reached its bandwidth
# target for FP8 as well as int4-sym (K-split lane mapping + N-blocking inside
# the K-split kernel, and no per-call routing sync), so it now stays ahead of
# the grouped-GEMM over the whole small-batch range rather than only at the
# bs1 extreme, and the cutoff moves up to 128 total tokens accordingly.
# Batches above that still hand enough rows to each expert to fill the DPAS M
# tile, which is where the prefill path wins. Mirrors vLLM-xpu-kernels' `w4a16`
# dispatch, which likewise keeps the GEMV for the low tokens-per-expert regime.
_MOE_AUTO_DECODE_MAX_TOTAL_TOKENS = 128

_MOE_VALID_PHASES = ("auto", "decode", "prefill")


def _moe_auto_decode_max_total_tokens() -> int:
    """Return auto decode threshold from env or the module default.

    ``ARK_MOE_AUTO_DECODE_MAX_TOKENS`` is accepted when it is a positive
    integer. Unset/empty/invalid values fall back to
    ``_MOE_AUTO_DECODE_MAX_TOTAL_TOKENS``.
    """
    env = os.environ.get("ARK_MOE_AUTO_DECODE_MAX_TOKENS")
    if env is None:
        return _MOE_AUTO_DECODE_MAX_TOTAL_TOKENS
    env = env.strip()
    if not env:
        return _MOE_AUTO_DECODE_MAX_TOTAL_TOKENS
    try:
        value = int(env)
    except ValueError:
        return _MOE_AUTO_DECODE_MAX_TOTAL_TOKENS
    if value <= 0:
        return _MOE_AUTO_DECODE_MAX_TOTAL_TOKENS
    return value


def moe(
    activations: torch.Tensor,
    weights: torch.Tensor,
    num_tokens_per_expert: torch.Tensor,
    *,
    scales: torch.Tensor | None = None,
    zeros: torch.Tensor | None = None,
    weight_bits: int = 4,
    group_size: int = 128,
    asym: bool = False,
    phase: str = "auto",
    decode_threshold: int | None = None,
) -> torch.Tensor:
    """Unified MoE GEMM entry point that dispatches to decode or prefill.

    This is a thin Python-side dispatcher over :func:`moe_gemm_decode` and
    :func:`moe_gemm_prefill`. The two underlying kernels accept the same
    argument shapes/dtypes (see :func:`moe_gemm_decode` for the full layout
    contract); ``moe`` simply picks the one that is faster for the current
    token distribution so model code can have a single call site for both
    prefill and decode phases.

    Args:
        activations: ``[total_tokens, K]`` in fp16 or bf16.
        weights: ``[E, N, K_packed]`` -- see :func:`moe_gemm_decode` for the
            quant-specific layout/dtype contract.
        num_tokens_per_expert: ``[E]`` int32. Sum must equal
            ``activations.shape[0]`` (see :func:`moe_gemm_decode`).
        scales, zeros, weight_bits, group_size, asym: forwarded to the
            underlying kernel; see :func:`moe_gemm_decode`.
        phase: dispatch mode.

            * ``"auto"`` (default): dispatch to decode when
              ``activations.shape[0] <= decode_threshold`` (total tokens),
              otherwise prefill.
            * ``"decode"``: always dispatch to :func:`moe_gemm_decode`. Use
              when the model's generation loop already knows it is in the
              decode phase.
            * ``"prefill"``: always dispatch to :func:`moe_gemm_prefill`.
              Use when the model knows it is in the prefill phase.
        decode_threshold: Total-token threshold for ``"auto"`` mode. If not
            provided, uses ``ARK_MOE_AUTO_DECODE_MAX_TOKENS`` when set to a
            valid positive integer, otherwise defaults to 128. Explicit
            argument values take precedence over the environment variable.

    Returns:
        ``[total_tokens, N]`` in the activations dtype. Bit-identical to the
        underlying kernel that was dispatched.
    """
    if phase not in _MOE_VALID_PHASES:
        raise ValueError(f"phase must be one of {_MOE_VALID_PHASES}, got {phase!r}")

    if phase == "auto":
        threshold = _moe_auto_decode_max_total_tokens() if decode_threshold is None else int(decode_threshold)
        if num_tokens_per_expert.numel() == 0:
            raise ValueError("num_tokens_per_expert must be non-empty")
        total_tokens = int(activations.shape[0])
        phase = "decode" if total_tokens <= threshold else "prefill"

    if phase == "decode":
        return moe_gemm_decode(
            activations,
            weights,
            num_tokens_per_expert,
            scales=scales,
            zeros=zeros,
            weight_bits=weight_bits,
            group_size=group_size,
            asym=asym,
        )
    # phase == "prefill"
    return moe_gemm_prefill(
        activations,
        weights,
        num_tokens_per_expert,
        scales=scales,
        zeros=zeros,
        weight_bits=weight_bits,
        group_size=group_size,
        asym=asym,
    )


def patch_torch_sdpa(*args, **kwargs):
    from .torch_sdpa_patch import patch_torch_sdpa_with_ark

    return patch_torch_sdpa_with_ark(*args, **kwargs)


def unpatch_torch_sdpa():
    from .torch_sdpa_patch import unpatch_torch_sdpa_with_ark

    return unpatch_torch_sdpa_with_ark()


class _ArkInternalCpuNamespace:
    """Internal/experimental CPU helpers and backend lifecycle tools."""

    debug_resolve_sdpa_route = staticmethod(debug_cpu_sdpa_route)
    kv_cache_alloc = staticmethod(ark_cpu_kv_cache_alloc)
    kv_update = staticmethod(ark_cpu_kv_update)
    packed_kv_descriptor = staticmethod(ark_cpu_packed_kv_descriptor)
    packed_kv_alloc_from_descriptor = staticmethod(ark_cpu_packed_kv_alloc_from_descriptor)
    packed_kv_alloc = staticmethod(ark_cpu_packed_kv_alloc)
    packed_kv_info = staticmethod(ark_cpu_packed_kv_info)
    update_packed_kv_from_descriptor = staticmethod(ark_cpu_update_packed_kv_from_descriptor)
    update_packed_kv = staticmethod(ark_cpu_update_packed_kv)
    copy_packed_kv = staticmethod(ark_cpu_copy_packed_kv)
    copy_packed_kv_from_descriptor = staticmethod(ark_cpu_copy_packed_kv_from_descriptor)
    shift_packed_k = staticmethod(ark_cpu_shift_packed_k)
    shift_packed_k_from_descriptor = staticmethod(ark_cpu_shift_packed_k_from_descriptor)
    bestla_sdpa_packed = staticmethod(ark_cpu_bestla_sdpa_packed)
    bestla_sdpa_packed_from_descriptor = staticmethod(ark_cpu_bestla_sdpa_packed_from_descriptor)
    PackedKVHandle = ArkCpuPackedKVHandle


class _ArkInternalNamespace:
    """Internal/experimental helper surface."""

    cpu = _ArkInternalCpuNamespace()


internal = _ArkInternalNamespace()


__all__ = ["patch_torch_sdpa", "unpatch_torch_sdpa"]


# -----------------------------------------------------------------------------
# Compatibility layer
#
# Some callers (e.g. auto_round_extension/ark/qlinear.py) historically imported
# this package as a module and expected certain functions to exist at the module
# level (e.g. ark.repack_quantized_weight, ark.woq_linear).
#
# The wrappers below keep backward compatibility for legacy positional call
# styles without changing the compiled extension.
# -----------------------------------------------------------------------------


def check_isa_supported(_isa: str) -> bool:
    # Best-effort: some builds expose ISA checks via native libs; keep safe.
    # Returning False is conservative and avoids misconfiguration.
    return False


def repack_quantized_weight(*args, **kwargs):
    """Repack quantized weights into ARK/BestLA packed format.

    Supports two call styles:

    1) New style (recommended):
       repack_quantized_weight(QB, scaleB, zp, groupsize, compute_type, weight_type, scale_type, asym)

    2) Legacy style used by qlinear.py:
       repack_quantized_weight(QB, scaleB, zp, g_idx, compute_type, weight_type, scale_type, asym, groupsize)
       repack_quantized_weight(QB, scaleB, zp, g_idx, weight_type, compute_type, scale_type, asym, groupsize)
       (g_idx is ignored)
    """

    if kwargs:
        return _repack_quantized_weight_core(**kwargs)

    if len(args) == 8:
        QB, scaleB, zp, groupsize, compute_type, weight_type, scale_type, asym = args
    elif len(args) == 9:
        QB, scaleB, zp, _g_idx, a4, a5, scale_type, asym, groupsize = args
        # Legacy call sites sometimes swap compute_type/weight_type.
        compute_types = {"fp16", "bf16", "fp32", "fp8_e4m3", "fp8_e5m2", "fp8_e8m0"}
        if isinstance(a4, str) and a4 in compute_types:
            compute_type, weight_type = a4, a5
        else:
            weight_type, compute_type = a4, a5
    else:
        raise TypeError("repack_quantized_weight() expects 8 or 9 positional arguments; " f"got {len(args)}")

    # Some native paths may still expect a valid zp pointer even when asym=False.
    if (zp is None) or (isinstance(zp, torch.Tensor) and zp.numel() == 0):
        if not bool(asym):
            k = QB.shape[0]
            n = QB.shape[1]
            zp = torch.zeros((k // int(groupsize), n), dtype=torch.int8, device=QB.device)
        else:
            zp = torch.empty(0, dtype=torch.int8, device=QB.device)

    return _repack_quantized_weight_core(
        QB,
        scaleB,
        zp,
        groupsize,
        compute_type,
        weight_type,
        scale_type,
        asym,
    )


def unpack_weight(
    blob: torch.Tensor,
    out_dtype: torch.dtype,
    n,
    k,
    groupsize,
    compute_type,
    weight_type,
    scale_type,
    asym,
):
    return _unpack_weight_core(blob, out_dtype, n, k, groupsize, compute_type, weight_type, scale_type, asym)


def packed_weight_size(A: torch.Tensor, n, k, groupsize, compute_type, weight_type, scale_type, asym):
    # Keep signature convenient for Python callers; native library needs a stream.
    lib = get_lib(A)
    stream = get_stream(A)
    ct = cvtstr_dtype(compute_type)
    wt = cvtstr_dtype(weight_type)
    st = cvtstr_dtype(scale_type)
    return lib.packed_weight_size(stream, n, k, groupsize, ct, wt, st, asym)


def woqgemm_linear(
    A: torch.Tensor,
    B: torch.Tensor,
    bias: torch.Tensor | None,
    n,
    k,
    groupsize,
    compute_type,
    weight_type,
    scale_type,
    asym,
):
    n = int(n)
    k = int(k)
    groupsize = int(groupsize)
    asym = bool(asym)

    if A.shape[-1] != k:
        raise ValueError(f"k must match A.shape[-1] ({A.shape[-1]}), got {k}")

    raw_input_dtype = A.dtype
    target_dtype = torch.float16 if A.device.type == "xpu" else torch.float32
    out_shape = A.shape[:-1] + (n,)
    A_2d = A.to(target_dtype).reshape(-1, A.shape[-1])
    if bias is None or bias.numel() == 0:
        bias = torch.empty(0, dtype=target_dtype, device=A.device)
    else:
        bias = bias.to(device=A.device, dtype=target_dtype)

    out = woqgemm(
        A_2d,
        B,
        bias,
        n,
        k,
        groupsize,
        compute_type,
        weight_type,
        scale_type,
        asym,
    )

    return out.to(raw_input_dtype).reshape(out_shape)


def woq_linear(
    A: torch.Tensor,
    packed_B: torch.Tensor,
    bias: torch.Tensor,
    out: torch.Tensor,
    compute_type,
    weight_type,
    scale_type,
    asym,
    groupsize=None,
):
    """Linear helper that writes into a preallocated output tensor."""

    if groupsize is None:
        groupsize = A.shape[-1]

    result = woqgemm(
        A,
        packed_B,
        bias,
        out.shape[-1],
        A.shape[-1],
        int(groupsize),
        compute_type,
        weight_type,
        scale_type,
        bool(asym),
    )
    out.copy_(result)
    return out


# -----------------------------------------------------------------------------
# Module-level lib loading (replaces the previous singleton ``ARK`` class).
#
# NOTE: placed at the end of the module to avoid circular imports during
# package initialization.  pybind11-compiled .so modules may trigger a
# PyImport_AddModule lookup of the parent ``auto_round_kernel`` package;
# deferring the import until all definitions are complete ensures that the
# parent is fully registered in sys.modules.
# -----------------------------------------------------------------------------

try:
    from . import auto_round_kernel_cpu as _cpu_lib_mod

    cpu_lib = _cpu_lib_mod
except ImportError as _e:
    print(f"ARK is unable to load CPU lib: {_e}")

if torch.xpu.is_available():
    try:
        from . import auto_round_kernel_xpu as _xpu_lib_mod

        xpu_lib = _xpu_lib_mod
    except ImportError as _e:
        print(f"ARK is unable to load XPU lib: {_e}")

# Activation fused HMT + MXFP4 quantization (XPU). Imported last so the lib
# handles above are already bound when the submodule looks them up.
from .mxfp4_hadamard import (  # noqa: E402
    HADAMARD_DIM,
    HADAMARD_DIM_128,
    MAX_LANES_PER_ROW,
    SUPPORTED_HADAMARD_DIMS,
    get_hadamard_matrix,
    mxfp4_hadamard_quant,
    mxfp4_hadamard_quant_reference,
    mxfp4_quant_reference,
    mxfp4_stream_reference,
)

if __name__ == "__main__":
    print(cpu_lib is None, xpu_lib is None)

    def matmul_test():
        m = n = k = 128
        dt = torch.int8
        device = "cpu"
        has_bias = False
        if dt == torch.int8:
            A = torch.randint(-128, 127, (m, k), dtype=dt, device=device)
            B = torch.randint(-128, 127, (n, k), dtype=dt, device=device)
            C = igemm_s8s8s32(A, B)
            print(C)
        else:
            A = torch.rand(m, k, dtype=dt, device=device) - 0.5
            B = torch.rand(k, n, dtype=dt, device=device) - 0.5
            bias = torch.rand(1, n, dtype=dt, device=device) if has_bias else torch.empty(0)
            C = matmul(A, B, bias)
        ref = torch.matmul(A, B.T)
        if has_bias:
            ref = ref + bias
        dff = abs(C - ref)
        if dt != torch.int8:
            print(dff.max(), dff.mean())
            print(torch.allclose(ref, C, 0.01, 0.1))

    def woq():
        m = n = k = 128
        dt = torch.float32
        device = "cpu"
        A = torch.rand(m, k, dtype=dt, device=device) - 0.5
        bias = torch.rand(1, n, dtype=dt, device=device) + 1000
        B = torch.randint(-128, 127, (n, k), dtype=torch.int8, device=device)
        scaleB = torch.rand(n, 1, dtype=dt, device=device)
        C = woqgemm_s8(A, B, scaleB, bias)
        print(C)
        DB = B.to(dt) * scaleB
        ref = torch.matmul(A, DB.T) + bias
        print(ref)
        dff = abs(C - ref)
        print(dff.max(), dff.mean())

    def pack_unpack():
        m = n = k = 128
        groupsize = 32
        dt = torch.float32
        device = "xpu"
        B = torch.randint(-8, 7, (k, n), dtype=torch.int8, device=device)
        zp = torch.randint(-8, 7, (k // groupsize, n), dtype=torch.int8, device=device)
        scaleB = torch.rand(k // groupsize, n, dtype=dt, device=device) / 100
        blob = repack_quantized_weight(B, scaleB, zp, groupsize, "fp32", "int4", "fp32", False)
        dq = unpack_weight(blob, dt, n, k, groupsize, "fp32", "int4", "fp32", False)
        print(blob, dq)
        scale_re = scaleB.repeat_interleave(repeats=groupsize, dim=0).to(dt)

        DB = B.to(dt) * scale_re
        dff = abs(DB.T - dq)
        print(dff.max(), dff.mean())

    pack_unpack()