PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[ARK] Add bf16 + NHD layout support, refactor sage_dynamic_quant (#1882)
Luo Yu08c5d6d4b94e
diff --git a/auto_round_extension/ark/README.md b/auto_round_extension/ark/README.mdindex c087147d9..5b66ccc63 100644--- a/auto_round_extension/ark/README.md+++ b/auto_round_extension/ark/README.md@@ -44,6 +44,8 @@ pip install auto-round-lib### 2. Install from Source```bash+pip install . --no-build-isolation+# orpython setup.py bdist_wheel;pip install dist/*```diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex 167cd0c08..b7b80dc6d 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -93,951 +93,1053 @@ def get_stream(A: torch.Tensor) -> int:return torch.xpu.current_stream().sycl_queue-def singleton(cls):- """- 一个简单的单例模式装饰器- """- instances = {} # 存储类与实例的映射关系+def _normalize_tensor_layout(tensor_layout: str) -> str:+ layout = tensor_layout.upper()+ if layout not in ("HND", "NHD"):+ raise ValueError(f"tensor_layout must be either 'HND' or 'NHD', got {tensor_layout!r}")+ return layout+++def _attention_shape(tensor: torch.Tensor, tensor_layout: str) -> tuple[int, int, int, int]:+ layout = _normalize_tensor_layout(tensor_layout)+ if layout == "HND":+ batch, num_heads, seq_len, head_dim = tensor.shape+ else:+ batch, seq_len, num_heads, head_dim = tensor.shape+ return batch, num_heads, seq_len, head_dim- def get_instance(*args, **kwargs):- # 如果类不在字典中,则创建一个并存入- if cls not in instances:- instances[cls] = cls(*args, **kwargs)- # 如果已经在字典中,直接返回之前创建的那个- return instances[cls]- return get_instance+def _attention_strides_qko(tensor: torch.Tensor, tensor_layout: str) -> tuple[int, int, int, int]:+ layout = _normalize_tensor_layout(tensor_layout)+ if layout == "HND":+ batch_stride, head_stride, seq_stride, dim_stride = tensor.stride()+ else:+ batch_stride, seq_stride, head_stride, dim_stride = tensor.stride()+ return seq_stride, dim_stride, head_stride, batch_stride+++def _attention_strides_v(tensor: torch.Tensor, tensor_layout: str) -> tuple[int, int, int, int]:+ layout = _normalize_tensor_layout(tensor_layout)+ if layout == "HND":+ batch_stride, head_stride, seq_stride, dim_stride = tensor.stride()+ else:+ batch_stride, seq_stride, head_stride, dim_stride = tensor.stride()+ return dim_stride, seq_stride, head_stride, batch_stride+++def _validate_attention_tensor(+ tensor: torch.Tensor,+ name: str,+ tensor_layout: str,+ *,+ expected_dtype: torch.dtype | None = None,+) -> tuple[int, int, int, int]:+ if tensor.ndim != 4:+ raise ValueError(f"{name} must be a 4D tensor")+ if expected_dtype is not None and tensor.dtype != expected_dtype:+ raise ValueError(f"{name} must have dtype {expected_dtype}, got {tensor.dtype}")++ qko_strides = _attention_strides_qko(tensor, tensor_layout)+ if qko_strides[1] != 1:+ raise ValueError(f"{name} must be contiguous along the head-dim axis; got stride {qko_strides[1]}")+ if any(stride <= 0 for stride in qko_strides):+ raise ValueError(f"{name} must have positive non-zero strides, got {tensor.stride()}")++ return _attention_shape(tensor, tensor_layout)+++def _empty_attention_output(+ batch: int,+ num_heads: int,+ seq_len: int,+ head_dim: int,+ *,+ dtype: torch.dtype,+ device: torch.device,+ tensor_layout: str,+) -> torch.Tensor:+ layout = _normalize_tensor_layout(tensor_layout)+ shape = (batch, num_heads, seq_len, head_dim) if layout == "HND" else (batch, seq_len, num_heads, head_dim)+ return torch.empty(shape, device=device, dtype=dtype)-@singleton-class ARK:+# -----------------------------------------------------------------------------+# Module-level lib loading (replaces the previous singleton ``ARK`` class).+# -----------------------------------------------------------------------------++cpu_lib = None+xpu_lib = None++try:+ from . import auto_round_kernel_cpu as _cpu_lib_mod++ cpu_lib = _cpu_lib_mod+except ImportError as _e:+ print(f"ARK is unable to load CPU lib: {_e}")cpu_lib = None- xpu_lib = None-- def __init__(self):- try:- from . import auto_round_kernel_cpu-- self.cpu_lib = auto_round_kernel_cpu- except ImportError as e:- print(f"ARK is unable to load CPU lib: {e}")- self.cpu_lib = None-- if torch.xpu.is_available():- try:- from . import auto_round_kernel_xpu-- self.xpu_lib = auto_round_kernel_xpu- except ImportError as e:- print(f"ARK is unable to load XPU lib: {e}")- self.xpu_lib = None-- def get_lib(self, A: torch.Tensor):- lib = None- if A.device.type == "xpu":- lib = self.xpu_lib- if A.device.type == "cpu":- lib = self.cpu_lib- if lib is None:- raise NotImplementedError(f"Current device {A.device} is not supported")- return lib-- # A: mxk, B: nxk, bias: n- def matmul(self, A: torch.Tensor, B: torch.Tensor, bias: torch.Tensor):- m = A.shape[0]- n = B.shape[0]- k = B.shape[1]- lib = self.get_lib(A)- ctype = A.dtype- if A.device.type == "cpu":- ctype = torch.float32- C = torch.zeros(m, n, dtype=ctype, device=A.device)- stream = get_stream(A)- lib.matmul(- stream,- m,- n,- k,- A.contiguous().data_ptr(),- cvt_dtype(A.dtype),- B.contiguous().data_ptr(),- cvt_dtype(B.dtype),- C.contiguous().data_ptr(),- cvt_dtype(C.dtype),- bias.to(C.dtype).contiguous().data_ptr(),- True,- )- return C-- # A: mxk:s8, B: nxk:s8, return: mxn:s32- def igemm_s8s8s32(self, A: torch.Tensor, B: torch.Tensor):- m = A.shape[0]- n = B.shape[0]- k = B.shape[1]- lib = self.get_lib(A)- if lib is None:- raise NotImplementedError(f"Current device {A.device} is not supported")- C = torch.zeros(m, n, dtype=torch.int32, device=A.device)- stream = get_stream(A)- lib.matmul(- stream,- m,- n,- k,- A.contiguous().data_ptr(),- cvt_dtype(A.dtype),- B.contiguous().data_ptr(),- cvt_dtype(B.dtype),- C.contiguous().data_ptr(),- cvt_dtype(C.dtype),- 0,- True,- )- return C-- # A: mxk:DT, B: nxk:s8, scaleB: n:DT- # return: mxn:DT- def woqgemm_s8(self, A: torch.Tensor, B: torch.Tensor, scaleB: torch.Tensor, bias: torch.Tensor):- m = A.shape[0]- n = B.shape[0]- k = B.shape[1]- lib = self.get_lib(A)-- C = torch.zeros(m, n, dtype=A.dtype, device=A.device)- stream = get_stream(A)- lib.woqgemm_s8(- stream,- m,- n,- k,- A.contiguous().data_ptr(),- cvt_dtype(A.dtype),- B.contiguous().data_ptr(),- C.contiguous().data_ptr(),- bias.contiguous().data_ptr(),- True,- scaleB.contiguous().data_ptr(),- )- return C-- # A: mxk:DT, B: BS:s8, bias: n:DT- # return: C: mxn:DT- def woqgemm(- self,- A: torch.Tensor,- B: torch.Tensor,- bias: torch.Tensor,++if torch.xpu.is_available():+ try:+ from . import auto_round_kernel_xpu as _xpu_lib_mod++ xpu_lib = _xpu_lib_mod+ except ImportError as _e:+ print(f"ARK is unable to load XPU lib: {_e}")+ xpu_lib = None+++def get_lib(A: torch.Tensor):+ lib = None+ if A.device.type == "xpu":+ lib = xpu_lib+ if A.device.type == "cpu":+ lib = cpu_lib+ if lib is None:+ raise NotImplementedError(f"Current device {A.device} is not supported")+ return lib+++# A: mxk, B: nxk, bias: n+def matmul(A: torch.Tensor, B: torch.Tensor, bias: torch.Tensor):+ m = A.shape[0]+ n = B.shape[0]+ k = B.shape[1]+ lib = get_lib(A)+ ctype = A.dtype+ if A.device.type == "cpu":+ ctype = torch.float32+ C = torch.zeros(m, n, dtype=ctype, device=A.device)+ stream = get_stream(A)+ lib.matmul(+ stream,+ m,n,k,+ A.contiguous().data_ptr(),+ cvt_dtype(A.dtype),+ B.contiguous().data_ptr(),+ cvt_dtype(B.dtype),+ C.contiguous().data_ptr(),+ cvt_dtype(C.dtype),+ bias.to(C.dtype).contiguous().data_ptr(),+ True,+ )+ return C+++# A: mxk:s8, B: nxk:s8, return: mxn:s32+def igemm_s8s8s32(A: torch.Tensor, B: torch.Tensor):+ m = A.shape[0]+ n = B.shape[0]+ k = B.shape[1]+ lib = get_lib(A)+ if lib is None:+ raise NotImplementedError(f"Current device {A.device} is not supported")+ C = torch.zeros(m, n, dtype=torch.int32, device=A.device)+ stream = get_stream(A)+ lib.matmul(+ stream,+ m,+ n,+ k,+ A.contiguous().data_ptr(),+ cvt_dtype(A.dtype),+ B.contiguous().data_ptr(),+ cvt_dtype(B.dtype),+ C.contiguous().data_ptr(),+ cvt_dtype(C.dtype),+ 0,+ True,+ )+ return C+++# A: mxk:DT, B: nxk:s8, scaleB: n:DT+# return: mxn:DT+def woqgemm_s8(A: torch.Tensor, B: torch.Tensor, scaleB: torch.Tensor, bias: torch.Tensor):+ m = A.shape[0]+ n = B.shape[0]+ k = B.shape[1]+ lib = get_lib(A)++ C = torch.zeros(m, n, dtype=A.dtype, device=A.device)+ stream = get_stream(A)+ lib.woqgemm_s8(+ stream,+ m,+ n,+ k,+ A.contiguous().data_ptr(),+ cvt_dtype(A.dtype),+ B.contiguous().data_ptr(),+ C.contiguous().data_ptr(),+ bias.contiguous().data_ptr(),+ True,+ scaleB.contiguous().data_ptr(),+ )+ return C+++# A: mxk:DT, B: BS:s8, bias: n:DT+# return: C: mxn:DT+def woqgemm(+ A: torch.Tensor,+ B: torch.Tensor,+ bias: torch.Tensor,+ n,+ k,+ groupsize,+ compute_type,+ weight_type,+ scale_type,+ asym,+):+ m = A.shape[0]+ lib = get_lib(A)+ ct = cvtstr_dtype(compute_type)+ wt = cvtstr_dtype(weight_type)+ st = cvtstr_dtype(scale_type)+ C = torch.zeros(m, n, dtype=A.dtype, device=A.device)+ stream = get_stream(A)+ lib.woqgemm(+ stream,+ m,+ n,+ k,+ A.contiguous().data_ptr(),+ cvt_dtype(A.dtype),+ B.contiguous().data_ptr(),+ C.contiguous().data_ptr(),+ bias.contiguous().data_ptr(),groupsize,- compute_type,- weight_type,- scale_type,+ ct,+ wt,+ st,asym,- ):- m = A.shape[0]- lib = self.get_lib(A)- ct = cvtstr_dtype(compute_type)- wt = cvtstr_dtype(weight_type)- st = cvtstr_dtype(scale_type)- C = torch.zeros(m, n, dtype=A.dtype, device=A.device)- stream = get_stream(A)- lib.woqgemm(- stream,- m,- n,- k,- A.contiguous().data_ptr(),- cvt_dtype(A.dtype),- B.contiguous().data_ptr(),- C.contiguous().data_ptr(),- bias.contiguous().data_ptr(),- groupsize,- ct,- wt,- st,- asym,- )- return C-- # QB: k*n:int8, scaleB: k/blocksize*n:DT- # return: blob:BS:int8- def repack_quantized_weight(- self,- QB: torch.Tensor,- scaleB: torch.Tensor,- zp: torch.Tensor,+ )+ return C+++# QB: k*n:int8, scaleB: k/blocksize*n:DT+# return: blob:BS:int8+def _repack_quantized_weight_core(+ QB: torch.Tensor,+ scaleB: torch.Tensor,+ zp: torch.Tensor,+ groupsize,+ compute_type,+ weight_type,+ scale_type,+ asym,+):+ k = QB.shape[0]+ n = QB.shape[1]+ lib = get_lib(QB)+ stream = get_stream(QB)+ ct = cvtstr_dtype(compute_type)+ wt = cvtstr_dtype(weight_type)+ st = cvtstr_dtype(scale_type)+ BS = lib.packed_weight_size(stream, n, k, groupsize, ct, wt, st, asym)+ blob = torch.zeros(BS, dtype=torch.int8, device=QB.device)+ lib.repack_quantized_weight(+ stream,+ QB.contiguous().data_ptr(),+ zp.contiguous().data_ptr(),+ scaleB.contiguous().data_ptr(),+ blob.data_ptr(),+ n,+ k,groupsize,- compute_type,- weight_type,- scale_type,+ ct,+ wt,+ st,asym,- ):- k = QB.shape[0]- n = QB.shape[1]- lib = self.get_lib(QB)- stream = get_stream(QB)- ct = cvtstr_dtype(compute_type)- wt = cvtstr_dtype(weight_type)- st = cvtstr_dtype(scale_type)- BS = lib.packed_weight_size(stream, n, k, groupsize, ct, wt, st, asym)- blob = torch.zeros(BS, dtype=torch.int8, device=QB.device)- lib.repack_quantized_weight(- stream,- QB.contiguous().data_ptr(),- zp.contiguous().data_ptr(),- scaleB.contiguous().data_ptr(),- blob.data_ptr(),- n,- k,- groupsize,- ct,- wt,- st,- asym,- )- return blob-- # QB: blob:BS:int8- # return: out:nxk:out_dtype- def unpack_weight(- self,- blob: torch.Tensor,- out_dtype: torch.dtype,+ )+ return blob+++# QB: blob:BS:int8+# return: out:nxk:out_dtype+def _unpack_weight_core(+ blob: torch.Tensor,+ out_dtype: torch.dtype,+ n,+ k,+ groupsize,+ compute_type,+ weight_type,+ scale_type,+ asym,+):+ lib = get_lib(blob)+ stream = get_stream(blob)+ ct = cvtstr_dtype(compute_type)+ wt = cvtstr_dtype(weight_type)++ st = cvtstr_dtype(scale_type)+ oshape = (n, k) if blob.device.type == "xpu" else (k, n)+ out = torch.zeros(oshape, dtype=out_dtype, device=blob.device)+ lib.unpack_weight(+ stream,+ blob.data_ptr(),+ out.data_ptr(),+ cvt_dtype(out_dtype),n,k,groupsize,- compute_type,- weight_type,- scale_type,+ ct,+ wt,+ st,asym,- ):- lib = self.get_lib(blob)- stream = get_stream(blob)- ct = cvtstr_dtype(compute_type)- wt = cvtstr_dtype(weight_type)-- st = cvtstr_dtype(scale_type)- oshape = (n, k) if blob.device.type == "xpu" else (k, n)- out = torch.zeros(oshape, dtype=out_dtype, device=blob.device)- lib.unpack_weight(- stream,- blob.data_ptr(),- out.data_ptr(),- cvt_dtype(out_dtype),- n,- k,- groupsize,- ct,- wt,- st,- asym,- )- if blob.device.type == "cpu":- return out.T- return out-- def sdpa(- self,- query: torch.Tensor,- key: torch.Tensor,- value: torch.Tensor,- attn_mask: torch.Tensor | None = None,- dropout_p: float = 0.0,- is_causal: bool = False,- scale: float | None = None,- ) -> torch.Tensor:- """Scaled dot-product attention (SDPA) prefill+decode.-- Expects contiguous layouts:- - query: [B, Hq, Sq, D]- - key: [B, Hkv, Skv, D]- - value: [B, Hkv, Skv, D]-- Args:- - scale: Softmax scale. Uses 1 / sqrt(D) when None.-- Returns:- - O: [B, Hq, Sq, D] (same dtype as value)- """- if query.device.type != "xpu":- raise NotImplementedError("sdpa is only supported on XPU")-- if query.dtype not in (torch.float16, torch.bfloat16):- raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")- if key.dtype != query.dtype or value.dtype != query.dtype:- raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")-- if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:- raise ValueError("Q/K/V must be 4D tensors")-- if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():- raise ValueError("Q/K/V must be contiguous")-- B, Hq, Sq, D = query.shape- Bk, Hkv, Skv, Dk = key.shape- Bv, Hkv2, Skv2, Dv = value.shape-- if Bk != B or Bv != B:- raise ValueError("Batch size mismatch between Q/K/V")- if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:- raise ValueError("K/V shape mismatch")- if Dk != D:- raise ValueError("Head dim mismatch between Q and K/V")- if D not in (64, 128, 96, 192):- raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128, 96, 192")-- if dropout_p != 0.0:- raise NotImplementedError(f"dropout_p must be 0.0 (got {dropout_p}); dropout is not supported")-- if attn_mask is not None:- if attn_mask.device.type != "xpu":- raise ValueError("attn_mask must be on XPU")- if not attn_mask.is_contiguous():- raise ValueError("attn_mask must be contiguous")- if attn_mask.dtype != torch.float32:- raise ValueError(f"attn_mask must be float32 (additive bias), got {attn_mask.dtype}")- expected_mask_shape = (B, 1, Sq, Skv)- if attn_mask.shape != expected_mask_shape:- raise ValueError(f"attn_mask shape must be {expected_mask_shape}, got {tuple(attn_mask.shape)}")-- lib = self.get_lib(query)- stream = get_stream(query)- O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)- lib.sdpa(- stream,- query.data_ptr(),- key.data_ptr(),- value.data_ptr(),- O.data_ptr(),- attn_mask.data_ptr() if attn_mask is not None else 0,- cvt_dtype(query.dtype),- cvt_dtype(key.dtype),- cvt_dtype(O.dtype),- B,- Hq,- Hkv,- Sq,- Skv,- D,- float(scale) if scale is not None else 1.0 / (D**0.5),- bool(is_causal),- )- return O-- def sage(- self,- query: torch.Tensor,- key: torch.Tensor,- value: torch.Tensor,- attn_mask: torch.Tensor | None = None,- dropout_p: float = 0.0,- is_causal: bool = False,- scale: float | None = None,- enable_gqa: bool = False,- quant_block_size: int = 64,- qscale: torch.Tensor = None,- kscale: torch.Tensor = None,- ) -> torch.Tensor:- """SAGE attention prefill+decode.-- Expects contiguous layouts:- - query: [B, Hq, Sq, D]- - key: [B, Hkv, Skv, D]- - value: [B, Hkv, Skv, D]-- Args:- - scale: Attention scale. Uses 1 / sqrt(D) when None.- - quant_block_size: Block size for qscale and kscale.-- Returns:- - O: [B, Hq, Sq, D] (same dtype as value)- """- if query.device.type != "xpu":- raise NotImplementedError("sdpa is only supported on XPU")-- # if query.dtype not in (torch.float16, torch.bfloat16):- # raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")- # if key.dtype != query.dtype or value.dtype != query.dtype:- # raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")-- if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:- raise ValueError("Q/K/V must be 4D tensors")-- if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():- raise ValueError("Q/K/V must be contiguous")-- B, Hq, Sq, D = query.shape- Bk, Hkv, Skv, Dk = key.shape- Bv, Hkv2, Skv2, Dv = value.shape-- if Bk != B or Bv != B:- raise ValueError("Batch size mismatch between Q/K/V")- if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:- raise ValueError("K/V shape mismatch")- if Dk != D:- raise ValueError("Head dim mismatch between Q and K/V")- if D not in (64, 128):- raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")-- lib = self.get_lib(query)- stream = get_stream(query)- O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)- lib.sage(- stream,- query.data_ptr(),- key.data_ptr(),- value.data_ptr(),- O.data_ptr(),- attn_mask.data_ptr() if attn_mask is not None else 0,- quant_block_size,- qscale.data_ptr() if qscale is not None else 0,- kscale.data_ptr() if kscale is not None else 0,- cvt_dtype(query.dtype),- cvt_dtype(key.dtype),- cvt_dtype(O.dtype),- B,- Hq,- Hkv,- Sq,- Skv,- D,- float(scale) if scale is not None else 1.0 / (D**0.5),- bool(is_causal),- )- return O-- def sage_pvi8(- self,- query: torch.Tensor,- key: torch.Tensor,- value: torch.Tensor,- attn_mask: torch.Tensor | None = None,- dropout_p: float = 0.0,- is_causal: bool = False,- scale: float | None = None,- enable_gqa: bool = False,- quant_block_size: int = 64,- qscale: torch.Tensor = None,- kscale: torch.Tensor = None,- vscale: torch.Tensor = None,- out_dtype: torch.dtype = torch.float16,- ) -> torch.Tensor:- """Low-level SAGE attention with pre-quantized INT8 Q/K/V and PV int8.-- Expects contiguous layouts:- - query: [B, Hq, Sq, D] int8- - key: [B, Hkv, Skv, D] int8- - value: [B, Hkv, Skv, D] int8- - qscale: [B, Hq, ceil(Sq / quant_block_size), 1] float32- - kscale: [B, Hkv, ceil(Skv / quant_block_size), 1] float32- - vscale: [B, Hkv, ceil(Skv / quant_block_size), D] float32-- Returns:- - O: [B, Hq, Sq, D] float16- """- if query.device.type != "xpu":- raise NotImplementedError("sage_pvi8 is only supported on XPU")- if query.dtype != torch.int8 or key.dtype != torch.int8 or value.dtype != torch.int8:- raise ValueError(f"Q/K/V must be int8, got Q={query.dtype}, K={key.dtype}, V={value.dtype}")- if out_dtype != torch.float16:- raise ValueError(f"sage_pvi8 output must be float16, got {out_dtype}")- if qscale is None or kscale is None or vscale is None:- raise ValueError("qscale, kscale and vscale must be provided for sage_pvi8")-- if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:- raise ValueError("Q/K/V must be 4D tensors")-- if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():- raise ValueError("Q/K/V must be contiguous")-- B, Hq, Sq, D = query.shape- Bk, Hkv, Skv, Dk = key.shape- Bv, Hkv2, Skv2, Dv = value.shape-- if Bk != B or Bv != B:- raise ValueError("Batch size mismatch between Q/K/V")- if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:- raise ValueError("K/V shape mismatch")- if Dk != D:- raise ValueError("Head dim mismatch between Q and K/V")- if D not in (64, 128):- raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")-- q_blocks = (Sq + quant_block_size - 1) // quant_block_size- kv_blocks = (Skv + quant_block_size - 1) // quant_block_size- if qscale.numel() != B * Hq * q_blocks:- raise ValueError(- f"qscale must have {B * Hq * q_blocks} elements for shape [B, Hq, ceil(Sq/block), 1], got {qscale.numel()}"- )- if kscale.numel() != B * Hkv * kv_blocks:- raise ValueError(- f"kscale must have {B * Hkv * kv_blocks} elements for shape [B, Hkv, ceil(Skv/block), 1], got {kscale.numel()}"- )- if vscale.numel() != B * Hkv * kv_blocks * D:- raise ValueError(- f"vscale must have {B * Hkv * kv_blocks * D} elements for shape [B, Hkv, ceil(Skv/block), D], got {vscale.numel()}"- )-- lib = self.get_lib(query)- stream = get_stream(query)- O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=out_dtype)- lib.sage_pvi8(- stream,- query.data_ptr(),- key.data_ptr(),- value.data_ptr(),- O.data_ptr(),- attn_mask.data_ptr() if attn_mask is not None else 0,- quant_block_size,- qscale.data_ptr(),- kscale.data_ptr(),- vscale.data_ptr(),- cvt_dtype(query.dtype),- cvt_dtype(key.dtype),- cvt_dtype(O.dtype),- B,- Hq,- Hkv,- Sq,- Skv,- D,- float(scale) if scale is not None else 1.0 / (D**0.5),- bool(is_causal),- )- return O-- def sagev1(- self,- query: torch.Tensor,- key: torch.Tensor,- value: torch.Tensor,- attn_mask: torch.Tensor | None = None,- dropout_p: float = 0.0,- is_causal: bool = False,- scale: float | None = None,- enable_gqa: bool = False,- quant_block_size: int = 64,- ) -> torch.Tensor:- """SAGE v1 attention prefill+decode.-- Expects contiguous layouts:- - query: [B, Hq, Sq, D]- - key: [B, Hkv, Skv, D]- - value: [B, Hkv, Skv, D]-- Args:- - scale: Attention scale. Uses 1 / sqrt(D) when None.- - quant_block_size: Quantization block size used by the kernel.-- Returns:- - O: [B, Hq, Sq, D] (same dtype as value)- """- if quant_block_size <= 0:- return self.sdpa(- query=query,- key=key,- value=value,- attn_mask=attn_mask,- dropout_p=dropout_p,- is_causal=is_causal,- scale=scale,- )- if query.device.type != "xpu":- raise NotImplementedError("sdpa is only supported on XPU")- if query.dtype not in (torch.float16,):- raise ValueError(f"Q must be float16, got {query.dtype}")- if key.dtype != query.dtype or value.dtype != query.dtype:- raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")-- if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:- raise ValueError("Q/K/V must be 4D tensors")-- if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():- raise ValueError("Q/K/V must be contiguous")-- B, Hq, Sq, D = query.shape- Bk, Hkv, Skv, Dk = key.shape- Bv, Hkv2, Skv2, Dv = value.shape-- if Bk != B or Bv != B:- raise ValueError("Batch size mismatch between Q/K/V")- if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:- raise ValueError("K/V shape mismatch")- if Dk != D:- raise ValueError("Head dim mismatch between Q and K/V")- if D not in (64, 128):- raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")-- lib = self.get_lib(query)- stream = get_stream(query)- O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)- lib.sagev1(- stream,- query.data_ptr(),- key.data_ptr(),- value.data_ptr(),- O.data_ptr(),- attn_mask.data_ptr() if attn_mask is not None else 0,- quant_block_size,- cvt_dtype(query.dtype),- cvt_dtype(key.dtype),- cvt_dtype(value.dtype),- cvt_dtype(O.dtype),- B,- Hq,- Hkv,- Sq,- Skv,- D,- float(scale) if scale is not None else 1.0 / (D**0.5),- bool(is_causal),- )- return O-- def sagev1_pvi8(- self,- query: torch.Tensor,- key: torch.Tensor,- value: torch.Tensor,- attn_mask: torch.Tensor | None = None,- dropout_p: float = 0.0,- is_causal: bool = False,- scale: float | None = None,- enable_gqa: bool = False,- quant_block_size: int = 64,- ) -> torch.Tensor:- """SAGE v1 attention with PV int8 path.-- Expects FP16 Q/K/V input and quantizes Q/K/V internally before calling- the PV int8 kernel.- """- if quant_block_size <= 0:- return self.sdpa(- query=query,- key=key,- value=value,- attn_mask=attn_mask,- dropout_p=dropout_p,- is_causal=is_causal,- scale=scale,- )- if query.device.type != "xpu":- raise NotImplementedError("sagev1_pvi8 is only supported on XPU")- if query.dtype not in (torch.float16,):- raise ValueError(f"Q must be float16, got {query.dtype}")- if key.dtype != query.dtype or value.dtype != query.dtype:- raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")-- if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:- raise ValueError("Q/K/V must be 4D tensors")-- if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():- raise ValueError("Q/K/V must be contiguous")-- B, Hq, Sq, D = query.shape- Bk, Hkv, Skv, Dk = key.shape- Bv, Hkv2, Skv2, Dv = value.shape-- if Bk != B or Bv != B:- raise ValueError("Batch size mismatch between Q/K/V")- if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:- raise ValueError("K/V shape mismatch")- if Dk != D:- raise ValueError("Head dim mismatch between Q and K/V")- if D not in (64, 128):- raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")-- lib = self.get_lib(query)- stream = get_stream(query)- O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)- lib.sagev1_pvi8(- stream,- query.data_ptr(),- key.data_ptr(),- value.data_ptr(),- O.data_ptr(),- attn_mask.data_ptr() if attn_mask is not None else 0,- quant_block_size,- cvt_dtype(query.dtype),- cvt_dtype(key.dtype),- cvt_dtype(value.dtype),- cvt_dtype(O.dtype),- B,- Hq,- Hkv,- Sq,- Skv,- D,- float(scale) if scale is not None else 1.0 / (D**0.5),- bool(is_causal),+ )+ if blob.device.type == "cpu":+ return out.T+ return out+++def sdpa(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ tensor_layout: str = "HND",+) -> torch.Tensor:+ """Scaled dot-product attention (SDPA) prefill+decode.++ Supported tensor layouts:+ - HND: [B, H, N, D]+ - NHD: [B, N, H, D]++ Args:+ - scale: Softmax scale. Uses 1 / sqrt(D) when None.+ - tensor_layout: Layout of Q/K/V/O tensors.++ Returns:+ - O: same layout as the input tensors.+ """+ if query.device.type != "xpu":+ raise NotImplementedError("sdpa is only supported on XPU")++ if query.dtype not in (torch.float16, torch.bfloat16):+ raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")+ if key.dtype != query.dtype or value.dtype != query.dtype:+ raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")++ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout, expected_dtype=query.dtype)+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout, expected_dtype=query.dtype)++ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")+ if D not in (64, 128, 96, 192):+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128, 96, 192")++ if dropout_p != 0.0:+ raise NotImplementedError(f"dropout_p must be 0.0 (got {dropout_p}); dropout is not supported")++ if attn_mask is not None:+ if attn_mask.device.type != "xpu":+ raise ValueError("attn_mask must be on XPU")+ if not attn_mask.is_contiguous():+ raise ValueError("attn_mask must be contiguous")+ if attn_mask.dtype != torch.float32:+ raise ValueError(f"attn_mask must be float32 (additive bias), got {attn_mask.dtype}")+ expected_mask_shape = (B, 1, Sq, Skv)+ if attn_mask.shape != expected_mask_shape:+ raise ValueError(f"attn_mask shape must be {expected_mask_shape}, got {tuple(attn_mask.shape)}")++ lib = get_lib(query)+ stream = get_stream(query)+ O = _empty_attention_output(+ B,+ Hq,+ Sq,+ D,+ dtype=value.dtype,+ device=query.device,+ tensor_layout=tensor_layout,+ )+ q_strides = _attention_strides_qko(query, tensor_layout)+ k_strides = _attention_strides_qko(key, tensor_layout)+ v_strides = _attention_strides_v(value, tensor_layout)+ o_strides = _attention_strides_qko(O, tensor_layout)+ lib.sdpa(+ stream,+ query.data_ptr(),+ key.data_ptr(),+ value.data_ptr(),+ O.data_ptr(),+ attn_mask.data_ptr() if attn_mask is not None else 0,+ *q_strides,+ *k_strides,+ *v_strides,+ *o_strides,+ cvt_dtype(query.dtype),+ cvt_dtype(key.dtype),+ cvt_dtype(O.dtype),+ B,+ Hq,+ Hkv,+ Sq,+ Skv,+ D,+ float(scale) if scale is not None else 1.0 / (D**0.5),+ bool(is_causal),+ )+ return O+++def sage(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ enable_gqa: bool = False,+ quant_block_size: int = 64,+ qscale: torch.Tensor = None,+ kscale: torch.Tensor = None,+ tensor_layout: str = "HND",+) -> torch.Tensor:+ """SAGE attention prefill+decode.++ Supported tensor layouts:+ - HND: [B, H, N, D]+ - NHD: [B, N, H, D]++ Args:+ - scale: Attention scale. Uses 1 / sqrt(D) when None.+ - quant_block_size: Block size for qscale and kscale.+ - tensor_layout: Layout of Q/K/V/O tensors.++ Returns:+ - O: same layout as the input tensors.+ """+ if query.device.type != "xpu":+ raise NotImplementedError("sdpa is only supported on XPU")++ # if query.dtype not in (torch.float16, torch.bfloat16):+ # raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")+ # if key.dtype != query.dtype or value.dtype != query.dtype:+ # raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")++ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout)+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout)++ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")+ if D not in (64, 128):+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")++ lib = get_lib(query)+ stream = get_stream(query)+ O = _empty_attention_output(+ B,+ Hq,+ Sq,+ D,+ dtype=value.dtype,+ device=query.device,+ tensor_layout=tensor_layout,+ )+ q_strides = _attention_strides_qko(query, tensor_layout)+ k_strides = _attention_strides_qko(key, tensor_layout)+ v_strides = _attention_strides_v(value, tensor_layout)+ o_strides = _attention_strides_qko(O, tensor_layout)+ lib.sage(+ stream,+ query.data_ptr(),+ key.data_ptr(),+ value.data_ptr(),+ O.data_ptr(),+ attn_mask.data_ptr() if attn_mask is not None else 0,+ quant_block_size,+ qscale.data_ptr() if qscale is not None else 0,+ kscale.data_ptr() if kscale is not None else 0,+ *q_strides,+ *k_strides,+ *v_strides,+ *o_strides,+ cvt_dtype(query.dtype),+ cvt_dtype(key.dtype),+ cvt_dtype(O.dtype),+ B,+ Hq,+ Hkv,+ Sq,+ Skv,+ D,+ float(scale) if scale is not None else 1.0 / (D**0.5),+ bool(is_causal),+ )+ return O+++def sage_pvi8(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ enable_gqa: bool = False,+ quant_block_size: int = 64,+ qscale: torch.Tensor = None,+ kscale: torch.Tensor = None,+ vscale: torch.Tensor = None,+ out_dtype: torch.dtype = torch.float16,+ tensor_layout: str = "HND",+) -> torch.Tensor:+ """Low-level SAGE attention with pre-quantized INT8 Q/K/V and PV int8.++ Expects contiguous layouts:+ - query: [B, Hq, Sq, D] int8+ - key: [B, Hkv, Skv, D] int8+ - value: [B, Hkv, Skv, D] int8+ - qscale: [B, Hq, ceil(Sq / quant_block_size), 1] float32+ - kscale: [B, Hkv, ceil(Skv / quant_block_size), 1] float32+ - vscale: [B, Hkv, ceil(Skv / quant_block_size), D] float32++ Returns:+ - O: [B, Hq, Sq, D] float16+ """+ if query.device.type != "xpu":+ raise NotImplementedError("sage_pvi8 is only supported on XPU")+ if query.dtype != torch.int8 or key.dtype != torch.int8 or value.dtype != torch.int8:+ raise ValueError(f"Q/K/V must be int8, got Q={query.dtype}, K={key.dtype}, V={value.dtype}")+ if out_dtype != torch.float16:+ raise ValueError(f"sage_pvi8 output must be float16, got {out_dtype}")+ if qscale is None or kscale is None or vscale is None:+ raise ValueError("qscale, kscale and vscale must be provided for sage_pvi8")++ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout)+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout)++ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")+ if D not in (64, 128):+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")++ q_blocks = (Sq + quant_block_size - 1) // quant_block_size+ kv_blocks = (Skv + quant_block_size - 1) // quant_block_size+ if qscale.numel() != B * Hq * q_blocks:+ raise ValueError(+ f"qscale must have {B * Hq * q_blocks} elements for shape [B, Hq, ceil(Sq/block), 1], got {qscale.numel()}")- return O-- def sage_dynquant(- self,- query: torch.Tensor,- key: torch.Tensor,- value: torch.Tensor,- attn_mask: torch.Tensor | None = None,- dropout_p: float = 0.0,- is_causal: bool = False,- scale: float | None = None,- enable_gqa: bool = False,- quant_block_size: int = 64,- ) -> torch.Tensor:- """SAGE Attention with dynamic INT8 block-wise quantization of Q/K.-- Takes FP16 Q, K, V inputs. Quantizes Q and K to INT8 per-block- using a fused SYCL kernel, then calls SAGE V1 with INT8 data.- API is like SDPA but with an extra quant_block_size parameter.-- Args:- query: [B, Hq, Sq, D] float16- key: [B, Hkv, Skv, D] float16- value: [B, Hkv, Skv, D] float16- quant_block_size: Number of tokens sharing one INT8 scale.- E.g. 64 means 64 consecutive tokens share one absmax.- 0 means per-token (block_size=1).-- Returns:- O: [B, Hq, Sq, D] float16- """- if query.device.type != "xpu":- raise NotImplementedError("sage_dynquant is only supported on XPU")-- if query.dtype not in (torch.float16,):- raise ValueError(f"Q must be float16, got {query.dtype}")-- B, Hq, Sq, D = query.shape- _, Hkv, Skv, _ = key.shape-- # block_size=0 means per-token- block_size = quant_block_size if quant_block_size > 0 else 1-- # SAGE V1 kernel uses K-tile size=32; quant_block_size must be 1 or >=32- if block_size != 1 and block_size < 32:- raise ValueError(- f"quant_block_size={block_size} is not supported. "- f"Must be 1 (per-token) or >= 32 (e.g. 32, 64, 128, 256)."- )-- lib = self.get_lib(query)- stream = get_stream(query)-- # Auto-pad Q and K/V seq lengths to be divisible by block_size- # so sage_dynquant works as a drop-in replacement for SDPA- def _ceil_div(a, b):- return (a + b - 1) // b-- Sq_pad = _ceil_div(Sq, block_size) * block_size- Skv_pad = _ceil_div(Skv, block_size) * block_size- need_pad_q = Sq_pad != Sq- need_pad_kv = Skv_pad != Skv-- if need_pad_q:- pad_q = Sq_pad - Sq- query = torch.nn.functional.pad(query, (0, 0, 0, pad_q)) # pad S dim with zeros- if need_pad_kv:- pad_kv = Skv_pad - Skv- key = torch.nn.functional.pad(key, (0, 0, 0, pad_kv))- value = torch.nn.functional.pad(value, (0, 0, 0, pad_kv))-- # Fused block-wise quantization via SYCL kernel- # Tensor layout: [B, H, S, D] is contiguous → [B*H*S, D] flattened- # block_size tokens share one scale → num_blocks = B*H*S / block_size- # For Q: num_rows = B*Hq*Sq_pad, scale shape = [B, Hq, Sq_pad/block_size, 1]- q_num_rows = B * Hq * Sq_pad- q_num_blocks = q_num_rows // block_size- q_i8 = torch.empty_like(query, dtype=torch.int8)- q_scale = torch.empty(q_num_blocks, dtype=torch.float32, device=query.device)- lib.sage_dynamic_quant(- stream,- query.data_ptr(),- 0,- q_i8.data_ptr(),- q_scale.data_ptr(),- q_num_rows,- D,- block_size,+ if kscale.numel() != B * Hkv * kv_blocks:+ raise ValueError(+ f"kscale must have {B * Hkv * kv_blocks} elements for shape [B, Hkv, ceil(Skv/block), 1], got {kscale.numel()}")- q_scale = q_scale.reshape(B, Hq, Sq_pad // block_size, 1)-- k_num_rows = B * Hkv * Skv_pad- k_num_blocks = k_num_rows // block_size- k_i8 = torch.empty_like(key, dtype=torch.int8)- k_scale = torch.empty(k_num_blocks, dtype=torch.float32, device=key.device)- lib.sage_dynamic_quant(- stream,- key.data_ptr(),- 0,- k_i8.data_ptr(),- k_scale.data_ptr(),- k_num_rows,- D,- block_size,+ if vscale.numel() != B * Hkv * kv_blocks * D:+ raise ValueError(+ f"vscale must have {B * Hkv * kv_blocks * D} elements for shape [B, Hkv, ceil(Skv/block), D], got {vscale.numel()}")- k_scale = k_scale.reshape(B, Hkv, Skv_pad // block_size, 1)- # Call SAGE v1 with matching quant_block_size- out = self.sage(- q_i8,- k_i8,- value,+ lib = get_lib(query)+ stream = get_stream(query)+ O = _empty_attention_output(+ B,+ Hq,+ Sq,+ D,+ dtype=out_dtype,+ device=query.device,+ tensor_layout=tensor_layout,+ )+ q_strides = _attention_strides_qko(query, tensor_layout)+ k_strides = _attention_strides_qko(key, tensor_layout)+ v_strides = _attention_strides_v(value, tensor_layout)+ o_strides = _attention_strides_qko(O, tensor_layout)+ lib.sage_pvi8(+ stream,+ query.data_ptr(),+ key.data_ptr(),+ value.data_ptr(),+ O.data_ptr(),+ attn_mask.data_ptr() if attn_mask is not None else 0,+ quant_block_size,+ qscale.data_ptr(),+ kscale.data_ptr(),+ vscale.data_ptr(),+ *q_strides,+ *k_strides,+ *v_strides,+ *o_strides,+ cvt_dtype(query.dtype),+ cvt_dtype(key.dtype),+ cvt_dtype(O.dtype),+ B,+ Hq,+ Hkv,+ Sq,+ Skv,+ D,+ float(scale) if scale is not None else 1.0 / (D**0.5),+ bool(is_causal),+ )+ return O+++def sagev1(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ enable_gqa: bool = False,+ quant_block_size: int = 64,+ tensor_layout: str = "HND",+) -> torch.Tensor:+ """SAGE v1 attention prefill+decode.++ Supported tensor layouts:+ - HND: [B, H, N, D]+ - NHD: [B, N, H, D]++ Args:+ - scale: Attention scale. Uses 1 / sqrt(D) when None.+ - quant_block_size: Quantization block size used by the kernel.+ - tensor_layout: Layout of Q/K/V/O tensors.++ Returns:+ - O: same layout as the input tensors.+ """+ if quant_block_size <= 0:+ return sdpa(+ query=query,+ key=key,+ value=value,+ attn_mask=attn_mask,+ dropout_p=dropout_p,+ is_causal=is_causal,+ scale=scale,+ tensor_layout=tensor_layout,+ )+ if query.device.type != "xpu":+ raise NotImplementedError("sdpa is only supported on XPU")+ if query.dtype not in (torch.float16, torch.bfloat16):+ raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")+ if key.dtype != query.dtype or value.dtype != query.dtype:+ raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")++ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout, expected_dtype=query.dtype)+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout, expected_dtype=query.dtype)++ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")+ if D not in (64, 128):+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")++ lib = get_lib(query)+ stream = get_stream(query)+ O = _empty_attention_output(+ B,+ Hq,+ Sq,+ D,+ dtype=value.dtype,+ device=query.device,+ tensor_layout=tensor_layout,+ )+ q_strides = _attention_strides_qko(query, tensor_layout)+ k_strides = _attention_strides_qko(key, tensor_layout)+ v_strides = _attention_strides_v(value, tensor_layout)+ o_strides = _attention_strides_qko(O, tensor_layout)+ lib.sagev1(+ stream,+ query.data_ptr(),+ key.data_ptr(),+ value.data_ptr(),+ O.data_ptr(),+ attn_mask.data_ptr() if attn_mask is not None else 0,+ quant_block_size,+ *q_strides,+ *k_strides,+ *v_strides,+ *o_strides,+ cvt_dtype(query.dtype),+ cvt_dtype(key.dtype),+ cvt_dtype(value.dtype),+ cvt_dtype(O.dtype),+ B,+ Hq,+ Hkv,+ Sq,+ Skv,+ D,+ float(scale) if scale is not None else 1.0 / (D**0.5),+ bool(is_causal),+ )+ return O+++def sagev1_pvi8(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ enable_gqa: bool = False,+ quant_block_size: int = 64,+ tensor_layout: str = "HND",+) -> torch.Tensor:+ """SAGE v1 attention with PV int8 path.++ Expects FP16 Q/K/V input and quantizes Q/K/V internally before calling+ the PV int8 kernel.+ """+ if quant_block_size <= 0:+ return sdpa(+ query=query,+ key=key,+ value=value,attn_mask=attn_mask,+ dropout_p=dropout_p,is_causal=is_causal,scale=scale,- enable_gqa=enable_gqa,- quant_block_size=block_size,- qscale=q_scale,- kscale=k_scale,+ tensor_layout=tensor_layout,)+ if query.device.type != "xpu":+ raise NotImplementedError("sagev1_pvi8 is only supported on XPU")+ if query.dtype not in (torch.float16, torch.bfloat16):+ raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")+ if key.dtype != query.dtype or value.dtype != query.dtype:+ raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")++ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout, expected_dtype=query.dtype)+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout, expected_dtype=query.dtype)++ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")+ if D not in (64, 128):+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")++ lib = get_lib(query)+ stream = get_stream(query)+ O = _empty_attention_output(+ B,+ Hq,+ Sq,+ D,+ dtype=value.dtype,+ device=query.device,+ tensor_layout=tensor_layout,+ )+ q_strides = _attention_strides_qko(query, tensor_layout)+ k_strides = _attention_strides_qko(key, tensor_layout)+ v_strides = _attention_strides_v(value, tensor_layout)+ o_strides = _attention_strides_qko(O, tensor_layout)+ lib.sagev1_pvi8(+ stream,+ query.data_ptr(),+ key.data_ptr(),+ value.data_ptr(),+ O.data_ptr(),+ attn_mask.data_ptr() if attn_mask is not None else 0,+ quant_block_size,+ *q_strides,+ *k_strides,+ *v_strides,+ *o_strides,+ cvt_dtype(query.dtype),+ cvt_dtype(key.dtype),+ cvt_dtype(value.dtype),+ cvt_dtype(O.dtype),+ B,+ Hq,+ Hkv,+ Sq,+ Skv,+ D,+ float(scale) if scale is not None else 1.0 / (D**0.5),+ bool(is_causal),+ )+ return O+++def sageattn(+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ tensor_layout: str = "HND",+ is_causal: bool = False,+ sm_scale: Optional[float] = None,+ return_lse: bool = False,+ kernel: str = "v1_pvhalf",+ **kwargs,+) -> torch.Tensor:+ """SAGE attention dispatcher.++ Signature mirrors ``sageattention.sageattn``.++ Args:+ - q, k, v: Query/Key/Value tensors. Layout selected by ``tensor_layout``.+ - tensor_layout: "HND" or "NHD".+ - is_causal: Whether to apply causal mask.+ - sm_scale: Softmax scale. Uses ``1 / sqrt(head_dim)`` when None.+ - return_lse: Not supported; must be False.+ - kernel: Which SAGE variant to dispatch to.+ - "v1_pvhalf" (default): PV in half precision (calls ``sagev1``).+ - "v1_pvi8": PV in INT8 precision (calls ``sagev1_pvi8``).+ - kwargs: Forwarded to the underlying kernel (e.g. ``attn_mask``,+ ``dropout_p``, ``enable_gqa``, ``quant_block_size``).++ Returns:+ - O: same layout as the input tensors.+ """+ if return_lse:+ raise NotImplementedError("return_lse is not supported in ARK sageattn")++ if kernel == "v1_pvhalf":+ impl = sagev1+ elif kernel == "v1_pvi8":+ impl = sagev1_pvi8+ else:+ raise ValueError(f"Unsupported sageattn kernel={kernel!r}; supported: 'v1_pvhalf', 'v1_pvi8'")++ return impl(+ query=q,+ key=k,+ value=v,+ is_causal=is_causal,+ scale=sm_scale,+ tensor_layout=tensor_layout,+ **kwargs,+ )- # Slice back to original seq length if padded- if need_pad_q:- out = out[:, :, :Sq, :]- return out-- def moe_gemm(- self,- activations: torch.Tensor,- weights: torch.Tensor,- num_tokens_per_expert: torch.Tensor,- *,- scales: Optional[torch.Tensor] = None,- ) -> torch.Tensor:- """MOE GEMM (Mixture of Experts Grouped GEMM).-- Computes grouped GEMM for MOE layers where different experts process- different numbers of tokens.-- Expects contiguous layouts:- - activations: [total_tokens, K] (BF16/FP16)- - weights: [num_experts, K, N] (BF16/FP16, Row major)- - num_tokens_per_expert: [num_experts] (int32)- - scales (optional): [num_experts, N] or None-- Returns:- - outputs: [total_tokens, N] (same dtype as activations)- """- if activations.device.type != "xpu":- raise NotImplementedError("moe_gemm is only supported on XPU")-- if activations.dtype not in (torch.float16, torch.bfloat16):- raise ValueError(f"activations must be fp16/bf16, got {activations.dtype}")- if weights.dtype != activations.dtype:- raise ValueError("weights dtype must match activations dtype")-- if activations.ndim != 2 or weights.ndim != 3:- raise ValueError("activations must be 2D [total_tokens, K], weights must be 3D [num_experts, K, N]")-- if not activations.is_contiguous() or not weights.is_contiguous():- raise ValueError("activations and weights must be contiguous")-- if num_tokens_per_expert.dtype != torch.int32:- num_tokens_per_expert = num_tokens_per_expert.to(torch.int32)-- if not num_tokens_per_expert.is_contiguous():- num_tokens_per_expert = num_tokens_per_expert.contiguous()-- total_tokens, K = activations.shape- num_experts, K_w, N = weights.shape # weights are [num_experts, K, N]-- if K != K_w:- raise ValueError(f"K dimension mismatch: activations K={K}, weights K={K_w}")-- if num_tokens_per_expert.shape[0] != num_experts:- raise ValueError(- f"num_tokens_per_expert length {num_tokens_per_expert.shape[0]} != num_experts {num_experts}"- )-- # Validate total tokens- expected_total = int(num_tokens_per_expert.sum().item())- if expected_total != total_tokens:- raise ValueError(f"Sum of num_tokens_per_expert ({expected_total}) != total_tokens ({total_tokens})")-- lib = self.get_lib(activations)- stream = get_stream(activations)- outputs = torch.empty((total_tokens, N), device=activations.device, dtype=activations.dtype)-- scales_ptr = scales.data_ptr() if scales is not None else 0-- lib.moe_gemm(- stream,- activations.data_ptr(),- weights.data_ptr(),- scales_ptr,- outputs.data_ptr(),- cvt_dtype(activations.dtype),- N,- K,- num_tokens_per_expert.data_ptr(),- num_experts,++def sage_dynquant(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ enable_gqa: bool = False,+ quant_block_size: int = 64,+) -> torch.Tensor:+ """SAGE Attention with dynamic INT8 block-wise quantization of Q/K.++ Takes FP16 Q, K, V inputs. Quantizes Q and K to INT8 per-block+ using a fused SYCL kernel, then calls SAGE V1 with INT8 data.+ API is like SDPA but with an extra quant_block_size parameter.++ Args:+ query: [B, Hq, Sq, D] float16+ key: [B, Hkv, Skv, D] float16+ value: [B, Hkv, Skv, D] float16+ quant_block_size: Number of tokens sharing one INT8 scale.+ E.g. 64 means 64 consecutive tokens share one absmax.+ 0 means per-token (block_size=1).++ Returns:+ O: [B, Hq, Sq, D] float16+ """+ if query.device.type != "xpu":+ raise NotImplementedError("sage_dynquant is only supported on XPU")++ if query.dtype not in (torch.float16, torch.bfloat16):+ raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")++ B, Hq, Sq, D = query.shape+ _, Hkv, Skv, _ = key.shape++ # block_size=0 means per-token+ block_size = quant_block_size if quant_block_size > 0 else 1++ # SAGE V1 kernel uses K-tile size=32; quant_block_size must be 1 or >=32+ if block_size != 1 and block_size < 32:+ raise ValueError(+ f"quant_block_size={block_size} is not supported. "+ f"Must be 1 (per-token) or >= 32 (e.g. 32, 64, 128, 256).")- return outputs+ lib = get_lib(query)+ stream = get_stream(query)++ # Auto-pad Q and K/V seq lengths to be divisible by block_size+ # so sage_dynquant works as a drop-in replacement for SDPA+ def _ceil_div(a, b):+ return (a + b - 1) // b++ Sq_pad = _ceil_div(Sq, block_size) * block_size+ Skv_pad = _ceil_div(Skv, block_size) * block_size+ need_pad_q = Sq_pad != Sq+ need_pad_kv = Skv_pad != Skv++ if need_pad_q:+ pad_q = Sq_pad - Sq+ query = torch.nn.functional.pad(query, (0, 0, 0, pad_q)) # pad S dim with zeros+ if need_pad_kv:+ pad_kv = Skv_pad - Skv+ key = torch.nn.functional.pad(key, (0, 0, 0, pad_kv))+ value = torch.nn.functional.pad(value, (0, 0, 0, pad_kv))++ # Fused block-wise quantization via SYCL kernel+ # Tensor layout: [B, H, S, D] is contiguous → [B*H*S, D] flattened+ # block_size tokens share one scale → num_blocks = B*H*S / block_size+ # For Q: num_rows = B*Hq*Sq_pad, scale shape = [B, Hq, Sq_pad/block_size, 1]+ q_num_rows = B * Hq * Sq_pad+ q_num_blocks = q_num_rows // block_size+ q_i8 = torch.empty_like(query, dtype=torch.int8)+ q_scale = torch.empty(q_num_blocks, dtype=torch.float32, device=query.device)+ lib.sage_dynamic_quant(+ stream,+ query.data_ptr(),+ 0,+ q_i8.data_ptr(),+ q_scale.data_ptr(),+ q_num_rows,+ D,+ block_size,+ )+ q_scale = q_scale.reshape(B, Hq, Sq_pad // block_size, 1)++ k_num_rows = B * Hkv * Skv_pad+ k_num_blocks = k_num_rows // block_size+ k_i8 = torch.empty_like(key, dtype=torch.int8)+ k_scale = torch.empty(k_num_blocks, dtype=torch.float32, device=key.device)+ lib.sage_dynamic_quant(+ stream,+ key.data_ptr(),+ 0,+ k_i8.data_ptr(),+ k_scale.data_ptr(),+ k_num_rows,+ D,+ block_size,+ )+ k_scale = k_scale.reshape(B, Hkv, Skv_pad // block_size, 1)++ # Call SAGE v1 with matching quant_block_size+ out = sage(+ q_i8,+ k_i8,+ value,+ attn_mask=attn_mask,+ is_causal=is_causal,+ scale=scale,+ enable_gqa=enable_gqa,+ quant_block_size=block_size,+ qscale=q_scale,+ kscale=k_scale,+ )-if __name__ == "__main__":- ark = ARK()- print(ark.cpu_lib is None, ark.xpu_lib is None)+ # Slice back to original seq length if padded+ if need_pad_q:+ out = out[:, :, :Sq, :]+ return out- def matmul():- m = n = k = 128- dt = torch.int8- device = "cpu"- has_bias = False- if dt == torch.int8:- A = torch.randint(-128, 127, (m, k), dtype=dt, device=device)- B = torch.randint(-128, 127, (n, k), dtype=dt, device=device)- C = ark.igemm_s8s8s32(A, B)- print(C)- else:- A = torch.rand(m, k, dtype=dt, device=device) - 0.5- B = torch.rand(k, n, dtype=dt, device=device) - 0.5- bias = torch.rand(1, n, dtype=dt, device=device) if has_bias else torch.empty(0)- C = ark.matmul(A, B, bias)- ref = torch.matmul(A, B.T)- if has_bias:- ref = ref + bias- dff = abs(C - ref)- if dt != torch.int8:- print(dff.max(), dff.mean())- print(torch.allclose(ref, C, 0.01, 0.1))- def woq():- m = n = k = 128- dt = torch.float32- device = "cpu"- A = torch.rand(m, k, dtype=dt, device=device) - 0.5- bias = torch.rand(1, n, dtype=dt, device=device) + 1000- B = torch.randint(-128, 127, (n, k), dtype=torch.int8, device=device)- scaleB = torch.rand(n, 1, dtype=dt, device=device)- C = ark.woqgemm_s8(A, B, scaleB, bias)- print(C)- DB = B.to(dt) * scaleB- ref = torch.matmul(A, DB.T) + bias- print(ref)- dff = abs(C - ref)- print(dff.max(), dff.mean())+def moe_gemm(+ activations: torch.Tensor,+ weights: torch.Tensor,+ num_tokens_per_expert: torch.Tensor,+ *,+ scales: Optional[torch.Tensor] = None,+) -> torch.Tensor:+ """MOE GEMM (Mixture of Experts Grouped GEMM).- def pack_unpack():- m = n = k = 128- groupsize = 32- dt = torch.float32- device = "xpu"- B = torch.randint(-8, 7, (k, n), dtype=torch.int8, device=device)- zp = torch.randint(-8, 7, (k // groupsize, n), dtype=torch.int8, device=device)- scaleB = torch.rand(k // groupsize, n, dtype=dt, device=device) / 100- blob = ark.repack_quantized_weight(B, scaleB, zp, groupsize, "fp32", "int4", "fp32", False)- dq = ark.unpack_weight(blob, dt, n, k, groupsize, "fp32", "int4", "fp32", False)- print(blob, dq)- scale_re = scaleB.repeat_interleave(repeats=groupsize, dim=0).to(dt)+ Computes grouped GEMM for MOE layers where different experts process+ different numbers of tokens.- DB = B.to(dt) * scale_re- dff = abs(DB.T - dq)- print(dff.max(), dff.mean())+ Expects contiguous layouts:+ - activations: [total_tokens, K] (BF16/FP16)+ - weights: [num_experts, K, N] (BF16/FP16, Row major)+ - num_tokens_per_expert: [num_experts] (int32)+ - scales (optional): [num_experts, N] or None- pack_unpack()+ Returns:+ - outputs: [total_tokens, N] (same dtype as activations)+ """+ if activations.device.type != "xpu":+ raise NotImplementedError("moe_gemm is only supported on XPU")++ if activations.dtype not in (torch.float16, torch.bfloat16):+ raise ValueError(f"activations must be fp16/bf16, got {activations.dtype}")+ if weights.dtype != activations.dtype:+ raise ValueError("weights dtype must match activations dtype")++ if activations.ndim != 2 or weights.ndim != 3:+ raise ValueError("activations must be 2D [total_tokens, K], weights must be 3D [num_experts, K, N]")++ if not activations.is_contiguous() or not weights.is_contiguous():+ raise ValueError("activations and weights must be contiguous")++ if num_tokens_per_expert.dtype != torch.int32:+ num_tokens_per_expert = num_tokens_per_expert.to(torch.int32)++ if not num_tokens_per_expert.is_contiguous():+ num_tokens_per_expert = num_tokens_per_expert.contiguous()++ total_tokens, K = activations.shape+ num_experts, K_w, N = weights.shape # weights are [num_experts, K, N]++ if K != K_w:+ raise ValueError(f"K dimension mismatch: activations K={K}, weights K={K_w}")++ if num_tokens_per_expert.shape[0] != num_experts:+ raise ValueError(f"num_tokens_per_expert length {num_tokens_per_expert.shape[0]} != num_experts {num_experts}")++ # Validate total tokens+ expected_total = int(num_tokens_per_expert.sum().item())+ if expected_total != total_tokens:+ raise ValueError(f"Sum of num_tokens_per_expert ({expected_total}) != total_tokens ({total_tokens})")++ lib = get_lib(activations)+ stream = get_stream(activations)+ outputs = torch.empty((total_tokens, N), device=activations.device, dtype=activations.dtype)++ scales_ptr = scales.data_ptr() if scales is not None else 0++ lib.moe_gemm(+ stream,+ activations.data_ptr(),+ weights.data_ptr(),+ scales_ptr,+ outputs.data_ptr(),+ cvt_dtype(activations.dtype),+ N,+ K,+ num_tokens_per_expert.data_ptr(),+ num_experts,+ )+ return outputsdef patch_torch_sdpa(*args, **kwargs):@@ -1052,7 +1154,7 @@ def unpatch_torch_sdpa():return unpatch_torch_sdpa_with_ark()-__all__ = ["ARK", "patch_torch_sdpa", "unpatch_torch_sdpa"]+__all__ = ["patch_torch_sdpa", "unpatch_torch_sdpa"]# -----------------------------------------------------------------------------@@ -1062,16 +1164,11 @@ __all__ = ["ARK", "patch_torch_sdpa", "unpatch_torch_sdpa"]# this package as a module and expected certain functions to exist at the module# level (e.g. ark.repack_quantized_weight, ark.woq_linear).#-# The current implementation exposes these as methods on the singleton ARK()-# instance. The wrappers below keep backward compatibility without changing the-# compiled extension.+# The wrappers below keep backward compatibility for legacy positional call+# styles without changing the compiled extension.# ------------------------------------------------------------------------------def _ark_instance():- return ARK()--def check_isa_supported(_isa: str) -> bool:# Best-effort: some builds expose ISA checks via native libs; keep safe.# Returning False is conservative and avoids misconfiguration.@@ -1093,7 +1190,7 @@ def repack_quantized_weight(*args, **kwargs):"""if kwargs:- return _ark_instance().repack_quantized_weight(**kwargs)+ return _repack_quantized_weight_core(**kwargs)if len(args) == 8:QB, scaleB, zp, groupsize, compute_type, weight_type, scale_type, asym = args@@ -1117,7 +1214,7 @@ def repack_quantized_weight(*args, **kwargs):else:zp = torch.empty(0, dtype=torch.int8, device=QB.device)- return _ark_instance().repack_quantized_weight(+ return _repack_quantized_weight_core(QB,scaleB,zp,@@ -1140,12 +1237,12 @@ def unpack_weight(scale_type,asym,):- return _ark_instance().unpack_weight(blob, out_dtype, n, k, groupsize, compute_type, weight_type, scale_type, asym)+ return _unpack_weight_core(blob, out_dtype, n, k, groupsize, compute_type, weight_type, scale_type, asym)def packed_weight_size(A: torch.Tensor, n, k, groupsize, compute_type, weight_type, scale_type, asym):# Keep signature convenient for Python callers; native library needs a stream.- lib = _ark_instance().get_lib(A)+ lib = get_lib(A)stream = get_stream(A)ct = cvtstr_dtype(compute_type)wt = cvtstr_dtype(weight_type)@@ -1169,7 +1266,7 @@ def woq_linear(if groupsize is None:groupsize = A.shape[-1]- result = _ark_instance().woqgemm(+ result = woqgemm(A,packed_B,bias,@@ -1183,3 +1280,65 @@ def woq_linear()out.copy_(result)return out+++if __name__ == "__main__":+ print(cpu_lib is None, xpu_lib is None)++ def matmul_test():+ m = n = k = 128+ dt = torch.int8+ device = "cpu"+ has_bias = False+ if dt == torch.int8:+ A = torch.randint(-128, 127, (m, k), dtype=dt, device=device)+ B = torch.randint(-128, 127, (n, k), dtype=dt, device=device)+ C = igemm_s8s8s32(A, B)+ print(C)+ else:+ A = torch.rand(m, k, dtype=dt, device=device) - 0.5+ B = torch.rand(k, n, dtype=dt, device=device) - 0.5+ bias = torch.rand(1, n, dtype=dt, device=device) if has_bias else torch.empty(0)+ C = matmul(A, B, bias)+ ref = torch.matmul(A, B.T)+ if has_bias:+ ref = ref + bias+ dff = abs(C - ref)+ if dt != torch.int8:+ print(dff.max(), dff.mean())+ print(torch.allclose(ref, C, 0.01, 0.1))++ def woq():+ m = n = k = 128+ dt = torch.float32+ device = "cpu"+ A = torch.rand(m, k, dtype=dt, device=device) - 0.5+ bias = torch.rand(1, n, dtype=dt, device=device) + 1000+ B = torch.randint(-128, 127, (n, k), dtype=torch.int8, device=device)+ scaleB = torch.rand(n, 1, dtype=dt, device=device)+ C = woqgemm_s8(A, B, scaleB, bias)+ print(C)+ DB = B.to(dt) * scaleB+ ref = torch.matmul(A, DB.T) + bias+ print(ref)+ dff = abs(C - ref)+ print(dff.max(), dff.mean())++ def pack_unpack():+ m = n = k = 128+ groupsize = 32+ dt = torch.float32+ device = "xpu"+ B = torch.randint(-8, 7, (k, n), dtype=torch.int8, device=device)+ zp = torch.randint(-8, 7, (k // groupsize, n), dtype=torch.int8, device=device)+ scaleB = torch.rand(k // groupsize, n, dtype=dt, device=device) / 100+ blob = repack_quantized_weight(B, scaleB, zp, groupsize, "fp32", "int4", "fp32", False)+ dq = unpack_weight(blob, dt, n, k, groupsize, "fp32", "int4", "fp32", False)+ print(blob, dq)+ scale_re = scaleB.repeat_interleave(repeats=groupsize, dim=0).to(dt)++ DB = B.to(dt) * scale_re+ dff = abs(DB.T - dq)+ print(dff.max(), dff.mean())++ pack_unpack()diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cppindex b623ed437..dbf0eac16 100755--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp@@ -98,9 +98,12 @@ static size_t packed_weight_size(torch_ptr stream, int n, int k, int blocksize,#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)-static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask, int q_dtype,- int k_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,- int head_dim, float softmax_scale, bool is_causal) {+static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,+ int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b,+ int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int q_dtype, int k_dtype, int o_dtype,+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ float softmax_scale, bool is_causal) {if (k_dtype != q_dtype || o_dtype != q_dtype) {throw std::invalid_argument("ark::sdpa: k_dtype and o_dtype must match q_dtype");}@@ -111,12 +114,17 @@ static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_throw std::invalid_argument("ark::sdpa: mask and is_causal cannot both be set");}ark::sdpa_impl((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, (BTLA_DTYPE)(q_dtype),+ q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b,+ v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);}static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,- int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch,- int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,+ int o_stride_h, int o_stride_b, int q_dtype, int k_dtype, int v_dtype, int o_dtype,+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,float softmax_scale, bool is_causal, bool use_int8_pv) {if (mask && is_causal) {throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");@@ -124,17 +132,22 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,if (q_dtype != k_dtype || o_dtype != q_dtype || v_dtype != q_dtype) {throw std::invalid_argument("ark::sagev1: k_dtype and o_dtype must match q_dtype");}- if (q_dtype != (int)BTLA_DTYPE::F16) {- throw std::invalid_argument("ark::sagev1: only F16 is supported for q_dtype");+ if (q_dtype != (int)BTLA_DTYPE::F16 && q_dtype != (int)BTLA_DTYPE::BF16) {+ throw std::invalid_argument("ark::sagev1: only F16 and BF16 are supported for q_dtype");}#ifdef ARK_XPUif (use_int8_pv) {XpuWrapper::sagev1_pvi8((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,- scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,- softmax_scale, is_causal);+ scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s,+ k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,+ o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,+ seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, (BTLA_DTYPE)q_dtype);} else {XpuWrapper::sagev1((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, scale_block_size,- batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h,+ k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,+ o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,+ softmax_scale, is_causal, (BTLA_DTYPE)q_dtype);}#elsethrow std::runtime_error("ark::sagev1 is only supported on XPU");@@ -142,42 +155,64 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,}static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,- int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q,- int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {- sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch, num_heads_q,- num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, false);+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,+ int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv,+ int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+ sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,+ o_stride_s, o_stride_d, o_stride_h, o_stride_b, q_dtype, k_dtype, v_dtype, o_dtype, batch,+ num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, false);}static void sagev1_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,- int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch,- int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,+ int o_stride_h, int o_stride_b, int q_dtype, int k_dtype, int v_dtype, int o_dtype,+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,float softmax_scale, bool is_causal) {- sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch, num_heads_q,- num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, true);+ sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,+ o_stride_s, o_stride_d, o_stride_h, o_stride_b, q_dtype, k_dtype, v_dtype, o_dtype, batch,+ num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, true);}static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,- int scale_block_size, torch_ptr qscale, torch_ptr kscale, int q_dtype, int k_dtype, int o_dtype,- int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,- float softmax_scale, bool is_causal) {+ int scale_block_size, torch_ptr qscale, torch_ptr kscale, int q_stride_s, int q_stride_d,+ int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b,+ int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,+ int o_stride_h, int o_stride_b, int q_dtype, int k_dtype, int o_dtype, int batch, int num_heads_q,+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal) {if (mask && is_causal) {throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");}ark::sdpa_impl_qks8_pvhalf((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,- scale_block_size, (void*)qscale, (void*)kscale, batch, num_heads_q, num_heads_kv,- seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ scale_block_size, (void*)qscale, (void*)kscale, q_stride_s, q_stride_d, q_stride_h,+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,+ v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch,+ num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal,+ (BTLA_DTYPE)o_dtype);}static void sage_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,- int scale_block_size, torch_ptr qscale, torch_ptr kscale, torch_ptr vscale, int q_dtype,+ int scale_block_size, torch_ptr qscale, torch_ptr kscale, torch_ptr vscale, int q_stride_s,+ int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,+ int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int q_dtype,int k_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {if (mask && is_causal) {throw std::invalid_argument("ark::sage_pvi8: mask and is_causal cannot both be set");}ark::sdpa_impl_qks8_pvi8((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,- scale_block_size, (void*)qscale, (void*)kscale, (void*)vscale, batch, num_heads_q,- num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ scale_block_size, (void*)qscale, (void*)kscale, (void*)vscale, q_stride_s, q_stride_d,+ q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,+ v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,+ is_causal, (BTLA_DTYPE)o_dtype);}static void moe_gemm_wrapper(torch_ptr stream, torch_ptr activations, torch_ptr weights, torch_ptr scales,@@ -309,6 +344,77 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr bias});}}++static void sage_compute_seq_mean_bias_layout(torch_ptr stream, torch_ptr input, torch_ptr output, int batch,+ int num_heads, int seq, int head_dim, int stride_seq, int stride_dim,+ int stride_head, int stride_batch) {+ auto* q = (sycl::queue*)stream;+ auto* in_ptr = (sycl::half*)input;+ auto* out_ptr = (sycl::half*)output;+ if (stride_dim != 1) {+ throw std::invalid_argument("ark::sage_compute_seq_mean_bias_layout: head-dim stride must be 1");+ }+ if (ark::XpuWrapper::is_packed_hnd(stride_seq, stride_dim, stride_head, stride_batch, num_heads, seq, head_dim)) {+ ark::XpuWrapper::compute_seq_mean_bias<sycl::half>(q, in_ptr, out_ptr, batch * num_heads, seq, head_dim);+ } else {+ ark::XpuWrapper::compute_seq_mean_bias_strided<sycl::half>(q, in_ptr, out_ptr, batch, num_heads, seq, head_dim,+ stride_seq, stride_dim, stride_head, stride_batch);+ }+}++static void sage_dynamic_quant_layout(torch_ptr stream, torch_ptr input, torch_ptr bias, torch_ptr output,+ torch_ptr scale_out, int batch, int num_heads, int seq, int head_dim,+ int block_size, int stride_seq, int stride_dim, int stride_head,+ int stride_batch) {+ auto* q = (sycl::queue*)stream;+ auto* in_ptr = (sycl::half*)input;+ auto* bias_ptr = bias ? (sycl::half*)bias : nullptr;+ auto* out_ptr = (int8_t*)output;+ auto* scale_ptr = (float*)scale_out;+ if (block_size <= 0) {+ throw std::invalid_argument("ark::sage_dynamic_quant_layout: block_size must be > 0");+ }+ if (stride_dim != 1) {+ throw std::invalid_argument("ark::sage_dynamic_quant_layout: head-dim stride must be 1");+ }+ int n_seq_blk = (seq + block_size - 1) / block_size;+ bool force_strided = ark::env_params::Instance()->sage_disable_packed_hnd_fast != 0;+ if (!force_strided &&+ ark::XpuWrapper::is_packed_hnd(stride_seq, stride_dim, stride_head, stride_batch, num_heads, seq, head_dim)) {+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, in_ptr, out_ptr, scale_ptr, batch * num_heads, seq, n_seq_blk,+ head_dim, block_size, bias_ptr);+ } else {+ ark::XpuWrapper::sage_dynamic_quant_strided<sycl::half>(q, in_ptr, out_ptr, scale_ptr, batch, num_heads, seq,+ n_seq_blk, head_dim, block_size, stride_seq, stride_dim,+ stride_head, stride_batch, bias_ptr);+ }+}++static void sage_dynamic_quant_v_layout(torch_ptr stream, torch_ptr input, torch_ptr output, torch_ptr scale_out,+ int batch, int num_heads, int seq, int head_dim, int block_size,+ int stride_dim, int stride_seq, int stride_head, int stride_batch) {+ auto* q = (sycl::queue*)stream;+ auto* in_ptr = (sycl::half*)input;+ auto* out_ptr = (int8_t*)output;+ auto* scale_ptr = (float*)scale_out;+ if (block_size <= 0) {+ throw std::invalid_argument("ark::sage_dynamic_quant_v_layout: block_size must be > 0");+ }+ if (stride_dim != 1) {+ throw std::invalid_argument("ark::sage_dynamic_quant_v_layout: head-dim stride must be 1");+ }+ int n_seq_blk = (seq + block_size - 1) / block_size;+ bool force_strided = ark::env_params::Instance()->sage_disable_packed_hnd_fast != 0;+ if (!force_strided &&+ ark::XpuWrapper::is_packed_hnd(stride_seq, stride_dim, stride_head, stride_batch, num_heads, seq, head_dim)) {+ ark::XpuWrapper::sage_dynamic_quant_v<sycl::half>(q, in_ptr, out_ptr, scale_ptr, batch * num_heads, seq,+ n_seq_blk, head_dim, block_size);+ } else {+ ark::XpuWrapper::sage_dynamic_quant_v_strided<sycl::half>(q, in_ptr, out_ptr, scale_ptr, batch, num_heads, seq,+ n_seq_blk, head_dim, block_size, stride_dim, stride_seq,+ stride_head, stride_batch);+ }+}#endif // ARK_XPU && ARK_SYCL_TLA} // namespace ark@@ -329,6 +435,9 @@ PYBIND11_MODULE(PY_NAME, m) {// Low-level SAGE PVi8 API: input Q/K/V are pre-quantized int8 with qscale/kscale/vscale.m.def("sage_pvi8", &ark::sage_pvi8);m.def("sage_dynamic_quant", &ark::sage_dynamic_quant);+ m.def("sage_compute_seq_mean_bias_layout", &ark::sage_compute_seq_mean_bias_layout);+ m.def("sage_dynamic_quant_layout", &ark::sage_dynamic_quant_layout);+ m.def("sage_dynamic_quant_v_layout", &ark::sage_dynamic_quant_v_layout);m.def("moe_gemm", &ark::moe_gemm_wrapper);#endif}\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/qlinear.py b/auto_round_extension/ark/auto_round_kernel/qlinear.pyindex 125e77e90..749d2db1c 100644--- a/auto_round_extension/ark/auto_round_kernel/qlinear.py+++ b/auto_round_extension/ark/auto_round_kernel/qlinear.py@@ -21,7 +21,7 @@ try:import auto_round_kernelARK_INSTALLED = True- ark = auto_round_kernel.ARK()+ ark = auto_round_kernelexcept:ARK_INSTALLED = False@@ -130,12 +130,16 @@ class QuantLinear(nn.Module):raise NotImplementedError("in_features must be divisible by group_size")if "awq" in self.QUANT_TYPE:self.register_buffer(- "qweight", torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32)+ "qweight",+ torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32),)else:self.register_buffer("qweight",- torch.zeros((self.infeatures // 32 * self.bits, self.outfeatures), dtype=torch.int32),+ torch.zeros(+ (self.infeatures // 32 * self.bits, self.outfeatures),+ dtype=torch.int32,+ ),)self.register_buffer("qzeros",@@ -394,7 +398,14 @@ class QuantLinearFP8(nn.Module):self.trainable = trainabledef _load_from_state_dict(- self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs+ self,+ state_dict,+ prefix,+ local_metadata,+ strict,+ missing_keys,+ unexpected_keys,+ error_msgs,):"""Override to handle weight_scale shape mismatch between checkpoint and model."""weight_scale_key = prefix + "weight_scale"@@ -404,7 +415,10 @@ class QuantLinearFP8(nn.Module):out_features, num_groups = expected_shapeexpected_numel = out_features * num_groups- if loaded_scale.ndim == 2 and loaded_scale.shape == (num_groups, out_features):+ if loaded_scale.ndim == 2 and loaded_scale.shape == (+ num_groups,+ out_features,+ ):state_dict[weight_scale_key] = loaded_scale.t().contiguous()loaded_scale = state_dict[weight_scale_key]@@ -424,7 +438,9 @@ class QuantLinearFP8(nn.Module):inferred_shape = (out_features, inferred_num_groups)state_dict[weight_scale_key] = loaded_scale.view(inferred_shape)self.weight_scale = torch.zeros(- inferred_shape, dtype=self.weight_scale.dtype, device=self.weight_scale.device+ inferred_shape,+ dtype=self.weight_scale.dtype,+ device=self.weight_scale.device,)else:@@ -432,7 +448,13 @@ class QuantLinearFP8(nn.Module):f"Cannot reshape scale! loaded_scale.numel()={numel}, expected={expected_numel}, out_features={out_features}")super()._load_from_state_dict(- state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs+ state_dict,+ prefix,+ local_metadata,+ strict,+ missing_keys,+ unexpected_keys,+ error_msgs,)def post_init(self):diff --git a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp b/auto_round_extension/ark/auto_round_kernel/sdpa.cppindex f5e30e7aa..03a9ada85 100644--- a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp+++ b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp@@ -50,16 +50,42 @@ int launch_prefill_kernel_f16_64_sage_i8pv(detail::Options const& options) {options);}-KernelLauncher select_sage_prefill_launcher(BTLA_DTYPE dtype, int head_dim, bool use_int8_pv) {+int launch_prefill_kernel_bf16_128_sage(detail::Options const& options) {+ return launch_sage_prefill_kernel_128<cute::int8_t, cute::int8_t, cute::bfloat16_t>(options);+}++int launch_prefill_kernel_bf16_128_sage_i8pv(detail::Options const& options) {+ return launch_sage_prefill_kernel_128<cute::int8_t, cute::int8_t, cute::int8_t, cute::bfloat16_t, true, true,+ true>(options);+}++int launch_prefill_kernel_bf16_64_sage(detail::Options const& options) {+ return launch_sage_prefill_kernel_64<cute::int8_t, cute::int8_t, cute::bfloat16_t>(options);+}++int launch_prefill_kernel_bf16_64_sage_i8pv(detail::Options const& options) {+ return launch_sage_prefill_kernel_64<cute::int8_t, cute::int8_t, cute::int8_t, cute::bfloat16_t, true, true,+ true>(options);+}++KernelLauncher select_sage_prefill_launcher(BTLA_DTYPE dtype, BTLA_DTYPE pv_dtype, int head_dim, bool use_int8_pv) {switch (dtype) {case BTLA_DTYPE::S8:switch (head_dim) {case 128:- if (!use_int8_pv) return launch_prefill_kernel_f16_128_sage;- return launch_prefill_kernel_f16_128_sage_i8pv;+ if (use_int8_pv) {+ if (pv_dtype == BTLA_DTYPE::BF16) return launch_prefill_kernel_bf16_128_sage_i8pv;+ return launch_prefill_kernel_f16_128_sage_i8pv;+ }+ if (pv_dtype == BTLA_DTYPE::BF16) return launch_prefill_kernel_bf16_128_sage;+ return launch_prefill_kernel_f16_128_sage;case 64:- if (!use_int8_pv) return launch_prefill_kernel_f16_64_sage;- return launch_prefill_kernel_f16_64_sage_i8pv;+ if (use_int8_pv) {+ if (pv_dtype == BTLA_DTYPE::BF16) return launch_prefill_kernel_bf16_64_sage_i8pv;+ return launch_prefill_kernel_f16_64_sage_i8pv;+ }+ if (pv_dtype == BTLA_DTYPE::BF16) return launch_prefill_kernel_bf16_64_sage;+ return launch_prefill_kernel_f16_64_sage;default:return nullptr;}@@ -153,15 +179,38 @@ KernelLauncher select_decode_launcher(BTLA_DTYPE dtype, int head_dim) {}}-detail::Options make_common_options(void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int batch,- int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,- float softmax_scale, bool is_causal) {+detail::Options make_common_options(void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int q_stride_s,+ int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,+ int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,+ int head_dim, float softmax_scale, bool is_causal) {+ if (q_stride_d != 1 || k_stride_d != 1 || v_stride_d != 1 || o_stride_d != 1) {+ throw std::invalid_argument("make_common_options: head-dim stride must be 1 for Q/K/V/O");+ }detail::Options options;options.q = Q_ptr;options.k = K_ptr;options.v = V_ptr;options.mask = mask;options.o = O_ptr;+ options.use_tensor_strides = true;+ options.q_stride_s = q_stride_s;+ options.q_stride_d = q_stride_d;+ options.q_stride_h = q_stride_h;+ options.q_stride_b = q_stride_b;+ options.k_stride_s = k_stride_s;+ options.k_stride_d = k_stride_d;+ options.k_stride_h = k_stride_h;+ options.k_stride_b = k_stride_b;+ options.v_stride_d = v_stride_d;+ options.v_stride_s = v_stride_s;+ options.v_stride_h = v_stride_h;+ options.v_stride_b = v_stride_b;+ options.o_stride_s = o_stride_s;+ options.o_stride_d = o_stride_d;+ options.o_stride_h = o_stride_h;+ options.o_stride_b = o_stride_b;options.batch = batch;options.num_heads_q = num_heads_q;options.num_heads_kv = num_heads_kv;@@ -176,32 +225,44 @@ detail::Options make_common_options(void* Q_ptr, void* K_ptr, void* V_ptr, void*} // namespace-void sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,- void* kscale, void* vscale, bool use_int8_pv,- BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {- detail::Options options = make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, batch, num_heads_q, num_heads_kv,- seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+void sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,+ int scale_block_size, void* qscale, void* kscale, void* vscale, bool use_int8_pv,+ BTLA_DTYPE q_dtype, BTLA_DTYPE pv_dtype, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ float softmax_scale, bool is_causal) {+ detail::Options options =+ make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);options.scale_block_size = scale_block_size;options.qscale = qscale;options.kscale = kscale;options.vscale = vscale;compat::set_default_queue(*q);- KernelLauncher launcher = select_sage_prefill_launcher(q_dtype, head_dim, use_int8_pv);+ KernelLauncher launcher = select_sage_prefill_launcher(q_dtype, pv_dtype, head_dim, use_int8_pv);if (launcher == nullptr) {throw std::runtime_error(- "Unsupported dtype or head dimension for sage_prefill / SAGE (only F16/BF16 and 64/96/128/192 are supported)");+ "Unsupported dtype or head dimension for sage_prefill / SAGE (only F16/BF16 PV and 64/128 are supported)");}launcher(options);}void flash_attn_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,- BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,- int head_dim, float softmax_scale, bool is_causal) {- detail::Options options = make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, batch, num_heads_q, num_heads_kv,- seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ BTLA_DTYPE q_dtype, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,+ int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,+ int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+ detail::Options options =+ make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);compat::set_default_queue(*q);KernelLauncher launcher = select_prefill_launcher(q_dtype, head_dim);@@ -214,10 +275,16 @@ void flash_attn_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, v}void flash_attn_decode(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,- BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_kv, int head_dim,- float softmax_scale, bool is_causal) {- detail::Options options = make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, batch, num_heads_q,- num_heads_kv, 1, seq_len_kv, head_dim, softmax_scale, is_causal);+ BTLA_DTYPE q_dtype, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,+ int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+ detail::Options options =+ make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,+ num_heads_kv, 1, seq_len_kv, head_dim, softmax_scale, is_causal);compat::set_default_queue(*q);KernelLauncher launcher = select_decode_launcher(q_dtype, head_dim);@@ -230,8 +297,11 @@ void flash_attn_decode(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, vo}void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, BTLA_DTYPE q_dtype,- int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,- float softmax_scale, bool is_causal) {+ int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b,+ int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch, int num_heads_q,+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal) {// if (q_dtype != BTLA_DTYPE::F16 && q_dtype != BTLA_DTYPE::BF16) {// throw std::invalid_argument("sdpa_impl: only FP16 and BF16 are supported");// }@@ -243,18 +313,25 @@ void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptthrow std::invalid_argument("sdpa_impl: seq_len_q and seq_len_kv must be greater than 0");}if (seq_len_q == 1) {- flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, batch, num_heads_q, num_heads_kv, seq_len_kv,- head_dim, softmax_scale, is_causal);+ flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b,+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,+ seq_len_kv, head_dim, softmax_scale, is_causal);return;}- flash_attn_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, batch, num_heads_q, num_heads_kv, seq_len_q,- seq_len_kv, head_dim, softmax_scale, is_causal);+ flash_attn_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b,+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,+ seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);}-void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,- void* kscale, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,+ int scale_block_size, void* qscale, void* kscale, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h,+ int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE pv_dtype) {if (mask && is_causal) {throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");}@@ -262,22 +339,31 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptrif (seq_len_q <= 0 || seq_len_kv <= 0) {throw std::invalid_argument("sdpa_impl: seq_len_q and seq_len_kv must be greater than 0");}+ if (pv_dtype != BTLA_DTYPE::F16 && pv_dtype != BTLA_DTYPE::BF16) {+ throw std::invalid_argument("sdpa_impl_qks8_pvhalf: only F16 and BF16 are supported for V/O dtype");+ }if (seq_len_q == 1) {- flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, BTLA_DTYPE::F16, batch, num_heads_q, num_heads_kv, seq_len_kv,- head_dim, softmax_scale, is_causal);+ flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, pv_dtype, q_stride_s, q_stride_d, q_stride_h,+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,+ v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,+ num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal);return;}-sage_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, nullptr, false,- BTLA_DTYPE::S8, batch, num_heads_q, num_heads_kv, seq_len_q,+ BTLA_DTYPE::S8, pv_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,+ k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s,+ o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q,seq_len_kv, head_dim, softmax_scale, is_causal);}void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,- int scale_block_size, void* qscale, void* kscale, void* vscale, int batch, int num_heads_q,- int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,- bool is_causal) {+ int scale_block_size, void* qscale, void* kscale, void* vscale, int q_stride_s,+ int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,+ int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,+ int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype) {if (mask && is_causal) {throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");}@@ -288,15 +374,22 @@ void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,if (vscale == nullptr) {throw std::invalid_argument("sdpa_impl_qks8_pvi8: vscale must be provided for int8 PV");}+ if (o_dtype != BTLA_DTYPE::F16 && o_dtype != BTLA_DTYPE::BF16) {+ throw std::invalid_argument("sdpa_impl_qks8_pvi8: only F16 and BF16 are supported for output dtype");+ }if (seq_len_q == 1) {- flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, BTLA_DTYPE::F16, batch, num_heads_q, num_heads_kv,- seq_len_kv, head_dim, softmax_scale, is_causal);+ flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, o_dtype, q_stride_s, q_stride_d, q_stride_h,+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,+ v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,+ num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal);return;}sage_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, vscale, true,- BTLA_DTYPE::S8, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,- is_causal);+ BTLA_DTYPE::S8, o_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,+ k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,+ o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,+ softmax_scale, is_causal);}} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.pyindex aa9f2ac30..17e2864f5 100644--- a/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py+++ b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py@@ -7,7 +7,9 @@ import loggingimport torch-from . import ARK+from . import sagev1, sagev1_pvi8+from . import sdpa as ark_sdpa+from . import xpu_liblogger = logging.getLogger(__name__)@@ -134,8 +136,7 @@ def patch_torch_sdpa_with_ark(*, strict: bool = False, backend: str = "sdpa", quraise RuntimeError("XPU is not available")return False- ark = ARK()- if ark.xpu_lib is None or not hasattr(ark.xpu_lib, backend):+ if xpu_lib is None or not hasattr(xpu_lib, backend):if strict:raise RuntimeError(f"ARK XPU {backend} kernel is not available")return False@@ -186,30 +187,30 @@ def patch_torch_sdpa_with_ark(*, strict: bool = False, backend: str = "sdpa", qutry:if backend == "sdpa":- return ark.sdpa(- query.contiguous(),- key.contiguous(),- value.contiguous(),+ return ark_sdpa(+ query,+ key,+ value,attn_mask=normalized_mask,dropout_p=dropout_p,is_causal=is_causal,scale=scale,)if backend == "sagev1":- return ark.sagev1(- query.contiguous(),- key.contiguous(),- value.contiguous(),+ return sagev1(+ query,+ key,+ value,attn_mask=normalized_mask,dropout_p=dropout_p,is_causal=is_causal,scale=scale,quant_block_size=quant_block_size,)- return ark.sagev1_pvi8(- query.contiguous(),- key.contiguous(),- value.contiguous(),+ return sagev1_pvi8(+ query,+ key,+ value,attn_mask=normalized_mask,dropout_p=dropout_p,is_causal=is_causal,diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hppindex 62850e46b..806dcb0ca 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp@@ -58,17 +58,27 @@ void moe_gemm(sycl::queue* q, void* activations, void* weights, void* scales, vo* @param is_causal Whether to apply causal mask*/void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, BTLA_DTYPE q_dtype,- int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b,+ int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch, int num_heads_q,+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,float softmax_scale, bool is_causal);-void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,- void* kscale, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal);+void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,+ int scale_block_size, void* qscale, void* kscale, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h,+ int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,+ BTLA_DTYPE pv_dtype = BTLA_DTYPE::F16);void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,- int scale_block_size, void* qscale, void* kscale, void* vscale, int batch, int num_heads_q,- int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,- bool is_causal);+ int scale_block_size, void* qscale, void* kscale, void* vscale, int q_stride_s,+ int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,+ int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype = BTLA_DTYPE::F16);#endif // ARK_SYCL_TLA} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hppindex 6c4cdb006..f80a89b47 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp@@ -74,10 +74,15 @@ struct Options {bool is_causal = false;bool varlen = false;bool use_paged_kv = false;+ bool use_tensor_strides = false;int batch = 0, num_heads_q = 0, num_heads_kv = 0, seq_len_qo = 0, seq_len_kv = 0, seq_len_kv_cache = 0, page_size = 0,head_size_qk = 0, head_size_vo = 0;int total_seqlen_q = 0, total_seqlen_kv = 0, total_seqlen_kv_cache = 0;int max_seqlen_q = 0, max_seqlen_kv = 0, max_seqlen_kv_cache = 0;+ int q_stride_s = 0, q_stride_d = 1, q_stride_h = 0, q_stride_b = 0;+ int k_stride_s = 0, k_stride_d = 1, k_stride_h = 0, k_stride_b = 0;+ int v_stride_d = 1, v_stride_s = 0, v_stride_h = 0, v_stride_b = 0;+ int o_stride_s = 0, o_stride_d = 1, o_stride_h = 0, o_stride_b = 0;float softmax_scale = 0.0f;bool persistent = false;@@ -95,6 +100,7 @@ struct Options {<< " is_causal: " << is_causal << "\n"<< " varlen: " << varlen << "\n"<< " use_paged_kv: " << use_paged_kv << "\n"+ << " use_tensor_strides: " << use_tensor_strides << "\n"<< " batch: " << batch << "\n"<< " num_heads_q: " << num_heads_q << "\n"<< " num_heads_kv: " << num_heads_kv << "\n"@@ -110,12 +116,25 @@ struct Options {<< " max_seqlen_q: " << max_seqlen_q << "\n"<< " max_seqlen_kv: " << max_seqlen_kv << "\n"<< " max_seqlen_kv_cache: " << max_seqlen_kv_cache << "\n"+ << " q_stride: (" << q_stride_s << ", " << q_stride_d << ", " << q_stride_h << ", " << q_stride_b << ")\n"+ << " k_stride: (" << k_stride_s << ", " << k_stride_d << ", " << k_stride_h << ", " << k_stride_b << ")\n"+ << " v_stride: (" << v_stride_d << ", " << v_stride_s << ", " << v_stride_h << ", " << v_stride_b << ")\n"+ << " o_stride: (" << o_stride_s << ", " << o_stride_d << ", " << o_stride_h << ", " << o_stride_b << ")\n"<< " softmax_scale: " << softmax_scale << "\n"<< " persistent: " << persistent << "\n"<< "}\n";}};+ template <typename StrideQ, typename StrideK, typename StrideV, typename StrideO>+ inline void set_tensor_strides_from_options(const Options& options, StrideQ& stride_Q, StrideK& stride_K,+ StrideV& stride_V, StrideO& stride_O) {+ stride_Q = cute::make_stride(options.q_stride_s, cute::_1{}, options.q_stride_h, options.q_stride_b);+ stride_K = cute::make_stride(options.k_stride_s, cute::_1{}, options.k_stride_h, options.k_stride_b);+ stride_V = cute::make_stride(cute::_1{}, options.v_stride_s, options.v_stride_h, options.v_stride_b);+ stride_O = cute::make_stride(options.o_stride_s, cute::_1{}, options.o_stride_h, options.o_stride_b);+ }+// 3 input matrices: (K)eys, (Q)ueries and (V)alues.using LayoutQ = cutlass::layout::RowMajor;using LayoutK = cutlass::layout::ColumnMajor;@@ -237,12 +256,23 @@ struct KernelRunner {auto shape_V_cache = cute::make_shape(head_size_vo, seq_len_kv_cache, num_heads_kv, batch);auto shape_O = cute::make_shape(seq_len_qo, head_size_vo, num_heads_q, batch);- stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);- stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);- stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);+ if constexpr (!isVarLen) {+ if (options.use_tensor_strides) {+ set_tensor_strides_from_options(options, stride_Q, stride_K, stride_V, stride_O);+ } else {+ stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);+ stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);+ stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);+ stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);+ }+ } else {+ stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);+ stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);+ stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);+ stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);+ }stride_K_cache = cutlass::make_cute_packed_stride(StrideK{}, shape_K_cache);stride_V_cache = cutlass::make_cute_packed_stride(StrideV{}, shape_V_cache);- stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);if constexpr (isVarLen) {// shape.seq_len_qo.cumulative_length = device_cumulative_seqlen_q.get();@@ -573,12 +603,23 @@ struct SageKernelRunner {auto shape_V_cache = cute::make_shape(head_size_vo, seq_len_kv_cache, num_heads_kv, batch);auto shape_O = cute::make_shape(seq_len_qo, head_size_vo, num_heads_q, batch);- stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);- stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);- stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);+ if constexpr (!isVarLen) {+ if (options.use_tensor_strides) {+ set_tensor_strides_from_options(options, stride_Q, stride_K, stride_V, stride_O);+ } else {+ stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);+ stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);+ stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);+ stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);+ }+ } else {+ stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);+ stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);+ stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);+ stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);+ }stride_K_cache = cutlass::make_cute_packed_stride(StrideK{}, shape_K_cache);stride_V_cache = cutlass::make_cute_packed_stride(StrideV{}, shape_V_cache);- stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);if constexpr (isVarLen) {// shape.seq_len_qo.cumulative_length = device_cumulative_seqlen_q.get();@@ -650,8 +691,12 @@ struct SageConfig {static constexpr int SGTileQ = get<0>(shape_div(TileShapeQK{}, shape(SubgroupLayoutQK{})))();using MMAOperation =cute::conditional_t<is_void_v<MMAOperation_>, XE_DPAS_TT<cute::gcd(SGTileQ, 8), int32_t, int8_t>, MMAOperation_>;+ // The PV "float" tiled MMA operates on the output element type. For UseInt8PV the kernel also+ // constructs a separate int8 quantized PV MMA internally, while this float MMA only needs to+ // describe the tile shape used for the accumulator and dequantized path. For the non-int8-PV+ // path, V is consumed directly so we use ElementO (== ElementV) which supports half_t / bfloat16_t.using MMAOperationPV = cute::conditional_t<is_void_v<MMAOperation_>,- XE_DPAS_TT<cute::gcd(SGTileQ, 8), float, cute::half_t>, MMAOperation_>;+ XE_DPAS_TT<cute::gcd(SGTileQ, 8), float, ElementO>, MMAOperation_>;using SubgroupLayoutPV =cute::conditional_t<is_void_v<SubgroupLayoutPV_>,decltype(cutlass::fmha::collective::get_sg_layout_pv(SubgroupLayoutQK{})), SubgroupLayoutPV_>;@@ -811,8 +856,8 @@ inline int launch_prefill_kernel_128(Options const& options) {template <typename ElementQ, typename ElementK, typename ElementV, typename ElementO = ElementV, bool UseInt8PV = false,bool WriteBackInt8PV = true, bool ExecuteInt8PV = true>inline int launch_sage_prefill_kernel_128(Options const& options) {- constexpr int PipelineStages = 1;- constexpr int PipelineStages1 = 1;+ constexpr int PipelineStages = 2;+ constexpr int PipelineStages1 = 2;using ShapeQK = Shape<_256, _64, _32>;using ShapePV = Shape<_256, _32, _64>;using ShapeOut = Shape<_256, _128>;diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hppindex 8b50ee769..bbf2f2bad 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp@@ -34,6 +34,7 @@ struct env_params {int auto_s8 = 0;int sage_use_mean_bias = 1;int sage_print_kbias = 0;+ int sage_disable_packed_hnd_fast = 0;static env_params* Instance() {static env_params instance;return &instance;@@ -43,6 +44,7 @@ struct env_params {env_i("ARK_AUTO_S8", auto_s8);env_i("ARK_SAGE_USE_MEAN_BIAS", sage_use_mean_bias);env_i("ARK_SAGE_PRINT_KBIAS", sage_print_kbias);+ env_i("ARK_SAGE_DISABLE_PACKED_HND_FAST", sage_disable_packed_hnd_fast);}static inline void env_i(const char* envstr, int& default_) {const char* log_level_env = getenv(envstr);diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hppindex 51a346e2b..2cc82ff19 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp@@ -665,6 +665,18 @@ class XpuWrapper {#endif}+ static inline size_t logical_hnd_offset(int batch_id, int head_id, int seq_id, int dim_id, int stride_seq,+ int stride_dim, int stride_head, int stride_batch) {+ return size_t(batch_id) * size_t(stride_batch) + size_t(head_id) * size_t(stride_head) ++ size_t(seq_id) * size_t(stride_seq) + size_t(dim_id) * size_t(stride_dim);+ }++ static inline bool is_packed_hnd(int stride_seq, int stride_dim, int stride_head, int stride_batch, int num_heads,+ int seq_len, int head_dim) {+ return stride_dim == 1 && stride_seq == head_dim && stride_head == seq_len * head_dim &&+ stride_batch == num_heads * seq_len * head_dim;+ }+static void woq_gemm(int m, const void* a, const void* b, void* c, const void* bias, BTLA_DTYPE acdt, QuantParam* p,sycl::queue* q) {auto ret = woq_gemv(q, m, p, a, b, c, bias, acdt);@@ -739,6 +751,56 @@ class XpuWrapper {});}+ template <typename T>+ static void compute_seq_mean_bias_strided(sycl::queue* q, const T* in_ptr, T* bias_ptr, int batch, int num_heads,+ int seq, int head_dim, int stride_seq, int stride_dim, int stride_head,+ int stride_batch) {+ constexpr int SG_SIZE = 32;+ constexpr int WG_SIZE = 512;+ constexpr int SG_PER_WG = WG_SIZE / SG_SIZE;+ constexpr int SeqUnroll = 4;+ size_t total = size_t(batch) * size_t(num_heads) * size_t(head_dim);+ size_t num_groups = (total + SG_PER_WG - 1) / SG_PER_WG;+ size_t global = num_groups * WG_SIZE;+ q->parallel_for(sycl::nd_range<1>(global, WG_SIZE),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ size_t group_id = item.get_group(0);+ int local_id = item.get_local_id(0);+ int subgroup_id = local_id / SG_SIZE;+ int lane_id = local_id % SG_SIZE;+ size_t index = group_id * SG_PER_WG + subgroup_id;+ if (index >= total) {+ return;+ }++ int row_id = int(index / size_t(head_dim));+ int dim_id = int(index % size_t(head_dim));+ int batch_id = row_id / num_heads;+ int head_id = row_id % num_heads;+ size_t base_offset = logical_hnd_offset(batch_id, head_id, 0, dim_id, stride_seq, stride_dim,+ stride_head, stride_batch);+ auto sg = item.get_sub_group();++ float sum = 0.0f;+ int token = lane_id;+ for (; token + (SeqUnroll - 1) * SG_SIZE < seq; token += SeqUnroll * SG_SIZE) {+#pragma unroll+ for (int unroll_idx = 0; unroll_idx < SeqUnroll; ++unroll_idx) {+ sum += static_cast<float>(in_ptr[base_offset ++ size_t(token + unroll_idx * SG_SIZE) *+ size_t(stride_seq)]);+ }+ }+ for (; token < seq; token += SG_SIZE) {+ sum += static_cast<float>(in_ptr[base_offset + size_t(token) * size_t(stride_seq)]);+ }+ sum = sycl::reduce_over_group(sg, sum, sycl::plus<float>{});+ if (lane_id == 0) {+ bias_ptr[index] = static_cast<T>(sum / static_cast<float>(seq));+ }+ });+ }+template <typename T>static void print_value_distribution(sycl::queue* q, const T* dev_ptr, size_t count, const char* name) {if (dev_ptr == nullptr || count == 0) {@@ -782,262 +844,82 @@ class XpuWrapper {bool fast_vector_bias = bias_ptr && (head_dim % Unroll == 0);bool power_of_two_head_dim = fast_vector_bias && ((head_dim & (head_dim - 1)) == 0);int bias_mask = head_dim - 1;- if (elems_per_block > MAX_Reg * MAX_WG_SIZE) {+ if (true) { // Always use two-pass load for better performance on Intel GPUint wg_size = (elems_per_block <= 256) ? SG_SIZE : 256;// Ensure wg_size is a multiple of SG_SIZEwg_size = ((wg_size + SG_SIZE - 1) / SG_SIZE) * SG_SIZE;- if (bias_ptr) {- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {- int block_id = item.get_group(0);- int row_id = block_id / n_seq_blk;- int seq_id = block_id % n_seq_blk;- int tid = item.get_local_id(0);- auto wg = item.get_group();- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;- auto* row_bias = bias_ptr + (size_t)row_id * head_dim;-- int elems_this_wg = (seq_id + 1) * block_size <= seq ? elems_per_block- : (seq - seq_id * block_size) * head_dim;-- float local_max = 0.0f;- sycl::vec<T, Unroll> local_data, local_max_vec;- local_max_vec = sycl::vec<T, Unroll>(0.0f);- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {- local_data = *(sycl::vec<T, Unroll>*)(&block_in[i]);- if (fast_vector_bias) {- int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);- local_data = local_data - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);- } else {-#pragma unroll- for (int j = 0; j < Unroll; ++j) {- local_data[j] = local_data[j] - row_bias[(i + j) % head_dim];- }- }- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));- }- for (int i = 0; i < Unroll; ++i) {- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));- }- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;+ q->parallel_for(sycl::nd_range<1>(size_t(num_blocks) * wg_size, wg_size),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int block_id = item.get_group(0);+ int row_id = block_id / n_seq_blk;+ int seq_id = block_id % n_seq_blk;+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + (size_t)seq_id * block_size * head_dim;+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + (size_t)seq_id * block_size * head_dim;+ auto* row_bias = bias_ptr ? bias_ptr + (size_t)row_id * head_dim : nullptr;++ int elems_this_wg = (seq_id + 1) * block_size <= seq ? elems_per_block+ : (seq - seq_id * block_size) * head_dim;- if (tid == 0) {- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;- }+ float local_max = 0.0f;+ sycl::vec<T, Unroll> input_vec;+ sycl::vec<float, Unroll> local_max_vec(0.0f);- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {- sycl::vec<T, Unroll> quant_input = *(sycl::vec<T, Unroll>*)(&block_in[i]);+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {+ input_vec = *(sycl::vec<T, Unroll>*)(&block_in[i]);+ sycl::vec<float, Unroll> input_f =+ input_vec.template convert<float, sycl::rounding_mode::automatic>();+ if (row_bias) {if (fast_vector_bias) {int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);- quant_input = quant_input - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);+ input_f = input_f - bias_vec.template convert<float, sycl::rounding_mode::automatic>();} else {#pragma unrollfor (int j = 0; j < Unroll; ++j) {- quant_input[j] = quant_input[j] - row_bias[(i + j) % head_dim];+ input_f[j] = input_f[j] - static_cast<float>(row_bias[(i + j) % head_dim]);}}- sycl::vec<float, Unroll> val =- quant_input.template convert<float, sycl::rounding_mode::automatic>();- val = val * inv_scale;- val = sycl::round(val);- val = sycl::clamp(val, -127, 127);- sycl::vec<int8_t, Unroll> qv =- val.template convert<int8_t, sycl::rounding_mode::automatic>();- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;- }- });- } else {- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {- int block_id = item.get_group(0);- int row_id = block_id / n_seq_blk;- int seq_id = block_id % n_seq_blk;- int tid = item.get_local_id(0);- auto wg = item.get_group();- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;-- int elems_this_wg = (seq_id + 1) * block_size <= seq ? elems_per_block- : (seq - seq_id * block_size) * head_dim;-- float local_max = 0.0f;- sycl::vec<T, Unroll> local_data, local_max_vec;- local_max_vec = sycl::vec<T, Unroll>(0.0f);- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {- local_data = *(sycl::vec<T, Unroll>*)(&block_in[i]);- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));- }- for (int i = 0; i < Unroll; ++i) {- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));- }- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;-- if (tid == 0) {- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;- }-- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {- sycl::vec<T, Unroll> quant_input = *(sycl::vec<T, Unroll>*)(&block_in[i]);- sycl::vec<float, Unroll> val =- quant_input.template convert<float, sycl::rounding_mode::automatic>();- val = val * inv_scale;- val = sycl::round(val);- val = sycl::clamp(val, -127, 127);- sycl::vec<int8_t, Unroll> qv =- val.template convert<int8_t, sycl::rounding_mode::automatic>();- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;}- });- }- } else {- int wg_size = MAX_WG_SIZE;- if (bias_ptr) {- if (fast_vector_bias) {- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {- int seq_id = item.get_group(0);- int row_id = seq_id / n_seq_blk;- seq_id = seq_id % n_seq_blk;- int tid = item.get_local_id(0);- auto wg = item.get_group();- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;- auto* row_bias = bias_ptr + (size_t)row_id * head_dim;-- float local_max = 0.0f;- sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;- local_max_vec = sycl::vec<T, Unroll>(0.0f);- int local_i = 0;- int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim- : (seq - seq_id * block_size) * head_dim;- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {- int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);- local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];- local_data[local_i] = local_data[local_i] - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));- }+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(input_f));+ }#pragma unroll- for (int i = 0; i < Unroll; ++i) {- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));- }- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, local_max_vec[i]);+ }+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;- if (tid == 0) {- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;- }+ if (tid == 0) {+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;+ }- local_i = 0;- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {- sycl::vec<float, Unroll> val =- local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();- val = val * inv_scale;- val = sycl::round(val);- val = sycl::clamp(val, -127, 127);- sycl::vec<int8_t, Unroll> qv =- val.template convert<int8_t, sycl::rounding_mode::automatic>();- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;- }- });- } else {- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {- int seq_id = item.get_group(0);- int row_id = seq_id / n_seq_blk;- seq_id = seq_id % n_seq_blk;- int tid = item.get_local_id(0);- auto wg = item.get_group();- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;- auto* row_bias = bias_ptr + (size_t)row_id * head_dim;-- float local_max = 0.0f;- sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;- local_max_vec = sycl::vec<T, Unroll>(0.0f);- int local_i = 0;- int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim- : (seq - seq_id * block_size) * head_dim;- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {- local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {+ input_vec = *(sycl::vec<T, Unroll>*)(&block_in[i]);+ sycl::vec<float, Unroll> val =+ input_vec.template convert<float, sycl::rounding_mode::automatic>();+ if (row_bias) {+ if (fast_vector_bias) {+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);+ val = val - bias_vec.template convert<float, sycl::rounding_mode::automatic>();+ } else {#pragma unrollfor (int j = 0; j < Unroll; ++j) {- local_data[local_i][j] = local_data[local_i][j] - row_bias[(i + j) % head_dim];+ val[j] = val[j] - static_cast<float>(row_bias[(i + j) % head_dim]);}- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));- }-#pragma unroll- for (int i = 0; i < Unroll; ++i) {- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));- }- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;-- if (tid == 0) {- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;}-- local_i = 0;- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {- sycl::vec<float, Unroll> val =- local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();- val = val * inv_scale;- val = sycl::round(val);- val = sycl::clamp(val, -127, 127);- sycl::vec<int8_t, Unroll> qv =- val.template convert<int8_t, sycl::rounding_mode::automatic>();- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;- }- });- }- } else {- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {- int seq_id = item.get_group(0);- int row_id = seq_id / n_seq_blk;- seq_id = seq_id % n_seq_blk;- int tid = item.get_local_id(0);- auto wg = item.get_group();- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;-- float local_max = 0.0f;- sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;- local_max_vec = sycl::vec<T, Unroll>(0.0f);- int local_i = 0;- int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim- : (seq - seq_id * block_size) * head_dim;- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {- local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));}-#pragma unroll- for (int i = 0; i < Unroll; ++i) {- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));- }- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;-- if (tid == 0) {- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;- }-- local_i = 0;- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {- sycl::vec<float, Unroll> val =- local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();- val = val * inv_scale;- val = sycl::round(val);- val = sycl::clamp(val, -127, 127);- sycl::vec<int8_t, Unroll> qv =- val.template convert<int8_t, sycl::rounding_mode::automatic>();- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;- }- });- }+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127.0f, 127.0f);+ sycl::vec<int8_t, Unroll> qv =+ val.template convert<int8_t, sycl::rounding_mode::automatic>();+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;+ }+ });}}@@ -1075,10 +957,519 @@ class XpuWrapper {});}+ template <typename T>+ static void sage_dynamic_quant_strided(sycl::queue* q, const T* in_ptr, int8_t* out_ptr, float* scale_ptr,+ int batch, int num_heads, int seq, int n_seq_blk, int head_dim,+ int block_size, int stride_seq, int stride_dim, int stride_head,+ int stride_batch, const T* bias_ptr = nullptr) {+ int num_blocks = batch * num_heads * n_seq_blk;+ constexpr int SG_SIZE = 32;+ constexpr int Unroll = 4;+ constexpr int MAX_Reg = 64;+ constexpr int WG_SIZE = 256;+ constexpr int HEADS_PER_WG = WG_SIZE / SG_SIZE;+ constexpr int TOKEN_MAJOR_WG_SIZE = WG_SIZE;+ int elems_per_block = block_size * head_dim;+ bool use_cache = elems_per_block <= MAX_Reg * WG_SIZE;+ bool use_vector_path = stride_dim == 1 && (head_dim % Unroll) == 0;+ bool use_multihead_nhd_q_path = false;+ bool use_token_major_nhd_q_path =+ use_cache && bias_ptr == nullptr && stride_dim == 1 && stride_head == head_dim &&+ stride_seq == num_heads * head_dim &&+ stride_batch == seq * stride_seq && head_dim <= WG_SIZE * Unroll && (head_dim % Unroll) == 0;+ // HND-packed fast path: the whole tensor is canonically packed HND, i.e. dims are contiguous+ // (stride_dim==1), tokens are contiguous within a head (stride_seq==head_dim), heads are+ // contiguous within a batch (stride_head==seq*head_dim), and batches are contiguous+ // (stride_batch==num_heads*seq*head_dim). The kernel addresses memory as+ // in_ptr + row_id*seq*head_dim (row_id = batch_id*num_heads + head_id), which is only valid+ // under these canonical strides; non-canonical head/batch strides (e.g. head slicing or+ // padding) would read the wrong data, so they fall through to the generic strided kernel.+ // This avoids div/mod and per-element stride MADs in the inner loop, matching the perf of+ // the non-strided sage_dynamic_quant kernel.+ bool use_packed_hnd_path = stride_dim == 1 && stride_seq == head_dim &&+ stride_head == seq * head_dim &&+ stride_batch == num_heads * seq * head_dim && (head_dim % Unroll) == 0;+ if (use_multihead_nhd_q_path) {+ int num_head_tiles = (num_heads + HEADS_PER_WG - 1) / HEADS_PER_WG;+ int total_blocks = batch * num_head_tiles * n_seq_blk;+ q->parallel_for(sycl::nd_range<1>(size_t(total_blocks) * WG_SIZE, WG_SIZE),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int block_id = item.get_group(0);+ int head_tile_id = (block_id / n_seq_blk) % num_head_tiles;+ int seq_block_id = block_id % n_seq_blk;+ int batch_id = block_id / (num_head_tiles * n_seq_blk);+ int tid = item.get_local_id(0);+ int head_slot = tid / SG_SIZE;+ int lane_id = tid % SG_SIZE;+ int head_id = head_tile_id * HEADS_PER_WG + head_slot;+ int vecs_per_head = head_dim / Unroll;+ if (head_id >= num_heads || lane_id >= vecs_per_head) {+ return;+ }++ int row_id = batch_id * num_heads + head_id;+ int seq_begin = seq_block_id * block_size;+ int seq_end = std::min(seq_begin + block_size, seq);+ int dim = lane_id * Unroll;+ auto sg = item.get_sub_group();++ float local_max = 0.0f;+ for (int token = seq_begin; token < seq_end; ++token) {+ size_t src_offset = size_t(batch_id) * size_t(stride_batch) + size_t(token) * size_t(stride_seq) ++ size_t(head_id) * size_t(stride_head) + size_t(dim);+ sycl::vec<T, Unroll> input_vec = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);+#pragma unroll+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, sycl::fabs(static_cast<float>(input_vec[i])));+ }+ }++ float absmax = sycl::reduce_over_group(sg, local_max, sycl::maximum<float>{});+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;+ if (lane_id == 0) {+ scale_ptr[size_t(row_id) * size_t(n_seq_blk) + size_t(seq_block_id)] = scale;+ }++ for (int token = seq_begin; token < seq_end; ++token) {+ size_t src_offset = size_t(batch_id) * size_t(stride_batch) + size_t(token) * size_t(stride_seq) ++ size_t(head_id) * size_t(stride_head) + size_t(dim);+ sycl::vec<T, Unroll> input_vec = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);+ sycl::vec<float, Unroll> val =+ input_vec.template convert<float, sycl::rounding_mode::automatic>();+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127, 127);+ sycl::vec<int8_t, Unroll> qv =+ val.template convert<int8_t, sycl::rounding_mode::automatic>();+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) ++ size_t(dim);+ *(sycl::vec<int8_t, Unroll>*)(&out_ptr[dst_offset]) = qv;+ }+ });+ return;+ }+ if (use_token_major_nhd_q_path) {+ q->parallel_for(sycl::nd_range<1>(size_t(num_blocks) * TOKEN_MAJOR_WG_SIZE, TOKEN_MAJOR_WG_SIZE),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int block_id = item.get_group(0);+ int row_id = block_id / n_seq_blk;+ int seq_block_id = block_id % n_seq_blk;+ int batch_id = row_id / num_heads;+ int head_id = row_id % num_heads;+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ int seq_begin = seq_block_id * block_size;+ int seq_end = std::min(seq_begin + block_size, seq);+ int vecs_per_token = head_dim / Unroll;+ int tokens_per_iter = TOKEN_MAJOR_WG_SIZE / vecs_per_token;+ int vec_id = tid % vecs_per_token;+ int token_slot = tid / vecs_per_token;+ int dim = vec_id * Unroll;+ float local_max = 0.0f;+ sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll];+ sycl::vec<float, Unroll> local_max_vec(0.0f);+ int local_i = 0;++ for (int token = seq_begin + token_slot; token < seq_end; token += tokens_per_iter, local_i++) {+ size_t src_offset = size_t(batch_id) * size_t(stride_batch) + size_t(token) * size_t(stride_seq) ++ size_t(head_id) * size_t(stride_head) + size_t(dim);+ local_data[local_i] = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);+ sycl::vec<float, Unroll> data_f =+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(data_f));+ }+ #pragma unroll+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, local_max_vec[i]);+ }++ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;+ if (tid == 0) {+ scale_ptr[block_id] = scale;+ }++ local_i = 0;+ for (int token = seq_begin + token_slot; token < seq_end; token += tokens_per_iter, local_i++) {+ sycl::vec<float, Unroll> val =+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127, 127);+ sycl::vec<int8_t, Unroll> qv =+ val.template convert<int8_t, sycl::rounding_mode::automatic>();+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) ++ size_t(dim);+ *(sycl::vec<int8_t, Unroll>*)(&out_ptr[dst_offset]) = qv;+ }+ });+ return;+ }+ if (use_packed_hnd_path) {+ bool fast_vector_bias = (head_dim % Unroll == 0);+ bool power_of_two_head_dim = fast_vector_bias && ((head_dim & (head_dim - 1)) == 0);+ int bias_mask = head_dim - 1;+ q->parallel_for(+ sycl::nd_range<1>(size_t(num_blocks) * WG_SIZE, WG_SIZE),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int block_id = item.get_group(0);+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ int row_id = block_id / n_seq_blk;+ int seq_block_id = block_id % n_seq_blk;+ int seq_begin = seq_block_id * block_size;+ int seq_end = std::min(seq_begin + block_size, seq);+ int elems_this_wg = (seq_end - seq_begin) * head_dim;++ const T* block_in = in_ptr + (size_t)row_id * seq * head_dim + (size_t)seq_begin * head_dim;+ int8_t* block_out = out_ptr + (size_t)row_id * seq * head_dim + (size_t)seq_begin * head_dim;+ const T* row_bias = bias_ptr ? bias_ptr + (size_t)row_id * head_dim : nullptr;++ float local_max = 0.0f;+ sycl::vec<T, Unroll> input_vec;+ sycl::vec<float, Unroll> local_max_vec(0.0f);+ for (int i = tid * Unroll; i < elems_this_wg; i += WG_SIZE * Unroll) {+ input_vec = *(sycl::vec<T, Unroll>*)(&block_in[i]);+ sycl::vec<float, Unroll> input_f =+ input_vec.template convert<float, sycl::rounding_mode::automatic>();+ if (row_bias) {+ if (fast_vector_bias) {+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);+ input_f = input_f - bias_vec.template convert<float, sycl::rounding_mode::automatic>();+ } else {+#pragma unroll+ for (int j = 0; j < Unroll; ++j) {+ input_f[j] = input_f[j] - static_cast<float>(row_bias[(i + j) % head_dim]);+ }+ }+ }+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(input_f));+ }+#pragma unroll+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, local_max_vec[i]);+ }+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;+ if (tid == 0) {+ scale_ptr[block_id] = absmax / 127.0f;+ }++ for (int i = tid * Unroll; i < elems_this_wg; i += WG_SIZE * Unroll) {+ input_vec = *(sycl::vec<T, Unroll>*)(&block_in[i]);+ sycl::vec<float, Unroll> val = input_vec.template convert<float, sycl::rounding_mode::automatic>();+ if (row_bias) {+ if (fast_vector_bias) {+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);+ val = val - bias_vec.template convert<float, sycl::rounding_mode::automatic>();+ } else {+#pragma unroll+ for (int j = 0; j < Unroll; ++j) {+ val[j] = val[j] - static_cast<float>(row_bias[(i + j) % head_dim]);+ }+ }+ }+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127.0f, 127.0f);+ sycl::vec<int8_t, Unroll> qv = val.template convert<int8_t, sycl::rounding_mode::automatic>();+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;+ }+ });+ return;+ }+ q->parallel_for(sycl::nd_range<1>(size_t(num_blocks) * WG_SIZE, WG_SIZE),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int block_id = item.get_group(0);+ int row_id = block_id / n_seq_blk;+ int seq_block_id = block_id % n_seq_blk;+ int batch_id = row_id / num_heads;+ int head_id = row_id % num_heads;+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ int seq_begin = seq_block_id * block_size;+ int seq_end = std::min(seq_begin + block_size, seq);+ int elems_this_wg = (seq_end - seq_begin) * head_dim;+ const T* row_bias = bias_ptr ? bias_ptr + size_t(row_id) * size_t(head_dim) : nullptr;++ if (use_vector_path && use_cache) {+ float local_max = 0.0f;+ sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll];+ sycl::vec<float, Unroll> local_max_vec(0.0f);+ int local_i = 0;+ for (int linear_idx = tid * Unroll; linear_idx < elems_this_wg;+ linear_idx += WG_SIZE * Unroll, local_i++) {+ int token_rel = linear_idx / head_dim;+ int dim = linear_idx % head_dim;+ size_t src_offset = logical_hnd_offset(batch_id, head_id, seq_begin + token_rel, dim,+ stride_seq, stride_dim, stride_head, stride_batch);+ local_data[local_i] = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);+ sycl::vec<float, Unroll> data_f =+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();+ if (row_bias) {+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[dim]);+ data_f = data_f - bias_vec.template convert<float, sycl::rounding_mode::automatic>();+ }+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(data_f));+ }+#pragma unroll+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, local_max_vec[i]);+ }++ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;+ if (tid == 0) {+ scale_ptr[block_id] = scale;+ }++ local_i = 0;+ for (int linear_idx = tid * Unroll; linear_idx < elems_this_wg;+ linear_idx += WG_SIZE * Unroll, local_i++) {+ int token_rel = linear_idx / head_dim;+ int dim = linear_idx % head_dim;+ sycl::vec<T, Unroll> quant_input = local_data[local_i];+ sycl::vec<float, Unroll> val =+ quant_input.template convert<float, sycl::rounding_mode::automatic>();+ if (row_bias) {+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[dim]);+ val = val - bias_vec.template convert<float, sycl::rounding_mode::automatic>();+ }+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127, 127);+ sycl::vec<int8_t, Unroll> qv =+ val.template convert<int8_t, sycl::rounding_mode::automatic>();+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(seq_begin + token_rel)) *+ size_t(head_dim) ++ size_t(dim);+ *(sycl::vec<int8_t, Unroll>*)(&out_ptr[dst_offset]) = qv;+ }+ } else if (use_vector_path) {+ // use_cache == false: block too large to cache in registers; recompute+ // from memory in the second pass instead of storing into local_data.+ float local_max = 0.0f;+ sycl::vec<float, Unroll> local_max_vec(0.0f);+ for (int linear_idx = tid * Unroll; linear_idx < elems_this_wg;+ linear_idx += WG_SIZE * Unroll) {+ int token_rel = linear_idx / head_dim;+ int dim = linear_idx % head_dim;+ size_t src_offset = logical_hnd_offset(batch_id, head_id, seq_begin + token_rel, dim,+ stride_seq, stride_dim, stride_head, stride_batch);+ sycl::vec<T, Unroll> input_vec = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);+ sycl::vec<float, Unroll> data_f =+ input_vec.template convert<float, sycl::rounding_mode::automatic>();+ if (row_bias) {+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[dim]);+ data_f = data_f - bias_vec.template convert<float, sycl::rounding_mode::automatic>();+ }+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(data_f));+ }+#pragma unroll+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, local_max_vec[i]);+ }++ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;+ if (tid == 0) {+ scale_ptr[block_id] = scale;+ }++ for (int linear_idx = tid * Unroll; linear_idx < elems_this_wg;+ linear_idx += WG_SIZE * Unroll) {+ int token_rel = linear_idx / head_dim;+ int dim = linear_idx % head_dim;+ size_t src_offset = logical_hnd_offset(batch_id, head_id, seq_begin + token_rel, dim,+ stride_seq, stride_dim, stride_head, stride_batch);+ sycl::vec<T, Unroll> quant_input = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);+ sycl::vec<float, Unroll> val =+ quant_input.template convert<float, sycl::rounding_mode::automatic>();+ if (row_bias) {+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[dim]);+ val = val - bias_vec.template convert<float, sycl::rounding_mode::automatic>();+ }+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127, 127);+ sycl::vec<int8_t, Unroll> qv =+ val.template convert<int8_t, sycl::rounding_mode::automatic>();+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(seq_begin + token_rel)) *+ size_t(head_dim) ++ size_t(dim);+ *(sycl::vec<int8_t, Unroll>*)(&out_ptr[dst_offset]) = qv;+ }+ } else {+ float local_max = 0.0f;+ T local_data[MAX_Reg];+ int local_count = 0;+ for (int linear_idx = tid; linear_idx < elems_this_wg; linear_idx += WG_SIZE) {+ int token_rel = linear_idx / head_dim;+ int dim = linear_idx % head_dim;+ size_t src_offset = logical_hnd_offset(batch_id, head_id, seq_begin + token_rel, dim,+ stride_seq, stride_dim, stride_head, stride_batch);+ float value = static_cast<float>(in_ptr[src_offset]);+ if (row_bias) {+ value -= static_cast<float>(row_bias[dim]);+ }+ if (use_cache) {+ local_data[local_count++] = static_cast<T>(value);+ }+ local_max = sycl::fmax(local_max, sycl::fabs(value));+ }++ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;+ if (tid == 0) {+ scale_ptr[block_id] = scale;+ }++ local_count = 0;+ for (int linear_idx = tid; linear_idx < elems_this_wg; linear_idx += WG_SIZE) {+ int token_rel = linear_idx / head_dim;+ int dim = linear_idx % head_dim;+ float value = 0.0f;+ if (use_cache) {+ value = static_cast<float>(local_data[local_count++]);+ } else {+ size_t src_offset = logical_hnd_offset(batch_id, head_id, seq_begin + token_rel, dim,+ stride_seq, stride_dim, stride_head,+ stride_batch);+ value = static_cast<float>(in_ptr[src_offset]);+ if (row_bias) {+ value -= static_cast<float>(row_bias[dim]);+ }+ }+ value *= inv_scale;+ value = sycl::round(value);+ value = sycl::clamp(value, -127.0f, 127.0f);+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(seq_begin + token_rel)) *+ size_t(head_dim) ++ size_t(dim);+ out_ptr[dst_offset] = static_cast<int8_t>(value);+ }+ }+ });+ }++ template <typename T>+ static void sage_dynamic_quant_v_strided(sycl::queue* q, const T* in_ptr, int8_t* out_ptr, float* scale_ptr,+ int batch, int num_heads, int seq, int n_seq_blk, int head_dim,+ int block_size, int stride_dim, int stride_seq, int stride_head,+ int stride_batch) {+ size_t num_scales = size_t(batch) * size_t(num_heads) * size_t(n_seq_blk) * size_t(head_dim);+ constexpr int SG_SIZE = 32;+ constexpr int WG_SIZE = 512;+ constexpr int SG_PER_WG = WG_SIZE / SG_SIZE;+ if (block_size <= SG_SIZE * 2) {+ q->parallel_for(sycl::range<1>(num_scales), [=](sycl::id<1> item) {+ size_t linear_idx = item[0];+ int dim = int(linear_idx % size_t(head_dim));+ size_t block_idx = linear_idx / size_t(head_dim);+ int seq_id = int(block_idx % size_t(n_seq_blk));+ int row_id = int(block_idx / size_t(n_seq_blk));+ int batch_id = row_id / num_heads;+ int head_id = row_id % num_heads;+ int seq_begin = seq_id * block_size;+ int seq_end = std::min(seq_begin + block_size, seq);++ float absmax = 0.0f;+ for (int token = seq_begin; token < seq_end; ++token) {+ size_t src_offset =+ logical_hnd_offset(batch_id, head_id, token, dim, stride_seq, stride_dim, stride_head, stride_batch);+ float value = static_cast<float>(in_ptr[src_offset]);+ absmax = sycl::fmax(absmax, sycl::fabs(value));+ }++ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;+ scale_ptr[linear_idx] = scale;++ for (int token = seq_begin; token < seq_end; ++token) {+ size_t src_offset =+ logical_hnd_offset(batch_id, head_id, token, dim, stride_seq, stride_dim, stride_head, stride_batch);+ float value = static_cast<float>(in_ptr[src_offset]) * inv_scale;+ value = sycl::round(value);+ value = sycl::clamp(value, -127.0f, 127.0f);+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) + size_t(dim);+ out_ptr[dst_offset] = static_cast<int8_t>(value);+ }+ });+ return;+ }++ size_t num_groups = (num_scales + SG_PER_WG - 1) / SG_PER_WG;+ size_t global = num_groups * WG_SIZE;+ q->parallel_for(sycl::nd_range<1>(global, WG_SIZE),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ size_t group_id = item.get_group(0);+ int local_id = item.get_local_id(0);+ int subgroup_id = local_id / SG_SIZE;+ int lane_id = local_id % SG_SIZE;+ size_t linear_idx = group_id * SG_PER_WG + subgroup_id;+ if (linear_idx >= num_scales) {+ return;+ }++ int dim = int(linear_idx % size_t(head_dim));+ size_t block_idx = linear_idx / size_t(head_dim);+ int seq_id = int(block_idx % size_t(n_seq_blk));+ int row_id = int(block_idx / size_t(n_seq_blk));+ int batch_id = row_id / num_heads;+ int head_id = row_id % num_heads;+ int seq_begin = seq_id * block_size;+ int seq_end = std::min(seq_begin + block_size, seq);+ auto sg = item.get_sub_group();++ float local_max = 0.0f;+ for (int token = seq_begin + lane_id; token < seq_end; token += SG_SIZE) {+ size_t src_offset = logical_hnd_offset(batch_id, head_id, token, dim, stride_seq, stride_dim,+ stride_head, stride_batch);+ float value = static_cast<float>(in_ptr[src_offset]);+ local_max = sycl::fmax(local_max, sycl::fabs(value));+ }++ float absmax = sycl::reduce_over_group(sg, local_max, sycl::maximum<float>{});+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;+ if (lane_id == 0) {+ scale_ptr[linear_idx] = scale;+ }++ for (int token = seq_begin + lane_id; token < seq_end; token += SG_SIZE) {+ size_t src_offset = logical_hnd_offset(batch_id, head_id, token, dim, stride_seq, stride_dim,+ stride_head, stride_batch);+ float value = static_cast<float>(in_ptr[src_offset]) * inv_scale;+ value = sycl::round(value);+ value = sycl::clamp(value, -127.0f, 127.0f);+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) ++ size_t(dim);+ out_ptr[dst_offset] = static_cast<int8_t>(value);+ }+ });+ }++ template <typename T = sycl::half>static void sagev1_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,- int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, bool use_int8_pv) {+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,+ int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,+ int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,+ bool use_int8_pv) {bool use_mean_bias = env_params::Instance()->sage_use_mean_bias != 0;+ bool q_packed = is_packed_hnd(q_stride_s, q_stride_d, q_stride_h, q_stride_b, num_heads_q, seq_len_q, head_dim);+ bool k_packed =+ is_packed_hnd(k_stride_s, k_stride_d, k_stride_h, k_stride_b, num_heads_kv, seq_len_kv, head_dim);+ bool v_packed =+ is_packed_hnd(v_stride_s, v_stride_d, v_stride_h, v_stride_b, num_heads_kv, seq_len_kv, head_dim);size_t seq_q_blk = (seq_len_q + scale_block_size - 1) / scale_block_size;size_t seq_kv_blk = (seq_len_kv + scale_block_size - 1) / scale_block_size;size_t q_size = num_heads_q * seq_len_q * head_dim * batch;@@ -1087,7 +1478,7 @@ class XpuWrapper {size_t k_scale_size = num_heads_kv * seq_kv_blk * batch;size_t v_tmp_size = use_int8_pv ? k_size : 0;size_t v_scale_size = use_int8_pv ? k_scale_size * head_dim * sizeof(float) : 0;- size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * batch * sizeof(sycl::half) : 0;+ size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * batch * sizeof(T) : 0;size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size +v_scale_size + k_bias_size;auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);@@ -1102,44 +1493,115 @@ class XpuWrapper {k_scale_size * sizeof(float) + k_size): nullptr;auto kbias = use_mean_bias- ? (sycl::half*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) +- k_scale_size * sizeof(float) + v_tmp_size + v_scale_size)+ ? (T*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) ++ k_scale_size * sizeof(float) + v_tmp_size + v_scale_size): nullptr;- sage_dynamic_quant<sycl::half>(q, (sycl::half*)Q_ptr, (int8_t*)q_out_ptr, (float*)qscale, batch * num_heads_q,- seq_len_q, seq_q_blk, head_dim, scale_block_size);+ if (q_packed) {+ sage_dynamic_quant<T>(q, (T*)Q_ptr, (int8_t*)q_out_ptr, (float*)qscale, batch * num_heads_q,+ seq_len_q, seq_q_blk, head_dim, scale_block_size);+ } else {+ sage_dynamic_quant_strided<T>(q, (T*)Q_ptr, (int8_t*)q_out_ptr, (float*)qscale, batch,+ num_heads_q, seq_len_q, seq_q_blk, head_dim, scale_block_size,+ q_stride_s, q_stride_d, q_stride_h, q_stride_b);+ }if (use_mean_bias) {- compute_seq_mean_bias<sycl::half>(q, (sycl::half*)K_ptr, kbias, batch * num_heads_kv, seq_len_kv, head_dim);+ if (k_packed) {+ compute_seq_mean_bias<T>(q, (T*)K_ptr, kbias, batch * num_heads_kv, seq_len_kv, head_dim);+ } else {+ compute_seq_mean_bias_strided<T>(q, (T*)K_ptr, kbias, batch, num_heads_kv, seq_len_kv,+ head_dim, k_stride_s, k_stride_d, k_stride_h, k_stride_b);+ }if (env_params::Instance()->sage_print_kbias != 0) {print_value_distribution(q, kbias, size_t(batch) * num_heads_kv * head_dim, "kbias");}}- sage_dynamic_quant<sycl::half>(q, (sycl::half*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch * num_heads_kv,- seq_len_kv, seq_kv_blk, head_dim, scale_block_size, kbias);+ if (k_packed) {+ sage_dynamic_quant<T>(q, (T*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch * num_heads_kv,+ seq_len_kv, seq_kv_blk, head_dim, scale_block_size, kbias);+ } else {+ sage_dynamic_quant_strided<T>(q, (T*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch,+ num_heads_kv, seq_len_kv, seq_kv_blk, head_dim, scale_block_size,+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, kbias);+ }+ int q_out_stride_s = head_dim;+ int q_out_stride_d = 1;+ int q_out_stride_h = seq_len_q * head_dim;+ int q_out_stride_b = num_heads_q * seq_len_q * head_dim;+ int k_out_stride_s = head_dim;+ int k_out_stride_d = 1;+ int k_out_stride_h = seq_len_kv * head_dim;+ int k_out_stride_b = num_heads_kv * seq_len_kv * head_dim;+ int v_out_stride_d = 1;+ int v_out_stride_s = head_dim;+ int v_out_stride_h = seq_len_kv * head_dim;+ int v_out_stride_b = num_heads_kv * seq_len_kv * head_dim;+ constexpr BTLA_DTYPE pv_dtype = std::is_same<T, sycl::half>::value ? BTLA_DTYPE::F16 : BTLA_DTYPE::BF16;if (use_int8_pv) {- sage_dynamic_quant_v<sycl::half>(q, (sycl::half*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale,- batch * num_heads_kv, seq_len_kv, seq_kv_blk, head_dim, scale_block_size);+ if (v_packed) {+ sage_dynamic_quant_v<T>(q, (T*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale,+ batch * num_heads_kv, seq_len_kv, seq_kv_blk, head_dim, scale_block_size);+ } else {+ sage_dynamic_quant_v_strided<T>(q, (T*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale, batch,+ num_heads_kv, seq_len_kv, seq_kv_blk, head_dim, scale_block_size,+ v_stride_d, v_stride_s, v_stride_h, v_stride_b);+ }ark::sdpa_impl_qks8_pvi8(q, q_out_ptr, k_out_ptr, v_out_ptr, O_ptr, mask, scale_block_size, qscale, kscale,- vscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,- softmax_scale, is_causal);+ vscale, q_out_stride_s, q_out_stride_d, q_out_stride_h, q_out_stride_b,+ k_out_stride_s, k_out_stride_d, k_out_stride_h, k_out_stride_b, v_out_stride_d,+ v_out_stride_s, v_out_stride_h, v_out_stride_b, o_stride_s, o_stride_d, o_stride_h,+ o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,+ softmax_scale, is_causal, pv_dtype);} else {ark::sdpa_impl_qks8_pvhalf(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale,- batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,- is_causal);+ q_out_stride_s, q_out_stride_d, q_out_stride_h, q_out_stride_b, k_out_stride_s,+ k_out_stride_d, k_out_stride_h, k_out_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, pv_dtype);}}static void sagev1(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,- int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,- int head_dim, float softmax_scale, bool is_causal) {- sagev1_impl(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q,- seq_len_kv, head_dim, softmax_scale, is_causal, false);+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,+ int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,+ BTLA_DTYPE dtype = BTLA_DTYPE::F16) {+ if (dtype == BTLA_DTYPE::BF16) {+ sagev1_impl<sycl::ext::oneapi::bfloat16>(+ q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,+ o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,+ head_dim, softmax_scale, is_causal, false);+ } else {+ sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,+ q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,+ v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,+ is_causal, false);+ }}static void sagev1_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,- int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {- sagev1_impl(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q,- seq_len_kv, head_dim, softmax_scale, is_causal, true);+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,+ int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,+ int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,+ BTLA_DTYPE dtype = BTLA_DTYPE::F16) {+ if (dtype == BTLA_DTYPE::BF16) {+ sagev1_impl<sycl::ext::oneapi::bfloat16>(+ q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,+ o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,+ head_dim, softmax_scale, is_causal, true);+ } else {+ sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,+ q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,+ v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,+ is_causal, true);+ }}};diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hppindex ac7d75eb7..b73137715 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp@@ -20,9 +20,11 @@ struct TestSDPA {#if defined(ARK_XPU) && defined(ARK_SYCL_TLA) && ARK_XPU && ARK_SYCL_TLA// test_sagev1_accuracy("prefill_gqa_128_bias", 1, 32, 8, 256, 512, 128, 128, false, true);// test_sagev1_accuracy("prefill_gqa_128_no_bias", 1, 32, 8, 256, 512, 128, 128, false, false);- test_sagev1_accuracy("prefill_gqa_128_causal_no_bias", 1, 32, 8, 256, 512, 128, 128, true, false);- benchmark_sagev1("bench_prefill_gqa_128_s4096", 1, 32, 8, 4096, 4096, 128, 128, false, 5, 20);- benchmark_sagev1("bench_bmg_hq96_d128_k8192", 1, 96, 8, 4096, 8192, 128, 128, false, 5, 20);+ // test_sagev1_accuracy(\"prefill_gqa_128_causal_no_bias\", 1, 32, 8, 256, 512, 128, 128, true, false);+ benchmark_dynamic_quant("dq_b1_h32_s4096_d128", 1, 32, 4096, 128, 128, 5, 50);+ benchmark_dynamic_quant("dq_b1_h96_s8192_d128", 1, 96, 8192, 128, 128, 5, 50);+ // benchmark_sagev1("bench_prefill_gqa_128_s4096", 1, 32, 8, 4096, 4096, 128, 128, false, 5, 20);+ // benchmark_sagev1("bench_bmg_hq96_d128_k8192", 1, 96, 8, 4096, 8192, 128, 128, false, 5, 20);#elsestd::cout << "[sagev1] skipped: requires ARK_XPU and ARK_SYCL_TLA\n";#endif@@ -79,6 +81,7 @@ struct TestSDPA {return std::chrono::duration<double, std::milli>(end - start).count() / double(iters);}+#if 0 // legacy benchmarks rely on outdated sdpa_impl_qks8_* signaturesvoid test_sagev1_accuracy(const std::string& name, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,int seq_len_kv, int head_dim, int scale_block_size, bool is_causal,bool use_mean_bias) {@@ -283,5 +286,55 @@ struct TestSDPA {ctx->deallocate(dev_kscale);ctx->deallocate(dev_vscale);}+#endif // legacy benchmarks++ template <typename T>+ void run_dynamic_quant_bench(const char* tag, sycl::queue* q, int num_rows, int seq, int n_seq_blk,+ int head_dim, int block_size, int warmup, int iters) {+ auto ctx = Context::Instance();+ size_t in_count = size_t(num_rows) * size_t(seq) * size_t(head_dim);+ size_t scale_count = size_t(num_rows) * size_t(n_seq_blk);+ auto* dev_in = reinterpret_cast<T*>(ctx->allocate(in_count * sizeof(T)));+ auto* dev_out = reinterpret_cast<int8_t*>(ctx->allocate(in_count * sizeof(int8_t)));+ auto* dev_scale = reinterpret_cast<float*>(ctx->allocate(scale_count * sizeof(float)));++ // initialize input with random fp values cast to T+ auto host_f = make_random_vector(in_count, -1.0f, 1.0f, 1234u + uint32_t(num_rows + seq));+ std::vector<T> host_t(in_count);+ for (size_t i = 0; i < in_count; ++i) host_t[i] = T(host_f[i]);+ q->memcpy(dev_in, host_t.data(), in_count * sizeof(T)).wait();++ double ms = run_bench(+ [&]() {+ ark::XpuWrapper::sage_dynamic_quant<T>(q, dev_in, dev_out, dev_scale, num_rows, seq, n_seq_blk,+ head_dim, block_size);+ },+ q, warmup, iters);++ double bytes = double(in_count) * double(sizeof(T)) + double(in_count) * 1.0 ++ double(scale_count) * double(sizeof(float));+ double gbps = bytes / (ms * 1e-3) / 1e9;+ std::cout << std::fixed << std::setprecision(4) << "[dq][" << tag << "] ms=" << ms+ << " bytes=" << bytes / 1e6 << "MB GBps=" << gbps << "\n";++ ctx->deallocate(dev_in);+ ctx->deallocate(dev_out);+ ctx->deallocate(dev_scale);+ }++ void benchmark_dynamic_quant(const std::string& name, int batch, int num_heads, int seq,+ int head_dim, int block_size, int warmup, int iters) {+ GETQ();+ LOG_LINE();+ int num_rows = batch * num_heads;+ int n_seq_blk = (seq + block_size - 1) / block_size;+ std::cout << "[dq][cfg] " << name << " rows=" << num_rows << " seq=" << seq+ << " head_dim=" << head_dim << " block=" << block_size+ << " n_seq_blk=" << n_seq_blk << "\n";+ run_dynamic_quant_bench<sycl::half>("f16", q, num_rows, seq, n_seq_blk, head_dim, block_size, warmup,+ iters);+ run_dynamic_quant_bench<sycl::ext::oneapi::bfloat16>("bf16", q, num_rows, seq, n_seq_blk, head_dim,+ block_size, warmup, iters);+ }#endif};diff --git a/auto_round_extension/ark/test/test_bench_bmg.py b/auto_round_extension/ark/test/test_bench_bmg.pyindex e4bddd382..d0762d830 100644--- a/auto_round_extension/ark/test/test_bench_bmg.py+++ b/auto_round_extension/ark/test/test_bench_bmg.py@@ -23,7 +23,7 @@ ark = Nonedef get_ark():global arkif ark is None:- ark = auto_round_kernel.ARK()+ ark = auto_round_kernelreturn arkdiff --git a/auto_round_extension/ark/test/test_flash_attn.py b/auto_round_extension/ark/test/test_flash_attn.pyindex ade89ee22..27aad5426 100644--- a/auto_round_extension/ark/test/test_flash_attn.py+++ b/auto_round_extension/ark/test/test_flash_attn.py@@ -27,7 +27,7 @@ import auto_round_kernelimport pytestimport torch-ark = auto_round_kernel.ARK()+ark = auto_round_kerneldef is_xpu_available():diff --git a/auto_round_extension/ark/test/test_matmul.py b/auto_round_extension/ark/test/test_matmul.pyindex 6debd7685..6b26aa337 100644--- a/auto_round_extension/ark/test/test_matmul.py+++ b/auto_round_extension/ark/test/test_matmul.py@@ -22,7 +22,7 @@ import auto_round_kernelimport pytestimport torch-ark = auto_round_kernel.ARK()+ark = auto_round_kerneldef main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device):diff --git a/auto_round_extension/ark/test/test_moe.py b/auto_round_extension/ark/test/test_moe.pyindex 1134f7b4d..360b603b6 100644--- a/auto_round_extension/ark/test/test_moe.py+++ b/auto_round_extension/ark/test/test_moe.py@@ -25,7 +25,7 @@ import auto_round_kernelimport pytestimport torch-ark = auto_round_kernel.ARK()+ark = auto_round_kerneldef is_xpu_available():diff --git a/auto_round_extension/ark/test/test_packq.py b/auto_round_extension/ark/test/test_packq.pyindex c77d6449c..d8c4aa0d6 100644--- a/auto_round_extension/ark/test/test_packq.py+++ b/auto_round_extension/ark/test/test_packq.py@@ -22,7 +22,7 @@ import pytestimport torchfrom ut_utils import compare2, decode_fp8_to_float, gen_weis8, get_scale_torch_dt, get_torch_dt, sample_valid_fp8-ark = auto_round_kernel.ARK()+ark = auto_round_kerneldef main_op(k, n, weight_type, scale_type, asym, blocksize, device, compute_type):diff --git a/auto_round_extension/ark/test/test_sage_dynquant.py b/auto_round_extension/ark/test/test_sage_dynquant.pyindex 72beb2fb4..5a2c3c4ac 100644--- a/auto_round_extension/ark/test/test_sage_dynquant.py+++ b/auto_round_extension/ark/test/test_sage_dynquant.py@@ -19,7 +19,7 @@ ark = Nonedef get_ark():global arkif ark is None:- ark = auto_round_kernel.ARK()+ ark = auto_round_kernelreturn arkdiff --git a/auto_round_extension/ark/test/test_sdpa.py b/auto_round_extension/ark/test/test_sdpa.pyindex 4aaabdc14..2e5091035 100644--- a/auto_round_extension/ark/test/test_sdpa.py+++ b/auto_round_extension/ark/test/test_sdpa.py@@ -251,8 +251,8 @@ def get_ark():global arkif ark is None:if not is_xpu_available():- raise RuntimeError("XPU is not available; cannot initialize auto_round_kernel.ARK()")- ark = auto_round_kernel.ARK()+ raise RuntimeError("XPU is not available; cannot initialize auto_round_kernel")+ ark = auto_round_kernelreturn ark@@ -618,44 +618,67 @@ def compare_sdpa_sage_scale(batch, seq, seq_kv, h_q, h_kv, H, H_v, dt=torch.floadef compare_sdpa_sage_dynquant(- batch, seq, seq_kv, h_q, h_kv, H, H_v, dt=torch.float16, is_causal=False, dev="xpu", quant_block_size=64+ batch,+ seq,+ seq_kv,+ h_q,+ h_kv,+ H,+ H_v,+ dt=torch.float16,+ is_causal=False,+ dev="xpu",+ quant_block_size=64,+ tensor_layout="HND",):"""Test dynamic quantization SAGE attention.- Q, K, V are FP16 inputs. The kernel dynamically quantizes Q, K to INT8+ Q, K, V are FP16/BF16 inputs. The kernel dynamically quantizes Q, K to INT8.+ tensor_layout: 'HND' = [B, H, N, D]; 'NHD' = [B, N, H, D]."""group = h_q // h_kv- q = torch.randn(batch, h_q, seq, H, dtype=dt, device=dev)- k = torch.randn(batch, h_kv, seq_kv, H, dtype=dt, device=dev)- v = torch.randn(batch, h_kv, seq_kv, H_v, dtype=dt, device=dev) * 100+ # Always build reference in HND for SDPA, then permute to requested layout.+ q_hnd = torch.randn(batch, h_q, seq, H, dtype=dt, device=dev)+ k_hnd = torch.randn(batch, h_kv, seq_kv, H, dtype=dt, device=dev)+ v_hnd = torch.randn(batch, h_kv, seq_kv, H_v, dtype=dt, device=dev) * 100+ if tensor_layout == "NHD":+ q = q_hnd.transpose(1, 2).contiguous()+ k = k_hnd.transpose(1, 2).contiguous()+ v = v_hnd.transpose(1, 2).contiguous()+ else:+ q, k, v = q_hnd, k_hnd, v_hndscale = 1 / math.sqrt(H)if seq_kv > 4096 and is_causal:if seq_kv == seq:ref = torch.nn.functional.scaled_dot_product_attention(- q, k, v, scale=scale, enable_gqa=group > 1, is_causal=is_causal+ q_hnd, k_hnd, v_hnd, scale=scale, enable_gqa=group > 1, is_causal=is_causal)else:- ref = torch.zeros_like(q)+ ref = torch.zeros_like(q_hnd)else:- attn_bias = torch.zeros(seq, seq_kv, dtype=q.dtype, device=q.device)- temp_mask = torch.ones(seq, seq_kv, dtype=torch.bool, device=q.device).tril(diagonal=seq_kv - seq)+ attn_bias = torch.zeros(seq, seq_kv, dtype=q_hnd.dtype, device=q_hnd.device)+ temp_mask = torch.ones(seq, seq_kv, dtype=torch.bool, device=q_hnd.device).tril(diagonal=seq_kv - seq)attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf"))scale = 1 / math.sqrt(H)if is_causal:ref = torch.nn.functional.scaled_dot_product_attention(- q, k, v, scale=scale, enable_gqa=group > 1, is_causal=False, attn_mask=attn_bias+ q_hnd, k_hnd, v_hnd, scale=scale, enable_gqa=group > 1, is_causal=False, attn_mask=attn_bias)else:ref = torch.nn.functional.scaled_dot_product_attention(- q, k, v, scale=scale, enable_gqa=group > 1, is_causal=is_causal+ q_hnd, k_hnd, v_hnd, scale=scale, enable_gqa=group > 1, is_causal=is_causal)+ if tensor_layout == "NHD":+ ref = ref.transpose(1, 2).contiguous()# Dynamic quantization SAGE kernel- out = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size)+ out = get_ark().sagev1(+ q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size, tensor_layout=tensor_layout+ )dff = abs(ref - out)- print(f"=== Dynamic Quantization SAGE Test (block_size={quant_block_size}) ===")+ print(f"=== Dynamic Quantization SAGE Test (block_size={quant_block_size}, layout={tensor_layout}, dt={dt}) ===")print(f"Batch:{batch} Seq_Q:{seq}, Seq_KV:{seq_kv}, HeadNum_Q:{h_q}, HeadNum_KV:{h_kv}, HeadDim_QK:{H}, HeadDim_V:{H_v}, Causal:{is_causal}")@@ -664,12 +687,16 @@ def compare_sdpa_sage_dynquant(# Benchmarkn_runs = 10 if seq > 8192 else 100for i in range(n_runs):- _ = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size)+ _ = get_ark().sagev1(+ q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size, tensor_layout=tensor_layout+ )if dev == "xpu":torch.xpu.synchronize()st = time.time()for i in range(n_runs):- _ = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size)+ _ = get_ark().sagev1(+ q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size, tensor_layout=tensor_layout+ )if dev == "xpu":torch.xpu.synchronize()et = time.time()@@ -684,7 +711,51 @@ def compare_sdpa_sage_dynquant(MEM += h_kv * seq_kv * H_v * dt.itemsize # vMEM *= batchprint(f"Time:{dur*1e3:.3f} FLOPS:{OPS/dur/1e9:.2f} G, MEM:{MEM/dur/1e9:.2f} GB/s")- return out, ref+ return {+ "layout": tensor_layout,+ "dtype": "fp16" if dt == torch.float16 else ("bf16" if dt == torch.bfloat16 else str(dt)),+ "causal": is_causal,+ "time_ms": dur * 1e3,+ "tflops": OPS / dur / 1e12,+ "max_diff": float(dff.max().item()),+ "mean_diff": float(dff.mean().item()),+ }+++def run_dynquant_layout_dtype_table(+ batch=1, seq=8192, seq_kv=8192, h_q=96, h_kv=8, H=128, H_v=128, quant_block_size=64, dev="xpu"+):+ """Sweep {layout} x {dtype} x {causal} for sage_dynquant and print a markdown table."""+ rows = []+ for layout in ("HND", "NHD"):+ for dt in (torch.float16, torch.bfloat16):+ for causal in (False, True):+ m = compare_sdpa_sage_dynquant(+ batch,+ seq,+ seq_kv,+ h_q,+ h_kv,+ H,+ H_v,+ dt=dt,+ is_causal=causal,+ dev=dev,+ quant_block_size=quant_block_size,+ tensor_layout=layout,+ )+ rows.append(m)++ print()+ print(f"=== sage_dynquant sweep: B={batch} Hq={h_q} Hkv={h_kv} S={seq} D={H} blk={quant_block_size} ===")+ print("| layout | dtype | causal | time (ms) | TFLOPS | max diff | mean diff |")+ print("|---|---|---|---|---|---|---|")+ for r in rows:+ print(+ f"| {r['layout']} | {r['dtype']} | {'T' if r['causal'] else 'F'} | "+ f"{r['time_ms']:.3f} | {r['tflops']:.2f} | {r['max_diff']:.4g} | {r['mean_diff']:.4g} |"+ )+ return rowsif __name__ == "__main__":@@ -732,8 +803,7 @@ if __name__ == "__main__":# bench_ark(1, 4096, 8192*1, 96, 8, 64, 64, dt=torch.float16, is_causal=True)# compare_sdpa_sage_scale(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16)# compare_sdpa_sage_scale(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16, is_causal=True)- compare_sdpa_sage_dynquant(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16)- compare_sdpa_sage_dynquant(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16, is_causal=True)+ run_dynquant_layout_dtype_table(batch=1, seq=8192, seq_kv=8192, h_q=96, h_kv=8, H=128, H_v=128, quant_block_size=64)# compare_sdpa_sage_scale(2, 4096, 4096, 96, 8, 128, 128, dt=torch.float16)# compare_sdpa_sage_scale(1, 4096, 4096, 96, 8, 128, 128, dt=torch.float16, is_causal=True)# compare_sdpa_sage(16, 512, 512, 32, 32, 128, 128, dt=torch.float16)diff --git a/auto_round_extension/ark/test/test_sdpa_parity.py b/auto_round_extension/ark/test/test_sdpa_parity.pyindex 77efc6f79..3b261e176 100644--- a/auto_round_extension/ark/test/test_sdpa_parity.py+++ b/auto_round_extension/ark/test/test_sdpa_parity.py@@ -2,17 +2,76 @@# SPDX-License-Identifier: Apache-2.0import math+import sys+from pathlib import Path-import auto_round_kernelimport pytestimport torch+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++import auto_round_kernel+pytestmark = pytest.mark.skipif(not (hasattr(torch, "xpu") and torch.xpu.is_available()),reason="XPU not available",)+def _make_noncontiguous_attention_inputs(layout: str):+ torch.manual_seed(3030)+ batch, heads, seq, head_dim = 1, 2, 64, 64+ q_hnd = torch.randn(batch, heads, seq * 2, head_dim, device="xpu", dtype=torch.float16)[:, :, ::2, :]+ k_hnd = torch.randn(batch, heads, seq * 2, head_dim, device="xpu", dtype=torch.float16)[:, :, ::2, :]+ v_hnd = torch.randn(batch, heads, seq * 2, head_dim, device="xpu", dtype=torch.float16)[:, :, ::2, :]++ if layout == "HND":+ return q_hnd, k_hnd, v_hnd+ if layout == "NHD":+ return q_hnd.transpose(1, 2), k_hnd.transpose(1, 2), v_hnd.transpose(1, 2)+ raise ValueError(f"Unsupported layout: {layout}")+++ ("api_name", "atol", "rtol"),+ [("sdpa", 1e-2, 1e-2), ("sagev1", 2e-2, 2e-2)],+)[email protected]("layout", ["HND", "NHD"])+def test_ark_attention_supports_noncontiguous_inputs_for_layouts(api_name, atol, rtol, layout):+ q, k, v = _make_noncontiguous_attention_inputs(layout)+ scale = 1 / math.sqrt(q.shape[-1])++ expected = torch.nn.functional.scaled_dot_product_attention(+ q if layout == "HND" else q.transpose(1, 2),+ k if layout == "HND" else k.transpose(1, 2),+ v if layout == "HND" else v.transpose(1, 2),+ scale=scale,+ is_causal=False,+ )+ if layout == "NHD":+ expected = expected.transpose(1, 2)++ ark = auto_round_kernel+ if api_name == "sdpa":+ actual = ark.sdpa(q, k, v, scale=scale, is_causal=False, tensor_layout=layout)+ else:+ actual = ark.sagev1(+ q,+ k,+ v,+ scale=scale,+ is_causal=False,+ quant_block_size=64,+ tensor_layout=layout,+ )+ torch.xpu.synchronize()++ assert not q.is_contiguous()+ assert not k.is_contiguous()+ assert not v.is_contiguous()+ torch.testing.assert_close(actual, expected, atol=atol, rtol=rtol)++@pytest.mark.parametrize("seq_q,seq_kv", [(64, 64), (80, 64), (64, 80), (226, 226)])def test_ark_sdpa_matches_torch_for_kv_remainders(seq_q, seq_kv):torch.manual_seed(2026)@@ -29,7 +88,7 @@ def test_ark_sdpa_matches_torch_for_kv_remainders(seq_q, seq_kv):scale=scale,is_causal=False,)- actual = auto_round_kernel.ARK().sdpa(q, k, v, scale=scale, is_causal=False)+ actual = auto_round_kernel.sdpa(q, k, v, scale=scale, is_causal=False)torch.xpu.synchronize()torch.testing.assert_close(actual, expected, atol=1e-2, rtol=1e-2)@@ -57,7 +116,7 @@ def test_ark_sagev1_matches_torch_for_kv_remainder_tile():scale=scale,is_causal=False,)- actual = auto_round_kernel.ARK().sagev1(+ actual = auto_round_kernel.sagev1(q,k,v,diff --git a/auto_round_extension/ark/test/test_weightonly.py b/auto_round_extension/ark/test/test_weightonly.pyindex d26e843fd..c81aed0cd 100644--- a/auto_round_extension/ark/test/test_weightonly.py+++ b/auto_round_extension/ark/test/test_weightonly.py@@ -23,7 +23,7 @@ import pytestimport torchfrom ut_utils import decode_fp8_to_float, gen_weis8, get_torch_dt, sample_valid_fp8, sample_valid_fp8_e8m0_xpu_safe-ark = auto_round_kernel.ARK()+ark = auto_round_kerneldef main_op(m, n, k, blocksize, compute_type, weight_type, scale_type, asym, device, is_ci=False):