Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[ARK] Add bf16 + NHD layout support, refactor sage_dynamic_quant (#1882)

Luo Yu08c5d6d4b94e
diff --git a/auto_round_extension/ark/README.md b/auto_round_extension/ark/README.md
index c087147d9..5b66ccc63 100644
--- a/auto_round_extension/ark/README.md
+++ b/auto_round_extension/ark/README.md
@@ -44,6 +44,8 @@ pip install auto-round-lib
### 2. Install from Source
```bash
+pip install . --no-build-isolation
+# or
python setup.py bdist_wheel;pip install dist/*
```
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.py
index 167cd0c08..b7b80dc6d 100644
--- a/auto_round_extension/ark/auto_round_kernel/__init__.py
+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py
@@ -93,951 +93,1053 @@ def get_stream(A: torch.Tensor) -> int:
return torch.xpu.current_stream().sycl_queue
-def singleton(cls):
- """
- 一个简单的单例模式装饰器
- """
- instances = {} # 存储类与实例的映射关系
+def _normalize_tensor_layout(tensor_layout: str) -> str:
+ layout = tensor_layout.upper()
+ if layout not in ("HND", "NHD"):
+ raise ValueError(f"tensor_layout must be either 'HND' or 'NHD', got {tensor_layout!r}")
+ return layout
+
+
+def _attention_shape(tensor: torch.Tensor, tensor_layout: str) -> tuple[int, int, int, int]:
+ layout = _normalize_tensor_layout(tensor_layout)
+ if layout == "HND":
+ batch, num_heads, seq_len, head_dim = tensor.shape
+ else:
+ batch, seq_len, num_heads, head_dim = tensor.shape
+ return batch, num_heads, seq_len, head_dim
- def get_instance(*args, **kwargs):
- # 如果类不在字典中,则创建一个并存入
- if cls not in instances:
- instances[cls] = cls(*args, **kwargs)
- # 如果已经在字典中,直接返回之前创建的那个
- return instances[cls]
- return get_instance
+def _attention_strides_qko(tensor: torch.Tensor, tensor_layout: str) -> tuple[int, int, int, int]:
+ layout = _normalize_tensor_layout(tensor_layout)
+ if layout == "HND":
+ batch_stride, head_stride, seq_stride, dim_stride = tensor.stride()
+ else:
+ batch_stride, seq_stride, head_stride, dim_stride = tensor.stride()
+ return seq_stride, dim_stride, head_stride, batch_stride
+
+
+def _attention_strides_v(tensor: torch.Tensor, tensor_layout: str) -> tuple[int, int, int, int]:
+ layout = _normalize_tensor_layout(tensor_layout)
+ if layout == "HND":
+ batch_stride, head_stride, seq_stride, dim_stride = tensor.stride()
+ else:
+ batch_stride, seq_stride, head_stride, dim_stride = tensor.stride()
+ return dim_stride, seq_stride, head_stride, batch_stride
+
+
+def _validate_attention_tensor(
+ tensor: torch.Tensor,
+ name: str,
+ tensor_layout: str,
+ *,
+ expected_dtype: torch.dtype | None = None,
+) -> tuple[int, int, int, int]:
+ if tensor.ndim != 4:
+ raise ValueError(f"{name} must be a 4D tensor")
+ if expected_dtype is not None and tensor.dtype != expected_dtype:
+ raise ValueError(f"{name} must have dtype {expected_dtype}, got {tensor.dtype}")
+
+ qko_strides = _attention_strides_qko(tensor, tensor_layout)
+ if qko_strides[1] != 1:
+ raise ValueError(f"{name} must be contiguous along the head-dim axis; got stride {qko_strides[1]}")
+ if any(stride <= 0 for stride in qko_strides):
+ raise ValueError(f"{name} must have positive non-zero strides, got {tensor.stride()}")
+
+ return _attention_shape(tensor, tensor_layout)
+
+
+def _empty_attention_output(
+ batch: int,
+ num_heads: int,
+ seq_len: int,
+ head_dim: int,
+ *,
+ dtype: torch.dtype,
+ device: torch.device,
+ tensor_layout: str,
+) -> torch.Tensor:
+ layout = _normalize_tensor_layout(tensor_layout)
+ shape = (batch, num_heads, seq_len, head_dim) if layout == "HND" else (batch, seq_len, num_heads, head_dim)
+ return torch.empty(shape, device=device, dtype=dtype)
-@singleton
-class ARK:
+# -----------------------------------------------------------------------------
+# Module-level lib loading (replaces the previous singleton ``ARK`` class).
+# -----------------------------------------------------------------------------
+
+cpu_lib = None
+xpu_lib = None
+
+try:
+ from . import auto_round_kernel_cpu as _cpu_lib_mod
+
+ cpu_lib = _cpu_lib_mod
+except ImportError as _e:
+ print(f"ARK is unable to load CPU lib: {_e}")
cpu_lib = None
- xpu_lib = None
-
- def __init__(self):
- try:
- from . import auto_round_kernel_cpu
-
- self.cpu_lib = auto_round_kernel_cpu
- except ImportError as e:
- print(f"ARK is unable to load CPU lib: {e}")
- self.cpu_lib = None
-
- if torch.xpu.is_available():
- try:
- from . import auto_round_kernel_xpu
-
- self.xpu_lib = auto_round_kernel_xpu
- except ImportError as e:
- print(f"ARK is unable to load XPU lib: {e}")
- self.xpu_lib = None
-
- def get_lib(self, A: torch.Tensor):
- lib = None
- if A.device.type == "xpu":
- lib = self.xpu_lib
- if A.device.type == "cpu":
- lib = self.cpu_lib
- if lib is None:
- raise NotImplementedError(f"Current device {A.device} is not supported")
- return lib
-
- # A: mxk, B: nxk, bias: n
- def matmul(self, A: torch.Tensor, B: torch.Tensor, bias: torch.Tensor):
- m = A.shape[0]
- n = B.shape[0]
- k = B.shape[1]
- lib = self.get_lib(A)
- ctype = A.dtype
- if A.device.type == "cpu":
- ctype = torch.float32
- C = torch.zeros(m, n, dtype=ctype, device=A.device)
- stream = get_stream(A)
- lib.matmul(
- stream,
- m,
- n,
- k,
- A.contiguous().data_ptr(),
- cvt_dtype(A.dtype),
- B.contiguous().data_ptr(),
- cvt_dtype(B.dtype),
- C.contiguous().data_ptr(),
- cvt_dtype(C.dtype),
- bias.to(C.dtype).contiguous().data_ptr(),
- True,
- )
- return C
-
- # A: mxk:s8, B: nxk:s8, return: mxn:s32
- def igemm_s8s8s32(self, A: torch.Tensor, B: torch.Tensor):
- m = A.shape[0]
- n = B.shape[0]
- k = B.shape[1]
- lib = self.get_lib(A)
- if lib is None:
- raise NotImplementedError(f"Current device {A.device} is not supported")
- C = torch.zeros(m, n, dtype=torch.int32, device=A.device)
- stream = get_stream(A)
- lib.matmul(
- stream,
- m,
- n,
- k,
- A.contiguous().data_ptr(),
- cvt_dtype(A.dtype),
- B.contiguous().data_ptr(),
- cvt_dtype(B.dtype),
- C.contiguous().data_ptr(),
- cvt_dtype(C.dtype),
- 0,
- True,
- )
- return C
-
- # A: mxk:DT, B: nxk:s8, scaleB: n:DT
- # return: mxn:DT
- def woqgemm_s8(self, A: torch.Tensor, B: torch.Tensor, scaleB: torch.Tensor, bias: torch.Tensor):
- m = A.shape[0]
- n = B.shape[0]
- k = B.shape[1]
- lib = self.get_lib(A)
-
- C = torch.zeros(m, n, dtype=A.dtype, device=A.device)
- stream = get_stream(A)
- lib.woqgemm_s8(
- stream,
- m,
- n,
- k,
- A.contiguous().data_ptr(),
- cvt_dtype(A.dtype),
- B.contiguous().data_ptr(),
- C.contiguous().data_ptr(),
- bias.contiguous().data_ptr(),
- True,
- scaleB.contiguous().data_ptr(),
- )
- return C
-
- # A: mxk:DT, B: BS:s8, bias: n:DT
- # return: C: mxn:DT
- def woqgemm(
- self,
- A: torch.Tensor,
- B: torch.Tensor,
- bias: torch.Tensor,
+
+if torch.xpu.is_available():
+ try:
+ from . import auto_round_kernel_xpu as _xpu_lib_mod
+
+ xpu_lib = _xpu_lib_mod
+ except ImportError as _e:
+ print(f"ARK is unable to load XPU lib: {_e}")
+ xpu_lib = None
+
+
+def get_lib(A: torch.Tensor):
+ lib = None
+ if A.device.type == "xpu":
+ lib = xpu_lib
+ if A.device.type == "cpu":
+ lib = cpu_lib
+ if lib is None:
+ raise NotImplementedError(f"Current device {A.device} is not supported")
+ return lib
+
+
+# A: mxk, B: nxk, bias: n
+def matmul(A: torch.Tensor, B: torch.Tensor, bias: torch.Tensor):
+ m = A.shape[0]
+ n = B.shape[0]
+ k = B.shape[1]
+ lib = get_lib(A)
+ ctype = A.dtype
+ if A.device.type == "cpu":
+ ctype = torch.float32
+ C = torch.zeros(m, n, dtype=ctype, device=A.device)
+ stream = get_stream(A)
+ lib.matmul(
+ stream,
+ m,
n,
k,
+ A.contiguous().data_ptr(),
+ cvt_dtype(A.dtype),
+ B.contiguous().data_ptr(),
+ cvt_dtype(B.dtype),
+ C.contiguous().data_ptr(),
+ cvt_dtype(C.dtype),
+ bias.to(C.dtype).contiguous().data_ptr(),
+ True,
+ )
+ return C
+
+
+# A: mxk:s8, B: nxk:s8, return: mxn:s32
+def igemm_s8s8s32(A: torch.Tensor, B: torch.Tensor):
+ m = A.shape[0]
+ n = B.shape[0]
+ k = B.shape[1]
+ lib = get_lib(A)
+ if lib is None:
+ raise NotImplementedError(f"Current device {A.device} is not supported")
+ C = torch.zeros(m, n, dtype=torch.int32, device=A.device)
+ stream = get_stream(A)
+ lib.matmul(
+ stream,
+ m,
+ n,
+ k,
+ A.contiguous().data_ptr(),
+ cvt_dtype(A.dtype),
+ B.contiguous().data_ptr(),
+ cvt_dtype(B.dtype),
+ C.contiguous().data_ptr(),
+ cvt_dtype(C.dtype),
+ 0,
+ True,
+ )
+ return C
+
+
+# A: mxk:DT, B: nxk:s8, scaleB: n:DT
+# return: mxn:DT
+def woqgemm_s8(A: torch.Tensor, B: torch.Tensor, scaleB: torch.Tensor, bias: torch.Tensor):
+ m = A.shape[0]
+ n = B.shape[0]
+ k = B.shape[1]
+ lib = get_lib(A)
+
+ C = torch.zeros(m, n, dtype=A.dtype, device=A.device)
+ stream = get_stream(A)
+ lib.woqgemm_s8(
+ stream,
+ m,
+ n,
+ k,
+ A.contiguous().data_ptr(),
+ cvt_dtype(A.dtype),
+ B.contiguous().data_ptr(),
+ C.contiguous().data_ptr(),
+ bias.contiguous().data_ptr(),
+ True,
+ scaleB.contiguous().data_ptr(),
+ )
+ return C
+
+
+# A: mxk:DT, B: BS:s8, bias: n:DT
+# return: C: mxn:DT
+def woqgemm(
+ A: torch.Tensor,
+ B: torch.Tensor,
+ bias: torch.Tensor,
+ n,
+ k,
+ groupsize,
+ compute_type,
+ weight_type,
+ scale_type,
+ asym,
+):
+ m = A.shape[0]
+ lib = get_lib(A)
+ ct = cvtstr_dtype(compute_type)
+ wt = cvtstr_dtype(weight_type)
+ st = cvtstr_dtype(scale_type)
+ C = torch.zeros(m, n, dtype=A.dtype, device=A.device)
+ stream = get_stream(A)
+ lib.woqgemm(
+ stream,
+ m,
+ n,
+ k,
+ A.contiguous().data_ptr(),
+ cvt_dtype(A.dtype),
+ B.contiguous().data_ptr(),
+ C.contiguous().data_ptr(),
+ bias.contiguous().data_ptr(),
groupsize,
- compute_type,
- weight_type,
- scale_type,
+ ct,
+ wt,
+ st,
asym,
- ):
- m = A.shape[0]
- lib = self.get_lib(A)
- ct = cvtstr_dtype(compute_type)
- wt = cvtstr_dtype(weight_type)
- st = cvtstr_dtype(scale_type)
- C = torch.zeros(m, n, dtype=A.dtype, device=A.device)
- stream = get_stream(A)
- lib.woqgemm(
- stream,
- m,
- n,
- k,
- A.contiguous().data_ptr(),
- cvt_dtype(A.dtype),
- B.contiguous().data_ptr(),
- C.contiguous().data_ptr(),
- bias.contiguous().data_ptr(),
- groupsize,
- ct,
- wt,
- st,
- asym,
- )
- return C
-
- # QB: k*n:int8, scaleB: k/blocksize*n:DT
- # return: blob:BS:int8
- def repack_quantized_weight(
- self,
- QB: torch.Tensor,
- scaleB: torch.Tensor,
- zp: torch.Tensor,
+ )
+ return C
+
+
+# QB: k*n:int8, scaleB: k/blocksize*n:DT
+# return: blob:BS:int8
+def _repack_quantized_weight_core(
+ QB: torch.Tensor,
+ scaleB: torch.Tensor,
+ zp: torch.Tensor,
+ groupsize,
+ compute_type,
+ weight_type,
+ scale_type,
+ asym,
+):
+ k = QB.shape[0]
+ n = QB.shape[1]
+ lib = get_lib(QB)
+ stream = get_stream(QB)
+ ct = cvtstr_dtype(compute_type)
+ wt = cvtstr_dtype(weight_type)
+ st = cvtstr_dtype(scale_type)
+ BS = lib.packed_weight_size(stream, n, k, groupsize, ct, wt, st, asym)
+ blob = torch.zeros(BS, dtype=torch.int8, device=QB.device)
+ lib.repack_quantized_weight(
+ stream,
+ QB.contiguous().data_ptr(),
+ zp.contiguous().data_ptr(),
+ scaleB.contiguous().data_ptr(),
+ blob.data_ptr(),
+ n,
+ k,
groupsize,
- compute_type,
- weight_type,
- scale_type,
+ ct,
+ wt,
+ st,
asym,
- ):
- k = QB.shape[0]
- n = QB.shape[1]
- lib = self.get_lib(QB)
- stream = get_stream(QB)
- ct = cvtstr_dtype(compute_type)
- wt = cvtstr_dtype(weight_type)
- st = cvtstr_dtype(scale_type)
- BS = lib.packed_weight_size(stream, n, k, groupsize, ct, wt, st, asym)
- blob = torch.zeros(BS, dtype=torch.int8, device=QB.device)
- lib.repack_quantized_weight(
- stream,
- QB.contiguous().data_ptr(),
- zp.contiguous().data_ptr(),
- scaleB.contiguous().data_ptr(),
- blob.data_ptr(),
- n,
- k,
- groupsize,
- ct,
- wt,
- st,
- asym,
- )
- return blob
-
- # QB: blob:BS:int8
- # return: out:nxk:out_dtype
- def unpack_weight(
- self,
- blob: torch.Tensor,
- out_dtype: torch.dtype,
+ )
+ return blob
+
+
+# QB: blob:BS:int8
+# return: out:nxk:out_dtype
+def _unpack_weight_core(
+ blob: torch.Tensor,
+ out_dtype: torch.dtype,
+ n,
+ k,
+ groupsize,
+ compute_type,
+ weight_type,
+ scale_type,
+ asym,
+):
+ lib = get_lib(blob)
+ stream = get_stream(blob)
+ ct = cvtstr_dtype(compute_type)
+ wt = cvtstr_dtype(weight_type)
+
+ st = cvtstr_dtype(scale_type)
+ oshape = (n, k) if blob.device.type == "xpu" else (k, n)
+ out = torch.zeros(oshape, dtype=out_dtype, device=blob.device)
+ lib.unpack_weight(
+ stream,
+ blob.data_ptr(),
+ out.data_ptr(),
+ cvt_dtype(out_dtype),
n,
k,
groupsize,
- compute_type,
- weight_type,
- scale_type,
+ ct,
+ wt,
+ st,
asym,
- ):
- lib = self.get_lib(blob)
- stream = get_stream(blob)
- ct = cvtstr_dtype(compute_type)
- wt = cvtstr_dtype(weight_type)
-
- st = cvtstr_dtype(scale_type)
- oshape = (n, k) if blob.device.type == "xpu" else (k, n)
- out = torch.zeros(oshape, dtype=out_dtype, device=blob.device)
- lib.unpack_weight(
- stream,
- blob.data_ptr(),
- out.data_ptr(),
- cvt_dtype(out_dtype),
- n,
- k,
- groupsize,
- ct,
- wt,
- st,
- asym,
- )
- if blob.device.type == "cpu":
- return out.T
- return out
-
- def sdpa(
- self,
- query: torch.Tensor,
- key: torch.Tensor,
- value: torch.Tensor,
- attn_mask: torch.Tensor | None = None,
- dropout_p: float = 0.0,
- is_causal: bool = False,
- scale: float | None = None,
- ) -> torch.Tensor:
- """Scaled dot-product attention (SDPA) prefill+decode.
-
- Expects contiguous layouts:
- - query: [B, Hq, Sq, D]
- - key: [B, Hkv, Skv, D]
- - value: [B, Hkv, Skv, D]
-
- Args:
- - scale: Softmax scale. Uses 1 / sqrt(D) when None.
-
- Returns:
- - O: [B, Hq, Sq, D] (same dtype as value)
- """
- if query.device.type != "xpu":
- raise NotImplementedError("sdpa is only supported on XPU")
-
- if query.dtype not in (torch.float16, torch.bfloat16):
- raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")
- if key.dtype != query.dtype or value.dtype != query.dtype:
- raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")
-
- if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:
- raise ValueError("Q/K/V must be 4D tensors")
-
- if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():
- raise ValueError("Q/K/V must be contiguous")
-
- B, Hq, Sq, D = query.shape
- Bk, Hkv, Skv, Dk = key.shape
- Bv, Hkv2, Skv2, Dv = value.shape
-
- if Bk != B or Bv != B:
- raise ValueError("Batch size mismatch between Q/K/V")
- if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
- raise ValueError("K/V shape mismatch")
- if Dk != D:
- raise ValueError("Head dim mismatch between Q and K/V")
- if D not in (64, 128, 96, 192):
- raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128, 96, 192")
-
- if dropout_p != 0.0:
- raise NotImplementedError(f"dropout_p must be 0.0 (got {dropout_p}); dropout is not supported")
-
- if attn_mask is not None:
- if attn_mask.device.type != "xpu":
- raise ValueError("attn_mask must be on XPU")
- if not attn_mask.is_contiguous():
- raise ValueError("attn_mask must be contiguous")
- if attn_mask.dtype != torch.float32:
- raise ValueError(f"attn_mask must be float32 (additive bias), got {attn_mask.dtype}")
- expected_mask_shape = (B, 1, Sq, Skv)
- if attn_mask.shape != expected_mask_shape:
- raise ValueError(f"attn_mask shape must be {expected_mask_shape}, got {tuple(attn_mask.shape)}")
-
- lib = self.get_lib(query)
- stream = get_stream(query)
- O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)
- lib.sdpa(
- stream,
- query.data_ptr(),
- key.data_ptr(),
- value.data_ptr(),
- O.data_ptr(),
- attn_mask.data_ptr() if attn_mask is not None else 0,
- cvt_dtype(query.dtype),
- cvt_dtype(key.dtype),
- cvt_dtype(O.dtype),
- B,
- Hq,
- Hkv,
- Sq,
- Skv,
- D,
- float(scale) if scale is not None else 1.0 / (D**0.5),
- bool(is_causal),
- )
- return O
-
- def sage(
- self,
- query: torch.Tensor,
- key: torch.Tensor,
- value: torch.Tensor,
- attn_mask: torch.Tensor | None = None,
- dropout_p: float = 0.0,
- is_causal: bool = False,
- scale: float | None = None,
- enable_gqa: bool = False,
- quant_block_size: int = 64,
- qscale: torch.Tensor = None,
- kscale: torch.Tensor = None,
- ) -> torch.Tensor:
- """SAGE attention prefill+decode.
-
- Expects contiguous layouts:
- - query: [B, Hq, Sq, D]
- - key: [B, Hkv, Skv, D]
- - value: [B, Hkv, Skv, D]
-
- Args:
- - scale: Attention scale. Uses 1 / sqrt(D) when None.
- - quant_block_size: Block size for qscale and kscale.
-
- Returns:
- - O: [B, Hq, Sq, D] (same dtype as value)
- """
- if query.device.type != "xpu":
- raise NotImplementedError("sdpa is only supported on XPU")
-
- # if query.dtype not in (torch.float16, torch.bfloat16):
- # raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")
- # if key.dtype != query.dtype or value.dtype != query.dtype:
- # raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")
-
- if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:
- raise ValueError("Q/K/V must be 4D tensors")
-
- if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():
- raise ValueError("Q/K/V must be contiguous")
-
- B, Hq, Sq, D = query.shape
- Bk, Hkv, Skv, Dk = key.shape
- Bv, Hkv2, Skv2, Dv = value.shape
-
- if Bk != B or Bv != B:
- raise ValueError("Batch size mismatch between Q/K/V")
- if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
- raise ValueError("K/V shape mismatch")
- if Dk != D:
- raise ValueError("Head dim mismatch between Q and K/V")
- if D not in (64, 128):
- raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
-
- lib = self.get_lib(query)
- stream = get_stream(query)
- O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)
- lib.sage(
- stream,
- query.data_ptr(),
- key.data_ptr(),
- value.data_ptr(),
- O.data_ptr(),
- attn_mask.data_ptr() if attn_mask is not None else 0,
- quant_block_size,
- qscale.data_ptr() if qscale is not None else 0,
- kscale.data_ptr() if kscale is not None else 0,
- cvt_dtype(query.dtype),
- cvt_dtype(key.dtype),
- cvt_dtype(O.dtype),
- B,
- Hq,
- Hkv,
- Sq,
- Skv,
- D,
- float(scale) if scale is not None else 1.0 / (D**0.5),
- bool(is_causal),
- )
- return O
-
- def sage_pvi8(
- self,
- query: torch.Tensor,
- key: torch.Tensor,
- value: torch.Tensor,
- attn_mask: torch.Tensor | None = None,
- dropout_p: float = 0.0,
- is_causal: bool = False,
- scale: float | None = None,
- enable_gqa: bool = False,
- quant_block_size: int = 64,
- qscale: torch.Tensor = None,
- kscale: torch.Tensor = None,
- vscale: torch.Tensor = None,
- out_dtype: torch.dtype = torch.float16,
- ) -> torch.Tensor:
- """Low-level SAGE attention with pre-quantized INT8 Q/K/V and PV int8.
-
- Expects contiguous layouts:
- - query: [B, Hq, Sq, D] int8
- - key: [B, Hkv, Skv, D] int8
- - value: [B, Hkv, Skv, D] int8
- - qscale: [B, Hq, ceil(Sq / quant_block_size), 1] float32
- - kscale: [B, Hkv, ceil(Skv / quant_block_size), 1] float32
- - vscale: [B, Hkv, ceil(Skv / quant_block_size), D] float32
-
- Returns:
- - O: [B, Hq, Sq, D] float16
- """
- if query.device.type != "xpu":
- raise NotImplementedError("sage_pvi8 is only supported on XPU")
- if query.dtype != torch.int8 or key.dtype != torch.int8 or value.dtype != torch.int8:
- raise ValueError(f"Q/K/V must be int8, got Q={query.dtype}, K={key.dtype}, V={value.dtype}")
- if out_dtype != torch.float16:
- raise ValueError(f"sage_pvi8 output must be float16, got {out_dtype}")
- if qscale is None or kscale is None or vscale is None:
- raise ValueError("qscale, kscale and vscale must be provided for sage_pvi8")
-
- if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:
- raise ValueError("Q/K/V must be 4D tensors")
-
- if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():
- raise ValueError("Q/K/V must be contiguous")
-
- B, Hq, Sq, D = query.shape
- Bk, Hkv, Skv, Dk = key.shape
- Bv, Hkv2, Skv2, Dv = value.shape
-
- if Bk != B or Bv != B:
- raise ValueError("Batch size mismatch between Q/K/V")
- if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
- raise ValueError("K/V shape mismatch")
- if Dk != D:
- raise ValueError("Head dim mismatch between Q and K/V")
- if D not in (64, 128):
- raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
-
- q_blocks = (Sq + quant_block_size - 1) // quant_block_size
- kv_blocks = (Skv + quant_block_size - 1) // quant_block_size
- if qscale.numel() != B * Hq * q_blocks:
- raise ValueError(
- f"qscale must have {B * Hq * q_blocks} elements for shape [B, Hq, ceil(Sq/block), 1], got {qscale.numel()}"
- )
- if kscale.numel() != B * Hkv * kv_blocks:
- raise ValueError(
- f"kscale must have {B * Hkv * kv_blocks} elements for shape [B, Hkv, ceil(Skv/block), 1], got {kscale.numel()}"
- )
- if vscale.numel() != B * Hkv * kv_blocks * D:
- raise ValueError(
- f"vscale must have {B * Hkv * kv_blocks * D} elements for shape [B, Hkv, ceil(Skv/block), D], got {vscale.numel()}"
- )
-
- lib = self.get_lib(query)
- stream = get_stream(query)
- O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=out_dtype)
- lib.sage_pvi8(
- stream,
- query.data_ptr(),
- key.data_ptr(),
- value.data_ptr(),
- O.data_ptr(),
- attn_mask.data_ptr() if attn_mask is not None else 0,
- quant_block_size,
- qscale.data_ptr(),
- kscale.data_ptr(),
- vscale.data_ptr(),
- cvt_dtype(query.dtype),
- cvt_dtype(key.dtype),
- cvt_dtype(O.dtype),
- B,
- Hq,
- Hkv,
- Sq,
- Skv,
- D,
- float(scale) if scale is not None else 1.0 / (D**0.5),
- bool(is_causal),
- )
- return O
-
- def sagev1(
- self,
- query: torch.Tensor,
- key: torch.Tensor,
- value: torch.Tensor,
- attn_mask: torch.Tensor | None = None,
- dropout_p: float = 0.0,
- is_causal: bool = False,
- scale: float | None = None,
- enable_gqa: bool = False,
- quant_block_size: int = 64,
- ) -> torch.Tensor:
- """SAGE v1 attention prefill+decode.
-
- Expects contiguous layouts:
- - query: [B, Hq, Sq, D]
- - key: [B, Hkv, Skv, D]
- - value: [B, Hkv, Skv, D]
-
- Args:
- - scale: Attention scale. Uses 1 / sqrt(D) when None.
- - quant_block_size: Quantization block size used by the kernel.
-
- Returns:
- - O: [B, Hq, Sq, D] (same dtype as value)
- """
- if quant_block_size <= 0:
- return self.sdpa(
- query=query,
- key=key,
- value=value,
- attn_mask=attn_mask,
- dropout_p=dropout_p,
- is_causal=is_causal,
- scale=scale,
- )
- if query.device.type != "xpu":
- raise NotImplementedError("sdpa is only supported on XPU")
- if query.dtype not in (torch.float16,):
- raise ValueError(f"Q must be float16, got {query.dtype}")
- if key.dtype != query.dtype or value.dtype != query.dtype:
- raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")
-
- if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:
- raise ValueError("Q/K/V must be 4D tensors")
-
- if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():
- raise ValueError("Q/K/V must be contiguous")
-
- B, Hq, Sq, D = query.shape
- Bk, Hkv, Skv, Dk = key.shape
- Bv, Hkv2, Skv2, Dv = value.shape
-
- if Bk != B or Bv != B:
- raise ValueError("Batch size mismatch between Q/K/V")
- if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
- raise ValueError("K/V shape mismatch")
- if Dk != D:
- raise ValueError("Head dim mismatch between Q and K/V")
- if D not in (64, 128):
- raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
-
- lib = self.get_lib(query)
- stream = get_stream(query)
- O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)
- lib.sagev1(
- stream,
- query.data_ptr(),
- key.data_ptr(),
- value.data_ptr(),
- O.data_ptr(),
- attn_mask.data_ptr() if attn_mask is not None else 0,
- quant_block_size,
- cvt_dtype(query.dtype),
- cvt_dtype(key.dtype),
- cvt_dtype(value.dtype),
- cvt_dtype(O.dtype),
- B,
- Hq,
- Hkv,
- Sq,
- Skv,
- D,
- float(scale) if scale is not None else 1.0 / (D**0.5),
- bool(is_causal),
- )
- return O
-
- def sagev1_pvi8(
- self,
- query: torch.Tensor,
- key: torch.Tensor,
- value: torch.Tensor,
- attn_mask: torch.Tensor | None = None,
- dropout_p: float = 0.0,
- is_causal: bool = False,
- scale: float | None = None,
- enable_gqa: bool = False,
- quant_block_size: int = 64,
- ) -> torch.Tensor:
- """SAGE v1 attention with PV int8 path.
-
- Expects FP16 Q/K/V input and quantizes Q/K/V internally before calling
- the PV int8 kernel.
- """
- if quant_block_size <= 0:
- return self.sdpa(
- query=query,
- key=key,
- value=value,
- attn_mask=attn_mask,
- dropout_p=dropout_p,
- is_causal=is_causal,
- scale=scale,
- )
- if query.device.type != "xpu":
- raise NotImplementedError("sagev1_pvi8 is only supported on XPU")
- if query.dtype not in (torch.float16,):
- raise ValueError(f"Q must be float16, got {query.dtype}")
- if key.dtype != query.dtype or value.dtype != query.dtype:
- raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")
-
- if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:
- raise ValueError("Q/K/V must be 4D tensors")
-
- if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():
- raise ValueError("Q/K/V must be contiguous")
-
- B, Hq, Sq, D = query.shape
- Bk, Hkv, Skv, Dk = key.shape
- Bv, Hkv2, Skv2, Dv = value.shape
-
- if Bk != B or Bv != B:
- raise ValueError("Batch size mismatch between Q/K/V")
- if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
- raise ValueError("K/V shape mismatch")
- if Dk != D:
- raise ValueError("Head dim mismatch between Q and K/V")
- if D not in (64, 128):
- raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
-
- lib = self.get_lib(query)
- stream = get_stream(query)
- O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)
- lib.sagev1_pvi8(
- stream,
- query.data_ptr(),
- key.data_ptr(),
- value.data_ptr(),
- O.data_ptr(),
- attn_mask.data_ptr() if attn_mask is not None else 0,
- quant_block_size,
- cvt_dtype(query.dtype),
- cvt_dtype(key.dtype),
- cvt_dtype(value.dtype),
- cvt_dtype(O.dtype),
- B,
- Hq,
- Hkv,
- Sq,
- Skv,
- D,
- float(scale) if scale is not None else 1.0 / (D**0.5),
- bool(is_causal),
+ )
+ if blob.device.type == "cpu":
+ return out.T
+ return out
+
+
+def sdpa(
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attn_mask: torch.Tensor | None = None,
+ dropout_p: float = 0.0,
+ is_causal: bool = False,
+ scale: float | None = None,
+ tensor_layout: str = "HND",
+) -> torch.Tensor:
+ """Scaled dot-product attention (SDPA) prefill+decode.
+
+ Supported tensor layouts:
+ - HND: [B, H, N, D]
+ - NHD: [B, N, H, D]
+
+ Args:
+ - scale: Softmax scale. Uses 1 / sqrt(D) when None.
+ - tensor_layout: Layout of Q/K/V/O tensors.
+
+ Returns:
+ - O: same layout as the input tensors.
+ """
+ if query.device.type != "xpu":
+ raise NotImplementedError("sdpa is only supported on XPU")
+
+ if query.dtype not in (torch.float16, torch.bfloat16):
+ raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")
+ if key.dtype != query.dtype or value.dtype != query.dtype:
+ raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")
+
+ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)
+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout, expected_dtype=query.dtype)
+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout, expected_dtype=query.dtype)
+
+ if Bk != B or Bv != B:
+ raise ValueError("Batch size mismatch between Q/K/V")
+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
+ raise ValueError("K/V shape mismatch")
+ if Dk != D:
+ raise ValueError("Head dim mismatch between Q and K/V")
+ if D not in (64, 128, 96, 192):
+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128, 96, 192")
+
+ if dropout_p != 0.0:
+ raise NotImplementedError(f"dropout_p must be 0.0 (got {dropout_p}); dropout is not supported")
+
+ if attn_mask is not None:
+ if attn_mask.device.type != "xpu":
+ raise ValueError("attn_mask must be on XPU")
+ if not attn_mask.is_contiguous():
+ raise ValueError("attn_mask must be contiguous")
+ if attn_mask.dtype != torch.float32:
+ raise ValueError(f"attn_mask must be float32 (additive bias), got {attn_mask.dtype}")
+ expected_mask_shape = (B, 1, Sq, Skv)
+ if attn_mask.shape != expected_mask_shape:
+ raise ValueError(f"attn_mask shape must be {expected_mask_shape}, got {tuple(attn_mask.shape)}")
+
+ lib = get_lib(query)
+ stream = get_stream(query)
+ O = _empty_attention_output(
+ B,
+ Hq,
+ Sq,
+ D,
+ dtype=value.dtype,
+ device=query.device,
+ tensor_layout=tensor_layout,
+ )
+ q_strides = _attention_strides_qko(query, tensor_layout)
+ k_strides = _attention_strides_qko(key, tensor_layout)
+ v_strides = _attention_strides_v(value, tensor_layout)
+ o_strides = _attention_strides_qko(O, tensor_layout)
+ lib.sdpa(
+ stream,
+ query.data_ptr(),
+ key.data_ptr(),
+ value.data_ptr(),
+ O.data_ptr(),
+ attn_mask.data_ptr() if attn_mask is not None else 0,
+ *q_strides,
+ *k_strides,
+ *v_strides,
+ *o_strides,
+ cvt_dtype(query.dtype),
+ cvt_dtype(key.dtype),
+ cvt_dtype(O.dtype),
+ B,
+ Hq,
+ Hkv,
+ Sq,
+ Skv,
+ D,
+ float(scale) if scale is not None else 1.0 / (D**0.5),
+ bool(is_causal),
+ )
+ return O
+
+
+def sage(
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attn_mask: torch.Tensor | None = None,
+ dropout_p: float = 0.0,
+ is_causal: bool = False,
+ scale: float | None = None,
+ enable_gqa: bool = False,
+ quant_block_size: int = 64,
+ qscale: torch.Tensor = None,
+ kscale: torch.Tensor = None,
+ tensor_layout: str = "HND",
+) -> torch.Tensor:
+ """SAGE attention prefill+decode.
+
+ Supported tensor layouts:
+ - HND: [B, H, N, D]
+ - NHD: [B, N, H, D]
+
+ Args:
+ - scale: Attention scale. Uses 1 / sqrt(D) when None.
+ - quant_block_size: Block size for qscale and kscale.
+ - tensor_layout: Layout of Q/K/V/O tensors.
+
+ Returns:
+ - O: same layout as the input tensors.
+ """
+ if query.device.type != "xpu":
+ raise NotImplementedError("sdpa is only supported on XPU")
+
+ # if query.dtype not in (torch.float16, torch.bfloat16):
+ # raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")
+ # if key.dtype != query.dtype or value.dtype != query.dtype:
+ # raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")
+
+ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)
+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout)
+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout)
+
+ if Bk != B or Bv != B:
+ raise ValueError("Batch size mismatch between Q/K/V")
+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
+ raise ValueError("K/V shape mismatch")
+ if Dk != D:
+ raise ValueError("Head dim mismatch between Q and K/V")
+ if D not in (64, 128):
+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
+
+ lib = get_lib(query)
+ stream = get_stream(query)
+ O = _empty_attention_output(
+ B,
+ Hq,
+ Sq,
+ D,
+ dtype=value.dtype,
+ device=query.device,
+ tensor_layout=tensor_layout,
+ )
+ q_strides = _attention_strides_qko(query, tensor_layout)
+ k_strides = _attention_strides_qko(key, tensor_layout)
+ v_strides = _attention_strides_v(value, tensor_layout)
+ o_strides = _attention_strides_qko(O, tensor_layout)
+ lib.sage(
+ stream,
+ query.data_ptr(),
+ key.data_ptr(),
+ value.data_ptr(),
+ O.data_ptr(),
+ attn_mask.data_ptr() if attn_mask is not None else 0,
+ quant_block_size,
+ qscale.data_ptr() if qscale is not None else 0,
+ kscale.data_ptr() if kscale is not None else 0,
+ *q_strides,
+ *k_strides,
+ *v_strides,
+ *o_strides,
+ cvt_dtype(query.dtype),
+ cvt_dtype(key.dtype),
+ cvt_dtype(O.dtype),
+ B,
+ Hq,
+ Hkv,
+ Sq,
+ Skv,
+ D,
+ float(scale) if scale is not None else 1.0 / (D**0.5),
+ bool(is_causal),
+ )
+ return O
+
+
+def sage_pvi8(
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attn_mask: torch.Tensor | None = None,
+ dropout_p: float = 0.0,
+ is_causal: bool = False,
+ scale: float | None = None,
+ enable_gqa: bool = False,
+ quant_block_size: int = 64,
+ qscale: torch.Tensor = None,
+ kscale: torch.Tensor = None,
+ vscale: torch.Tensor = None,
+ out_dtype: torch.dtype = torch.float16,
+ tensor_layout: str = "HND",
+) -> torch.Tensor:
+ """Low-level SAGE attention with pre-quantized INT8 Q/K/V and PV int8.
+
+ Expects contiguous layouts:
+ - query: [B, Hq, Sq, D] int8
+ - key: [B, Hkv, Skv, D] int8
+ - value: [B, Hkv, Skv, D] int8
+ - qscale: [B, Hq, ceil(Sq / quant_block_size), 1] float32
+ - kscale: [B, Hkv, ceil(Skv / quant_block_size), 1] float32
+ - vscale: [B, Hkv, ceil(Skv / quant_block_size), D] float32
+
+ Returns:
+ - O: [B, Hq, Sq, D] float16
+ """
+ if query.device.type != "xpu":
+ raise NotImplementedError("sage_pvi8 is only supported on XPU")
+ if query.dtype != torch.int8 or key.dtype != torch.int8 or value.dtype != torch.int8:
+ raise ValueError(f"Q/K/V must be int8, got Q={query.dtype}, K={key.dtype}, V={value.dtype}")
+ if out_dtype != torch.float16:
+ raise ValueError(f"sage_pvi8 output must be float16, got {out_dtype}")
+ if qscale is None or kscale is None or vscale is None:
+ raise ValueError("qscale, kscale and vscale must be provided for sage_pvi8")
+
+ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)
+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout)
+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout)
+
+ if Bk != B or Bv != B:
+ raise ValueError("Batch size mismatch between Q/K/V")
+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
+ raise ValueError("K/V shape mismatch")
+ if Dk != D:
+ raise ValueError("Head dim mismatch between Q and K/V")
+ if D not in (64, 128):
+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
+
+ q_blocks = (Sq + quant_block_size - 1) // quant_block_size
+ kv_blocks = (Skv + quant_block_size - 1) // quant_block_size
+ if qscale.numel() != B * Hq * q_blocks:
+ raise ValueError(
+ f"qscale must have {B * Hq * q_blocks} elements for shape [B, Hq, ceil(Sq/block), 1], got {qscale.numel()}"
)
- return O
-
- def sage_dynquant(
- self,
- query: torch.Tensor,
- key: torch.Tensor,
- value: torch.Tensor,
- attn_mask: torch.Tensor | None = None,
- dropout_p: float = 0.0,
- is_causal: bool = False,
- scale: float | None = None,
- enable_gqa: bool = False,
- quant_block_size: int = 64,
- ) -> torch.Tensor:
- """SAGE Attention with dynamic INT8 block-wise quantization of Q/K.
-
- Takes FP16 Q, K, V inputs. Quantizes Q and K to INT8 per-block
- using a fused SYCL kernel, then calls SAGE V1 with INT8 data.
- API is like SDPA but with an extra quant_block_size parameter.
-
- Args:
- query: [B, Hq, Sq, D] float16
- key: [B, Hkv, Skv, D] float16
- value: [B, Hkv, Skv, D] float16
- quant_block_size: Number of tokens sharing one INT8 scale.
- E.g. 64 means 64 consecutive tokens share one absmax.
- 0 means per-token (block_size=1).
-
- Returns:
- O: [B, Hq, Sq, D] float16
- """
- if query.device.type != "xpu":
- raise NotImplementedError("sage_dynquant is only supported on XPU")
-
- if query.dtype not in (torch.float16,):
- raise ValueError(f"Q must be float16, got {query.dtype}")
-
- B, Hq, Sq, D = query.shape
- _, Hkv, Skv, _ = key.shape
-
- # block_size=0 means per-token
- block_size = quant_block_size if quant_block_size > 0 else 1
-
- # SAGE V1 kernel uses K-tile size=32; quant_block_size must be 1 or >=32
- if block_size != 1 and block_size < 32:
- raise ValueError(
- f"quant_block_size={block_size} is not supported. "
- f"Must be 1 (per-token) or >= 32 (e.g. 32, 64, 128, 256)."
- )
-
- lib = self.get_lib(query)
- stream = get_stream(query)
-
- # Auto-pad Q and K/V seq lengths to be divisible by block_size
- # so sage_dynquant works as a drop-in replacement for SDPA
- def _ceil_div(a, b):
- return (a + b - 1) // b
-
- Sq_pad = _ceil_div(Sq, block_size) * block_size
- Skv_pad = _ceil_div(Skv, block_size) * block_size
- need_pad_q = Sq_pad != Sq
- need_pad_kv = Skv_pad != Skv
-
- if need_pad_q:
- pad_q = Sq_pad - Sq
- query = torch.nn.functional.pad(query, (0, 0, 0, pad_q)) # pad S dim with zeros
- if need_pad_kv:
- pad_kv = Skv_pad - Skv
- key = torch.nn.functional.pad(key, (0, 0, 0, pad_kv))
- value = torch.nn.functional.pad(value, (0, 0, 0, pad_kv))
-
- # Fused block-wise quantization via SYCL kernel
- # Tensor layout: [B, H, S, D] is contiguous → [B*H*S, D] flattened
- # block_size tokens share one scale → num_blocks = B*H*S / block_size
- # For Q: num_rows = B*Hq*Sq_pad, scale shape = [B, Hq, Sq_pad/block_size, 1]
- q_num_rows = B * Hq * Sq_pad
- q_num_blocks = q_num_rows // block_size
- q_i8 = torch.empty_like(query, dtype=torch.int8)
- q_scale = torch.empty(q_num_blocks, dtype=torch.float32, device=query.device)
- lib.sage_dynamic_quant(
- stream,
- query.data_ptr(),
- 0,
- q_i8.data_ptr(),
- q_scale.data_ptr(),
- q_num_rows,
- D,
- block_size,
+ if kscale.numel() != B * Hkv * kv_blocks:
+ raise ValueError(
+ f"kscale must have {B * Hkv * kv_blocks} elements for shape [B, Hkv, ceil(Skv/block), 1], got {kscale.numel()}"
)
- q_scale = q_scale.reshape(B, Hq, Sq_pad // block_size, 1)
-
- k_num_rows = B * Hkv * Skv_pad
- k_num_blocks = k_num_rows // block_size
- k_i8 = torch.empty_like(key, dtype=torch.int8)
- k_scale = torch.empty(k_num_blocks, dtype=torch.float32, device=key.device)
- lib.sage_dynamic_quant(
- stream,
- key.data_ptr(),
- 0,
- k_i8.data_ptr(),
- k_scale.data_ptr(),
- k_num_rows,
- D,
- block_size,
+ if vscale.numel() != B * Hkv * kv_blocks * D:
+ raise ValueError(
+ f"vscale must have {B * Hkv * kv_blocks * D} elements for shape [B, Hkv, ceil(Skv/block), D], got {vscale.numel()}"
)
- k_scale = k_scale.reshape(B, Hkv, Skv_pad // block_size, 1)
- # Call SAGE v1 with matching quant_block_size
- out = self.sage(
- q_i8,
- k_i8,
- value,
+ lib = get_lib(query)
+ stream = get_stream(query)
+ O = _empty_attention_output(
+ B,
+ Hq,
+ Sq,
+ D,
+ dtype=out_dtype,
+ device=query.device,
+ tensor_layout=tensor_layout,
+ )
+ q_strides = _attention_strides_qko(query, tensor_layout)
+ k_strides = _attention_strides_qko(key, tensor_layout)
+ v_strides = _attention_strides_v(value, tensor_layout)
+ o_strides = _attention_strides_qko(O, tensor_layout)
+ lib.sage_pvi8(
+ stream,
+ query.data_ptr(),
+ key.data_ptr(),
+ value.data_ptr(),
+ O.data_ptr(),
+ attn_mask.data_ptr() if attn_mask is not None else 0,
+ quant_block_size,
+ qscale.data_ptr(),
+ kscale.data_ptr(),
+ vscale.data_ptr(),
+ *q_strides,
+ *k_strides,
+ *v_strides,
+ *o_strides,
+ cvt_dtype(query.dtype),
+ cvt_dtype(key.dtype),
+ cvt_dtype(O.dtype),
+ B,
+ Hq,
+ Hkv,
+ Sq,
+ Skv,
+ D,
+ float(scale) if scale is not None else 1.0 / (D**0.5),
+ bool(is_causal),
+ )
+ return O
+
+
+def sagev1(
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attn_mask: torch.Tensor | None = None,
+ dropout_p: float = 0.0,
+ is_causal: bool = False,
+ scale: float | None = None,
+ enable_gqa: bool = False,
+ quant_block_size: int = 64,
+ tensor_layout: str = "HND",
+) -> torch.Tensor:
+ """SAGE v1 attention prefill+decode.
+
+ Supported tensor layouts:
+ - HND: [B, H, N, D]
+ - NHD: [B, N, H, D]
+
+ Args:
+ - scale: Attention scale. Uses 1 / sqrt(D) when None.
+ - quant_block_size: Quantization block size used by the kernel.
+ - tensor_layout: Layout of Q/K/V/O tensors.
+
+ Returns:
+ - O: same layout as the input tensors.
+ """
+ if quant_block_size <= 0:
+ return sdpa(
+ query=query,
+ key=key,
+ value=value,
+ attn_mask=attn_mask,
+ dropout_p=dropout_p,
+ is_causal=is_causal,
+ scale=scale,
+ tensor_layout=tensor_layout,
+ )
+ if query.device.type != "xpu":
+ raise NotImplementedError("sdpa is only supported on XPU")
+ if query.dtype not in (torch.float16, torch.bfloat16):
+ raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")
+ if key.dtype != query.dtype or value.dtype != query.dtype:
+ raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")
+
+ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)
+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout, expected_dtype=query.dtype)
+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout, expected_dtype=query.dtype)
+
+ if Bk != B or Bv != B:
+ raise ValueError("Batch size mismatch between Q/K/V")
+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
+ raise ValueError("K/V shape mismatch")
+ if Dk != D:
+ raise ValueError("Head dim mismatch between Q and K/V")
+ if D not in (64, 128):
+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
+
+ lib = get_lib(query)
+ stream = get_stream(query)
+ O = _empty_attention_output(
+ B,
+ Hq,
+ Sq,
+ D,
+ dtype=value.dtype,
+ device=query.device,
+ tensor_layout=tensor_layout,
+ )
+ q_strides = _attention_strides_qko(query, tensor_layout)
+ k_strides = _attention_strides_qko(key, tensor_layout)
+ v_strides = _attention_strides_v(value, tensor_layout)
+ o_strides = _attention_strides_qko(O, tensor_layout)
+ lib.sagev1(
+ stream,
+ query.data_ptr(),
+ key.data_ptr(),
+ value.data_ptr(),
+ O.data_ptr(),
+ attn_mask.data_ptr() if attn_mask is not None else 0,
+ quant_block_size,
+ *q_strides,
+ *k_strides,
+ *v_strides,
+ *o_strides,
+ cvt_dtype(query.dtype),
+ cvt_dtype(key.dtype),
+ cvt_dtype(value.dtype),
+ cvt_dtype(O.dtype),
+ B,
+ Hq,
+ Hkv,
+ Sq,
+ Skv,
+ D,
+ float(scale) if scale is not None else 1.0 / (D**0.5),
+ bool(is_causal),
+ )
+ return O
+
+
+def sagev1_pvi8(
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attn_mask: torch.Tensor | None = None,
+ dropout_p: float = 0.0,
+ is_causal: bool = False,
+ scale: float | None = None,
+ enable_gqa: bool = False,
+ quant_block_size: int = 64,
+ tensor_layout: str = "HND",
+) -> torch.Tensor:
+ """SAGE v1 attention with PV int8 path.
+
+ Expects FP16 Q/K/V input and quantizes Q/K/V internally before calling
+ the PV int8 kernel.
+ """
+ if quant_block_size <= 0:
+ return sdpa(
+ query=query,
+ key=key,
+ value=value,
attn_mask=attn_mask,
+ dropout_p=dropout_p,
is_causal=is_causal,
scale=scale,
- enable_gqa=enable_gqa,
- quant_block_size=block_size,
- qscale=q_scale,
- kscale=k_scale,
+ tensor_layout=tensor_layout,
)
+ if query.device.type != "xpu":
+ raise NotImplementedError("sagev1_pvi8 is only supported on XPU")
+ if query.dtype not in (torch.float16, torch.bfloat16):
+ raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")
+ if key.dtype != query.dtype or value.dtype != query.dtype:
+ raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")
+
+ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)
+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout, expected_dtype=query.dtype)
+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout, expected_dtype=query.dtype)
+
+ if Bk != B or Bv != B:
+ raise ValueError("Batch size mismatch between Q/K/V")
+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
+ raise ValueError("K/V shape mismatch")
+ if Dk != D:
+ raise ValueError("Head dim mismatch between Q and K/V")
+ if D not in (64, 128):
+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
+
+ lib = get_lib(query)
+ stream = get_stream(query)
+ O = _empty_attention_output(
+ B,
+ Hq,
+ Sq,
+ D,
+ dtype=value.dtype,
+ device=query.device,
+ tensor_layout=tensor_layout,
+ )
+ q_strides = _attention_strides_qko(query, tensor_layout)
+ k_strides = _attention_strides_qko(key, tensor_layout)
+ v_strides = _attention_strides_v(value, tensor_layout)
+ o_strides = _attention_strides_qko(O, tensor_layout)
+ lib.sagev1_pvi8(
+ stream,
+ query.data_ptr(),
+ key.data_ptr(),
+ value.data_ptr(),
+ O.data_ptr(),
+ attn_mask.data_ptr() if attn_mask is not None else 0,
+ quant_block_size,
+ *q_strides,
+ *k_strides,
+ *v_strides,
+ *o_strides,
+ cvt_dtype(query.dtype),
+ cvt_dtype(key.dtype),
+ cvt_dtype(value.dtype),
+ cvt_dtype(O.dtype),
+ B,
+ Hq,
+ Hkv,
+ Sq,
+ Skv,
+ D,
+ float(scale) if scale is not None else 1.0 / (D**0.5),
+ bool(is_causal),
+ )
+ return O
+
+
+def sageattn(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ tensor_layout: str = "HND",
+ is_causal: bool = False,
+ sm_scale: Optional[float] = None,
+ return_lse: bool = False,
+ kernel: str = "v1_pvhalf",
+ **kwargs,
+) -> torch.Tensor:
+ """SAGE attention dispatcher.
+
+ Signature mirrors ``sageattention.sageattn``.
+
+ Args:
+ - q, k, v: Query/Key/Value tensors. Layout selected by ``tensor_layout``.
+ - tensor_layout: "HND" or "NHD".
+ - is_causal: Whether to apply causal mask.
+ - sm_scale: Softmax scale. Uses ``1 / sqrt(head_dim)`` when None.
+ - return_lse: Not supported; must be False.
+ - kernel: Which SAGE variant to dispatch to.
+ - "v1_pvhalf" (default): PV in half precision (calls ``sagev1``).
+ - "v1_pvi8": PV in INT8 precision (calls ``sagev1_pvi8``).
+ - kwargs: Forwarded to the underlying kernel (e.g. ``attn_mask``,
+ ``dropout_p``, ``enable_gqa``, ``quant_block_size``).
+
+ Returns:
+ - O: same layout as the input tensors.
+ """
+ if return_lse:
+ raise NotImplementedError("return_lse is not supported in ARK sageattn")
+
+ if kernel == "v1_pvhalf":
+ impl = sagev1
+ elif kernel == "v1_pvi8":
+ impl = sagev1_pvi8
+ else:
+ raise ValueError(f"Unsupported sageattn kernel={kernel!r}; supported: 'v1_pvhalf', 'v1_pvi8'")
+
+ return impl(
+ query=q,
+ key=k,
+ value=v,
+ is_causal=is_causal,
+ scale=sm_scale,
+ tensor_layout=tensor_layout,
+ **kwargs,
+ )
- # Slice back to original seq length if padded
- if need_pad_q:
- out = out[:, :, :Sq, :]
- return out
-
- def moe_gemm(
- self,
- activations: torch.Tensor,
- weights: torch.Tensor,
- num_tokens_per_expert: torch.Tensor,
- *,
- scales: Optional[torch.Tensor] = None,
- ) -> torch.Tensor:
- """MOE GEMM (Mixture of Experts Grouped GEMM).
-
- Computes grouped GEMM for MOE layers where different experts process
- different numbers of tokens.
-
- Expects contiguous layouts:
- - activations: [total_tokens, K] (BF16/FP16)
- - weights: [num_experts, K, N] (BF16/FP16, Row major)
- - num_tokens_per_expert: [num_experts] (int32)
- - scales (optional): [num_experts, N] or None
-
- Returns:
- - outputs: [total_tokens, N] (same dtype as activations)
- """
- if activations.device.type != "xpu":
- raise NotImplementedError("moe_gemm is only supported on XPU")
-
- if activations.dtype not in (torch.float16, torch.bfloat16):
- raise ValueError(f"activations must be fp16/bf16, got {activations.dtype}")
- if weights.dtype != activations.dtype:
- raise ValueError("weights dtype must match activations dtype")
-
- if activations.ndim != 2 or weights.ndim != 3:
- raise ValueError("activations must be 2D [total_tokens, K], weights must be 3D [num_experts, K, N]")
-
- if not activations.is_contiguous() or not weights.is_contiguous():
- raise ValueError("activations and weights must be contiguous")
-
- if num_tokens_per_expert.dtype != torch.int32:
- num_tokens_per_expert = num_tokens_per_expert.to(torch.int32)
-
- if not num_tokens_per_expert.is_contiguous():
- num_tokens_per_expert = num_tokens_per_expert.contiguous()
-
- total_tokens, K = activations.shape
- num_experts, K_w, N = weights.shape # weights are [num_experts, K, N]
-
- if K != K_w:
- raise ValueError(f"K dimension mismatch: activations K={K}, weights K={K_w}")
-
- if num_tokens_per_expert.shape[0] != num_experts:
- raise ValueError(
- f"num_tokens_per_expert length {num_tokens_per_expert.shape[0]} != num_experts {num_experts}"
- )
-
- # Validate total tokens
- expected_total = int(num_tokens_per_expert.sum().item())
- if expected_total != total_tokens:
- raise ValueError(f"Sum of num_tokens_per_expert ({expected_total}) != total_tokens ({total_tokens})")
-
- lib = self.get_lib(activations)
- stream = get_stream(activations)
- outputs = torch.empty((total_tokens, N), device=activations.device, dtype=activations.dtype)
-
- scales_ptr = scales.data_ptr() if scales is not None else 0
-
- lib.moe_gemm(
- stream,
- activations.data_ptr(),
- weights.data_ptr(),
- scales_ptr,
- outputs.data_ptr(),
- cvt_dtype(activations.dtype),
- N,
- K,
- num_tokens_per_expert.data_ptr(),
- num_experts,
+
+def sage_dynquant(
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attn_mask: torch.Tensor | None = None,
+ dropout_p: float = 0.0,
+ is_causal: bool = False,
+ scale: float | None = None,
+ enable_gqa: bool = False,
+ quant_block_size: int = 64,
+) -> torch.Tensor:
+ """SAGE Attention with dynamic INT8 block-wise quantization of Q/K.
+
+ Takes FP16 Q, K, V inputs. Quantizes Q and K to INT8 per-block
+ using a fused SYCL kernel, then calls SAGE V1 with INT8 data.
+ API is like SDPA but with an extra quant_block_size parameter.
+
+ Args:
+ query: [B, Hq, Sq, D] float16
+ key: [B, Hkv, Skv, D] float16
+ value: [B, Hkv, Skv, D] float16
+ quant_block_size: Number of tokens sharing one INT8 scale.
+ E.g. 64 means 64 consecutive tokens share one absmax.
+ 0 means per-token (block_size=1).
+
+ Returns:
+ O: [B, Hq, Sq, D] float16
+ """
+ if query.device.type != "xpu":
+ raise NotImplementedError("sage_dynquant is only supported on XPU")
+
+ if query.dtype not in (torch.float16, torch.bfloat16):
+ raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")
+
+ B, Hq, Sq, D = query.shape
+ _, Hkv, Skv, _ = key.shape
+
+ # block_size=0 means per-token
+ block_size = quant_block_size if quant_block_size > 0 else 1
+
+ # SAGE V1 kernel uses K-tile size=32; quant_block_size must be 1 or >=32
+ if block_size != 1 and block_size < 32:
+ raise ValueError(
+ f"quant_block_size={block_size} is not supported. "
+ f"Must be 1 (per-token) or >= 32 (e.g. 32, 64, 128, 256)."
)
- return outputs
+ lib = get_lib(query)
+ stream = get_stream(query)
+
+ # Auto-pad Q and K/V seq lengths to be divisible by block_size
+ # so sage_dynquant works as a drop-in replacement for SDPA
+ def _ceil_div(a, b):
+ return (a + b - 1) // b
+
+ Sq_pad = _ceil_div(Sq, block_size) * block_size
+ Skv_pad = _ceil_div(Skv, block_size) * block_size
+ need_pad_q = Sq_pad != Sq
+ need_pad_kv = Skv_pad != Skv
+
+ if need_pad_q:
+ pad_q = Sq_pad - Sq
+ query = torch.nn.functional.pad(query, (0, 0, 0, pad_q)) # pad S dim with zeros
+ if need_pad_kv:
+ pad_kv = Skv_pad - Skv
+ key = torch.nn.functional.pad(key, (0, 0, 0, pad_kv))
+ value = torch.nn.functional.pad(value, (0, 0, 0, pad_kv))
+
+ # Fused block-wise quantization via SYCL kernel
+ # Tensor layout: [B, H, S, D] is contiguous → [B*H*S, D] flattened
+ # block_size tokens share one scale → num_blocks = B*H*S / block_size
+ # For Q: num_rows = B*Hq*Sq_pad, scale shape = [B, Hq, Sq_pad/block_size, 1]
+ q_num_rows = B * Hq * Sq_pad
+ q_num_blocks = q_num_rows // block_size
+ q_i8 = torch.empty_like(query, dtype=torch.int8)
+ q_scale = torch.empty(q_num_blocks, dtype=torch.float32, device=query.device)
+ lib.sage_dynamic_quant(
+ stream,
+ query.data_ptr(),
+ 0,
+ q_i8.data_ptr(),
+ q_scale.data_ptr(),
+ q_num_rows,
+ D,
+ block_size,
+ )
+ q_scale = q_scale.reshape(B, Hq, Sq_pad // block_size, 1)
+
+ k_num_rows = B * Hkv * Skv_pad
+ k_num_blocks = k_num_rows // block_size
+ k_i8 = torch.empty_like(key, dtype=torch.int8)
+ k_scale = torch.empty(k_num_blocks, dtype=torch.float32, device=key.device)
+ lib.sage_dynamic_quant(
+ stream,
+ key.data_ptr(),
+ 0,
+ k_i8.data_ptr(),
+ k_scale.data_ptr(),
+ k_num_rows,
+ D,
+ block_size,
+ )
+ k_scale = k_scale.reshape(B, Hkv, Skv_pad // block_size, 1)
+
+ # Call SAGE v1 with matching quant_block_size
+ out = sage(
+ q_i8,
+ k_i8,
+ value,
+ attn_mask=attn_mask,
+ is_causal=is_causal,
+ scale=scale,
+ enable_gqa=enable_gqa,
+ quant_block_size=block_size,
+ qscale=q_scale,
+ kscale=k_scale,
+ )
-if __name__ == "__main__":
- ark = ARK()
- print(ark.cpu_lib is None, ark.xpu_lib is None)
+ # Slice back to original seq length if padded
+ if need_pad_q:
+ out = out[:, :, :Sq, :]
+ return out
- def matmul():
- m = n = k = 128
- dt = torch.int8
- device = "cpu"
- has_bias = False
- if dt == torch.int8:
- A = torch.randint(-128, 127, (m, k), dtype=dt, device=device)
- B = torch.randint(-128, 127, (n, k), dtype=dt, device=device)
- C = ark.igemm_s8s8s32(A, B)
- print(C)
- else:
- A = torch.rand(m, k, dtype=dt, device=device) - 0.5
- B = torch.rand(k, n, dtype=dt, device=device) - 0.5
- bias = torch.rand(1, n, dtype=dt, device=device) if has_bias else torch.empty(0)
- C = ark.matmul(A, B, bias)
- ref = torch.matmul(A, B.T)
- if has_bias:
- ref = ref + bias
- dff = abs(C - ref)
- if dt != torch.int8:
- print(dff.max(), dff.mean())
- print(torch.allclose(ref, C, 0.01, 0.1))
- def woq():
- m = n = k = 128
- dt = torch.float32
- device = "cpu"
- A = torch.rand(m, k, dtype=dt, device=device) - 0.5
- bias = torch.rand(1, n, dtype=dt, device=device) + 1000
- B = torch.randint(-128, 127, (n, k), dtype=torch.int8, device=device)
- scaleB = torch.rand(n, 1, dtype=dt, device=device)
- C = ark.woqgemm_s8(A, B, scaleB, bias)
- print(C)
- DB = B.to(dt) * scaleB
- ref = torch.matmul(A, DB.T) + bias
- print(ref)
- dff = abs(C - ref)
- print(dff.max(), dff.mean())
+def moe_gemm(
+ activations: torch.Tensor,
+ weights: torch.Tensor,
+ num_tokens_per_expert: torch.Tensor,
+ *,
+ scales: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+ """MOE GEMM (Mixture of Experts Grouped GEMM).
- def pack_unpack():
- m = n = k = 128
- groupsize = 32
- dt = torch.float32
- device = "xpu"
- B = torch.randint(-8, 7, (k, n), dtype=torch.int8, device=device)
- zp = torch.randint(-8, 7, (k // groupsize, n), dtype=torch.int8, device=device)
- scaleB = torch.rand(k // groupsize, n, dtype=dt, device=device) / 100
- blob = ark.repack_quantized_weight(B, scaleB, zp, groupsize, "fp32", "int4", "fp32", False)
- dq = ark.unpack_weight(blob, dt, n, k, groupsize, "fp32", "int4", "fp32", False)
- print(blob, dq)
- scale_re = scaleB.repeat_interleave(repeats=groupsize, dim=0).to(dt)
+ Computes grouped GEMM for MOE layers where different experts process
+ different numbers of tokens.
- DB = B.to(dt) * scale_re
- dff = abs(DB.T - dq)
- print(dff.max(), dff.mean())
+ Expects contiguous layouts:
+ - activations: [total_tokens, K] (BF16/FP16)
+ - weights: [num_experts, K, N] (BF16/FP16, Row major)
+ - num_tokens_per_expert: [num_experts] (int32)
+ - scales (optional): [num_experts, N] or None
- pack_unpack()
+ Returns:
+ - outputs: [total_tokens, N] (same dtype as activations)
+ """
+ if activations.device.type != "xpu":
+ raise NotImplementedError("moe_gemm is only supported on XPU")
+
+ if activations.dtype not in (torch.float16, torch.bfloat16):
+ raise ValueError(f"activations must be fp16/bf16, got {activations.dtype}")
+ if weights.dtype != activations.dtype:
+ raise ValueError("weights dtype must match activations dtype")
+
+ if activations.ndim != 2 or weights.ndim != 3:
+ raise ValueError("activations must be 2D [total_tokens, K], weights must be 3D [num_experts, K, N]")
+
+ if not activations.is_contiguous() or not weights.is_contiguous():
+ raise ValueError("activations and weights must be contiguous")
+
+ if num_tokens_per_expert.dtype != torch.int32:
+ num_tokens_per_expert = num_tokens_per_expert.to(torch.int32)
+
+ if not num_tokens_per_expert.is_contiguous():
+ num_tokens_per_expert = num_tokens_per_expert.contiguous()
+
+ total_tokens, K = activations.shape
+ num_experts, K_w, N = weights.shape # weights are [num_experts, K, N]
+
+ if K != K_w:
+ raise ValueError(f"K dimension mismatch: activations K={K}, weights K={K_w}")
+
+ if num_tokens_per_expert.shape[0] != num_experts:
+ raise ValueError(f"num_tokens_per_expert length {num_tokens_per_expert.shape[0]} != num_experts {num_experts}")
+
+ # Validate total tokens
+ expected_total = int(num_tokens_per_expert.sum().item())
+ if expected_total != total_tokens:
+ raise ValueError(f"Sum of num_tokens_per_expert ({expected_total}) != total_tokens ({total_tokens})")
+
+ lib = get_lib(activations)
+ stream = get_stream(activations)
+ outputs = torch.empty((total_tokens, N), device=activations.device, dtype=activations.dtype)
+
+ scales_ptr = scales.data_ptr() if scales is not None else 0
+
+ lib.moe_gemm(
+ stream,
+ activations.data_ptr(),
+ weights.data_ptr(),
+ scales_ptr,
+ outputs.data_ptr(),
+ cvt_dtype(activations.dtype),
+ N,
+ K,
+ num_tokens_per_expert.data_ptr(),
+ num_experts,
+ )
+ return outputs
def patch_torch_sdpa(*args, **kwargs):
@@ -1052,7 +1154,7 @@ def unpatch_torch_sdpa():
return unpatch_torch_sdpa_with_ark()
-__all__ = ["ARK", "patch_torch_sdpa", "unpatch_torch_sdpa"]
+__all__ = ["patch_torch_sdpa", "unpatch_torch_sdpa"]
# -----------------------------------------------------------------------------
@@ -1062,16 +1164,11 @@ __all__ = ["ARK", "patch_torch_sdpa", "unpatch_torch_sdpa"]
# this package as a module and expected certain functions to exist at the module
# level (e.g. ark.repack_quantized_weight, ark.woq_linear).
#
-# The current implementation exposes these as methods on the singleton ARK()
-# instance. The wrappers below keep backward compatibility without changing the
-# compiled extension.
+# The wrappers below keep backward compatibility for legacy positional call
+# styles without changing the compiled extension.
# -----------------------------------------------------------------------------
-def _ark_instance():
- return ARK()
-
-
def check_isa_supported(_isa: str) -> bool:
# Best-effort: some builds expose ISA checks via native libs; keep safe.
# Returning False is conservative and avoids misconfiguration.
@@ -1093,7 +1190,7 @@ def repack_quantized_weight(*args, **kwargs):
"""
if kwargs:
- return _ark_instance().repack_quantized_weight(**kwargs)
+ return _repack_quantized_weight_core(**kwargs)
if len(args) == 8:
QB, scaleB, zp, groupsize, compute_type, weight_type, scale_type, asym = args
@@ -1117,7 +1214,7 @@ def repack_quantized_weight(*args, **kwargs):
else:
zp = torch.empty(0, dtype=torch.int8, device=QB.device)
- return _ark_instance().repack_quantized_weight(
+ return _repack_quantized_weight_core(
QB,
scaleB,
zp,
@@ -1140,12 +1237,12 @@ def unpack_weight(
scale_type,
asym,
):
- return _ark_instance().unpack_weight(blob, out_dtype, n, k, groupsize, compute_type, weight_type, scale_type, asym)
+ return _unpack_weight_core(blob, out_dtype, n, k, groupsize, compute_type, weight_type, scale_type, asym)
def packed_weight_size(A: torch.Tensor, n, k, groupsize, compute_type, weight_type, scale_type, asym):
# Keep signature convenient for Python callers; native library needs a stream.
- lib = _ark_instance().get_lib(A)
+ lib = get_lib(A)
stream = get_stream(A)
ct = cvtstr_dtype(compute_type)
wt = cvtstr_dtype(weight_type)
@@ -1169,7 +1266,7 @@ def woq_linear(
if groupsize is None:
groupsize = A.shape[-1]
- result = _ark_instance().woqgemm(
+ result = woqgemm(
A,
packed_B,
bias,
@@ -1183,3 +1280,65 @@ def woq_linear(
)
out.copy_(result)
return out
+
+
+if __name__ == "__main__":
+ print(cpu_lib is None, xpu_lib is None)
+
+ def matmul_test():
+ m = n = k = 128
+ dt = torch.int8
+ device = "cpu"
+ has_bias = False
+ if dt == torch.int8:
+ A = torch.randint(-128, 127, (m, k), dtype=dt, device=device)
+ B = torch.randint(-128, 127, (n, k), dtype=dt, device=device)
+ C = igemm_s8s8s32(A, B)
+ print(C)
+ else:
+ A = torch.rand(m, k, dtype=dt, device=device) - 0.5
+ B = torch.rand(k, n, dtype=dt, device=device) - 0.5
+ bias = torch.rand(1, n, dtype=dt, device=device) if has_bias else torch.empty(0)
+ C = matmul(A, B, bias)
+ ref = torch.matmul(A, B.T)
+ if has_bias:
+ ref = ref + bias
+ dff = abs(C - ref)
+ if dt != torch.int8:
+ print(dff.max(), dff.mean())
+ print(torch.allclose(ref, C, 0.01, 0.1))
+
+ def woq():
+ m = n = k = 128
+ dt = torch.float32
+ device = "cpu"
+ A = torch.rand(m, k, dtype=dt, device=device) - 0.5
+ bias = torch.rand(1, n, dtype=dt, device=device) + 1000
+ B = torch.randint(-128, 127, (n, k), dtype=torch.int8, device=device)
+ scaleB = torch.rand(n, 1, dtype=dt, device=device)
+ C = woqgemm_s8(A, B, scaleB, bias)
+ print(C)
+ DB = B.to(dt) * scaleB
+ ref = torch.matmul(A, DB.T) + bias
+ print(ref)
+ dff = abs(C - ref)
+ print(dff.max(), dff.mean())
+
+ def pack_unpack():
+ m = n = k = 128
+ groupsize = 32
+ dt = torch.float32
+ device = "xpu"
+ B = torch.randint(-8, 7, (k, n), dtype=torch.int8, device=device)
+ zp = torch.randint(-8, 7, (k // groupsize, n), dtype=torch.int8, device=device)
+ scaleB = torch.rand(k // groupsize, n, dtype=dt, device=device) / 100
+ blob = repack_quantized_weight(B, scaleB, zp, groupsize, "fp32", "int4", "fp32", False)
+ dq = unpack_weight(blob, dt, n, k, groupsize, "fp32", "int4", "fp32", False)
+ print(blob, dq)
+ scale_re = scaleB.repeat_interleave(repeats=groupsize, dim=0).to(dt)
+
+ DB = B.to(dt) * scale_re
+ dff = abs(DB.T - dq)
+ print(dff.max(), dff.mean())
+
+ pack_unpack()
diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cpp
index b623ed437..dbf0eac16 100755
--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp
@@ -98,9 +98,12 @@ static size_t packed_weight_size(torch_ptr stream, int n, int k, int blocksize,
#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)
-static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask, int q_dtype,
- int k_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,
- int head_dim, float softmax_scale, bool is_causal) {
+static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
+ int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,
+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b,
+ int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int q_dtype, int k_dtype, int o_dtype,
+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
+ float softmax_scale, bool is_causal) {
if (k_dtype != q_dtype || o_dtype != q_dtype) {
throw std::invalid_argument("ark::sdpa: k_dtype and o_dtype must match q_dtype");
}
@@ -111,12 +114,17 @@ static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_
throw std::invalid_argument("ark::sdpa: mask and is_causal cannot both be set");
}
ark::sdpa_impl((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, (BTLA_DTYPE)(q_dtype),
+ q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b,
+ v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,
batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
}
static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
- int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch,
- int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,
+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
+ int o_stride_h, int o_stride_b, int q_dtype, int k_dtype, int v_dtype, int o_dtype,
+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
float softmax_scale, bool is_causal, bool use_int8_pv) {
if (mask && is_causal) {
throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");
@@ -124,17 +132,22 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,
if (q_dtype != k_dtype || o_dtype != q_dtype || v_dtype != q_dtype) {
throw std::invalid_argument("ark::sagev1: k_dtype and o_dtype must match q_dtype");
}
- if (q_dtype != (int)BTLA_DTYPE::F16) {
- throw std::invalid_argument("ark::sagev1: only F16 is supported for q_dtype");
+ if (q_dtype != (int)BTLA_DTYPE::F16 && q_dtype != (int)BTLA_DTYPE::BF16) {
+ throw std::invalid_argument("ark::sagev1: only F16 and BF16 are supported for q_dtype");
}
#ifdef ARK_XPU
if (use_int8_pv) {
XpuWrapper::sagev1_pvi8((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,
- scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
- softmax_scale, is_causal);
+ scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s,
+ k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,
+ o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,
+ seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, (BTLA_DTYPE)q_dtype);
} else {
XpuWrapper::sagev1((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, scale_block_size,
- batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+ q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h,
+ k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,
+ o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
+ softmax_scale, is_causal, (BTLA_DTYPE)q_dtype);
}
#else
throw std::runtime_error("ark::sagev1 is only supported on XPU");
@@ -142,42 +155,64 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,
}
static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
- int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q,
- int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
- sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch, num_heads_q,
- num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, false);
+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,
+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,
+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,
+ int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv,
+ int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
+ sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,
+ o_stride_s, o_stride_d, o_stride_h, o_stride_b, q_dtype, k_dtype, v_dtype, o_dtype, batch,
+ num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, false);
}
static void sagev1_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
- int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch,
- int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,
+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
+ int o_stride_h, int o_stride_b, int q_dtype, int k_dtype, int v_dtype, int o_dtype,
+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
float softmax_scale, bool is_causal) {
- sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch, num_heads_q,
- num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, true);
+ sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,
+ o_stride_s, o_stride_d, o_stride_h, o_stride_b, q_dtype, k_dtype, v_dtype, o_dtype, batch,
+ num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, true);
}
static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
- int scale_block_size, torch_ptr qscale, torch_ptr kscale, int q_dtype, int k_dtype, int o_dtype,
- int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
- float softmax_scale, bool is_causal) {
+ int scale_block_size, torch_ptr qscale, torch_ptr kscale, int q_stride_s, int q_stride_d,
+ int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b,
+ int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
+ int o_stride_h, int o_stride_b, int q_dtype, int k_dtype, int o_dtype, int batch, int num_heads_q,
+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,
+ bool is_causal) {
if (mask && is_causal) {
throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");
}
ark::sdpa_impl_qks8_pvhalf((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,
- scale_block_size, (void*)qscale, (void*)kscale, batch, num_heads_q, num_heads_kv,
- seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+ scale_block_size, (void*)qscale, (void*)kscale, q_stride_s, q_stride_d, q_stride_h,
+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,
+ v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch,
+ num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal,
+ (BTLA_DTYPE)o_dtype);
}
static void sage_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
- int scale_block_size, torch_ptr qscale, torch_ptr kscale, torch_ptr vscale, int q_dtype,
+ int scale_block_size, torch_ptr qscale, torch_ptr kscale, torch_ptr vscale, int q_stride_s,
+ int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,
+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,
+ int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int q_dtype,
int k_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
if (mask && is_causal) {
throw std::invalid_argument("ark::sage_pvi8: mask and is_causal cannot both be set");
}
ark::sdpa_impl_qks8_pvi8((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,
- scale_block_size, (void*)qscale, (void*)kscale, (void*)vscale, batch, num_heads_q,
- num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+ scale_block_size, (void*)qscale, (void*)kscale, (void*)vscale, q_stride_s, q_stride_d,
+ q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,
+ v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,
+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
+ is_causal, (BTLA_DTYPE)o_dtype);
}
static void moe_gemm_wrapper(torch_ptr stream, torch_ptr activations, torch_ptr weights, torch_ptr scales,
@@ -309,6 +344,77 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr bias
});
}
}
+
+static void sage_compute_seq_mean_bias_layout(torch_ptr stream, torch_ptr input, torch_ptr output, int batch,
+ int num_heads, int seq, int head_dim, int stride_seq, int stride_dim,
+ int stride_head, int stride_batch) {
+ auto* q = (sycl::queue*)stream;
+ auto* in_ptr = (sycl::half*)input;
+ auto* out_ptr = (sycl::half*)output;
+ if (stride_dim != 1) {
+ throw std::invalid_argument("ark::sage_compute_seq_mean_bias_layout: head-dim stride must be 1");
+ }
+ if (ark::XpuWrapper::is_packed_hnd(stride_seq, stride_dim, stride_head, stride_batch, num_heads, seq, head_dim)) {
+ ark::XpuWrapper::compute_seq_mean_bias<sycl::half>(q, in_ptr, out_ptr, batch * num_heads, seq, head_dim);
+ } else {
+ ark::XpuWrapper::compute_seq_mean_bias_strided<sycl::half>(q, in_ptr, out_ptr, batch, num_heads, seq, head_dim,
+ stride_seq, stride_dim, stride_head, stride_batch);
+ }
+}
+
+static void sage_dynamic_quant_layout(torch_ptr stream, torch_ptr input, torch_ptr bias, torch_ptr output,
+ torch_ptr scale_out, int batch, int num_heads, int seq, int head_dim,
+ int block_size, int stride_seq, int stride_dim, int stride_head,
+ int stride_batch) {
+ auto* q = (sycl::queue*)stream;
+ auto* in_ptr = (sycl::half*)input;
+ auto* bias_ptr = bias ? (sycl::half*)bias : nullptr;
+ auto* out_ptr = (int8_t*)output;
+ auto* scale_ptr = (float*)scale_out;
+ if (block_size <= 0) {
+ throw std::invalid_argument("ark::sage_dynamic_quant_layout: block_size must be > 0");
+ }
+ if (stride_dim != 1) {
+ throw std::invalid_argument("ark::sage_dynamic_quant_layout: head-dim stride must be 1");
+ }
+ int n_seq_blk = (seq + block_size - 1) / block_size;
+ bool force_strided = ark::env_params::Instance()->sage_disable_packed_hnd_fast != 0;
+ if (!force_strided &&
+ ark::XpuWrapper::is_packed_hnd(stride_seq, stride_dim, stride_head, stride_batch, num_heads, seq, head_dim)) {
+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, in_ptr, out_ptr, scale_ptr, batch * num_heads, seq, n_seq_blk,
+ head_dim, block_size, bias_ptr);
+ } else {
+ ark::XpuWrapper::sage_dynamic_quant_strided<sycl::half>(q, in_ptr, out_ptr, scale_ptr, batch, num_heads, seq,
+ n_seq_blk, head_dim, block_size, stride_seq, stride_dim,
+ stride_head, stride_batch, bias_ptr);
+ }
+}
+
+static void sage_dynamic_quant_v_layout(torch_ptr stream, torch_ptr input, torch_ptr output, torch_ptr scale_out,
+ int batch, int num_heads, int seq, int head_dim, int block_size,
+ int stride_dim, int stride_seq, int stride_head, int stride_batch) {
+ auto* q = (sycl::queue*)stream;
+ auto* in_ptr = (sycl::half*)input;
+ auto* out_ptr = (int8_t*)output;
+ auto* scale_ptr = (float*)scale_out;
+ if (block_size <= 0) {
+ throw std::invalid_argument("ark::sage_dynamic_quant_v_layout: block_size must be > 0");
+ }
+ if (stride_dim != 1) {
+ throw std::invalid_argument("ark::sage_dynamic_quant_v_layout: head-dim stride must be 1");
+ }
+ int n_seq_blk = (seq + block_size - 1) / block_size;
+ bool force_strided = ark::env_params::Instance()->sage_disable_packed_hnd_fast != 0;
+ if (!force_strided &&
+ ark::XpuWrapper::is_packed_hnd(stride_seq, stride_dim, stride_head, stride_batch, num_heads, seq, head_dim)) {
+ ark::XpuWrapper::sage_dynamic_quant_v<sycl::half>(q, in_ptr, out_ptr, scale_ptr, batch * num_heads, seq,
+ n_seq_blk, head_dim, block_size);
+ } else {
+ ark::XpuWrapper::sage_dynamic_quant_v_strided<sycl::half>(q, in_ptr, out_ptr, scale_ptr, batch, num_heads, seq,
+ n_seq_blk, head_dim, block_size, stride_dim, stride_seq,
+ stride_head, stride_batch);
+ }
+}
#endif // ARK_XPU && ARK_SYCL_TLA
} // namespace ark
@@ -329,6 +435,9 @@ PYBIND11_MODULE(PY_NAME, m) {
// Low-level SAGE PVi8 API: input Q/K/V are pre-quantized int8 with qscale/kscale/vscale.
m.def("sage_pvi8", &ark::sage_pvi8);
m.def("sage_dynamic_quant", &ark::sage_dynamic_quant);
+ m.def("sage_compute_seq_mean_bias_layout", &ark::sage_compute_seq_mean_bias_layout);
+ m.def("sage_dynamic_quant_layout", &ark::sage_dynamic_quant_layout);
+ m.def("sage_dynamic_quant_v_layout", &ark::sage_dynamic_quant_v_layout);
m.def("moe_gemm", &ark::moe_gemm_wrapper);
#endif
}
\ No newline at end of file
diff --git a/auto_round_extension/ark/auto_round_kernel/qlinear.py b/auto_round_extension/ark/auto_round_kernel/qlinear.py
index 125e77e90..749d2db1c 100644
--- a/auto_round_extension/ark/auto_round_kernel/qlinear.py
+++ b/auto_round_extension/ark/auto_round_kernel/qlinear.py
@@ -21,7 +21,7 @@ try:
import auto_round_kernel
ARK_INSTALLED = True
- ark = auto_round_kernel.ARK()
+ ark = auto_round_kernel
except:
ARK_INSTALLED = False
@@ -130,12 +130,16 @@ class QuantLinear(nn.Module):
raise NotImplementedError("in_features must be divisible by group_size")
if "awq" in self.QUANT_TYPE:
self.register_buffer(
- "qweight", torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32)
+ "qweight",
+ torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32),
)
else:
self.register_buffer(
"qweight",
- torch.zeros((self.infeatures // 32 * self.bits, self.outfeatures), dtype=torch.int32),
+ torch.zeros(
+ (self.infeatures // 32 * self.bits, self.outfeatures),
+ dtype=torch.int32,
+ ),
)
self.register_buffer(
"qzeros",
@@ -394,7 +398,14 @@ class QuantLinearFP8(nn.Module):
self.trainable = trainable
def _load_from_state_dict(
- self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
+ self,
+ state_dict,
+ prefix,
+ local_metadata,
+ strict,
+ missing_keys,
+ unexpected_keys,
+ error_msgs,
):
"""Override to handle weight_scale shape mismatch between checkpoint and model."""
weight_scale_key = prefix + "weight_scale"
@@ -404,7 +415,10 @@ class QuantLinearFP8(nn.Module):
out_features, num_groups = expected_shape
expected_numel = out_features * num_groups
- if loaded_scale.ndim == 2 and loaded_scale.shape == (num_groups, out_features):
+ if loaded_scale.ndim == 2 and loaded_scale.shape == (
+ num_groups,
+ out_features,
+ ):
state_dict[weight_scale_key] = loaded_scale.t().contiguous()
loaded_scale = state_dict[weight_scale_key]
@@ -424,7 +438,9 @@ class QuantLinearFP8(nn.Module):
inferred_shape = (out_features, inferred_num_groups)
state_dict[weight_scale_key] = loaded_scale.view(inferred_shape)
self.weight_scale = torch.zeros(
- inferred_shape, dtype=self.weight_scale.dtype, device=self.weight_scale.device
+ inferred_shape,
+ dtype=self.weight_scale.dtype,
+ device=self.weight_scale.device,
)
else:
@@ -432,7 +448,13 @@ class QuantLinearFP8(nn.Module):
f"Cannot reshape scale! loaded_scale.numel()={numel}, expected={expected_numel}, out_features={out_features}"
)
super()._load_from_state_dict(
- state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
+ state_dict,
+ prefix,
+ local_metadata,
+ strict,
+ missing_keys,
+ unexpected_keys,
+ error_msgs,
)
def post_init(self):
diff --git a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
index f5e30e7aa..03a9ada85 100644
--- a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
@@ -50,16 +50,42 @@ int launch_prefill_kernel_f16_64_sage_i8pv(detail::Options const& options) {
options);
}
-KernelLauncher select_sage_prefill_launcher(BTLA_DTYPE dtype, int head_dim, bool use_int8_pv) {
+int launch_prefill_kernel_bf16_128_sage(detail::Options const& options) {
+ return launch_sage_prefill_kernel_128<cute::int8_t, cute::int8_t, cute::bfloat16_t>(options);
+}
+
+int launch_prefill_kernel_bf16_128_sage_i8pv(detail::Options const& options) {
+ return launch_sage_prefill_kernel_128<cute::int8_t, cute::int8_t, cute::int8_t, cute::bfloat16_t, true, true,
+ true>(options);
+}
+
+int launch_prefill_kernel_bf16_64_sage(detail::Options const& options) {
+ return launch_sage_prefill_kernel_64<cute::int8_t, cute::int8_t, cute::bfloat16_t>(options);
+}
+
+int launch_prefill_kernel_bf16_64_sage_i8pv(detail::Options const& options) {
+ return launch_sage_prefill_kernel_64<cute::int8_t, cute::int8_t, cute::int8_t, cute::bfloat16_t, true, true,
+ true>(options);
+}
+
+KernelLauncher select_sage_prefill_launcher(BTLA_DTYPE dtype, BTLA_DTYPE pv_dtype, int head_dim, bool use_int8_pv) {
switch (dtype) {
case BTLA_DTYPE::S8:
switch (head_dim) {
case 128:
- if (!use_int8_pv) return launch_prefill_kernel_f16_128_sage;
- return launch_prefill_kernel_f16_128_sage_i8pv;
+ if (use_int8_pv) {
+ if (pv_dtype == BTLA_DTYPE::BF16) return launch_prefill_kernel_bf16_128_sage_i8pv;
+ return launch_prefill_kernel_f16_128_sage_i8pv;
+ }
+ if (pv_dtype == BTLA_DTYPE::BF16) return launch_prefill_kernel_bf16_128_sage;
+ return launch_prefill_kernel_f16_128_sage;
case 64:
- if (!use_int8_pv) return launch_prefill_kernel_f16_64_sage;
- return launch_prefill_kernel_f16_64_sage_i8pv;
+ if (use_int8_pv) {
+ if (pv_dtype == BTLA_DTYPE::BF16) return launch_prefill_kernel_bf16_64_sage_i8pv;
+ return launch_prefill_kernel_f16_64_sage_i8pv;
+ }
+ if (pv_dtype == BTLA_DTYPE::BF16) return launch_prefill_kernel_bf16_64_sage;
+ return launch_prefill_kernel_f16_64_sage;
default:
return nullptr;
}
@@ -153,15 +179,38 @@ KernelLauncher select_decode_launcher(BTLA_DTYPE dtype, int head_dim) {
}
}
-detail::Options make_common_options(void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int batch,
- int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
- float softmax_scale, bool is_causal) {
+detail::Options make_common_options(void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int q_stride_s,
+ int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,
+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,
+ int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,
+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,
+ int head_dim, float softmax_scale, bool is_causal) {
+ if (q_stride_d != 1 || k_stride_d != 1 || v_stride_d != 1 || o_stride_d != 1) {
+ throw std::invalid_argument("make_common_options: head-dim stride must be 1 for Q/K/V/O");
+ }
detail::Options options;
options.q = Q_ptr;
options.k = K_ptr;
options.v = V_ptr;
options.mask = mask;
options.o = O_ptr;
+ options.use_tensor_strides = true;
+ options.q_stride_s = q_stride_s;
+ options.q_stride_d = q_stride_d;
+ options.q_stride_h = q_stride_h;
+ options.q_stride_b = q_stride_b;
+ options.k_stride_s = k_stride_s;
+ options.k_stride_d = k_stride_d;
+ options.k_stride_h = k_stride_h;
+ options.k_stride_b = k_stride_b;
+ options.v_stride_d = v_stride_d;
+ options.v_stride_s = v_stride_s;
+ options.v_stride_h = v_stride_h;
+ options.v_stride_b = v_stride_b;
+ options.o_stride_s = o_stride_s;
+ options.o_stride_d = o_stride_d;
+ options.o_stride_h = o_stride_h;
+ options.o_stride_b = o_stride_b;
options.batch = batch;
options.num_heads_q = num_heads_q;
options.num_heads_kv = num_heads_kv;
@@ -176,32 +225,44 @@ detail::Options make_common_options(void* Q_ptr, void* K_ptr, void* V_ptr, void*
} // namespace
-void sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,
- void* kscale, void* vscale, bool use_int8_pv,
- BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
- detail::Options options = make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, batch, num_heads_q, num_heads_kv,
- seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+void sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
+ int scale_block_size, void* qscale, void* kscale, void* vscale, bool use_int8_pv,
+ BTLA_DTYPE q_dtype, BTLA_DTYPE pv_dtype, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,
+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,
+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,
+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
+ float softmax_scale, bool is_causal) {
+ detail::Options options =
+ make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,
+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,
+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
options.scale_block_size = scale_block_size;
options.qscale = qscale;
options.kscale = kscale;
options.vscale = vscale;
compat::set_default_queue(*q);
- KernelLauncher launcher = select_sage_prefill_launcher(q_dtype, head_dim, use_int8_pv);
+ KernelLauncher launcher = select_sage_prefill_launcher(q_dtype, pv_dtype, head_dim, use_int8_pv);
if (launcher == nullptr) {
throw std::runtime_error(
- "Unsupported dtype or head dimension for sage_prefill / SAGE (only F16/BF16 and 64/96/128/192 are supported)");
+ "Unsupported dtype or head dimension for sage_prefill / SAGE (only F16/BF16 PV and 64/128 are supported)");
}
launcher(options);
}
void flash_attn_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
- BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,
- int head_dim, float softmax_scale, bool is_causal) {
- detail::Options options = make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, batch, num_heads_q, num_heads_kv,
- seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+ BTLA_DTYPE q_dtype, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,
+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
+ int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
+ int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
+ detail::Options options =
+ make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,
+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,
+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
compat::set_default_queue(*q);
KernelLauncher launcher = select_prefill_launcher(q_dtype, head_dim);
@@ -214,10 +275,16 @@ void flash_attn_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, v
}
void flash_attn_decode(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
- BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_kv, int head_dim,
- float softmax_scale, bool is_causal) {
- detail::Options options = make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, batch, num_heads_q,
- num_heads_kv, 1, seq_len_kv, head_dim, softmax_scale, is_causal);
+ BTLA_DTYPE q_dtype, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,
+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
+ int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
+ detail::Options options =
+ make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,
+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,
+ num_heads_kv, 1, seq_len_kv, head_dim, softmax_scale, is_causal);
compat::set_default_queue(*q);
KernelLauncher launcher = select_decode_launcher(q_dtype, head_dim);
@@ -230,8 +297,11 @@ void flash_attn_decode(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, vo
}
void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, BTLA_DTYPE q_dtype,
- int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
- float softmax_scale, bool is_causal) {
+ int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,
+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b,
+ int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch, int num_heads_q,
+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,
+ bool is_causal) {
// if (q_dtype != BTLA_DTYPE::F16 && q_dtype != BTLA_DTYPE::BF16) {
// throw std::invalid_argument("sdpa_impl: only FP16 and BF16 are supported");
// }
@@ -243,18 +313,25 @@ void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_pt
throw std::invalid_argument("sdpa_impl: seq_len_q and seq_len_kv must be greater than 0");
}
if (seq_len_q == 1) {
- flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, batch, num_heads_q, num_heads_kv, seq_len_kv,
- head_dim, softmax_scale, is_causal);
+ flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,
+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,
+ seq_len_kv, head_dim, softmax_scale, is_causal);
return;
}
- flash_attn_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, batch, num_heads_q, num_heads_kv, seq_len_q,
- seq_len_kv, head_dim, softmax_scale, is_causal);
+ flash_attn_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,
+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,
+ seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
}
-void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,
- void* kscale, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
+void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
+ int scale_block_size, void* qscale, void* kscale, int q_stride_s, int q_stride_d, int q_stride_h,
+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h,
+ int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE pv_dtype) {
if (mask && is_causal) {
throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");
}
@@ -262,22 +339,31 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr
if (seq_len_q <= 0 || seq_len_kv <= 0) {
throw std::invalid_argument("sdpa_impl: seq_len_q and seq_len_kv must be greater than 0");
}
+ if (pv_dtype != BTLA_DTYPE::F16 && pv_dtype != BTLA_DTYPE::BF16) {
+ throw std::invalid_argument("sdpa_impl_qks8_pvhalf: only F16 and BF16 are supported for V/O dtype");
+ }
if (seq_len_q == 1) {
- flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, BTLA_DTYPE::F16, batch, num_heads_q, num_heads_kv, seq_len_kv,
- head_dim, softmax_scale, is_causal);
+ flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, pv_dtype, q_stride_s, q_stride_d, q_stride_h,
+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,
+ v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,
+ num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal);
return;
}
-
sage_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, nullptr, false,
- BTLA_DTYPE::S8, batch, num_heads_q, num_heads_kv, seq_len_q,
+ BTLA_DTYPE::S8, pv_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,
+ k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s,
+ o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q,
seq_len_kv, head_dim, softmax_scale, is_causal);
}
void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
- int scale_block_size, void* qscale, void* kscale, void* vscale, int batch, int num_heads_q,
- int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,
- bool is_causal) {
+ int scale_block_size, void* qscale, void* kscale, void* vscale, int q_stride_s,
+ int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,
+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,
+ int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,
+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,
+ int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype) {
if (mask && is_causal) {
throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");
}
@@ -288,15 +374,22 @@ void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,
if (vscale == nullptr) {
throw std::invalid_argument("sdpa_impl_qks8_pvi8: vscale must be provided for int8 PV");
}
+ if (o_dtype != BTLA_DTYPE::F16 && o_dtype != BTLA_DTYPE::BF16) {
+ throw std::invalid_argument("sdpa_impl_qks8_pvi8: only F16 and BF16 are supported for output dtype");
+ }
if (seq_len_q == 1) {
- flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, BTLA_DTYPE::F16, batch, num_heads_q, num_heads_kv,
- seq_len_kv, head_dim, softmax_scale, is_causal);
+ flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, o_dtype, q_stride_s, q_stride_d, q_stride_h,
+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,
+ v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,
+ num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal);
return;
}
sage_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, vscale, true,
- BTLA_DTYPE::S8, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
- is_causal);
+ BTLA_DTYPE::S8, o_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,
+ k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,
+ o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
+ softmax_scale, is_causal);
}
} // namespace ark
diff --git a/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py
index aa9f2ac30..17e2864f5 100644
--- a/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py
+++ b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py
@@ -7,7 +7,9 @@ import logging
import torch
-from . import ARK
+from . import sagev1, sagev1_pvi8
+from . import sdpa as ark_sdpa
+from . import xpu_lib
logger = logging.getLogger(__name__)
@@ -134,8 +136,7 @@ def patch_torch_sdpa_with_ark(*, strict: bool = False, backend: str = "sdpa", qu
raise RuntimeError("XPU is not available")
return False
- ark = ARK()
- if ark.xpu_lib is None or not hasattr(ark.xpu_lib, backend):
+ if xpu_lib is None or not hasattr(xpu_lib, backend):
if strict:
raise RuntimeError(f"ARK XPU {backend} kernel is not available")
return False
@@ -186,30 +187,30 @@ def patch_torch_sdpa_with_ark(*, strict: bool = False, backend: str = "sdpa", qu
try:
if backend == "sdpa":
- return ark.sdpa(
- query.contiguous(),
- key.contiguous(),
- value.contiguous(),
+ return ark_sdpa(
+ query,
+ key,
+ value,
attn_mask=normalized_mask,
dropout_p=dropout_p,
is_causal=is_causal,
scale=scale,
)
if backend == "sagev1":
- return ark.sagev1(
- query.contiguous(),
- key.contiguous(),
- value.contiguous(),
+ return sagev1(
+ query,
+ key,
+ value,
attn_mask=normalized_mask,
dropout_p=dropout_p,
is_causal=is_causal,
scale=scale,
quant_block_size=quant_block_size,
)
- return ark.sagev1_pvi8(
- query.contiguous(),
- key.contiguous(),
- value.contiguous(),
+ return sagev1_pvi8(
+ query,
+ key,
+ value,
attn_mask=normalized_mask,
dropout_p=dropout_p,
is_causal=is_causal,
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp
index 62850e46b..806dcb0ca 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp
@@ -58,17 +58,27 @@ void moe_gemm(sycl::queue* q, void* activations, void* weights, void* scales, vo
* @param is_causal Whether to apply causal mask
*/
void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, BTLA_DTYPE q_dtype,
- int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
+ int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,
+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b,
+ int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch, int num_heads_q,
+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
float softmax_scale, bool is_causal);
-void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,
- void* kscale, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal);
+void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
+ int scale_block_size, void* qscale, void* kscale, int q_stride_s, int q_stride_d, int q_stride_h,
+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h,
+ int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
+ BTLA_DTYPE pv_dtype = BTLA_DTYPE::F16);
void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
- int scale_block_size, void* qscale, void* kscale, void* vscale, int batch, int num_heads_q,
- int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,
- bool is_causal);
+ int scale_block_size, void* qscale, void* kscale, void* vscale, int q_stride_s,
+ int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,
+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,
+ int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,
+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
+ float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype = BTLA_DTYPE::F16);
#endif // ARK_SYCL_TLA
} // namespace ark
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp
index 6c4cdb006..f80a89b47 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp
@@ -74,10 +74,15 @@ struct Options {
bool is_causal = false;
bool varlen = false;
bool use_paged_kv = false;
+ bool use_tensor_strides = false;
int batch = 0, num_heads_q = 0, num_heads_kv = 0, seq_len_qo = 0, seq_len_kv = 0, seq_len_kv_cache = 0, page_size = 0,
head_size_qk = 0, head_size_vo = 0;
int total_seqlen_q = 0, total_seqlen_kv = 0, total_seqlen_kv_cache = 0;
int max_seqlen_q = 0, max_seqlen_kv = 0, max_seqlen_kv_cache = 0;
+ int q_stride_s = 0, q_stride_d = 1, q_stride_h = 0, q_stride_b = 0;
+ int k_stride_s = 0, k_stride_d = 1, k_stride_h = 0, k_stride_b = 0;
+ int v_stride_d = 1, v_stride_s = 0, v_stride_h = 0, v_stride_b = 0;
+ int o_stride_s = 0, o_stride_d = 1, o_stride_h = 0, o_stride_b = 0;
float softmax_scale = 0.0f;
bool persistent = false;
@@ -95,6 +100,7 @@ struct Options {
<< " is_causal: " << is_causal << "\n"
<< " varlen: " << varlen << "\n"
<< " use_paged_kv: " << use_paged_kv << "\n"
+ << " use_tensor_strides: " << use_tensor_strides << "\n"
<< " batch: " << batch << "\n"
<< " num_heads_q: " << num_heads_q << "\n"
<< " num_heads_kv: " << num_heads_kv << "\n"
@@ -110,12 +116,25 @@ struct Options {
<< " max_seqlen_q: " << max_seqlen_q << "\n"
<< " max_seqlen_kv: " << max_seqlen_kv << "\n"
<< " max_seqlen_kv_cache: " << max_seqlen_kv_cache << "\n"
+ << " q_stride: (" << q_stride_s << ", " << q_stride_d << ", " << q_stride_h << ", " << q_stride_b << ")\n"
+ << " k_stride: (" << k_stride_s << ", " << k_stride_d << ", " << k_stride_h << ", " << k_stride_b << ")\n"
+ << " v_stride: (" << v_stride_d << ", " << v_stride_s << ", " << v_stride_h << ", " << v_stride_b << ")\n"
+ << " o_stride: (" << o_stride_s << ", " << o_stride_d << ", " << o_stride_h << ", " << o_stride_b << ")\n"
<< " softmax_scale: " << softmax_scale << "\n"
<< " persistent: " << persistent << "\n"
<< "}\n";
}
};
+ template <typename StrideQ, typename StrideK, typename StrideV, typename StrideO>
+ inline void set_tensor_strides_from_options(const Options& options, StrideQ& stride_Q, StrideK& stride_K,
+ StrideV& stride_V, StrideO& stride_O) {
+ stride_Q = cute::make_stride(options.q_stride_s, cute::_1{}, options.q_stride_h, options.q_stride_b);
+ stride_K = cute::make_stride(options.k_stride_s, cute::_1{}, options.k_stride_h, options.k_stride_b);
+ stride_V = cute::make_stride(cute::_1{}, options.v_stride_s, options.v_stride_h, options.v_stride_b);
+ stride_O = cute::make_stride(options.o_stride_s, cute::_1{}, options.o_stride_h, options.o_stride_b);
+ }
+
// 3 input matrices: (K)eys, (Q)ueries and (V)alues.
using LayoutQ = cutlass::layout::RowMajor;
using LayoutK = cutlass::layout::ColumnMajor;
@@ -237,12 +256,23 @@ struct KernelRunner {
auto shape_V_cache = cute::make_shape(head_size_vo, seq_len_kv_cache, num_heads_kv, batch);
auto shape_O = cute::make_shape(seq_len_qo, head_size_vo, num_heads_q, batch);
- stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);
- stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);
- stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);
+ if constexpr (!isVarLen) {
+ if (options.use_tensor_strides) {
+ set_tensor_strides_from_options(options, stride_Q, stride_K, stride_V, stride_O);
+ } else {
+ stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);
+ stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);
+ stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);
+ stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);
+ }
+ } else {
+ stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);
+ stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);
+ stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);
+ stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);
+ }
stride_K_cache = cutlass::make_cute_packed_stride(StrideK{}, shape_K_cache);
stride_V_cache = cutlass::make_cute_packed_stride(StrideV{}, shape_V_cache);
- stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);
if constexpr (isVarLen) {
// shape.seq_len_qo.cumulative_length = device_cumulative_seqlen_q.get();
@@ -573,12 +603,23 @@ struct SageKernelRunner {
auto shape_V_cache = cute::make_shape(head_size_vo, seq_len_kv_cache, num_heads_kv, batch);
auto shape_O = cute::make_shape(seq_len_qo, head_size_vo, num_heads_q, batch);
- stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);
- stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);
- stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);
+ if constexpr (!isVarLen) {
+ if (options.use_tensor_strides) {
+ set_tensor_strides_from_options(options, stride_Q, stride_K, stride_V, stride_O);
+ } else {
+ stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);
+ stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);
+ stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);
+ stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);
+ }
+ } else {
+ stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);
+ stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);
+ stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);
+ stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);
+ }
stride_K_cache = cutlass::make_cute_packed_stride(StrideK{}, shape_K_cache);
stride_V_cache = cutlass::make_cute_packed_stride(StrideV{}, shape_V_cache);
- stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);
if constexpr (isVarLen) {
// shape.seq_len_qo.cumulative_length = device_cumulative_seqlen_q.get();
@@ -650,8 +691,12 @@ struct SageConfig {
static constexpr int SGTileQ = get<0>(shape_div(TileShapeQK{}, shape(SubgroupLayoutQK{})))();
using MMAOperation =
cute::conditional_t<is_void_v<MMAOperation_>, XE_DPAS_TT<cute::gcd(SGTileQ, 8), int32_t, int8_t>, MMAOperation_>;
+ // The PV "float" tiled MMA operates on the output element type. For UseInt8PV the kernel also
+ // constructs a separate int8 quantized PV MMA internally, while this float MMA only needs to
+ // describe the tile shape used for the accumulator and dequantized path. For the non-int8-PV
+ // path, V is consumed directly so we use ElementO (== ElementV) which supports half_t / bfloat16_t.
using MMAOperationPV = cute::conditional_t<is_void_v<MMAOperation_>,
- XE_DPAS_TT<cute::gcd(SGTileQ, 8), float, cute::half_t>, MMAOperation_>;
+ XE_DPAS_TT<cute::gcd(SGTileQ, 8), float, ElementO>, MMAOperation_>;
using SubgroupLayoutPV =
cute::conditional_t<is_void_v<SubgroupLayoutPV_>,
decltype(cutlass::fmha::collective::get_sg_layout_pv(SubgroupLayoutQK{})), SubgroupLayoutPV_>;
@@ -811,8 +856,8 @@ inline int launch_prefill_kernel_128(Options const& options) {
template <typename ElementQ, typename ElementK, typename ElementV, typename ElementO = ElementV, bool UseInt8PV = false,
bool WriteBackInt8PV = true, bool ExecuteInt8PV = true>
inline int launch_sage_prefill_kernel_128(Options const& options) {
- constexpr int PipelineStages = 1;
- constexpr int PipelineStages1 = 1;
+ constexpr int PipelineStages = 2;
+ constexpr int PipelineStages1 = 2;
using ShapeQK = Shape<_256, _64, _32>;
using ShapePV = Shape<_256, _32, _64>;
using ShapeOut = Shape<_256, _128>;
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
index 8b50ee769..bbf2f2bad 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
@@ -34,6 +34,7 @@ struct env_params {
int auto_s8 = 0;
int sage_use_mean_bias = 1;
int sage_print_kbias = 0;
+ int sage_disable_packed_hnd_fast = 0;
static env_params* Instance() {
static env_params instance;
return &instance;
@@ -43,6 +44,7 @@ struct env_params {
env_i("ARK_AUTO_S8", auto_s8);
env_i("ARK_SAGE_USE_MEAN_BIAS", sage_use_mean_bias);
env_i("ARK_SAGE_PRINT_KBIAS", sage_print_kbias);
+ env_i("ARK_SAGE_DISABLE_PACKED_HND_FAST", sage_disable_packed_hnd_fast);
}
static inline void env_i(const char* envstr, int& default_) {
const char* log_level_env = getenv(envstr);
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
index 51a346e2b..2cc82ff19 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
@@ -665,6 +665,18 @@ class XpuWrapper {
#endif
}
+ static inline size_t logical_hnd_offset(int batch_id, int head_id, int seq_id, int dim_id, int stride_seq,
+ int stride_dim, int stride_head, int stride_batch) {
+ return size_t(batch_id) * size_t(stride_batch) + size_t(head_id) * size_t(stride_head) +
+ size_t(seq_id) * size_t(stride_seq) + size_t(dim_id) * size_t(stride_dim);
+ }
+
+ static inline bool is_packed_hnd(int stride_seq, int stride_dim, int stride_head, int stride_batch, int num_heads,
+ int seq_len, int head_dim) {
+ return stride_dim == 1 && stride_seq == head_dim && stride_head == seq_len * head_dim &&
+ stride_batch == num_heads * seq_len * head_dim;
+ }
+
static void woq_gemm(int m, const void* a, const void* b, void* c, const void* bias, BTLA_DTYPE acdt, QuantParam* p,
sycl::queue* q) {
auto ret = woq_gemv(q, m, p, a, b, c, bias, acdt);
@@ -739,6 +751,56 @@ class XpuWrapper {
});
}
+ template <typename T>
+ static void compute_seq_mean_bias_strided(sycl::queue* q, const T* in_ptr, T* bias_ptr, int batch, int num_heads,
+ int seq, int head_dim, int stride_seq, int stride_dim, int stride_head,
+ int stride_batch) {
+ constexpr int SG_SIZE = 32;
+ constexpr int WG_SIZE = 512;
+ constexpr int SG_PER_WG = WG_SIZE / SG_SIZE;
+ constexpr int SeqUnroll = 4;
+ size_t total = size_t(batch) * size_t(num_heads) * size_t(head_dim);
+ size_t num_groups = (total + SG_PER_WG - 1) / SG_PER_WG;
+ size_t global = num_groups * WG_SIZE;
+ q->parallel_for(sycl::nd_range<1>(global, WG_SIZE),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ size_t group_id = item.get_group(0);
+ int local_id = item.get_local_id(0);
+ int subgroup_id = local_id / SG_SIZE;
+ int lane_id = local_id % SG_SIZE;
+ size_t index = group_id * SG_PER_WG + subgroup_id;
+ if (index >= total) {
+ return;
+ }
+
+ int row_id = int(index / size_t(head_dim));
+ int dim_id = int(index % size_t(head_dim));
+ int batch_id = row_id / num_heads;
+ int head_id = row_id % num_heads;
+ size_t base_offset = logical_hnd_offset(batch_id, head_id, 0, dim_id, stride_seq, stride_dim,
+ stride_head, stride_batch);
+ auto sg = item.get_sub_group();
+
+ float sum = 0.0f;
+ int token = lane_id;
+ for (; token + (SeqUnroll - 1) * SG_SIZE < seq; token += SeqUnroll * SG_SIZE) {
+#pragma unroll
+ for (int unroll_idx = 0; unroll_idx < SeqUnroll; ++unroll_idx) {
+ sum += static_cast<float>(in_ptr[base_offset +
+ size_t(token + unroll_idx * SG_SIZE) *
+ size_t(stride_seq)]);
+ }
+ }
+ for (; token < seq; token += SG_SIZE) {
+ sum += static_cast<float>(in_ptr[base_offset + size_t(token) * size_t(stride_seq)]);
+ }
+ sum = sycl::reduce_over_group(sg, sum, sycl::plus<float>{});
+ if (lane_id == 0) {
+ bias_ptr[index] = static_cast<T>(sum / static_cast<float>(seq));
+ }
+ });
+ }
+
template <typename T>
static void print_value_distribution(sycl::queue* q, const T* dev_ptr, size_t count, const char* name) {
if (dev_ptr == nullptr || count == 0) {
@@ -782,262 +844,82 @@ class XpuWrapper {
bool fast_vector_bias = bias_ptr && (head_dim % Unroll == 0);
bool power_of_two_head_dim = fast_vector_bias && ((head_dim & (head_dim - 1)) == 0);
int bias_mask = head_dim - 1;
- if (elems_per_block > MAX_Reg * MAX_WG_SIZE) {
+ if (true) { // Always use two-pass load for better performance on Intel GPU
int wg_size = (elems_per_block <= 256) ? SG_SIZE : 256;
// Ensure wg_size is a multiple of SG_SIZE
wg_size = ((wg_size + SG_SIZE - 1) / SG_SIZE) * SG_SIZE;
- if (bias_ptr) {
- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
- int block_id = item.get_group(0);
- int row_id = block_id / n_seq_blk;
- int seq_id = block_id % n_seq_blk;
- int tid = item.get_local_id(0);
- auto wg = item.get_group();
- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
- auto* row_bias = bias_ptr + (size_t)row_id * head_dim;
-
- int elems_this_wg = (seq_id + 1) * block_size <= seq ? elems_per_block
- : (seq - seq_id * block_size) * head_dim;
-
- float local_max = 0.0f;
- sycl::vec<T, Unroll> local_data, local_max_vec;
- local_max_vec = sycl::vec<T, Unroll>(0.0f);
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
- local_data = *(sycl::vec<T, Unroll>*)(&block_in[i]);
- if (fast_vector_bias) {
- int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);
- local_data = local_data - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);
- } else {
-#pragma unroll
- for (int j = 0; j < Unroll; ++j) {
- local_data[j] = local_data[j] - row_bias[(i + j) % head_dim];
- }
- }
- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));
- }
- for (int i = 0; i < Unroll; ++i) {
- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
- }
- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
+ q->parallel_for(sycl::nd_range<1>(size_t(num_blocks) * wg_size, wg_size),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ int block_id = item.get_group(0);
+ int row_id = block_id / n_seq_blk;
+ int seq_id = block_id % n_seq_blk;
+ int tid = item.get_local_id(0);
+ auto wg = item.get_group();
+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + (size_t)seq_id * block_size * head_dim;
+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + (size_t)seq_id * block_size * head_dim;
+ auto* row_bias = bias_ptr ? bias_ptr + (size_t)row_id * head_dim : nullptr;
+
+ int elems_this_wg = (seq_id + 1) * block_size <= seq ? elems_per_block
+ : (seq - seq_id * block_size) * head_dim;
- if (tid == 0) {
- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
- }
+ float local_max = 0.0f;
+ sycl::vec<T, Unroll> input_vec;
+ sycl::vec<float, Unroll> local_max_vec(0.0f);
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
- sycl::vec<T, Unroll> quant_input = *(sycl::vec<T, Unroll>*)(&block_in[i]);
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
+ input_vec = *(sycl::vec<T, Unroll>*)(&block_in[i]);
+ sycl::vec<float, Unroll> input_f =
+ input_vec.template convert<float, sycl::rounding_mode::automatic>();
+ if (row_bias) {
if (fast_vector_bias) {
int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);
- quant_input = quant_input - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);
+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);
+ input_f = input_f - bias_vec.template convert<float, sycl::rounding_mode::automatic>();
} else {
#pragma unroll
for (int j = 0; j < Unroll; ++j) {
- quant_input[j] = quant_input[j] - row_bias[(i + j) % head_dim];
+ input_f[j] = input_f[j] - static_cast<float>(row_bias[(i + j) % head_dim]);
}
}
- sycl::vec<float, Unroll> val =
- quant_input.template convert<float, sycl::rounding_mode::automatic>();
- val = val * inv_scale;
- val = sycl::round(val);
- val = sycl::clamp(val, -127, 127);
- sycl::vec<int8_t, Unroll> qv =
- val.template convert<int8_t, sycl::rounding_mode::automatic>();
- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
- }
- });
- } else {
- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
- int block_id = item.get_group(0);
- int row_id = block_id / n_seq_blk;
- int seq_id = block_id % n_seq_blk;
- int tid = item.get_local_id(0);
- auto wg = item.get_group();
- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
-
- int elems_this_wg = (seq_id + 1) * block_size <= seq ? elems_per_block
- : (seq - seq_id * block_size) * head_dim;
-
- float local_max = 0.0f;
- sycl::vec<T, Unroll> local_data, local_max_vec;
- local_max_vec = sycl::vec<T, Unroll>(0.0f);
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
- local_data = *(sycl::vec<T, Unroll>*)(&block_in[i]);
- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));
- }
- for (int i = 0; i < Unroll; ++i) {
- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
- }
- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
-
- if (tid == 0) {
- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
- }
-
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
- sycl::vec<T, Unroll> quant_input = *(sycl::vec<T, Unroll>*)(&block_in[i]);
- sycl::vec<float, Unroll> val =
- quant_input.template convert<float, sycl::rounding_mode::automatic>();
- val = val * inv_scale;
- val = sycl::round(val);
- val = sycl::clamp(val, -127, 127);
- sycl::vec<int8_t, Unroll> qv =
- val.template convert<int8_t, sycl::rounding_mode::automatic>();
- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
}
- });
- }
- } else {
- int wg_size = MAX_WG_SIZE;
- if (bias_ptr) {
- if (fast_vector_bias) {
- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
- int seq_id = item.get_group(0);
- int row_id = seq_id / n_seq_blk;
- seq_id = seq_id % n_seq_blk;
- int tid = item.get_local_id(0);
- auto wg = item.get_group();
- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
- auto* row_bias = bias_ptr + (size_t)row_id * head_dim;
-
- float local_max = 0.0f;
- sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;
- local_max_vec = sycl::vec<T, Unroll>(0.0f);
- int local_i = 0;
- int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim
- : (seq - seq_id * block_size) * head_dim;
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
- int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);
- local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];
- local_data[local_i] = local_data[local_i] - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);
- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));
- }
+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(input_f));
+ }
#pragma unroll
- for (int i = 0; i < Unroll; ++i) {
- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
- }
- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
+ for (int i = 0; i < Unroll; ++i) {
+ local_max = sycl::fmax(local_max, local_max_vec[i]);
+ }
+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
- if (tid == 0) {
- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
- }
+ if (tid == 0) {
+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
+ }
- local_i = 0;
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
- sycl::vec<float, Unroll> val =
- local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();
- val = val * inv_scale;
- val = sycl::round(val);
- val = sycl::clamp(val, -127, 127);
- sycl::vec<int8_t, Unroll> qv =
- val.template convert<int8_t, sycl::rounding_mode::automatic>();
- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
- }
- });
- } else {
- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
- int seq_id = item.get_group(0);
- int row_id = seq_id / n_seq_blk;
- seq_id = seq_id % n_seq_blk;
- int tid = item.get_local_id(0);
- auto wg = item.get_group();
- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
- auto* row_bias = bias_ptr + (size_t)row_id * head_dim;
-
- float local_max = 0.0f;
- sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;
- local_max_vec = sycl::vec<T, Unroll>(0.0f);
- int local_i = 0;
- int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim
- : (seq - seq_id * block_size) * head_dim;
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
- local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
+ input_vec = *(sycl::vec<T, Unroll>*)(&block_in[i]);
+ sycl::vec<float, Unroll> val =
+ input_vec.template convert<float, sycl::rounding_mode::automatic>();
+ if (row_bias) {
+ if (fast_vector_bias) {
+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);
+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);
+ val = val - bias_vec.template convert<float, sycl::rounding_mode::automatic>();
+ } else {
#pragma unroll
for (int j = 0; j < Unroll; ++j) {
- local_data[local_i][j] = local_data[local_i][j] - row_bias[(i + j) % head_dim];
+ val[j] = val[j] - static_cast<float>(row_bias[(i + j) % head_dim]);
}
- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));
- }
-#pragma unroll
- for (int i = 0; i < Unroll; ++i) {
- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
- }
- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
-
- if (tid == 0) {
- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
}
-
- local_i = 0;
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
- sycl::vec<float, Unroll> val =
- local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();
- val = val * inv_scale;
- val = sycl::round(val);
- val = sycl::clamp(val, -127, 127);
- sycl::vec<int8_t, Unroll> qv =
- val.template convert<int8_t, sycl::rounding_mode::automatic>();
- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
- }
- });
- }
- } else {
- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
- int seq_id = item.get_group(0);
- int row_id = seq_id / n_seq_blk;
- seq_id = seq_id % n_seq_blk;
- int tid = item.get_local_id(0);
- auto wg = item.get_group();
- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
-
- float local_max = 0.0f;
- sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;
- local_max_vec = sycl::vec<T, Unroll>(0.0f);
- int local_i = 0;
- int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim
- : (seq - seq_id * block_size) * head_dim;
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
- local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];
- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));
}
-#pragma unroll
- for (int i = 0; i < Unroll; ++i) {
- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
- }
- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
-
- if (tid == 0) {
- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
- }
-
- local_i = 0;
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
- sycl::vec<float, Unroll> val =
- local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();
- val = val * inv_scale;
- val = sycl::round(val);
- val = sycl::clamp(val, -127, 127);
- sycl::vec<int8_t, Unroll> qv =
- val.template convert<int8_t, sycl::rounding_mode::automatic>();
- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
- }
- });
- }
+ val = val * inv_scale;
+ val = sycl::round(val);
+ val = sycl::clamp(val, -127.0f, 127.0f);
+ sycl::vec<int8_t, Unroll> qv =
+ val.template convert<int8_t, sycl::rounding_mode::automatic>();
+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
+ }
+ });
}
}
@@ -1075,10 +957,519 @@ class XpuWrapper {
});
}
+ template <typename T>
+ static void sage_dynamic_quant_strided(sycl::queue* q, const T* in_ptr, int8_t* out_ptr, float* scale_ptr,
+ int batch, int num_heads, int seq, int n_seq_blk, int head_dim,
+ int block_size, int stride_seq, int stride_dim, int stride_head,
+ int stride_batch, const T* bias_ptr = nullptr) {
+ int num_blocks = batch * num_heads * n_seq_blk;
+ constexpr int SG_SIZE = 32;
+ constexpr int Unroll = 4;
+ constexpr int MAX_Reg = 64;
+ constexpr int WG_SIZE = 256;
+ constexpr int HEADS_PER_WG = WG_SIZE / SG_SIZE;
+ constexpr int TOKEN_MAJOR_WG_SIZE = WG_SIZE;
+ int elems_per_block = block_size * head_dim;
+ bool use_cache = elems_per_block <= MAX_Reg * WG_SIZE;
+ bool use_vector_path = stride_dim == 1 && (head_dim % Unroll) == 0;
+ bool use_multihead_nhd_q_path = false;
+ bool use_token_major_nhd_q_path =
+ use_cache && bias_ptr == nullptr && stride_dim == 1 && stride_head == head_dim &&
+ stride_seq == num_heads * head_dim &&
+ stride_batch == seq * stride_seq && head_dim <= WG_SIZE * Unroll && (head_dim % Unroll) == 0;
+ // HND-packed fast path: the whole tensor is canonically packed HND, i.e. dims are contiguous
+ // (stride_dim==1), tokens are contiguous within a head (stride_seq==head_dim), heads are
+ // contiguous within a batch (stride_head==seq*head_dim), and batches are contiguous
+ // (stride_batch==num_heads*seq*head_dim). The kernel addresses memory as
+ // in_ptr + row_id*seq*head_dim (row_id = batch_id*num_heads + head_id), which is only valid
+ // under these canonical strides; non-canonical head/batch strides (e.g. head slicing or
+ // padding) would read the wrong data, so they fall through to the generic strided kernel.
+ // This avoids div/mod and per-element stride MADs in the inner loop, matching the perf of
+ // the non-strided sage_dynamic_quant kernel.
+ bool use_packed_hnd_path = stride_dim == 1 && stride_seq == head_dim &&
+ stride_head == seq * head_dim &&
+ stride_batch == num_heads * seq * head_dim && (head_dim % Unroll) == 0;
+ if (use_multihead_nhd_q_path) {
+ int num_head_tiles = (num_heads + HEADS_PER_WG - 1) / HEADS_PER_WG;
+ int total_blocks = batch * num_head_tiles * n_seq_blk;
+ q->parallel_for(sycl::nd_range<1>(size_t(total_blocks) * WG_SIZE, WG_SIZE),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ int block_id = item.get_group(0);
+ int head_tile_id = (block_id / n_seq_blk) % num_head_tiles;
+ int seq_block_id = block_id % n_seq_blk;
+ int batch_id = block_id / (num_head_tiles * n_seq_blk);
+ int tid = item.get_local_id(0);
+ int head_slot = tid / SG_SIZE;
+ int lane_id = tid % SG_SIZE;
+ int head_id = head_tile_id * HEADS_PER_WG + head_slot;
+ int vecs_per_head = head_dim / Unroll;
+ if (head_id >= num_heads || lane_id >= vecs_per_head) {
+ return;
+ }
+
+ int row_id = batch_id * num_heads + head_id;
+ int seq_begin = seq_block_id * block_size;
+ int seq_end = std::min(seq_begin + block_size, seq);
+ int dim = lane_id * Unroll;
+ auto sg = item.get_sub_group();
+
+ float local_max = 0.0f;
+ for (int token = seq_begin; token < seq_end; ++token) {
+ size_t src_offset = size_t(batch_id) * size_t(stride_batch) + size_t(token) * size_t(stride_seq) +
+ size_t(head_id) * size_t(stride_head) + size_t(dim);
+ sycl::vec<T, Unroll> input_vec = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);
+#pragma unroll
+ for (int i = 0; i < Unroll; ++i) {
+ local_max = sycl::fmax(local_max, sycl::fabs(static_cast<float>(input_vec[i])));
+ }
+ }
+
+ float absmax = sycl::reduce_over_group(sg, local_max, sycl::maximum<float>{});
+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;
+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;
+ if (lane_id == 0) {
+ scale_ptr[size_t(row_id) * size_t(n_seq_blk) + size_t(seq_block_id)] = scale;
+ }
+
+ for (int token = seq_begin; token < seq_end; ++token) {
+ size_t src_offset = size_t(batch_id) * size_t(stride_batch) + size_t(token) * size_t(stride_seq) +
+ size_t(head_id) * size_t(stride_head) + size_t(dim);
+ sycl::vec<T, Unroll> input_vec = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);
+ sycl::vec<float, Unroll> val =
+ input_vec.template convert<float, sycl::rounding_mode::automatic>();
+ val = val * inv_scale;
+ val = sycl::round(val);
+ val = sycl::clamp(val, -127, 127);
+ sycl::vec<int8_t, Unroll> qv =
+ val.template convert<int8_t, sycl::rounding_mode::automatic>();
+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) +
+ size_t(dim);
+ *(sycl::vec<int8_t, Unroll>*)(&out_ptr[dst_offset]) = qv;
+ }
+ });
+ return;
+ }
+ if (use_token_major_nhd_q_path) {
+ q->parallel_for(sycl::nd_range<1>(size_t(num_blocks) * TOKEN_MAJOR_WG_SIZE, TOKEN_MAJOR_WG_SIZE),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ int block_id = item.get_group(0);
+ int row_id = block_id / n_seq_blk;
+ int seq_block_id = block_id % n_seq_blk;
+ int batch_id = row_id / num_heads;
+ int head_id = row_id % num_heads;
+ int tid = item.get_local_id(0);
+ auto wg = item.get_group();
+ int seq_begin = seq_block_id * block_size;
+ int seq_end = std::min(seq_begin + block_size, seq);
+ int vecs_per_token = head_dim / Unroll;
+ int tokens_per_iter = TOKEN_MAJOR_WG_SIZE / vecs_per_token;
+ int vec_id = tid % vecs_per_token;
+ int token_slot = tid / vecs_per_token;
+ int dim = vec_id * Unroll;
+ float local_max = 0.0f;
+ sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll];
+ sycl::vec<float, Unroll> local_max_vec(0.0f);
+ int local_i = 0;
+
+ for (int token = seq_begin + token_slot; token < seq_end; token += tokens_per_iter, local_i++) {
+ size_t src_offset = size_t(batch_id) * size_t(stride_batch) + size_t(token) * size_t(stride_seq) +
+ size_t(head_id) * size_t(stride_head) + size_t(dim);
+ local_data[local_i] = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);
+ sycl::vec<float, Unroll> data_f =
+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();
+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(data_f));
+ }
+ #pragma unroll
+ for (int i = 0; i < Unroll; ++i) {
+ local_max = sycl::fmax(local_max, local_max_vec[i]);
+ }
+
+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;
+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;
+ if (tid == 0) {
+ scale_ptr[block_id] = scale;
+ }
+
+ local_i = 0;
+ for (int token = seq_begin + token_slot; token < seq_end; token += tokens_per_iter, local_i++) {
+ sycl::vec<float, Unroll> val =
+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();
+ val = val * inv_scale;
+ val = sycl::round(val);
+ val = sycl::clamp(val, -127, 127);
+ sycl::vec<int8_t, Unroll> qv =
+ val.template convert<int8_t, sycl::rounding_mode::automatic>();
+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) +
+ size_t(dim);
+ *(sycl::vec<int8_t, Unroll>*)(&out_ptr[dst_offset]) = qv;
+ }
+ });
+ return;
+ }
+ if (use_packed_hnd_path) {
+ bool fast_vector_bias = (head_dim % Unroll == 0);
+ bool power_of_two_head_dim = fast_vector_bias && ((head_dim & (head_dim - 1)) == 0);
+ int bias_mask = head_dim - 1;
+ q->parallel_for(
+ sycl::nd_range<1>(size_t(num_blocks) * WG_SIZE, WG_SIZE),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ int block_id = item.get_group(0);
+ int tid = item.get_local_id(0);
+ auto wg = item.get_group();
+ int row_id = block_id / n_seq_blk;
+ int seq_block_id = block_id % n_seq_blk;
+ int seq_begin = seq_block_id * block_size;
+ int seq_end = std::min(seq_begin + block_size, seq);
+ int elems_this_wg = (seq_end - seq_begin) * head_dim;
+
+ const T* block_in = in_ptr + (size_t)row_id * seq * head_dim + (size_t)seq_begin * head_dim;
+ int8_t* block_out = out_ptr + (size_t)row_id * seq * head_dim + (size_t)seq_begin * head_dim;
+ const T* row_bias = bias_ptr ? bias_ptr + (size_t)row_id * head_dim : nullptr;
+
+ float local_max = 0.0f;
+ sycl::vec<T, Unroll> input_vec;
+ sycl::vec<float, Unroll> local_max_vec(0.0f);
+ for (int i = tid * Unroll; i < elems_this_wg; i += WG_SIZE * Unroll) {
+ input_vec = *(sycl::vec<T, Unroll>*)(&block_in[i]);
+ sycl::vec<float, Unroll> input_f =
+ input_vec.template convert<float, sycl::rounding_mode::automatic>();
+ if (row_bias) {
+ if (fast_vector_bias) {
+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);
+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);
+ input_f = input_f - bias_vec.template convert<float, sycl::rounding_mode::automatic>();
+ } else {
+#pragma unroll
+ for (int j = 0; j < Unroll; ++j) {
+ input_f[j] = input_f[j] - static_cast<float>(row_bias[(i + j) % head_dim]);
+ }
+ }
+ }
+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(input_f));
+ }
+#pragma unroll
+ for (int i = 0; i < Unroll; ++i) {
+ local_max = sycl::fmax(local_max, local_max_vec[i]);
+ }
+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
+ if (tid == 0) {
+ scale_ptr[block_id] = absmax / 127.0f;
+ }
+
+ for (int i = tid * Unroll; i < elems_this_wg; i += WG_SIZE * Unroll) {
+ input_vec = *(sycl::vec<T, Unroll>*)(&block_in[i]);
+ sycl::vec<float, Unroll> val = input_vec.template convert<float, sycl::rounding_mode::automatic>();
+ if (row_bias) {
+ if (fast_vector_bias) {
+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);
+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);
+ val = val - bias_vec.template convert<float, sycl::rounding_mode::automatic>();
+ } else {
+#pragma unroll
+ for (int j = 0; j < Unroll; ++j) {
+ val[j] = val[j] - static_cast<float>(row_bias[(i + j) % head_dim]);
+ }
+ }
+ }
+ val = val * inv_scale;
+ val = sycl::round(val);
+ val = sycl::clamp(val, -127.0f, 127.0f);
+ sycl::vec<int8_t, Unroll> qv = val.template convert<int8_t, sycl::rounding_mode::automatic>();
+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
+ }
+ });
+ return;
+ }
+ q->parallel_for(sycl::nd_range<1>(size_t(num_blocks) * WG_SIZE, WG_SIZE),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ int block_id = item.get_group(0);
+ int row_id = block_id / n_seq_blk;
+ int seq_block_id = block_id % n_seq_blk;
+ int batch_id = row_id / num_heads;
+ int head_id = row_id % num_heads;
+ int tid = item.get_local_id(0);
+ auto wg = item.get_group();
+ int seq_begin = seq_block_id * block_size;
+ int seq_end = std::min(seq_begin + block_size, seq);
+ int elems_this_wg = (seq_end - seq_begin) * head_dim;
+ const T* row_bias = bias_ptr ? bias_ptr + size_t(row_id) * size_t(head_dim) : nullptr;
+
+ if (use_vector_path && use_cache) {
+ float local_max = 0.0f;
+ sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll];
+ sycl::vec<float, Unroll> local_max_vec(0.0f);
+ int local_i = 0;
+ for (int linear_idx = tid * Unroll; linear_idx < elems_this_wg;
+ linear_idx += WG_SIZE * Unroll, local_i++) {
+ int token_rel = linear_idx / head_dim;
+ int dim = linear_idx % head_dim;
+ size_t src_offset = logical_hnd_offset(batch_id, head_id, seq_begin + token_rel, dim,
+ stride_seq, stride_dim, stride_head, stride_batch);
+ local_data[local_i] = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);
+ sycl::vec<float, Unroll> data_f =
+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();
+ if (row_bias) {
+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[dim]);
+ data_f = data_f - bias_vec.template convert<float, sycl::rounding_mode::automatic>();
+ }
+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(data_f));
+ }
+#pragma unroll
+ for (int i = 0; i < Unroll; ++i) {
+ local_max = sycl::fmax(local_max, local_max_vec[i]);
+ }
+
+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;
+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;
+ if (tid == 0) {
+ scale_ptr[block_id] = scale;
+ }
+
+ local_i = 0;
+ for (int linear_idx = tid * Unroll; linear_idx < elems_this_wg;
+ linear_idx += WG_SIZE * Unroll, local_i++) {
+ int token_rel = linear_idx / head_dim;
+ int dim = linear_idx % head_dim;
+ sycl::vec<T, Unroll> quant_input = local_data[local_i];
+ sycl::vec<float, Unroll> val =
+ quant_input.template convert<float, sycl::rounding_mode::automatic>();
+ if (row_bias) {
+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[dim]);
+ val = val - bias_vec.template convert<float, sycl::rounding_mode::automatic>();
+ }
+ val = val * inv_scale;
+ val = sycl::round(val);
+ val = sycl::clamp(val, -127, 127);
+ sycl::vec<int8_t, Unroll> qv =
+ val.template convert<int8_t, sycl::rounding_mode::automatic>();
+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(seq_begin + token_rel)) *
+ size_t(head_dim) +
+ size_t(dim);
+ *(sycl::vec<int8_t, Unroll>*)(&out_ptr[dst_offset]) = qv;
+ }
+ } else if (use_vector_path) {
+ // use_cache == false: block too large to cache in registers; recompute
+ // from memory in the second pass instead of storing into local_data.
+ float local_max = 0.0f;
+ sycl::vec<float, Unroll> local_max_vec(0.0f);
+ for (int linear_idx = tid * Unroll; linear_idx < elems_this_wg;
+ linear_idx += WG_SIZE * Unroll) {
+ int token_rel = linear_idx / head_dim;
+ int dim = linear_idx % head_dim;
+ size_t src_offset = logical_hnd_offset(batch_id, head_id, seq_begin + token_rel, dim,
+ stride_seq, stride_dim, stride_head, stride_batch);
+ sycl::vec<T, Unroll> input_vec = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);
+ sycl::vec<float, Unroll> data_f =
+ input_vec.template convert<float, sycl::rounding_mode::automatic>();
+ if (row_bias) {
+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[dim]);
+ data_f = data_f - bias_vec.template convert<float, sycl::rounding_mode::automatic>();
+ }
+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(data_f));
+ }
+#pragma unroll
+ for (int i = 0; i < Unroll; ++i) {
+ local_max = sycl::fmax(local_max, local_max_vec[i]);
+ }
+
+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;
+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;
+ if (tid == 0) {
+ scale_ptr[block_id] = scale;
+ }
+
+ for (int linear_idx = tid * Unroll; linear_idx < elems_this_wg;
+ linear_idx += WG_SIZE * Unroll) {
+ int token_rel = linear_idx / head_dim;
+ int dim = linear_idx % head_dim;
+ size_t src_offset = logical_hnd_offset(batch_id, head_id, seq_begin + token_rel, dim,
+ stride_seq, stride_dim, stride_head, stride_batch);
+ sycl::vec<T, Unroll> quant_input = *(sycl::vec<T, Unroll>*)(&in_ptr[src_offset]);
+ sycl::vec<float, Unroll> val =
+ quant_input.template convert<float, sycl::rounding_mode::automatic>();
+ if (row_bias) {
+ sycl::vec<T, Unroll> bias_vec = *(sycl::vec<T, Unroll>*)(&row_bias[dim]);
+ val = val - bias_vec.template convert<float, sycl::rounding_mode::automatic>();
+ }
+ val = val * inv_scale;
+ val = sycl::round(val);
+ val = sycl::clamp(val, -127, 127);
+ sycl::vec<int8_t, Unroll> qv =
+ val.template convert<int8_t, sycl::rounding_mode::automatic>();
+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(seq_begin + token_rel)) *
+ size_t(head_dim) +
+ size_t(dim);
+ *(sycl::vec<int8_t, Unroll>*)(&out_ptr[dst_offset]) = qv;
+ }
+ } else {
+ float local_max = 0.0f;
+ T local_data[MAX_Reg];
+ int local_count = 0;
+ for (int linear_idx = tid; linear_idx < elems_this_wg; linear_idx += WG_SIZE) {
+ int token_rel = linear_idx / head_dim;
+ int dim = linear_idx % head_dim;
+ size_t src_offset = logical_hnd_offset(batch_id, head_id, seq_begin + token_rel, dim,
+ stride_seq, stride_dim, stride_head, stride_batch);
+ float value = static_cast<float>(in_ptr[src_offset]);
+ if (row_bias) {
+ value -= static_cast<float>(row_bias[dim]);
+ }
+ if (use_cache) {
+ local_data[local_count++] = static_cast<T>(value);
+ }
+ local_max = sycl::fmax(local_max, sycl::fabs(value));
+ }
+
+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;
+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;
+ if (tid == 0) {
+ scale_ptr[block_id] = scale;
+ }
+
+ local_count = 0;
+ for (int linear_idx = tid; linear_idx < elems_this_wg; linear_idx += WG_SIZE) {
+ int token_rel = linear_idx / head_dim;
+ int dim = linear_idx % head_dim;
+ float value = 0.0f;
+ if (use_cache) {
+ value = static_cast<float>(local_data[local_count++]);
+ } else {
+ size_t src_offset = logical_hnd_offset(batch_id, head_id, seq_begin + token_rel, dim,
+ stride_seq, stride_dim, stride_head,
+ stride_batch);
+ value = static_cast<float>(in_ptr[src_offset]);
+ if (row_bias) {
+ value -= static_cast<float>(row_bias[dim]);
+ }
+ }
+ value *= inv_scale;
+ value = sycl::round(value);
+ value = sycl::clamp(value, -127.0f, 127.0f);
+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(seq_begin + token_rel)) *
+ size_t(head_dim) +
+ size_t(dim);
+ out_ptr[dst_offset] = static_cast<int8_t>(value);
+ }
+ }
+ });
+ }
+
+ template <typename T>
+ static void sage_dynamic_quant_v_strided(sycl::queue* q, const T* in_ptr, int8_t* out_ptr, float* scale_ptr,
+ int batch, int num_heads, int seq, int n_seq_blk, int head_dim,
+ int block_size, int stride_dim, int stride_seq, int stride_head,
+ int stride_batch) {
+ size_t num_scales = size_t(batch) * size_t(num_heads) * size_t(n_seq_blk) * size_t(head_dim);
+ constexpr int SG_SIZE = 32;
+ constexpr int WG_SIZE = 512;
+ constexpr int SG_PER_WG = WG_SIZE / SG_SIZE;
+ if (block_size <= SG_SIZE * 2) {
+ q->parallel_for(sycl::range<1>(num_scales), [=](sycl::id<1> item) {
+ size_t linear_idx = item[0];
+ int dim = int(linear_idx % size_t(head_dim));
+ size_t block_idx = linear_idx / size_t(head_dim);
+ int seq_id = int(block_idx % size_t(n_seq_blk));
+ int row_id = int(block_idx / size_t(n_seq_blk));
+ int batch_id = row_id / num_heads;
+ int head_id = row_id % num_heads;
+ int seq_begin = seq_id * block_size;
+ int seq_end = std::min(seq_begin + block_size, seq);
+
+ float absmax = 0.0f;
+ for (int token = seq_begin; token < seq_end; ++token) {
+ size_t src_offset =
+ logical_hnd_offset(batch_id, head_id, token, dim, stride_seq, stride_dim, stride_head, stride_batch);
+ float value = static_cast<float>(in_ptr[src_offset]);
+ absmax = sycl::fmax(absmax, sycl::fabs(value));
+ }
+
+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;
+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;
+ scale_ptr[linear_idx] = scale;
+
+ for (int token = seq_begin; token < seq_end; ++token) {
+ size_t src_offset =
+ logical_hnd_offset(batch_id, head_id, token, dim, stride_seq, stride_dim, stride_head, stride_batch);
+ float value = static_cast<float>(in_ptr[src_offset]) * inv_scale;
+ value = sycl::round(value);
+ value = sycl::clamp(value, -127.0f, 127.0f);
+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) + size_t(dim);
+ out_ptr[dst_offset] = static_cast<int8_t>(value);
+ }
+ });
+ return;
+ }
+
+ size_t num_groups = (num_scales + SG_PER_WG - 1) / SG_PER_WG;
+ size_t global = num_groups * WG_SIZE;
+ q->parallel_for(sycl::nd_range<1>(global, WG_SIZE),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ size_t group_id = item.get_group(0);
+ int local_id = item.get_local_id(0);
+ int subgroup_id = local_id / SG_SIZE;
+ int lane_id = local_id % SG_SIZE;
+ size_t linear_idx = group_id * SG_PER_WG + subgroup_id;
+ if (linear_idx >= num_scales) {
+ return;
+ }
+
+ int dim = int(linear_idx % size_t(head_dim));
+ size_t block_idx = linear_idx / size_t(head_dim);
+ int seq_id = int(block_idx % size_t(n_seq_blk));
+ int row_id = int(block_idx / size_t(n_seq_blk));
+ int batch_id = row_id / num_heads;
+ int head_id = row_id % num_heads;
+ int seq_begin = seq_id * block_size;
+ int seq_end = std::min(seq_begin + block_size, seq);
+ auto sg = item.get_sub_group();
+
+ float local_max = 0.0f;
+ for (int token = seq_begin + lane_id; token < seq_end; token += SG_SIZE) {
+ size_t src_offset = logical_hnd_offset(batch_id, head_id, token, dim, stride_seq, stride_dim,
+ stride_head, stride_batch);
+ float value = static_cast<float>(in_ptr[src_offset]);
+ local_max = sycl::fmax(local_max, sycl::fabs(value));
+ }
+
+ float absmax = sycl::reduce_over_group(sg, local_max, sycl::maximum<float>{});
+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;
+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;
+ if (lane_id == 0) {
+ scale_ptr[linear_idx] = scale;
+ }
+
+ for (int token = seq_begin + lane_id; token < seq_end; token += SG_SIZE) {
+ size_t src_offset = logical_hnd_offset(batch_id, head_id, token, dim, stride_seq, stride_dim,
+ stride_head, stride_batch);
+ float value = static_cast<float>(in_ptr[src_offset]) * inv_scale;
+ value = sycl::round(value);
+ value = sycl::clamp(value, -127.0f, 127.0f);
+ size_t dst_offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) +
+ size_t(dim);
+ out_ptr[dst_offset] = static_cast<int8_t>(value);
+ }
+ });
+ }
+
+ template <typename T = sycl::half>
static void sagev1_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
- int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, bool use_int8_pv) {
+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,
+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
+ int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
+ int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
+ bool use_int8_pv) {
bool use_mean_bias = env_params::Instance()->sage_use_mean_bias != 0;
+ bool q_packed = is_packed_hnd(q_stride_s, q_stride_d, q_stride_h, q_stride_b, num_heads_q, seq_len_q, head_dim);
+ bool k_packed =
+ is_packed_hnd(k_stride_s, k_stride_d, k_stride_h, k_stride_b, num_heads_kv, seq_len_kv, head_dim);
+ bool v_packed =
+ is_packed_hnd(v_stride_s, v_stride_d, v_stride_h, v_stride_b, num_heads_kv, seq_len_kv, head_dim);
size_t seq_q_blk = (seq_len_q + scale_block_size - 1) / scale_block_size;
size_t seq_kv_blk = (seq_len_kv + scale_block_size - 1) / scale_block_size;
size_t q_size = num_heads_q * seq_len_q * head_dim * batch;
@@ -1087,7 +1478,7 @@ class XpuWrapper {
size_t k_scale_size = num_heads_kv * seq_kv_blk * batch;
size_t v_tmp_size = use_int8_pv ? k_size : 0;
size_t v_scale_size = use_int8_pv ? k_scale_size * head_dim * sizeof(float) : 0;
- size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * batch * sizeof(sycl::half) : 0;
+ size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * batch * sizeof(T) : 0;
size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size +
v_scale_size + k_bias_size;
auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);
@@ -1102,44 +1493,115 @@ class XpuWrapper {
k_scale_size * sizeof(float) + k_size)
: nullptr;
auto kbias = use_mean_bias
- ? (sycl::half*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) +
- k_scale_size * sizeof(float) + v_tmp_size + v_scale_size)
+ ? (T*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) +
+ k_scale_size * sizeof(float) + v_tmp_size + v_scale_size)
: nullptr;
- sage_dynamic_quant<sycl::half>(q, (sycl::half*)Q_ptr, (int8_t*)q_out_ptr, (float*)qscale, batch * num_heads_q,
- seq_len_q, seq_q_blk, head_dim, scale_block_size);
+ if (q_packed) {
+ sage_dynamic_quant<T>(q, (T*)Q_ptr, (int8_t*)q_out_ptr, (float*)qscale, batch * num_heads_q,
+ seq_len_q, seq_q_blk, head_dim, scale_block_size);
+ } else {
+ sage_dynamic_quant_strided<T>(q, (T*)Q_ptr, (int8_t*)q_out_ptr, (float*)qscale, batch,
+ num_heads_q, seq_len_q, seq_q_blk, head_dim, scale_block_size,
+ q_stride_s, q_stride_d, q_stride_h, q_stride_b);
+ }
if (use_mean_bias) {
- compute_seq_mean_bias<sycl::half>(q, (sycl::half*)K_ptr, kbias, batch * num_heads_kv, seq_len_kv, head_dim);
+ if (k_packed) {
+ compute_seq_mean_bias<T>(q, (T*)K_ptr, kbias, batch * num_heads_kv, seq_len_kv, head_dim);
+ } else {
+ compute_seq_mean_bias_strided<T>(q, (T*)K_ptr, kbias, batch, num_heads_kv, seq_len_kv,
+ head_dim, k_stride_s, k_stride_d, k_stride_h, k_stride_b);
+ }
if (env_params::Instance()->sage_print_kbias != 0) {
print_value_distribution(q, kbias, size_t(batch) * num_heads_kv * head_dim, "kbias");
}
}
- sage_dynamic_quant<sycl::half>(q, (sycl::half*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch * num_heads_kv,
- seq_len_kv, seq_kv_blk, head_dim, scale_block_size, kbias);
+ if (k_packed) {
+ sage_dynamic_quant<T>(q, (T*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch * num_heads_kv,
+ seq_len_kv, seq_kv_blk, head_dim, scale_block_size, kbias);
+ } else {
+ sage_dynamic_quant_strided<T>(q, (T*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch,
+ num_heads_kv, seq_len_kv, seq_kv_blk, head_dim, scale_block_size,
+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, kbias);
+ }
+ int q_out_stride_s = head_dim;
+ int q_out_stride_d = 1;
+ int q_out_stride_h = seq_len_q * head_dim;
+ int q_out_stride_b = num_heads_q * seq_len_q * head_dim;
+ int k_out_stride_s = head_dim;
+ int k_out_stride_d = 1;
+ int k_out_stride_h = seq_len_kv * head_dim;
+ int k_out_stride_b = num_heads_kv * seq_len_kv * head_dim;
+ int v_out_stride_d = 1;
+ int v_out_stride_s = head_dim;
+ int v_out_stride_h = seq_len_kv * head_dim;
+ int v_out_stride_b = num_heads_kv * seq_len_kv * head_dim;
+ constexpr BTLA_DTYPE pv_dtype = std::is_same<T, sycl::half>::value ? BTLA_DTYPE::F16 : BTLA_DTYPE::BF16;
if (use_int8_pv) {
- sage_dynamic_quant_v<sycl::half>(q, (sycl::half*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale,
- batch * num_heads_kv, seq_len_kv, seq_kv_blk, head_dim, scale_block_size);
+ if (v_packed) {
+ sage_dynamic_quant_v<T>(q, (T*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale,
+ batch * num_heads_kv, seq_len_kv, seq_kv_blk, head_dim, scale_block_size);
+ } else {
+ sage_dynamic_quant_v_strided<T>(q, (T*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale, batch,
+ num_heads_kv, seq_len_kv, seq_kv_blk, head_dim, scale_block_size,
+ v_stride_d, v_stride_s, v_stride_h, v_stride_b);
+ }
ark::sdpa_impl_qks8_pvi8(q, q_out_ptr, k_out_ptr, v_out_ptr, O_ptr, mask, scale_block_size, qscale, kscale,
- vscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
- softmax_scale, is_causal);
+ vscale, q_out_stride_s, q_out_stride_d, q_out_stride_h, q_out_stride_b,
+ k_out_stride_s, k_out_stride_d, k_out_stride_h, k_out_stride_b, v_out_stride_d,
+ v_out_stride_s, v_out_stride_h, v_out_stride_b, o_stride_s, o_stride_d, o_stride_h,
+ o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
+ softmax_scale, is_causal, pv_dtype);
} else {
ark::sdpa_impl_qks8_pvhalf(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale,
- batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
- is_causal);
+ q_out_stride_s, q_out_stride_d, q_out_stride_h, q_out_stride_b, k_out_stride_s,
+ k_out_stride_d, k_out_stride_h, k_out_stride_b, v_stride_d, v_stride_s, v_stride_h,
+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,
+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, pv_dtype);
}
}
static void sagev1(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
- int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,
- int head_dim, float softmax_scale, bool is_causal) {
- sagev1_impl(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q,
- seq_len_kv, head_dim, softmax_scale, is_causal, false);
+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,
+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
+ int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
+ BTLA_DTYPE dtype = BTLA_DTYPE::F16) {
+ if (dtype == BTLA_DTYPE::BF16) {
+ sagev1_impl<sycl::ext::oneapi::bfloat16>(
+ q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,
+ o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,
+ head_dim, softmax_scale, is_causal, false);
+ } else {
+ sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,
+ q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,
+ v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,
+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
+ is_causal, false);
+ }
}
static void sagev1_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
- int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
- sagev1_impl(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q,
- seq_len_kv, head_dim, softmax_scale, is_causal, true);
+ int scale_block_size, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,
+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
+ int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
+ int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
+ BTLA_DTYPE dtype = BTLA_DTYPE::F16) {
+ if (dtype == BTLA_DTYPE::BF16) {
+ sagev1_impl<sycl::ext::oneapi::bfloat16>(
+ q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,
+ o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,
+ head_dim, softmax_scale, is_causal, true);
+ } else {
+ sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,
+ q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,
+ v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,
+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
+ is_causal, true);
+ }
}
};
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp
index ac7d75eb7..b73137715 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp
@@ -20,9 +20,11 @@ struct TestSDPA {
#if defined(ARK_XPU) && defined(ARK_SYCL_TLA) && ARK_XPU && ARK_SYCL_TLA
// test_sagev1_accuracy("prefill_gqa_128_bias", 1, 32, 8, 256, 512, 128, 128, false, true);
// test_sagev1_accuracy("prefill_gqa_128_no_bias", 1, 32, 8, 256, 512, 128, 128, false, false);
- test_sagev1_accuracy("prefill_gqa_128_causal_no_bias", 1, 32, 8, 256, 512, 128, 128, true, false);
- benchmark_sagev1("bench_prefill_gqa_128_s4096", 1, 32, 8, 4096, 4096, 128, 128, false, 5, 20);
- benchmark_sagev1("bench_bmg_hq96_d128_k8192", 1, 96, 8, 4096, 8192, 128, 128, false, 5, 20);
+ // test_sagev1_accuracy(\"prefill_gqa_128_causal_no_bias\", 1, 32, 8, 256, 512, 128, 128, true, false);
+ benchmark_dynamic_quant("dq_b1_h32_s4096_d128", 1, 32, 4096, 128, 128, 5, 50);
+ benchmark_dynamic_quant("dq_b1_h96_s8192_d128", 1, 96, 8192, 128, 128, 5, 50);
+ // benchmark_sagev1("bench_prefill_gqa_128_s4096", 1, 32, 8, 4096, 4096, 128, 128, false, 5, 20);
+ // benchmark_sagev1("bench_bmg_hq96_d128_k8192", 1, 96, 8, 4096, 8192, 128, 128, false, 5, 20);
#else
std::cout << "[sagev1] skipped: requires ARK_XPU and ARK_SYCL_TLA\n";
#endif
@@ -79,6 +81,7 @@ struct TestSDPA {
return std::chrono::duration<double, std::milli>(end - start).count() / double(iters);
}
+#if 0 // legacy benchmarks rely on outdated sdpa_impl_qks8_* signatures
void test_sagev1_accuracy(const std::string& name, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
int seq_len_kv, int head_dim, int scale_block_size, bool is_causal,
bool use_mean_bias) {
@@ -283,5 +286,55 @@ struct TestSDPA {
ctx->deallocate(dev_kscale);
ctx->deallocate(dev_vscale);
}
+#endif // legacy benchmarks
+
+ template <typename T>
+ void run_dynamic_quant_bench(const char* tag, sycl::queue* q, int num_rows, int seq, int n_seq_blk,
+ int head_dim, int block_size, int warmup, int iters) {
+ auto ctx = Context::Instance();
+ size_t in_count = size_t(num_rows) * size_t(seq) * size_t(head_dim);
+ size_t scale_count = size_t(num_rows) * size_t(n_seq_blk);
+ auto* dev_in = reinterpret_cast<T*>(ctx->allocate(in_count * sizeof(T)));
+ auto* dev_out = reinterpret_cast<int8_t*>(ctx->allocate(in_count * sizeof(int8_t)));
+ auto* dev_scale = reinterpret_cast<float*>(ctx->allocate(scale_count * sizeof(float)));
+
+ // initialize input with random fp values cast to T
+ auto host_f = make_random_vector(in_count, -1.0f, 1.0f, 1234u + uint32_t(num_rows + seq));
+ std::vector<T> host_t(in_count);
+ for (size_t i = 0; i < in_count; ++i) host_t[i] = T(host_f[i]);
+ q->memcpy(dev_in, host_t.data(), in_count * sizeof(T)).wait();
+
+ double ms = run_bench(
+ [&]() {
+ ark::XpuWrapper::sage_dynamic_quant<T>(q, dev_in, dev_out, dev_scale, num_rows, seq, n_seq_blk,
+ head_dim, block_size);
+ },
+ q, warmup, iters);
+
+ double bytes = double(in_count) * double(sizeof(T)) + double(in_count) * 1.0 +
+ double(scale_count) * double(sizeof(float));
+ double gbps = bytes / (ms * 1e-3) / 1e9;
+ std::cout << std::fixed << std::setprecision(4) << "[dq][" << tag << "] ms=" << ms
+ << " bytes=" << bytes / 1e6 << "MB GBps=" << gbps << "\n";
+
+ ctx->deallocate(dev_in);
+ ctx->deallocate(dev_out);
+ ctx->deallocate(dev_scale);
+ }
+
+ void benchmark_dynamic_quant(const std::string& name, int batch, int num_heads, int seq,
+ int head_dim, int block_size, int warmup, int iters) {
+ GETQ();
+ LOG_LINE();
+ int num_rows = batch * num_heads;
+ int n_seq_blk = (seq + block_size - 1) / block_size;
+ std::cout << "[dq][cfg] " << name << " rows=" << num_rows << " seq=" << seq
+ << " head_dim=" << head_dim << " block=" << block_size
+ << " n_seq_blk=" << n_seq_blk << "\n";
+ run_dynamic_quant_bench<sycl::half>("f16", q, num_rows, seq, n_seq_blk, head_dim, block_size, warmup,
+ iters);
+ run_dynamic_quant_bench<sycl::ext::oneapi::bfloat16>("bf16", q, num_rows, seq, n_seq_blk, head_dim,
+ block_size, warmup, iters);
+ }
#endif
};
diff --git a/auto_round_extension/ark/test/test_bench_bmg.py b/auto_round_extension/ark/test/test_bench_bmg.py
index e4bddd382..d0762d830 100644
--- a/auto_round_extension/ark/test/test_bench_bmg.py
+++ b/auto_round_extension/ark/test/test_bench_bmg.py
@@ -23,7 +23,7 @@ ark = None
def get_ark():
global ark
if ark is None:
- ark = auto_round_kernel.ARK()
+ ark = auto_round_kernel
return ark
diff --git a/auto_round_extension/ark/test/test_flash_attn.py b/auto_round_extension/ark/test/test_flash_attn.py
index ade89ee22..27aad5426 100644
--- a/auto_round_extension/ark/test/test_flash_attn.py
+++ b/auto_round_extension/ark/test/test_flash_attn.py
@@ -27,7 +27,7 @@ import auto_round_kernel
import pytest
import torch
-ark = auto_round_kernel.ARK()
+ark = auto_round_kernel
def is_xpu_available():
diff --git a/auto_round_extension/ark/test/test_matmul.py b/auto_round_extension/ark/test/test_matmul.py
index 6debd7685..6b26aa337 100644
--- a/auto_round_extension/ark/test/test_matmul.py
+++ b/auto_round_extension/ark/test/test_matmul.py
@@ -22,7 +22,7 @@ import auto_round_kernel
import pytest
import torch
-ark = auto_round_kernel.ARK()
+ark = auto_round_kernel
def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device):
diff --git a/auto_round_extension/ark/test/test_moe.py b/auto_round_extension/ark/test/test_moe.py
index 1134f7b4d..360b603b6 100644
--- a/auto_round_extension/ark/test/test_moe.py
+++ b/auto_round_extension/ark/test/test_moe.py
@@ -25,7 +25,7 @@ import auto_round_kernel
import pytest
import torch
-ark = auto_round_kernel.ARK()
+ark = auto_round_kernel
def is_xpu_available():
diff --git a/auto_round_extension/ark/test/test_packq.py b/auto_round_extension/ark/test/test_packq.py
index c77d6449c..d8c4aa0d6 100644
--- a/auto_round_extension/ark/test/test_packq.py
+++ b/auto_round_extension/ark/test/test_packq.py
@@ -22,7 +22,7 @@ import pytest
import torch
from ut_utils import compare2, decode_fp8_to_float, gen_weis8, get_scale_torch_dt, get_torch_dt, sample_valid_fp8
-ark = auto_round_kernel.ARK()
+ark = auto_round_kernel
def main_op(k, n, weight_type, scale_type, asym, blocksize, device, compute_type):
diff --git a/auto_round_extension/ark/test/test_sage_dynquant.py b/auto_round_extension/ark/test/test_sage_dynquant.py
index 72beb2fb4..5a2c3c4ac 100644
--- a/auto_round_extension/ark/test/test_sage_dynquant.py
+++ b/auto_round_extension/ark/test/test_sage_dynquant.py
@@ -19,7 +19,7 @@ ark = None
def get_ark():
global ark
if ark is None:
- ark = auto_round_kernel.ARK()
+ ark = auto_round_kernel
return ark
diff --git a/auto_round_extension/ark/test/test_sdpa.py b/auto_round_extension/ark/test/test_sdpa.py
index 4aaabdc14..2e5091035 100644
--- a/auto_round_extension/ark/test/test_sdpa.py
+++ b/auto_round_extension/ark/test/test_sdpa.py
@@ -251,8 +251,8 @@ def get_ark():
global ark
if ark is None:
if not is_xpu_available():
- raise RuntimeError("XPU is not available; cannot initialize auto_round_kernel.ARK()")
- ark = auto_round_kernel.ARK()
+ raise RuntimeError("XPU is not available; cannot initialize auto_round_kernel")
+ ark = auto_round_kernel
return ark
@@ -618,44 +618,67 @@ def compare_sdpa_sage_scale(batch, seq, seq_kv, h_q, h_kv, H, H_v, dt=torch.floa
def compare_sdpa_sage_dynquant(
- batch, seq, seq_kv, h_q, h_kv, H, H_v, dt=torch.float16, is_causal=False, dev="xpu", quant_block_size=64
+ batch,
+ seq,
+ seq_kv,
+ h_q,
+ h_kv,
+ H,
+ H_v,
+ dt=torch.float16,
+ is_causal=False,
+ dev="xpu",
+ quant_block_size=64,
+ tensor_layout="HND",
):
"""Test dynamic quantization SAGE attention.
- Q, K, V are FP16 inputs. The kernel dynamically quantizes Q, K to INT8
+ Q, K, V are FP16/BF16 inputs. The kernel dynamically quantizes Q, K to INT8.
+ tensor_layout: 'HND' = [B, H, N, D]; 'NHD' = [B, N, H, D].
"""
group = h_q // h_kv
- q = torch.randn(batch, h_q, seq, H, dtype=dt, device=dev)
- k = torch.randn(batch, h_kv, seq_kv, H, dtype=dt, device=dev)
- v = torch.randn(batch, h_kv, seq_kv, H_v, dtype=dt, device=dev) * 100
+ # Always build reference in HND for SDPA, then permute to requested layout.
+ q_hnd = torch.randn(batch, h_q, seq, H, dtype=dt, device=dev)
+ k_hnd = torch.randn(batch, h_kv, seq_kv, H, dtype=dt, device=dev)
+ v_hnd = torch.randn(batch, h_kv, seq_kv, H_v, dtype=dt, device=dev) * 100
+ if tensor_layout == "NHD":
+ q = q_hnd.transpose(1, 2).contiguous()
+ k = k_hnd.transpose(1, 2).contiguous()
+ v = v_hnd.transpose(1, 2).contiguous()
+ else:
+ q, k, v = q_hnd, k_hnd, v_hnd
scale = 1 / math.sqrt(H)
if seq_kv > 4096 and is_causal:
if seq_kv == seq:
ref = torch.nn.functional.scaled_dot_product_attention(
- q, k, v, scale=scale, enable_gqa=group > 1, is_causal=is_causal
+ q_hnd, k_hnd, v_hnd, scale=scale, enable_gqa=group > 1, is_causal=is_causal
)
else:
- ref = torch.zeros_like(q)
+ ref = torch.zeros_like(q_hnd)
else:
- attn_bias = torch.zeros(seq, seq_kv, dtype=q.dtype, device=q.device)
- temp_mask = torch.ones(seq, seq_kv, dtype=torch.bool, device=q.device).tril(diagonal=seq_kv - seq)
+ attn_bias = torch.zeros(seq, seq_kv, dtype=q_hnd.dtype, device=q_hnd.device)
+ temp_mask = torch.ones(seq, seq_kv, dtype=torch.bool, device=q_hnd.device).tril(diagonal=seq_kv - seq)
attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf"))
scale = 1 / math.sqrt(H)
if is_causal:
ref = torch.nn.functional.scaled_dot_product_attention(
- q, k, v, scale=scale, enable_gqa=group > 1, is_causal=False, attn_mask=attn_bias
+ q_hnd, k_hnd, v_hnd, scale=scale, enable_gqa=group > 1, is_causal=False, attn_mask=attn_bias
)
else:
ref = torch.nn.functional.scaled_dot_product_attention(
- q, k, v, scale=scale, enable_gqa=group > 1, is_causal=is_causal
+ q_hnd, k_hnd, v_hnd, scale=scale, enable_gqa=group > 1, is_causal=is_causal
)
+ if tensor_layout == "NHD":
+ ref = ref.transpose(1, 2).contiguous()
# Dynamic quantization SAGE kernel
- out = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size)
+ out = get_ark().sagev1(
+ q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size, tensor_layout=tensor_layout
+ )
dff = abs(ref - out)
- print(f"=== Dynamic Quantization SAGE Test (block_size={quant_block_size}) ===")
+ print(f"=== Dynamic Quantization SAGE Test (block_size={quant_block_size}, layout={tensor_layout}, dt={dt}) ===")
print(
f"Batch:{batch} Seq_Q:{seq}, Seq_KV:{seq_kv}, HeadNum_Q:{h_q}, HeadNum_KV:{h_kv}, HeadDim_QK:{H}, HeadDim_V:{H_v}, Causal:{is_causal}"
)
@@ -664,12 +687,16 @@ def compare_sdpa_sage_dynquant(
# Benchmark
n_runs = 10 if seq > 8192 else 100
for i in range(n_runs):
- _ = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size)
+ _ = get_ark().sagev1(
+ q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size, tensor_layout=tensor_layout
+ )
if dev == "xpu":
torch.xpu.synchronize()
st = time.time()
for i in range(n_runs):
- _ = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size)
+ _ = get_ark().sagev1(
+ q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size, tensor_layout=tensor_layout
+ )
if dev == "xpu":
torch.xpu.synchronize()
et = time.time()
@@ -684,7 +711,51 @@ def compare_sdpa_sage_dynquant(
MEM += h_kv * seq_kv * H_v * dt.itemsize # v
MEM *= batch
print(f"Time:{dur*1e3:.3f} FLOPS:{OPS/dur/1e9:.2f} G, MEM:{MEM/dur/1e9:.2f} GB/s")
- return out, ref
+ return {
+ "layout": tensor_layout,
+ "dtype": "fp16" if dt == torch.float16 else ("bf16" if dt == torch.bfloat16 else str(dt)),
+ "causal": is_causal,
+ "time_ms": dur * 1e3,
+ "tflops": OPS / dur / 1e12,
+ "max_diff": float(dff.max().item()),
+ "mean_diff": float(dff.mean().item()),
+ }
+
+
+def run_dynquant_layout_dtype_table(
+ batch=1, seq=8192, seq_kv=8192, h_q=96, h_kv=8, H=128, H_v=128, quant_block_size=64, dev="xpu"
+):
+ """Sweep {layout} x {dtype} x {causal} for sage_dynquant and print a markdown table."""
+ rows = []
+ for layout in ("HND", "NHD"):
+ for dt in (torch.float16, torch.bfloat16):
+ for causal in (False, True):
+ m = compare_sdpa_sage_dynquant(
+ batch,
+ seq,
+ seq_kv,
+ h_q,
+ h_kv,
+ H,
+ H_v,
+ dt=dt,
+ is_causal=causal,
+ dev=dev,
+ quant_block_size=quant_block_size,
+ tensor_layout=layout,
+ )
+ rows.append(m)
+
+ print()
+ print(f"=== sage_dynquant sweep: B={batch} Hq={h_q} Hkv={h_kv} S={seq} D={H} blk={quant_block_size} ===")
+ print("| layout | dtype | causal | time (ms) | TFLOPS | max diff | mean diff |")
+ print("|---|---|---|---|---|---|---|")
+ for r in rows:
+ print(
+ f"| {r['layout']} | {r['dtype']} | {'T' if r['causal'] else 'F'} | "
+ f"{r['time_ms']:.3f} | {r['tflops']:.2f} | {r['max_diff']:.4g} | {r['mean_diff']:.4g} |"
+ )
+ return rows
if __name__ == "__main__":
@@ -732,8 +803,7 @@ if __name__ == "__main__":
# bench_ark(1, 4096, 8192*1, 96, 8, 64, 64, dt=torch.float16, is_causal=True)
# compare_sdpa_sage_scale(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16)
# compare_sdpa_sage_scale(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16, is_causal=True)
- compare_sdpa_sage_dynquant(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16)
- compare_sdpa_sage_dynquant(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16, is_causal=True)
+ run_dynquant_layout_dtype_table(batch=1, seq=8192, seq_kv=8192, h_q=96, h_kv=8, H=128, H_v=128, quant_block_size=64)
# compare_sdpa_sage_scale(2, 4096, 4096, 96, 8, 128, 128, dt=torch.float16)
# compare_sdpa_sage_scale(1, 4096, 4096, 96, 8, 128, 128, dt=torch.float16, is_causal=True)
# compare_sdpa_sage(16, 512, 512, 32, 32, 128, 128, dt=torch.float16)
diff --git a/auto_round_extension/ark/test/test_sdpa_parity.py b/auto_round_extension/ark/test/test_sdpa_parity.py
index 77efc6f79..3b261e176 100644
--- a/auto_round_extension/ark/test/test_sdpa_parity.py
+++ b/auto_round_extension/ark/test/test_sdpa_parity.py
@@ -2,17 +2,76 @@
# SPDX-License-Identifier: Apache-2.0
import math
+import sys
+from pathlib import Path
-import auto_round_kernel
import pytest
import torch
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import auto_round_kernel
+
pytestmark = pytest.mark.skipif(
not (hasattr(torch, "xpu") and torch.xpu.is_available()),
reason="XPU not available",
)
+def _make_noncontiguous_attention_inputs(layout: str):
+ torch.manual_seed(3030)
+ batch, heads, seq, head_dim = 1, 2, 64, 64
+ q_hnd = torch.randn(batch, heads, seq * 2, head_dim, device="xpu", dtype=torch.float16)[:, :, ::2, :]
+ k_hnd = torch.randn(batch, heads, seq * 2, head_dim, device="xpu", dtype=torch.float16)[:, :, ::2, :]
+ v_hnd = torch.randn(batch, heads, seq * 2, head_dim, device="xpu", dtype=torch.float16)[:, :, ::2, :]
+
+ if layout == "HND":
+ return q_hnd, k_hnd, v_hnd
+ if layout == "NHD":
+ return q_hnd.transpose(1, 2), k_hnd.transpose(1, 2), v_hnd.transpose(1, 2)
+ raise ValueError(f"Unsupported layout: {layout}")
+
+
+ ("api_name", "atol", "rtol"),
+ [("sdpa", 1e-2, 1e-2), ("sagev1", 2e-2, 2e-2)],
+)
[email protected]("layout", ["HND", "NHD"])
+def test_ark_attention_supports_noncontiguous_inputs_for_layouts(api_name, atol, rtol, layout):
+ q, k, v = _make_noncontiguous_attention_inputs(layout)
+ scale = 1 / math.sqrt(q.shape[-1])
+
+ expected = torch.nn.functional.scaled_dot_product_attention(
+ q if layout == "HND" else q.transpose(1, 2),
+ k if layout == "HND" else k.transpose(1, 2),
+ v if layout == "HND" else v.transpose(1, 2),
+ scale=scale,
+ is_causal=False,
+ )
+ if layout == "NHD":
+ expected = expected.transpose(1, 2)
+
+ ark = auto_round_kernel
+ if api_name == "sdpa":
+ actual = ark.sdpa(q, k, v, scale=scale, is_causal=False, tensor_layout=layout)
+ else:
+ actual = ark.sagev1(
+ q,
+ k,
+ v,
+ scale=scale,
+ is_causal=False,
+ quant_block_size=64,
+ tensor_layout=layout,
+ )
+ torch.xpu.synchronize()
+
+ assert not q.is_contiguous()
+ assert not k.is_contiguous()
+ assert not v.is_contiguous()
+ torch.testing.assert_close(actual, expected, atol=atol, rtol=rtol)
+
+
@pytest.mark.parametrize("seq_q,seq_kv", [(64, 64), (80, 64), (64, 80), (226, 226)])
def test_ark_sdpa_matches_torch_for_kv_remainders(seq_q, seq_kv):
torch.manual_seed(2026)
@@ -29,7 +88,7 @@ def test_ark_sdpa_matches_torch_for_kv_remainders(seq_q, seq_kv):
scale=scale,
is_causal=False,
)
- actual = auto_round_kernel.ARK().sdpa(q, k, v, scale=scale, is_causal=False)
+ actual = auto_round_kernel.sdpa(q, k, v, scale=scale, is_causal=False)
torch.xpu.synchronize()
torch.testing.assert_close(actual, expected, atol=1e-2, rtol=1e-2)
@@ -57,7 +116,7 @@ def test_ark_sagev1_matches_torch_for_kv_remainder_tile():
scale=scale,
is_causal=False,
)
- actual = auto_round_kernel.ARK().sagev1(
+ actual = auto_round_kernel.sagev1(
q,
k,
v,
diff --git a/auto_round_extension/ark/test/test_weightonly.py b/auto_round_extension/ark/test/test_weightonly.py
index d26e843fd..c81aed0cd 100644
--- a/auto_round_extension/ark/test/test_weightonly.py
+++ b/auto_round_extension/ark/test/test_weightonly.py
@@ -23,7 +23,7 @@ import pytest
import torch
from ut_utils import decode_fp8_to_float, gen_weis8, get_torch_dt, sample_valid_fp8, sample_valid_fp8_e8m0_xpu_safe
-ark = auto_round_kernel.ARK()
+ark = auto_round_kernel
def main_op(m, n, k, blocksize, compute_type, weight_type, scale_type, asym, device, is_ci=False):