Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

int8 kv cache layout, update, prepare

CCYeh09ef61fd8505
diff --git a/crates/uzu-engine/build/common/gpu_types/item_constant.rs b/crates/uzu-engine/build/common/gpu_types/item_constant.rs
index 10a1731ce..dc77a04c4 100644
--- a/crates/uzu-engine/build/common/gpu_types/item_constant.rs
+++ b/crates/uzu-engine/build/common/gpu_types/item_constant.rs
@@ -1,24 +1,30 @@
use anyhow::bail;
use quote::ToTokens;
use syn::{
- BinOp, Expr, ExprBinary, ExprGroup, ExprLit, ExprParen, ExprPath, ExprUnary, ItemConst, Lit, Type, TypePath, UnOp,
+ BinOp, Expr, ExprArray, ExprBinary, ExprGroup, ExprLit, ExprParen, ExprPath, ExprUnary, ItemConst, Lit, Type,
+ TypePath, UnOp,
};
#[derive(Debug)]
pub struct GpuTypeConstant {
pub name: Box<str>,
pub ty: Box<str>,
+ pub array_length: Option<Box<str>>,
pub value_expression: Box<str>,
}
impl GpuTypeConstant {
pub fn parse(item: ItemConst) -> anyhow::Result<Self> {
+ let (ty, array_length) = match *item.ty {
+ Type::Array(array) => (*array.elem, Some(render_const_expr(&array.len, false)?.into())),
+ ty => (ty, None),
+ };
let Type::Path(TypePath {
qself: None,
path,
- }) = *item.ty
+ }) = ty
else {
- bail!("GPU constant type must be a path, found {:?}", item.ty);
+ bail!("GPU constant element type must be a path, found {ty:?}");
};
let is_bool = path.is_ident("bool");
@@ -27,6 +33,7 @@ impl GpuTypeConstant {
Ok(Self {
name: item.ident.to_string().into(),
ty: path.into_token_stream().to_string().into(),
+ array_length,
value_expression,
})
}
@@ -37,6 +44,14 @@ fn render_const_expr(
is_bool: bool,
) -> anyhow::Result<String> {
match expr {
+ Expr::Array(ExprArray {
+ elems,
+ ..
+ }) => {
+ let elements =
+ elems.iter().map(|element| render_const_expr(element, is_bool)).collect::<anyhow::Result<Vec<_>>>()?;
+ Ok(format!("{{{}}}", elements.join(", ")))
+ },
Expr::Lit(ExprLit {
lit: Lit::Int(value),
..
diff --git a/crates/uzu-engine/build/metal/gpu_types/gpu_type_constant.rs b/crates/uzu-engine/build/metal/gpu_types/gpu_type_constant.rs
index 114b94e76..da9b02f16 100644
--- a/crates/uzu-engine/build/metal/gpu_types/gpu_type_constant.rs
+++ b/crates/uzu-engine/build/metal/gpu_types/gpu_type_constant.rs
@@ -3,5 +3,14 @@ use crate::common::gpu_types::GpuTypeConstant;
pub fn gpu_type_gen_constant(constant: &GpuTypeConstant) -> anyhow::Result<String> {
let ty = rust_to_metal(&constant.ty)?;
- Ok(format!("static constant constexpr {ty} {} = {};", constant.name, constant.value_expression))
+ let array_suffix = constant.array_length.as_ref().map(|length| format!("[{length}]")).unwrap_or_default();
+ let separator = if constant.array_length.is_some() {
+ "\n "
+ } else {
+ " "
+ };
+ Ok(format!(
+ "static constant constexpr {ty} {}{array_suffix} ={separator}{};",
+ constant.name, constant.value_expression
+ ))
}
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/kv_layout.rs b/crates/uzu-engine/src/backends/common/gpu_types/kv_layout.rs
new file mode 100644
index 000000000..2bbfa5c6d
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/gpu_types/kv_layout.rs
@@ -0,0 +1,13 @@
+pub const TILE_KEYS: u32 = 32;
+pub const TILE_DIMS: u32 = 16;
+pub const FRAGMENT_BYTES: u32 = TILE_KEYS * TILE_DIMS;
+pub const SLICE_DIMS: u32 = 64;
+pub const V_CODE_BIAS: i16 = 128;
+
+pub const GROUP_ROWS: u32 = 8;
+pub const LANE_COLS: u32 = 4;
+pub const LANE_BYTES: u32 = TILE_KEYS / GROUP_ROWS * LANE_COLS;
+
+pub const WORD_BITS: u32 = 32;
+pub const ROTATION_SIGN_BITS: [u32; 8] =
+ [0x9e3779b9, 0x85ebca6b, 0xc2b2ae35, 0x27d4eb2f, 0x165667b1, 0xbf58476d, 0x94d049bb, 0xd6e8feb8];
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
index 91947e9a0..902858285 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
@@ -12,6 +12,7 @@ pub mod gated_act_mul;
pub mod gemm;
pub mod hadamard_order;
pub mod kv_cache_update;
+pub mod kv_layout;
pub mod matmul;
pub mod quantization;
pub mod quantization_method;
diff --git a/crates/uzu-engine/src/backends/common/kernel/kv_layout.rs b/crates/uzu-engine/src/backends/common/kernel/kv_layout.rs
new file mode 100644
index 000000000..7ffc660f2
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/kernel/kv_layout.rs
@@ -0,0 +1,69 @@
+use crate::backends::common::gpu_types::kv_layout;
+
+pub struct Int8KVLayout {
+ pub head_dim: u32,
+ pub num_kv_heads: u32,
+}
+
+impl Int8KVLayout {
+ pub fn new(
+ head_dim: u32,
+ num_kv_heads: u32,
+ ) -> Self {
+ Self {
+ head_dim,
+ num_kv_heads,
+ }
+ }
+
+ /// Scales: [key block][head][key in block].
+ pub fn scale_index(
+ &self,
+ key: u32,
+ head: u32,
+ ) -> usize {
+ ((key / kv_layout::TILE_KEYS * self.num_kv_heads + head) * kv_layout::TILE_KEYS + key % kv_layout::TILE_KEYS)
+ as usize
+ }
+
+ pub fn k_code_index(
+ &self,
+ key: u32,
+ head: u32,
+ channel: u32,
+ ) -> usize {
+ let row = key % kv_layout::TILE_KEYS;
+ let lane = mma_lane(row % kv_layout::GROUP_ROWS, channel % kv_layout::TILE_DIMS / kv_layout::LANE_COLS);
+ let lane_offset = row / kv_layout::GROUP_ROWS * kv_layout::LANE_COLS + channel % kv_layout::LANE_COLS;
+ ((key / kv_layout::TILE_KEYS * self.num_kv_heads + head) * kv_layout::TILE_KEYS * self.head_dim) as usize
+ + (channel / kv_layout::TILE_DIMS * kv_layout::FRAGMENT_BYTES + lane * kv_layout::LANE_BYTES + lane_offset)
+ as usize
+ }
+
+ pub fn v_code_index(
+ &self,
+ key: u32,
+ head: u32,
+ channel: u32,
+ ) -> usize {
+ let (row, slice_channel) = (key % kv_layout::TILE_KEYS, channel % kv_layout::SLICE_DIMS);
+ let tile = slice_channel / kv_layout::TILE_DIMS;
+ let tiles = kv_layout::SLICE_DIMS / kv_layout::TILE_DIMS;
+ let half_rows = kv_layout::TILE_KEYS / 2;
+ let lane = mma_lane(row % kv_layout::GROUP_ROWS, slice_channel % kv_layout::TILE_DIMS / kv_layout::LANE_COLS);
+ let pair = tiles * (channel / kv_layout::SLICE_DIMS) + tiles / 2 * (row / half_rows) + tile / 2;
+ let lane_offset = tile % 2 * (kv_layout::LANE_BYTES / 2)
+ + row % half_rows / kv_layout::GROUP_ROWS * kv_layout::LANE_COLS
+ + channel % kv_layout::LANE_COLS;
+ ((key / kv_layout::TILE_KEYS * self.num_kv_heads + head) * kv_layout::TILE_KEYS * self.head_dim) as usize
+ + (pair * kv_layout::FRAGMENT_BYTES + lane * kv_layout::LANE_BYTES + lane_offset) as usize
+ }
+}
+
+fn mma_lane(
+ row: u32,
+ group: u32,
+) -> u32 {
+ // Lane bits: row[2], group[1], row[1], row[0], group[0].
+ ((row >> 2) << 4) | ((group >> 1) << 3) | (((row >> 1) & 1) << 2) | ((row & 1) << 1) | (group & 1)
+}
diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rs
index 0e1e89e05..e1206e9d5 100644
--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs
@@ -8,6 +8,8 @@ pub mod attention;
pub mod delta_net_chunked_prefill;
pub mod delta_net_tree_verify;
pub mod gated_act_mul;
+#[cfg(any(backend = "cpu", test))]
+pub mod kv_layout;
pub mod matmul;
pub mod radix_top_k_small;
#[cfg(any(backend = "cpu", test))]
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs
index cbd24aee9..4da298f46 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs
@@ -4,7 +4,7 @@ use uzu_engine_macros::kernel;
use crate::data_type::DataTypeRepr;
-fn apply_rope<ElementT: DataTypeRepr + Float>(
+pub(super) fn apply_rope<ElementT: DataTypeRepr + Float>(
head: *const ElementT,
cosines: *const (impl DataTypeRepr + Float),
sines: *const (impl DataTypeRepr + Float),
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs
index 3f0fea1e3..51be1992f 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs
@@ -2,7 +2,7 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::data_type::DataTypeRepr;
+use crate::{backends::common::kernel::kv_layout::Int8KVLayout, data_type::DataTypeRepr};
#[kernel(KVCacheUpdate)]
#[variants(T, f32, bf16)]
@@ -26,3 +26,37 @@ pub fn kv_cache_update<T: DataTypeRepr + Float>(
}
}
}
+
+#[kernel(KVCacheUpdateInt8)]
+pub fn kv_cache_update_int8(
+ in_place_k_codes: *mut i8,
+ in_place_v_codes: *mut u8,
+ in_place_k_scales: *mut bf16,
+ in_place_v_scales: *mut bf16,
+ copies: &[crate::backends::common::gpu_types::kv_cache_update::Copy],
+ copy_count: u32,
+ #[specialize] head_dim: u32,
+ #[specialize] num_kv_heads: u32,
+) {
+ let layout = Int8KVLayout::new(head_dim, num_kv_heads);
+ for copy in copies.iter().take(copy_count as usize) {
+ for head in 0..num_kv_heads {
+ for channel in 0..head_dim {
+ let source_k = layout.k_code_index(copy.source, head, channel);
+ let destination_k = layout.k_code_index(copy.destination, head, channel);
+ let source_v = layout.v_code_index(copy.source, head, channel);
+ let destination_v = layout.v_code_index(copy.destination, head, channel);
+ unsafe {
+ *in_place_k_codes.add(destination_k) = *in_place_k_codes.add(source_k);
+ *in_place_v_codes.add(destination_v) = *in_place_v_codes.add(source_v);
+ }
+ }
+ let source = layout.scale_index(copy.source, head);
+ let destination = layout.scale_index(copy.destination, head);
+ unsafe {
+ *in_place_k_scales.add(destination) = *in_place_k_scales.add(source);
+ *in_place_v_scales.add(destination) = *in_place_v_scales.add(source);
+ }
+ }
+ }
+}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
index e357123a1..ab202b81f 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
@@ -4,6 +4,7 @@ pub mod ancestor_attention;
pub mod attention_prepare;
pub mod attention_single_pass;
pub mod kv_cache_update;
+pub mod prepare_int8;
pub mod qkv_norm;
pub mod sigmoid_gate;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs
new file mode 100644
index 000000000..2b251a10e
--- /dev/null
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs
@@ -0,0 +1,122 @@
+use half::bf16;
+use num_traits::Float;
+use uzu_engine_macros::kernel;
+
+use super::attention_prepare::apply_rope;
+use crate::{
+ backends::{
+ common::{gpu_types::kv_layout, kernel::kv_layout::Int8KVLayout},
+ cpu::kernel::activation_transform::{INT8_SYMMETRIC_QUANTIZATION_MAXIMUM as INT8_MAX, hadamard_transform},
+ },
+ data_type::DataTypeRepr,
+};
+
+#[kernel(AttentionPrepareInt8)]
+#[variants(ElementT, bf16)]
+#[variants(RopeT, f32)]
+#[variants(HEAD_DIM, 256)]
+pub fn attention_prepare_int8<ElementT: DataTypeRepr + Float, RopeT: DataTypeRepr + Float, const HEAD_DIM: u32>(
+ qkvg: *const ElementT,
+ #[optional(!quantize_queries)] queries: Option<*mut ElementT>,
+ #[optional(quantize_queries)] q_codes: Option<*mut i8>,
+ #[optional(quantize_queries)] q_scales: Option<*mut ElementT>,
+ k_codes: *mut i8,
+ k_scales: *mut ElementT,
+ v_codes: *mut u8,
+ v_scales: *mut ElementT,
+ #[optional(has_rope)] cosines: Option<*const RopeT>,
+ #[optional(has_rope)] sines: Option<*const RopeT>,
+ num_q_heads: u32,
+ #[optional(has_rope)] rope_dim: Option<u32>,
+ kv_token_offset: u32,
+ input_row_stride: u32,
+ suffix_length: u32,
+ padded_suffix: u32,
+ #[specialize] has_rope: bool,
+ #[specialize] quantize_queries: bool,
+ #[specialize] num_kv_heads: u32,
+) {
+ let head_dim = HEAD_DIM as usize;
+ let rope_dim = if has_rope {
+ rope_dim.unwrap() as usize
+ } else {
+ 0
+ };
+ let layout = Int8KVLayout::new(HEAD_DIM, num_kv_heads);
+ let mut values = vec![0.0f32; head_dim];
+ let v_head_start = num_q_heads + num_kv_heads;
+ for head in 0..v_head_start + num_kv_heads {
+ let is_query = head < num_q_heads;
+ let is_key = !is_query && head < v_head_start;
+ let quantize = !is_query || quantize_queries;
+ let (kv_head, scales) = match (is_query, is_key) {
+ (true, _) => (0, q_scales),
+ (_, true) => (head - num_q_heads, Some(k_scales)),
+ _ => (head - v_head_start, Some(v_scales)),
+ };
+ for token in 0..padded_suffix {
+ let query_row = (head * padded_suffix + token) as usize;
+ if token >= suffix_length {
+ if is_query && quantize_queries {
+ unsafe {
+ std::ptr::write_bytes(q_codes.unwrap().add(query_row * head_dim), 0, head_dim);
+ *scales.unwrap().add(query_row) = ElementT::one();
+ }
+ }
+ continue;
+ }
+ let source = unsafe { qkvg.add((token * input_row_stride + head * HEAD_DIM) as usize) };
+ for (channel, value) in values.iter_mut().enumerate() {
+ let element = if (is_query || is_key) && channel < rope_dim {
+ apply_rope(source, cosines.unwrap(), sines.unwrap(), token as usize, channel, rope_dim)
+ } else {
+ unsafe { *source.add(channel) }
+ }
+ .to_f32()
+ .unwrap();
+ if !quantize {
+ let row = (head * suffix_length + token) as usize * head_dim;
+ unsafe { *queries.unwrap().add(row + channel) = ElementT::from(element).unwrap() };
+ } else {
+ let sign_word = kv_layout::ROTATION_SIGN_BITS[channel / kv_layout::WORD_BITS as usize];
+ let flip_sign = (sign_word >> (channel % kv_layout::WORD_BITS as usize)) & 1 != 0;
+ *value = if flip_sign {
+ -element
+ } else {
+ element
+ };
+ }
+ }
+ if !quantize {
+ continue;
+ }
+ hadamard_transform(&mut values);
+ let maximum = values.iter().fold(0.0f32, |maximum, value| maximum.max(value.abs()));
+ let scale = ElementT::from(if maximum > 0.0 {
+ maximum / INT8_MAX
+ } else {
+ 1.0
+ })
+ .unwrap();
+ let scale_index = if is_query {
+ query_row
+ } else {
+ layout.scale_index(kv_token_offset + token, kv_head)
+ };
+ for (channel, &value) in values.iter().enumerate() {
+ let code = (value / scale.to_f32().unwrap()).round_ties_even().clamp(-INT8_MAX, INT8_MAX) as i8;
+ unsafe {
+ if is_query {
+ *q_codes.unwrap().add(scale_index * head_dim + channel) = code;
+ } else if is_key {
+ *k_codes.add(layout.k_code_index(kv_token_offset + token, kv_head, channel as u32)) = code;
+ } else {
+ *v_codes.add(layout.v_code_index(kv_token_offset + token, kv_head, channel as u32)) =
+ (i16::from(code) + kv_layout::V_CODE_BIAS) as u8;
+ }
+ }
+ }
+ unsafe { *scales.unwrap().add(scale_index) = scale };
+ }
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_prepare.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_prepare.metal
index 07aee1240..aa20b7811 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_prepare.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_prepare.metal
@@ -1,27 +1,5 @@
#include "../common/dsl.h"
-
-// TODO: very ugly and wasteful, clean up and optimize
-
-template <typename ElementT, typename RopeT>
-METAL_FUNC ElementT apply_rope(
- const device ElementT* head,
- const device RopeT* cosines,
- const device RopeT* sines,
- uint32_t batch_idx,
- uint32_t head_dim_idx,
- uint32_t rope_dim
-) {
- const uint32_t half_rope_dim = rope_dim / 2;
- const uint32_t paired_idx =
- head_dim_idx < half_rope_dim ? head_dim_idx + half_rope_dim : head_dim_idx - half_rope_dim;
- const float input = float(head[head_dim_idx]);
- const float paired = float(head[paired_idx]);
- const float signed_paired = head_dim_idx < half_rope_dim ? -paired : paired;
- const float cos_val = float(cosines[batch_idx * rope_dim + head_dim_idx]);
- const float sin_val = float(sines[batch_idx * rope_dim + head_dim_idx]);
-
- return static_cast<ElementT>(input * cos_val + signed_paired * sin_val);
-}
+#include "rope.h"
template <typename ElementT, typename RopeT>
VARIANTS(ElementT, bfloat)
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal
index 9fdfab026..2360ee8b8 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal
@@ -2,6 +2,9 @@
#include "../common/dsl.h"
#include "kv_cache_update.h"
+#include "kv_layout.h"
+
+namespace kv_layout = uzu::kv_layout;
template <typename T>
VARIANTS(T, bfloat)
@@ -21,3 +24,33 @@ PUBLIC KERNEL(KVCacheUpdate) (
in_place_values[destIdx] = in_place_values[sourceIdx];
}
}
+
+PUBLIC KERNEL(KVCacheUpdateInt8)(
+ device int8_t* in_place_k_codes,
+ device uchar* in_place_v_codes,
+ device bfloat* in_place_k_scales,
+ device bfloat* in_place_v_scales,
+ const constant uzu::kv_cache_update::Copy* copies,
+ const constant uint& copy_count,
+ const uint head_dim SPECIALIZE,
+ const uint num_kv_heads SPECIALIZE,
+ const uint element AXIS(num_kv_heads* head_dim, 256)
+) {
+ const kv_layout::Int8KVShape shape{head_dim, num_kv_heads};
+ const uint kv_head = element / head_dim;
+ const uint channel = element % head_dim;
+ for (uint i = 0; i < copy_count; ++i) {
+ const uint source = copies[i].source;
+ const uint destination = copies[i].destination;
+ in_place_k_codes[shape.k_code_index(destination, kv_head, channel)] =
+ in_place_k_codes[shape.k_code_index(source, kv_head, channel)];
+ in_place_v_codes[shape.v_code_index(destination, kv_head, channel)] =
+ in_place_v_codes[shape.v_code_index(source, kv_head, channel)];
+ if (channel == 0) {
+ const size_t source_scale = shape.scale_index(source, kv_head);
+ const size_t destination_scale = shape.scale_index(destination, kv_head);
+ in_place_k_scales[destination_scale] = in_place_k_scales[source_scale];
+ in_place_v_scales[destination_scale] = in_place_v_scales[source_scale];
+ }
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_layout.h b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_layout.h
new file mode 100644
index 000000000..b103146d4
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_layout.h
@@ -0,0 +1,62 @@
+#pragma once
+
+#include <metal_stdlib>
+#include "../common/defines.h"
+#include "../generated/kv_layout.h"
+
+// K: [block][head][fragment][lane][LANE_BYTES].
+// V: [block][head][slice][chunk group][lane][LANE_BYTES].
+
+namespace uzu::kv_layout {
+
+// Lane bits: row[2], group[1], row[1], row[0], group[0].
+METAL_FUNC uint mma_lane(const uint row_in_group, const uint channel_group) {
+ return ((row_in_group >> 2) << 4) | ((channel_group >> 1) << 3) | (((row_in_group >> 1) & 1u) << 2) |
+ ((row_in_group & 1u) << 1) | (channel_group & 1u);
+}
+
+struct Int8KVShape {
+ uint head_dim;
+ uint num_kv_heads;
+
+ METAL_FUNC size_t block_region(const uint block, const uint kv_head) const {
+ return (size_t(block) * num_kv_heads + kv_head) * (kv_layout::TILE_KEYS * head_dim);
+ }
+
+ METAL_FUNC size_t scale_index(const uint key, const uint kv_head) const {
+ return (size_t(key / kv_layout::TILE_KEYS) * num_kv_heads + kv_head) * kv_layout::TILE_KEYS +
+ key % kv_layout::TILE_KEYS;
+ }
+
+ METAL_FUNC size_t k_code_index(const uint key, const uint kv_head, const uint channel) const {
+ const uint row = key % kv_layout::TILE_KEYS;
+ const uint lane = mma_lane(row % kv_layout::GROUP_ROWS, channel % kv_layout::TILE_DIMS / kv_layout::LANE_COLS);
+ const uint lane_offset = row / kv_layout::GROUP_ROWS * kv_layout::LANE_COLS + channel % kv_layout::LANE_COLS;
+ return block_region(key / kv_layout::TILE_KEYS, kv_head) +
+ channel / kv_layout::TILE_DIMS * kv_layout::FRAGMENT_BYTES + lane * kv_layout::LANE_BYTES + lane_offset;
+ }
+
+ METAL_FUNC size_t v_code_index(const uint key, const uint kv_head, const uint channel) const {
+ const uint row = key % kv_layout::TILE_KEYS;
+ const uint slice_channel = channel % kv_layout::SLICE_DIMS;
+ const uint tile = slice_channel / kv_layout::TILE_DIMS;
+ const uint tiles = kv_layout::SLICE_DIMS / kv_layout::TILE_DIMS;
+ const uint half_rows = kv_layout::TILE_KEYS / 2;
+ const uint lane =
+ mma_lane(row % kv_layout::GROUP_ROWS, slice_channel % kv_layout::TILE_DIMS / kv_layout::LANE_COLS);
+ const uint pair = tiles * (channel / kv_layout::SLICE_DIMS) + tiles / 2 * (row / half_rows) + tile / 2;
+ const uint lane_offset = tile % 2 * (kv_layout::LANE_BYTES / 2) +
+ row % half_rows / kv_layout::GROUP_ROWS * kv_layout::LANE_COLS +
+ channel % kv_layout::LANE_COLS;
+ return block_region(key / kv_layout::TILE_KEYS, kv_head) + pair * kv_layout::FRAGMENT_BYTES +
+ lane * kv_layout::LANE_BYTES + lane_offset;
+ }
+};
+
+METAL_FUNC float rotation_sign(const uint channel) {
+ const uint word = kv_layout::ROTATION_SIGN_BITS[channel / kv_layout::WORD_BITS];
+ const uint bit = channel % kv_layout::WORD_BITS;
+ return (word >> bit) & 1u ? -1.0f : 1.0f;
+}
+
+} // namespace uzu::kv_layout
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal
new file mode 100644
index 000000000..5e7c8220a
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal
@@ -0,0 +1,123 @@
+#include <metal_stdlib>
+#include "../common/defines.h"
+#include "../common/dsl.h"
+#include "../hadamard_transform/hadamard_transform.h"
+#include "kv_layout.h"
+#include "rope.h"
+
+namespace kv_layout = uzu::kv_layout;
+
+template <typename ElementT, ushort VALUES_PER_LANE>
+METAL_FUNC ElementT kv_quantize_row(const thread float* values, thread int* codes) {
+ constexpr float CODE_MAX = static_cast<float>(metal::numeric_limits<int8_t>::max());
+ float abs_max = 0.0f;
+ METAL_PRAGMA_UNROLL
+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {
+ abs_max = metal::max(abs_max, metal::abs(values[value_index]));
+ }
+ abs_max = metal::simd_max(abs_max);
+ const ElementT scale = ElementT(abs_max > 0.0f ? metal::precise::divide(abs_max, CODE_MAX) : 1.0f);
+ METAL_PRAGMA_UNROLL
+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {
+ codes[value_index] =
+ int(metal::clamp(metal::rint(metal::precise::divide(values[value_index], float(scale))), -CODE_MAX, CODE_MAX));
+ }
+ return scale;
+}
+
+template <typename ElementT, typename RopeT, uint HEAD_DIM>
+VARIANTS(ElementT, bfloat)
+VARIANTS(RopeT, float)
+VARIANTS(HEAD_DIM, 256)
+PUBLIC KERNEL(AttentionPrepareInt8)(
+ const device ElementT* qkvg,
+ device ElementT* queries OPTIONAL(!quantize_queries),
+ device int8_t* q_codes OPTIONAL(quantize_queries),
+ device ElementT* q_scales OPTIONAL(quantize_queries),
+ device int8_t* k_codes,
+ device ElementT* k_scales,
+ device uchar* v_codes,
+ device ElementT* v_scales,
+ const device RopeT* cosines OPTIONAL(has_rope),
+ const device RopeT* sines OPTIONAL(has_rope),
+ const constant uint32_t& num_q_heads,
+ const constant uint32_t& rope_dim OPTIONAL(has_rope),
+ const constant uint32_t& kv_token_offset,
+ const constant uint32_t& input_row_stride,
+ const constant uint32_t& suffix_length,
+ const constant uint32_t& padded_suffix,
+ const bool has_rope SPECIALIZE,
+ const bool quantize_queries SPECIALIZE,
+ const uint32_t num_kv_heads SPECIALIZE,
+ const uint32_t token_idx GROUPS(padded_suffix),
+ const uint32_t head_idx GROUPS(num_q_heads + num_kv_heads * 2),
+ const uint32_t lane THREADS(METAL_SIMD_SIZE)
+) {
+ constexpr ushort VALUES_PER_LANE = HEAD_DIM / METAL_SIMD_SIZE;
+ const kv_layout::Int8KVShape shape{HEAD_DIM, num_kv_heads};
+ const ushort lane_index = ushort(lane);
+ const bool is_query = head_idx < num_q_heads;
+ const bool is_key = !is_query && head_idx < num_q_heads + num_kv_heads;
+ const uint32_t kv_head = is_key ? head_idx - num_q_heads : head_idx - num_q_heads - num_kv_heads;
+
+ if (token_idx >= suffix_length) {
+ if (is_query && quantize_queries) {
+ const size_t row = size_t(head_idx) * padded_suffix + token_idx;
+ METAL_PRAGMA_UNROLL
+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {
+ q_codes[row * HEAD_DIM + value_index * METAL_SIMD_SIZE + lane_index] = 0;
+ }
+ if (lane_index == 0) {
+ q_scales[row] = ElementT(1.0f);
+ }
+ }
+ return;
+ }
+
+ const device ElementT* head = qkvg + size_t(token_idx) * input_row_stride + size_t(head_idx) * HEAD_DIM;
+ float values[VALUES_PER_LANE];
+ METAL_PRAGMA_UNROLL
+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {
+ const uint32_t channel = value_index * METAL_SIMD_SIZE + lane_index;
+ ElementT element = head[channel];
+ if (has_rope && (is_query || is_key) && channel < rope_dim) {
+ element = apply_rope(head, cosines, sines, token_idx, channel, rope_dim);
+ }
+ values[value_index] = float(element);
+ }
+
+ if (is_query && !quantize_queries) {
+ const size_t row = size_t(head_idx) * suffix_length + token_idx;
+ METAL_PRAGMA_UNROLL
+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {
+ queries[row * HEAD_DIM + value_index * METAL_SIMD_SIZE + lane_index] = ElementT(values[value_index]);
+ }
+ return;
+ }
+
+ METAL_PRAGMA_UNROLL
+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {
+ values[value_index] *= kv_layout::rotation_sign(value_index * METAL_SIMD_SIZE + lane_index);
+ }
+ simdgroup_hadamard_transform<VALUES_PER_LANE, METAL_SIMD_SIZE>(lane_index, values);
+
+ int codes[VALUES_PER_LANE];
+ const ElementT scale = kv_quantize_row<ElementT, VALUES_PER_LANE>(values, codes);
+
+ const uint32_t key = kv_token_offset + token_idx;
+ const size_t scale_index = is_query ? size_t(head_idx) * padded_suffix + token_idx : shape.scale_index(key, kv_head);
+ METAL_PRAGMA_UNROLL
+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {
+ const uint32_t channel = value_index * METAL_SIMD_SIZE + lane_index;
+ if (is_query) {
+ q_codes[scale_index * HEAD_DIM + channel] = int8_t(codes[value_index]);
+ } else if (is_key) {
+ k_codes[shape.k_code_index(key, kv_head, channel)] = int8_t(codes[value_index]);
+ } else {
+ v_codes[shape.v_code_index(key, kv_head, channel)] = uchar(codes[value_index] + kv_layout::V_CODE_BIAS);
+ }
+ }
+ if (lane_index == 0) {
+ (is_query ? q_scales : is_key ? k_scales : v_scales)[scale_index] = scale;
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/rope.h b/crates/uzu-engine/src/backends/metal/kernel/attention/rope.h
new file mode 100644
index 000000000..d0fbcbd4d
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/rope.h
@@ -0,0 +1,26 @@
+#pragma once
+
+#include <metal_stdlib>
+#include "../common/defines.h"
+
+// TODO: very ugly and wasteful, clean up and optimize
+template <typename ElementT, typename RopeT>
+METAL_FUNC ElementT apply_rope(
+ const device ElementT* head,
+ const device RopeT* cosines,
+ const device RopeT* sines,
+ uint32_t batch_idx,
+ uint32_t head_dim_idx,
+ uint32_t rope_dim
+) {
+ const uint32_t half_rope_dim = rope_dim / 2;
+ const uint32_t paired_idx =
+ head_dim_idx < half_rope_dim ? head_dim_idx + half_rope_dim : head_dim_idx - half_rope_dim;
+ const float input = float(head[head_dim_idx]);
+ const float paired = float(head[paired_idx]);
+ const float signed_paired = head_dim_idx < half_rope_dim ? -paired : paired;
+ const float cos_val = float(cosines[batch_idx * rope_dim + head_dim_idx]);
+ const float sin_val = float(sines[batch_idx * rope_dim + head_dim_idx]);
+
+ return static_cast<ElementT>(input * cos_val + signed_paired * sin_val);
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/kv_layout.h b/crates/uzu-engine/src/backends/metal/kernel/generated/kv_layout.h
new file mode 100644
index 000000000..de5139242
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/kv_layout.h
@@ -0,0 +1,28 @@
+// Auto-generated from gpu_types/kv_layout - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::kv_layout {
+static constant constexpr uint32_t TILE_KEYS = 32;
+
+static constant constexpr uint32_t TILE_DIMS = 16;
+
+static constant constexpr uint32_t FRAGMENT_BYTES = TILE_KEYS * TILE_DIMS;
+
+static constant constexpr uint32_t SLICE_DIMS = 64;
+
+static constant constexpr int16_t V_CODE_BIAS = 128;
+
+static constant constexpr uint32_t GROUP_ROWS = 8;
+
+static constant constexpr uint32_t LANE_COLS = 4;
+
+static constant constexpr uint32_t LANE_BYTES = TILE_KEYS / GROUP_ROWS * LANE_COLS;
+
+static constant constexpr uint32_t WORD_BITS = 32;
+
+static constant constexpr uint32_t ROTATION_SIGN_BITS[8] =
+ {2654435769, 2246822507, 3266489909, 668265263, 374761393, 3210233709, 2496678331, 3605593784};
+} // namespace uzu::kv_layout
diff --git a/crates/uzu-engine/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/uzu-engine/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
index 7777464e8..4d6afe56b 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
@@ -13,13 +13,47 @@ static_assert(
"Hadamard transform block size must match the Metal SIMD width"
);
-static METAL_FUNC float simdgroup_hadamard_transform(ushort lane_index, float lane_value) {
- for (ushort stride = 1; stride < HADAMARD_TRANSFORM_BLOCK_SIZE; stride <<= 1) {
- float partner_lane_value = simd_shuffle_xor(lane_value, stride);
- lane_value = (lane_index & stride) ? (partner_lane_value - lane_value) : (partner_lane_value + lane_value);
+/// Normalized Walsh-Hadamard transform of `VALUES_PER_LANE` values in each of `LANES` lanes.
+template <ushort VALUES_PER_LANE, ushort LANES>
+static METAL_FUNC void simdgroup_hadamard_transform(ushort lane_index, thread float* values) {
+ constexpr ushort SIZE = VALUES_PER_LANE * LANES;
+ constexpr ushort CHUNK = VALUES_PER_LANE < 4 ? VALUES_PER_LANE : 4;
+ using Chunk = metal::conditional_t<CHUNK == 1, float, vec<float, CHUNK>>;
+ static_assert(
+ LANES <= METAL_SIMD_SIZE && (LANES & (LANES - 1)) == 0,
+ "lanes must be a power of two within a simdgroup"
+ );
+ static_assert(SIZE == 32 || SIZE == 64 || SIZE == 128 || SIZE == 256, "transform size must be 32, 64, 128 or 256");
+ METAL_PRAGMA_UNROLL
+ for (ushort stride = 1; stride < VALUES_PER_LANE; stride <<= 1) {
+ METAL_PRAGMA_UNROLL
+ for (ushort index = 0; index < VALUES_PER_LANE; ++index) {
+ if (!(index & stride)) {
+ const float value = values[index];
+ const float partner_value = values[index + stride];
+ values[index] = partner_value + value;
+ values[index + stride] = value - partner_value;
+ }
+ }
+ }
+ METAL_PRAGMA_UNROLL
+ for (ushort lane_stride = 1; lane_stride < LANES; lane_stride <<= 1) {
+ METAL_PRAGMA_UNROLL
+ for (ushort index = 0; index < VALUES_PER_LANE; index += CHUNK) {
+ thread Chunk& chunk = *reinterpret_cast<thread Chunk*>(values + index);
+ const Chunk partner = simd_shuffle_xor(chunk, lane_stride);
+ chunk = (lane_index & lane_stride) ? (partner - chunk) : (partner + chunk);
+ }
+ }
+ METAL_PRAGMA_UNROLL
+ for (ushort index = 0; index < VALUES_PER_LANE; ++index) {
+ values[index] /= sqrt(static_cast<float>(SIZE));
}
+}
- return lane_value / sqrt(static_cast<float>(HADAMARD_TRANSFORM_BLOCK_SIZE));
+static METAL_FUNC float simdgroup_hadamard_transform(ushort lane_index, float lane_value) {
+ simdgroup_hadamard_transform<1, HADAMARD_TRANSFORM_BLOCK_SIZE>(lane_index, &lane_value);
+ return lane_value;
}
template <typename T>
@@ -37,22 +71,11 @@ static METAL_FUNC T simdgroup_output_random_hadamard_transform(ushort lane_index
UZU_CONST ushort HADAMARD_LANES_PER_BLOCK = HADAMARD_TRANSFORM_BLOCK_SIZE / HADAMARD_VECTOR_SIZE;
static METAL_FUNC float4 simdgroup_hadamard_transform_vector(ushort lane_index, float4 lane_values) {
- for (ushort stride = 1; stride < HADAMARD_VECTOR_SIZE; stride <<= 1) {
- METAL_PRAGMA_UNROLL
- for (ushort index = 0; index < HADAMARD_VECTOR_SIZE; ++index) {
- if (!(index & stride)) {
- const float value = lane_values[index];
- const float partner_value = lane_values[index + stride];
- lane_values[index] = partner_value + value;
- lane_values[index + stride] = value - partner_value;
- }
- }
- }
- for (ushort lane_stride = 1; lane_stride < HADAMARD_LANES_PER_BLOCK; lane_stride <<= 1) {
- const float4 partner_values = simd_shuffle_xor(lane_values, lane_stride);
- lane_values = (lane_index & lane_stride) ? (partner_values - lane_values) : (partner_values + lane_values);
- }
- return lane_values / sqrt(static_cast<float>(HADAMARD_TRANSFORM_BLOCK_SIZE));
+ simdgroup_hadamard_transform<HADAMARD_VECTOR_SIZE, HADAMARD_LANES_PER_BLOCK>(
+ lane_index,
+ reinterpret_cast<thread float*>(&lane_values)
+ );
+ return lane_values;
}
template <typename T>
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_prepare_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_prepare_test.rs
index 1f1bd14ff..d213fc357 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_prepare_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_prepare_test.rs
@@ -5,7 +5,7 @@ use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::AttentionPrepareKernel,
+ kernel::{AttentionPrepareInt8Kernel, AttentionPrepareKernel},
},
cpu::Cpu,
},
@@ -131,3 +131,68 @@ fn test_attention_prepare_uses_query_gate_row_stride() {
assert_eq!(run_query_only::<B>(), expected, "query mismatch on {}", std::any::type_name::<B>());
});
}
+
+#[derive(Debug, PartialEq)]
+struct Int8Output {
+ codes: [Vec<u8>; 3],
+ scales: [Vec<bf16>; 3],
+}
+
+fn run_int8<B: Backend>() -> Int8Output {
+ const DIM: usize = 256;
+ const CACHE_TOKENS: usize = 64;
+ let code_bytes = CACHE_TOKENS * DIM;
+ let rows: Vec<bf16> = (0..3 * DIM).map(|i| bf16::from_f32((i as f32 * 0.37).sin())).collect();
+ let context = B::Context::new().unwrap();
+ let qkvg = create_buffer_with_data::<B, bf16>(&context, &rows);
+ let mut q_codes = create_buffer::<B, i8>(&context, DIM);
+ let mut q_scales = create_buffer::<B, bf16>(&context, 1);
+ let mut k_codes = create_buffer_with_data::<B, i8>(&context, &vec![0; code_bytes]);
+ let mut v_codes = create_buffer_with_data::<B, u8>(&context, &vec![0; code_bytes]);
+ let mut k_scales = create_buffer_with_data::<B, bf16>(&context, &[bf16::ZERO; CACHE_TOKENS]);
+ let mut v_scales = create_buffer_with_data::<B, bf16>(&context, &[bf16::ZERO; CACHE_TOKENS]);
+ let kernel = <B::Kernels as Kernels>::AttentionPrepareInt8Kernel::new(
+ &context,
+ DataType::BF16,
+ DataType::F32,
+ DIM as u32,
+ false,
+ true,
+ 1,
+ )
+ .unwrap();
+ let mut command_buffer = context.create_command_buffer(None, None).unwrap();
+ kernel.encode(
+ &qkvg,
+ None::<&mut B::GlobalBuffer>,
+ Some(&mut q_codes),
+ Some(&mut q_scales),
+ &mut k_codes,
+ &mut k_scales,
+ &mut v_codes,
+ &mut v_scales,
+ None::<&B::GlobalBuffer>,
+ None::<&B::GlobalBuffer>,
+ 1,
+ None,
+ 33,
+ (3 * DIM) as u32,
+ 1,
+ 1,
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+
+ Int8Output {
+ codes: [&q_codes, &k_codes, &v_codes].map(buffer_to_vec::<B, u8>),
+ scales: [&q_scales, &k_scales, &v_scales].map(buffer_to_vec::<B, bf16>),
+ }
+}
+
+#[uzu_test]
+fn test_attention_prepare_int8() {
+ let cpu = run_int8::<Cpu>();
+ for_each_non_cpu_backend!(|B| {
+ assert_eq!(cpu, run_int8::<B>(), "INT8 prepare mismatch on {}", std::any::type_name::<B>());
+ });
+}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
index 8f6df0224..765aabb56 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
@@ -8,7 +8,8 @@ use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::kv_cache_update::Copy, kernel::KVCacheUpdateKernel,
+ gpu_types::kv_cache_update::Copy,
+ kernel::{KVCacheUpdateInt8Kernel, KVCacheUpdateKernel, kv_layout::Int8KVLayout},
},
cpu::Cpu,
},
@@ -238,3 +239,44 @@ fn test_no_copy_bf16() {
fn test_large_bf16() {
test_large_internal::<bf16>();
}
+
+#[uzu_test]
+fn test_int8_copy() {
+ let expected = get_int8_output::<Cpu>();
+ for_each_non_cpu_backend!(|B| {
+ assert_eq!(expected, get_int8_output::<B>(), "INT8 cache update mismatch on {}", std::any::type_name::<B>());
+ });
+}
+
+fn get_int8_output<B: Backend>() -> ([Vec<u8>; 2], [Vec<bf16>; 2]) {
+ let layout = Int8KVLayout::new(256, 2);
+ let scale_count = 64 * layout.num_kv_heads as usize;
+ let code_bytes = scale_count * layout.head_dim as usize;
+ let k_codes: Vec<i8> = (0..code_bytes).map(|i| ((i * 37) ^ (i >> 8)) as i8).collect();
+ let v_codes: Vec<u8> = (0..code_bytes).map(|i| ((i * 19) ^ (i >> 7)) as u8).collect();
+ let k_scales: Vec<bf16> = (0..scale_count).map(|i| bf16::from_f32(i as f32 + 1.0)).collect();
+ let v_scales: Vec<bf16> = (0..scale_count).map(|i| bf16::from_f32(i as f32 + 129.0)).collect();
+ let copies = [Copy {
+ source: 40,
+ destination: 5,
+ }];
+ let context = B::Context::new().unwrap();
+ let mut keys = create_buffer_with_data::<B, i8>(&context, &k_codes);
+ let mut values = create_buffer_with_data::<B, u8>(&context, &v_codes);
+ let mut key_scales = create_buffer_with_data::<B, bf16>(&context, &k_scales);
+ let mut value_scales = create_buffer_with_data::<B, bf16>(&context, &v_scales);
+ let kernel =
+ <B::Kernels as Kernels>::KVCacheUpdateInt8Kernel::new(&context, layout.head_dim, layout.num_kv_heads).unwrap();
+ let mut command_buffer = context.create_command_buffer(None, None).unwrap();
+ kernel.encode(
+ &mut keys,
+ &mut values,
+ &mut key_scales,
+ &mut value_scales,
+ &copies,
+ copies.len() as u32,
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ ([&keys, &values].map(buffer_to_vec::<B, u8>), [&key_scales, &value_scales].map(buffer_to_vec::<B, bf16>))
+}