A high-performance inference engine for AI models
int8 kv cache layout, update, prepare
CCYeh09ef61fd8505
diff --git a/crates/uzu-engine/build/common/gpu_types/item_constant.rs b/crates/uzu-engine/build/common/gpu_types/item_constant.rsindex 10a1731ce..dc77a04c4 100644--- a/crates/uzu-engine/build/common/gpu_types/item_constant.rs+++ b/crates/uzu-engine/build/common/gpu_types/item_constant.rs@@ -1,24 +1,30 @@use anyhow::bail;use quote::ToTokens;use syn::{- BinOp, Expr, ExprBinary, ExprGroup, ExprLit, ExprParen, ExprPath, ExprUnary, ItemConst, Lit, Type, TypePath, UnOp,+ BinOp, Expr, ExprArray, ExprBinary, ExprGroup, ExprLit, ExprParen, ExprPath, ExprUnary, ItemConst, Lit, Type,+ TypePath, UnOp,};#[derive(Debug)]pub struct GpuTypeConstant {pub name: Box<str>,pub ty: Box<str>,+ pub array_length: Option<Box<str>>,pub value_expression: Box<str>,}impl GpuTypeConstant {pub fn parse(item: ItemConst) -> anyhow::Result<Self> {+ let (ty, array_length) = match *item.ty {+ Type::Array(array) => (*array.elem, Some(render_const_expr(&array.len, false)?.into())),+ ty => (ty, None),+ };let Type::Path(TypePath {qself: None,path,- }) = *item.ty+ }) = tyelse {- bail!("GPU constant type must be a path, found {:?}", item.ty);+ bail!("GPU constant element type must be a path, found {ty:?}");};let is_bool = path.is_ident("bool");@@ -27,6 +33,7 @@ impl GpuTypeConstant {Ok(Self {name: item.ident.to_string().into(),ty: path.into_token_stream().to_string().into(),+ array_length,value_expression,})}@@ -37,6 +44,14 @@ fn render_const_expr(is_bool: bool,) -> anyhow::Result<String> {match expr {+ Expr::Array(ExprArray {+ elems,+ ..+ }) => {+ let elements =+ elems.iter().map(|element| render_const_expr(element, is_bool)).collect::<anyhow::Result<Vec<_>>>()?;+ Ok(format!("{{{}}}", elements.join(", ")))+ },Expr::Lit(ExprLit {lit: Lit::Int(value),..diff --git a/crates/uzu-engine/build/metal/gpu_types/gpu_type_constant.rs b/crates/uzu-engine/build/metal/gpu_types/gpu_type_constant.rsindex 114b94e76..da9b02f16 100644--- a/crates/uzu-engine/build/metal/gpu_types/gpu_type_constant.rs+++ b/crates/uzu-engine/build/metal/gpu_types/gpu_type_constant.rs@@ -3,5 +3,14 @@ use crate::common::gpu_types::GpuTypeConstant;pub fn gpu_type_gen_constant(constant: &GpuTypeConstant) -> anyhow::Result<String> {let ty = rust_to_metal(&constant.ty)?;- Ok(format!("static constant constexpr {ty} {} = {};", constant.name, constant.value_expression))+ let array_suffix = constant.array_length.as_ref().map(|length| format!("[{length}]")).unwrap_or_default();+ let separator = if constant.array_length.is_some() {+ "\n "+ } else {+ " "+ };+ Ok(format!(+ "static constant constexpr {ty} {}{array_suffix} ={separator}{};",+ constant.name, constant.value_expression+ ))}diff --git a/crates/uzu-engine/src/backends/common/gpu_types/kv_layout.rs b/crates/uzu-engine/src/backends/common/gpu_types/kv_layout.rsnew file mode 100644index 000000000..2bbfa5c6d--- /dev/null+++ b/crates/uzu-engine/src/backends/common/gpu_types/kv_layout.rs@@ -0,0 +1,13 @@+pub const TILE_KEYS: u32 = 32;+pub const TILE_DIMS: u32 = 16;+pub const FRAGMENT_BYTES: u32 = TILE_KEYS * TILE_DIMS;+pub const SLICE_DIMS: u32 = 64;+pub const V_CODE_BIAS: i16 = 128;++pub const GROUP_ROWS: u32 = 8;+pub const LANE_COLS: u32 = 4;+pub const LANE_BYTES: u32 = TILE_KEYS / GROUP_ROWS * LANE_COLS;++pub const WORD_BITS: u32 = 32;+pub const ROTATION_SIGN_BITS: [u32; 8] =+ [0x9e3779b9, 0x85ebca6b, 0xc2b2ae35, 0x27d4eb2f, 0x165667b1, 0xbf58476d, 0x94d049bb, 0xd6e8feb8];diff --git a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs b/crates/uzu-engine/src/backends/common/gpu_types/mod.rsindex 91947e9a0..902858285 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs@@ -12,6 +12,7 @@ pub mod gated_act_mul;pub mod gemm;pub mod hadamard_order;pub mod kv_cache_update;+pub mod kv_layout;pub mod matmul;pub mod quantization;pub mod quantization_method;diff --git a/crates/uzu-engine/src/backends/common/kernel/kv_layout.rs b/crates/uzu-engine/src/backends/common/kernel/kv_layout.rsnew file mode 100644index 000000000..7ffc660f2--- /dev/null+++ b/crates/uzu-engine/src/backends/common/kernel/kv_layout.rs@@ -0,0 +1,69 @@+use crate::backends::common::gpu_types::kv_layout;++pub struct Int8KVLayout {+ pub head_dim: u32,+ pub num_kv_heads: u32,+}++impl Int8KVLayout {+ pub fn new(+ head_dim: u32,+ num_kv_heads: u32,+ ) -> Self {+ Self {+ head_dim,+ num_kv_heads,+ }+ }++ /// Scales: [key block][head][key in block].+ pub fn scale_index(+ &self,+ key: u32,+ head: u32,+ ) -> usize {+ ((key / kv_layout::TILE_KEYS * self.num_kv_heads + head) * kv_layout::TILE_KEYS + key % kv_layout::TILE_KEYS)+ as usize+ }++ pub fn k_code_index(+ &self,+ key: u32,+ head: u32,+ channel: u32,+ ) -> usize {+ let row = key % kv_layout::TILE_KEYS;+ let lane = mma_lane(row % kv_layout::GROUP_ROWS, channel % kv_layout::TILE_DIMS / kv_layout::LANE_COLS);+ let lane_offset = row / kv_layout::GROUP_ROWS * kv_layout::LANE_COLS + channel % kv_layout::LANE_COLS;+ ((key / kv_layout::TILE_KEYS * self.num_kv_heads + head) * kv_layout::TILE_KEYS * self.head_dim) as usize+ + (channel / kv_layout::TILE_DIMS * kv_layout::FRAGMENT_BYTES + lane * kv_layout::LANE_BYTES + lane_offset)+ as usize+ }++ pub fn v_code_index(+ &self,+ key: u32,+ head: u32,+ channel: u32,+ ) -> usize {+ let (row, slice_channel) = (key % kv_layout::TILE_KEYS, channel % kv_layout::SLICE_DIMS);+ let tile = slice_channel / kv_layout::TILE_DIMS;+ let tiles = kv_layout::SLICE_DIMS / kv_layout::TILE_DIMS;+ let half_rows = kv_layout::TILE_KEYS / 2;+ let lane = mma_lane(row % kv_layout::GROUP_ROWS, slice_channel % kv_layout::TILE_DIMS / kv_layout::LANE_COLS);+ let pair = tiles * (channel / kv_layout::SLICE_DIMS) + tiles / 2 * (row / half_rows) + tile / 2;+ let lane_offset = tile % 2 * (kv_layout::LANE_BYTES / 2)+ + row % half_rows / kv_layout::GROUP_ROWS * kv_layout::LANE_COLS+ + channel % kv_layout::LANE_COLS;+ ((key / kv_layout::TILE_KEYS * self.num_kv_heads + head) * kv_layout::TILE_KEYS * self.head_dim) as usize+ + (pair * kv_layout::FRAGMENT_BYTES + lane * kv_layout::LANE_BYTES + lane_offset) as usize+ }+}++fn mma_lane(+ row: u32,+ group: u32,+) -> u32 {+ // Lane bits: row[2], group[1], row[1], row[0], group[0].+ ((row >> 2) << 4) | ((group >> 1) << 3) | (((row >> 1) & 1) << 2) | ((row & 1) << 1) | (group & 1)+}diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rsindex 0e1e89e05..e1206e9d5 100644--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs@@ -8,6 +8,8 @@ pub mod attention;pub mod delta_net_chunked_prefill;pub mod delta_net_tree_verify;pub mod gated_act_mul;+#[cfg(any(backend = "cpu", test))]+pub mod kv_layout;pub mod matmul;pub mod radix_top_k_small;#[cfg(any(backend = "cpu", test))]diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rsindex cbd24aee9..4da298f46 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs@@ -4,7 +4,7 @@ use uzu_engine_macros::kernel;use crate::data_type::DataTypeRepr;-fn apply_rope<ElementT: DataTypeRepr + Float>(+pub(super) fn apply_rope<ElementT: DataTypeRepr + Float>(head: *const ElementT,cosines: *const (impl DataTypeRepr + Float),sines: *const (impl DataTypeRepr + Float),diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rsindex 3f0fea1e3..51be1992f 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs@@ -2,7 +2,7 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::data_type::DataTypeRepr;+use crate::{backends::common::kernel::kv_layout::Int8KVLayout, data_type::DataTypeRepr};#[kernel(KVCacheUpdate)]#[variants(T, f32, bf16)]@@ -26,3 +26,37 @@ pub fn kv_cache_update<T: DataTypeRepr + Float>(}}}++#[kernel(KVCacheUpdateInt8)]+pub fn kv_cache_update_int8(+ in_place_k_codes: *mut i8,+ in_place_v_codes: *mut u8,+ in_place_k_scales: *mut bf16,+ in_place_v_scales: *mut bf16,+ copies: &[crate::backends::common::gpu_types::kv_cache_update::Copy],+ copy_count: u32,+ #[specialize] head_dim: u32,+ #[specialize] num_kv_heads: u32,+) {+ let layout = Int8KVLayout::new(head_dim, num_kv_heads);+ for copy in copies.iter().take(copy_count as usize) {+ for head in 0..num_kv_heads {+ for channel in 0..head_dim {+ let source_k = layout.k_code_index(copy.source, head, channel);+ let destination_k = layout.k_code_index(copy.destination, head, channel);+ let source_v = layout.v_code_index(copy.source, head, channel);+ let destination_v = layout.v_code_index(copy.destination, head, channel);+ unsafe {+ *in_place_k_codes.add(destination_k) = *in_place_k_codes.add(source_k);+ *in_place_v_codes.add(destination_v) = *in_place_v_codes.add(source_v);+ }+ }+ let source = layout.scale_index(copy.source, head);+ let destination = layout.scale_index(copy.destination, head);+ unsafe {+ *in_place_k_scales.add(destination) = *in_place_k_scales.add(source);+ *in_place_v_scales.add(destination) = *in_place_v_scales.add(source);+ }+ }+ }+}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rsindex e357123a1..ab202b81f 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs@@ -4,6 +4,7 @@ pub mod ancestor_attention;pub mod attention_prepare;pub mod attention_single_pass;pub mod kv_cache_update;+pub mod prepare_int8;pub mod qkv_norm;pub mod sigmoid_gate;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rsnew file mode 100644index 000000000..2b251a10e--- /dev/null+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs@@ -0,0 +1,122 @@+use half::bf16;+use num_traits::Float;+use uzu_engine_macros::kernel;++use super::attention_prepare::apply_rope;+use crate::{+ backends::{+ common::{gpu_types::kv_layout, kernel::kv_layout::Int8KVLayout},+ cpu::kernel::activation_transform::{INT8_SYMMETRIC_QUANTIZATION_MAXIMUM as INT8_MAX, hadamard_transform},+ },+ data_type::DataTypeRepr,+};++#[kernel(AttentionPrepareInt8)]+#[variants(ElementT, bf16)]+#[variants(RopeT, f32)]+#[variants(HEAD_DIM, 256)]+pub fn attention_prepare_int8<ElementT: DataTypeRepr + Float, RopeT: DataTypeRepr + Float, const HEAD_DIM: u32>(+ qkvg: *const ElementT,+ #[optional(!quantize_queries)] queries: Option<*mut ElementT>,+ #[optional(quantize_queries)] q_codes: Option<*mut i8>,+ #[optional(quantize_queries)] q_scales: Option<*mut ElementT>,+ k_codes: *mut i8,+ k_scales: *mut ElementT,+ v_codes: *mut u8,+ v_scales: *mut ElementT,+ #[optional(has_rope)] cosines: Option<*const RopeT>,+ #[optional(has_rope)] sines: Option<*const RopeT>,+ num_q_heads: u32,+ #[optional(has_rope)] rope_dim: Option<u32>,+ kv_token_offset: u32,+ input_row_stride: u32,+ suffix_length: u32,+ padded_suffix: u32,+ #[specialize] has_rope: bool,+ #[specialize] quantize_queries: bool,+ #[specialize] num_kv_heads: u32,+) {+ let head_dim = HEAD_DIM as usize;+ let rope_dim = if has_rope {+ rope_dim.unwrap() as usize+ } else {+ 0+ };+ let layout = Int8KVLayout::new(HEAD_DIM, num_kv_heads);+ let mut values = vec![0.0f32; head_dim];+ let v_head_start = num_q_heads + num_kv_heads;+ for head in 0..v_head_start + num_kv_heads {+ let is_query = head < num_q_heads;+ let is_key = !is_query && head < v_head_start;+ let quantize = !is_query || quantize_queries;+ let (kv_head, scales) = match (is_query, is_key) {+ (true, _) => (0, q_scales),+ (_, true) => (head - num_q_heads, Some(k_scales)),+ _ => (head - v_head_start, Some(v_scales)),+ };+ for token in 0..padded_suffix {+ let query_row = (head * padded_suffix + token) as usize;+ if token >= suffix_length {+ if is_query && quantize_queries {+ unsafe {+ std::ptr::write_bytes(q_codes.unwrap().add(query_row * head_dim), 0, head_dim);+ *scales.unwrap().add(query_row) = ElementT::one();+ }+ }+ continue;+ }+ let source = unsafe { qkvg.add((token * input_row_stride + head * HEAD_DIM) as usize) };+ for (channel, value) in values.iter_mut().enumerate() {+ let element = if (is_query || is_key) && channel < rope_dim {+ apply_rope(source, cosines.unwrap(), sines.unwrap(), token as usize, channel, rope_dim)+ } else {+ unsafe { *source.add(channel) }+ }+ .to_f32()+ .unwrap();+ if !quantize {+ let row = (head * suffix_length + token) as usize * head_dim;+ unsafe { *queries.unwrap().add(row + channel) = ElementT::from(element).unwrap() };+ } else {+ let sign_word = kv_layout::ROTATION_SIGN_BITS[channel / kv_layout::WORD_BITS as usize];+ let flip_sign = (sign_word >> (channel % kv_layout::WORD_BITS as usize)) & 1 != 0;+ *value = if flip_sign {+ -element+ } else {+ element+ };+ }+ }+ if !quantize {+ continue;+ }+ hadamard_transform(&mut values);+ let maximum = values.iter().fold(0.0f32, |maximum, value| maximum.max(value.abs()));+ let scale = ElementT::from(if maximum > 0.0 {+ maximum / INT8_MAX+ } else {+ 1.0+ })+ .unwrap();+ let scale_index = if is_query {+ query_row+ } else {+ layout.scale_index(kv_token_offset + token, kv_head)+ };+ for (channel, &value) in values.iter().enumerate() {+ let code = (value / scale.to_f32().unwrap()).round_ties_even().clamp(-INT8_MAX, INT8_MAX) as i8;+ unsafe {+ if is_query {+ *q_codes.unwrap().add(scale_index * head_dim + channel) = code;+ } else if is_key {+ *k_codes.add(layout.k_code_index(kv_token_offset + token, kv_head, channel as u32)) = code;+ } else {+ *v_codes.add(layout.v_code_index(kv_token_offset + token, kv_head, channel as u32)) =+ (i16::from(code) + kv_layout::V_CODE_BIAS) as u8;+ }+ }+ }+ unsafe { *scales.unwrap().add(scale_index) = scale };+ }+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_prepare.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_prepare.metalindex 07aee1240..aa20b7811 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_prepare.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_prepare.metal@@ -1,27 +1,5 @@#include "../common/dsl.h"--// TODO: very ugly and wasteful, clean up and optimize--template <typename ElementT, typename RopeT>-METAL_FUNC ElementT apply_rope(- const device ElementT* head,- const device RopeT* cosines,- const device RopeT* sines,- uint32_t batch_idx,- uint32_t head_dim_idx,- uint32_t rope_dim-) {- const uint32_t half_rope_dim = rope_dim / 2;- const uint32_t paired_idx =- head_dim_idx < half_rope_dim ? head_dim_idx + half_rope_dim : head_dim_idx - half_rope_dim;- const float input = float(head[head_dim_idx]);- const float paired = float(head[paired_idx]);- const float signed_paired = head_dim_idx < half_rope_dim ? -paired : paired;- const float cos_val = float(cosines[batch_idx * rope_dim + head_dim_idx]);- const float sin_val = float(sines[batch_idx * rope_dim + head_dim_idx]);-- return static_cast<ElementT>(input * cos_val + signed_paired * sin_val);-}+#include "rope.h"template <typename ElementT, typename RopeT>VARIANTS(ElementT, bfloat)diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metalindex 9fdfab026..2360ee8b8 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal@@ -2,6 +2,9 @@#include "../common/dsl.h"#include "kv_cache_update.h"+#include "kv_layout.h"++namespace kv_layout = uzu::kv_layout;template <typename T>VARIANTS(T, bfloat)@@ -21,3 +24,33 @@ PUBLIC KERNEL(KVCacheUpdate) (in_place_values[destIdx] = in_place_values[sourceIdx];}}++PUBLIC KERNEL(KVCacheUpdateInt8)(+ device int8_t* in_place_k_codes,+ device uchar* in_place_v_codes,+ device bfloat* in_place_k_scales,+ device bfloat* in_place_v_scales,+ const constant uzu::kv_cache_update::Copy* copies,+ const constant uint& copy_count,+ const uint head_dim SPECIALIZE,+ const uint num_kv_heads SPECIALIZE,+ const uint element AXIS(num_kv_heads* head_dim, 256)+) {+ const kv_layout::Int8KVShape shape{head_dim, num_kv_heads};+ const uint kv_head = element / head_dim;+ const uint channel = element % head_dim;+ for (uint i = 0; i < copy_count; ++i) {+ const uint source = copies[i].source;+ const uint destination = copies[i].destination;+ in_place_k_codes[shape.k_code_index(destination, kv_head, channel)] =+ in_place_k_codes[shape.k_code_index(source, kv_head, channel)];+ in_place_v_codes[shape.v_code_index(destination, kv_head, channel)] =+ in_place_v_codes[shape.v_code_index(source, kv_head, channel)];+ if (channel == 0) {+ const size_t source_scale = shape.scale_index(source, kv_head);+ const size_t destination_scale = shape.scale_index(destination, kv_head);+ in_place_k_scales[destination_scale] = in_place_k_scales[source_scale];+ in_place_v_scales[destination_scale] = in_place_v_scales[source_scale];+ }+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_layout.h b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_layout.hnew file mode 100644index 000000000..b103146d4--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_layout.h@@ -0,0 +1,62 @@+#pragma once++#include <metal_stdlib>+#include "../common/defines.h"+#include "../generated/kv_layout.h"++// K: [block][head][fragment][lane][LANE_BYTES].+// V: [block][head][slice][chunk group][lane][LANE_BYTES].++namespace uzu::kv_layout {++// Lane bits: row[2], group[1], row[1], row[0], group[0].+METAL_FUNC uint mma_lane(const uint row_in_group, const uint channel_group) {+ return ((row_in_group >> 2) << 4) | ((channel_group >> 1) << 3) | (((row_in_group >> 1) & 1u) << 2) |+ ((row_in_group & 1u) << 1) | (channel_group & 1u);+}++struct Int8KVShape {+ uint head_dim;+ uint num_kv_heads;++ METAL_FUNC size_t block_region(const uint block, const uint kv_head) const {+ return (size_t(block) * num_kv_heads + kv_head) * (kv_layout::TILE_KEYS * head_dim);+ }++ METAL_FUNC size_t scale_index(const uint key, const uint kv_head) const {+ return (size_t(key / kv_layout::TILE_KEYS) * num_kv_heads + kv_head) * kv_layout::TILE_KEYS ++ key % kv_layout::TILE_KEYS;+ }++ METAL_FUNC size_t k_code_index(const uint key, const uint kv_head, const uint channel) const {+ const uint row = key % kv_layout::TILE_KEYS;+ const uint lane = mma_lane(row % kv_layout::GROUP_ROWS, channel % kv_layout::TILE_DIMS / kv_layout::LANE_COLS);+ const uint lane_offset = row / kv_layout::GROUP_ROWS * kv_layout::LANE_COLS + channel % kv_layout::LANE_COLS;+ return block_region(key / kv_layout::TILE_KEYS, kv_head) ++ channel / kv_layout::TILE_DIMS * kv_layout::FRAGMENT_BYTES + lane * kv_layout::LANE_BYTES + lane_offset;+ }++ METAL_FUNC size_t v_code_index(const uint key, const uint kv_head, const uint channel) const {+ const uint row = key % kv_layout::TILE_KEYS;+ const uint slice_channel = channel % kv_layout::SLICE_DIMS;+ const uint tile = slice_channel / kv_layout::TILE_DIMS;+ const uint tiles = kv_layout::SLICE_DIMS / kv_layout::TILE_DIMS;+ const uint half_rows = kv_layout::TILE_KEYS / 2;+ const uint lane =+ mma_lane(row % kv_layout::GROUP_ROWS, slice_channel % kv_layout::TILE_DIMS / kv_layout::LANE_COLS);+ const uint pair = tiles * (channel / kv_layout::SLICE_DIMS) + tiles / 2 * (row / half_rows) + tile / 2;+ const uint lane_offset = tile % 2 * (kv_layout::LANE_BYTES / 2) ++ row % half_rows / kv_layout::GROUP_ROWS * kv_layout::LANE_COLS ++ channel % kv_layout::LANE_COLS;+ return block_region(key / kv_layout::TILE_KEYS, kv_head) + pair * kv_layout::FRAGMENT_BYTES ++ lane * kv_layout::LANE_BYTES + lane_offset;+ }+};++METAL_FUNC float rotation_sign(const uint channel) {+ const uint word = kv_layout::ROTATION_SIGN_BITS[channel / kv_layout::WORD_BITS];+ const uint bit = channel % kv_layout::WORD_BITS;+ return (word >> bit) & 1u ? -1.0f : 1.0f;+}++} // namespace uzu::kv_layoutdiff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metalnew file mode 100644index 000000000..5e7c8220a--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal@@ -0,0 +1,123 @@+#include <metal_stdlib>+#include "../common/defines.h"+#include "../common/dsl.h"+#include "../hadamard_transform/hadamard_transform.h"+#include "kv_layout.h"+#include "rope.h"++namespace kv_layout = uzu::kv_layout;++template <typename ElementT, ushort VALUES_PER_LANE>+METAL_FUNC ElementT kv_quantize_row(const thread float* values, thread int* codes) {+ constexpr float CODE_MAX = static_cast<float>(metal::numeric_limits<int8_t>::max());+ float abs_max = 0.0f;+ METAL_PRAGMA_UNROLL+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {+ abs_max = metal::max(abs_max, metal::abs(values[value_index]));+ }+ abs_max = metal::simd_max(abs_max);+ const ElementT scale = ElementT(abs_max > 0.0f ? metal::precise::divide(abs_max, CODE_MAX) : 1.0f);+ METAL_PRAGMA_UNROLL+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {+ codes[value_index] =+ int(metal::clamp(metal::rint(metal::precise::divide(values[value_index], float(scale))), -CODE_MAX, CODE_MAX));+ }+ return scale;+}++template <typename ElementT, typename RopeT, uint HEAD_DIM>+VARIANTS(ElementT, bfloat)+VARIANTS(RopeT, float)+VARIANTS(HEAD_DIM, 256)+PUBLIC KERNEL(AttentionPrepareInt8)(+ const device ElementT* qkvg,+ device ElementT* queries OPTIONAL(!quantize_queries),+ device int8_t* q_codes OPTIONAL(quantize_queries),+ device ElementT* q_scales OPTIONAL(quantize_queries),+ device int8_t* k_codes,+ device ElementT* k_scales,+ device uchar* v_codes,+ device ElementT* v_scales,+ const device RopeT* cosines OPTIONAL(has_rope),+ const device RopeT* sines OPTIONAL(has_rope),+ const constant uint32_t& num_q_heads,+ const constant uint32_t& rope_dim OPTIONAL(has_rope),+ const constant uint32_t& kv_token_offset,+ const constant uint32_t& input_row_stride,+ const constant uint32_t& suffix_length,+ const constant uint32_t& padded_suffix,+ const bool has_rope SPECIALIZE,+ const bool quantize_queries SPECIALIZE,+ const uint32_t num_kv_heads SPECIALIZE,+ const uint32_t token_idx GROUPS(padded_suffix),+ const uint32_t head_idx GROUPS(num_q_heads + num_kv_heads * 2),+ const uint32_t lane THREADS(METAL_SIMD_SIZE)+) {+ constexpr ushort VALUES_PER_LANE = HEAD_DIM / METAL_SIMD_SIZE;+ const kv_layout::Int8KVShape shape{HEAD_DIM, num_kv_heads};+ const ushort lane_index = ushort(lane);+ const bool is_query = head_idx < num_q_heads;+ const bool is_key = !is_query && head_idx < num_q_heads + num_kv_heads;+ const uint32_t kv_head = is_key ? head_idx - num_q_heads : head_idx - num_q_heads - num_kv_heads;++ if (token_idx >= suffix_length) {+ if (is_query && quantize_queries) {+ const size_t row = size_t(head_idx) * padded_suffix + token_idx;+ METAL_PRAGMA_UNROLL+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {+ q_codes[row * HEAD_DIM + value_index * METAL_SIMD_SIZE + lane_index] = 0;+ }+ if (lane_index == 0) {+ q_scales[row] = ElementT(1.0f);+ }+ }+ return;+ }++ const device ElementT* head = qkvg + size_t(token_idx) * input_row_stride + size_t(head_idx) * HEAD_DIM;+ float values[VALUES_PER_LANE];+ METAL_PRAGMA_UNROLL+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {+ const uint32_t channel = value_index * METAL_SIMD_SIZE + lane_index;+ ElementT element = head[channel];+ if (has_rope && (is_query || is_key) && channel < rope_dim) {+ element = apply_rope(head, cosines, sines, token_idx, channel, rope_dim);+ }+ values[value_index] = float(element);+ }++ if (is_query && !quantize_queries) {+ const size_t row = size_t(head_idx) * suffix_length + token_idx;+ METAL_PRAGMA_UNROLL+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {+ queries[row * HEAD_DIM + value_index * METAL_SIMD_SIZE + lane_index] = ElementT(values[value_index]);+ }+ return;+ }++ METAL_PRAGMA_UNROLL+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {+ values[value_index] *= kv_layout::rotation_sign(value_index * METAL_SIMD_SIZE + lane_index);+ }+ simdgroup_hadamard_transform<VALUES_PER_LANE, METAL_SIMD_SIZE>(lane_index, values);++ int codes[VALUES_PER_LANE];+ const ElementT scale = kv_quantize_row<ElementT, VALUES_PER_LANE>(values, codes);++ const uint32_t key = kv_token_offset + token_idx;+ const size_t scale_index = is_query ? size_t(head_idx) * padded_suffix + token_idx : shape.scale_index(key, kv_head);+ METAL_PRAGMA_UNROLL+ for (ushort value_index = 0; value_index < VALUES_PER_LANE; ++value_index) {+ const uint32_t channel = value_index * METAL_SIMD_SIZE + lane_index;+ if (is_query) {+ q_codes[scale_index * HEAD_DIM + channel] = int8_t(codes[value_index]);+ } else if (is_key) {+ k_codes[shape.k_code_index(key, kv_head, channel)] = int8_t(codes[value_index]);+ } else {+ v_codes[shape.v_code_index(key, kv_head, channel)] = uchar(codes[value_index] + kv_layout::V_CODE_BIAS);+ }+ }+ if (lane_index == 0) {+ (is_query ? q_scales : is_key ? k_scales : v_scales)[scale_index] = scale;+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/rope.h b/crates/uzu-engine/src/backends/metal/kernel/attention/rope.hnew file mode 100644index 000000000..d0fbcbd4d--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/rope.h@@ -0,0 +1,26 @@+#pragma once++#include <metal_stdlib>+#include "../common/defines.h"++// TODO: very ugly and wasteful, clean up and optimize+template <typename ElementT, typename RopeT>+METAL_FUNC ElementT apply_rope(+ const device ElementT* head,+ const device RopeT* cosines,+ const device RopeT* sines,+ uint32_t batch_idx,+ uint32_t head_dim_idx,+ uint32_t rope_dim+) {+ const uint32_t half_rope_dim = rope_dim / 2;+ const uint32_t paired_idx =+ head_dim_idx < half_rope_dim ? head_dim_idx + half_rope_dim : head_dim_idx - half_rope_dim;+ const float input = float(head[head_dim_idx]);+ const float paired = float(head[paired_idx]);+ const float signed_paired = head_dim_idx < half_rope_dim ? -paired : paired;+ const float cos_val = float(cosines[batch_idx * rope_dim + head_dim_idx]);+ const float sin_val = float(sines[batch_idx * rope_dim + head_dim_idx]);++ return static_cast<ElementT>(input * cos_val + signed_paired * sin_val);+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/kv_layout.h b/crates/uzu-engine/src/backends/metal/kernel/generated/kv_layout.hnew file mode 100644index 000000000..de5139242--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/kv_layout.h@@ -0,0 +1,28 @@+// Auto-generated from gpu_types/kv_layout - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::kv_layout {+static constant constexpr uint32_t TILE_KEYS = 32;++static constant constexpr uint32_t TILE_DIMS = 16;++static constant constexpr uint32_t FRAGMENT_BYTES = TILE_KEYS * TILE_DIMS;++static constant constexpr uint32_t SLICE_DIMS = 64;++static constant constexpr int16_t V_CODE_BIAS = 128;++static constant constexpr uint32_t GROUP_ROWS = 8;++static constant constexpr uint32_t LANE_COLS = 4;++static constant constexpr uint32_t LANE_BYTES = TILE_KEYS / GROUP_ROWS * LANE_COLS;++static constant constexpr uint32_t WORD_BITS = 32;++static constant constexpr uint32_t ROTATION_SIGN_BITS[8] =+ {2654435769, 2246822507, 3266489909, 668265263, 374761393, 3210233709, 2496678331, 3605593784};+} // namespace uzu::kv_layoutdiff --git a/crates/uzu-engine/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/uzu-engine/src/backends/metal/kernel/hadamard_transform/hadamard_transform.hindex 7777464e8..4d6afe56b 100644--- a/crates/uzu-engine/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h+++ b/crates/uzu-engine/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h@@ -13,13 +13,47 @@ static_assert("Hadamard transform block size must match the Metal SIMD width");-static METAL_FUNC float simdgroup_hadamard_transform(ushort lane_index, float lane_value) {- for (ushort stride = 1; stride < HADAMARD_TRANSFORM_BLOCK_SIZE; stride <<= 1) {- float partner_lane_value = simd_shuffle_xor(lane_value, stride);- lane_value = (lane_index & stride) ? (partner_lane_value - lane_value) : (partner_lane_value + lane_value);+/// Normalized Walsh-Hadamard transform of `VALUES_PER_LANE` values in each of `LANES` lanes.+template <ushort VALUES_PER_LANE, ushort LANES>+static METAL_FUNC void simdgroup_hadamard_transform(ushort lane_index, thread float* values) {+ constexpr ushort SIZE = VALUES_PER_LANE * LANES;+ constexpr ushort CHUNK = VALUES_PER_LANE < 4 ? VALUES_PER_LANE : 4;+ using Chunk = metal::conditional_t<CHUNK == 1, float, vec<float, CHUNK>>;+ static_assert(+ LANES <= METAL_SIMD_SIZE && (LANES & (LANES - 1)) == 0,+ "lanes must be a power of two within a simdgroup"+ );+ static_assert(SIZE == 32 || SIZE == 64 || SIZE == 128 || SIZE == 256, "transform size must be 32, 64, 128 or 256");+ METAL_PRAGMA_UNROLL+ for (ushort stride = 1; stride < VALUES_PER_LANE; stride <<= 1) {+ METAL_PRAGMA_UNROLL+ for (ushort index = 0; index < VALUES_PER_LANE; ++index) {+ if (!(index & stride)) {+ const float value = values[index];+ const float partner_value = values[index + stride];+ values[index] = partner_value + value;+ values[index + stride] = value - partner_value;+ }+ }+ }+ METAL_PRAGMA_UNROLL+ for (ushort lane_stride = 1; lane_stride < LANES; lane_stride <<= 1) {+ METAL_PRAGMA_UNROLL+ for (ushort index = 0; index < VALUES_PER_LANE; index += CHUNK) {+ thread Chunk& chunk = *reinterpret_cast<thread Chunk*>(values + index);+ const Chunk partner = simd_shuffle_xor(chunk, lane_stride);+ chunk = (lane_index & lane_stride) ? (partner - chunk) : (partner + chunk);+ }+ }+ METAL_PRAGMA_UNROLL+ for (ushort index = 0; index < VALUES_PER_LANE; ++index) {+ values[index] /= sqrt(static_cast<float>(SIZE));}+}- return lane_value / sqrt(static_cast<float>(HADAMARD_TRANSFORM_BLOCK_SIZE));+static METAL_FUNC float simdgroup_hadamard_transform(ushort lane_index, float lane_value) {+ simdgroup_hadamard_transform<1, HADAMARD_TRANSFORM_BLOCK_SIZE>(lane_index, &lane_value);+ return lane_value;}template <typename T>@@ -37,22 +71,11 @@ static METAL_FUNC T simdgroup_output_random_hadamard_transform(ushort lane_indexUZU_CONST ushort HADAMARD_LANES_PER_BLOCK = HADAMARD_TRANSFORM_BLOCK_SIZE / HADAMARD_VECTOR_SIZE;static METAL_FUNC float4 simdgroup_hadamard_transform_vector(ushort lane_index, float4 lane_values) {- for (ushort stride = 1; stride < HADAMARD_VECTOR_SIZE; stride <<= 1) {- METAL_PRAGMA_UNROLL- for (ushort index = 0; index < HADAMARD_VECTOR_SIZE; ++index) {- if (!(index & stride)) {- const float value = lane_values[index];- const float partner_value = lane_values[index + stride];- lane_values[index] = partner_value + value;- lane_values[index + stride] = value - partner_value;- }- }- }- for (ushort lane_stride = 1; lane_stride < HADAMARD_LANES_PER_BLOCK; lane_stride <<= 1) {- const float4 partner_values = simd_shuffle_xor(lane_values, lane_stride);- lane_values = (lane_index & lane_stride) ? (partner_values - lane_values) : (partner_values + lane_values);- }- return lane_values / sqrt(static_cast<float>(HADAMARD_TRANSFORM_BLOCK_SIZE));+ simdgroup_hadamard_transform<HADAMARD_VECTOR_SIZE, HADAMARD_LANES_PER_BLOCK>(+ lane_index,+ reinterpret_cast<thread float*>(&lane_values)+ );+ return lane_values;}template <typename T>diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_prepare_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_prepare_test.rsindex 1f1bd14ff..d213fc357 100644--- a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_prepare_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_prepare_test.rs@@ -5,7 +5,7 @@ use crate::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::AttentionPrepareKernel,+ kernel::{AttentionPrepareInt8Kernel, AttentionPrepareKernel},},cpu::Cpu,},@@ -131,3 +131,68 @@ fn test_attention_prepare_uses_query_gate_row_stride() {assert_eq!(run_query_only::<B>(), expected, "query mismatch on {}", std::any::type_name::<B>());});}++#[derive(Debug, PartialEq)]+struct Int8Output {+ codes: [Vec<u8>; 3],+ scales: [Vec<bf16>; 3],+}++fn run_int8<B: Backend>() -> Int8Output {+ const DIM: usize = 256;+ const CACHE_TOKENS: usize = 64;+ let code_bytes = CACHE_TOKENS * DIM;+ let rows: Vec<bf16> = (0..3 * DIM).map(|i| bf16::from_f32((i as f32 * 0.37).sin())).collect();+ let context = B::Context::new().unwrap();+ let qkvg = create_buffer_with_data::<B, bf16>(&context, &rows);+ let mut q_codes = create_buffer::<B, i8>(&context, DIM);+ let mut q_scales = create_buffer::<B, bf16>(&context, 1);+ let mut k_codes = create_buffer_with_data::<B, i8>(&context, &vec![0; code_bytes]);+ let mut v_codes = create_buffer_with_data::<B, u8>(&context, &vec![0; code_bytes]);+ let mut k_scales = create_buffer_with_data::<B, bf16>(&context, &[bf16::ZERO; CACHE_TOKENS]);+ let mut v_scales = create_buffer_with_data::<B, bf16>(&context, &[bf16::ZERO; CACHE_TOKENS]);+ let kernel = <B::Kernels as Kernels>::AttentionPrepareInt8Kernel::new(+ &context,+ DataType::BF16,+ DataType::F32,+ DIM as u32,+ false,+ true,+ 1,+ )+ .unwrap();+ let mut command_buffer = context.create_command_buffer(None, None).unwrap();+ kernel.encode(+ &qkvg,+ None::<&mut B::GlobalBuffer>,+ Some(&mut q_codes),+ Some(&mut q_scales),+ &mut k_codes,+ &mut k_scales,+ &mut v_codes,+ &mut v_scales,+ None::<&B::GlobalBuffer>,+ None::<&B::GlobalBuffer>,+ 1,+ None,+ 33,+ (3 * DIM) as u32,+ 1,+ 1,+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();++ Int8Output {+ codes: [&q_codes, &k_codes, &v_codes].map(buffer_to_vec::<B, u8>),+ scales: [&q_scales, &k_scales, &v_scales].map(buffer_to_vec::<B, bf16>),+ }+}++#[uzu_test]+fn test_attention_prepare_int8() {+ let cpu = run_int8::<Cpu>();+ for_each_non_cpu_backend!(|B| {+ assert_eq!(cpu, run_int8::<B>(), "INT8 prepare mismatch on {}", std::any::type_name::<B>());+ });+}diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rsindex 8f6df0224..765aabb56 100644--- a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs@@ -8,7 +8,8 @@ use crate::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::kv_cache_update::Copy, kernel::KVCacheUpdateKernel,+ gpu_types::kv_cache_update::Copy,+ kernel::{KVCacheUpdateInt8Kernel, KVCacheUpdateKernel, kv_layout::Int8KVLayout},},cpu::Cpu,},@@ -238,3 +239,44 @@ fn test_no_copy_bf16() {fn test_large_bf16() {test_large_internal::<bf16>();}++#[uzu_test]+fn test_int8_copy() {+ let expected = get_int8_output::<Cpu>();+ for_each_non_cpu_backend!(|B| {+ assert_eq!(expected, get_int8_output::<B>(), "INT8 cache update mismatch on {}", std::any::type_name::<B>());+ });+}++fn get_int8_output<B: Backend>() -> ([Vec<u8>; 2], [Vec<bf16>; 2]) {+ let layout = Int8KVLayout::new(256, 2);+ let scale_count = 64 * layout.num_kv_heads as usize;+ let code_bytes = scale_count * layout.head_dim as usize;+ let k_codes: Vec<i8> = (0..code_bytes).map(|i| ((i * 37) ^ (i >> 8)) as i8).collect();+ let v_codes: Vec<u8> = (0..code_bytes).map(|i| ((i * 19) ^ (i >> 7)) as u8).collect();+ let k_scales: Vec<bf16> = (0..scale_count).map(|i| bf16::from_f32(i as f32 + 1.0)).collect();+ let v_scales: Vec<bf16> = (0..scale_count).map(|i| bf16::from_f32(i as f32 + 129.0)).collect();+ let copies = [Copy {+ source: 40,+ destination: 5,+ }];+ let context = B::Context::new().unwrap();+ let mut keys = create_buffer_with_data::<B, i8>(&context, &k_codes);+ let mut values = create_buffer_with_data::<B, u8>(&context, &v_codes);+ let mut key_scales = create_buffer_with_data::<B, bf16>(&context, &k_scales);+ let mut value_scales = create_buffer_with_data::<B, bf16>(&context, &v_scales);+ let kernel =+ <B::Kernels as Kernels>::KVCacheUpdateInt8Kernel::new(&context, layout.head_dim, layout.num_kv_heads).unwrap();+ let mut command_buffer = context.create_command_buffer(None, None).unwrap();+ kernel.encode(+ &mut keys,+ &mut values,+ &mut key_scales,+ &mut value_scales,+ &copies,+ copies.len() as u32,+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ ([&keys, &values].map(buffer_to_vec::<B, u8>), [&key_scales, &value_scales].map(buffer_to_vec::<B, bf16>))+}