A high-performance inference engine for AI models
use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
use crate::{
backends::{
common::gpu_types::{
d4s4, embedding::EmbeddingTableKind, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
quantization::QuantizationMode, quantization_method::QuantizationMethod,
},
cpu::kernel::activation_transform::hadamard_transform,
},
data_type::DataTypeRepr,
};
#[kernel(InputEmbeddingLookup)]
#[variants(T, f32, bf16)]
pub fn input_embedding_lookup<T: DataTypeRepr + Float>(
token_ids: *const u32,
values: *const u8,
#[optional(table_kind != EmbeddingTableKind::Dense)] scales: Option<*const T>,
#[optional(quantization_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,
#[optional(quantization_method == QuantizationMethod::ScaleBias)] biases: Option<*const T>,
#[optional(use_hadamard)] hadamard_factors: Option<*const i32>,
#[optional(table_kind == EmbeddingTableKind::D4S4)] ladder_indices: Option<*const u8>,
#[optional(table_kind == EmbeddingTableKind::D4S4)] ladder: Option<*const f16>,
#[optional(table_kind == EmbeddingTableKind::D4S4)] codebook: Option<*const i8>,
output: *mut T,
batch_size: u32,
vocab_size: u32,
model_dim: u32,
input_scale: f32,
#[specialize] table_kind: EmbeddingTableKind,
#[specialize]
#[optional(table_kind == EmbeddingTableKind::Quantized)]
group_size: Option<u32>,
#[specialize]
#[optional(table_kind == EmbeddingTableKind::Quantized)]
quantization_mode: Option<QuantizationMode>,
#[specialize]
#[optional(table_kind == EmbeddingTableKind::Quantized)]
quantization_method: Option<QuantizationMethod>,
#[specialize] use_hadamard: bool,
) {
let factors = if use_hadamard {
hadamard_factors
} else {
None
};
let dim = model_dim as usize;
let quantization =
if let (Some(group_size), Some(mode), Some(method)) = (group_size, quantization_mode, quantization_method) {
let num_groups = dim.div_ceil(group_size as usize);
let weights_stride = dim / mode.packing_divisor() as usize;
Some((group_size, mode, method, num_groups, weights_stride))
} else {
None
};
let dense_scale = T::from(input_scale).unwrap();
for batch in 0..batch_size as usize {
let row = unsafe { std::slice::from_raw_parts_mut(output.add(batch * dim), dim) };
let token = unsafe { *token_ids.add(batch) };
if token >= vocab_size {
row.fill(T::zero());
continue;
}
let token = token as usize;
let load = |column: usize| -> f32 {
match table_kind {
EmbeddingTableKind::Dense => unsafe {
(*(values as *const T).add(token * dim + column) * dense_scale).to_f32().unwrap()
},
EmbeddingTableKind::Quantized => {
let (group_size, quantization_mode, quantization_method, num_groups, weights_stride) =
quantization.expect("quantization settings");
let scales = scales.expect("quantized lookup requires scales");
let group = column / group_size as usize;
let index = token * num_groups + group;
let scale = unsafe { (*scales.add(index)).to_f32().unwrap() };
let offset = token * weights_stride;
let code = match quantization_mode {
QuantizationMode::U2 => unreachable!("2-bit weights are not embedding tables"),
QuantizationMode::U4 => read_u4(values, 2 * offset + column) as f32,
QuantizationMode::I8 => (unsafe { *(values as *const i8).add(offset + column) }) as f32,
QuantizationMode::U8 => (unsafe { *values.add(offset + column) }) as f32,
};
let bias = match quantization_method {
QuantizationMethod::ScaleBias => unsafe {
(*biases.expect("quantized lookup requires biases").add(index)).to_f32().unwrap()
},
QuantizationMethod::ScaleZeroPoint => {
let zero_points = zero_points.expect("quantized lookup requires zero points");
let zero_point = if quantization_mode == QuantizationMode::U4 {
read_u4(zero_points, 2 * token * num_groups.div_ceil(2) + group)
} else {
unsafe { *zero_points.add(index) }
};
-scale * zero_point as f32
},
QuantizationMethod::ScaleSymmetric => {
let midpoint = if quantization_mode == QuantizationMode::U4 {
8.0
} else {
128.0
};
-scale * midpoint
},
};
// Ordinary quantization rounds to T before the transform.
T::from((scale * code + bias) * input_scale).unwrap().to_f32().unwrap()
},
EmbeddingTableKind::D4S4 => {
let row_scales = scales.expect("D4S4 lookup requires row scales");
let ladder_indices = ladder_indices.expect("D4S4 lookup requires ladder indices");
let ladder = ladder.expect("D4S4 lookup requires a ladder");
let codebook = codebook.expect("D4S4 lookup requires a codebook");
let values_per_code = d4s4::VALUES_PER_CODE as usize;
let columns_per_scale = d4s4::COLUMNS_PER_LADDER_SCALE as usize;
let ladder_index =
read_u4(ladder_indices, token * (dim / columns_per_scale) + column / columns_per_scale);
let code =
unsafe { *values.add(token * (dim / values_per_code) + column / values_per_code) } as usize;
let point = unsafe { *codebook.add(values_per_code * code + column % values_per_code) };
let row_scale = unsafe { (*row_scales.add(token)).to_f32().unwrap() };
let step = unsafe { (*ladder.add(ladder_index as usize)).to_f32() };
row_scale * step * point as f32 * input_scale
},
}
};
if let Some(factors) = factors {
for block_start in (0..row.len()).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {
let mut block: [f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize] =
std::array::from_fn(|lane| load(block_start + lane));
hadamard_transform(&mut block);
for (lane, value) in block.into_iter().enumerate() {
row[block_start + lane] =
T::from(value * unsafe { *factors.add(block_start + lane) } as f32).unwrap();
}
}
} else {
for (column, value) in row.iter_mut().enumerate() {
*value = T::from(load(column)).unwrap();
}
}
}
}
fn read_u4(
values: *const u8,
nibble: usize,
) -> u8 {
(unsafe { *values.add(nibble / 2) } >> (4 * (nibble % 2))) & 15
}