A high-performance inference engine for AI models
use crate::{
backends::{
common::{
Backend, BufferMut, BufferRef, CpuBuffer, Kernels,
gpu_types::quantization::QuantizationMode,
kernel::{
ActivationTransform, TensorAddBiasKernel,
matmul::{Int8CodeLayout, MatmulA, MatmulArguments, MatmulKernel},
},
},
cpu::{
Cpu,
buffer::CpuBufferExt,
command_buffer::CpuCommandBufferEncoding,
context::CpuContext,
kernel::matmul::{
MatmulCpuError,
reference::{WeightData, load, store},
},
},
},
data_type::DataType,
utils::pointers::{SendPtr, SendPtrMut},
};
pub struct MatmulCpuKernel {
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
output_rht: ActivationTransform<Cpu>,
bias_add: <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel,
}
impl MatmulKernel for MatmulCpuKernel {
type Backend = Cpu;
type Error = MatmulCpuError;
type WeightsLayout = ();
fn new(
context: &CpuContext,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
) -> Result<Self, Self::Error> {
for data_type in [weights_data_type, input_data_type, output_data_type] {
if !matches!(data_type, DataType::BF16 | DataType::F32) {
return Err(MatmulCpuError::UnsupportedDataType(data_type));
}
}
let output_rht = ActivationTransform::output_rht(context, output_data_type, None, true)?;
let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(
context,
output_data_type,
weights_data_type,
true,
)?;
Ok(Self {
weights_data_type,
input_data_type,
output_data_type,
output_rht,
bias_add,
})
}
fn encode(
&mut self,
arguments: MatmulArguments<
'_,
Cpu,
impl BufferRef<Backend = Cpu>,
impl BufferRef<Backend = Cpu>,
impl BufferMut<Backend = Cpu>,
impl BufferRef<Backend = Cpu>,
>,
command_buffer: &mut CpuCommandBufferEncoding,
) -> Result<(), Self::Error> {
let MatmulArguments {
a,
b,
b_leading_dimension,
b_transpose,
output,
m,
n,
k,
gather_indices,
..
} = arguments;
let (mut d, ops) = output.into_contiguous(n, "CpuMatmul")?;
let output_scale = ops.ab_scale;
let bias_buffer = ops.bias;
let post_rht = ops.rht_factors;
let soft_cap = ops.soft_cap;
let m_u = m as usize;
let n_u = n as usize;
let k_u = k as usize;
#[derive(Clone, Copy)]
enum AData {
FullPrecision(SendPtr<u8>),
Int8 {
values: SendPtr<u8>,
scales: SendPtr<u8>,
group_size: usize,
code_layout: Int8CodeLayout,
},
}
let a_data = match a {
MatmulA::FullPrecision {
values,
offset,
} => {
let (values, range) = values.parts();
let byte_offset = range.start + offset;
AData::FullPrecision(SendPtr(
values.cpu_address().as_ptr().cast::<u8>().cast_const().wrapping_byte_add(byte_offset),
))
},
MatmulA::Int8Symmetric {
values,
scales,
group_sums: _,
quantization,
} => {
let a_group_size = quantization.scale_group_size();
let code_layout = quantization.code_layout();
let compatible = matches!(a_group_size, 32 | 64 | 128)
&& k.is_multiple_of(a_group_size)
&& b.quantized().is_some_and(|quantized| {
matches!(quantized.mode, QuantizationMode::U2 | QuantizationMode::U4 | QuantizationMode::U8)
&& (quantized.params.is_per_row() || matches!(quantized.group_size, 32 | 64 | 128))
});
if !compatible {
return Err(MatmulCpuError::IncompatibleA {
path: "CpuMatmul",
reason: "symmetric int8 activations require a supported 32/64/128 activation and weight group",
});
}
let (values, values_range) = values.parts();
let (scales, scales_range) = scales.parts();
AData::Int8 {
values: SendPtr(
values.cpu_address().as_ptr().cast::<u8>().cast_const().wrapping_byte_add(values_range.start),
),
scales: SendPtr(
scales.cpu_address().as_ptr().cast::<u8>().cast_const().wrapping_byte_add(scales_range.start),
),
group_size: a_group_size as usize,
code_layout,
}
},
MatmulA::PackedHalf {
..
}
| MatmulA::Trellis {
..
} => {
return Err(MatmulCpuError::UnsupportedLayout {
path: "CpuMatmul",
});
},
};
let bias_ptr = bias_buffer.map(|bias| SendPtr(bias.cpu_ptr().as_ptr().cast::<u8>().cast_const()));
let gather_ptr = gather_indices.map(|indices| {
let (indices, range) = indices.parts();
SendPtr(indices.cpu_address().as_ptr().cast::<u32>().cast_const().wrapping_byte_add(range.start))
});
let (d_buffer, d_range) = d.reborrow().parts();
let d_ptr = SendPtrMut(d_buffer.cpu_address().as_ptr().cast::<u8>().wrapping_byte_add(d_range.start));
let weight_data = WeightData::from_b(b, b_leading_dimension, b_transpose, k_u, n_u)?;
let weights_data_type = self.weights_data_type;
let input_data_type = self.input_data_type;
let output_data_type = self.output_data_type;
let bias_after_rht = post_rht.is_some();
command_buffer.push_command(move || {
unsafe {
for row in 0..m_u {
for col in 0..n_u {
// Gather remaps output column `col` to B-row `gather_indices[row * n + col]`.
let b_col = match gather_ptr {
Some(g) => *g.as_ptr().add(row * n_u + col) as usize,
None => col,
};
let mut accumulator = 0.0f32;
for inner in 0..k_u {
let a_value = match a_data {
AData::FullPrecision(ptr) => load(ptr.as_ptr(), input_data_type, row * k_u + inner),
AData::Int8 {
values,
scales,
group_size,
code_layout,
} => {
let groups = k_u.div_ceil(group_size);
let group = inner / group_size;
let code_index = code_layout.index(inner);
let q = *(values.as_ptr() as *const i8).add(row * k_u + code_index) as f32;
let scale = *(scales.as_ptr() as *const f32).add(row * groups + group);
q * scale
},
};
let b_value = match &weight_data {
WeightData::FullPrecision {
ptr,
leading_dimension,
transpose,
} => {
let index = if *transpose {
b_col * leading_dimension + inner
} else {
inner * leading_dimension + b_col
};
load(ptr.as_ptr(), weights_data_type, index)
},
WeightData::Quantized {
weights,
scales,
zero_points,
biases,
scale_strides,
bits,
group_size,
signed_codes,
} => {
let pack_factor = 32 / *bits;
let weight_linear_index = b_col * k_u + inner;
let word_index = weight_linear_index / pack_factor;
let code_index_in_word = weight_linear_index % pack_factor;
let bit_offset = code_index_in_word * *bits;
let weights_words = weights.as_ptr() as *const u32;
let word = weights_words.add(word_index).read_unaligned();
let code_mask = (1u32 << bits) - 1;
let mut weight_code = ((word >> bit_offset) & code_mask) as u8;
if *signed_codes {
weight_code ^= 1u8 << (bits - 1);
}
let quantized_value = f32::from(weight_code);
let group_index = inner / group_size;
let metadata_index = b_col * scale_strides.output_stride as usize
+ group_index * scale_strides.group_stride as usize;
let scale = load(scales.as_ptr(), weights_data_type, metadata_index);
let midpoint = (1u32 << (bits - 1)) as f32;
let zero_point = zero_points.as_ref().map(|(zp, strides)| {
let zero_point_index = b_col * strides.output_stride as usize
+ group_index * strides.group_stride as usize;
if *bits == 4 {
let byte_index = zero_point_index / 2;
let byte_value = *zp.as_ptr().add(byte_index);
if zero_point_index.is_multiple_of(2) {
(byte_value & 0x0F) as f32
} else {
((byte_value >> 4) & 0x0F) as f32
}
} else {
*zp.as_ptr().add(zero_point_index) as f32
}
});
let bias_term = if let Some(zp) = zero_point {
-scale * zp
} else if let Some(b) = biases {
load(b.as_ptr(), weights_data_type, metadata_index)
} else {
-scale * midpoint
};
scale * quantized_value + bias_term
},
};
accumulator += a_value * b_value;
}
let output_index = row * n_u + col;
let mut value = output_scale * accumulator;
if !bias_after_rht && let Some(bias) = bias_ptr {
value += load(bias.as_ptr(), weights_data_type, col);
}
if let Some(cap) = soft_cap {
value = cap * (value / cap).tanh();
}
store(d_ptr.as_ptr(), output_data_type, output_index, value);
}
}
}
});
if let Some(factors) = post_rht {
self.output_rht.encode_fp_in_place(d.reborrow(), factors, None, m, n, command_buffer);
if let Some(bias) = bias_buffer {
let output_length = m.checked_mul(n).expect("matmul output length must fit in u32");
self.bias_add.encode(
None::<&<Cpu as Backend>::ScratchBuffer>,
bias,
d,
n,
output_length,
command_buffer,
);
}
}
Ok(())
}
}