A high-performance inference engine for AI models
use crate::{
backends::common::{
Backend, BufferMut, BufferRef, CommandBuffer, Kernels,
gpu_types::{activation_transform::ActivationTransformOp, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},
kernel::{
ActivationTransformKernel,
matmul::{ActivationFormat, Int8CodeLayout},
},
},
data_type::DataType,
};
fn assert_row_width(element_count: u32) {
assert!(
element_count.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE),
"activation transform requires element_count ({element_count}) to be a multiple of {HADAMARD_TRANSFORM_BLOCK_SIZE}"
);
}
pub const ACTIVATION_SCALE_GROUP_SIZE: u32 = 128;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ActivationQuantization {
scale_group_size: u32,
sum_group_size: Option<u32>,
code_layout: Int8CodeLayout,
}
impl ActivationQuantization {
pub fn new(
scale_group_size: u32,
weight_group_size: u32,
emit_group_sums: bool,
code_layout: Int8CodeLayout,
) -> Option<Self> {
(scale_group_size == ACTIVATION_SCALE_GROUP_SIZE
&& (!emit_group_sums || matches!(weight_group_size, 32 | 64 | 128)))
.then_some(Self {
scale_group_size,
sum_group_size: emit_group_sums.then_some(weight_group_size),
code_layout,
})
}
pub const fn scale_group_size(self) -> u32 {
self.scale_group_size
}
pub const fn sum_group_size(self) -> Option<u32> {
self.sum_group_size
}
pub const fn code_layout(self) -> Int8CodeLayout {
self.code_layout
}
}
pub struct ActivationTransform<B: Backend> {
kernel: <B::Kernels as Kernels>::ActivationTransformKernel,
in_place: bool,
has_bias: bool,
format: ActivationFormat,
}
impl<B: Backend> ActivationTransform<B> {
fn new(
context: &B::Context,
data_type: DataType,
bias_data_type: Option<DataType>,
ops: ActivationTransformOp,
in_place: bool,
format: ActivationFormat,
) -> Result<Self, B::Error> {
let has_bias = bias_data_type.is_some();
let (codes_grouped_by_nibble, scale_group_size, sum_group_size, weight_bits) = match format {
ActivationFormat::Int8(q) => (
q.code_layout().is_grouped_by_nibble(),
q.scale_group_size(),
q.sum_group_size().unwrap_or(q.scale_group_size()),
0,
),
ActivationFormat::PackedHalf {
block_k,
bits,
} => (false, HADAMARD_TRANSFORM_BLOCK_SIZE, block_k, bits),
ActivationFormat::Bf16 | ActivationFormat::Trellis => {
(false, HADAMARD_TRANSFORM_BLOCK_SIZE, HADAMARD_TRANSFORM_BLOCK_SIZE, 0)
},
};
let kernel = <B::Kernels as Kernels>::ActivationTransformKernel::new(
context,
data_type,
bias_data_type.unwrap_or(data_type),
ops,
codes_grouped_by_nibble,
in_place,
scale_group_size,
sum_group_size,
weight_bits,
has_bias,
)?;
Ok(Self {
kernel,
in_place,
has_bias,
format,
})
}
pub fn input_rht(
context: &B::Context,
data_type: DataType,
in_place: bool,
) -> Result<Self, B::Error> {
Self::new(context, data_type, None, ActivationTransformOp::InputRht, in_place, ActivationFormat::Bf16)
}
pub fn output_rht(
context: &B::Context,
data_type: DataType,
bias_data_type: Option<DataType>,
in_place: bool,
) -> Result<Self, B::Error> {
Self::new(
context,
data_type,
bias_data_type,
ActivationTransformOp::OutputRht,
in_place,
ActivationFormat::Bf16,
)
}
pub fn quantize(
context: &B::Context,
data_type: DataType,
quantization: ActivationFormat,
) -> Result<Self, B::Error> {
let op = match quantization {
ActivationFormat::Int8(q) if q.sum_group_size().is_some() => ActivationTransformOp::QuantizeWithGroupSums,
ActivationFormat::Int8(_) => ActivationTransformOp::Quantize,
ActivationFormat::PackedHalf {
..
} => ActivationTransformOp::PackHalf,
ActivationFormat::Bf16 | ActivationFormat::Trellis => unreachable!(),
};
Self::new(context, data_type, None, op, false, quantization)
}
pub fn format(&self) -> ActivationFormat {
self.format
}
/// `input` and `output` must be distinct buffers.
pub fn encode_fp(
&self,
input: impl BufferRef<Backend = B>,
output: impl BufferMut<Backend = B>,
rht_factors: impl BufferRef<Backend = B>,
batch_size: u32,
element_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
assert!(self.format == ActivationFormat::Bf16 && !self.in_place && !self.has_bias);
assert_row_width(element_count);
self.kernel.encode(
Some(input),
Some(output),
None::<&B::GlobalGpuBuffer>,
None::<&mut B::ScratchBuffer>,
None::<&mut B::ScratchBuffer>,
None::<&mut B::ScratchBuffer>,
rht_factors,
batch_size,
element_count,
None::<&mut B::ScratchBuffer>,
command_buffer,
);
}
pub fn encode_fp_in_place(
&self,
data: impl BufferMut<Backend = B>,
rht_factors: impl BufferRef<Backend = B>,
bias: Option<&B::GlobalGpuBuffer>,
batch_size: u32,
element_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
assert!(self.format == ActivationFormat::Bf16 && self.in_place && self.has_bias == bias.is_some());
assert_row_width(element_count);
self.kernel.encode(
None::<&B::ScratchBuffer>,
Some(data),
bias,
None::<&mut B::ScratchBuffer>,
None::<&mut B::ScratchBuffer>,
None::<&mut B::ScratchBuffer>,
rht_factors,
batch_size,
element_count,
None::<&mut B::ScratchBuffer>,
command_buffer,
);
}
pub fn encode_quantize(
&self,
input: impl BufferRef<Backend = B>,
q_out: impl BufferMut<Backend = B>,
scales_out: impl BufferMut<Backend = B>,
group_sums_out: Option<impl BufferMut<Backend = B>>,
rht_factors: impl BufferRef<Backend = B>,
batch_size: u32,
element_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
let ActivationFormat::Int8(quantization) = self.format else {
panic!("quantized activation transform required")
};
assert_row_width(element_count);
assert!(
element_count.is_multiple_of(quantization.scale_group_size()),
"quantized activation row ({element_count}) must be a multiple of scale group ({})",
quantization.scale_group_size()
);
if let Some(group_size) = quantization.sum_group_size() {
assert!(element_count.is_multiple_of(group_size));
}
self.kernel.encode(
Some(input),
None::<&mut B::ScratchBuffer>,
None::<&B::GlobalGpuBuffer>,
Some(q_out),
Some(scales_out),
group_sums_out,
rht_factors,
batch_size,
element_count,
None::<&mut B::ScratchBuffer>,
command_buffer,
);
}
pub fn encode_pack_half(
&self,
input: impl BufferRef<Backend = B>,
output: impl BufferMut<Backend = B>,
rht_factors: impl BufferRef<Backend = B>,
batch_size: u32,
element_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
assert!(matches!(self.format, ActivationFormat::PackedHalf { .. }) && batch_size <= 8);
assert_row_width(element_count);
self.kernel.encode(
Some(input),
None::<&mut B::ScratchBuffer>,
None::<&B::GlobalGpuBuffer>,
None::<&mut B::ScratchBuffer>,
None::<&mut B::ScratchBuffer>,
None::<&mut B::ScratchBuffer>,
rht_factors,
batch_size,
element_count,
Some(output),
command_buffer,
);
}
}