A high-performance inference engine for AI models
use std::sync::Arc;
use derive_more::Debug;
use thiserror::Error;
use crate::{
backends::common::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding,
kernel::{Kernels, NormalizationKernel},
},
config::normalization::{NormalizationConfig, UpcastMode},
data_type::DataType,
safetensors::{ParameterLoaderError, ParameterTree},
utils::size_for_shape::size_for_shape,
};
// TODO: clean up
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum PostLayerScalar {
None,
ScaleResidualSum(f32),
ScaleOutput(f32),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ShortcutMode {
None,
Copy,
Add,
}
#[derive(Debug, Error)]
pub enum NormalizationNewError<B: Backend> {
#[error("Backend error: {0}")]
Backend(#[source] B::Error),
#[error("Parameter loading error: {0}")]
Parameter(#[from] ParameterLoaderError<B>),
}
pub struct Normalization<B: Backend> {
epsilon: f32,
scale_offset: Option<f32>,
scales: Option<B::GlobalGpuBuffer>,
biases: Option<B::GlobalGpuBuffer>,
element_count: u32,
hadamard_factors: Option<Arc<B::GlobalGpuBuffer>>,
post_layer_scalar_value: f32,
input_data_type: DataType,
output_data_type: DataType,
kernel: <B::Kernels as Kernels>::NormalizationKernel,
}
impl<B: Backend> Normalization<B> {
pub async fn new(
element_count: u32,
input_data_type: DataType,
output_data_type: DataType,
hadamard_factors: Option<Arc<B::GlobalGpuBuffer>>,
shortcut_mode: ShortcutMode,
post_layer_scalar: PostLayerScalar,
config: &NormalizationConfig,
parameter_tree: &ParameterTree<'_, B>,
context: &B::Context,
) -> Result<Self, NormalizationNewError<B>> {
let (copy_to_shortcut, residual_add) = match shortcut_mode {
ShortcutMode::None => (false, false),
ShortcutMode::Copy => (true, false),
ShortcutMode::Add => (true, true),
};
let scales = if config.has_scale {
Some(parameter_tree.read_buffer("scales", &[element_count], DataType::F32).await?)
} else {
None
};
let biases = if config.has_biases {
Some(parameter_tree.read_buffer("biases", &[element_count], DataType::F32).await?)
} else {
None
};
let (scale_residual_sum, scale_output, post_layer_scalar_value) = match post_layer_scalar {
PostLayerScalar::None => (false, false, 1.0),
PostLayerScalar::ScaleResidualSum(value) => (true, false, value),
PostLayerScalar::ScaleOutput(value) => (false, true, value),
};
let kernel = <B::Kernels as Kernels>::NormalizationKernel::new(
context,
input_data_type,
DataType::F32,
output_data_type,
DataType::F32,
false,
config.subtract_mean,
config.upcast_mode == UpcastMode::FullLayer,
copy_to_shortcut,
residual_add,
hadamard_factors.is_some(),
scale_residual_sum,
scale_output,
biases.is_some(),
scales.is_some(),
)
.map_err(NormalizationNewError::Backend)?;
Ok(Self {
epsilon: config.epsilon,
scale_offset: config.scale_offset,
scales,
biases,
element_count,
hadamard_factors,
post_layer_scalar_value,
input_data_type,
output_data_type,
kernel,
})
}
pub fn encode(
&self,
input: impl BufferRef<Backend = B>,
row_offset: u32,
row_count: u32,
shortcut: Option<impl BufferMut<Backend = B>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
let row_size = size_for_shape(&[self.element_count], self.input_data_type);
let row_offset_bytes = row_offset as usize * row_size;
let shortcut = shortcut.map(|shortcut| shortcut.subrange_mut(row_offset_bytes..));
let mut output =
command_buffer.allocate_scratch_for_shape(&[row_count, self.element_count], self.output_data_type)?;
self.kernel.encode(
Some(input.subrange(row_offset_bytes..)),
self.scales.as_ref(),
self.biases.as_ref(),
&mut output,
shortcut,
self.hadamard_factors.as_deref(),
row_count,
self.element_count,
self.epsilon,
self.scale_offset.unwrap_or(0.0),
self.post_layer_scalar_value,
command_buffer,
);
Ok(output)
}
}