Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Dflash v2 (#814)

CCYehde0e0987b2ee
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/convolution/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/convolution/mod.rs
new file mode 100644
index 000000000..c74404d25
--- /dev/null
+++ b/crates/uzu-engine/src/backends/cpu/kernel/convolution/mod.rs
@@ -0,0 +1 @@
+pub mod separable_causal_conv;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs b/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs
new file mode 100644
index 000000000..aaa9a4190
--- /dev/null
+++ b/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs
@@ -0,0 +1,58 @@
+use half::bf16;
+use num_traits::Float;
+use uzu_engine_macros::kernel;
+
+use crate::array::ArrayElement;
+
+#[kernel(SeparableCausalConv)]
+#[variants(T, bf16)]
+pub fn separable_causal_conv<T: ArrayElement + Float>(
+ input: *const T,
+ coefficient_deltas: *const T,
+ weights: *const T,
+ #[optional(has_bias)] bias: Option<*const T>,
+ output: *mut T,
+ sequence_length: u32,
+ coefficient_row_stride: u32,
+ #[specialize] model_dim: u32,
+ #[specialize] kernel_size: u32,
+ #[specialize] group_size: u32,
+ #[specialize] has_bias: bool,
+) {
+ let sequence_length = sequence_length as usize;
+ let coefficient_row_stride = coefficient_row_stride as usize;
+ let model_dim = model_dim as usize;
+ let kernel_size = kernel_size as usize;
+ let group_size = group_size as usize;
+
+ let num_groups = model_dim / group_size;
+
+ for token in 0..sequence_length {
+ for channel in 0..model_dim {
+ let group = channel / group_size;
+ let mut output_value = if has_bias {
+ unsafe { (*bias.unwrap().add(channel)).to_f32().unwrap() }
+ } else {
+ 0.0
+ };
+
+ for tokens_back in 0..kernel_size.min(token + 1) {
+ let input_token = token - tokens_back;
+ let input_index = input_token * model_dim + channel;
+ let weight_index = channel * kernel_size + (kernel_size - 1 - tokens_back);
+ let coefficient_index = token * coefficient_row_stride + tokens_back * num_groups + group;
+
+ let input_value = unsafe { (*input.add(input_index)).to_f32().unwrap() };
+ let base_weight = unsafe { (*weights.add(weight_index)).to_f32().unwrap() };
+ let coefficient_delta = unsafe { (*coefficient_deltas.add(coefficient_index)).to_f32().unwrap() };
+
+ output_value += input_value * (base_weight + coefficient_delta);
+ }
+
+ let output_index = token * model_dim + channel;
+ unsafe {
+ *output.add(output_index) = T::from(output_value).unwrap();
+ }
+ }
+ }
+}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
index 5cbd2a504..a1a3f0467 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
@@ -6,6 +6,7 @@ use crate::backends::{
mod activation;
pub(crate) mod activation_transform;
mod attention;
+mod convolution;
mod embedding;
mod gated_act_mul;
mod gdn;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/convolution/separable_causal_conv.metal b/crates/uzu-engine/src/backends/metal/kernel/convolution/separable_causal_conv.metal
new file mode 100644
index 000000000..ea9644820
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/convolution/separable_causal_conv.metal
@@ -0,0 +1,58 @@
+#include <metal_stdlib>
+#include "../common/defines.h"
+#include "../common/dsl.h"
+
+using namespace metal;
+
+#define CHANNELS_PER_BLOCK 4u
+#define TOKENS_PER_THREADGROUP 1u
+#define CHANNEL_BLOCKS_PER_THREADGROUP 128u
+
+template<typename T>
+VARIANTS(T, bfloat)
+PUBLIC KERNEL(SeparableCausalConv)(
+ device const T* input,
+ device const T* coefficient_deltas,
+ device const T* weights,
+ device const T* bias OPTIONAL(has_bias),
+ device T* output,
+ constant uint& sequence_length,
+ constant uint& coefficient_row_stride,
+ const uint model_dim SPECIALIZE,
+ const uint kernel_size SPECIALIZE,
+ const uint group_size SPECIALIZE,
+ const bool has_bias SPECIALIZE,
+ uint token AXIS(sequence_length, TOKENS_PER_THREADGROUP),
+ uint channel_block_index AXIS((model_dim + CHANNELS_PER_BLOCK - 1u) / CHANNELS_PER_BLOCK, CHANNEL_BLOCKS_PER_THREADGROUP)
+){
+ using ValueBlock = vec<T, CHANNELS_PER_BLOCK>;
+ using AccumulatorBlock = vec<float, CHANNELS_PER_BLOCK>;
+
+ const uint channel = channel_block_index * CHANNELS_PER_BLOCK;
+ const uint num_groups = model_dim / group_size;
+ const uint available_tokens = min(kernel_size, token + 1u);
+ const uint group = channel / group_size;
+ AccumulatorBlock output_value = 0.0f;
+ if (has_bias) {
+ output_value = AccumulatorBlock(*reinterpret_cast<const device ValueBlock*>(bias + channel));
+ }
+
+ METAL_PRAGMA_UNROLL
+ for (uint tokens_back = 0; tokens_back < available_tokens; ++tokens_back) {
+ const uint input_token = token - tokens_back;
+ const uint input_index = input_token * model_dim + channel;
+ const uint stored_weight = kernel_size - 1u - tokens_back;
+ const uint coefficient_index = token * coefficient_row_stride + tokens_back * num_groups + group;
+ const AccumulatorBlock weight = AccumulatorBlock(
+ float(weights[(channel + 0u) * kernel_size + stored_weight]),
+ float(weights[(channel + 1u) * kernel_size + stored_weight]),
+ float(weights[(channel + 2u) * kernel_size + stored_weight]),
+ float(weights[(channel + 3u) * kernel_size + stored_weight])
+ );
+
+ output_value += (weight + float(coefficient_deltas[coefficient_index])) *
+ AccumulatorBlock(*reinterpret_cast<const device ValueBlock*>(input + input_index));
+ }
+
+ *reinterpret_cast<device ValueBlock*>(output + token * model_dim + channel) = ValueBlock(output_value);
+}
diff --git a/crates/uzu-engine/src/config/transformer_layer.rs b/crates/uzu-engine/src/config/transformer_layer.rs
index 91cd71cb5..3944b8b95 100644
--- a/crates/uzu-engine/src/config/transformer_layer.rs
+++ b/crates/uzu-engine/src/config/transformer_layer.rs
@@ -1,17 +1,31 @@
use uzu_engine_macros::uzu_config;
use crate::config::{
- mlp::AnyMLPConfig, normalization::NormalizationConfig, per_layer_embedding::PLELayerConfig, rope::AnyRoPEConfig,
- token_mixer::AnyTokenMixerConfig,
+ linear::LinearConfig,
+ mlp::AnyMLPConfig,
+ normalization::NormalizationConfig,
+ per_layer_embedding::PLELayerConfig,
+ rope::AnyRoPEConfig,
+ token_mixer::{AnyTokenMixerConfig, convolutions::SeparableCausalConvConfig},
};
+#[uzu_config]
+pub struct TransformerLayerConvConfig {
+ pub conv_config: SeparableCausalConvConfig,
+ pub kernel_projection_config: LinearConfig,
+ pub conv_kernel_size: u32,
+ pub conv_group_size: u32,
+}
+
#[uzu_config]
pub struct TransformerLayerConfig {
pub pre_mixer_norm_config: Option<NormalizationConfig>,
pub mixer_config: AnyTokenMixerConfig,
+ pub mixer_conv_config: Option<TransformerLayerConvConfig>,
pub post_mixer_norm_config: Option<NormalizationConfig>,
pub pre_mlp_norm_config: NormalizationConfig,
pub mlp_config: AnyMLPConfig,
+ pub mlp_conv_config: Option<TransformerLayerConvConfig>,
pub post_mlp_norm_config: Option<NormalizationConfig>,
pub hidden_dim: Option<u32>,
pub ple_config: Option<PLELayerConfig>,
diff --git a/crates/uzu-engine/src/encodable_block/convolution.rs b/crates/uzu-engine/src/encodable_block/convolution.rs
new file mode 100644
index 000000000..437bee19d
--- /dev/null
+++ b/crates/uzu-engine/src/encodable_block/convolution.rs
@@ -0,0 +1,99 @@
+use thiserror::Error;
+
+use crate::{
+ array::size_for_shape,
+ backends::common::{
+ Allocation, Backend, Encoder,
+ kernel::{Kernels, SeparableCausalConvKernel},
+ },
+ config::token_mixer::convolutions::SeparableCausalConvConfig,
+ data_type::DataType,
+ parameters::{ParameterLoaderError, ParameterTree},
+};
+
+#[derive(Debug, Error)]
+pub enum ConvolutionNewError<B: Backend> {
+ #[error("Backend error: {0}")]
+ Backend(#[source] B::Error),
+ #[error("Parameter loading error: {0}")]
+ Parameter(#[from] ParameterLoaderError<B>),
+}
+
+pub struct SeparableCausalConv<B: Backend> {
+ model_dim: u32,
+ data_type: DataType,
+ weights: Allocation<B>,
+ biases: Option<Allocation<B>>,
+ kernel: <B::Kernels as Kernels>::SeparableCausalConvKernel,
+}
+
+impl<B: Backend> SeparableCausalConv<B> {
+ pub fn new(
+ model_dim: u32,
+ kernel_size: u32,
+ group_size: u32,
+ data_type: DataType,
+ config: &SeparableCausalConvConfig,
+ parameter_tree: &ParameterTree<B>,
+ context: &B::Context,
+ ) -> Result<Self, ConvolutionNewError<B>> {
+ assert!(model_dim.is_multiple_of(4));
+ assert!(group_size.is_multiple_of(4));
+ assert!(model_dim.is_multiple_of(group_size));
+ assert_eq!(data_type, DataType::BF16, "SeparableCausalConv only supports BF16");
+
+ let weights =
+ parameter_tree.leaf("weights")?.validate(&[model_dim, kernel_size], DataType::BF16)?.read_allocation()?;
+
+ let biases = config
+ .has_biases
+ .then(|| parameter_tree.leaf("biases")?.validate(&[model_dim], DataType::BF16)?.read_allocation())
+ .transpose()?;
+
+ let kernel = <B::Kernels as Kernels>::SeparableCausalConvKernel::new(
+ context,
+ data_type,
+ model_dim,
+ kernel_size,
+ group_size,
+ config.has_biases,
+ )
+ .map_err(ConvolutionNewError::Backend)?;
+
+ Ok(Self {
+ model_dim,
+ data_type,
+ weights,
+ biases,
+ kernel,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ input: &Allocation<B>,
+ coefficient_deltas: &Allocation<B>,
+ coefficient_row_stride: u32,
+ coefficient_column_offset: u32,
+ sequence_length: u32,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ encoder.push_debug_group("SeparableCausalConv");
+
+ let mut output = encoder.allocate_scratch_for_shape(&[sequence_length, self.model_dim], self.data_type)?;
+ let coefficients_offset_bytes = size_for_shape(&[coefficient_column_offset], self.data_type);
+ self.kernel.encode(
+ input,
+ (coefficient_deltas, coefficients_offset_bytes),
+ &self.weights,
+ self.biases.as_ref(),
+ &mut output,
+ sequence_length,
+ coefficient_row_stride,
+ encoder,
+ );
+
+ encoder.pop_debug_group();
+ Ok(output)
+ }
+}
diff --git a/crates/uzu-engine/src/encodable_block/mod.rs b/crates/uzu-engine/src/encodable_block/mod.rs
index 909d783ea..9f6c79366 100644
--- a/crates/uzu-engine/src/encodable_block/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mod.rs
@@ -1,5 +1,6 @@
pub mod batch_topology;
pub mod classifier;
+pub mod convolution;
pub mod decoder;
pub mod dflash;
pub mod embedding;
diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
index 180c87113..4091001c2 100644
--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
@@ -1,11 +1,13 @@
use thiserror::Error;
use crate::{
- backends::common::{Allocation, Backend, Encoder},
- config::transformer_layer::TransformerLayerConfig,
+ backends::common::{Allocation, Backend, Encoder, kernel::ActivationTransform},
+ config::transformer_layer::{TransformerLayerConfig, TransformerLayerConvConfig},
data_type::DataType,
encodable_block::{
batch_topology::BatchTopology,
+ convolution::{ConvolutionNewError, SeparableCausalConv},
+ linear::{Linear, LinearBlockError},
mixer::{Mixer, MixerNewError, MixerState, attention::rope::PrecalculatedRoPE},
mlp::{Mlp, MlpBlockError},
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
@@ -17,6 +19,8 @@ use crate::{
#[derive(Debug, Error)]
pub enum TransformerLayerError<B: Backend> {
+ #[error("Backend error: {0}")]
+ Backend(#[source] B::Error),
#[error("Parameter loader error: {0}")]
ParameterLoader(#[from] ParameterLoaderError<B>),
#[error("Mixer error: {0}")]
@@ -25,6 +29,10 @@ pub enum TransformerLayerError<B: Backend> {
MlpBlock(#[from] MlpBlockError<B>),
#[error("Normalization error: {0}")]
Normalization(#[from] NormalizationNewError<B>),
+ #[error("Convolution error: {0}")]
+ Convolution(#[from] ConvolutionNewError<B>),
+ #[error("Linear error: {0}")]
+ Linear(#[from] LinearBlockError<B>),
#[error("Layer {layer_index} sets post_layer_scalar but has no post_mlp_norm")]
PostLayerScalarWithoutPostMlpNorm {
layer_index: u32,
@@ -33,20 +41,120 @@ pub enum TransformerLayerError<B: Backend> {
MissingPreMixerNormConfig,
}
+struct TransformerLayerConv<B: Backend> {
+ model_dim: u32,
+ pre_conv: SeparableCausalConv<B>,
+ kernel_projection: Box<dyn Linear<B>>,
+ post_conv: SeparableCausalConv<B>,
+ input_rht: Option<(ActivationTransform<B>, Allocation<B>)>,
+ coefficient_count: u32,
+}
+
// TODO: saner shortcut
pub struct TransformerLayer<B: Backend> {
pub layer_index: u32,
- pub pre_mixer_norm: Option<Normalization<B>>,
pub kv_source_layer_index: Option<u32>,
+ pub pre_mixer_norm: Option<Normalization<B>>,
+ mixer_conv: Option<TransformerLayerConv<B>>,
pub mixer: Box<dyn Mixer<B>>,
pub post_mixer_norm: Option<Normalization<B>>,
pub pre_mlp_norm: Normalization<B>,
+ mlp_conv: Option<TransformerLayerConv<B>>,
pub mlp: Box<dyn Mlp<B>>,
pub post_mlp_norm: Option<Normalization<B>>,
pub ple_projection: Option<PerLayerEmbeddingProjection<B>>,
}
+impl<B: Backend> TransformerLayerConv<B> {
+ fn new(
+ context: &B::Context,
+ model_dim: u32,
+ config: &TransformerLayerConvConfig,
+ parameter_tree: &ParameterTree<B>,
+ data_type: DataType,
+ input_hadamard_factors: Option<Allocation<B>>,
+ ) -> Result<Self, TransformerLayerError<B>> {
+ let pre_conv = SeparableCausalConv::new(
+ model_dim,
+ config.conv_kernel_size,
+ config.conv_group_size,
+ data_type,
+ &config.conv_config,
+ &parameter_tree.subtree("pre_conv"),
+ context,
+ )?;
+ let post_conv = SeparableCausalConv::new(
+ model_dim,
+ config.conv_kernel_size,
+ config.conv_group_size,
+ data_type,
+ &config.conv_config,
+ &parameter_tree.subtree("post_conv"),
+ context,
+ )?;
+
+ let coefficient_count = config.conv_kernel_size * (model_dim / config.conv_group_size);
+ let projection_dim = coefficient_count * 2;
+ let kernel_projection = <dyn Linear<B>>::new(
+ model_dim,
+ [projection_dim],
+ false,
+ context,
+ data_type,
+ &parameter_tree.subtree("kernel_projection"),
+ )?;
+
+ let input_rht = input_hadamard_factors
+ .map(|factors| ActivationTransform::input_rht(context, data_type, true).map(|kernel| (kernel, factors)))
+ .transpose()
+ .map_err(TransformerLayerError::Backend)?;
+
+ Ok(Self {
+ model_dim,
+ pre_conv,
+ kernel_projection,
+ post_conv,
+ input_rht,
+ coefficient_count,
+ })
+ }
+
+ fn encode_pre_convolution(
+ &self,
+ input: &Allocation<B>,
+ sequence_length: u32,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {
+ let mut projection_input = encoder.allocate_scratch(input.size())?;
+ encoder.encode_copy(input, .., &mut projection_input, ..);
+ let coefficients = self.kernel_projection.encode(projection_input, sequence_length, encoder)?;
+ let mut output =
+ self.pre_conv.encode(input, &coefficients, 2 * self.coefficient_count, 0, sequence_length, encoder)?;
+ if let Some((transform, factors)) = &self.input_rht {
+ transform.encode_fp_in_place(&mut output, factors, sequence_length, self.model_dim, encoder);
+ }
+ Ok((output, coefficients))
+ }
+
+ fn encode_post_convolution(
+ &self,
+ input: &Allocation<B>,
+ coefficients: &Allocation<B>,
+ sequence_length: u32,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ self.post_conv.encode(
+ input,
+ coefficients,
+ 2 * self.coefficient_count,
+ self.coefficient_count,
+ sequence_length,
+ encoder,
+ )
+ }
+}
+
impl<B: Backend> TransformerLayer<B> {
pub fn new(
context: &B::Context,
@@ -92,6 +200,21 @@ impl<B: Backend> TransformerLayer<B> {
context,
)?;
+ let (mixer_conv, mixer_hadamard_factors) = match &layer_config.mixer_conv_config {
+ Some(config) => (
+ Some(TransformerLayerConv::new(
+ context,
+ model_dim,
+ config,
+ &parameter_tree.subtree("mixer_conv"),
+ data_type,
+ mixer_hadamard_factors,
+ )?),
+ None,
+ ),
+ None => (None, mixer_hadamard_factors),
+ };
+
let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.pre_mixer_norm_config {
Some(Normalization::new(
model_dim,
@@ -138,6 +261,21 @@ impl<B: Backend> TransformerLayer<B> {
data_type,
)?;
+ let (mlp_conv, mlp_input_hadamard_factors) = match &layer_config.mlp_conv_config {
+ Some(config) => (
+ Some(TransformerLayerConv::new(
+ context,
+ model_dim,
+ config,
+ &parameter_tree.subtree("mlp_conv"),
+ data_type,
+ mlp_input_hadamard_factors,
+ )?),
+ None,
+ ),
+ None => (None, mlp_input_hadamard_factors),
+ };
+
let pre_mlp_norm = Normalization::new(
model_dim,
mlp_input_hadamard_factors,
@@ -180,11 +318,13 @@ impl<B: Backend> TransformerLayer<B> {
Ok(Self {
layer_index,
- pre_mixer_norm,
kv_source_layer_index: layer_config.kv_source_layer_index,
+ pre_mixer_norm,
+ mixer_conv,
mixer,
post_mixer_norm,
pre_mlp_norm,
+ mlp_conv,
mlp,
post_mlp_norm,
ple_projection,
@@ -203,7 +343,7 @@ impl<B: Backend> TransformerLayer<B> {
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group(&format!("transformer layer {}", self.layer_index));
- let hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {
+ let mut hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {
pre_mixer_norm.encode(&input, 0, batch_dim.size(), Some(shortcut), encoder)?
} else {
assert!(self.layer_index == 0);
@@ -211,8 +351,21 @@ impl<B: Backend> TransformerLayer<B> {
input
};
+ let mixer_coefficients = if let Some(convolution) = &self.mixer_conv {
+ let (output, coefficients) = convolution.encode_pre_convolution(&hidden, batch_dim.size(), encoder)?;
+ hidden = output;
+ Some(coefficients)
+ } else {
+ None
+ };
+
// TODO: In prefill outside of sampling suffix in last layer part of mixer (ie out projection) and everything after is dead code
- let mut hidden = self.mixer.encode(hidden, precalculated_rope, batch_dim, state, encoder)?;
+ hidden = self.mixer.encode(hidden, precalculated_rope, batch_dim, state, encoder)?;
+
+ if let Some(coefficients) = mixer_coefficients {
+ let convolution = self.mixer_conv.as_ref().expect("mixer convolution required");
+ hidden = convolution.encode_post_convolution(&hidden, &coefficients, batch_dim.size(), encoder)?;
+ }
if let Some(post_mixer_norm) = &self.post_mixer_norm {
hidden = post_mixer_norm.encode(&hidden, 0, batch_dim.size(), None, encoder)?;
@@ -220,8 +373,21 @@ impl<B: Backend> TransformerLayer<B> {
hidden = self.pre_mlp_norm.encode(&hidden, 0, batch_dim.size(), Some(shortcut), encoder)?;
+ let mlp_coefficients = if let Some(convolution) = &self.mlp_conv {
+ let (output, coefficients) = convolution.encode_pre_convolution(&hidden, batch_dim.size(), encoder)?;
+ hidden = output;
+ Some(coefficients)
+ } else {
+ None
+ };
+
hidden = self.mlp.encode(hidden, batch_dim.size(), encoder)?;
+ if let Some(coefficients) = mlp_coefficients {
+ let convolution = self.mlp_conv.as_ref().expect("mlp convolution required");
+ hidden = convolution.encode_post_convolution(&hidden, &coefficients, batch_dim.size(), encoder)?;
+ }
+
if let Some(post_mlp_norm) = &self.post_mlp_norm {
hidden = post_mlp_norm.encode(&hidden, 0, batch_dim.size(), None, encoder)?;
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/mod.rs
index e56e9b0d9..6194246f5 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/mod.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/mod.rs
@@ -10,6 +10,7 @@ mod moe;
mod normalization_test;
mod pooling;
mod radix_top_k_small_test;
+mod separable_causal_conv_test;
mod short_conv;
mod ssm;
mod tensor_add_bias_test;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs
new file mode 100644
index 000000000..3cd771618
--- /dev/null
+++ b/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs
@@ -0,0 +1,72 @@
+use half::bf16;
+use uzu_engine_macros::uzu_test;
+
+use crate::{
+ array::ArrayElement,
+ backends::{
+ common::{Allocation, Backend, Context, Encoder, Kernels, kernel::SeparableCausalConvKernel},
+ cpu::Cpu,
+ },
+ tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},
+};
+
+fn run_kernel<B: Backend>() -> Vec<bf16> {
+ const SEQUENCE_LENGTH: u32 = 2;
+ const MODEL_DIM: u32 = 16;
+ const KERNEL_SIZE: u32 = 2;
+ const GROUP_SIZE: u32 = 16;
+ const COEFFICIENT_ROW_STRIDE: u32 = 2 * KERNEL_SIZE * (MODEL_DIM / GROUP_SIZE);
+
+ let input = [
+ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0,
+ 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0,
+ ]
+ .map(bf16::from_f32);
+ let weights = [
+ 10.0, 1.0, 20.0, 2.0, 30.0, 3.0, 40.0, 4.0, 50.0, 5.0, 60.0, 6.0, 70.0, 7.0, 80.0, 8.0, 90.0, 9.0, 100.0, 10.0,
+ 110.0, 11.0, 120.0, 12.0, 130.0, 13.0, 140.0, 14.0, 150.0, 15.0, 160.0, 16.0,
+ ]
+ .map(bf16::from_f32);
+ let coefficient_deltas = [1.0, 2.0, 99.0, 99.0, 5.0, 6.0, 99.0, 99.0].map(bf16::from_f32);
+
+ let context = B::Context::new().expect("create context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::SeparableCausalConvKernel::new(
+ &context,
+ bf16::data_type(),
+ MODEL_DIM,
+ KERNEL_SIZE,
+ GROUP_SIZE,
+ false,
+ )
+ .expect("create separable causal convolution kernel");
+
+ let input = alloc_allocation_with_data::<B, bf16>(&context, &input);
+ let coefficient_deltas = alloc_allocation_with_data::<B, bf16>(&context, &coefficient_deltas);
+ let weights = alloc_allocation_with_data::<B, bf16>(&context, &weights);
+ let mut output = alloc_allocation::<B, bf16>(&context, (SEQUENCE_LENGTH * MODEL_DIM) as usize);
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("create encoder");
+ kernel.encode(
+ &input,
+ &coefficient_deltas,
+ &weights,
+ None::<&Allocation<B>>,
+ &mut output,
+ SEQUENCE_LENGTH,
+ COEFFICIENT_ROW_STRIDE,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ allocation_to_vec::<B, bf16>(&output)
+}
+
+#[uzu_test]
+fn test_separable_causal_convolution() {
+ let expected = run_kernel::<Cpu>();
+
+ for_each_non_cpu_backend!(|B| {
+ let actual = run_kernel::<B>();
+ assert_eq!(expected, actual);
+ });
+}