A high-performance inference engine for AI models
Dflash v2 (#814)
CCYehde0e0987b2ee
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/convolution/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/convolution/mod.rsnew file mode 100644index 000000000..c74404d25--- /dev/null+++ b/crates/uzu-engine/src/backends/cpu/kernel/convolution/mod.rs@@ -0,0 +1 @@+pub mod separable_causal_conv;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs b/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rsnew file mode 100644index 000000000..aaa9a4190--- /dev/null+++ b/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs@@ -0,0 +1,58 @@+use half::bf16;+use num_traits::Float;+use uzu_engine_macros::kernel;++use crate::array::ArrayElement;++#[kernel(SeparableCausalConv)]+#[variants(T, bf16)]+pub fn separable_causal_conv<T: ArrayElement + Float>(+ input: *const T,+ coefficient_deltas: *const T,+ weights: *const T,+ #[optional(has_bias)] bias: Option<*const T>,+ output: *mut T,+ sequence_length: u32,+ coefficient_row_stride: u32,+ #[specialize] model_dim: u32,+ #[specialize] kernel_size: u32,+ #[specialize] group_size: u32,+ #[specialize] has_bias: bool,+) {+ let sequence_length = sequence_length as usize;+ let coefficient_row_stride = coefficient_row_stride as usize;+ let model_dim = model_dim as usize;+ let kernel_size = kernel_size as usize;+ let group_size = group_size as usize;++ let num_groups = model_dim / group_size;++ for token in 0..sequence_length {+ for channel in 0..model_dim {+ let group = channel / group_size;+ let mut output_value = if has_bias {+ unsafe { (*bias.unwrap().add(channel)).to_f32().unwrap() }+ } else {+ 0.0+ };++ for tokens_back in 0..kernel_size.min(token + 1) {+ let input_token = token - tokens_back;+ let input_index = input_token * model_dim + channel;+ let weight_index = channel * kernel_size + (kernel_size - 1 - tokens_back);+ let coefficient_index = token * coefficient_row_stride + tokens_back * num_groups + group;++ let input_value = unsafe { (*input.add(input_index)).to_f32().unwrap() };+ let base_weight = unsafe { (*weights.add(weight_index)).to_f32().unwrap() };+ let coefficient_delta = unsafe { (*coefficient_deltas.add(coefficient_index)).to_f32().unwrap() };++ output_value += input_value * (base_weight + coefficient_delta);+ }++ let output_index = token * model_dim + channel;+ unsafe {+ *output.add(output_index) = T::from(output_value).unwrap();+ }+ }+ }+}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/mod.rsindex 5cbd2a504..a1a3f0467 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs@@ -6,6 +6,7 @@ use crate::backends::{mod activation;pub(crate) mod activation_transform;mod attention;+mod convolution;mod embedding;mod gated_act_mul;mod gdn;diff --git a/crates/uzu-engine/src/backends/metal/kernel/convolution/separable_causal_conv.metal b/crates/uzu-engine/src/backends/metal/kernel/convolution/separable_causal_conv.metalnew file mode 100644index 000000000..ea9644820--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/convolution/separable_causal_conv.metal@@ -0,0 +1,58 @@+#include <metal_stdlib>+#include "../common/defines.h"+#include "../common/dsl.h"++using namespace metal;++#define CHANNELS_PER_BLOCK 4u+#define TOKENS_PER_THREADGROUP 1u+#define CHANNEL_BLOCKS_PER_THREADGROUP 128u++template<typename T>+VARIANTS(T, bfloat)+PUBLIC KERNEL(SeparableCausalConv)(+ device const T* input,+ device const T* coefficient_deltas,+ device const T* weights,+ device const T* bias OPTIONAL(has_bias),+ device T* output,+ constant uint& sequence_length,+ constant uint& coefficient_row_stride,+ const uint model_dim SPECIALIZE,+ const uint kernel_size SPECIALIZE,+ const uint group_size SPECIALIZE,+ const bool has_bias SPECIALIZE,+ uint token AXIS(sequence_length, TOKENS_PER_THREADGROUP),+ uint channel_block_index AXIS((model_dim + CHANNELS_PER_BLOCK - 1u) / CHANNELS_PER_BLOCK, CHANNEL_BLOCKS_PER_THREADGROUP)+){+ using ValueBlock = vec<T, CHANNELS_PER_BLOCK>;+ using AccumulatorBlock = vec<float, CHANNELS_PER_BLOCK>;++ const uint channel = channel_block_index * CHANNELS_PER_BLOCK;+ const uint num_groups = model_dim / group_size;+ const uint available_tokens = min(kernel_size, token + 1u);+ const uint group = channel / group_size;+ AccumulatorBlock output_value = 0.0f;+ if (has_bias) {+ output_value = AccumulatorBlock(*reinterpret_cast<const device ValueBlock*>(bias + channel));+ }++ METAL_PRAGMA_UNROLL+ for (uint tokens_back = 0; tokens_back < available_tokens; ++tokens_back) {+ const uint input_token = token - tokens_back;+ const uint input_index = input_token * model_dim + channel;+ const uint stored_weight = kernel_size - 1u - tokens_back;+ const uint coefficient_index = token * coefficient_row_stride + tokens_back * num_groups + group;+ const AccumulatorBlock weight = AccumulatorBlock(+ float(weights[(channel + 0u) * kernel_size + stored_weight]),+ float(weights[(channel + 1u) * kernel_size + stored_weight]),+ float(weights[(channel + 2u) * kernel_size + stored_weight]),+ float(weights[(channel + 3u) * kernel_size + stored_weight])+ );++ output_value += (weight + float(coefficient_deltas[coefficient_index])) *+ AccumulatorBlock(*reinterpret_cast<const device ValueBlock*>(input + input_index));+ }++ *reinterpret_cast<device ValueBlock*>(output + token * model_dim + channel) = ValueBlock(output_value);+}diff --git a/crates/uzu-engine/src/config/transformer_layer.rs b/crates/uzu-engine/src/config/transformer_layer.rsindex 91cd71cb5..3944b8b95 100644--- a/crates/uzu-engine/src/config/transformer_layer.rs+++ b/crates/uzu-engine/src/config/transformer_layer.rs@@ -1,17 +1,31 @@use uzu_engine_macros::uzu_config;use crate::config::{- mlp::AnyMLPConfig, normalization::NormalizationConfig, per_layer_embedding::PLELayerConfig, rope::AnyRoPEConfig,- token_mixer::AnyTokenMixerConfig,+ linear::LinearConfig,+ mlp::AnyMLPConfig,+ normalization::NormalizationConfig,+ per_layer_embedding::PLELayerConfig,+ rope::AnyRoPEConfig,+ token_mixer::{AnyTokenMixerConfig, convolutions::SeparableCausalConvConfig},};+#[uzu_config]+pub struct TransformerLayerConvConfig {+ pub conv_config: SeparableCausalConvConfig,+ pub kernel_projection_config: LinearConfig,+ pub conv_kernel_size: u32,+ pub conv_group_size: u32,+}+#[uzu_config]pub struct TransformerLayerConfig {pub pre_mixer_norm_config: Option<NormalizationConfig>,pub mixer_config: AnyTokenMixerConfig,+ pub mixer_conv_config: Option<TransformerLayerConvConfig>,pub post_mixer_norm_config: Option<NormalizationConfig>,pub pre_mlp_norm_config: NormalizationConfig,pub mlp_config: AnyMLPConfig,+ pub mlp_conv_config: Option<TransformerLayerConvConfig>,pub post_mlp_norm_config: Option<NormalizationConfig>,pub hidden_dim: Option<u32>,pub ple_config: Option<PLELayerConfig>,diff --git a/crates/uzu-engine/src/encodable_block/convolution.rs b/crates/uzu-engine/src/encodable_block/convolution.rsnew file mode 100644index 000000000..437bee19d--- /dev/null+++ b/crates/uzu-engine/src/encodable_block/convolution.rs@@ -0,0 +1,99 @@+use thiserror::Error;++use crate::{+ array::size_for_shape,+ backends::common::{+ Allocation, Backend, Encoder,+ kernel::{Kernels, SeparableCausalConvKernel},+ },+ config::token_mixer::convolutions::SeparableCausalConvConfig,+ data_type::DataType,+ parameters::{ParameterLoaderError, ParameterTree},+};++#[derive(Debug, Error)]+pub enum ConvolutionNewError<B: Backend> {+ #[error("Backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("Parameter loading error: {0}")]+ Parameter(#[from] ParameterLoaderError<B>),+}++pub struct SeparableCausalConv<B: Backend> {+ model_dim: u32,+ data_type: DataType,+ weights: Allocation<B>,+ biases: Option<Allocation<B>>,+ kernel: <B::Kernels as Kernels>::SeparableCausalConvKernel,+}++impl<B: Backend> SeparableCausalConv<B> {+ pub fn new(+ model_dim: u32,+ kernel_size: u32,+ group_size: u32,+ data_type: DataType,+ config: &SeparableCausalConvConfig,+ parameter_tree: &ParameterTree<B>,+ context: &B::Context,+ ) -> Result<Self, ConvolutionNewError<B>> {+ assert!(model_dim.is_multiple_of(4));+ assert!(group_size.is_multiple_of(4));+ assert!(model_dim.is_multiple_of(group_size));+ assert_eq!(data_type, DataType::BF16, "SeparableCausalConv only supports BF16");++ let weights =+ parameter_tree.leaf("weights")?.validate(&[model_dim, kernel_size], DataType::BF16)?.read_allocation()?;++ let biases = config+ .has_biases+ .then(|| parameter_tree.leaf("biases")?.validate(&[model_dim], DataType::BF16)?.read_allocation())+ .transpose()?;++ let kernel = <B::Kernels as Kernels>::SeparableCausalConvKernel::new(+ context,+ data_type,+ model_dim,+ kernel_size,+ group_size,+ config.has_biases,+ )+ .map_err(ConvolutionNewError::Backend)?;++ Ok(Self {+ model_dim,+ data_type,+ weights,+ biases,+ kernel,+ })+ }++ pub fn encode(+ &self,+ input: &Allocation<B>,+ coefficient_deltas: &Allocation<B>,+ coefficient_row_stride: u32,+ coefficient_column_offset: u32,+ sequence_length: u32,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("SeparableCausalConv");++ let mut output = encoder.allocate_scratch_for_shape(&[sequence_length, self.model_dim], self.data_type)?;+ let coefficients_offset_bytes = size_for_shape(&[coefficient_column_offset], self.data_type);+ self.kernel.encode(+ input,+ (coefficient_deltas, coefficients_offset_bytes),+ &self.weights,+ self.biases.as_ref(),+ &mut output,+ sequence_length,+ coefficient_row_stride,+ encoder,+ );++ encoder.pop_debug_group();+ Ok(output)+ }+}diff --git a/crates/uzu-engine/src/encodable_block/mod.rs b/crates/uzu-engine/src/encodable_block/mod.rsindex 909d783ea..9f6c79366 100644--- a/crates/uzu-engine/src/encodable_block/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mod.rs@@ -1,5 +1,6 @@pub mod batch_topology;pub mod classifier;+pub mod convolution;pub mod decoder;pub mod dflash;pub mod embedding;diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rsindex 180c87113..4091001c2 100644--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs@@ -1,11 +1,13 @@use thiserror::Error;use crate::{- backends::common::{Allocation, Backend, Encoder},- config::transformer_layer::TransformerLayerConfig,+ backends::common::{Allocation, Backend, Encoder, kernel::ActivationTransform},+ config::transformer_layer::{TransformerLayerConfig, TransformerLayerConvConfig},data_type::DataType,encodable_block::{batch_topology::BatchTopology,+ convolution::{ConvolutionNewError, SeparableCausalConv},+ linear::{Linear, LinearBlockError},mixer::{Mixer, MixerNewError, MixerState, attention::rope::PrecalculatedRoPE},mlp::{Mlp, MlpBlockError},normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},@@ -17,6 +19,8 @@ use crate::{#[derive(Debug, Error)]pub enum TransformerLayerError<B: Backend> {+ #[error("Backend error: {0}")]+ Backend(#[source] B::Error),#[error("Parameter loader error: {0}")]ParameterLoader(#[from] ParameterLoaderError<B>),#[error("Mixer error: {0}")]@@ -25,6 +29,10 @@ pub enum TransformerLayerError<B: Backend> {MlpBlock(#[from] MlpBlockError<B>),#[error("Normalization error: {0}")]Normalization(#[from] NormalizationNewError<B>),+ #[error("Convolution error: {0}")]+ Convolution(#[from] ConvolutionNewError<B>),+ #[error("Linear error: {0}")]+ Linear(#[from] LinearBlockError<B>),#[error("Layer {layer_index} sets post_layer_scalar but has no post_mlp_norm")]PostLayerScalarWithoutPostMlpNorm {layer_index: u32,@@ -33,20 +41,120 @@ pub enum TransformerLayerError<B: Backend> {MissingPreMixerNormConfig,}+struct TransformerLayerConv<B: Backend> {+ model_dim: u32,+ pre_conv: SeparableCausalConv<B>,+ kernel_projection: Box<dyn Linear<B>>,+ post_conv: SeparableCausalConv<B>,+ input_rht: Option<(ActivationTransform<B>, Allocation<B>)>,+ coefficient_count: u32,+}+// TODO: saner shortcutpub struct TransformerLayer<B: Backend> {pub layer_index: u32,- pub pre_mixer_norm: Option<Normalization<B>>,pub kv_source_layer_index: Option<u32>,+ pub pre_mixer_norm: Option<Normalization<B>>,+ mixer_conv: Option<TransformerLayerConv<B>>,pub mixer: Box<dyn Mixer<B>>,pub post_mixer_norm: Option<Normalization<B>>,pub pre_mlp_norm: Normalization<B>,+ mlp_conv: Option<TransformerLayerConv<B>>,pub mlp: Box<dyn Mlp<B>>,pub post_mlp_norm: Option<Normalization<B>>,pub ple_projection: Option<PerLayerEmbeddingProjection<B>>,}+impl<B: Backend> TransformerLayerConv<B> {+ fn new(+ context: &B::Context,+ model_dim: u32,+ config: &TransformerLayerConvConfig,+ parameter_tree: &ParameterTree<B>,+ data_type: DataType,+ input_hadamard_factors: Option<Allocation<B>>,+ ) -> Result<Self, TransformerLayerError<B>> {+ let pre_conv = SeparableCausalConv::new(+ model_dim,+ config.conv_kernel_size,+ config.conv_group_size,+ data_type,+ &config.conv_config,+ ¶meter_tree.subtree("pre_conv"),+ context,+ )?;+ let post_conv = SeparableCausalConv::new(+ model_dim,+ config.conv_kernel_size,+ config.conv_group_size,+ data_type,+ &config.conv_config,+ ¶meter_tree.subtree("post_conv"),+ context,+ )?;++ let coefficient_count = config.conv_kernel_size * (model_dim / config.conv_group_size);+ let projection_dim = coefficient_count * 2;+ let kernel_projection = <dyn Linear<B>>::new(+ model_dim,+ [projection_dim],+ false,+ context,+ data_type,+ ¶meter_tree.subtree("kernel_projection"),+ )?;++ let input_rht = input_hadamard_factors+ .map(|factors| ActivationTransform::input_rht(context, data_type, true).map(|kernel| (kernel, factors)))+ .transpose()+ .map_err(TransformerLayerError::Backend)?;++ Ok(Self {+ model_dim,+ pre_conv,+ kernel_projection,+ post_conv,+ input_rht,+ coefficient_count,+ })+ }++ fn encode_pre_convolution(+ &self,+ input: &Allocation<B>,+ sequence_length: u32,+ encoder: &mut Encoder<B>,+ ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {+ let mut projection_input = encoder.allocate_scratch(input.size())?;+ encoder.encode_copy(input, .., &mut projection_input, ..);+ let coefficients = self.kernel_projection.encode(projection_input, sequence_length, encoder)?;+ let mut output =+ self.pre_conv.encode(input, &coefficients, 2 * self.coefficient_count, 0, sequence_length, encoder)?;+ if let Some((transform, factors)) = &self.input_rht {+ transform.encode_fp_in_place(&mut output, factors, sequence_length, self.model_dim, encoder);+ }+ Ok((output, coefficients))+ }++ fn encode_post_convolution(+ &self,+ input: &Allocation<B>,+ coefficients: &Allocation<B>,+ sequence_length: u32,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ self.post_conv.encode(+ input,+ coefficients,+ 2 * self.coefficient_count,+ self.coefficient_count,+ sequence_length,+ encoder,+ )+ }+}+impl<B: Backend> TransformerLayer<B> {pub fn new(context: &B::Context,@@ -92,6 +200,21 @@ impl<B: Backend> TransformerLayer<B> {context,)?;+ let (mixer_conv, mixer_hadamard_factors) = match &layer_config.mixer_conv_config {+ Some(config) => (+ Some(TransformerLayerConv::new(+ context,+ model_dim,+ config,+ ¶meter_tree.subtree("mixer_conv"),+ data_type,+ mixer_hadamard_factors,+ )?),+ None,+ ),+ None => (None, mixer_hadamard_factors),+ };+let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.pre_mixer_norm_config {Some(Normalization::new(model_dim,@@ -138,6 +261,21 @@ impl<B: Backend> TransformerLayer<B> {data_type,)?;+ let (mlp_conv, mlp_input_hadamard_factors) = match &layer_config.mlp_conv_config {+ Some(config) => (+ Some(TransformerLayerConv::new(+ context,+ model_dim,+ config,+ ¶meter_tree.subtree("mlp_conv"),+ data_type,+ mlp_input_hadamard_factors,+ )?),+ None,+ ),+ None => (None, mlp_input_hadamard_factors),+ };+let pre_mlp_norm = Normalization::new(model_dim,mlp_input_hadamard_factors,@@ -180,11 +318,13 @@ impl<B: Backend> TransformerLayer<B> {Ok(Self {layer_index,- pre_mixer_norm,kv_source_layer_index: layer_config.kv_source_layer_index,+ pre_mixer_norm,+ mixer_conv,mixer,post_mixer_norm,pre_mlp_norm,+ mlp_conv,mlp,post_mlp_norm,ple_projection,@@ -203,7 +343,7 @@ impl<B: Backend> TransformerLayer<B> {) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group(&format!("transformer layer {}", self.layer_index));- let hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {+ let mut hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {pre_mixer_norm.encode(&input, 0, batch_dim.size(), Some(shortcut), encoder)?} else {assert!(self.layer_index == 0);@@ -211,8 +351,21 @@ impl<B: Backend> TransformerLayer<B> {input};+ let mixer_coefficients = if let Some(convolution) = &self.mixer_conv {+ let (output, coefficients) = convolution.encode_pre_convolution(&hidden, batch_dim.size(), encoder)?;+ hidden = output;+ Some(coefficients)+ } else {+ None+ };+// TODO: In prefill outside of sampling suffix in last layer part of mixer (ie out projection) and everything after is dead code- let mut hidden = self.mixer.encode(hidden, precalculated_rope, batch_dim, state, encoder)?;+ hidden = self.mixer.encode(hidden, precalculated_rope, batch_dim, state, encoder)?;++ if let Some(coefficients) = mixer_coefficients {+ let convolution = self.mixer_conv.as_ref().expect("mixer convolution required");+ hidden = convolution.encode_post_convolution(&hidden, &coefficients, batch_dim.size(), encoder)?;+ }if let Some(post_mixer_norm) = &self.post_mixer_norm {hidden = post_mixer_norm.encode(&hidden, 0, batch_dim.size(), None, encoder)?;@@ -220,8 +373,21 @@ impl<B: Backend> TransformerLayer<B> {hidden = self.pre_mlp_norm.encode(&hidden, 0, batch_dim.size(), Some(shortcut), encoder)?;+ let mlp_coefficients = if let Some(convolution) = &self.mlp_conv {+ let (output, coefficients) = convolution.encode_pre_convolution(&hidden, batch_dim.size(), encoder)?;+ hidden = output;+ Some(coefficients)+ } else {+ None+ };+hidden = self.mlp.encode(hidden, batch_dim.size(), encoder)?;+ if let Some(coefficients) = mlp_coefficients {+ let convolution = self.mlp_conv.as_ref().expect("mlp convolution required");+ hidden = convolution.encode_post_convolution(&hidden, &coefficients, batch_dim.size(), encoder)?;+ }+if let Some(post_mlp_norm) = &self.post_mlp_norm {hidden = post_mlp_norm.encode(&hidden, 0, batch_dim.size(), None, encoder)?;}diff --git a/crates/uzu-engine/unit/backends/common/kernel/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/mod.rsindex e56e9b0d9..6194246f5 100644--- a/crates/uzu-engine/unit/backends/common/kernel/mod.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/mod.rs@@ -10,6 +10,7 @@ mod moe;mod normalization_test;mod pooling;mod radix_top_k_small_test;+mod separable_causal_conv_test;mod short_conv;mod ssm;mod tensor_add_bias_test;diff --git a/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rsnew file mode 100644index 000000000..3cd771618--- /dev/null+++ b/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs@@ -0,0 +1,72 @@+use half::bf16;+use uzu_engine_macros::uzu_test;++use crate::{+ array::ArrayElement,+ backends::{+ common::{Allocation, Backend, Context, Encoder, Kernels, kernel::SeparableCausalConvKernel},+ cpu::Cpu,+ },+ tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},+};++fn run_kernel<B: Backend>() -> Vec<bf16> {+ const SEQUENCE_LENGTH: u32 = 2;+ const MODEL_DIM: u32 = 16;+ const KERNEL_SIZE: u32 = 2;+ const GROUP_SIZE: u32 = 16;+ const COEFFICIENT_ROW_STRIDE: u32 = 2 * KERNEL_SIZE * (MODEL_DIM / GROUP_SIZE);++ let input = [+ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0,+ 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0,+ ]+ .map(bf16::from_f32);+ let weights = [+ 10.0, 1.0, 20.0, 2.0, 30.0, 3.0, 40.0, 4.0, 50.0, 5.0, 60.0, 6.0, 70.0, 7.0, 80.0, 8.0, 90.0, 9.0, 100.0, 10.0,+ 110.0, 11.0, 120.0, 12.0, 130.0, 13.0, 140.0, 14.0, 150.0, 15.0, 160.0, 16.0,+ ]+ .map(bf16::from_f32);+ let coefficient_deltas = [1.0, 2.0, 99.0, 99.0, 5.0, 6.0, 99.0, 99.0].map(bf16::from_f32);++ let context = B::Context::new().expect("create context");+ let kernel = <<B as Backend>::Kernels as Kernels>::SeparableCausalConvKernel::new(+ &context,+ bf16::data_type(),+ MODEL_DIM,+ KERNEL_SIZE,+ GROUP_SIZE,+ false,+ )+ .expect("create separable causal convolution kernel");++ let input = alloc_allocation_with_data::<B, bf16>(&context, &input);+ let coefficient_deltas = alloc_allocation_with_data::<B, bf16>(&context, &coefficient_deltas);+ let weights = alloc_allocation_with_data::<B, bf16>(&context, &weights);+ let mut output = alloc_allocation::<B, bf16>(&context, (SEQUENCE_LENGTH * MODEL_DIM) as usize);++ let mut encoder = Encoder::new(context.as_ref()).expect("create encoder");+ kernel.encode(+ &input,+ &coefficient_deltas,+ &weights,+ None::<&Allocation<B>>,+ &mut output,+ SEQUENCE_LENGTH,+ COEFFICIENT_ROW_STRIDE,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ allocation_to_vec::<B, bf16>(&output)+}++#[uzu_test]+fn test_separable_causal_convolution() {+ let expected = run_kernel::<Cpu>();++ for_each_non_cpu_backend!(|B| {+ let actual = run_kernel::<B>();+ assert_eq!(expected, actual);+ });+}