A high-performance inference engine for AI models
Honor configured SiLU alpha and clipping in GatedActMul (#747)
xhjkl0ad4f8719f33
diff --git a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rsindex 198261601..7bcc47cec 100644--- a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs+++ b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs@@ -6,6 +6,7 @@ use crate::{gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE},kernel::GatedActMulKernel,},+ config::clipping::ClippingBounds,data_type::DataType,};@@ -36,10 +37,19 @@ bitflags! {}}+/// Value transforms baked into a gated-activation kernel specialization.+#[derive(Debug, Clone, Copy, Default)]+pub struct GatedActMulSettings {+ pub activation_alpha: Option<f32>,+ pub gate_clipping: ClippingBounds,+ pub value_clipping: ClippingBounds,+}+pub struct GatedActMul<B: Backend> {kernel: <B::Kernels as Kernels>::GatedActMulKernel,ops: GatedActMulOp,options: GatedActMulOptions,+ settings: GatedActMulSettings,activation_group_size: u32,sum_group_size: u32,}@@ -50,6 +60,7 @@ impl<B: Backend> GatedActMul<B> {data_type: DataType,interleaved: bool,use_hadamard: bool,+ settings: GatedActMulSettings,) -> Result<Self, B::Error> {let mut options = GatedActMulOptions::empty();options.set(GatedActMulOptions::INTERLEAVED, interleaved);@@ -61,6 +72,7 @@ impl<B: Backend> GatedActMul<B> {options,HADAMARD_TRANSFORM_BLOCK_SIZE,HADAMARD_TRANSFORM_BLOCK_SIZE,+ settings,)}@@ -69,6 +81,7 @@ impl<B: Backend> GatedActMul<B> {data_type: DataType,activation_group_size: u32,sum_group_size: Option<u32>,+ settings: GatedActMulSettings,) -> Result<Self, B::Error> {let activation_group_size = GatedActMulGroupSize::from_u32(activation_group_size);let sum_group_size = sum_group_size.map(GatedActMulGroupSize::from_u32);@@ -79,6 +92,7 @@ impl<B: Backend> GatedActMul<B> {GatedActMulOptions::INTERLEAVED | GatedActMulOptions::HADAMARD,activation_group_size as u32,sum_group_size.unwrap_or(activation_group_size) as u32,+ settings,)}@@ -89,6 +103,7 @@ impl<B: Backend> GatedActMul<B> {options: GatedActMulOptions,activation_group_size: u32,sum_group_size: u32,+ settings: GatedActMulSettings,) -> Result<Self, B::Error> {let kernel = <B::Kernels as Kernels>::GatedActMulKernel::new(context,@@ -98,11 +113,15 @@ impl<B: Backend> GatedActMul<B> {options.contains(GatedActMulOptions::HADAMARD),activation_group_size,sum_group_size,+ settings.activation_alpha.is_some(),+ settings.gate_clipping.into_pair().is_some(),+ settings.value_clipping.into_pair().is_some(),)?;Ok(Self {kernel,ops,options,+ settings,activation_group_size,sum_group_size,})@@ -128,6 +147,8 @@ impl<B: Backend> GatedActMul<B> {!self.options.contains(GatedActMulOptions::HADAMARD)|| gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));+ let (gate_clip_min, gate_clip_max) = self.settings.gate_clipping.into_pair().unzip();+ let (value_clip_min, value_clip_max) = self.settings.value_clipping.into_pair().unzip();self.kernel.encode(act_operand,value_operand,@@ -141,6 +162,11 @@ impl<B: Backend> GatedActMul<B> {value_offset,value_row_stride,act_type,+ self.settings.activation_alpha,+ gate_clip_min,+ gate_clip_max,+ value_clip_min,+ value_clip_max,encoder,);}@@ -166,6 +192,8 @@ impl<B: Backend> GatedActMul<B> {if self.ops == GatedActMulOp::QuantizeWithGroupSums {assert!(gated_dim.is_multiple_of(self.sum_group_size));}+ let (gate_clip_min, gate_clip_max) = self.settings.gate_clipping.into_pair().unzip();+ let (value_clip_min, value_clip_max) = self.settings.value_clipping.into_pair().unzip();self.kernel.encode(act_operand,None::<&Allocation<B>>,@@ -179,6 +207,11 @@ impl<B: Backend> GatedActMul<B> {0,0,act_type,+ self.settings.activation_alpha,+ gate_clip_min,+ gate_clip_max,+ value_clip_min,+ value_clip_max,encoder,);}diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rsindex 6f426140f..4a700860e 100644--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs@@ -9,7 +9,7 @@ pub mod matmul;pub mod radix_top_k_small;pub use activation_transform::ActivationTransform;-pub use gated_act_mul::GatedActMul;+pub use gated_act_mul::{GatedActMul, GatedActMulSettings};include!(concat!(env!("OUT_DIR"), "/traits.rs"));diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rsindex 740952329..74373cd83 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs@@ -5,7 +5,7 @@ use uzu_engine_macros::kernel;use crate::{array::ArrayElement,backends::{- common::gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE},+ common::gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE, activation_silu_alpha},cpu::kernel::activation_transform::{hadamard_transform, quantize_transformed_row},},};@@ -27,11 +27,19 @@ pub fn gated_act_mul<T: ArrayElement + Float>(value_offset: u32,value_row_stride: u32,act_type: ActivationType,+ #[optional(custom_activation_alpha)] activation_alpha: Option<f32>,+ #[optional(clip_gate)] gate_clip_min: Option<f32>,+ #[optional(clip_gate)] gate_clip_max: Option<f32>,+ #[optional(clip_value)] value_clip_min: Option<f32>,+ #[optional(clip_value)] value_clip_max: Option<f32>,#[specialize] ops: GatedActMulOp,#[specialize] interleaved: bool,#[specialize] use_hadamard: bool,#[specialize] activation_scale_group_size: u32,#[specialize] sum_group_size: u32,+ #[specialize] custom_activation_alpha: bool,+ #[specialize] clip_gate: bool,+ #[specialize] clip_value: bool,) {assert_eq!(hadamard_factors.is_some(), use_hadamard);let quantize = matches!(ops, GatedActMulOp::Quantize | GatedActMulOp::QuantizeWithGroupSums);@@ -57,7 +65,24 @@ pub fn gated_act_mul<T: ArrayElement + Float>((batch * gated_dim + gated, unsafe { *value_operand.unwrap().add(value_index) })};let gate = unsafe { *act_operand.add(act_index) };- let result = super::gated_act_mul(value, gate, act_type);+ let gate = if clip_gate {+ let gate = gate.to_f32().unwrap().clamp(gate_clip_min.unwrap(), gate_clip_max.unwrap());+ T::from(gate).unwrap()+ } else {+ gate+ };+ let activated: T = if custom_activation_alpha && act_type == ActivationType::SILU {+ activation_silu_alpha(gate, activation_alpha.unwrap())+ } else {+ act_type.activate(gate)+ };+ let value = if clip_value {+ let value = value.to_f32().unwrap().clamp(value_clip_min.unwrap(), value_clip_max.unwrap());+ T::from(value).unwrap()+ } else {+ value+ };+ let result = (value * activated).to_f32().unwrap();if let Some(transformed) = transformed.as_mut() {transformed[gated] = result;} else {diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/mod.rsindex ef320ee64..95bdbdf41 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/mod.rs@@ -1,14 +1 @@-use num_traits::Float;--use crate::backends::common::gpu_types::activation_type::ActivationType;--#[inline]-pub(super) fn gated_act_mul<T: Float>(- value: T,- gate: T,- act_type: ActivationType,-) -> f32 {- (value * act_type.activate(gate)).to_f32().unwrap()-}-pub mod gated_act_mul;diff --git a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metalindex b4867bb7e..9eeae5d80 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal@@ -31,11 +31,19 @@ PUBLIC KERNEL(GatedActMul) (const constant uint& value_offset,const constant uint& value_row_stride,const constant ActivationType& act_type,+ const constant float& activation_alpha OPTIONAL(custom_activation_alpha),+ const constant float& gate_clip_min OPTIONAL(clip_gate),+ const constant float& gate_clip_max OPTIONAL(clip_gate),+ const constant float& value_clip_min OPTIONAL(clip_value),+ const constant float& value_clip_max OPTIONAL(clip_value),const GatedActMulOp ops SPECIALIZE,const bool interleaved SPECIALIZE,const bool use_hadamard SPECIALIZE,const uint activation_scale_group_size SPECIALIZE,const uint sum_group_size SPECIALIZE,+ const bool custom_activation_alpha SPECIALIZE,+ const bool clip_gate SPECIALIZE,+ const bool clip_value SPECIALIZE,threadgroup float partial_max OPTIONAL(QUANTIZED && activation_scale_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],threadgroup int partial_sums OPTIONAL(EMITS_GROUP_SUMS && sum_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],uint activation_tile_index GROUPS(gated_dim.div_ceil(NUM_THREADS)),@@ -60,19 +68,31 @@ PUBLIC KERNEL(GatedActMul) (gate = act_operand[batch_idx * gated_dim + gated_idx];}}-+ if (clip_gate) {+ gate = static_cast<T>(clamp(float(gate), gate_clip_min, gate_clip_max));+ }+ if (clip_value) {+ value = static_cast<T>(clamp(float(value), value_clip_min, value_clip_max));+ }if (!QUANTIZED) {T result = static_cast<T>(0);if (element_in_bounds && (!use_hadamard || simdgroup_in_bounds)) {- float gated = gated_act_mul(value, gate, act_type);+ float transformed;+ if (custom_activation_alpha && act_type == ActivationType::SILU) {+ const T activated = activate_silu_alpha(gate, activation_alpha);+ const T gated = value * activated;+ transformed = static_cast<float>(gated);+ } else {+ transformed = gated_act_mul(value, gate, act_type);+ }if (use_hadamard) {- gated = simdgroup_input_random_hadamard_transform(+ transformed = simdgroup_input_random_hadamard_transform(static_cast<ushort>(gated_idx % METAL_SIMD_SIZE),- gated,+ transformed,hadamard_factors[gated_idx]);}- result = static_cast<T>(gated);+ result = static_cast<T>(transformed);}if (element_in_bounds) {fp_out[batch_idx * gated_dim + gated_idx] = result;@@ -82,7 +102,13 @@ PUBLIC KERNEL(GatedActMul) (float result = 0.0f;if (simdgroup_in_bounds) {- result = gated_act_mul(value, gate, act_type);+ if (custom_activation_alpha && act_type == ActivationType::SILU) {+ const T activated = activate_silu_alpha(gate, activation_alpha);+ const T gated = value * activated;+ result = static_cast<float>(gated);+ } else {+ result = gated_act_mul(value, gate, act_type);+ }result = simdgroup_input_random_hadamard_transform(static_cast<ushort>(gated_idx % METAL_SIMD_SIZE),result,diff --git a/crates/uzu-engine/src/config/activation/mod.rs b/crates/uzu-engine/src/config/activation/mod.rsindex 48e043b6f..8ccc592b2 100644--- a/crates/uzu-engine/src/config/activation/mod.rs+++ b/crates/uzu-engine/src/config/activation/mod.rs@@ -31,4 +31,12 @@ impl AnyActivation {AnyActivation::GELU(_) | AnyActivation::Identity(_) => 1.0,}}++ /// Non-default activation slope encoded by kernels that specialize it.+ pub fn custom_alpha(&self) -> Option<f32> {+ match self {+ AnyActivation::SiLU(config) if config.alpha != 1.0 => Some(config.alpha),+ _ => None,+ }+ }}diff --git a/crates/uzu-engine/src/config/clipping.rs b/crates/uzu-engine/src/config/clipping.rsnew file mode 100644index 000000000..4adc8ef8d--- /dev/null+++ b/crates/uzu-engine/src/config/clipping.rs@@ -0,0 +1,44 @@+use serde::{Deserialize, Serialize};++use crate::utils::strict_serde::DeserializeStrict;++type WireClippingBounds = Option<(Option<f32>, Option<f32>)>;++/// Optional inclusive clipping interval with one-sided bounds normalized to finite limits.+#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]+#[serde(from = "WireClippingBounds", into = "WireClippingBounds")]+pub struct ClippingBounds(Option<(f32, f32)>);++impl ClippingBounds {+ /// Finite bounds when clipping is configured.+ pub(crate) fn into_pair(self) -> Option<(f32, f32)> {+ self.0+ }+}++impl From<WireClippingBounds> for ClippingBounds {+ fn from(bounds: WireClippingBounds) -> Self {+ let Some((min, max)) = bounds else {+ return Self(None);+ };+ if min.is_none() && max.is_none() {+ return Self(None);+ }+ Self(Some((min.unwrap_or(f32::MIN), max.unwrap_or(f32::MAX))))+ }+}++impl From<ClippingBounds> for WireClippingBounds {+ fn from(bounds: ClippingBounds) -> Self {+ let (min, max) = bounds.0?;+ let min = (min != f32::MIN).then_some(min);+ let max = (max != f32::MAX).then_some(max);+ if min.is_none() && max.is_none() {+ return None;+ }++ Some((min, max))+ }+}++impl<'de> DeserializeStrict<'de> for ClippingBounds {}diff --git a/crates/uzu-engine/src/config/mlp/dense_mlp.rs b/crates/uzu-engine/src/config/mlp/dense_mlp.rsindex 8a74fd9f2..648b5e7e6 100644--- a/crates/uzu-engine/src/config/mlp/dense_mlp.rs+++ b/crates/uzu-engine/src/config/mlp/dense_mlp.rs@@ -1,6 +1,6 @@use uzu_engine_macros::uzu_config;-use crate::config::{activation::AnyActivation, linear::LinearConfig};+use crate::config::{activation::AnyActivation, clipping::ClippingBounds, linear::LinearConfig};#[uzu_config(super::MLPConfig)]pub struct DenseMLPConfig {@@ -8,8 +8,8 @@ pub struct DenseMLPConfig {pub activation: AnyActivation,pub has_up_biases: bool,pub has_down_biases: bool,- pub gate_clipping: Option<(Option<f32>, Option<f32>)>,- pub up_clipping: Option<(Option<f32>, Option<f32>)>,+ pub gate_clipping: ClippingBounds,+ pub up_clipping: ClippingBounds,}impl DenseMLPConfig {@@ -25,8 +25,8 @@ impl DenseMLPConfig {activation,has_up_biases,has_down_biases,- gate_clipping: None,- up_clipping: None,+ gate_clipping: ClippingBounds::default(),+ up_clipping: ClippingBounds::default(),}}}diff --git a/crates/uzu-engine/src/config/mod.rs b/crates/uzu-engine/src/config/mod.rsindex 964ec30c9..91d66c95a 100644--- a/crates/uzu-engine/src/config/mod.rs+++ b/crates/uzu-engine/src/config/mod.rs@@ -1,5 +1,6 @@pub mod activation;pub mod classifier;+pub mod clipping;pub mod decoder;pub mod dflash;pub mod embedding;diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rsindex bd629fe69..8e58a3cbe 100644--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs@@ -4,11 +4,11 @@ use crate::{Allocation, Backend, Encoder,gpu_types::ActivationType,kernel::{- GatedActMul,+ GatedActMul, GatedActMulSettings,matmul::{A8ActivationPlan, ActivationFormat},},},- config::activation::AnyActivation,+ config::{activation::AnyActivation, clipping::ClippingBounds},data_type::DataType,encodable_block::linear::{LinearInput, LinearInputPreparation},};@@ -28,14 +28,23 @@ impl<B: Backend> MlpGateActMulEncodable<B> {context: &B::Context,data_type: DataType,activation: AnyActivation,+ gate_clipping: ClippingBounds,+ value_clipping: ClippingBounds,hidden_dim: u32,input_preparation: Option<LinearInputPreparation<B>>,) -> Result<Self, B::Error> {let (hadamard_factors, a8_plan) = input_preparation.map_or((None, None), |preparation| (Some(preparation.input_factors), preparation.a8_plan));- let fp_kernel = GatedActMul::full_precision(context, data_type, true, hadamard_factors.is_some())?;+ let settings = GatedActMulSettings {+ activation_alpha: activation.custom_alpha(),+ gate_clipping,+ value_clipping,+ };+ let fp_kernel = GatedActMul::full_precision(context, data_type, true, hadamard_factors.is_some(), settings)?;let quantized_kernel = a8_plan- .map(|plan| GatedActMul::quantized(context, data_type, plan.activation_group_size, plan.sum_group_size))+ .map(|plan| {+ GatedActMul::quantized(context, data_type, plan.activation_group_size, plan.sum_group_size, settings)+ }).transpose()?;Ok(Self {fp_kernel,diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rsindex 606ad0220..614961941 100644--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs@@ -67,6 +67,8 @@ impl<B: Backend> dyn Mlp<B> {context,data_type,dense_config.activation.clone(),+ dense_config.gate_clipping,+ dense_config.up_clipping,hidden_dimension,down_input_preparation,)diff --git a/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rsindex 0e1a90c10..f47f6cec6 100644--- a/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs@@ -175,8 +175,10 @@ impl<B: Backend> MoeBlock<B> {let finalize_kernel =<B::Kernels as Kernels>::MoeFinalizeKernel::new(context, data_type).map_err(MoeBlockError::BackendError)?;- let (gate_lo, gate_hi) = moe_config.expert_config.gate_clipping.unwrap_or_default();- let (up_lo, up_hi) = moe_config.expert_config.up_clipping.unwrap_or_default();+ let gate_clipping = moe_config.expert_config.gate_clipping;+ let gate_clipping = gate_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));+ let up_clipping = moe_config.expert_config.up_clipping;+ let up_clipping = up_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));Ok(Self {router_topk_kernel,@@ -198,10 +200,10 @@ impl<B: Backend> MoeBlock<B> {hidden_dim: moe_config.expert_hidden_dim,num_routed_experts: moe_config.num_routed_experts,num_active_experts: moe_config.num_active_routed_experts,- gate_clip_min: gate_lo.unwrap_or(f32::NEG_INFINITY),- gate_clip_max: gate_hi.unwrap_or(f32::INFINITY),- up_clip_min: up_lo.unwrap_or(f32::NEG_INFINITY),- up_clip_max: up_hi.unwrap_or(f32::INFINITY),+ gate_clip_min: gate_clipping.0,+ gate_clip_max: gate_clipping.1,+ up_clip_min: up_clipping.0,+ up_clip_max: up_clipping.1,silu_alpha: moe_config.expert_config.activation.alpha(),data_type,})diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rsindex 919b54abf..74b4b36ec 100644--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs@@ -4,7 +4,7 @@ use crate::{array::size_for_shape,backends::common::{Allocation, Backend, Encoder,- kernel::{GatedActMul, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},+ kernel::{GatedActMul, GatedActMulSettings, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},},config::{activation::AnyActivation,@@ -196,7 +196,12 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {context,)?;- let gate_act_mul = GatedActMul::full_precision(context, data_type, false, false)+ let activation_alpha = config.activation.alpha();+ let settings = GatedActMulSettings {+ activation_alpha: (activation_alpha != 1.0).then_some(activation_alpha),+ ..Default::default()+ };+ let gate_act_mul = GatedActMul::full_precision(context, data_type, false, false, settings).map_err(PerLayerEmbeddingError::BackendError)?;let residual_finalize = <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, data_type, true).map_err(PerLayerEmbeddingError::BackendError)?;diff --git a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rsindex 5ab6a5b23..829cb9d52 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs@@ -7,9 +7,14 @@ use uzu_engine_macros::uzu_test;use crate::{array::ArrayElement,backends::{- common::{Allocation, Backend, Context, Encoder, gpu_types::ActivationType, kernel::GatedActMul},+ common::{+ Allocation, Backend, Context, Encoder,+ gpu_types::ActivationType,+ kernel::{GatedActMul, GatedActMulSettings},+ },cpu::Cpu,},+ config::clipping::ClippingBounds,data_type::DataType,tests::{assert::assert_eq_float,@@ -50,7 +55,8 @@ fn run_interleaved<T: ArrayElement + Float, B: Backend>() -> Vec<T> {let context = B::Context::new().expect("create context");let kernel =- GatedActMul::<B>::full_precision(&context, T::data_type(), true, use_hadamard).expect("create GatedActMul");+ GatedActMul::<B>::full_precision(&context, T::data_type(), true, use_hadamard, GatedActMulSettings::default())+ .expect("create GatedActMul");let fused_length = (input.batch_dim * 2 * input.gated_dim) as usize;let output_length = (input.batch_dim * input.gated_dim) as usize;@@ -161,7 +167,9 @@ fn separate_input<T: ArrayElement + Float>() -> (SeparateInput<T>, Vec<T>) {fn run_separate<T: ArrayElement + Float, B: Backend>(input: &SeparateInput<T>) -> Vec<T> {let context = B::Context::new().expect("create context");- let kernel = GatedActMul::<B>::full_precision(&context, T::data_type(), false, false).expect("create GatedActMul");+ let kernel =+ GatedActMul::<B>::full_precision(&context, T::data_type(), false, false, GatedActMulSettings::default())+ .expect("create GatedActMul");let gate_out = alloc_allocation_with_data::<B, T>(&context, &input.gate_out);let per_layer_input = alloc_allocation_with_data::<B, T>(&context, &input.per_layer_input);@@ -202,3 +210,119 @@ fn test_gated_act_mul_separate_f32() {fn test_gated_act_mul_separate_bf16() {separate_test::<bf16>();}++fn transformed_interleaved_test<T: ArrayElement + Float + Debug + Display>(+ gate_clipping: ClippingBounds,+ value_clipping: ClippingBounds,+) {+ const GATED_DIM: u32 = 4;+ const ACTIVATION_ALPHA: f32 = 0.5;++ let values = [-4.0f32, -1.0, 2.0, 5.0];+ let gates = [-3.0f32, -0.5, 1.0, 4.0];+ let (gate_clip_min, gate_clip_max) = gate_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));+ let (value_clip_min, value_clip_max) = value_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));+ let fused_up = values.into_iter().chain(gates).map(|value| T::from(value).unwrap()).collect::<Vec<_>>();+ let expected = values+ .into_iter()+ .zip(gates)+ .map(|(value, gate)| {+ let value = T::from(value.clamp(value_clip_min, value_clip_max)).unwrap();+ let gate = gate.clamp(gate_clip_min, gate_clip_max);+ let activated = T::from(gate / (1.0 + (-ACTIVATION_ALPHA * gate).exp())).unwrap();+ value * activated+ })+ .collect::<Vec<_>>();+ let tolerance = if T::data_type() == DataType::BF16 {+ 0.02+ } else {+ 1e-5+ };+ let settings = GatedActMulSettings {+ activation_alpha: Some(ACTIVATION_ALPHA),+ gate_clipping,+ value_clipping,+ };++ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().expect("create context");+ let kernel = GatedActMul::<B>::full_precision(&context, T::data_type(), true, false, settings)+ .expect("create transformed GatedActMul");+ let fused_up = alloc_allocation_with_data::<B, T>(&context, &fused_up);+ let mut output = alloc_allocation::<B, T>(&context, GATED_DIM as usize);+ let mut encoder = Encoder::new(context.as_ref()).expect("create encoder");+ kernel.encode_fp(+ &fused_up,+ None::<&Allocation<B>>,+ &mut output,+ None::<&Allocation<B>>,+ GATED_DIM,+ 1,+ 0,+ 0,+ ActivationType::SILU,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ let output = allocation_to_vec::<B, T>(&output);+ assert_eq_float(+ &expected,+ &output,+ tolerance,+ &format!("transformed mismatch for backend {}", <B as Backend>::NAME),+ );+ });+}++#[uzu_test]+fn test_gated_act_mul_transforms_f32() {+ transformed_interleaved_test::<f32>(+ serde_json::from_str("[-1.0,2.0]").unwrap(),+ serde_json::from_str("[-2.0,3.0]").unwrap(),+ );+}++#[uzu_test]+fn test_gated_act_mul_transforms_bf16() {+ transformed_interleaved_test::<bf16>(+ serde_json::from_str("[-1.0,2.0]").unwrap(),+ serde_json::from_str("[-2.0,3.0]").unwrap(),+ );+}++#[uzu_test]+fn test_gated_act_mul_one_sided_clipping_f32() {+ transformed_interleaved_test::<f32>(+ serde_json::from_str("[-1.0,null]").unwrap(),+ serde_json::from_str("[null,3.0]").unwrap(),+ );+}++#[uzu_test]+fn test_gated_act_mul_one_sided_clipping_bf16() {+ transformed_interleaved_test::<bf16>(+ serde_json::from_str("[-1.0,null]").unwrap(),+ serde_json::from_str("[null,3.0]").unwrap(),+ );+}++#[uzu_test]+fn test_gated_act_mul_clipping_bounds_wire_format() {+ let cases = [+ ("null", None),+ ("[-1.0,2.0]", Some((-1.0, 2.0))),+ ("[-1.0,null]", Some((-1.0, f32::MAX))),+ ("[null,3.0]", Some((f32::MIN, 3.0))),+ ];++ for (json, expected) in cases {+ let bounds = serde_json::from_str::<ClippingBounds>(json).expect("deserialize clipping bounds");+ assert_eq!(bounds.into_pair(), expected);+ assert_eq!(serde_json::to_string(&bounds).expect("serialize clipping bounds"), json);+ }++ let empty_pair = serde_json::from_str::<ClippingBounds>("[null,null]").expect("deserialize empty bounds");+ assert_eq!(empty_pair.into_pair(), None);+ assert_eq!(serde_json::to_string(&empty_pair).expect("serialize empty bounds"), "null");+}