Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Honor configured SiLU alpha and clipping in GatedActMul (#747)

xhjkl0ad4f8719f33
diff --git a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs
index 198261601..7bcc47cec 100644
--- a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs
@@ -6,6 +6,7 @@ use crate::{
gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE},
kernel::GatedActMulKernel,
},
+ config::clipping::ClippingBounds,
data_type::DataType,
};
@@ -36,10 +37,19 @@ bitflags! {
}
}
+/// Value transforms baked into a gated-activation kernel specialization.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct GatedActMulSettings {
+ pub activation_alpha: Option<f32>,
+ pub gate_clipping: ClippingBounds,
+ pub value_clipping: ClippingBounds,
+}
+
pub struct GatedActMul<B: Backend> {
kernel: <B::Kernels as Kernels>::GatedActMulKernel,
ops: GatedActMulOp,
options: GatedActMulOptions,
+ settings: GatedActMulSettings,
activation_group_size: u32,
sum_group_size: u32,
}
@@ -50,6 +60,7 @@ impl<B: Backend> GatedActMul<B> {
data_type: DataType,
interleaved: bool,
use_hadamard: bool,
+ settings: GatedActMulSettings,
) -> Result<Self, B::Error> {
let mut options = GatedActMulOptions::empty();
options.set(GatedActMulOptions::INTERLEAVED, interleaved);
@@ -61,6 +72,7 @@ impl<B: Backend> GatedActMul<B> {
options,
HADAMARD_TRANSFORM_BLOCK_SIZE,
HADAMARD_TRANSFORM_BLOCK_SIZE,
+ settings,
)
}
@@ -69,6 +81,7 @@ impl<B: Backend> GatedActMul<B> {
data_type: DataType,
activation_group_size: u32,
sum_group_size: Option<u32>,
+ settings: GatedActMulSettings,
) -> Result<Self, B::Error> {
let activation_group_size = GatedActMulGroupSize::from_u32(activation_group_size);
let sum_group_size = sum_group_size.map(GatedActMulGroupSize::from_u32);
@@ -79,6 +92,7 @@ impl<B: Backend> GatedActMul<B> {
GatedActMulOptions::INTERLEAVED | GatedActMulOptions::HADAMARD,
activation_group_size as u32,
sum_group_size.unwrap_or(activation_group_size) as u32,
+ settings,
)
}
@@ -89,6 +103,7 @@ impl<B: Backend> GatedActMul<B> {
options: GatedActMulOptions,
activation_group_size: u32,
sum_group_size: u32,
+ settings: GatedActMulSettings,
) -> Result<Self, B::Error> {
let kernel = <B::Kernels as Kernels>::GatedActMulKernel::new(
context,
@@ -98,11 +113,15 @@ impl<B: Backend> GatedActMul<B> {
options.contains(GatedActMulOptions::HADAMARD),
activation_group_size,
sum_group_size,
+ settings.activation_alpha.is_some(),
+ settings.gate_clipping.into_pair().is_some(),
+ settings.value_clipping.into_pair().is_some(),
)?;
Ok(Self {
kernel,
ops,
options,
+ settings,
activation_group_size,
sum_group_size,
})
@@ -128,6 +147,8 @@ impl<B: Backend> GatedActMul<B> {
!self.options.contains(GatedActMulOptions::HADAMARD)
|| gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE)
);
+ let (gate_clip_min, gate_clip_max) = self.settings.gate_clipping.into_pair().unzip();
+ let (value_clip_min, value_clip_max) = self.settings.value_clipping.into_pair().unzip();
self.kernel.encode(
act_operand,
value_operand,
@@ -141,6 +162,11 @@ impl<B: Backend> GatedActMul<B> {
value_offset,
value_row_stride,
act_type,
+ self.settings.activation_alpha,
+ gate_clip_min,
+ gate_clip_max,
+ value_clip_min,
+ value_clip_max,
encoder,
);
}
@@ -166,6 +192,8 @@ impl<B: Backend> GatedActMul<B> {
if self.ops == GatedActMulOp::QuantizeWithGroupSums {
assert!(gated_dim.is_multiple_of(self.sum_group_size));
}
+ let (gate_clip_min, gate_clip_max) = self.settings.gate_clipping.into_pair().unzip();
+ let (value_clip_min, value_clip_max) = self.settings.value_clipping.into_pair().unzip();
self.kernel.encode(
act_operand,
None::<&Allocation<B>>,
@@ -179,6 +207,11 @@ impl<B: Backend> GatedActMul<B> {
0,
0,
act_type,
+ self.settings.activation_alpha,
+ gate_clip_min,
+ gate_clip_max,
+ value_clip_min,
+ value_clip_max,
encoder,
);
}
diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rs
index 6f426140f..4a700860e 100644
--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs
@@ -9,7 +9,7 @@ pub mod matmul;
pub mod radix_top_k_small;
pub use activation_transform::ActivationTransform;
-pub use gated_act_mul::GatedActMul;
+pub use gated_act_mul::{GatedActMul, GatedActMulSettings};
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
index 740952329..74373cd83 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
@@ -5,7 +5,7 @@ use uzu_engine_macros::kernel;
use crate::{
array::ArrayElement,
backends::{
- common::gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE},
+ common::gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE, activation_silu_alpha},
cpu::kernel::activation_transform::{hadamard_transform, quantize_transformed_row},
},
};
@@ -27,11 +27,19 @@ pub fn gated_act_mul<T: ArrayElement + Float>(
value_offset: u32,
value_row_stride: u32,
act_type: ActivationType,
+ #[optional(custom_activation_alpha)] activation_alpha: Option<f32>,
+ #[optional(clip_gate)] gate_clip_min: Option<f32>,
+ #[optional(clip_gate)] gate_clip_max: Option<f32>,
+ #[optional(clip_value)] value_clip_min: Option<f32>,
+ #[optional(clip_value)] value_clip_max: Option<f32>,
#[specialize] ops: GatedActMulOp,
#[specialize] interleaved: bool,
#[specialize] use_hadamard: bool,
#[specialize] activation_scale_group_size: u32,
#[specialize] sum_group_size: u32,
+ #[specialize] custom_activation_alpha: bool,
+ #[specialize] clip_gate: bool,
+ #[specialize] clip_value: bool,
) {
assert_eq!(hadamard_factors.is_some(), use_hadamard);
let quantize = matches!(ops, GatedActMulOp::Quantize | GatedActMulOp::QuantizeWithGroupSums);
@@ -57,7 +65,24 @@ pub fn gated_act_mul<T: ArrayElement + Float>(
(batch * gated_dim + gated, unsafe { *value_operand.unwrap().add(value_index) })
};
let gate = unsafe { *act_operand.add(act_index) };
- let result = super::gated_act_mul(value, gate, act_type);
+ let gate = if clip_gate {
+ let gate = gate.to_f32().unwrap().clamp(gate_clip_min.unwrap(), gate_clip_max.unwrap());
+ T::from(gate).unwrap()
+ } else {
+ gate
+ };
+ let activated: T = if custom_activation_alpha && act_type == ActivationType::SILU {
+ activation_silu_alpha(gate, activation_alpha.unwrap())
+ } else {
+ act_type.activate(gate)
+ };
+ let value = if clip_value {
+ let value = value.to_f32().unwrap().clamp(value_clip_min.unwrap(), value_clip_max.unwrap());
+ T::from(value).unwrap()
+ } else {
+ value
+ };
+ let result = (value * activated).to_f32().unwrap();
if let Some(transformed) = transformed.as_mut() {
transformed[gated] = result;
} else {
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/mod.rs
index ef320ee64..95bdbdf41 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/mod.rs
@@ -1,14 +1 @@
-use num_traits::Float;
-
-use crate::backends::common::gpu_types::activation_type::ActivationType;
-
-#[inline]
-pub(super) fn gated_act_mul<T: Float>(
- value: T,
- gate: T,
- act_type: ActivationType,
-) -> f32 {
- (value * act_type.activate(gate)).to_f32().unwrap()
-}
-
pub mod gated_act_mul;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
index b4867bb7e..9eeae5d80 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
@@ -31,11 +31,19 @@ PUBLIC KERNEL(GatedActMul) (
const constant uint& value_offset,
const constant uint& value_row_stride,
const constant ActivationType& act_type,
+ const constant float& activation_alpha OPTIONAL(custom_activation_alpha),
+ const constant float& gate_clip_min OPTIONAL(clip_gate),
+ const constant float& gate_clip_max OPTIONAL(clip_gate),
+ const constant float& value_clip_min OPTIONAL(clip_value),
+ const constant float& value_clip_max OPTIONAL(clip_value),
const GatedActMulOp ops SPECIALIZE,
const bool interleaved SPECIALIZE,
const bool use_hadamard SPECIALIZE,
const uint activation_scale_group_size SPECIALIZE,
const uint sum_group_size SPECIALIZE,
+ const bool custom_activation_alpha SPECIALIZE,
+ const bool clip_gate SPECIALIZE,
+ const bool clip_value SPECIALIZE,
threadgroup float partial_max OPTIONAL(QUANTIZED && activation_scale_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],
threadgroup int partial_sums OPTIONAL(EMITS_GROUP_SUMS && sum_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],
uint activation_tile_index GROUPS(gated_dim.div_ceil(NUM_THREADS)),
@@ -60,19 +68,31 @@ PUBLIC KERNEL(GatedActMul) (
gate = act_operand[batch_idx * gated_dim + gated_idx];
}
}
-
+ if (clip_gate) {
+ gate = static_cast<T>(clamp(float(gate), gate_clip_min, gate_clip_max));
+ }
+ if (clip_value) {
+ value = static_cast<T>(clamp(float(value), value_clip_min, value_clip_max));
+ }
if (!QUANTIZED) {
T result = static_cast<T>(0);
if (element_in_bounds && (!use_hadamard || simdgroup_in_bounds)) {
- float gated = gated_act_mul(value, gate, act_type);
+ float transformed;
+ if (custom_activation_alpha && act_type == ActivationType::SILU) {
+ const T activated = activate_silu_alpha(gate, activation_alpha);
+ const T gated = value * activated;
+ transformed = static_cast<float>(gated);
+ } else {
+ transformed = gated_act_mul(value, gate, act_type);
+ }
if (use_hadamard) {
- gated = simdgroup_input_random_hadamard_transform(
+ transformed = simdgroup_input_random_hadamard_transform(
static_cast<ushort>(gated_idx % METAL_SIMD_SIZE),
- gated,
+ transformed,
hadamard_factors[gated_idx]
);
}
- result = static_cast<T>(gated);
+ result = static_cast<T>(transformed);
}
if (element_in_bounds) {
fp_out[batch_idx * gated_dim + gated_idx] = result;
@@ -82,7 +102,13 @@ PUBLIC KERNEL(GatedActMul) (
float result = 0.0f;
if (simdgroup_in_bounds) {
- result = gated_act_mul(value, gate, act_type);
+ if (custom_activation_alpha && act_type == ActivationType::SILU) {
+ const T activated = activate_silu_alpha(gate, activation_alpha);
+ const T gated = value * activated;
+ result = static_cast<float>(gated);
+ } else {
+ result = gated_act_mul(value, gate, act_type);
+ }
result = simdgroup_input_random_hadamard_transform(
static_cast<ushort>(gated_idx % METAL_SIMD_SIZE),
result,
diff --git a/crates/uzu-engine/src/config/activation/mod.rs b/crates/uzu-engine/src/config/activation/mod.rs
index 48e043b6f..8ccc592b2 100644
--- a/crates/uzu-engine/src/config/activation/mod.rs
+++ b/crates/uzu-engine/src/config/activation/mod.rs
@@ -31,4 +31,12 @@ impl AnyActivation {
AnyActivation::GELU(_) | AnyActivation::Identity(_) => 1.0,
}
}
+
+ /// Non-default activation slope encoded by kernels that specialize it.
+ pub fn custom_alpha(&self) -> Option<f32> {
+ match self {
+ AnyActivation::SiLU(config) if config.alpha != 1.0 => Some(config.alpha),
+ _ => None,
+ }
+ }
}
diff --git a/crates/uzu-engine/src/config/clipping.rs b/crates/uzu-engine/src/config/clipping.rs
new file mode 100644
index 000000000..4adc8ef8d
--- /dev/null
+++ b/crates/uzu-engine/src/config/clipping.rs
@@ -0,0 +1,44 @@
+use serde::{Deserialize, Serialize};
+
+use crate::utils::strict_serde::DeserializeStrict;
+
+type WireClippingBounds = Option<(Option<f32>, Option<f32>)>;
+
+/// Optional inclusive clipping interval with one-sided bounds normalized to finite limits.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
+#[serde(from = "WireClippingBounds", into = "WireClippingBounds")]
+pub struct ClippingBounds(Option<(f32, f32)>);
+
+impl ClippingBounds {
+ /// Finite bounds when clipping is configured.
+ pub(crate) fn into_pair(self) -> Option<(f32, f32)> {
+ self.0
+ }
+}
+
+impl From<WireClippingBounds> for ClippingBounds {
+ fn from(bounds: WireClippingBounds) -> Self {
+ let Some((min, max)) = bounds else {
+ return Self(None);
+ };
+ if min.is_none() && max.is_none() {
+ return Self(None);
+ }
+ Self(Some((min.unwrap_or(f32::MIN), max.unwrap_or(f32::MAX))))
+ }
+}
+
+impl From<ClippingBounds> for WireClippingBounds {
+ fn from(bounds: ClippingBounds) -> Self {
+ let (min, max) = bounds.0?;
+ let min = (min != f32::MIN).then_some(min);
+ let max = (max != f32::MAX).then_some(max);
+ if min.is_none() && max.is_none() {
+ return None;
+ }
+
+ Some((min, max))
+ }
+}
+
+impl<'de> DeserializeStrict<'de> for ClippingBounds {}
diff --git a/crates/uzu-engine/src/config/mlp/dense_mlp.rs b/crates/uzu-engine/src/config/mlp/dense_mlp.rs
index 8a74fd9f2..648b5e7e6 100644
--- a/crates/uzu-engine/src/config/mlp/dense_mlp.rs
+++ b/crates/uzu-engine/src/config/mlp/dense_mlp.rs
@@ -1,6 +1,6 @@
use uzu_engine_macros::uzu_config;
-use crate::config::{activation::AnyActivation, linear::LinearConfig};
+use crate::config::{activation::AnyActivation, clipping::ClippingBounds, linear::LinearConfig};
#[uzu_config(super::MLPConfig)]
pub struct DenseMLPConfig {
@@ -8,8 +8,8 @@ pub struct DenseMLPConfig {
pub activation: AnyActivation,
pub has_up_biases: bool,
pub has_down_biases: bool,
- pub gate_clipping: Option<(Option<f32>, Option<f32>)>,
- pub up_clipping: Option<(Option<f32>, Option<f32>)>,
+ pub gate_clipping: ClippingBounds,
+ pub up_clipping: ClippingBounds,
}
impl DenseMLPConfig {
@@ -25,8 +25,8 @@ impl DenseMLPConfig {
activation,
has_up_biases,
has_down_biases,
- gate_clipping: None,
- up_clipping: None,
+ gate_clipping: ClippingBounds::default(),
+ up_clipping: ClippingBounds::default(),
}
}
}
diff --git a/crates/uzu-engine/src/config/mod.rs b/crates/uzu-engine/src/config/mod.rs
index 964ec30c9..91d66c95a 100644
--- a/crates/uzu-engine/src/config/mod.rs
+++ b/crates/uzu-engine/src/config/mod.rs
@@ -1,5 +1,6 @@
pub mod activation;
pub mod classifier;
+pub mod clipping;
pub mod decoder;
pub mod dflash;
pub mod embedding;
diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
index bd629fe69..8e58a3cbe 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
@@ -4,11 +4,11 @@ use crate::{
Allocation, Backend, Encoder,
gpu_types::ActivationType,
kernel::{
- GatedActMul,
+ GatedActMul, GatedActMulSettings,
matmul::{A8ActivationPlan, ActivationFormat},
},
},
- config::activation::AnyActivation,
+ config::{activation::AnyActivation, clipping::ClippingBounds},
data_type::DataType,
encodable_block::linear::{LinearInput, LinearInputPreparation},
};
@@ -28,14 +28,23 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
context: &B::Context,
data_type: DataType,
activation: AnyActivation,
+ gate_clipping: ClippingBounds,
+ value_clipping: ClippingBounds,
hidden_dim: u32,
input_preparation: Option<LinearInputPreparation<B>>,
) -> Result<Self, B::Error> {
let (hadamard_factors, a8_plan) = input_preparation
.map_or((None, None), |preparation| (Some(preparation.input_factors), preparation.a8_plan));
- let fp_kernel = GatedActMul::full_precision(context, data_type, true, hadamard_factors.is_some())?;
+ let settings = GatedActMulSettings {
+ activation_alpha: activation.custom_alpha(),
+ gate_clipping,
+ value_clipping,
+ };
+ let fp_kernel = GatedActMul::full_precision(context, data_type, true, hadamard_factors.is_some(), settings)?;
let quantized_kernel = a8_plan
- .map(|plan| GatedActMul::quantized(context, data_type, plan.activation_group_size, plan.sum_group_size))
+ .map(|plan| {
+ GatedActMul::quantized(context, data_type, plan.activation_group_size, plan.sum_group_size, settings)
+ })
.transpose()?;
Ok(Self {
fp_kernel,
diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
index 606ad0220..614961941 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
@@ -67,6 +67,8 @@ impl<B: Backend> dyn Mlp<B> {
context,
data_type,
dense_config.activation.clone(),
+ dense_config.gate_clipping,
+ dense_config.up_clipping,
hidden_dimension,
down_input_preparation,
)
diff --git a/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs
index 0e1a90c10..f47f6cec6 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs
@@ -175,8 +175,10 @@ impl<B: Backend> MoeBlock<B> {
let finalize_kernel =
<B::Kernels as Kernels>::MoeFinalizeKernel::new(context, data_type).map_err(MoeBlockError::BackendError)?;
- let (gate_lo, gate_hi) = moe_config.expert_config.gate_clipping.unwrap_or_default();
- let (up_lo, up_hi) = moe_config.expert_config.up_clipping.unwrap_or_default();
+ let gate_clipping = moe_config.expert_config.gate_clipping;
+ let gate_clipping = gate_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));
+ let up_clipping = moe_config.expert_config.up_clipping;
+ let up_clipping = up_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));
Ok(Self {
router_topk_kernel,
@@ -198,10 +200,10 @@ impl<B: Backend> MoeBlock<B> {
hidden_dim: moe_config.expert_hidden_dim,
num_routed_experts: moe_config.num_routed_experts,
num_active_experts: moe_config.num_active_routed_experts,
- gate_clip_min: gate_lo.unwrap_or(f32::NEG_INFINITY),
- gate_clip_max: gate_hi.unwrap_or(f32::INFINITY),
- up_clip_min: up_lo.unwrap_or(f32::NEG_INFINITY),
- up_clip_max: up_hi.unwrap_or(f32::INFINITY),
+ gate_clip_min: gate_clipping.0,
+ gate_clip_max: gate_clipping.1,
+ up_clip_min: up_clipping.0,
+ up_clip_max: up_clipping.1,
silu_alpha: moe_config.expert_config.activation.alpha(),
data_type,
})
diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
index 919b54abf..74b4b36ec 100644
--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
@@ -4,7 +4,7 @@ use crate::{
array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder,
- kernel::{GatedActMul, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},
+ kernel::{GatedActMul, GatedActMulSettings, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},
},
config::{
activation::AnyActivation,
@@ -196,7 +196,12 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
context,
)?;
- let gate_act_mul = GatedActMul::full_precision(context, data_type, false, false)
+ let activation_alpha = config.activation.alpha();
+ let settings = GatedActMulSettings {
+ activation_alpha: (activation_alpha != 1.0).then_some(activation_alpha),
+ ..Default::default()
+ };
+ let gate_act_mul = GatedActMul::full_precision(context, data_type, false, false, settings)
.map_err(PerLayerEmbeddingError::BackendError)?;
let residual_finalize = <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, data_type, true)
.map_err(PerLayerEmbeddingError::BackendError)?;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
index 5ab6a5b23..829cb9d52 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
@@ -7,9 +7,14 @@ use uzu_engine_macros::uzu_test;
use crate::{
array::ArrayElement,
backends::{
- common::{Allocation, Backend, Context, Encoder, gpu_types::ActivationType, kernel::GatedActMul},
+ common::{
+ Allocation, Backend, Context, Encoder,
+ gpu_types::ActivationType,
+ kernel::{GatedActMul, GatedActMulSettings},
+ },
cpu::Cpu,
},
+ config::clipping::ClippingBounds,
data_type::DataType,
tests::{
assert::assert_eq_float,
@@ -50,7 +55,8 @@ fn run_interleaved<T: ArrayElement + Float, B: Backend>(
) -> Vec<T> {
let context = B::Context::new().expect("create context");
let kernel =
- GatedActMul::<B>::full_precision(&context, T::data_type(), true, use_hadamard).expect("create GatedActMul");
+ GatedActMul::<B>::full_precision(&context, T::data_type(), true, use_hadamard, GatedActMulSettings::default())
+ .expect("create GatedActMul");
let fused_length = (input.batch_dim * 2 * input.gated_dim) as usize;
let output_length = (input.batch_dim * input.gated_dim) as usize;
@@ -161,7 +167,9 @@ fn separate_input<T: ArrayElement + Float>() -> (SeparateInput<T>, Vec<T>) {
fn run_separate<T: ArrayElement + Float, B: Backend>(input: &SeparateInput<T>) -> Vec<T> {
let context = B::Context::new().expect("create context");
- let kernel = GatedActMul::<B>::full_precision(&context, T::data_type(), false, false).expect("create GatedActMul");
+ let kernel =
+ GatedActMul::<B>::full_precision(&context, T::data_type(), false, false, GatedActMulSettings::default())
+ .expect("create GatedActMul");
let gate_out = alloc_allocation_with_data::<B, T>(&context, &input.gate_out);
let per_layer_input = alloc_allocation_with_data::<B, T>(&context, &input.per_layer_input);
@@ -202,3 +210,119 @@ fn test_gated_act_mul_separate_f32() {
fn test_gated_act_mul_separate_bf16() {
separate_test::<bf16>();
}
+
+fn transformed_interleaved_test<T: ArrayElement + Float + Debug + Display>(
+ gate_clipping: ClippingBounds,
+ value_clipping: ClippingBounds,
+) {
+ const GATED_DIM: u32 = 4;
+ const ACTIVATION_ALPHA: f32 = 0.5;
+
+ let values = [-4.0f32, -1.0, 2.0, 5.0];
+ let gates = [-3.0f32, -0.5, 1.0, 4.0];
+ let (gate_clip_min, gate_clip_max) = gate_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));
+ let (value_clip_min, value_clip_max) = value_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));
+ let fused_up = values.into_iter().chain(gates).map(|value| T::from(value).unwrap()).collect::<Vec<_>>();
+ let expected = values
+ .into_iter()
+ .zip(gates)
+ .map(|(value, gate)| {
+ let value = T::from(value.clamp(value_clip_min, value_clip_max)).unwrap();
+ let gate = gate.clamp(gate_clip_min, gate_clip_max);
+ let activated = T::from(gate / (1.0 + (-ACTIVATION_ALPHA * gate).exp())).unwrap();
+ value * activated
+ })
+ .collect::<Vec<_>>();
+ let tolerance = if T::data_type() == DataType::BF16 {
+ 0.02
+ } else {
+ 1e-5
+ };
+ let settings = GatedActMulSettings {
+ activation_alpha: Some(ACTIVATION_ALPHA),
+ gate_clipping,
+ value_clipping,
+ };
+
+ for_each_backend!(|B| {
+ let context = <B as Backend>::Context::new().expect("create context");
+ let kernel = GatedActMul::<B>::full_precision(&context, T::data_type(), true, false, settings)
+ .expect("create transformed GatedActMul");
+ let fused_up = alloc_allocation_with_data::<B, T>(&context, &fused_up);
+ let mut output = alloc_allocation::<B, T>(&context, GATED_DIM as usize);
+ let mut encoder = Encoder::new(context.as_ref()).expect("create encoder");
+ kernel.encode_fp(
+ &fused_up,
+ None::<&Allocation<B>>,
+ &mut output,
+ None::<&Allocation<B>>,
+ GATED_DIM,
+ 1,
+ 0,
+ 0,
+ ActivationType::SILU,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ let output = allocation_to_vec::<B, T>(&output);
+ assert_eq_float(
+ &expected,
+ &output,
+ tolerance,
+ &format!("transformed mismatch for backend {}", <B as Backend>::NAME),
+ );
+ });
+}
+
+#[uzu_test]
+fn test_gated_act_mul_transforms_f32() {
+ transformed_interleaved_test::<f32>(
+ serde_json::from_str("[-1.0,2.0]").unwrap(),
+ serde_json::from_str("[-2.0,3.0]").unwrap(),
+ );
+}
+
+#[uzu_test]
+fn test_gated_act_mul_transforms_bf16() {
+ transformed_interleaved_test::<bf16>(
+ serde_json::from_str("[-1.0,2.0]").unwrap(),
+ serde_json::from_str("[-2.0,3.0]").unwrap(),
+ );
+}
+
+#[uzu_test]
+fn test_gated_act_mul_one_sided_clipping_f32() {
+ transformed_interleaved_test::<f32>(
+ serde_json::from_str("[-1.0,null]").unwrap(),
+ serde_json::from_str("[null,3.0]").unwrap(),
+ );
+}
+
+#[uzu_test]
+fn test_gated_act_mul_one_sided_clipping_bf16() {
+ transformed_interleaved_test::<bf16>(
+ serde_json::from_str("[-1.0,null]").unwrap(),
+ serde_json::from_str("[null,3.0]").unwrap(),
+ );
+}
+
+#[uzu_test]
+fn test_gated_act_mul_clipping_bounds_wire_format() {
+ let cases = [
+ ("null", None),
+ ("[-1.0,2.0]", Some((-1.0, 2.0))),
+ ("[-1.0,null]", Some((-1.0, f32::MAX))),
+ ("[null,3.0]", Some((f32::MIN, 3.0))),
+ ];
+
+ for (json, expected) in cases {
+ let bounds = serde_json::from_str::<ClippingBounds>(json).expect("deserialize clipping bounds");
+ assert_eq!(bounds.into_pair(), expected);
+ assert_eq!(serde_json::to_string(&bounds).expect("serialize clipping bounds"), json);
+ }
+
+ let empty_pair = serde_json::from_str::<ClippingBounds>("[null,null]").expect("deserialize empty bounds");
+ assert_eq!(empty_pair.into_pair(), None);
+ assert_eq!(serde_json::to_string(&empty_pair).expect("serialize empty bounds"), "null");
+}