A high-performance inference engine for AI models
add in_place TensorAddScale (#457)
CCYeh4d0861105cd3
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rsindex 774fbff45..a9c153d43 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs@@ -7,18 +7,24 @@ use crate::ArrayElement;#[kernel(TensorAddScale)]#[variants(T, f32, f16, bf16)]pub fn tensor_add_scale<T: ArrayElement + Float>(- input: *const T,+ #[optional(!in_place)] input: Option<*const T>,bias: *const T,output: *mut T,num_cols: u32,length: u32,scale: f32,+ #[allow(unused)]+ #[specialize]+ in_place: bool,) {let num_cols = num_cols as usize;for position in 0..length as usize {let col = position % num_cols;unsafe {- let input_value = (*input.add(position)).to_f32().unwrap();+ let input_value = match input {+ Some(input) => (*input.add(position)).to_f32().unwrap(),+ None => (*output.add(position)).to_f32().unwrap(),+ };let bias_value = (*bias.add(col)).to_f32().unwrap();*output.add(position) = T::from((input_value + bias_value) * scale).unwrap();}diff --git a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metalindex 45afc5789..08efae8b8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal@@ -4,14 +4,18 @@template <typename T>VARIANTS(T, float, half, bfloat)PUBLIC KERNEL(TensorAddScale)(- const device T* input,+ const device T* input OPTIONAL(!in_place),const device T* bias,device T* output,constant uint& num_cols,constant uint& length,constant float& scale,- const uint position AXIS(length, 32)+ const uint position AXIS(length, 32),+ const bool in_place SPECIALIZE) {+ if (in_place) {+ input = output;+ }uint col = position % num_cols;float value = (float(input[position]) + float(bias[col])) * scale;output[position] = T(value);diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rsindex d6198f064..c29ed84ca 100644--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs@@ -222,7 +222,7 @@ impl<B: Backend> TokenDecoderRunner<B> {let context: Rc<B::Context> = Rc::clone(ctx.context());let model_dim = ctx.decoder_config.transformer_config.model_dim;let tensor_add_scale =- <B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), ctx.model_shape.data_type)+ <B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), ctx.model_shape.data_type, false).map_err(unable_to_create_context)?;let single_hidden_capture =context.create_array_zeros(&[1, model_dim], ctx.model_shape.data_type).into_allocation();@@ -1099,7 +1099,7 @@ impl<B: Backend> TokenDecoderRunner<B> {let mut output =encoder.allocate_scratch(main.as_buffer_range_ref().range().len()).map_err(unable_to_create_context)?;self.tensor_add_scale.encode(- &*main,+ Some(&*main),&self.single_override_embedding,&mut output,model_dim_u32,diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rsindex 4355a7f56..3000f4e38 100644--- a/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs@@ -36,21 +36,27 @@ fn get_input<T: ArrayElement + Float>((input, bias)}-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+fn get_output<T: ArrayElement + Float, B: Backend>(+ input: &Input<T>,+ in_place: bool,+) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddScaleKernel::new(&context, T::data_type())+ let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddScaleKernel::new(&context, T::data_type(), in_place).expect("Failed to create TensorAddScaleKernel");let length = input.length as usize;let num_cols = input.num_cols as usize;- let input_allocation = alloc_allocation_with_data::<B, T>(&context, &input.input[..length]);+ let input_allocation = (!in_place).then(|| alloc_allocation_with_data::<B, T>(&context, &input.input[..length]));let bias_allocation = alloc_allocation_with_data::<B, T>(&context, &input.bias[..num_cols]);- let mut output_allocation = alloc_allocation::<B, T>(&context, length);+ let mut output_allocation = match in_place {+ true => alloc_allocation_with_data::<B, T>(&context, &input.input[..length]),+ false => alloc_allocation::<B, T>(&context, length),+ };let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- &input_allocation,+ input_allocation.as_ref(),&bias_allocation,&mut output_allocation,input.num_cols,@@ -76,7 +82,7 @@ fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {scale: 0.5,};- let expected = get_output::<T, Cpu>(&input);+ let expected = get_output::<T, Cpu>(&input, false);(input, expected)}@@ -93,7 +99,7 @@ fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {scale: 2.0,};- let expected = get_output::<T, Cpu>(&input);+ let expected = get_output::<T, Cpu>(&input, false);(input, expected)}@@ -112,7 +118,7 @@ fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {scale: 1.0,};- let expected = get_output::<T, Cpu>(&input);+ let expected = get_output::<T, Cpu>(&input, false);(input, expected)}@@ -124,7 +130,7 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {};let (input, expected) = get_test_data_basic::<T>();for_each_non_cpu_backend!(|B| {- let output = get_output::<T, B>(&input);+ let output = get_output::<T, B>(&input, false);assert_eq_float::<T>(&expected,&output,@@ -142,7 +148,7 @@ fn test_large<T: ArrayElement + Float + Debug + Display>() {};let (input, expected) = get_test_data_large::<T>();for_each_non_cpu_backend!(|B| {- let output = get_output::<T, B>(&input);+ let output = get_output::<T, B>(&input, false);assert_eq_float::<T>(&expected,&output,@@ -160,7 +166,7 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>() {};let (input, expected) = get_test_data_edge::<T>();for_each_non_cpu_backend!(|B| {- let output = get_output::<T, B>(&input);+ let output = get_output::<T, B>(&input, false);assert_eq_float::<T>(&expected,&output,@@ -170,6 +176,24 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>() {});}+fn test_in_place<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.05f32+ } else {+ 1e-5+ };+ let (input, expected) = get_test_data_basic::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input, true);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("TensorAddScale in-place test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}+// basic tests#[uzu_test]fn test_basic_f32() {@@ -217,3 +241,18 @@ fn test_edge_f16() {fn test_edge_bf16() {test_edge::<bf16>();}++#[uzu_test]+fn test_in_place_f32() {+ test_in_place::<f32>();+}++#[uzu_test]+fn test_in_place_f16() {+ test_in_place::<f16>();+}++#[uzu_test]+fn test_in_place_bf16() {+ test_in_place::<bf16>();+}