Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add in_place TensorAddScale (#457)

CCYeh4d0861105cd3
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs
index 774fbff45..a9c153d43 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs
@@ -7,18 +7,24 @@ use crate::ArrayElement;
#[kernel(TensorAddScale)]
#[variants(T, f32, f16, bf16)]
pub fn tensor_add_scale<T: ArrayElement + Float>(
- input: *const T,
+ #[optional(!in_place)] input: Option<*const T>,
bias: *const T,
output: *mut T,
num_cols: u32,
length: u32,
scale: f32,
+ #[allow(unused)]
+ #[specialize]
+ in_place: bool,
) {
let num_cols = num_cols as usize;
for position in 0..length as usize {
let col = position % num_cols;
unsafe {
- let input_value = (*input.add(position)).to_f32().unwrap();
+ let input_value = match input {
+ Some(input) => (*input.add(position)).to_f32().unwrap(),
+ None => (*output.add(position)).to_f32().unwrap(),
+ };
let bias_value = (*bias.add(col)).to_f32().unwrap();
*output.add(position) = T::from((input_value + bias_value) * scale).unwrap();
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal
index 45afc5789..08efae8b8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal
@@ -4,14 +4,18 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
PUBLIC KERNEL(TensorAddScale)(
- const device T* input,
+ const device T* input OPTIONAL(!in_place),
const device T* bias,
device T* output,
constant uint& num_cols,
constant uint& length,
constant float& scale,
- const uint position AXIS(length, 32)
+ const uint position AXIS(length, 32),
+ const bool in_place SPECIALIZE
) {
+ if (in_place) {
+ input = output;
+ }
uint col = position % num_cols;
float value = (float(input[position]) + float(bias[col])) * scale;
output[position] = T(value);
diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
index d6198f064..c29ed84ca 100644
--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
@@ -222,7 +222,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
let context: Rc<B::Context> = Rc::clone(ctx.context());
let model_dim = ctx.decoder_config.transformer_config.model_dim;
let tensor_add_scale =
- <B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), ctx.model_shape.data_type)
+ <B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), ctx.model_shape.data_type, false)
.map_err(unable_to_create_context)?;
let single_hidden_capture =
context.create_array_zeros(&[1, model_dim], ctx.model_shape.data_type).into_allocation();
@@ -1099,7 +1099,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
let mut output =
encoder.allocate_scratch(main.as_buffer_range_ref().range().len()).map_err(unable_to_create_context)?;
self.tensor_add_scale.encode(
- &*main,
+ Some(&*main),
&self.single_override_embedding,
&mut output,
model_dim_u32,
diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs
index 4355a7f56..3000f4e38 100644
--- a/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs
@@ -36,21 +36,27 @@ fn get_input<T: ArrayElement + Float>(
(input, bias)
}
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+fn get_output<T: ArrayElement + Float, B: Backend>(
+ input: &Input<T>,
+ in_place: bool,
+) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddScaleKernel::new(&context, T::data_type())
+ let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddScaleKernel::new(&context, T::data_type(), in_place)
.expect("Failed to create TensorAddScaleKernel");
let length = input.length as usize;
let num_cols = input.num_cols as usize;
- let input_allocation = alloc_allocation_with_data::<B, T>(&context, &input.input[..length]);
+ let input_allocation = (!in_place).then(|| alloc_allocation_with_data::<B, T>(&context, &input.input[..length]));
let bias_allocation = alloc_allocation_with_data::<B, T>(&context, &input.bias[..num_cols]);
- let mut output_allocation = alloc_allocation::<B, T>(&context, length);
+ let mut output_allocation = match in_place {
+ true => alloc_allocation_with_data::<B, T>(&context, &input.input[..length]),
+ false => alloc_allocation::<B, T>(&context, length),
+ };
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- &input_allocation,
+ input_allocation.as_ref(),
&bias_allocation,
&mut output_allocation,
input.num_cols,
@@ -76,7 +82,7 @@ fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
scale: 0.5,
};
- let expected = get_output::<T, Cpu>(&input);
+ let expected = get_output::<T, Cpu>(&input, false);
(input, expected)
}
@@ -93,7 +99,7 @@ fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
scale: 2.0,
};
- let expected = get_output::<T, Cpu>(&input);
+ let expected = get_output::<T, Cpu>(&input, false);
(input, expected)
}
@@ -112,7 +118,7 @@ fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
scale: 1.0,
};
- let expected = get_output::<T, Cpu>(&input);
+ let expected = get_output::<T, Cpu>(&input, false);
(input, expected)
}
@@ -124,7 +130,7 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {
};
let (input, expected) = get_test_data_basic::<T>();
for_each_non_cpu_backend!(|B| {
- let output = get_output::<T, B>(&input);
+ let output = get_output::<T, B>(&input, false);
assert_eq_float::<T>(
&expected,
&output,
@@ -142,7 +148,7 @@ fn test_large<T: ArrayElement + Float + Debug + Display>() {
};
let (input, expected) = get_test_data_large::<T>();
for_each_non_cpu_backend!(|B| {
- let output = get_output::<T, B>(&input);
+ let output = get_output::<T, B>(&input, false);
assert_eq_float::<T>(
&expected,
&output,
@@ -160,7 +166,7 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>() {
};
let (input, expected) = get_test_data_edge::<T>();
for_each_non_cpu_backend!(|B| {
- let output = get_output::<T, B>(&input);
+ let output = get_output::<T, B>(&input, false);
assert_eq_float::<T>(
&expected,
&output,
@@ -170,6 +176,24 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>() {
});
}
+fn test_in_place<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.05f32
+ } else {
+ 1e-5
+ };
+ let (input, expected) = get_test_data_basic::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input, true);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("TensorAddScale in-place test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
// basic tests
#[uzu_test]
fn test_basic_f32() {
@@ -217,3 +241,18 @@ fn test_edge_f16() {
fn test_edge_bf16() {
test_edge::<bf16>();
}
+
+#[uzu_test]
+fn test_in_place_f32() {
+ test_in_place::<f32>();
+}
+
+#[uzu_test]
+fn test_in_place_f16() {
+ test_in_place::<f16>();
+}
+
+#[uzu_test]
+fn test_in_place_bf16() {
+ test_in_place::<bf16>();
+}