Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

7.2 KB
Raw Blame
use std::fmt::{Debug, Display};

use futures::executor::block_on;
use half::bf16;
use num_traits::Float;
use uzu_engine_macros::uzu_test;

use crate::{
    backends::{
        common::{
            Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
            kernel::{ActivationTransform, NormalizationKernel},
        },
        cpu::Cpu,
    },
    data_type::{DataType, DataTypeRepr},
    tests::{
        assert::assert_eq_float,
        helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend, for_each_non_cpu_backend},
    },
};

fn get_output<
    B: Backend,
    InputT: DataTypeRepr + Float,
    AffineT: DataTypeRepr + Float,
    OutputT: DataTypeRepr + Float,
>(
    input: &[InputT],
    scales: Option<&[AffineT]>,
    batch_size: u32,
    element_count: u32,
    epsilon: f32,
    full_layer: bool,
    hadamard_factors: Option<&[i32]>,
) -> Vec<OutputT> {
    let context = block_on(B::Context::new()).expect("Failed to create Context");
    let kernel = <<B as Backend>::Kernels as Kernels>::NormalizationKernel::new(
        &context,
        InputT::data_type(),
        AffineT::data_type(),
        OutputT::data_type(),
        DataType::F32,
        false,
        false,
        full_layer,
        false,
        false,
        hadamard_factors.is_some(),
        false,
        false,
        false,
        scales.is_some(),
    )
    .expect("Failed to create NormalizationKernel");

    let input_buffer = create_buffer_with_data::<B, InputT>(&context, input);
    let scales_buffer = scales.map(|scales| create_buffer_with_data::<B, AffineT>(&context, scales));
    let hadamard_factors_buffer =
        hadamard_factors.map(|hadamard_factors| create_buffer_with_data::<B, i32>(&context, hadamard_factors));
    let mut output_buffer = create_buffer_with_data::<B, OutputT>(&context, &vec![OutputT::zero(); input.len()]);

    let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
    kernel.encode(
        Some(&input_buffer),
        scales_buffer.as_ref(),
        None::<&B::GlobalGpuBuffer>,
        &mut output_buffer,
        None::<&mut B::GlobalGpuBuffer>,
        hadamard_factors_buffer.as_ref(),
        batch_size,
        element_count,
        epsilon,
        0.0,
        1.0,
        &mut command_buffer,
    );
    block_on(command_buffer.end_encoding().submit().wait_until_completed()).expect("Failed to wait command buffer");

    buffer_to_vec::<B, OutputT>(&context, &output_buffer)
}

fn test_internal<
    InputT: DataTypeRepr + Float,
    AffineT: DataTypeRepr + Float,
    OutputT: DataTypeRepr + Float + Debug + Display,
>(
    has_scales: bool,
    full_layer: bool,
) {
    let batch_size = 2u32;
    let element_count = 64u32;
    let epsilon = 1e-6f32;

    let input: Vec<InputT> =
        (0..(batch_size * element_count)).map(|index| InputT::from(0.5f32 + (index as f32) * 0.01).unwrap()).collect();
    let scales: Vec<AffineT> =
        (0..element_count).map(|index| AffineT::from(1.0f32 + (index as f32) * 0.001).unwrap()).collect();
    let scales = has_scales.then_some(scales);

    let expected = get_output::<Cpu, InputT, AffineT, OutputT>(
        &input,
        scales.as_deref(),
        batch_size,
        element_count,
        epsilon,
        full_layer,
        None,
    );

    let eps = if matches!(InputT::data_type(), DataType::F16 | DataType::BF16)
        || matches!(AffineT::data_type(), DataType::F16 | DataType::BF16)
        || matches!(OutputT::data_type(), DataType::F16 | DataType::BF16)
    {
        1e-2
    } else {
        1e-5
    };

    for_each_non_cpu_backend!(|B| {
        let actual = get_output::<B, InputT, AffineT, OutputT>(
            &input,
            scales.as_deref(),
            batch_size,
            element_count,
            epsilon,
            full_layer,
            None,
        );
        let message = format!(
            "Normalization kernel test failed with backend={}, has_scales={}, full_layer={}",
            std::any::type_name::<B>(),
            has_scales,
            full_layer,
        );
        assert_eq_float::<OutputT>(&expected, &actual, eps, &message);
    });
}

fn test_normalization<
    InputT: DataTypeRepr + Float,
    AffineT: DataTypeRepr + Float,
    OutputT: DataTypeRepr + Float + Debug + Display,
>() {
    for has_scales in [true, false] {
        for full_layer in [true, false] {
            test_internal::<InputT, AffineT, OutputT>(has_scales, full_layer);
        }
    }
}

// The fused transform has to match plain normalization followed by the standalone input RHT
fn test_hadamard<T: DataTypeRepr + Float + Debug + Display>() {
    let batch_size = 2u32;
    let element_count = 64u32;
    let epsilon = 1e-6f32;

    let input: Vec<T> =
        (0..(batch_size * element_count)).map(|index| T::from(0.5f32 + (index as f32) * 0.01).unwrap()).collect();
    let scales: Vec<T> = (0..element_count).map(|index| T::from(1.0f32 + (index as f32) * 0.001).unwrap()).collect();
    let hadamard_factors: Vec<i32> = (0..element_count)
        .map(|index| {
            if index % 3 == 0 {
                -1
            } else {
                1
            }
        })
        .collect();

    let plain = get_output::<Cpu, T, T, T>(&input, Some(&scales), batch_size, element_count, epsilon, true, None);

    let context = block_on(<Cpu as Backend>::Context::new()).expect("Failed to create Context");
    let input_rht = ActivationTransform::<Cpu>::input_rht(context.as_ref(), T::data_type(), false)
        .expect("Failed to create ActivationTransform");
    let plain_buffer = create_buffer_with_data::<Cpu, T>(&context, &plain);
    let hadamard_factors_buffer = create_buffer_with_data::<Cpu, i32>(&context, &hadamard_factors);
    let mut expected_buffer = create_buffer::<Cpu, T>(&context, plain.len());
    let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
    input_rht.encode_fp(
        &plain_buffer,
        &mut expected_buffer,
        &hadamard_factors_buffer,
        batch_size,
        element_count,
        &mut command_buffer,
    );
    block_on(command_buffer.end_encoding().submit().wait_until_completed()).expect("Failed to wait command buffer");
    let expected = buffer_to_vec::<Cpu, T>(&context, &expected_buffer);

    let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
        1e-2
    } else {
        1e-5
    };

    for_each_backend!(|B| {
        let actual = get_output::<B, T, T, T>(
            &input,
            Some(&scales),
            batch_size,
            element_count,
            epsilon,
            true,
            Some(&hadamard_factors),
        );
        let message = format!("Normalization hadamard kernel test failed with backend={}", std::any::type_name::<B>());
        assert_eq_float::<T>(&expected, &actual, eps, &message);
    });
}

#[uzu_test]
fn test_normalization_hadamard_f32() {
    test_hadamard::<f32>();
}

#[uzu_test]
fn test_normalization_hadamard_bf16() {
    test_hadamard::<bf16>();
}

#[uzu_test]
fn test_normalization_f32_f32_f32() {
    test_normalization::<f32, f32, f32>();
}

#[uzu_test]
fn test_normalization_bf16_bf16_bf16() {
    test_normalization::<bf16, bf16, bf16>();
}