Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

2.9 KB
Raw Blame
use num_traits::AsPrimitive;

use crate::{
    backends::{
        common::{
            BufferRef,
            kernel::matmul::{MatmulB, QuantParamsStrides},
        },
        cpu::{Cpu, buffer::CpuBufferExt, kernel::matmul::MatmulCpuError},
    },
    data_type::DataType,
    utils::{dispatch_dtype::dispatch_dtype, pointers::SendPtr},
};

pub(super) enum WeightData {
    FullPrecision {
        ptr: SendPtr<u8>,
        leading_dimension: usize,
        transpose: bool,
    },
    Quantized {
        weights: SendPtr<u8>,
        scales: SendPtr<u8>,
        zero_points: Option<(SendPtr<u8>, QuantParamsStrides)>,
        biases: Option<SendPtr<u8>>,
        scale_strides: QuantParamsStrides,
        bits: usize,
        group_size: usize,
        signed_codes: bool,
    },
}

impl WeightData {
    pub(super) fn from_b(
        b: MatmulB<impl BufferRef<Backend = Cpu>>,
        b_leading_dimension: Option<u32>,
        b_transpose: bool,
        k: usize,
        n: usize,
    ) -> Result<Self, MatmulCpuError> {
        fn buffer_ptr(view: impl BufferRef<Backend = Cpu>) -> SendPtr<u8> {
            let (buffer, range) = view.parts();
            SendPtr(buffer.cpu_address().as_ptr().cast::<u8>().cast_const().wrapping_byte_add(range.start))
        }
        match b {
            MatmulB::FullPrecision {
                b: weights,
            } => {
                let leading_dimension = b_leading_dimension.map(|ld| ld as usize).unwrap_or(if b_transpose {
                    k
                } else {
                    n
                });
                Ok(WeightData::FullPrecision {
                    ptr: buffer_ptr(weights),
                    leading_dimension,
                    transpose: b_transpose,
                })
            },
            MatmulB::Trellis {
                ..
            } => Err(MatmulCpuError::UnsupportedLayout {
                path: "CpuMatmul",
            }),
            MatmulB::Quantized(quantized) => Ok(WeightData::Quantized {
                weights: buffer_ptr(quantized.codes),
                scales: buffer_ptr(quantized.scales),
                zero_points: quantized.zero_points().map(|values| (buffer_ptr(values), quantized.zero_point_strides())),
                biases: quantized.biases().map(buffer_ptr),
                scale_strides: quantized.params.scale_strides(),
                bits: quantized.bits() as usize,
                group_size: quantized.group_size as usize,
                signed_codes: quantized.signed_codes,
            }),
        }
    }
}

#[inline]
pub(super) unsafe fn load(
    base: *const u8,
    data_type: DataType,
    index: usize,
) -> f32 {
    dispatch_dtype!(|(T: data_type)| unsafe { (*base.cast::<T>().add(index)).as_() })
}

#[inline]
pub(super) unsafe fn store(
    base: *mut u8,
    data_type: DataType,
    index: usize,
    value: f32,
) {
    dispatch_dtype!(|(T: data_type)| unsafe { *base.cast::<T>().add(index) = value.as_() })
}