A high-performance inference engine for AI models
mxfp4
xhjkl49ce0ceeb652
diff --git a/crates/uzu-engine/Cargo.toml b/crates/uzu-engine/Cargo.tomlindex 31c25959c..d59c2240c 100644--- a/crates/uzu-engine/Cargo.toml+++ b/crates/uzu-engine/Cargo.toml@@ -85,6 +85,7 @@ xgrammar = { workspace = true, optional = true }schemars = { workspace = true, optional = true }[dev-dependencies]+tempfile.workspace = truetest-tag.workspace = trueproptest.workspace = truecriterion.workspace = truediff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rsindex 96be2b02a..89e7a0bd0 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs@@ -1,7 +1,7 @@use thiserror::Error;use crate::{- backends::common::{Backend, gpu_types::gemm::GemmDTransform},+ backends::common::{Backend, gpu_types::gemm::GemmDTransform, microfloat::MicrofloatFormat},data_type::DataType,};@@ -20,6 +20,10 @@ pub enum MatmulError<B: Backend> {UnsupportedLayout {path: &'static str,},+ #[error("Microfloat storage does not match the requested matrix")]+ InvalidMicrofloatStorage,+ #[error("Unsupported microfloat weight format: {0:?}")]+ UnsupportedMicrofloat(MicrofloatFormat),#[error("Incompatible A operand for {path}: {reason}")]IncompatibleA {path: &'static str,diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rsindex e472a5477..2026d5dad 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs@@ -2,6 +2,7 @@ use crate::{backends::common::{Allocation, Backend, BufferArg,gpu_types::{QuantizationMode, gemm::GemmBPrologueKind},+ microfloat::MicrofloatMetadata,},data_type::DataType,};@@ -10,6 +11,12 @@ pub enum MatmulB<'a, B: Backend, TB: BufferArg<'a, B> = &'a Allocation<B>> {FullPrecision {b: TB,},+ Microfloat {+ codes: &'a Allocation<B>,+ scales: &'a Allocation<B>,+ outer_scales: &'a Allocation<B>,+ metadata: MicrofloatMetadata,+ },ScaleBiasDequant {b: &'a Allocation<B>,scales: &'a Allocation<B>,@@ -40,6 +47,9 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {match self {Self::FullPrecision {..+ }+ | Self::Microfloat {+ ..} => GemmBPrologueKind::FullPrecision,Self::ScaleBiasDequant {..@@ -58,6 +68,10 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {Self::FullPrecision {..} => None,+ Self::Microfloat {+ metadata,+ ..+ } => Some(metadata.encoding.bits),Self::ScaleBiasDequant {mode,..@@ -78,6 +92,10 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {Self::FullPrecision {..} => None,+ Self::Microfloat {+ metadata,+ ..+ } => Some(metadata.encoding.group_size),Self::ScaleBiasDequant {group_size,..@@ -97,6 +115,9 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {match self {Self::FullPrecision {..+ }+ | Self::Microfloat {+ ..} => false,Self::ScaleBiasDequant {signed_codes,diff --git a/crates/uzu-engine/src/backends/common/microfloat/decode.rs b/crates/uzu-engine/src/backends/common/microfloat/decode.rsnew file mode 100644index 000000000..c5242eeb9--- /dev/null+++ b/crates/uzu-engine/src/backends/common/microfloat/decode.rs@@ -0,0 +1,23 @@+#[inline]+pub fn decode_e2m1(code: u8) -> f32 {+ const VALUES: [f32; 16] = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0];+ VALUES[usize::from(code & 0x0f)]+}++#[inline]+pub fn decode_e8m0(exponent: u8) -> f32 {+ match exponent {+ 0 => f32::from_bits(0x0040_0000),+ 255 => f32::NAN,+ exponent => f32::from_bits(u32::from(exponent) << 23),+ }+}++#[inline]+pub fn decode_mxfp4(+ code: u8,+ exponent: u8,+ outer_scale: f32,+) -> f32 {+ decode_e2m1(code) * decode_e8m0(exponent) * outer_scale+}diff --git a/crates/uzu-engine/src/backends/common/microfloat/encoding.rs b/crates/uzu-engine/src/backends/common/microfloat/encoding.rsnew file mode 100644index 000000000..8c0c3c7fa--- /dev/null+++ b/crates/uzu-engine/src/backends/common/microfloat/encoding.rs@@ -0,0 +1,38 @@+use super::{MicrofloatError, MicrofloatFormat};++/// How packed microfloat bytes are interpreted, separate from matrix dimensions.+///+/// With MXFP4 group size 16, every 16 values along the input axis occupy eight+/// packed code bytes and share one E8M0 scale.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct MicrofloatEncoding {+ pub format: MicrofloatFormat,+ pub bits: u32,+ pub group_size: u32,+}++impl MicrofloatEncoding {+ pub fn new(+ format: MicrofloatFormat,+ bits: u32,+ group_size: u32,+ ) -> Result<Self, MicrofloatError> {+ if bits != 4 {+ return Err(MicrofloatError::UnsupportedBits {+ format,+ bits,+ });+ }+ if !matches!(group_size, 16 | 32) {+ return Err(MicrofloatError::UnsupportedGroupSize {+ format,+ group_size,+ });+ }+ Ok(Self {+ format,+ bits,+ group_size,+ })+ }+}diff --git a/crates/uzu-engine/src/backends/common/microfloat/error.rs b/crates/uzu-engine/src/backends/common/microfloat/error.rsnew file mode 100644index 000000000..abe796eb4--- /dev/null+++ b/crates/uzu-engine/src/backends/common/microfloat/error.rs@@ -0,0 +1,26 @@+use thiserror::Error;++use super::MicrofloatFormat;++#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]+pub enum MicrofloatError {+ #[error("unsupported {format:?} bit width: {bits}")]+ UnsupportedBits {+ format: MicrofloatFormat,+ bits: u32,+ },+ #[error("unsupported {format:?} group size: {group_size}")]+ UnsupportedGroupSize {+ format: MicrofloatFormat,+ group_size: u32,+ },+ #[error("microfloat rows and columns must be nonzero")]+ EmptyShape,+ #[error("microfloat columns {columns} are not divisible by group size {group_size}")]+ MisalignedColumns {+ columns: u32,+ group_size: u32,+ },+ #[error("microfloat storage size overflows usize")]+ SizeOverflow,+}diff --git a/crates/uzu-engine/src/backends/common/microfloat/format.rs b/crates/uzu-engine/src/backends/common/microfloat/format.rsnew file mode 100644index 000000000..8356176d7--- /dev/null+++ b/crates/uzu-engine/src/backends/common/microfloat/format.rs@@ -0,0 +1,8 @@+use uzu_engine_macros::uzu_config;++#[derive(Copy, Eq)]+#[uzu_config]+#[serde(rename_all = "snake_case")]+pub enum MicrofloatFormat {+ Mxfp4,+}diff --git a/crates/uzu-engine/src/backends/common/microfloat/metadata.rs b/crates/uzu-engine/src/backends/common/microfloat/metadata.rsnew file mode 100644index 000000000..b234b44c5--- /dev/null+++ b/crates/uzu-engine/src/backends/common/microfloat/metadata.rs@@ -0,0 +1,67 @@+use super::{MicrofloatEncoding, MicrofloatError};++/// Physical shape and derived strides for one microfloat matrix.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct MicrofloatMetadata {+ pub encoding: MicrofloatEncoding,+ pub rows: u32,+ pub columns: u32,+}++impl MicrofloatMetadata {+ pub fn new(+ encoding: MicrofloatEncoding,+ rows: u32,+ columns: u32,+ ) -> Result<Self, MicrofloatError> {+ if rows == 0 || columns == 0 {+ return Err(MicrofloatError::EmptyShape);+ }+ if !columns.is_multiple_of(encoding.group_size) {+ return Err(MicrofloatError::MisalignedColumns {+ columns,+ group_size: encoding.group_size,+ });+ }+ let metadata = Self {+ encoding,+ rows,+ columns,+ };+ metadata.checked_code_matrix_stride().ok_or(MicrofloatError::SizeOverflow)?;+ metadata.checked_scale_matrix_stride().ok_or(MicrofloatError::SizeOverflow)?;+ Ok(metadata)+ }++ pub fn code_row_stride(self) -> usize {+ self.columns as usize / 2+ }++ pub fn scale_row_stride(self) -> usize {+ self.columns as usize / self.encoding.group_size as usize+ }++ pub fn code_matrix_stride(self) -> usize {+ self.checked_code_matrix_stride().expect("MicrofloatMetadata validates code storage size")+ }++ pub fn scale_matrix_stride(self) -> usize {+ self.checked_scale_matrix_stride().expect("MicrofloatMetadata validates scale storage size")+ }++ pub fn required_code_bytes(self) -> usize {+ self.code_matrix_stride()+ }++ pub fn required_scale_bytes(self) -> usize {+ self.scale_matrix_stride()+ }++ fn checked_code_matrix_stride(self) -> Option<usize> {+ (self.rows as usize).checked_mul(self.code_row_stride())+ }++ fn checked_scale_matrix_stride(self) -> Option<usize> {+ (self.rows as usize).checked_mul(self.scale_row_stride())+ }+}diff --git a/crates/uzu-engine/src/backends/common/microfloat/mod.rs b/crates/uzu-engine/src/backends/common/microfloat/mod.rsnew file mode 100644index 000000000..21f7c3f6e--- /dev/null+++ b/crates/uzu-engine/src/backends/common/microfloat/mod.rs@@ -0,0 +1,15 @@+mod decode;+mod encoding;+mod error;+mod format;+mod metadata;++pub use decode::{decode_e2m1, decode_e8m0, decode_mxfp4};+pub use encoding::MicrofloatEncoding;+pub use error::MicrofloatError;+pub use format::MicrofloatFormat;+pub use metadata::MicrofloatMetadata;++#[cfg(test)]+#[path = "../../../../unit/backends/common/microfloat/decode_test.rs"]+mod decode_test;diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rsindex d3b57f26d..ff845b66b 100644--- a/crates/uzu-engine/src/backends/common/mod.rs+++ b/crates/uzu-engine/src/backends/common/mod.rs@@ -8,6 +8,7 @@ mod encoder;pub mod gpu_types;mod hazard_tracker;pub mod kernel;+pub mod microfloat;pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};pub use backend::Backend;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rsindex c3db18e05..d64f2fd38 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs@@ -58,6 +58,26 @@ impl MatmulKernel for MatmulCpuKernel {arguments: MatmulArguments<'a, 'b, 'd, Cpu, TB>,encoder: &mut Encoder<Cpu>,) -> Result<(), CpuError> {+ if let MatmulB::Microfloat {+ codes,+ scales,+ outer_scales,+ metadata,+ } = &arguments.b+ {+ let rows_match = arguments.gather_indices.is_some() || metadata.rows == arguments.n;+ if !arguments.b_transpose+ || arguments.b_leading_dimension.is_some()+ || !rows_match+ || metadata.columns != arguments.k+ || codes.size() < metadata.required_code_bytes()+ || scales.size() < metadata.required_scale_bytes()+ || outer_scales.size() < self.weights_data_type.size_in_bytes()+ {+ return Err(MatmulError::InvalidMicrofloatStorage.into());+ }+ }+let output_scale = arguments.d_transform.ab_scale;let accumulate = arguments.d_transform.accumulate;let bias_alloc = arguments.d_transform.bias;@@ -183,6 +203,9 @@ impl MatmulKernel for MatmulCpuKernel {},WeightData::FullPrecision {..+ }+ | WeightData::Microfloat {+ ..} => None,};@@ -273,6 +296,25 @@ impl MatmulKernel for MatmulCpuKernel {};scale * quantized_value + bias_term},+ WeightData::Microfloat {+ codes,+ scales,+ outer_scales,+ metadata,+ } => {+ let code_index = b_col * metadata.code_row_stride() + inner / 2;+ let packed = *codes.as_ptr().add(code_index);+ let code = if inner.is_multiple_of(2) {+ packed & 0x0f+ } else {+ packed >> 4+ };+ let scale_index = b_col * metadata.scale_row_stride()+ + inner / metadata.encoding.group_size as usize;+ let exponent = *scales.as_ptr().add(scale_index);+ let outer_scale = read_f32(outer_scales.as_ptr(), weights_data_type, 0);+ crate::backends::common::microfloat::decode_mxfp4(code, exponent, outer_scale)+ },};accumulator += a_value * b_value;}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rsindex 41c92a735..4e48ce623 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs@@ -2,7 +2,10 @@ use half::{bf16, f16};use crate::{backends::{- common::{AsBufferRangeRef, BufferArg, gpu_types::QuantizationMode, kernel::matmul::MatmulB},+ common::{+ AsBufferRangeRef, BufferArg, gpu_types::QuantizationMode, kernel::matmul::MatmulB,+ microfloat::MicrofloatMetadata,+ },cpu::Cpu,},data_type::DataType,@@ -24,6 +27,12 @@ pub(super) enum WeightData {group_size: usize,signed_codes: bool,},+ Microfloat {+ codes: SendPtr<u8>,+ scales: SendPtr<u8>,+ outer_scales: SendPtr<u8>,+ metadata: MicrofloatMetadata,+ },}impl WeightData {@@ -58,6 +67,17 @@ impl WeightData {transpose: b_transpose,}},+ MatmulB::Microfloat {+ codes,+ scales,+ outer_scales,+ metadata,+ } => WeightData::Microfloat {+ codes: alloc_ptr(codes),+ scales: alloc_ptr(scales),+ outer_scales: alloc_ptr(outer_scales),+ metadata,+ },MatmulB::ScaleBiasDequant {b: weights,scales,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 3b27f7299..532fc8b0a 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -191,6 +191,12 @@ impl GemmKernel {let use_mxu = plan.engine == GemmEngine::Mxu;match b {+ MatmulB::Microfloat {+ metadata,+ ..+ } => {+ return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format).into());+ },MatmulB::FullPrecision {b: weights,} => {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex a78ae14ed..7f425f552 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -266,6 +266,9 @@ impl GemvKernel {let (scales, zero_points, biases) = match &b {MatmulB::FullPrecision {..+ }+ | MatmulB::Microfloat {+ ..} => (None, None, None),MatmulB::ScaleBiasDequant {scales,@@ -287,6 +290,12 @@ impl GemvKernel {let context = encoder.context();let pipeline = self.get_or_create(context, specialization)?;match b {+ MatmulB::Microfloat {+ metadata,+ ..+ } => {+ return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format));+ },MatmulB::FullPrecision {b: weights,} => pipeline.encode(diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rsindex 14fa5a4c1..eb187459e 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs@@ -17,7 +17,10 @@ use crate::{gpu_types::gemm::{GemmBPrologueKind, GemmTiling},kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE,- matmul::{A8ActivationPlan, ActivationFormat, MatmulArguments, MatmulError, MatmulKernel, MatmulShape},+ matmul::{+ A8ActivationPlan, ActivationFormat, MatmulArguments, MatmulB, MatmulError, MatmulKernel,+ MatmulShape,+ },},},metal::{Metal, context::MetalContext, error::MetalError},@@ -217,6 +220,13 @@ impl MatmulKernel for MatmulMetalKernel {arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {+ if let MatmulB::Microfloat {+ metadata,+ ..+ } = &arguments.b+ {+ return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format).into());+ }let shape = MatmulShape::from_arguments(&arguments);let plan = match self.select_dispatch(&shape, encoder.context()) {MatmulDispatch::Gemv(gemv) => {diff --git a/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rs b/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rsnew file mode 100644index 000000000..45a7e99b0--- /dev/null+++ b/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rs@@ -0,0 +1,11 @@+use uzu_engine_macros::uzu_config;++use crate::{backends::common::microfloat::MicrofloatFormat, config::weight_matrix::Layout};++#[uzu_config(super::WeightMatrixSpec)]+pub struct MicrofloatSpec {+ pub bits: u32,+ pub group_size: usize,+ pub scale_mode: MicrofloatFormat,+ pub layout: Layout,+}diff --git a/crates/uzu-engine/src/config/weight_matrix/mod.rs b/crates/uzu-engine/src/config/weight_matrix/mod.rsindex 5f10ee506..405745762 100644--- a/crates/uzu-engine/src/config/weight_matrix/mod.rs+++ b/crates/uzu-engine/src/config/weight_matrix/mod.rs@@ -4,6 +4,7 @@ pub mod full_precision_spec;pub mod hybrid_spec;pub mod int_spec;pub mod low_rank_spec;+pub mod microfloat_spec;pub mod mlx_spec;#[uzu_config]@@ -18,6 +19,7 @@ pub enum Layout {low_rank_spec::LowRankSpec,hybrid_spec::HybridSpec,int_spec::IntSpec,+ microfloat_spec::MicrofloatSpec,mlx_spec::MLXSpec)]pub struct WeightMatrixSpec;diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rsindex 250a1e70c..a48d7771b 100644--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs@@ -7,7 +7,7 @@ use crate::{},config::weight_matrix::{AnyWeightMatrixSpec, Layout},data_type::DataType,- encodable_block::weight_matrix::{WeightMatrix, WeightMatrixError},+ encodable_block::weight_matrix::{QuantizationInfo, WeightMatrix, WeightMatrixError},parameters::{ParameterLoaderError, ParameterTree},};@@ -65,21 +65,32 @@ impl<B: Backend> EmbeddingTable<B> {}let lookup = match matrix.quantization() {- None => LookupKernel::FullPrecision(- <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)- .map_err(EmbeddingTableError::BackendError)?,- ),- Some(info) => LookupKernel::Quantized(- <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(+ None => {+ let kernel = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)+ .map_err(EmbeddingTableError::BackendError)?;+ LookupKernel::FullPrecision(kernel)+ },+ Some(QuantizationInfo::Microfloat(_)) => {+ return Err(EmbeddingTableError::UnsupportedConfiguration(+ "microfloat embedding tables are not supported".into(),+ ));+ },+ Some(QuantizationInfo::Integer {+ mode,+ method,+ group_size,+ }) => {+ let kernel = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(context,data_type,- info.group_size,- info.mode,- info.method,+ group_size,+ mode,+ method,output_hadamard_factors.is_some(),)- .map_err(EmbeddingTableError::BackendError)?,- ),+ .map_err(EmbeddingTableError::BackendError)?;+ LookupKernel::Quantized(kernel)+ },};Ok(Self {diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rsindex 06c435434..9dbb66aeb 100644--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs@@ -5,8 +5,9 @@ use crate::{Allocation, Backend,gpu_types::{QuantizationMethod, QuantizationMode},kernel::matmul::MatmulB,+ microfloat::{MicrofloatEncoding, MicrofloatMetadata},},- config::weight_matrix::{AnyWeightMatrixSpec, Layout},+ config::weight_matrix::{AnyWeightMatrixSpec, Layout, microfloat_spec::MicrofloatSpec},data_type::DataType,parameters::{ParameterLoaderError, ParameterTree},};@@ -19,11 +20,14 @@ pub enum WeightMatrixError<B: Backend> {UnsupportedConfiguration(String),}-#[derive(Clone, Copy)]-pub struct QuantizationInfo {- pub mode: QuantizationMode,- pub method: QuantizationMethod,- pub group_size: u32,+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum QuantizationInfo {+ Integer {+ mode: QuantizationMode,+ method: QuantizationMethod,+ group_size: u32,+ },+ Microfloat(MicrofloatEncoding),}pub struct ParsedWeightSpec {@@ -32,46 +36,41 @@ pub struct ParsedWeightSpec {}pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeightSpec, WeightMatrixError<B>> {- let (layout, quantized) = match spec {+ let (layout, quantization) = match spec {AnyWeightMatrixSpec::FullPrecisionSpec(spec) => (spec.layout.clone(), None),AnyWeightMatrixSpec::MLXSpec(spec) => {- (spec.layout.clone(), Some((spec.bits, spec.group_size, QuantizationMethod::ScaleBias)))+ let quantization = integer_quantization::<B>(spec.bits, spec.group_size, QuantizationMethod::ScaleBias)?;+ (spec.layout.clone(), Some(quantization))},- AnyWeightMatrixSpec::IntSpec(spec) => (- spec.layout.clone(),- Some((- spec.bits,- spec.group_size,- if spec.is_symmetric {- QuantizationMethod::ScaleSymmetric- } else {- QuantizationMethod::ScaleZeroPoint- },- )),- ),- spec => return Err(WeightMatrixError::UnsupportedConfiguration(format!("{spec:?}"))),- };- let quantization = match quantized {- None => None,- Some((bits, group_size, method)) => {- let mode = match bits {- 4 => QuantizationMode::U4,- 8 => QuantizationMode::U8,- _ => {- return Err(WeightMatrixError::UnsupportedConfiguration(format!(- "{method} bits={bits}, group_size={group_size}"- )));- },+ AnyWeightMatrixSpec::IntSpec(spec) => {+ let method = if spec.is_symmetric {+ QuantizationMethod::ScaleSymmetric+ } else {+ QuantizationMethod::ScaleZeroPoint};- if group_size == 0 {- return Err(WeightMatrixError::UnsupportedConfiguration("group size must be non-zero".into()));+ let quantization = integer_quantization::<B>(spec.bits, spec.group_size, method)?;+ (spec.layout.clone(), Some(quantization))+ },+ AnyWeightMatrixSpec::MicrofloatSpec(MicrofloatSpec {+ bits,+ group_size,+ scale_mode,+ layout,+ ..+ }) => {+ if layout != &Layout::OutputInput {+ return Err(WeightMatrixError::UnsupportedConfiguration(format!(+ "microfloat matrices require output-input layout, got {layout:?}"+ )));}- Some(QuantizationInfo {- mode,- method,- group_size,- })+ let group_size = u32::try_from(*group_size).map_err(|_| {+ WeightMatrixError::UnsupportedConfiguration(format!("microfloat group size {group_size} exceeds u32"))+ })?;+ let encoding = MicrofloatEncoding::new(*scale_mode, *bits, group_size)+ .map_err(|error| WeightMatrixError::UnsupportedConfiguration(error.to_string()))?;+ (layout.clone(), Some(QuantizationInfo::Microfloat(encoding)))},+ spec => return Err(WeightMatrixError::UnsupportedConfiguration(format!("{spec:?}"))),};Ok(ParsedWeightSpec {layout,@@ -79,6 +78,30 @@ pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeight})}+fn integer_quantization<B: Backend>(+ bits: u32,+ group_size: u32,+ method: QuantizationMethod,+) -> Result<QuantizationInfo, WeightMatrixError<B>> {+ let mode = match bits {+ 4 => QuantizationMode::U4,+ 8 => QuantizationMode::U8,+ _ => {+ return Err(WeightMatrixError::UnsupportedConfiguration(format!(+ "{method} bits={bits}, group_size={group_size}"+ )));+ },+ };+ if group_size == 0 {+ return Err(WeightMatrixError::UnsupportedConfiguration("group size must be non-zero".into()));+ }+ Ok(QuantizationInfo::Integer {+ mode,+ method,+ group_size,+ })+}+enum QuantizedCorrection<B: Backend> {Symmetric,Biases(Allocation<B>),@@ -88,13 +111,27 @@ enum QuantizedCorrection<B: Backend> {struct Quantized<B: Backend> {scales: Allocation<B>,correction: QuantizedCorrection<B>,- info: QuantizationInfo,+ mode: QuantizationMode,+ method: QuantizationMethod,+ group_size: u32,signed_codes: bool,}+struct Microfloat<B: Backend> {+ scales: Allocation<B>,+ outer_scales: Allocation<B>,+ metadata: MicrofloatMetadata,+}++enum WeightFormat<B: Backend> {+ FullPrecision,+ Integer(Quantized<B>),+ Microfloat(Microfloat<B>),+}+pub struct WeightMatrix<B: Backend> {values: Allocation<B>,- quantized: Option<Quantized<B>>,+ format: WeightFormat<B>,}impl<B: Backend> WeightMatrix<B> {@@ -117,28 +154,53 @@ impl<B: Backend> WeightMatrix<B> {}let (rows, columns) = physical_shape(&layout, output_dim, input_dim);- let Some(info) = quantization else {+ let Some(quantization) = quantization else {let values = tree.leaf("weights")?.validate(&[rows, columns], data_type)?.read_allocation()?;return Ok(Self {values,- quantized: None,+ format: WeightFormat::FullPrecision,});};- let group_size = info.group_size;- let packing_divisor = info.mode.packing_divisor();- let storage_data_type = info.mode.storage_type();+ let (mode, method, group_size) = match quantization {+ QuantizationInfo::Microfloat(encoding) => {+ let metadata = MicrofloatMetadata::new(encoding, rows, columns)+ .map_err(|error| WeightMatrixError::UnsupportedConfiguration(error.to_string()))?;+ let values = tree.leaf("weights")?.validate(&[rows, columns / 2], DataType::U8)?.read_allocation()?;+ let scales = tree+ .leaf("scales")?+ .validate(&[rows, columns / encoding.group_size], DataType::U8)?+ .read_allocation()?;+ // Preserving the artifact's established tensor name for the outer scale.+ let outer_scales = tree.leaf("global_scale")?.validate(&[1], data_type)?.read_allocation()?;+ return Ok(Self {+ values,+ format: WeightFormat::Microfloat(Microfloat {+ scales,+ outer_scales,+ metadata,+ }),+ });+ },+ QuantizationInfo::Integer {+ mode,+ method,+ group_size,+ } => (mode, method, group_size),+ };++ let packing_divisor = mode.packing_divisor();+ let storage_data_type = mode.storage_type();if !columns.is_multiple_of(packing_divisor) {return Err(WeightMatrixError::UnsupportedConfiguration(format!("stored columns {columns} are not divisible by packing divisor {packing_divisor}")));}let groups = columns.div_ceil(group_size);-let values =tree.leaf("weights")?.validate(&[rows, columns / packing_divisor], storage_data_type)?.read_allocation()?;let scales = tree.leaf("scales")?.validate(&[rows, groups], data_type)?.read_allocation()?;- let correction = match info.method {+ let correction = match method {QuantizationMethod::ScaleBias => QuantizedCorrection::Biases(tree.leaf("biases")?.validate(&[rows, groups], data_type)?.read_allocation()?,),@@ -149,13 +211,14 @@ impl<B: Backend> WeightMatrix<B> {),QuantizationMethod::ScaleSymmetric => QuantizedCorrection::Symmetric,};-Ok(Self {values,- quantized: Some(Quantized {+ format: WeightFormat::Integer(Quantized {scales,correction,- info,+ mode,+ method,+ group_size,signed_codes: false,}),})@@ -166,35 +229,64 @@ impl<B: Backend> WeightMatrix<B> {}pub fn quantization(&self) -> Option<QuantizationInfo> {- self.quantized.as_ref().map(|quantized| quantized.info)+ match &self.format {+ WeightFormat::FullPrecision => None,+ WeightFormat::Integer(quantized) => Some(QuantizationInfo::Integer {+ mode: quantized.mode,+ method: quantized.method,+ group_size: quantized.group_size,+ }),+ WeightFormat::Microfloat(microfloat) => Some(QuantizationInfo::Microfloat(microfloat.metadata.encoding)),+ }}pub fn scales(&self) -> Option<&Allocation<B>> {- self.quantized.as_ref().map(|quantized| &quantized.scales)+ match &self.format {+ WeightFormat::Integer(quantized) => Some(&quantized.scales),+ WeightFormat::Microfloat(microfloat) => Some(µfloat.scales),+ WeightFormat::FullPrecision => None,+ }}pub fn zero_points(&self) -> Option<&Allocation<B>> {- match &self.quantized.as_ref()?.correction {- QuantizedCorrection::ZeroPoints(zero_points) => Some(zero_points),- QuantizedCorrection::Biases(_) | QuantizedCorrection::Symmetric => None,+ match &self.format {+ WeightFormat::Integer(Quantized {+ correction: QuantizedCorrection::ZeroPoints(zero_points),+ ..+ }) => Some(zero_points),+ WeightFormat::FullPrecision | WeightFormat::Integer(_) | WeightFormat::Microfloat(_) => None,}}pub fn biases(&self) -> Option<&Allocation<B>> {- match &self.quantized.as_ref()?.correction {- QuantizedCorrection::Biases(biases) => Some(biases),- QuantizedCorrection::ZeroPoints(_) | QuantizedCorrection::Symmetric => None,+ match &self.format {+ WeightFormat::Integer(Quantized {+ correction: QuantizedCorrection::Biases(biases),+ ..+ }) => Some(biases),+ WeightFormat::FullPrecision | WeightFormat::Integer(_) | WeightFormat::Microfloat(_) => None,}}pub fn matmul_b(&self) -> MatmulB<'_, B> {- let Some(quantized) = self.quantized.as_ref() else {- return MatmulB::FullPrecision {- b: &self.values,- };+ let quantized = match &self.format {+ WeightFormat::FullPrecision => {+ return MatmulB::FullPrecision {+ b: &self.values,+ };+ },+ WeightFormat::Microfloat(microfloat) => {+ return MatmulB::Microfloat {+ codes: &self.values,+ scales: µfloat.scales,+ outer_scales: µfloat.outer_scales,+ metadata: microfloat.metadata,+ };+ },+ WeightFormat::Integer(quantized) => quantized,};- let mode = quantized.info.mode;- let group_size = quantized.info.group_size;+ let mode = quantized.mode;+ let group_size = quantized.group_size;let signed_codes = quantized.signed_codes;match &quantized.correction {QuantizedCorrection::Biases(biases) => MatmulB::ScaleBiasDequant {@@ -224,13 +316,13 @@ impl<B: Backend> WeightMatrix<B> {}pub fn make_codes_signed(&mut self) {- let Some(quantized) = self.quantized.as_mut() else {+ let WeightFormat::Integer(quantized) = &mut self.format else {return;};if quantized.signed_codes {return;}- let Some(sign_flip_mask) = quantized.info.mode.weight_codes_sign_flip_mask() else {+ let Some(sign_flip_mask) = quantized.mode.weight_codes_sign_flip_mask() else {return;};let broadcast_mask = u64::from(sign_flip_mask) * 0x0101_0101_0101_0101;@@ -251,3 +343,7 @@ fn physical_shape(Layout::InputOutput => (input_dim, output_dim),}}++#[cfg(test)]+#[path = "../../unit/encodable_block/weight_matrix/microfloat_test.rs"]+mod microfloat_test;diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rsnew file mode 100644index 000000000..6bac69a5f--- /dev/null+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rs@@ -0,0 +1,111 @@+use uzu_engine_macros::uzu_test;++use crate::{+ backends::{+ common::{+ Backend, Context, Encoder, Kernels,+ kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ microfloat::{MicrofloatEncoding, MicrofloatFormat, MicrofloatMetadata, decode_mxfp4},+ },+ cpu::Cpu,+ },+ data_type::DataType,+ tests::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ },+};++const K: usize = 32;+const N: usize = 4;++fn packed_codes() -> Vec<u8> {+ (0..N * K / 2)+ .map(|index| {+ let low = (index % 7 + 1) as u8;+ let high = ((index * 3 + 1) % 7 + 1) as u8;+ low | (high << 4)+ })+ .collect()+}++#[uzu_test]+fn cpu_executes_dense_mxfp4_matmul() {+ for row_count in [1, 5] {+ for group_size in [16, 32] {+ // Given packed MXFP4 weights with multiple batch and scale-group shapes.+ let input_values: Vec<f32> = (0..row_count * K).map(|index| (index % 13) as f32 * 0.125 - 0.5).collect();+ let codes = packed_codes();+ let scales: Vec<u8> = (0..N * K / group_size).map(|index| 126 + (index % 3) as u8).collect();+ let outer_scales = [1.25f32];+ let encoding =+ MicrofloatEncoding::new(MicrofloatFormat::Mxfp4, 4, group_size as u32).expect("valid MXFP4 encoding");+ let metadata = MicrofloatMetadata::new(encoding, N as u32, K as u32).expect("valid dense MXFP4 metadata");++ let context = <Cpu as Backend>::Context::new().expect("create CPU context");+ let input = alloc_allocation_with_data::<Cpu, f32>(context.as_ref(), &input_values);+ let codes = alloc_allocation_with_data::<Cpu, u8>(context.as_ref(), &codes);+ let scales = alloc_allocation_with_data::<Cpu, u8>(context.as_ref(), &scales);+ let outer_scales = alloc_allocation_with_data::<Cpu, f32>(context.as_ref(), &outer_scales);+ let mut output = alloc_allocation::<Cpu, f32>(context.as_ref(), row_count * N);+ let mut kernel = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(+ context.as_ref(),+ DataType::F32,+ DataType::F32,+ DataType::F32,+ )+ .expect("create CPU matmul kernel");+ let mut encoder = Encoder::<Cpu>::new(context.as_ref()).expect("create CPU encoder");++ // When the CPU backend executes a dense microfloat matrix multiplication.+ kernel+ .encode(+ MatmulArguments {+ a: MatmulA::FullPrecision {+ values: &input,+ offset: 0,+ },+ b: MatmulB::<Cpu>::Microfloat {+ codes: &codes,+ scales: &scales,+ outer_scales: &outer_scales,+ metadata,+ },+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut output,+ d_transform: MatmulDOps::none(),+ gather_indices: None,+ m: row_count as u32,+ n: N as u32,+ k: K as u32,+ },+ &mut encoder,+ )+ .expect("encode dense MXFP4 matmul");+ encoder.end_encoding().submit().wait_until_completed().expect("execute dense MXFP4 matmul");+ let actual = allocation_to_vec::<Cpu, f32>(&output);++ // Then its output agrees with independently decoded scalar accumulation.+ let codes: &[u8] = codes.as_slice();+ let scales: &[u8] = scales.as_slice();+ let mut expected = vec![0.0f32; row_count * N];+ for row in 0..row_count {+ for output_row in 0..N {+ for inner in 0..K {+ let packed = codes[output_row * K / 2 + inner / 2];+ let code = if inner.is_multiple_of(2) {+ packed & 0x0f+ } else {+ packed >> 4+ };+ let scale = scales[output_row * K / group_size + inner / group_size];+ expected[row * N + output_row] +=+ input_values[row * K + inner] * decode_mxfp4(code, scale, 1.25);+ }+ }+ }+ assert_eq_float(&expected, &actual, 1e-5, "CPU dense MXFP4");+ }+ }+}diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rsindex ab1716ec5..f543c08b3 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs@@ -2,6 +2,7 @@ mod a8w_bench;mod dispatch_paths_test;mod gemm_bench;mod gemv_test;+mod microfloat_test;mod quant_dispatch_test;mod quant_gemm_bench;mod quant_gemv_bench;diff --git a/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rs b/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rsnew file mode 100644index 000000000..b6d77344b--- /dev/null+++ b/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rs@@ -0,0 +1,20 @@+use uzu_engine_macros::uzu_test;++use super::{decode_e2m1, decode_e8m0};++#[uzu_test]+fn decodes_e2m1_and_e8m0_edges() {+ // Given every finite E2M1 magnitude and the E8M0 boundary encodings.+ let expected = [0.0f32, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0];++ // When the positive, negative, minimum-exponent, unit, and NaN codes are decoded.+ // Then E2M1 preserves every value and signed zero, while E8M0 preserves its edge semantics.+ for (code, value) in expected.into_iter().enumerate() {+ assert_eq!(decode_e2m1(code as u8), value);+ assert_eq!(decode_e2m1(code as u8 | 0b1000), -value);+ }+ assert_eq!(decode_e2m1(8).to_bits(), (-0.0f32).to_bits());+ assert_eq!(decode_e8m0(0).to_bits(), 0x0040_0000);+ assert_eq!(decode_e8m0(127), 1.0);+ assert!(decode_e8m0(255).is_nan());+}diff --git a/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rs b/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rsnew file mode 100644index 000000000..31e8b8f7b--- /dev/null+++ b/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rs@@ -0,0 +1,93 @@+use std::io::Write;++use half::bf16;+use serde_json::{Map, Value, json};+use tempfile::NamedTempFile;+use uzu_engine_macros::uzu_test;++use super::*;+use crate::{+ backends::{common::Context, cpu::Cpu},+ parameters::ParameterLoader,+};++fn add_tensor(+ header: &mut Map<String, Value>,+ payload: &mut Vec<u8>,+ name: &str,+ shape: &[u32],+ data_type: &str,+ data: &[u8],+) {+ let begin = payload.len();+ payload.extend_from_slice(data);+ header.insert(+ name.into(),+ json!({+ "dtype": data_type,+ "shape": shape,+ "data_offsets": [begin, payload.len()]+ }),+ );+}++fn dense_microfloat_parameter_file() -> NamedTempFile {+ const ROWS: u32 = 2;+ const COLUMNS: u32 = 32;+ const GROUP_SIZE: u32 = 16;++ let mut header = Map::new();+ header.insert(+ "__metadata__".into(),+ json!({+ "spec": json!({+ "type": "MicrofloatSpec",+ "bits": 4,+ "group_size": GROUP_SIZE,+ "scale_mode": "mxfp4",+ "layout": "output_input"+ }).to_string()+ }),+ );+ let mut payload = Vec::new();+ let codes: Vec<u8> = (0..ROWS * COLUMNS / 2).map(|index| 0x10 | (index % 8) as u8).collect();+ let scales: Vec<u8> = (0..ROWS * COLUMNS / GROUP_SIZE).map(|index| 126 + (index % 3) as u8).collect();+ let global_scale = bf16::from_f32(0.75).to_le_bytes();+ add_tensor(&mut header, &mut payload, "weights", &[ROWS, COLUMNS / 2], "U8", &codes);+ add_tensor(&mut header, &mut payload, "scales", &[ROWS, COLUMNS / GROUP_SIZE], "U8", &scales);+ add_tensor(&mut header, &mut payload, "global_scale", &[1], "BF16", &global_scale);++ let mut header = serde_json::to_vec(&Value::Object(header)).expect("serialize safetensors header");+ header.extend(std::iter::repeat_n(b' ', (8 - header.len() % 8) % 8));+ let mut file = NamedTempFile::new().expect("create dense MXFP4 fixture");+ file.write_all(&(header.len() as u64).to_le_bytes()).expect("write safetensors header length");+ file.write_all(&header).expect("write safetensors header");+ file.write_all(&payload).expect("write safetensors payload");+ file+}++#[uzu_test]+fn loads_dense_mxfp4_storage() {+ // Given a safetensors artifact whose metadata and tensors describe dense MXFP4 weights.+ let context = <Cpu as Backend>::Context::new().expect("create CPU context");+ let file = dense_microfloat_parameter_file();+ let loader = ParameterLoader::<Cpu>::new(file.as_file(), context.as_ref()).expect("load dense MXFP4 fixture");+ let tree = loader.tree();+ let spec = tree.metadata::<AnyWeightMatrixSpec>("spec").expect("load dense MXFP4 spec");++ // When the artifact is loaded through ParameterLoader and WeightMatrix.+ let matrix =+ WeightMatrix::load(&tree, spec, Layout::OutputInput, 2, 32, DataType::BF16).expect("load dense MXFP4 matrix");++ // Then the matrix retains its microfloat operand shape and consumes every source tensor.+ let MatmulB::Microfloat {+ metadata,+ ..+ } = matrix.matmul_b()+ else {+ panic!("dense MXFP4 matrix did not preserve its operand format");+ };+ assert_eq!(metadata.rows, 2);+ assert_eq!(metadata.columns, 32);+ tree.assert_all_tensors_validated().expect("validate dense MXFP4 tensors");+}