Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

mxfp4

xhjkl49ce0ceeb652
diff --git a/crates/uzu-engine/Cargo.toml b/crates/uzu-engine/Cargo.toml
index 31c25959c..d59c2240c 100644
--- a/crates/uzu-engine/Cargo.toml
+++ b/crates/uzu-engine/Cargo.toml
@@ -85,6 +85,7 @@ xgrammar = { workspace = true, optional = true }
schemars = { workspace = true, optional = true }
[dev-dependencies]
+tempfile.workspace = true
test-tag.workspace = true
proptest.workspace = true
criterion.workspace = true
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs
index 96be2b02a..89e7a0bd0 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs
@@ -1,7 +1,7 @@
use thiserror::Error;
use crate::{
- backends::common::{Backend, gpu_types::gemm::GemmDTransform},
+ backends::common::{Backend, gpu_types::gemm::GemmDTransform, microfloat::MicrofloatFormat},
data_type::DataType,
};
@@ -20,6 +20,10 @@ pub enum MatmulError<B: Backend> {
UnsupportedLayout {
path: &'static str,
},
+ #[error("Microfloat storage does not match the requested matrix")]
+ InvalidMicrofloatStorage,
+ #[error("Unsupported microfloat weight format: {0:?}")]
+ UnsupportedMicrofloat(MicrofloatFormat),
#[error("Incompatible A operand for {path}: {reason}")]
IncompatibleA {
path: &'static str,
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
index e472a5477..2026d5dad 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
@@ -2,6 +2,7 @@ use crate::{
backends::common::{
Allocation, Backend, BufferArg,
gpu_types::{QuantizationMode, gemm::GemmBPrologueKind},
+ microfloat::MicrofloatMetadata,
},
data_type::DataType,
};
@@ -10,6 +11,12 @@ pub enum MatmulB<'a, B: Backend, TB: BufferArg<'a, B> = &'a Allocation<B>> {
FullPrecision {
b: TB,
},
+ Microfloat {
+ codes: &'a Allocation<B>,
+ scales: &'a Allocation<B>,
+ outer_scales: &'a Allocation<B>,
+ metadata: MicrofloatMetadata,
+ },
ScaleBiasDequant {
b: &'a Allocation<B>,
scales: &'a Allocation<B>,
@@ -40,6 +47,9 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {
match self {
Self::FullPrecision {
..
+ }
+ | Self::Microfloat {
+ ..
} => GemmBPrologueKind::FullPrecision,
Self::ScaleBiasDequant {
..
@@ -58,6 +68,10 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {
Self::FullPrecision {
..
} => None,
+ Self::Microfloat {
+ metadata,
+ ..
+ } => Some(metadata.encoding.bits),
Self::ScaleBiasDequant {
mode,
..
@@ -78,6 +92,10 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {
Self::FullPrecision {
..
} => None,
+ Self::Microfloat {
+ metadata,
+ ..
+ } => Some(metadata.encoding.group_size),
Self::ScaleBiasDequant {
group_size,
..
@@ -97,6 +115,9 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {
match self {
Self::FullPrecision {
..
+ }
+ | Self::Microfloat {
+ ..
} => false,
Self::ScaleBiasDequant {
signed_codes,
diff --git a/crates/uzu-engine/src/backends/common/microfloat/decode.rs b/crates/uzu-engine/src/backends/common/microfloat/decode.rs
new file mode 100644
index 000000000..c5242eeb9
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/microfloat/decode.rs
@@ -0,0 +1,23 @@
+#[inline]
+pub fn decode_e2m1(code: u8) -> f32 {
+ const VALUES: [f32; 16] = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0];
+ VALUES[usize::from(code & 0x0f)]
+}
+
+#[inline]
+pub fn decode_e8m0(exponent: u8) -> f32 {
+ match exponent {
+ 0 => f32::from_bits(0x0040_0000),
+ 255 => f32::NAN,
+ exponent => f32::from_bits(u32::from(exponent) << 23),
+ }
+}
+
+#[inline]
+pub fn decode_mxfp4(
+ code: u8,
+ exponent: u8,
+ outer_scale: f32,
+) -> f32 {
+ decode_e2m1(code) * decode_e8m0(exponent) * outer_scale
+}
diff --git a/crates/uzu-engine/src/backends/common/microfloat/encoding.rs b/crates/uzu-engine/src/backends/common/microfloat/encoding.rs
new file mode 100644
index 000000000..8c0c3c7fa
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/microfloat/encoding.rs
@@ -0,0 +1,38 @@
+use super::{MicrofloatError, MicrofloatFormat};
+
+/// How packed microfloat bytes are interpreted, separate from matrix dimensions.
+///
+/// With MXFP4 group size 16, every 16 values along the input axis occupy eight
+/// packed code bytes and share one E8M0 scale.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct MicrofloatEncoding {
+ pub format: MicrofloatFormat,
+ pub bits: u32,
+ pub group_size: u32,
+}
+
+impl MicrofloatEncoding {
+ pub fn new(
+ format: MicrofloatFormat,
+ bits: u32,
+ group_size: u32,
+ ) -> Result<Self, MicrofloatError> {
+ if bits != 4 {
+ return Err(MicrofloatError::UnsupportedBits {
+ format,
+ bits,
+ });
+ }
+ if !matches!(group_size, 16 | 32) {
+ return Err(MicrofloatError::UnsupportedGroupSize {
+ format,
+ group_size,
+ });
+ }
+ Ok(Self {
+ format,
+ bits,
+ group_size,
+ })
+ }
+}
diff --git a/crates/uzu-engine/src/backends/common/microfloat/error.rs b/crates/uzu-engine/src/backends/common/microfloat/error.rs
new file mode 100644
index 000000000..abe796eb4
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/microfloat/error.rs
@@ -0,0 +1,26 @@
+use thiserror::Error;
+
+use super::MicrofloatFormat;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
+pub enum MicrofloatError {
+ #[error("unsupported {format:?} bit width: {bits}")]
+ UnsupportedBits {
+ format: MicrofloatFormat,
+ bits: u32,
+ },
+ #[error("unsupported {format:?} group size: {group_size}")]
+ UnsupportedGroupSize {
+ format: MicrofloatFormat,
+ group_size: u32,
+ },
+ #[error("microfloat rows and columns must be nonzero")]
+ EmptyShape,
+ #[error("microfloat columns {columns} are not divisible by group size {group_size}")]
+ MisalignedColumns {
+ columns: u32,
+ group_size: u32,
+ },
+ #[error("microfloat storage size overflows usize")]
+ SizeOverflow,
+}
diff --git a/crates/uzu-engine/src/backends/common/microfloat/format.rs b/crates/uzu-engine/src/backends/common/microfloat/format.rs
new file mode 100644
index 000000000..8356176d7
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/microfloat/format.rs
@@ -0,0 +1,8 @@
+use uzu_engine_macros::uzu_config;
+
+#[derive(Copy, Eq)]
+#[uzu_config]
+#[serde(rename_all = "snake_case")]
+pub enum MicrofloatFormat {
+ Mxfp4,
+}
diff --git a/crates/uzu-engine/src/backends/common/microfloat/metadata.rs b/crates/uzu-engine/src/backends/common/microfloat/metadata.rs
new file mode 100644
index 000000000..b234b44c5
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/microfloat/metadata.rs
@@ -0,0 +1,67 @@
+use super::{MicrofloatEncoding, MicrofloatError};
+
+/// Physical shape and derived strides for one microfloat matrix.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct MicrofloatMetadata {
+ pub encoding: MicrofloatEncoding,
+ pub rows: u32,
+ pub columns: u32,
+}
+
+impl MicrofloatMetadata {
+ pub fn new(
+ encoding: MicrofloatEncoding,
+ rows: u32,
+ columns: u32,
+ ) -> Result<Self, MicrofloatError> {
+ if rows == 0 || columns == 0 {
+ return Err(MicrofloatError::EmptyShape);
+ }
+ if !columns.is_multiple_of(encoding.group_size) {
+ return Err(MicrofloatError::MisalignedColumns {
+ columns,
+ group_size: encoding.group_size,
+ });
+ }
+ let metadata = Self {
+ encoding,
+ rows,
+ columns,
+ };
+ metadata.checked_code_matrix_stride().ok_or(MicrofloatError::SizeOverflow)?;
+ metadata.checked_scale_matrix_stride().ok_or(MicrofloatError::SizeOverflow)?;
+ Ok(metadata)
+ }
+
+ pub fn code_row_stride(self) -> usize {
+ self.columns as usize / 2
+ }
+
+ pub fn scale_row_stride(self) -> usize {
+ self.columns as usize / self.encoding.group_size as usize
+ }
+
+ pub fn code_matrix_stride(self) -> usize {
+ self.checked_code_matrix_stride().expect("MicrofloatMetadata validates code storage size")
+ }
+
+ pub fn scale_matrix_stride(self) -> usize {
+ self.checked_scale_matrix_stride().expect("MicrofloatMetadata validates scale storage size")
+ }
+
+ pub fn required_code_bytes(self) -> usize {
+ self.code_matrix_stride()
+ }
+
+ pub fn required_scale_bytes(self) -> usize {
+ self.scale_matrix_stride()
+ }
+
+ fn checked_code_matrix_stride(self) -> Option<usize> {
+ (self.rows as usize).checked_mul(self.code_row_stride())
+ }
+
+ fn checked_scale_matrix_stride(self) -> Option<usize> {
+ (self.rows as usize).checked_mul(self.scale_row_stride())
+ }
+}
diff --git a/crates/uzu-engine/src/backends/common/microfloat/mod.rs b/crates/uzu-engine/src/backends/common/microfloat/mod.rs
new file mode 100644
index 000000000..21f7c3f6e
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/microfloat/mod.rs
@@ -0,0 +1,15 @@
+mod decode;
+mod encoding;
+mod error;
+mod format;
+mod metadata;
+
+pub use decode::{decode_e2m1, decode_e8m0, decode_mxfp4};
+pub use encoding::MicrofloatEncoding;
+pub use error::MicrofloatError;
+pub use format::MicrofloatFormat;
+pub use metadata::MicrofloatMetadata;
+
+#[cfg(test)]
+#[path = "../../../../unit/backends/common/microfloat/decode_test.rs"]
+mod decode_test;
diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rs
index d3b57f26d..ff845b66b 100644
--- a/crates/uzu-engine/src/backends/common/mod.rs
+++ b/crates/uzu-engine/src/backends/common/mod.rs
@@ -8,6 +8,7 @@ mod encoder;
pub mod gpu_types;
mod hazard_tracker;
pub mod kernel;
+pub mod microfloat;
pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};
pub use backend::Backend;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
index c3db18e05..d64f2fd38 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
@@ -58,6 +58,26 @@ impl MatmulKernel for MatmulCpuKernel {
arguments: MatmulArguments<'a, 'b, 'd, Cpu, TB>,
encoder: &mut Encoder<Cpu>,
) -> Result<(), CpuError> {
+ if let MatmulB::Microfloat {
+ codes,
+ scales,
+ outer_scales,
+ metadata,
+ } = &arguments.b
+ {
+ let rows_match = arguments.gather_indices.is_some() || metadata.rows == arguments.n;
+ if !arguments.b_transpose
+ || arguments.b_leading_dimension.is_some()
+ || !rows_match
+ || metadata.columns != arguments.k
+ || codes.size() < metadata.required_code_bytes()
+ || scales.size() < metadata.required_scale_bytes()
+ || outer_scales.size() < self.weights_data_type.size_in_bytes()
+ {
+ return Err(MatmulError::InvalidMicrofloatStorage.into());
+ }
+ }
+
let output_scale = arguments.d_transform.ab_scale;
let accumulate = arguments.d_transform.accumulate;
let bias_alloc = arguments.d_transform.bias;
@@ -183,6 +203,9 @@ impl MatmulKernel for MatmulCpuKernel {
},
WeightData::FullPrecision {
..
+ }
+ | WeightData::Microfloat {
+ ..
} => None,
};
@@ -273,6 +296,25 @@ impl MatmulKernel for MatmulCpuKernel {
};
scale * quantized_value + bias_term
},
+ WeightData::Microfloat {
+ codes,
+ scales,
+ outer_scales,
+ metadata,
+ } => {
+ let code_index = b_col * metadata.code_row_stride() + inner / 2;
+ let packed = *codes.as_ptr().add(code_index);
+ let code = if inner.is_multiple_of(2) {
+ packed & 0x0f
+ } else {
+ packed >> 4
+ };
+ let scale_index = b_col * metadata.scale_row_stride()
+ + inner / metadata.encoding.group_size as usize;
+ let exponent = *scales.as_ptr().add(scale_index);
+ let outer_scale = read_f32(outer_scales.as_ptr(), weights_data_type, 0);
+ crate::backends::common::microfloat::decode_mxfp4(code, exponent, outer_scale)
+ },
};
accumulator += a_value * b_value;
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
index 41c92a735..4e48ce623 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
@@ -2,7 +2,10 @@ use half::{bf16, f16};
use crate::{
backends::{
- common::{AsBufferRangeRef, BufferArg, gpu_types::QuantizationMode, kernel::matmul::MatmulB},
+ common::{
+ AsBufferRangeRef, BufferArg, gpu_types::QuantizationMode, kernel::matmul::MatmulB,
+ microfloat::MicrofloatMetadata,
+ },
cpu::Cpu,
},
data_type::DataType,
@@ -24,6 +27,12 @@ pub(super) enum WeightData {
group_size: usize,
signed_codes: bool,
},
+ Microfloat {
+ codes: SendPtr<u8>,
+ scales: SendPtr<u8>,
+ outer_scales: SendPtr<u8>,
+ metadata: MicrofloatMetadata,
+ },
}
impl WeightData {
@@ -58,6 +67,17 @@ impl WeightData {
transpose: b_transpose,
}
},
+ MatmulB::Microfloat {
+ codes,
+ scales,
+ outer_scales,
+ metadata,
+ } => WeightData::Microfloat {
+ codes: alloc_ptr(codes),
+ scales: alloc_ptr(scales),
+ outer_scales: alloc_ptr(outer_scales),
+ metadata,
+ },
MatmulB::ScaleBiasDequant {
b: weights,
scales,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 3b27f7299..532fc8b0a 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -191,6 +191,12 @@ impl GemmKernel {
let use_mxu = plan.engine == GemmEngine::Mxu;
match b {
+ MatmulB::Microfloat {
+ metadata,
+ ..
+ } => {
+ return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format).into());
+ },
MatmulB::FullPrecision {
b: weights,
} => {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index a78ae14ed..7f425f552 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -266,6 +266,9 @@ impl GemvKernel {
let (scales, zero_points, biases) = match &b {
MatmulB::FullPrecision {
..
+ }
+ | MatmulB::Microfloat {
+ ..
} => (None, None, None),
MatmulB::ScaleBiasDequant {
scales,
@@ -287,6 +290,12 @@ impl GemvKernel {
let context = encoder.context();
let pipeline = self.get_or_create(context, specialization)?;
match b {
+ MatmulB::Microfloat {
+ metadata,
+ ..
+ } => {
+ return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format));
+ },
MatmulB::FullPrecision {
b: weights,
} => pipeline.encode(
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
index 14fa5a4c1..eb187459e 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
@@ -17,7 +17,10 @@ use crate::{
gpu_types::gemm::{GemmBPrologueKind, GemmTiling},
kernel::{
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
- matmul::{A8ActivationPlan, ActivationFormat, MatmulArguments, MatmulError, MatmulKernel, MatmulShape},
+ matmul::{
+ A8ActivationPlan, ActivationFormat, MatmulArguments, MatmulB, MatmulError, MatmulKernel,
+ MatmulShape,
+ },
},
},
metal::{Metal, context::MetalContext, error::MetalError},
@@ -217,6 +220,13 @@ impl MatmulKernel for MatmulMetalKernel {
arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
+ if let MatmulB::Microfloat {
+ metadata,
+ ..
+ } = &arguments.b
+ {
+ return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format).into());
+ }
let shape = MatmulShape::from_arguments(&arguments);
let plan = match self.select_dispatch(&shape, encoder.context()) {
MatmulDispatch::Gemv(gemv) => {
diff --git a/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rs b/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rs
new file mode 100644
index 000000000..45a7e99b0
--- /dev/null
+++ b/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rs
@@ -0,0 +1,11 @@
+use uzu_engine_macros::uzu_config;
+
+use crate::{backends::common::microfloat::MicrofloatFormat, config::weight_matrix::Layout};
+
+#[uzu_config(super::WeightMatrixSpec)]
+pub struct MicrofloatSpec {
+ pub bits: u32,
+ pub group_size: usize,
+ pub scale_mode: MicrofloatFormat,
+ pub layout: Layout,
+}
diff --git a/crates/uzu-engine/src/config/weight_matrix/mod.rs b/crates/uzu-engine/src/config/weight_matrix/mod.rs
index 5f10ee506..405745762 100644
--- a/crates/uzu-engine/src/config/weight_matrix/mod.rs
+++ b/crates/uzu-engine/src/config/weight_matrix/mod.rs
@@ -4,6 +4,7 @@ pub mod full_precision_spec;
pub mod hybrid_spec;
pub mod int_spec;
pub mod low_rank_spec;
+pub mod microfloat_spec;
pub mod mlx_spec;
#[uzu_config]
@@ -18,6 +19,7 @@ pub enum Layout {
low_rank_spec::LowRankSpec,
hybrid_spec::HybridSpec,
int_spec::IntSpec,
+ microfloat_spec::MicrofloatSpec,
mlx_spec::MLXSpec
)]
pub struct WeightMatrixSpec;
diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rs
index 250a1e70c..a48d7771b 100644
--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs
@@ -7,7 +7,7 @@ use crate::{
},
config::weight_matrix::{AnyWeightMatrixSpec, Layout},
data_type::DataType,
- encodable_block::weight_matrix::{WeightMatrix, WeightMatrixError},
+ encodable_block::weight_matrix::{QuantizationInfo, WeightMatrix, WeightMatrixError},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -65,21 +65,32 @@ impl<B: Backend> EmbeddingTable<B> {
}
let lookup = match matrix.quantization() {
- None => LookupKernel::FullPrecision(
- <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
- .map_err(EmbeddingTableError::BackendError)?,
- ),
- Some(info) => LookupKernel::Quantized(
- <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
+ None => {
+ let kernel = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
+ .map_err(EmbeddingTableError::BackendError)?;
+ LookupKernel::FullPrecision(kernel)
+ },
+ Some(QuantizationInfo::Microfloat(_)) => {
+ return Err(EmbeddingTableError::UnsupportedConfiguration(
+ "microfloat embedding tables are not supported".into(),
+ ));
+ },
+ Some(QuantizationInfo::Integer {
+ mode,
+ method,
+ group_size,
+ }) => {
+ let kernel = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
context,
data_type,
- info.group_size,
- info.mode,
- info.method,
+ group_size,
+ mode,
+ method,
output_hadamard_factors.is_some(),
)
- .map_err(EmbeddingTableError::BackendError)?,
- ),
+ .map_err(EmbeddingTableError::BackendError)?;
+ LookupKernel::Quantized(kernel)
+ },
};
Ok(Self {
diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
index 06c435434..9dbb66aeb 100644
--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs
+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
@@ -5,8 +5,9 @@ use crate::{
Allocation, Backend,
gpu_types::{QuantizationMethod, QuantizationMode},
kernel::matmul::MatmulB,
+ microfloat::{MicrofloatEncoding, MicrofloatMetadata},
},
- config::weight_matrix::{AnyWeightMatrixSpec, Layout},
+ config::weight_matrix::{AnyWeightMatrixSpec, Layout, microfloat_spec::MicrofloatSpec},
data_type::DataType,
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -19,11 +20,14 @@ pub enum WeightMatrixError<B: Backend> {
UnsupportedConfiguration(String),
}
-#[derive(Clone, Copy)]
-pub struct QuantizationInfo {
- pub mode: QuantizationMode,
- pub method: QuantizationMethod,
- pub group_size: u32,
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum QuantizationInfo {
+ Integer {
+ mode: QuantizationMode,
+ method: QuantizationMethod,
+ group_size: u32,
+ },
+ Microfloat(MicrofloatEncoding),
}
pub struct ParsedWeightSpec {
@@ -32,46 +36,41 @@ pub struct ParsedWeightSpec {
}
pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeightSpec, WeightMatrixError<B>> {
- let (layout, quantized) = match spec {
+ let (layout, quantization) = match spec {
AnyWeightMatrixSpec::FullPrecisionSpec(spec) => (spec.layout.clone(), None),
AnyWeightMatrixSpec::MLXSpec(spec) => {
- (spec.layout.clone(), Some((spec.bits, spec.group_size, QuantizationMethod::ScaleBias)))
+ let quantization = integer_quantization::<B>(spec.bits, spec.group_size, QuantizationMethod::ScaleBias)?;
+ (spec.layout.clone(), Some(quantization))
},
- AnyWeightMatrixSpec::IntSpec(spec) => (
- spec.layout.clone(),
- Some((
- spec.bits,
- spec.group_size,
- if spec.is_symmetric {
- QuantizationMethod::ScaleSymmetric
- } else {
- QuantizationMethod::ScaleZeroPoint
- },
- )),
- ),
- spec => return Err(WeightMatrixError::UnsupportedConfiguration(format!("{spec:?}"))),
- };
- let quantization = match quantized {
- None => None,
- Some((bits, group_size, method)) => {
- let mode = match bits {
- 4 => QuantizationMode::U4,
- 8 => QuantizationMode::U8,
- _ => {
- return Err(WeightMatrixError::UnsupportedConfiguration(format!(
- "{method} bits={bits}, group_size={group_size}"
- )));
- },
+ AnyWeightMatrixSpec::IntSpec(spec) => {
+ let method = if spec.is_symmetric {
+ QuantizationMethod::ScaleSymmetric
+ } else {
+ QuantizationMethod::ScaleZeroPoint
};
- if group_size == 0 {
- return Err(WeightMatrixError::UnsupportedConfiguration("group size must be non-zero".into()));
+ let quantization = integer_quantization::<B>(spec.bits, spec.group_size, method)?;
+ (spec.layout.clone(), Some(quantization))
+ },
+ AnyWeightMatrixSpec::MicrofloatSpec(MicrofloatSpec {
+ bits,
+ group_size,
+ scale_mode,
+ layout,
+ ..
+ }) => {
+ if layout != &Layout::OutputInput {
+ return Err(WeightMatrixError::UnsupportedConfiguration(format!(
+ "microfloat matrices require output-input layout, got {layout:?}"
+ )));
}
- Some(QuantizationInfo {
- mode,
- method,
- group_size,
- })
+ let group_size = u32::try_from(*group_size).map_err(|_| {
+ WeightMatrixError::UnsupportedConfiguration(format!("microfloat group size {group_size} exceeds u32"))
+ })?;
+ let encoding = MicrofloatEncoding::new(*scale_mode, *bits, group_size)
+ .map_err(|error| WeightMatrixError::UnsupportedConfiguration(error.to_string()))?;
+ (layout.clone(), Some(QuantizationInfo::Microfloat(encoding)))
},
+ spec => return Err(WeightMatrixError::UnsupportedConfiguration(format!("{spec:?}"))),
};
Ok(ParsedWeightSpec {
layout,
@@ -79,6 +78,30 @@ pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeight
})
}
+fn integer_quantization<B: Backend>(
+ bits: u32,
+ group_size: u32,
+ method: QuantizationMethod,
+) -> Result<QuantizationInfo, WeightMatrixError<B>> {
+ let mode = match bits {
+ 4 => QuantizationMode::U4,
+ 8 => QuantizationMode::U8,
+ _ => {
+ return Err(WeightMatrixError::UnsupportedConfiguration(format!(
+ "{method} bits={bits}, group_size={group_size}"
+ )));
+ },
+ };
+ if group_size == 0 {
+ return Err(WeightMatrixError::UnsupportedConfiguration("group size must be non-zero".into()));
+ }
+ Ok(QuantizationInfo::Integer {
+ mode,
+ method,
+ group_size,
+ })
+}
+
enum QuantizedCorrection<B: Backend> {
Symmetric,
Biases(Allocation<B>),
@@ -88,13 +111,27 @@ enum QuantizedCorrection<B: Backend> {
struct Quantized<B: Backend> {
scales: Allocation<B>,
correction: QuantizedCorrection<B>,
- info: QuantizationInfo,
+ mode: QuantizationMode,
+ method: QuantizationMethod,
+ group_size: u32,
signed_codes: bool,
}
+struct Microfloat<B: Backend> {
+ scales: Allocation<B>,
+ outer_scales: Allocation<B>,
+ metadata: MicrofloatMetadata,
+}
+
+enum WeightFormat<B: Backend> {
+ FullPrecision,
+ Integer(Quantized<B>),
+ Microfloat(Microfloat<B>),
+}
+
pub struct WeightMatrix<B: Backend> {
values: Allocation<B>,
- quantized: Option<Quantized<B>>,
+ format: WeightFormat<B>,
}
impl<B: Backend> WeightMatrix<B> {
@@ -117,28 +154,53 @@ impl<B: Backend> WeightMatrix<B> {
}
let (rows, columns) = physical_shape(&layout, output_dim, input_dim);
- let Some(info) = quantization else {
+ let Some(quantization) = quantization else {
let values = tree.leaf("weights")?.validate(&[rows, columns], data_type)?.read_allocation()?;
return Ok(Self {
values,
- quantized: None,
+ format: WeightFormat::FullPrecision,
});
};
- let group_size = info.group_size;
- let packing_divisor = info.mode.packing_divisor();
- let storage_data_type = info.mode.storage_type();
+ let (mode, method, group_size) = match quantization {
+ QuantizationInfo::Microfloat(encoding) => {
+ let metadata = MicrofloatMetadata::new(encoding, rows, columns)
+ .map_err(|error| WeightMatrixError::UnsupportedConfiguration(error.to_string()))?;
+ let values = tree.leaf("weights")?.validate(&[rows, columns / 2], DataType::U8)?.read_allocation()?;
+ let scales = tree
+ .leaf("scales")?
+ .validate(&[rows, columns / encoding.group_size], DataType::U8)?
+ .read_allocation()?;
+ // Preserving the artifact's established tensor name for the outer scale.
+ let outer_scales = tree.leaf("global_scale")?.validate(&[1], data_type)?.read_allocation()?;
+ return Ok(Self {
+ values,
+ format: WeightFormat::Microfloat(Microfloat {
+ scales,
+ outer_scales,
+ metadata,
+ }),
+ });
+ },
+ QuantizationInfo::Integer {
+ mode,
+ method,
+ group_size,
+ } => (mode, method, group_size),
+ };
+
+ let packing_divisor = mode.packing_divisor();
+ let storage_data_type = mode.storage_type();
if !columns.is_multiple_of(packing_divisor) {
return Err(WeightMatrixError::UnsupportedConfiguration(format!(
"stored columns {columns} are not divisible by packing divisor {packing_divisor}"
)));
}
let groups = columns.div_ceil(group_size);
-
let values =
tree.leaf("weights")?.validate(&[rows, columns / packing_divisor], storage_data_type)?.read_allocation()?;
let scales = tree.leaf("scales")?.validate(&[rows, groups], data_type)?.read_allocation()?;
- let correction = match info.method {
+ let correction = match method {
QuantizationMethod::ScaleBias => QuantizedCorrection::Biases(
tree.leaf("biases")?.validate(&[rows, groups], data_type)?.read_allocation()?,
),
@@ -149,13 +211,14 @@ impl<B: Backend> WeightMatrix<B> {
),
QuantizationMethod::ScaleSymmetric => QuantizedCorrection::Symmetric,
};
-
Ok(Self {
values,
- quantized: Some(Quantized {
+ format: WeightFormat::Integer(Quantized {
scales,
correction,
- info,
+ mode,
+ method,
+ group_size,
signed_codes: false,
}),
})
@@ -166,35 +229,64 @@ impl<B: Backend> WeightMatrix<B> {
}
pub fn quantization(&self) -> Option<QuantizationInfo> {
- self.quantized.as_ref().map(|quantized| quantized.info)
+ match &self.format {
+ WeightFormat::FullPrecision => None,
+ WeightFormat::Integer(quantized) => Some(QuantizationInfo::Integer {
+ mode: quantized.mode,
+ method: quantized.method,
+ group_size: quantized.group_size,
+ }),
+ WeightFormat::Microfloat(microfloat) => Some(QuantizationInfo::Microfloat(microfloat.metadata.encoding)),
+ }
}
pub fn scales(&self) -> Option<&Allocation<B>> {
- self.quantized.as_ref().map(|quantized| &quantized.scales)
+ match &self.format {
+ WeightFormat::Integer(quantized) => Some(&quantized.scales),
+ WeightFormat::Microfloat(microfloat) => Some(&microfloat.scales),
+ WeightFormat::FullPrecision => None,
+ }
}
pub fn zero_points(&self) -> Option<&Allocation<B>> {
- match &self.quantized.as_ref()?.correction {
- QuantizedCorrection::ZeroPoints(zero_points) => Some(zero_points),
- QuantizedCorrection::Biases(_) | QuantizedCorrection::Symmetric => None,
+ match &self.format {
+ WeightFormat::Integer(Quantized {
+ correction: QuantizedCorrection::ZeroPoints(zero_points),
+ ..
+ }) => Some(zero_points),
+ WeightFormat::FullPrecision | WeightFormat::Integer(_) | WeightFormat::Microfloat(_) => None,
}
}
pub fn biases(&self) -> Option<&Allocation<B>> {
- match &self.quantized.as_ref()?.correction {
- QuantizedCorrection::Biases(biases) => Some(biases),
- QuantizedCorrection::ZeroPoints(_) | QuantizedCorrection::Symmetric => None,
+ match &self.format {
+ WeightFormat::Integer(Quantized {
+ correction: QuantizedCorrection::Biases(biases),
+ ..
+ }) => Some(biases),
+ WeightFormat::FullPrecision | WeightFormat::Integer(_) | WeightFormat::Microfloat(_) => None,
}
}
pub fn matmul_b(&self) -> MatmulB<'_, B> {
- let Some(quantized) = self.quantized.as_ref() else {
- return MatmulB::FullPrecision {
- b: &self.values,
- };
+ let quantized = match &self.format {
+ WeightFormat::FullPrecision => {
+ return MatmulB::FullPrecision {
+ b: &self.values,
+ };
+ },
+ WeightFormat::Microfloat(microfloat) => {
+ return MatmulB::Microfloat {
+ codes: &self.values,
+ scales: &microfloat.scales,
+ outer_scales: &microfloat.outer_scales,
+ metadata: microfloat.metadata,
+ };
+ },
+ WeightFormat::Integer(quantized) => quantized,
};
- let mode = quantized.info.mode;
- let group_size = quantized.info.group_size;
+ let mode = quantized.mode;
+ let group_size = quantized.group_size;
let signed_codes = quantized.signed_codes;
match &quantized.correction {
QuantizedCorrection::Biases(biases) => MatmulB::ScaleBiasDequant {
@@ -224,13 +316,13 @@ impl<B: Backend> WeightMatrix<B> {
}
pub fn make_codes_signed(&mut self) {
- let Some(quantized) = self.quantized.as_mut() else {
+ let WeightFormat::Integer(quantized) = &mut self.format else {
return;
};
if quantized.signed_codes {
return;
}
- let Some(sign_flip_mask) = quantized.info.mode.weight_codes_sign_flip_mask() else {
+ let Some(sign_flip_mask) = quantized.mode.weight_codes_sign_flip_mask() else {
return;
};
let broadcast_mask = u64::from(sign_flip_mask) * 0x0101_0101_0101_0101;
@@ -251,3 +343,7 @@ fn physical_shape(
Layout::InputOutput => (input_dim, output_dim),
}
}
+
+#[cfg(test)]
+#[path = "../../unit/encodable_block/weight_matrix/microfloat_test.rs"]
+mod microfloat_test;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rs
new file mode 100644
index 000000000..6bac69a5f
--- /dev/null
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rs
@@ -0,0 +1,111 @@
+use uzu_engine_macros::uzu_test;
+
+use crate::{
+ backends::{
+ common::{
+ Backend, Context, Encoder, Kernels,
+ kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
+ microfloat::{MicrofloatEncoding, MicrofloatFormat, MicrofloatMetadata, decode_mxfp4},
+ },
+ cpu::Cpu,
+ },
+ data_type::DataType,
+ tests::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ },
+};
+
+const K: usize = 32;
+const N: usize = 4;
+
+fn packed_codes() -> Vec<u8> {
+ (0..N * K / 2)
+ .map(|index| {
+ let low = (index % 7 + 1) as u8;
+ let high = ((index * 3 + 1) % 7 + 1) as u8;
+ low | (high << 4)
+ })
+ .collect()
+}
+
+#[uzu_test]
+fn cpu_executes_dense_mxfp4_matmul() {
+ for row_count in [1, 5] {
+ for group_size in [16, 32] {
+ // Given packed MXFP4 weights with multiple batch and scale-group shapes.
+ let input_values: Vec<f32> = (0..row_count * K).map(|index| (index % 13) as f32 * 0.125 - 0.5).collect();
+ let codes = packed_codes();
+ let scales: Vec<u8> = (0..N * K / group_size).map(|index| 126 + (index % 3) as u8).collect();
+ let outer_scales = [1.25f32];
+ let encoding =
+ MicrofloatEncoding::new(MicrofloatFormat::Mxfp4, 4, group_size as u32).expect("valid MXFP4 encoding");
+ let metadata = MicrofloatMetadata::new(encoding, N as u32, K as u32).expect("valid dense MXFP4 metadata");
+
+ let context = <Cpu as Backend>::Context::new().expect("create CPU context");
+ let input = alloc_allocation_with_data::<Cpu, f32>(context.as_ref(), &input_values);
+ let codes = alloc_allocation_with_data::<Cpu, u8>(context.as_ref(), &codes);
+ let scales = alloc_allocation_with_data::<Cpu, u8>(context.as_ref(), &scales);
+ let outer_scales = alloc_allocation_with_data::<Cpu, f32>(context.as_ref(), &outer_scales);
+ let mut output = alloc_allocation::<Cpu, f32>(context.as_ref(), row_count * N);
+ let mut kernel = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ context.as_ref(),
+ DataType::F32,
+ DataType::F32,
+ DataType::F32,
+ )
+ .expect("create CPU matmul kernel");
+ let mut encoder = Encoder::<Cpu>::new(context.as_ref()).expect("create CPU encoder");
+
+ // When the CPU backend executes a dense microfloat matrix multiplication.
+ kernel
+ .encode(
+ MatmulArguments {
+ a: MatmulA::FullPrecision {
+ values: &input,
+ offset: 0,
+ },
+ b: MatmulB::<Cpu>::Microfloat {
+ codes: &codes,
+ scales: &scales,
+ outer_scales: &outer_scales,
+ metadata,
+ },
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut output,
+ d_transform: MatmulDOps::none(),
+ gather_indices: None,
+ m: row_count as u32,
+ n: N as u32,
+ k: K as u32,
+ },
+ &mut encoder,
+ )
+ .expect("encode dense MXFP4 matmul");
+ encoder.end_encoding().submit().wait_until_completed().expect("execute dense MXFP4 matmul");
+ let actual = allocation_to_vec::<Cpu, f32>(&output);
+
+ // Then its output agrees with independently decoded scalar accumulation.
+ let codes: &[u8] = codes.as_slice();
+ let scales: &[u8] = scales.as_slice();
+ let mut expected = vec![0.0f32; row_count * N];
+ for row in 0..row_count {
+ for output_row in 0..N {
+ for inner in 0..K {
+ let packed = codes[output_row * K / 2 + inner / 2];
+ let code = if inner.is_multiple_of(2) {
+ packed & 0x0f
+ } else {
+ packed >> 4
+ };
+ let scale = scales[output_row * K / group_size + inner / group_size];
+ expected[row * N + output_row] +=
+ input_values[row * K + inner] * decode_mxfp4(code, scale, 1.25);
+ }
+ }
+ }
+ assert_eq_float(&expected, &actual, 1e-5, "CPU dense MXFP4");
+ }
+ }
+}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs
index ab1716ec5..f543c08b3 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs
@@ -2,6 +2,7 @@ mod a8w_bench;
mod dispatch_paths_test;
mod gemm_bench;
mod gemv_test;
+mod microfloat_test;
mod quant_dispatch_test;
mod quant_gemm_bench;
mod quant_gemv_bench;
diff --git a/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rs b/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rs
new file mode 100644
index 000000000..b6d77344b
--- /dev/null
+++ b/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rs
@@ -0,0 +1,20 @@
+use uzu_engine_macros::uzu_test;
+
+use super::{decode_e2m1, decode_e8m0};
+
+#[uzu_test]
+fn decodes_e2m1_and_e8m0_edges() {
+ // Given every finite E2M1 magnitude and the E8M0 boundary encodings.
+ let expected = [0.0f32, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0];
+
+ // When the positive, negative, minimum-exponent, unit, and NaN codes are decoded.
+ // Then E2M1 preserves every value and signed zero, while E8M0 preserves its edge semantics.
+ for (code, value) in expected.into_iter().enumerate() {
+ assert_eq!(decode_e2m1(code as u8), value);
+ assert_eq!(decode_e2m1(code as u8 | 0b1000), -value);
+ }
+ assert_eq!(decode_e2m1(8).to_bits(), (-0.0f32).to_bits());
+ assert_eq!(decode_e8m0(0).to_bits(), 0x0040_0000);
+ assert_eq!(decode_e8m0(127), 1.0);
+ assert!(decode_e8m0(255).is_nan());
+}
diff --git a/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rs b/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rs
new file mode 100644
index 000000000..31e8b8f7b
--- /dev/null
+++ b/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rs
@@ -0,0 +1,93 @@
+use std::io::Write;
+
+use half::bf16;
+use serde_json::{Map, Value, json};
+use tempfile::NamedTempFile;
+use uzu_engine_macros::uzu_test;
+
+use super::*;
+use crate::{
+ backends::{common::Context, cpu::Cpu},
+ parameters::ParameterLoader,
+};
+
+fn add_tensor(
+ header: &mut Map<String, Value>,
+ payload: &mut Vec<u8>,
+ name: &str,
+ shape: &[u32],
+ data_type: &str,
+ data: &[u8],
+) {
+ let begin = payload.len();
+ payload.extend_from_slice(data);
+ header.insert(
+ name.into(),
+ json!({
+ "dtype": data_type,
+ "shape": shape,
+ "data_offsets": [begin, payload.len()]
+ }),
+ );
+}
+
+fn dense_microfloat_parameter_file() -> NamedTempFile {
+ const ROWS: u32 = 2;
+ const COLUMNS: u32 = 32;
+ const GROUP_SIZE: u32 = 16;
+
+ let mut header = Map::new();
+ header.insert(
+ "__metadata__".into(),
+ json!({
+ "spec": json!({
+ "type": "MicrofloatSpec",
+ "bits": 4,
+ "group_size": GROUP_SIZE,
+ "scale_mode": "mxfp4",
+ "layout": "output_input"
+ }).to_string()
+ }),
+ );
+ let mut payload = Vec::new();
+ let codes: Vec<u8> = (0..ROWS * COLUMNS / 2).map(|index| 0x10 | (index % 8) as u8).collect();
+ let scales: Vec<u8> = (0..ROWS * COLUMNS / GROUP_SIZE).map(|index| 126 + (index % 3) as u8).collect();
+ let global_scale = bf16::from_f32(0.75).to_le_bytes();
+ add_tensor(&mut header, &mut payload, "weights", &[ROWS, COLUMNS / 2], "U8", &codes);
+ add_tensor(&mut header, &mut payload, "scales", &[ROWS, COLUMNS / GROUP_SIZE], "U8", &scales);
+ add_tensor(&mut header, &mut payload, "global_scale", &[1], "BF16", &global_scale);
+
+ let mut header = serde_json::to_vec(&Value::Object(header)).expect("serialize safetensors header");
+ header.extend(std::iter::repeat_n(b' ', (8 - header.len() % 8) % 8));
+ let mut file = NamedTempFile::new().expect("create dense MXFP4 fixture");
+ file.write_all(&(header.len() as u64).to_le_bytes()).expect("write safetensors header length");
+ file.write_all(&header).expect("write safetensors header");
+ file.write_all(&payload).expect("write safetensors payload");
+ file
+}
+
+#[uzu_test]
+fn loads_dense_mxfp4_storage() {
+ // Given a safetensors artifact whose metadata and tensors describe dense MXFP4 weights.
+ let context = <Cpu as Backend>::Context::new().expect("create CPU context");
+ let file = dense_microfloat_parameter_file();
+ let loader = ParameterLoader::<Cpu>::new(file.as_file(), context.as_ref()).expect("load dense MXFP4 fixture");
+ let tree = loader.tree();
+ let spec = tree.metadata::<AnyWeightMatrixSpec>("spec").expect("load dense MXFP4 spec");
+
+ // When the artifact is loaded through ParameterLoader and WeightMatrix.
+ let matrix =
+ WeightMatrix::load(&tree, spec, Layout::OutputInput, 2, 32, DataType::BF16).expect("load dense MXFP4 matrix");
+
+ // Then the matrix retains its microfloat operand shape and consumes every source tensor.
+ let MatmulB::Microfloat {
+ metadata,
+ ..
+ } = matrix.matmul_b()
+ else {
+ panic!("dense MXFP4 matrix did not preserve its operand format");
+ };
+ assert_eq!(metadata.rows, 2);
+ assert_eq!(metadata.columns, 32);
+ tree.assert_all_tensors_validated().expect("validate dense MXFP4 tensors");
+}