Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Revert "mxfp4" (#804)

uuuvnd0699c22ccfa
diff --git a/crates/uzu-engine/Cargo.toml b/crates/uzu-engine/Cargo.toml
index d59c2240c..31c25959c 100644
--- a/crates/uzu-engine/Cargo.toml
+++ b/crates/uzu-engine/Cargo.toml
@@ -85,7 +85,6 @@ xgrammar = { workspace = true, optional = true }
schemars = { workspace = true, optional = true }
[dev-dependencies]
-tempfile.workspace = true
test-tag.workspace = true
proptest.workspace = true
criterion.workspace = true
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs
index 89e7a0bd0..96be2b02a 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs
@@ -1,7 +1,7 @@
use thiserror::Error;
use crate::{
- backends::common::{Backend, gpu_types::gemm::GemmDTransform, microfloat::MicrofloatFormat},
+ backends::common::{Backend, gpu_types::gemm::GemmDTransform},
data_type::DataType,
};
@@ -20,10 +20,6 @@ pub enum MatmulError<B: Backend> {
UnsupportedLayout {
path: &'static str,
},
- #[error("Microfloat storage does not match the requested matrix")]
- InvalidMicrofloatStorage,
- #[error("Unsupported microfloat weight format: {0:?}")]
- UnsupportedMicrofloat(MicrofloatFormat),
#[error("Incompatible A operand for {path}: {reason}")]
IncompatibleA {
path: &'static str,
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
index 2026d5dad..e472a5477 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
@@ -2,7 +2,6 @@ use crate::{
backends::common::{
Allocation, Backend, BufferArg,
gpu_types::{QuantizationMode, gemm::GemmBPrologueKind},
- microfloat::MicrofloatMetadata,
},
data_type::DataType,
};
@@ -11,12 +10,6 @@ pub enum MatmulB<'a, B: Backend, TB: BufferArg<'a, B> = &'a Allocation<B>> {
FullPrecision {
b: TB,
},
- Microfloat {
- codes: &'a Allocation<B>,
- scales: &'a Allocation<B>,
- outer_scales: &'a Allocation<B>,
- metadata: MicrofloatMetadata,
- },
ScaleBiasDequant {
b: &'a Allocation<B>,
scales: &'a Allocation<B>,
@@ -47,9 +40,6 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {
match self {
Self::FullPrecision {
..
- }
- | Self::Microfloat {
- ..
} => GemmBPrologueKind::FullPrecision,
Self::ScaleBiasDequant {
..
@@ -68,10 +58,6 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {
Self::FullPrecision {
..
} => None,
- Self::Microfloat {
- metadata,
- ..
- } => Some(metadata.encoding.bits),
Self::ScaleBiasDequant {
mode,
..
@@ -92,10 +78,6 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {
Self::FullPrecision {
..
} => None,
- Self::Microfloat {
- metadata,
- ..
- } => Some(metadata.encoding.group_size),
Self::ScaleBiasDequant {
group_size,
..
@@ -115,9 +97,6 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {
match self {
Self::FullPrecision {
..
- }
- | Self::Microfloat {
- ..
} => false,
Self::ScaleBiasDequant {
signed_codes,
diff --git a/crates/uzu-engine/src/backends/common/microfloat/decode.rs b/crates/uzu-engine/src/backends/common/microfloat/decode.rs
deleted file mode 100644
index c5242eeb9..000000000
--- a/crates/uzu-engine/src/backends/common/microfloat/decode.rs
+++ /dev/null
@@ -1,23 +0,0 @@
-#[inline]
-pub fn decode_e2m1(code: u8) -> f32 {
- const VALUES: [f32; 16] = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0];
- VALUES[usize::from(code & 0x0f)]
-}
-
-#[inline]
-pub fn decode_e8m0(exponent: u8) -> f32 {
- match exponent {
- 0 => f32::from_bits(0x0040_0000),
- 255 => f32::NAN,
- exponent => f32::from_bits(u32::from(exponent) << 23),
- }
-}
-
-#[inline]
-pub fn decode_mxfp4(
- code: u8,
- exponent: u8,
- outer_scale: f32,
-) -> f32 {
- decode_e2m1(code) * decode_e8m0(exponent) * outer_scale
-}
diff --git a/crates/uzu-engine/src/backends/common/microfloat/encoding.rs b/crates/uzu-engine/src/backends/common/microfloat/encoding.rs
deleted file mode 100644
index 8c0c3c7fa..000000000
--- a/crates/uzu-engine/src/backends/common/microfloat/encoding.rs
+++ /dev/null
@@ -1,38 +0,0 @@
-use super::{MicrofloatError, MicrofloatFormat};
-
-/// How packed microfloat bytes are interpreted, separate from matrix dimensions.
-///
-/// With MXFP4 group size 16, every 16 values along the input axis occupy eight
-/// packed code bytes and share one E8M0 scale.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct MicrofloatEncoding {
- pub format: MicrofloatFormat,
- pub bits: u32,
- pub group_size: u32,
-}
-
-impl MicrofloatEncoding {
- pub fn new(
- format: MicrofloatFormat,
- bits: u32,
- group_size: u32,
- ) -> Result<Self, MicrofloatError> {
- if bits != 4 {
- return Err(MicrofloatError::UnsupportedBits {
- format,
- bits,
- });
- }
- if !matches!(group_size, 16 | 32) {
- return Err(MicrofloatError::UnsupportedGroupSize {
- format,
- group_size,
- });
- }
- Ok(Self {
- format,
- bits,
- group_size,
- })
- }
-}
diff --git a/crates/uzu-engine/src/backends/common/microfloat/error.rs b/crates/uzu-engine/src/backends/common/microfloat/error.rs
deleted file mode 100644
index abe796eb4..000000000
--- a/crates/uzu-engine/src/backends/common/microfloat/error.rs
+++ /dev/null
@@ -1,26 +0,0 @@
-use thiserror::Error;
-
-use super::MicrofloatFormat;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
-pub enum MicrofloatError {
- #[error("unsupported {format:?} bit width: {bits}")]
- UnsupportedBits {
- format: MicrofloatFormat,
- bits: u32,
- },
- #[error("unsupported {format:?} group size: {group_size}")]
- UnsupportedGroupSize {
- format: MicrofloatFormat,
- group_size: u32,
- },
- #[error("microfloat rows and columns must be nonzero")]
- EmptyShape,
- #[error("microfloat columns {columns} are not divisible by group size {group_size}")]
- MisalignedColumns {
- columns: u32,
- group_size: u32,
- },
- #[error("microfloat storage size overflows usize")]
- SizeOverflow,
-}
diff --git a/crates/uzu-engine/src/backends/common/microfloat/format.rs b/crates/uzu-engine/src/backends/common/microfloat/format.rs
deleted file mode 100644
index 8356176d7..000000000
--- a/crates/uzu-engine/src/backends/common/microfloat/format.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-use uzu_engine_macros::uzu_config;
-
-#[derive(Copy, Eq)]
-#[uzu_config]
-#[serde(rename_all = "snake_case")]
-pub enum MicrofloatFormat {
- Mxfp4,
-}
diff --git a/crates/uzu-engine/src/backends/common/microfloat/metadata.rs b/crates/uzu-engine/src/backends/common/microfloat/metadata.rs
deleted file mode 100644
index b234b44c5..000000000
--- a/crates/uzu-engine/src/backends/common/microfloat/metadata.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-use super::{MicrofloatEncoding, MicrofloatError};
-
-/// Physical shape and derived strides for one microfloat matrix.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct MicrofloatMetadata {
- pub encoding: MicrofloatEncoding,
- pub rows: u32,
- pub columns: u32,
-}
-
-impl MicrofloatMetadata {
- pub fn new(
- encoding: MicrofloatEncoding,
- rows: u32,
- columns: u32,
- ) -> Result<Self, MicrofloatError> {
- if rows == 0 || columns == 0 {
- return Err(MicrofloatError::EmptyShape);
- }
- if !columns.is_multiple_of(encoding.group_size) {
- return Err(MicrofloatError::MisalignedColumns {
- columns,
- group_size: encoding.group_size,
- });
- }
- let metadata = Self {
- encoding,
- rows,
- columns,
- };
- metadata.checked_code_matrix_stride().ok_or(MicrofloatError::SizeOverflow)?;
- metadata.checked_scale_matrix_stride().ok_or(MicrofloatError::SizeOverflow)?;
- Ok(metadata)
- }
-
- pub fn code_row_stride(self) -> usize {
- self.columns as usize / 2
- }
-
- pub fn scale_row_stride(self) -> usize {
- self.columns as usize / self.encoding.group_size as usize
- }
-
- pub fn code_matrix_stride(self) -> usize {
- self.checked_code_matrix_stride().expect("MicrofloatMetadata validates code storage size")
- }
-
- pub fn scale_matrix_stride(self) -> usize {
- self.checked_scale_matrix_stride().expect("MicrofloatMetadata validates scale storage size")
- }
-
- pub fn required_code_bytes(self) -> usize {
- self.code_matrix_stride()
- }
-
- pub fn required_scale_bytes(self) -> usize {
- self.scale_matrix_stride()
- }
-
- fn checked_code_matrix_stride(self) -> Option<usize> {
- (self.rows as usize).checked_mul(self.code_row_stride())
- }
-
- fn checked_scale_matrix_stride(self) -> Option<usize> {
- (self.rows as usize).checked_mul(self.scale_row_stride())
- }
-}
diff --git a/crates/uzu-engine/src/backends/common/microfloat/mod.rs b/crates/uzu-engine/src/backends/common/microfloat/mod.rs
deleted file mode 100644
index 21f7c3f6e..000000000
--- a/crates/uzu-engine/src/backends/common/microfloat/mod.rs
+++ /dev/null
@@ -1,15 +0,0 @@
-mod decode;
-mod encoding;
-mod error;
-mod format;
-mod metadata;
-
-pub use decode::{decode_e2m1, decode_e8m0, decode_mxfp4};
-pub use encoding::MicrofloatEncoding;
-pub use error::MicrofloatError;
-pub use format::MicrofloatFormat;
-pub use metadata::MicrofloatMetadata;
-
-#[cfg(test)]
-#[path = "../../../../unit/backends/common/microfloat/decode_test.rs"]
-mod decode_test;
diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rs
index ff845b66b..d3b57f26d 100644
--- a/crates/uzu-engine/src/backends/common/mod.rs
+++ b/crates/uzu-engine/src/backends/common/mod.rs
@@ -8,7 +8,6 @@ mod encoder;
pub mod gpu_types;
mod hazard_tracker;
pub mod kernel;
-pub mod microfloat;
pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};
pub use backend::Backend;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
index d64f2fd38..c3db18e05 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
@@ -58,26 +58,6 @@ impl MatmulKernel for MatmulCpuKernel {
arguments: MatmulArguments<'a, 'b, 'd, Cpu, TB>,
encoder: &mut Encoder<Cpu>,
) -> Result<(), CpuError> {
- if let MatmulB::Microfloat {
- codes,
- scales,
- outer_scales,
- metadata,
- } = &arguments.b
- {
- let rows_match = arguments.gather_indices.is_some() || metadata.rows == arguments.n;
- if !arguments.b_transpose
- || arguments.b_leading_dimension.is_some()
- || !rows_match
- || metadata.columns != arguments.k
- || codes.size() < metadata.required_code_bytes()
- || scales.size() < metadata.required_scale_bytes()
- || outer_scales.size() < self.weights_data_type.size_in_bytes()
- {
- return Err(MatmulError::InvalidMicrofloatStorage.into());
- }
- }
-
let output_scale = arguments.d_transform.ab_scale;
let accumulate = arguments.d_transform.accumulate;
let bias_alloc = arguments.d_transform.bias;
@@ -203,9 +183,6 @@ impl MatmulKernel for MatmulCpuKernel {
},
WeightData::FullPrecision {
..
- }
- | WeightData::Microfloat {
- ..
} => None,
};
@@ -296,25 +273,6 @@ impl MatmulKernel for MatmulCpuKernel {
};
scale * quantized_value + bias_term
},
- WeightData::Microfloat {
- codes,
- scales,
- outer_scales,
- metadata,
- } => {
- let code_index = b_col * metadata.code_row_stride() + inner / 2;
- let packed = *codes.as_ptr().add(code_index);
- let code = if inner.is_multiple_of(2) {
- packed & 0x0f
- } else {
- packed >> 4
- };
- let scale_index = b_col * metadata.scale_row_stride()
- + inner / metadata.encoding.group_size as usize;
- let exponent = *scales.as_ptr().add(scale_index);
- let outer_scale = read_f32(outer_scales.as_ptr(), weights_data_type, 0);
- crate::backends::common::microfloat::decode_mxfp4(code, exponent, outer_scale)
- },
};
accumulator += a_value * b_value;
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
index 4e48ce623..41c92a735 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
@@ -2,10 +2,7 @@ use half::{bf16, f16};
use crate::{
backends::{
- common::{
- AsBufferRangeRef, BufferArg, gpu_types::QuantizationMode, kernel::matmul::MatmulB,
- microfloat::MicrofloatMetadata,
- },
+ common::{AsBufferRangeRef, BufferArg, gpu_types::QuantizationMode, kernel::matmul::MatmulB},
cpu::Cpu,
},
data_type::DataType,
@@ -27,12 +24,6 @@ pub(super) enum WeightData {
group_size: usize,
signed_codes: bool,
},
- Microfloat {
- codes: SendPtr<u8>,
- scales: SendPtr<u8>,
- outer_scales: SendPtr<u8>,
- metadata: MicrofloatMetadata,
- },
}
impl WeightData {
@@ -67,17 +58,6 @@ impl WeightData {
transpose: b_transpose,
}
},
- MatmulB::Microfloat {
- codes,
- scales,
- outer_scales,
- metadata,
- } => WeightData::Microfloat {
- codes: alloc_ptr(codes),
- scales: alloc_ptr(scales),
- outer_scales: alloc_ptr(outer_scales),
- metadata,
- },
MatmulB::ScaleBiasDequant {
b: weights,
scales,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 532fc8b0a..3b27f7299 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -191,12 +191,6 @@ impl GemmKernel {
let use_mxu = plan.engine == GemmEngine::Mxu;
match b {
- MatmulB::Microfloat {
- metadata,
- ..
- } => {
- return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format).into());
- },
MatmulB::FullPrecision {
b: weights,
} => {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 7f425f552..a78ae14ed 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -266,9 +266,6 @@ impl GemvKernel {
let (scales, zero_points, biases) = match &b {
MatmulB::FullPrecision {
..
- }
- | MatmulB::Microfloat {
- ..
} => (None, None, None),
MatmulB::ScaleBiasDequant {
scales,
@@ -290,12 +287,6 @@ impl GemvKernel {
let context = encoder.context();
let pipeline = self.get_or_create(context, specialization)?;
match b {
- MatmulB::Microfloat {
- metadata,
- ..
- } => {
- return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format));
- },
MatmulB::FullPrecision {
b: weights,
} => pipeline.encode(
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
index eb187459e..14fa5a4c1 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
@@ -17,10 +17,7 @@ use crate::{
gpu_types::gemm::{GemmBPrologueKind, GemmTiling},
kernel::{
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
- matmul::{
- A8ActivationPlan, ActivationFormat, MatmulArguments, MatmulB, MatmulError, MatmulKernel,
- MatmulShape,
- },
+ matmul::{A8ActivationPlan, ActivationFormat, MatmulArguments, MatmulError, MatmulKernel, MatmulShape},
},
},
metal::{Metal, context::MetalContext, error::MetalError},
@@ -220,13 +217,6 @@ impl MatmulKernel for MatmulMetalKernel {
arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
- if let MatmulB::Microfloat {
- metadata,
- ..
- } = &arguments.b
- {
- return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format).into());
- }
let shape = MatmulShape::from_arguments(&arguments);
let plan = match self.select_dispatch(&shape, encoder.context()) {
MatmulDispatch::Gemv(gemv) => {
diff --git a/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rs b/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rs
deleted file mode 100644
index 45a7e99b0..000000000
--- a/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rs
+++ /dev/null
@@ -1,11 +0,0 @@
-use uzu_engine_macros::uzu_config;
-
-use crate::{backends::common::microfloat::MicrofloatFormat, config::weight_matrix::Layout};
-
-#[uzu_config(super::WeightMatrixSpec)]
-pub struct MicrofloatSpec {
- pub bits: u32,
- pub group_size: usize,
- pub scale_mode: MicrofloatFormat,
- pub layout: Layout,
-}
diff --git a/crates/uzu-engine/src/config/weight_matrix/mod.rs b/crates/uzu-engine/src/config/weight_matrix/mod.rs
index 405745762..5f10ee506 100644
--- a/crates/uzu-engine/src/config/weight_matrix/mod.rs
+++ b/crates/uzu-engine/src/config/weight_matrix/mod.rs
@@ -4,7 +4,6 @@ pub mod full_precision_spec;
pub mod hybrid_spec;
pub mod int_spec;
pub mod low_rank_spec;
-pub mod microfloat_spec;
pub mod mlx_spec;
#[uzu_config]
@@ -19,7 +18,6 @@ pub enum Layout {
low_rank_spec::LowRankSpec,
hybrid_spec::HybridSpec,
int_spec::IntSpec,
- microfloat_spec::MicrofloatSpec,
mlx_spec::MLXSpec
)]
pub struct WeightMatrixSpec;
diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rs
index a48d7771b..250a1e70c 100644
--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs
@@ -7,7 +7,7 @@ use crate::{
},
config::weight_matrix::{AnyWeightMatrixSpec, Layout},
data_type::DataType,
- encodable_block::weight_matrix::{QuantizationInfo, WeightMatrix, WeightMatrixError},
+ encodable_block::weight_matrix::{WeightMatrix, WeightMatrixError},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -65,32 +65,21 @@ impl<B: Backend> EmbeddingTable<B> {
}
let lookup = match matrix.quantization() {
- None => {
- let kernel = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
- .map_err(EmbeddingTableError::BackendError)?;
- LookupKernel::FullPrecision(kernel)
- },
- Some(QuantizationInfo::Microfloat(_)) => {
- return Err(EmbeddingTableError::UnsupportedConfiguration(
- "microfloat embedding tables are not supported".into(),
- ));
- },
- Some(QuantizationInfo::Integer {
- mode,
- method,
- group_size,
- }) => {
- let kernel = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
+ None => LookupKernel::FullPrecision(
+ <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
+ .map_err(EmbeddingTableError::BackendError)?,
+ ),
+ Some(info) => LookupKernel::Quantized(
+ <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
context,
data_type,
- group_size,
- mode,
- method,
+ info.group_size,
+ info.mode,
+ info.method,
output_hadamard_factors.is_some(),
)
- .map_err(EmbeddingTableError::BackendError)?;
- LookupKernel::Quantized(kernel)
- },
+ .map_err(EmbeddingTableError::BackendError)?,
+ ),
};
Ok(Self {
diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
index 9dbb66aeb..06c435434 100644
--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs
+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
@@ -5,9 +5,8 @@ use crate::{
Allocation, Backend,
gpu_types::{QuantizationMethod, QuantizationMode},
kernel::matmul::MatmulB,
- microfloat::{MicrofloatEncoding, MicrofloatMetadata},
},
- config::weight_matrix::{AnyWeightMatrixSpec, Layout, microfloat_spec::MicrofloatSpec},
+ config::weight_matrix::{AnyWeightMatrixSpec, Layout},
data_type::DataType,
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -20,14 +19,11 @@ pub enum WeightMatrixError<B: Backend> {
UnsupportedConfiguration(String),
}
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum QuantizationInfo {
- Integer {
- mode: QuantizationMode,
- method: QuantizationMethod,
- group_size: u32,
- },
- Microfloat(MicrofloatEncoding),
+#[derive(Clone, Copy)]
+pub struct QuantizationInfo {
+ pub mode: QuantizationMode,
+ pub method: QuantizationMethod,
+ pub group_size: u32,
}
pub struct ParsedWeightSpec {
@@ -36,41 +32,46 @@ pub struct ParsedWeightSpec {
}
pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeightSpec, WeightMatrixError<B>> {
- let (layout, quantization) = match spec {
+ let (layout, quantized) = match spec {
AnyWeightMatrixSpec::FullPrecisionSpec(spec) => (spec.layout.clone(), None),
AnyWeightMatrixSpec::MLXSpec(spec) => {
- let quantization = integer_quantization::<B>(spec.bits, spec.group_size, QuantizationMethod::ScaleBias)?;
- (spec.layout.clone(), Some(quantization))
+ (spec.layout.clone(), Some((spec.bits, spec.group_size, QuantizationMethod::ScaleBias)))
},
- AnyWeightMatrixSpec::IntSpec(spec) => {
- let method = if spec.is_symmetric {
- QuantizationMethod::ScaleSymmetric
- } else {
- QuantizationMethod::ScaleZeroPoint
+ AnyWeightMatrixSpec::IntSpec(spec) => (
+ spec.layout.clone(),
+ Some((
+ spec.bits,
+ spec.group_size,
+ if spec.is_symmetric {
+ QuantizationMethod::ScaleSymmetric
+ } else {
+ QuantizationMethod::ScaleZeroPoint
+ },
+ )),
+ ),
+ spec => return Err(WeightMatrixError::UnsupportedConfiguration(format!("{spec:?}"))),
+ };
+ let quantization = match quantized {
+ None => None,
+ Some((bits, group_size, method)) => {
+ let mode = match bits {
+ 4 => QuantizationMode::U4,
+ 8 => QuantizationMode::U8,
+ _ => {
+ return Err(WeightMatrixError::UnsupportedConfiguration(format!(
+ "{method} bits={bits}, group_size={group_size}"
+ )));
+ },
};
- let quantization = integer_quantization::<B>(spec.bits, spec.group_size, method)?;
- (spec.layout.clone(), Some(quantization))
- },
- AnyWeightMatrixSpec::MicrofloatSpec(MicrofloatSpec {
- bits,
- group_size,
- scale_mode,
- layout,
- ..
- }) => {
- if layout != &Layout::OutputInput {
- return Err(WeightMatrixError::UnsupportedConfiguration(format!(
- "microfloat matrices require output-input layout, got {layout:?}"
- )));
+ if group_size == 0 {
+ return Err(WeightMatrixError::UnsupportedConfiguration("group size must be non-zero".into()));
}
- let group_size = u32::try_from(*group_size).map_err(|_| {
- WeightMatrixError::UnsupportedConfiguration(format!("microfloat group size {group_size} exceeds u32"))
- })?;
- let encoding = MicrofloatEncoding::new(*scale_mode, *bits, group_size)
- .map_err(|error| WeightMatrixError::UnsupportedConfiguration(error.to_string()))?;
- (layout.clone(), Some(QuantizationInfo::Microfloat(encoding)))
+ Some(QuantizationInfo {
+ mode,
+ method,
+ group_size,
+ })
},
- spec => return Err(WeightMatrixError::UnsupportedConfiguration(format!("{spec:?}"))),
};
Ok(ParsedWeightSpec {
layout,
@@ -78,30 +79,6 @@ pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeight
})
}
-fn integer_quantization<B: Backend>(
- bits: u32,
- group_size: u32,
- method: QuantizationMethod,
-) -> Result<QuantizationInfo, WeightMatrixError<B>> {
- let mode = match bits {
- 4 => QuantizationMode::U4,
- 8 => QuantizationMode::U8,
- _ => {
- return Err(WeightMatrixError::UnsupportedConfiguration(format!(
- "{method} bits={bits}, group_size={group_size}"
- )));
- },
- };
- if group_size == 0 {
- return Err(WeightMatrixError::UnsupportedConfiguration("group size must be non-zero".into()));
- }
- Ok(QuantizationInfo::Integer {
- mode,
- method,
- group_size,
- })
-}
-
enum QuantizedCorrection<B: Backend> {
Symmetric,
Biases(Allocation<B>),
@@ -111,27 +88,13 @@ enum QuantizedCorrection<B: Backend> {
struct Quantized<B: Backend> {
scales: Allocation<B>,
correction: QuantizedCorrection<B>,
- mode: QuantizationMode,
- method: QuantizationMethod,
- group_size: u32,
+ info: QuantizationInfo,
signed_codes: bool,
}
-struct Microfloat<B: Backend> {
- scales: Allocation<B>,
- outer_scales: Allocation<B>,
- metadata: MicrofloatMetadata,
-}
-
-enum WeightFormat<B: Backend> {
- FullPrecision,
- Integer(Quantized<B>),
- Microfloat(Microfloat<B>),
-}
-
pub struct WeightMatrix<B: Backend> {
values: Allocation<B>,
- format: WeightFormat<B>,
+ quantized: Option<Quantized<B>>,
}
impl<B: Backend> WeightMatrix<B> {
@@ -154,53 +117,28 @@ impl<B: Backend> WeightMatrix<B> {
}
let (rows, columns) = physical_shape(&layout, output_dim, input_dim);
- let Some(quantization) = quantization else {
+ let Some(info) = quantization else {
let values = tree.leaf("weights")?.validate(&[rows, columns], data_type)?.read_allocation()?;
return Ok(Self {
values,
- format: WeightFormat::FullPrecision,
+ quantized: None,
});
};
- let (mode, method, group_size) = match quantization {
- QuantizationInfo::Microfloat(encoding) => {
- let metadata = MicrofloatMetadata::new(encoding, rows, columns)
- .map_err(|error| WeightMatrixError::UnsupportedConfiguration(error.to_string()))?;
- let values = tree.leaf("weights")?.validate(&[rows, columns / 2], DataType::U8)?.read_allocation()?;
- let scales = tree
- .leaf("scales")?
- .validate(&[rows, columns / encoding.group_size], DataType::U8)?
- .read_allocation()?;
- // Preserving the artifact's established tensor name for the outer scale.
- let outer_scales = tree.leaf("global_scale")?.validate(&[1], data_type)?.read_allocation()?;
- return Ok(Self {
- values,
- format: WeightFormat::Microfloat(Microfloat {
- scales,
- outer_scales,
- metadata,
- }),
- });
- },
- QuantizationInfo::Integer {
- mode,
- method,
- group_size,
- } => (mode, method, group_size),
- };
-
- let packing_divisor = mode.packing_divisor();
- let storage_data_type = mode.storage_type();
+ let group_size = info.group_size;
+ let packing_divisor = info.mode.packing_divisor();
+ let storage_data_type = info.mode.storage_type();
if !columns.is_multiple_of(packing_divisor) {
return Err(WeightMatrixError::UnsupportedConfiguration(format!(
"stored columns {columns} are not divisible by packing divisor {packing_divisor}"
)));
}
let groups = columns.div_ceil(group_size);
+
let values =
tree.leaf("weights")?.validate(&[rows, columns / packing_divisor], storage_data_type)?.read_allocation()?;
let scales = tree.leaf("scales")?.validate(&[rows, groups], data_type)?.read_allocation()?;
- let correction = match method {
+ let correction = match info.method {
QuantizationMethod::ScaleBias => QuantizedCorrection::Biases(
tree.leaf("biases")?.validate(&[rows, groups], data_type)?.read_allocation()?,
),
@@ -211,14 +149,13 @@ impl<B: Backend> WeightMatrix<B> {
),
QuantizationMethod::ScaleSymmetric => QuantizedCorrection::Symmetric,
};
+
Ok(Self {
values,
- format: WeightFormat::Integer(Quantized {
+ quantized: Some(Quantized {
scales,
correction,
- mode,
- method,
- group_size,
+ info,
signed_codes: false,
}),
})
@@ -229,64 +166,35 @@ impl<B: Backend> WeightMatrix<B> {
}
pub fn quantization(&self) -> Option<QuantizationInfo> {
- match &self.format {
- WeightFormat::FullPrecision => None,
- WeightFormat::Integer(quantized) => Some(QuantizationInfo::Integer {
- mode: quantized.mode,
- method: quantized.method,
- group_size: quantized.group_size,
- }),
- WeightFormat::Microfloat(microfloat) => Some(QuantizationInfo::Microfloat(microfloat.metadata.encoding)),
- }
+ self.quantized.as_ref().map(|quantized| quantized.info)
}
pub fn scales(&self) -> Option<&Allocation<B>> {
- match &self.format {
- WeightFormat::Integer(quantized) => Some(&quantized.scales),
- WeightFormat::Microfloat(microfloat) => Some(&microfloat.scales),
- WeightFormat::FullPrecision => None,
- }
+ self.quantized.as_ref().map(|quantized| &quantized.scales)
}
pub fn zero_points(&self) -> Option<&Allocation<B>> {
- match &self.format {
- WeightFormat::Integer(Quantized {
- correction: QuantizedCorrection::ZeroPoints(zero_points),
- ..
- }) => Some(zero_points),
- WeightFormat::FullPrecision | WeightFormat::Integer(_) | WeightFormat::Microfloat(_) => None,
+ match &self.quantized.as_ref()?.correction {
+ QuantizedCorrection::ZeroPoints(zero_points) => Some(zero_points),
+ QuantizedCorrection::Biases(_) | QuantizedCorrection::Symmetric => None,
}
}
pub fn biases(&self) -> Option<&Allocation<B>> {
- match &self.format {
- WeightFormat::Integer(Quantized {
- correction: QuantizedCorrection::Biases(biases),
- ..
- }) => Some(biases),
- WeightFormat::FullPrecision | WeightFormat::Integer(_) | WeightFormat::Microfloat(_) => None,
+ match &self.quantized.as_ref()?.correction {
+ QuantizedCorrection::Biases(biases) => Some(biases),
+ QuantizedCorrection::ZeroPoints(_) | QuantizedCorrection::Symmetric => None,
}
}
pub fn matmul_b(&self) -> MatmulB<'_, B> {
- let quantized = match &self.format {
- WeightFormat::FullPrecision => {
- return MatmulB::FullPrecision {
- b: &self.values,
- };
- },
- WeightFormat::Microfloat(microfloat) => {
- return MatmulB::Microfloat {
- codes: &self.values,
- scales: &microfloat.scales,
- outer_scales: &microfloat.outer_scales,
- metadata: microfloat.metadata,
- };
- },
- WeightFormat::Integer(quantized) => quantized,
+ let Some(quantized) = self.quantized.as_ref() else {
+ return MatmulB::FullPrecision {
+ b: &self.values,
+ };
};
- let mode = quantized.mode;
- let group_size = quantized.group_size;
+ let mode = quantized.info.mode;
+ let group_size = quantized.info.group_size;
let signed_codes = quantized.signed_codes;
match &quantized.correction {
QuantizedCorrection::Biases(biases) => MatmulB::ScaleBiasDequant {
@@ -316,13 +224,13 @@ impl<B: Backend> WeightMatrix<B> {
}
pub fn make_codes_signed(&mut self) {
- let WeightFormat::Integer(quantized) = &mut self.format else {
+ let Some(quantized) = self.quantized.as_mut() else {
return;
};
if quantized.signed_codes {
return;
}
- let Some(sign_flip_mask) = quantized.mode.weight_codes_sign_flip_mask() else {
+ let Some(sign_flip_mask) = quantized.info.mode.weight_codes_sign_flip_mask() else {
return;
};
let broadcast_mask = u64::from(sign_flip_mask) * 0x0101_0101_0101_0101;
@@ -343,7 +251,3 @@ fn physical_shape(
Layout::InputOutput => (input_dim, output_dim),
}
}
-
-#[cfg(test)]
-#[path = "../../unit/encodable_block/weight_matrix/microfloat_test.rs"]
-mod microfloat_test;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rs
deleted file mode 100644
index 6bac69a5f..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rs
+++ /dev/null
@@ -1,111 +0,0 @@
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{
- Backend, Context, Encoder, Kernels,
- kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
- microfloat::{MicrofloatEncoding, MicrofloatFormat, MicrofloatMetadata, decode_mxfp4},
- },
- cpu::Cpu,
- },
- data_type::DataType,
- tests::{
- assert::assert_eq_float,
- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
- },
-};
-
-const K: usize = 32;
-const N: usize = 4;
-
-fn packed_codes() -> Vec<u8> {
- (0..N * K / 2)
- .map(|index| {
- let low = (index % 7 + 1) as u8;
- let high = ((index * 3 + 1) % 7 + 1) as u8;
- low | (high << 4)
- })
- .collect()
-}
-
-#[uzu_test]
-fn cpu_executes_dense_mxfp4_matmul() {
- for row_count in [1, 5] {
- for group_size in [16, 32] {
- // Given packed MXFP4 weights with multiple batch and scale-group shapes.
- let input_values: Vec<f32> = (0..row_count * K).map(|index| (index % 13) as f32 * 0.125 - 0.5).collect();
- let codes = packed_codes();
- let scales: Vec<u8> = (0..N * K / group_size).map(|index| 126 + (index % 3) as u8).collect();
- let outer_scales = [1.25f32];
- let encoding =
- MicrofloatEncoding::new(MicrofloatFormat::Mxfp4, 4, group_size as u32).expect("valid MXFP4 encoding");
- let metadata = MicrofloatMetadata::new(encoding, N as u32, K as u32).expect("valid dense MXFP4 metadata");
-
- let context = <Cpu as Backend>::Context::new().expect("create CPU context");
- let input = alloc_allocation_with_data::<Cpu, f32>(context.as_ref(), &input_values);
- let codes = alloc_allocation_with_data::<Cpu, u8>(context.as_ref(), &codes);
- let scales = alloc_allocation_with_data::<Cpu, u8>(context.as_ref(), &scales);
- let outer_scales = alloc_allocation_with_data::<Cpu, f32>(context.as_ref(), &outer_scales);
- let mut output = alloc_allocation::<Cpu, f32>(context.as_ref(), row_count * N);
- let mut kernel = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
- context.as_ref(),
- DataType::F32,
- DataType::F32,
- DataType::F32,
- )
- .expect("create CPU matmul kernel");
- let mut encoder = Encoder::<Cpu>::new(context.as_ref()).expect("create CPU encoder");
-
- // When the CPU backend executes a dense microfloat matrix multiplication.
- kernel
- .encode(
- MatmulArguments {
- a: MatmulA::FullPrecision {
- values: &input,
- offset: 0,
- },
- b: MatmulB::<Cpu>::Microfloat {
- codes: &codes,
- scales: &scales,
- outer_scales: &outer_scales,
- metadata,
- },
- b_leading_dimension: None,
- b_transpose: true,
- d: &mut output,
- d_transform: MatmulDOps::none(),
- gather_indices: None,
- m: row_count as u32,
- n: N as u32,
- k: K as u32,
- },
- &mut encoder,
- )
- .expect("encode dense MXFP4 matmul");
- encoder.end_encoding().submit().wait_until_completed().expect("execute dense MXFP4 matmul");
- let actual = allocation_to_vec::<Cpu, f32>(&output);
-
- // Then its output agrees with independently decoded scalar accumulation.
- let codes: &[u8] = codes.as_slice();
- let scales: &[u8] = scales.as_slice();
- let mut expected = vec![0.0f32; row_count * N];
- for row in 0..row_count {
- for output_row in 0..N {
- for inner in 0..K {
- let packed = codes[output_row * K / 2 + inner / 2];
- let code = if inner.is_multiple_of(2) {
- packed & 0x0f
- } else {
- packed >> 4
- };
- let scale = scales[output_row * K / group_size + inner / group_size];
- expected[row * N + output_row] +=
- input_values[row * K + inner] * decode_mxfp4(code, scale, 1.25);
- }
- }
- }
- assert_eq_float(&expected, &actual, 1e-5, "CPU dense MXFP4");
- }
- }
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs
index f543c08b3..ab1716ec5 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs
@@ -2,7 +2,6 @@ mod a8w_bench;
mod dispatch_paths_test;
mod gemm_bench;
mod gemv_test;
-mod microfloat_test;
mod quant_dispatch_test;
mod quant_gemm_bench;
mod quant_gemv_bench;
diff --git a/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rs b/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rs
deleted file mode 100644
index b6d77344b..000000000
--- a/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rs
+++ /dev/null
@@ -1,20 +0,0 @@
-use uzu_engine_macros::uzu_test;
-
-use super::{decode_e2m1, decode_e8m0};
-
-#[uzu_test]
-fn decodes_e2m1_and_e8m0_edges() {
- // Given every finite E2M1 magnitude and the E8M0 boundary encodings.
- let expected = [0.0f32, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0];
-
- // When the positive, negative, minimum-exponent, unit, and NaN codes are decoded.
- // Then E2M1 preserves every value and signed zero, while E8M0 preserves its edge semantics.
- for (code, value) in expected.into_iter().enumerate() {
- assert_eq!(decode_e2m1(code as u8), value);
- assert_eq!(decode_e2m1(code as u8 | 0b1000), -value);
- }
- assert_eq!(decode_e2m1(8).to_bits(), (-0.0f32).to_bits());
- assert_eq!(decode_e8m0(0).to_bits(), 0x0040_0000);
- assert_eq!(decode_e8m0(127), 1.0);
- assert!(decode_e8m0(255).is_nan());
-}
diff --git a/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rs b/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rs
deleted file mode 100644
index 31e8b8f7b..000000000
--- a/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-use std::io::Write;
-
-use half::bf16;
-use serde_json::{Map, Value, json};
-use tempfile::NamedTempFile;
-use uzu_engine_macros::uzu_test;
-
-use super::*;
-use crate::{
- backends::{common::Context, cpu::Cpu},
- parameters::ParameterLoader,
-};
-
-fn add_tensor(
- header: &mut Map<String, Value>,
- payload: &mut Vec<u8>,
- name: &str,
- shape: &[u32],
- data_type: &str,
- data: &[u8],
-) {
- let begin = payload.len();
- payload.extend_from_slice(data);
- header.insert(
- name.into(),
- json!({
- "dtype": data_type,
- "shape": shape,
- "data_offsets": [begin, payload.len()]
- }),
- );
-}
-
-fn dense_microfloat_parameter_file() -> NamedTempFile {
- const ROWS: u32 = 2;
- const COLUMNS: u32 = 32;
- const GROUP_SIZE: u32 = 16;
-
- let mut header = Map::new();
- header.insert(
- "__metadata__".into(),
- json!({
- "spec": json!({
- "type": "MicrofloatSpec",
- "bits": 4,
- "group_size": GROUP_SIZE,
- "scale_mode": "mxfp4",
- "layout": "output_input"
- }).to_string()
- }),
- );
- let mut payload = Vec::new();
- let codes: Vec<u8> = (0..ROWS * COLUMNS / 2).map(|index| 0x10 | (index % 8) as u8).collect();
- let scales: Vec<u8> = (0..ROWS * COLUMNS / GROUP_SIZE).map(|index| 126 + (index % 3) as u8).collect();
- let global_scale = bf16::from_f32(0.75).to_le_bytes();
- add_tensor(&mut header, &mut payload, "weights", &[ROWS, COLUMNS / 2], "U8", &codes);
- add_tensor(&mut header, &mut payload, "scales", &[ROWS, COLUMNS / GROUP_SIZE], "U8", &scales);
- add_tensor(&mut header, &mut payload, "global_scale", &[1], "BF16", &global_scale);
-
- let mut header = serde_json::to_vec(&Value::Object(header)).expect("serialize safetensors header");
- header.extend(std::iter::repeat_n(b' ', (8 - header.len() % 8) % 8));
- let mut file = NamedTempFile::new().expect("create dense MXFP4 fixture");
- file.write_all(&(header.len() as u64).to_le_bytes()).expect("write safetensors header length");
- file.write_all(&header).expect("write safetensors header");
- file.write_all(&payload).expect("write safetensors payload");
- file
-}
-
-#[uzu_test]
-fn loads_dense_mxfp4_storage() {
- // Given a safetensors artifact whose metadata and tensors describe dense MXFP4 weights.
- let context = <Cpu as Backend>::Context::new().expect("create CPU context");
- let file = dense_microfloat_parameter_file();
- let loader = ParameterLoader::<Cpu>::new(file.as_file(), context.as_ref()).expect("load dense MXFP4 fixture");
- let tree = loader.tree();
- let spec = tree.metadata::<AnyWeightMatrixSpec>("spec").expect("load dense MXFP4 spec");
-
- // When the artifact is loaded through ParameterLoader and WeightMatrix.
- let matrix =
- WeightMatrix::load(&tree, spec, Layout::OutputInput, 2, 32, DataType::BF16).expect("load dense MXFP4 matrix");
-
- // Then the matrix retains its microfloat operand shape and consumes every source tensor.
- let MatmulB::Microfloat {
- metadata,
- ..
- } = matrix.matmul_b()
- else {
- panic!("dense MXFP4 matrix did not preserve its operand format");
- };
- assert_eq!(metadata.rows, 2);
- assert_eq!(metadata.columns, 32);
- tree.assert_all_tensors_validated().expect("validate dense MXFP4 tensors");
-}