A high-performance inference engine for AI models
Revert "mxfp4" (#804)
uuuvnd0699c22ccfa
diff --git a/crates/uzu-engine/Cargo.toml b/crates/uzu-engine/Cargo.tomlindex d59c2240c..31c25959c 100644--- a/crates/uzu-engine/Cargo.toml+++ b/crates/uzu-engine/Cargo.toml@@ -85,7 +85,6 @@ xgrammar = { workspace = true, optional = true }schemars = { workspace = true, optional = true }[dev-dependencies]-tempfile.workspace = truetest-tag.workspace = trueproptest.workspace = truecriterion.workspace = truediff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rsindex 89e7a0bd0..96be2b02a 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs@@ -1,7 +1,7 @@use thiserror::Error;use crate::{- backends::common::{Backend, gpu_types::gemm::GemmDTransform, microfloat::MicrofloatFormat},+ backends::common::{Backend, gpu_types::gemm::GemmDTransform},data_type::DataType,};@@ -20,10 +20,6 @@ pub enum MatmulError<B: Backend> {UnsupportedLayout {path: &'static str,},- #[error("Microfloat storage does not match the requested matrix")]- InvalidMicrofloatStorage,- #[error("Unsupported microfloat weight format: {0:?}")]- UnsupportedMicrofloat(MicrofloatFormat),#[error("Incompatible A operand for {path}: {reason}")]IncompatibleA {path: &'static str,diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rsindex 2026d5dad..e472a5477 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs@@ -2,7 +2,6 @@ use crate::{backends::common::{Allocation, Backend, BufferArg,gpu_types::{QuantizationMode, gemm::GemmBPrologueKind},- microfloat::MicrofloatMetadata,},data_type::DataType,};@@ -11,12 +10,6 @@ pub enum MatmulB<'a, B: Backend, TB: BufferArg<'a, B> = &'a Allocation<B>> {FullPrecision {b: TB,},- Microfloat {- codes: &'a Allocation<B>,- scales: &'a Allocation<B>,- outer_scales: &'a Allocation<B>,- metadata: MicrofloatMetadata,- },ScaleBiasDequant {b: &'a Allocation<B>,scales: &'a Allocation<B>,@@ -47,9 +40,6 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {match self {Self::FullPrecision {..- }- | Self::Microfloat {- ..} => GemmBPrologueKind::FullPrecision,Self::ScaleBiasDequant {..@@ -68,10 +58,6 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {Self::FullPrecision {..} => None,- Self::Microfloat {- metadata,- ..- } => Some(metadata.encoding.bits),Self::ScaleBiasDequant {mode,..@@ -92,10 +78,6 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {Self::FullPrecision {..} => None,- Self::Microfloat {- metadata,- ..- } => Some(metadata.encoding.group_size),Self::ScaleBiasDequant {group_size,..@@ -115,9 +97,6 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {match self {Self::FullPrecision {..- }- | Self::Microfloat {- ..} => false,Self::ScaleBiasDequant {signed_codes,diff --git a/crates/uzu-engine/src/backends/common/microfloat/decode.rs b/crates/uzu-engine/src/backends/common/microfloat/decode.rsdeleted file mode 100644index c5242eeb9..000000000--- a/crates/uzu-engine/src/backends/common/microfloat/decode.rs+++ /dev/null@@ -1,23 +0,0 @@-#[inline]-pub fn decode_e2m1(code: u8) -> f32 {- const VALUES: [f32; 16] = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0];- VALUES[usize::from(code & 0x0f)]-}--#[inline]-pub fn decode_e8m0(exponent: u8) -> f32 {- match exponent {- 0 => f32::from_bits(0x0040_0000),- 255 => f32::NAN,- exponent => f32::from_bits(u32::from(exponent) << 23),- }-}--#[inline]-pub fn decode_mxfp4(- code: u8,- exponent: u8,- outer_scale: f32,-) -> f32 {- decode_e2m1(code) * decode_e8m0(exponent) * outer_scale-}diff --git a/crates/uzu-engine/src/backends/common/microfloat/encoding.rs b/crates/uzu-engine/src/backends/common/microfloat/encoding.rsdeleted file mode 100644index 8c0c3c7fa..000000000--- a/crates/uzu-engine/src/backends/common/microfloat/encoding.rs+++ /dev/null@@ -1,38 +0,0 @@-use super::{MicrofloatError, MicrofloatFormat};--/// How packed microfloat bytes are interpreted, separate from matrix dimensions.-///-/// With MXFP4 group size 16, every 16 values along the input axis occupy eight-/// packed code bytes and share one E8M0 scale.-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub struct MicrofloatEncoding {- pub format: MicrofloatFormat,- pub bits: u32,- pub group_size: u32,-}--impl MicrofloatEncoding {- pub fn new(- format: MicrofloatFormat,- bits: u32,- group_size: u32,- ) -> Result<Self, MicrofloatError> {- if bits != 4 {- return Err(MicrofloatError::UnsupportedBits {- format,- bits,- });- }- if !matches!(group_size, 16 | 32) {- return Err(MicrofloatError::UnsupportedGroupSize {- format,- group_size,- });- }- Ok(Self {- format,- bits,- group_size,- })- }-}diff --git a/crates/uzu-engine/src/backends/common/microfloat/error.rs b/crates/uzu-engine/src/backends/common/microfloat/error.rsdeleted file mode 100644index abe796eb4..000000000--- a/crates/uzu-engine/src/backends/common/microfloat/error.rs+++ /dev/null@@ -1,26 +0,0 @@-use thiserror::Error;--use super::MicrofloatFormat;--#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]-pub enum MicrofloatError {- #[error("unsupported {format:?} bit width: {bits}")]- UnsupportedBits {- format: MicrofloatFormat,- bits: u32,- },- #[error("unsupported {format:?} group size: {group_size}")]- UnsupportedGroupSize {- format: MicrofloatFormat,- group_size: u32,- },- #[error("microfloat rows and columns must be nonzero")]- EmptyShape,- #[error("microfloat columns {columns} are not divisible by group size {group_size}")]- MisalignedColumns {- columns: u32,- group_size: u32,- },- #[error("microfloat storage size overflows usize")]- SizeOverflow,-}diff --git a/crates/uzu-engine/src/backends/common/microfloat/format.rs b/crates/uzu-engine/src/backends/common/microfloat/format.rsdeleted file mode 100644index 8356176d7..000000000--- a/crates/uzu-engine/src/backends/common/microfloat/format.rs+++ /dev/null@@ -1,8 +0,0 @@-use uzu_engine_macros::uzu_config;--#[derive(Copy, Eq)]-#[uzu_config]-#[serde(rename_all = "snake_case")]-pub enum MicrofloatFormat {- Mxfp4,-}diff --git a/crates/uzu-engine/src/backends/common/microfloat/metadata.rs b/crates/uzu-engine/src/backends/common/microfloat/metadata.rsdeleted file mode 100644index b234b44c5..000000000--- a/crates/uzu-engine/src/backends/common/microfloat/metadata.rs+++ /dev/null@@ -1,67 +0,0 @@-use super::{MicrofloatEncoding, MicrofloatError};--/// Physical shape and derived strides for one microfloat matrix.-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub struct MicrofloatMetadata {- pub encoding: MicrofloatEncoding,- pub rows: u32,- pub columns: u32,-}--impl MicrofloatMetadata {- pub fn new(- encoding: MicrofloatEncoding,- rows: u32,- columns: u32,- ) -> Result<Self, MicrofloatError> {- if rows == 0 || columns == 0 {- return Err(MicrofloatError::EmptyShape);- }- if !columns.is_multiple_of(encoding.group_size) {- return Err(MicrofloatError::MisalignedColumns {- columns,- group_size: encoding.group_size,- });- }- let metadata = Self {- encoding,- rows,- columns,- };- metadata.checked_code_matrix_stride().ok_or(MicrofloatError::SizeOverflow)?;- metadata.checked_scale_matrix_stride().ok_or(MicrofloatError::SizeOverflow)?;- Ok(metadata)- }-- pub fn code_row_stride(self) -> usize {- self.columns as usize / 2- }-- pub fn scale_row_stride(self) -> usize {- self.columns as usize / self.encoding.group_size as usize- }-- pub fn code_matrix_stride(self) -> usize {- self.checked_code_matrix_stride().expect("MicrofloatMetadata validates code storage size")- }-- pub fn scale_matrix_stride(self) -> usize {- self.checked_scale_matrix_stride().expect("MicrofloatMetadata validates scale storage size")- }-- pub fn required_code_bytes(self) -> usize {- self.code_matrix_stride()- }-- pub fn required_scale_bytes(self) -> usize {- self.scale_matrix_stride()- }-- fn checked_code_matrix_stride(self) -> Option<usize> {- (self.rows as usize).checked_mul(self.code_row_stride())- }-- fn checked_scale_matrix_stride(self) -> Option<usize> {- (self.rows as usize).checked_mul(self.scale_row_stride())- }-}diff --git a/crates/uzu-engine/src/backends/common/microfloat/mod.rs b/crates/uzu-engine/src/backends/common/microfloat/mod.rsdeleted file mode 100644index 21f7c3f6e..000000000--- a/crates/uzu-engine/src/backends/common/microfloat/mod.rs+++ /dev/null@@ -1,15 +0,0 @@-mod decode;-mod encoding;-mod error;-mod format;-mod metadata;--pub use decode::{decode_e2m1, decode_e8m0, decode_mxfp4};-pub use encoding::MicrofloatEncoding;-pub use error::MicrofloatError;-pub use format::MicrofloatFormat;-pub use metadata::MicrofloatMetadata;--#[cfg(test)]-#[path = "../../../../unit/backends/common/microfloat/decode_test.rs"]-mod decode_test;diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rsindex ff845b66b..d3b57f26d 100644--- a/crates/uzu-engine/src/backends/common/mod.rs+++ b/crates/uzu-engine/src/backends/common/mod.rs@@ -8,7 +8,6 @@ mod encoder;pub mod gpu_types;mod hazard_tracker;pub mod kernel;-pub mod microfloat;pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};pub use backend::Backend;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rsindex d64f2fd38..c3db18e05 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs@@ -58,26 +58,6 @@ impl MatmulKernel for MatmulCpuKernel {arguments: MatmulArguments<'a, 'b, 'd, Cpu, TB>,encoder: &mut Encoder<Cpu>,) -> Result<(), CpuError> {- if let MatmulB::Microfloat {- codes,- scales,- outer_scales,- metadata,- } = &arguments.b- {- let rows_match = arguments.gather_indices.is_some() || metadata.rows == arguments.n;- if !arguments.b_transpose- || arguments.b_leading_dimension.is_some()- || !rows_match- || metadata.columns != arguments.k- || codes.size() < metadata.required_code_bytes()- || scales.size() < metadata.required_scale_bytes()- || outer_scales.size() < self.weights_data_type.size_in_bytes()- {- return Err(MatmulError::InvalidMicrofloatStorage.into());- }- }-let output_scale = arguments.d_transform.ab_scale;let accumulate = arguments.d_transform.accumulate;let bias_alloc = arguments.d_transform.bias;@@ -203,9 +183,6 @@ impl MatmulKernel for MatmulCpuKernel {},WeightData::FullPrecision {..- }- | WeightData::Microfloat {- ..} => None,};@@ -296,25 +273,6 @@ impl MatmulKernel for MatmulCpuKernel {};scale * quantized_value + bias_term},- WeightData::Microfloat {- codes,- scales,- outer_scales,- metadata,- } => {- let code_index = b_col * metadata.code_row_stride() + inner / 2;- let packed = *codes.as_ptr().add(code_index);- let code = if inner.is_multiple_of(2) {- packed & 0x0f- } else {- packed >> 4- };- let scale_index = b_col * metadata.scale_row_stride()- + inner / metadata.encoding.group_size as usize;- let exponent = *scales.as_ptr().add(scale_index);- let outer_scale = read_f32(outer_scales.as_ptr(), weights_data_type, 0);- crate::backends::common::microfloat::decode_mxfp4(code, exponent, outer_scale)- },};accumulator += a_value * b_value;}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rsindex 4e48ce623..41c92a735 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs@@ -2,10 +2,7 @@ use half::{bf16, f16};use crate::{backends::{- common::{- AsBufferRangeRef, BufferArg, gpu_types::QuantizationMode, kernel::matmul::MatmulB,- microfloat::MicrofloatMetadata,- },+ common::{AsBufferRangeRef, BufferArg, gpu_types::QuantizationMode, kernel::matmul::MatmulB},cpu::Cpu,},data_type::DataType,@@ -27,12 +24,6 @@ pub(super) enum WeightData {group_size: usize,signed_codes: bool,},- Microfloat {- codes: SendPtr<u8>,- scales: SendPtr<u8>,- outer_scales: SendPtr<u8>,- metadata: MicrofloatMetadata,- },}impl WeightData {@@ -67,17 +58,6 @@ impl WeightData {transpose: b_transpose,}},- MatmulB::Microfloat {- codes,- scales,- outer_scales,- metadata,- } => WeightData::Microfloat {- codes: alloc_ptr(codes),- scales: alloc_ptr(scales),- outer_scales: alloc_ptr(outer_scales),- metadata,- },MatmulB::ScaleBiasDequant {b: weights,scales,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 532fc8b0a..3b27f7299 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -191,12 +191,6 @@ impl GemmKernel {let use_mxu = plan.engine == GemmEngine::Mxu;match b {- MatmulB::Microfloat {- metadata,- ..- } => {- return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format).into());- },MatmulB::FullPrecision {b: weights,} => {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 7f425f552..a78ae14ed 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -266,9 +266,6 @@ impl GemvKernel {let (scales, zero_points, biases) = match &b {MatmulB::FullPrecision {..- }- | MatmulB::Microfloat {- ..} => (None, None, None),MatmulB::ScaleBiasDequant {scales,@@ -290,12 +287,6 @@ impl GemvKernel {let context = encoder.context();let pipeline = self.get_or_create(context, specialization)?;match b {- MatmulB::Microfloat {- metadata,- ..- } => {- return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format));- },MatmulB::FullPrecision {b: weights,} => pipeline.encode(diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rsindex eb187459e..14fa5a4c1 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs@@ -17,10 +17,7 @@ use crate::{gpu_types::gemm::{GemmBPrologueKind, GemmTiling},kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE,- matmul::{- A8ActivationPlan, ActivationFormat, MatmulArguments, MatmulB, MatmulError, MatmulKernel,- MatmulShape,- },+ matmul::{A8ActivationPlan, ActivationFormat, MatmulArguments, MatmulError, MatmulKernel, MatmulShape},},},metal::{Metal, context::MetalContext, error::MetalError},@@ -220,13 +217,6 @@ impl MatmulKernel for MatmulMetalKernel {arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {- if let MatmulB::Microfloat {- metadata,- ..- } = &arguments.b- {- return Err(MatmulError::UnsupportedMicrofloat(metadata.encoding.format).into());- }let shape = MatmulShape::from_arguments(&arguments);let plan = match self.select_dispatch(&shape, encoder.context()) {MatmulDispatch::Gemv(gemv) => {diff --git a/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rs b/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rsdeleted file mode 100644index 45a7e99b0..000000000--- a/crates/uzu-engine/src/config/weight_matrix/microfloat_spec.rs+++ /dev/null@@ -1,11 +0,0 @@-use uzu_engine_macros::uzu_config;--use crate::{backends::common::microfloat::MicrofloatFormat, config::weight_matrix::Layout};--#[uzu_config(super::WeightMatrixSpec)]-pub struct MicrofloatSpec {- pub bits: u32,- pub group_size: usize,- pub scale_mode: MicrofloatFormat,- pub layout: Layout,-}diff --git a/crates/uzu-engine/src/config/weight_matrix/mod.rs b/crates/uzu-engine/src/config/weight_matrix/mod.rsindex 405745762..5f10ee506 100644--- a/crates/uzu-engine/src/config/weight_matrix/mod.rs+++ b/crates/uzu-engine/src/config/weight_matrix/mod.rs@@ -4,7 +4,6 @@ pub mod full_precision_spec;pub mod hybrid_spec;pub mod int_spec;pub mod low_rank_spec;-pub mod microfloat_spec;pub mod mlx_spec;#[uzu_config]@@ -19,7 +18,6 @@ pub enum Layout {low_rank_spec::LowRankSpec,hybrid_spec::HybridSpec,int_spec::IntSpec,- microfloat_spec::MicrofloatSpec,mlx_spec::MLXSpec)]pub struct WeightMatrixSpec;diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rsindex a48d7771b..250a1e70c 100644--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs@@ -7,7 +7,7 @@ use crate::{},config::weight_matrix::{AnyWeightMatrixSpec, Layout},data_type::DataType,- encodable_block::weight_matrix::{QuantizationInfo, WeightMatrix, WeightMatrixError},+ encodable_block::weight_matrix::{WeightMatrix, WeightMatrixError},parameters::{ParameterLoaderError, ParameterTree},};@@ -65,32 +65,21 @@ impl<B: Backend> EmbeddingTable<B> {}let lookup = match matrix.quantization() {- None => {- let kernel = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)- .map_err(EmbeddingTableError::BackendError)?;- LookupKernel::FullPrecision(kernel)- },- Some(QuantizationInfo::Microfloat(_)) => {- return Err(EmbeddingTableError::UnsupportedConfiguration(- "microfloat embedding tables are not supported".into(),- ));- },- Some(QuantizationInfo::Integer {- mode,- method,- group_size,- }) => {- let kernel = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(+ None => LookupKernel::FullPrecision(+ <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)+ .map_err(EmbeddingTableError::BackendError)?,+ ),+ Some(info) => LookupKernel::Quantized(+ <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(context,data_type,- group_size,- mode,- method,+ info.group_size,+ info.mode,+ info.method,output_hadamard_factors.is_some(),)- .map_err(EmbeddingTableError::BackendError)?;- LookupKernel::Quantized(kernel)- },+ .map_err(EmbeddingTableError::BackendError)?,+ ),};Ok(Self {diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rsindex 9dbb66aeb..06c435434 100644--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs@@ -5,9 +5,8 @@ use crate::{Allocation, Backend,gpu_types::{QuantizationMethod, QuantizationMode},kernel::matmul::MatmulB,- microfloat::{MicrofloatEncoding, MicrofloatMetadata},},- config::weight_matrix::{AnyWeightMatrixSpec, Layout, microfloat_spec::MicrofloatSpec},+ config::weight_matrix::{AnyWeightMatrixSpec, Layout},data_type::DataType,parameters::{ParameterLoaderError, ParameterTree},};@@ -20,14 +19,11 @@ pub enum WeightMatrixError<B: Backend> {UnsupportedConfiguration(String),}-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum QuantizationInfo {- Integer {- mode: QuantizationMode,- method: QuantizationMethod,- group_size: u32,- },- Microfloat(MicrofloatEncoding),+#[derive(Clone, Copy)]+pub struct QuantizationInfo {+ pub mode: QuantizationMode,+ pub method: QuantizationMethod,+ pub group_size: u32,}pub struct ParsedWeightSpec {@@ -36,41 +32,46 @@ pub struct ParsedWeightSpec {}pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeightSpec, WeightMatrixError<B>> {- let (layout, quantization) = match spec {+ let (layout, quantized) = match spec {AnyWeightMatrixSpec::FullPrecisionSpec(spec) => (spec.layout.clone(), None),AnyWeightMatrixSpec::MLXSpec(spec) => {- let quantization = integer_quantization::<B>(spec.bits, spec.group_size, QuantizationMethod::ScaleBias)?;- (spec.layout.clone(), Some(quantization))+ (spec.layout.clone(), Some((spec.bits, spec.group_size, QuantizationMethod::ScaleBias)))},- AnyWeightMatrixSpec::IntSpec(spec) => {- let method = if spec.is_symmetric {- QuantizationMethod::ScaleSymmetric- } else {- QuantizationMethod::ScaleZeroPoint+ AnyWeightMatrixSpec::IntSpec(spec) => (+ spec.layout.clone(),+ Some((+ spec.bits,+ spec.group_size,+ if spec.is_symmetric {+ QuantizationMethod::ScaleSymmetric+ } else {+ QuantizationMethod::ScaleZeroPoint+ },+ )),+ ),+ spec => return Err(WeightMatrixError::UnsupportedConfiguration(format!("{spec:?}"))),+ };+ let quantization = match quantized {+ None => None,+ Some((bits, group_size, method)) => {+ let mode = match bits {+ 4 => QuantizationMode::U4,+ 8 => QuantizationMode::U8,+ _ => {+ return Err(WeightMatrixError::UnsupportedConfiguration(format!(+ "{method} bits={bits}, group_size={group_size}"+ )));+ },};- let quantization = integer_quantization::<B>(spec.bits, spec.group_size, method)?;- (spec.layout.clone(), Some(quantization))- },- AnyWeightMatrixSpec::MicrofloatSpec(MicrofloatSpec {- bits,- group_size,- scale_mode,- layout,- ..- }) => {- if layout != &Layout::OutputInput {- return Err(WeightMatrixError::UnsupportedConfiguration(format!(- "microfloat matrices require output-input layout, got {layout:?}"- )));+ if group_size == 0 {+ return Err(WeightMatrixError::UnsupportedConfiguration("group size must be non-zero".into()));}- let group_size = u32::try_from(*group_size).map_err(|_| {- WeightMatrixError::UnsupportedConfiguration(format!("microfloat group size {group_size} exceeds u32"))- })?;- let encoding = MicrofloatEncoding::new(*scale_mode, *bits, group_size)- .map_err(|error| WeightMatrixError::UnsupportedConfiguration(error.to_string()))?;- (layout.clone(), Some(QuantizationInfo::Microfloat(encoding)))+ Some(QuantizationInfo {+ mode,+ method,+ group_size,+ })},- spec => return Err(WeightMatrixError::UnsupportedConfiguration(format!("{spec:?}"))),};Ok(ParsedWeightSpec {layout,@@ -78,30 +79,6 @@ pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeight})}-fn integer_quantization<B: Backend>(- bits: u32,- group_size: u32,- method: QuantizationMethod,-) -> Result<QuantizationInfo, WeightMatrixError<B>> {- let mode = match bits {- 4 => QuantizationMode::U4,- 8 => QuantizationMode::U8,- _ => {- return Err(WeightMatrixError::UnsupportedConfiguration(format!(- "{method} bits={bits}, group_size={group_size}"- )));- },- };- if group_size == 0 {- return Err(WeightMatrixError::UnsupportedConfiguration("group size must be non-zero".into()));- }- Ok(QuantizationInfo::Integer {- mode,- method,- group_size,- })-}-enum QuantizedCorrection<B: Backend> {Symmetric,Biases(Allocation<B>),@@ -111,27 +88,13 @@ enum QuantizedCorrection<B: Backend> {struct Quantized<B: Backend> {scales: Allocation<B>,correction: QuantizedCorrection<B>,- mode: QuantizationMode,- method: QuantizationMethod,- group_size: u32,+ info: QuantizationInfo,signed_codes: bool,}-struct Microfloat<B: Backend> {- scales: Allocation<B>,- outer_scales: Allocation<B>,- metadata: MicrofloatMetadata,-}--enum WeightFormat<B: Backend> {- FullPrecision,- Integer(Quantized<B>),- Microfloat(Microfloat<B>),-}-pub struct WeightMatrix<B: Backend> {values: Allocation<B>,- format: WeightFormat<B>,+ quantized: Option<Quantized<B>>,}impl<B: Backend> WeightMatrix<B> {@@ -154,53 +117,28 @@ impl<B: Backend> WeightMatrix<B> {}let (rows, columns) = physical_shape(&layout, output_dim, input_dim);- let Some(quantization) = quantization else {+ let Some(info) = quantization else {let values = tree.leaf("weights")?.validate(&[rows, columns], data_type)?.read_allocation()?;return Ok(Self {values,- format: WeightFormat::FullPrecision,+ quantized: None,});};- let (mode, method, group_size) = match quantization {- QuantizationInfo::Microfloat(encoding) => {- let metadata = MicrofloatMetadata::new(encoding, rows, columns)- .map_err(|error| WeightMatrixError::UnsupportedConfiguration(error.to_string()))?;- let values = tree.leaf("weights")?.validate(&[rows, columns / 2], DataType::U8)?.read_allocation()?;- let scales = tree- .leaf("scales")?- .validate(&[rows, columns / encoding.group_size], DataType::U8)?- .read_allocation()?;- // Preserving the artifact's established tensor name for the outer scale.- let outer_scales = tree.leaf("global_scale")?.validate(&[1], data_type)?.read_allocation()?;- return Ok(Self {- values,- format: WeightFormat::Microfloat(Microfloat {- scales,- outer_scales,- metadata,- }),- });- },- QuantizationInfo::Integer {- mode,- method,- group_size,- } => (mode, method, group_size),- };-- let packing_divisor = mode.packing_divisor();- let storage_data_type = mode.storage_type();+ let group_size = info.group_size;+ let packing_divisor = info.mode.packing_divisor();+ let storage_data_type = info.mode.storage_type();if !columns.is_multiple_of(packing_divisor) {return Err(WeightMatrixError::UnsupportedConfiguration(format!("stored columns {columns} are not divisible by packing divisor {packing_divisor}")));}let groups = columns.div_ceil(group_size);+let values =tree.leaf("weights")?.validate(&[rows, columns / packing_divisor], storage_data_type)?.read_allocation()?;let scales = tree.leaf("scales")?.validate(&[rows, groups], data_type)?.read_allocation()?;- let correction = match method {+ let correction = match info.method {QuantizationMethod::ScaleBias => QuantizedCorrection::Biases(tree.leaf("biases")?.validate(&[rows, groups], data_type)?.read_allocation()?,),@@ -211,14 +149,13 @@ impl<B: Backend> WeightMatrix<B> {),QuantizationMethod::ScaleSymmetric => QuantizedCorrection::Symmetric,};+Ok(Self {values,- format: WeightFormat::Integer(Quantized {+ quantized: Some(Quantized {scales,correction,- mode,- method,- group_size,+ info,signed_codes: false,}),})@@ -229,64 +166,35 @@ impl<B: Backend> WeightMatrix<B> {}pub fn quantization(&self) -> Option<QuantizationInfo> {- match &self.format {- WeightFormat::FullPrecision => None,- WeightFormat::Integer(quantized) => Some(QuantizationInfo::Integer {- mode: quantized.mode,- method: quantized.method,- group_size: quantized.group_size,- }),- WeightFormat::Microfloat(microfloat) => Some(QuantizationInfo::Microfloat(microfloat.metadata.encoding)),- }+ self.quantized.as_ref().map(|quantized| quantized.info)}pub fn scales(&self) -> Option<&Allocation<B>> {- match &self.format {- WeightFormat::Integer(quantized) => Some(&quantized.scales),- WeightFormat::Microfloat(microfloat) => Some(µfloat.scales),- WeightFormat::FullPrecision => None,- }+ self.quantized.as_ref().map(|quantized| &quantized.scales)}pub fn zero_points(&self) -> Option<&Allocation<B>> {- match &self.format {- WeightFormat::Integer(Quantized {- correction: QuantizedCorrection::ZeroPoints(zero_points),- ..- }) => Some(zero_points),- WeightFormat::FullPrecision | WeightFormat::Integer(_) | WeightFormat::Microfloat(_) => None,+ match &self.quantized.as_ref()?.correction {+ QuantizedCorrection::ZeroPoints(zero_points) => Some(zero_points),+ QuantizedCorrection::Biases(_) | QuantizedCorrection::Symmetric => None,}}pub fn biases(&self) -> Option<&Allocation<B>> {- match &self.format {- WeightFormat::Integer(Quantized {- correction: QuantizedCorrection::Biases(biases),- ..- }) => Some(biases),- WeightFormat::FullPrecision | WeightFormat::Integer(_) | WeightFormat::Microfloat(_) => None,+ match &self.quantized.as_ref()?.correction {+ QuantizedCorrection::Biases(biases) => Some(biases),+ QuantizedCorrection::ZeroPoints(_) | QuantizedCorrection::Symmetric => None,}}pub fn matmul_b(&self) -> MatmulB<'_, B> {- let quantized = match &self.format {- WeightFormat::FullPrecision => {- return MatmulB::FullPrecision {- b: &self.values,- };- },- WeightFormat::Microfloat(microfloat) => {- return MatmulB::Microfloat {- codes: &self.values,- scales: µfloat.scales,- outer_scales: µfloat.outer_scales,- metadata: microfloat.metadata,- };- },- WeightFormat::Integer(quantized) => quantized,+ let Some(quantized) = self.quantized.as_ref() else {+ return MatmulB::FullPrecision {+ b: &self.values,+ };};- let mode = quantized.mode;- let group_size = quantized.group_size;+ let mode = quantized.info.mode;+ let group_size = quantized.info.group_size;let signed_codes = quantized.signed_codes;match &quantized.correction {QuantizedCorrection::Biases(biases) => MatmulB::ScaleBiasDequant {@@ -316,13 +224,13 @@ impl<B: Backend> WeightMatrix<B> {}pub fn make_codes_signed(&mut self) {- let WeightFormat::Integer(quantized) = &mut self.format else {+ let Some(quantized) = self.quantized.as_mut() else {return;};if quantized.signed_codes {return;}- let Some(sign_flip_mask) = quantized.mode.weight_codes_sign_flip_mask() else {+ let Some(sign_flip_mask) = quantized.info.mode.weight_codes_sign_flip_mask() else {return;};let broadcast_mask = u64::from(sign_flip_mask) * 0x0101_0101_0101_0101;@@ -343,7 +251,3 @@ fn physical_shape(Layout::InputOutput => (input_dim, output_dim),}}--#[cfg(test)]-#[path = "../../unit/encodable_block/weight_matrix/microfloat_test.rs"]-mod microfloat_test;diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rsdeleted file mode 100644index 6bac69a5f..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/microfloat_test.rs+++ /dev/null@@ -1,111 +0,0 @@-use uzu_engine_macros::uzu_test;--use crate::{- backends::{- common::{- Backend, Context, Encoder, Kernels,- kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},- microfloat::{MicrofloatEncoding, MicrofloatFormat, MicrofloatMetadata, decode_mxfp4},- },- cpu::Cpu,- },- data_type::DataType,- tests::{- assert::assert_eq_float,- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},- },-};--const K: usize = 32;-const N: usize = 4;--fn packed_codes() -> Vec<u8> {- (0..N * K / 2)- .map(|index| {- let low = (index % 7 + 1) as u8;- let high = ((index * 3 + 1) % 7 + 1) as u8;- low | (high << 4)- })- .collect()-}--#[uzu_test]-fn cpu_executes_dense_mxfp4_matmul() {- for row_count in [1, 5] {- for group_size in [16, 32] {- // Given packed MXFP4 weights with multiple batch and scale-group shapes.- let input_values: Vec<f32> = (0..row_count * K).map(|index| (index % 13) as f32 * 0.125 - 0.5).collect();- let codes = packed_codes();- let scales: Vec<u8> = (0..N * K / group_size).map(|index| 126 + (index % 3) as u8).collect();- let outer_scales = [1.25f32];- let encoding =- MicrofloatEncoding::new(MicrofloatFormat::Mxfp4, 4, group_size as u32).expect("valid MXFP4 encoding");- let metadata = MicrofloatMetadata::new(encoding, N as u32, K as u32).expect("valid dense MXFP4 metadata");-- let context = <Cpu as Backend>::Context::new().expect("create CPU context");- let input = alloc_allocation_with_data::<Cpu, f32>(context.as_ref(), &input_values);- let codes = alloc_allocation_with_data::<Cpu, u8>(context.as_ref(), &codes);- let scales = alloc_allocation_with_data::<Cpu, u8>(context.as_ref(), &scales);- let outer_scales = alloc_allocation_with_data::<Cpu, f32>(context.as_ref(), &outer_scales);- let mut output = alloc_allocation::<Cpu, f32>(context.as_ref(), row_count * N);- let mut kernel = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(- context.as_ref(),- DataType::F32,- DataType::F32,- DataType::F32,- )- .expect("create CPU matmul kernel");- let mut encoder = Encoder::<Cpu>::new(context.as_ref()).expect("create CPU encoder");-- // When the CPU backend executes a dense microfloat matrix multiplication.- kernel- .encode(- MatmulArguments {- a: MatmulA::FullPrecision {- values: &input,- offset: 0,- },- b: MatmulB::<Cpu>::Microfloat {- codes: &codes,- scales: &scales,- outer_scales: &outer_scales,- metadata,- },- b_leading_dimension: None,- b_transpose: true,- d: &mut output,- d_transform: MatmulDOps::none(),- gather_indices: None,- m: row_count as u32,- n: N as u32,- k: K as u32,- },- &mut encoder,- )- .expect("encode dense MXFP4 matmul");- encoder.end_encoding().submit().wait_until_completed().expect("execute dense MXFP4 matmul");- let actual = allocation_to_vec::<Cpu, f32>(&output);-- // Then its output agrees with independently decoded scalar accumulation.- let codes: &[u8] = codes.as_slice();- let scales: &[u8] = scales.as_slice();- let mut expected = vec![0.0f32; row_count * N];- for row in 0..row_count {- for output_row in 0..N {- for inner in 0..K {- let packed = codes[output_row * K / 2 + inner / 2];- let code = if inner.is_multiple_of(2) {- packed & 0x0f- } else {- packed >> 4- };- let scale = scales[output_row * K / group_size + inner / group_size];- expected[row * N + output_row] +=- input_values[row * K + inner] * decode_mxfp4(code, scale, 1.25);- }- }- }- assert_eq_float(&expected, &actual, 1e-5, "CPU dense MXFP4");- }- }-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rsindex f543c08b3..ab1716ec5 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs@@ -2,7 +2,6 @@ mod a8w_bench;mod dispatch_paths_test;mod gemm_bench;mod gemv_test;-mod microfloat_test;mod quant_dispatch_test;mod quant_gemm_bench;mod quant_gemv_bench;diff --git a/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rs b/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rsdeleted file mode 100644index b6d77344b..000000000--- a/crates/uzu-engine/unit/backends/common/microfloat/decode_test.rs+++ /dev/null@@ -1,20 +0,0 @@-use uzu_engine_macros::uzu_test;--use super::{decode_e2m1, decode_e8m0};--#[uzu_test]-fn decodes_e2m1_and_e8m0_edges() {- // Given every finite E2M1 magnitude and the E8M0 boundary encodings.- let expected = [0.0f32, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0];-- // When the positive, negative, minimum-exponent, unit, and NaN codes are decoded.- // Then E2M1 preserves every value and signed zero, while E8M0 preserves its edge semantics.- for (code, value) in expected.into_iter().enumerate() {- assert_eq!(decode_e2m1(code as u8), value);- assert_eq!(decode_e2m1(code as u8 | 0b1000), -value);- }- assert_eq!(decode_e2m1(8).to_bits(), (-0.0f32).to_bits());- assert_eq!(decode_e8m0(0).to_bits(), 0x0040_0000);- assert_eq!(decode_e8m0(127), 1.0);- assert!(decode_e8m0(255).is_nan());-}diff --git a/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rs b/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rsdeleted file mode 100644index 31e8b8f7b..000000000--- a/crates/uzu-engine/unit/encodable_block/weight_matrix/microfloat_test.rs+++ /dev/null@@ -1,93 +0,0 @@-use std::io::Write;--use half::bf16;-use serde_json::{Map, Value, json};-use tempfile::NamedTempFile;-use uzu_engine_macros::uzu_test;--use super::*;-use crate::{- backends::{common::Context, cpu::Cpu},- parameters::ParameterLoader,-};--fn add_tensor(- header: &mut Map<String, Value>,- payload: &mut Vec<u8>,- name: &str,- shape: &[u32],- data_type: &str,- data: &[u8],-) {- let begin = payload.len();- payload.extend_from_slice(data);- header.insert(- name.into(),- json!({- "dtype": data_type,- "shape": shape,- "data_offsets": [begin, payload.len()]- }),- );-}--fn dense_microfloat_parameter_file() -> NamedTempFile {- const ROWS: u32 = 2;- const COLUMNS: u32 = 32;- const GROUP_SIZE: u32 = 16;-- let mut header = Map::new();- header.insert(- "__metadata__".into(),- json!({- "spec": json!({- "type": "MicrofloatSpec",- "bits": 4,- "group_size": GROUP_SIZE,- "scale_mode": "mxfp4",- "layout": "output_input"- }).to_string()- }),- );- let mut payload = Vec::new();- let codes: Vec<u8> = (0..ROWS * COLUMNS / 2).map(|index| 0x10 | (index % 8) as u8).collect();- let scales: Vec<u8> = (0..ROWS * COLUMNS / GROUP_SIZE).map(|index| 126 + (index % 3) as u8).collect();- let global_scale = bf16::from_f32(0.75).to_le_bytes();- add_tensor(&mut header, &mut payload, "weights", &[ROWS, COLUMNS / 2], "U8", &codes);- add_tensor(&mut header, &mut payload, "scales", &[ROWS, COLUMNS / GROUP_SIZE], "U8", &scales);- add_tensor(&mut header, &mut payload, "global_scale", &[1], "BF16", &global_scale);-- let mut header = serde_json::to_vec(&Value::Object(header)).expect("serialize safetensors header");- header.extend(std::iter::repeat_n(b' ', (8 - header.len() % 8) % 8));- let mut file = NamedTempFile::new().expect("create dense MXFP4 fixture");- file.write_all(&(header.len() as u64).to_le_bytes()).expect("write safetensors header length");- file.write_all(&header).expect("write safetensors header");- file.write_all(&payload).expect("write safetensors payload");- file-}--#[uzu_test]-fn loads_dense_mxfp4_storage() {- // Given a safetensors artifact whose metadata and tensors describe dense MXFP4 weights.- let context = <Cpu as Backend>::Context::new().expect("create CPU context");- let file = dense_microfloat_parameter_file();- let loader = ParameterLoader::<Cpu>::new(file.as_file(), context.as_ref()).expect("load dense MXFP4 fixture");- let tree = loader.tree();- let spec = tree.metadata::<AnyWeightMatrixSpec>("spec").expect("load dense MXFP4 spec");-- // When the artifact is loaded through ParameterLoader and WeightMatrix.- let matrix =- WeightMatrix::load(&tree, spec, Layout::OutputInput, 2, 32, DataType::BF16).expect("load dense MXFP4 matrix");-- // Then the matrix retains its microfloat operand shape and consumes every source tensor.- let MatmulB::Microfloat {- metadata,- ..- } = matrix.matmul_b()- else {- panic!("dense MXFP4 matrix did not preserve its operand format");- };- assert_eq!(metadata.rows, 2);- assert_eq!(metadata.columns, 32);- tree.assert_all_tensors_validated().expect("validate dense MXFP4 tensors");-}