Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

RHT Block improvements + ActivationTransform fusion (#704)

CCYehaae4e743ebf8
diff --git a/crates/backend-uzu/src/backends/common/device_capabilities.rs b/crates/backend-uzu/src/backends/common/device_capabilities.rs
index 8125a319a..cc6852156 100644
--- a/crates/backend-uzu/src/backends/common/device_capabilities.rs
+++ b/crates/backend-uzu/src/backends/common/device_capabilities.rs
@@ -5,6 +5,5 @@ bitflags! {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct DeviceCapabilities: u32 {
const SPARSE_BUFFERS = 1 << 0;
- const NATIVE_INT8_MATMUL = 1 << 1;
}
}
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/gated_act_mul.rs b/crates/backend-uzu/src/backends/common/gpu_types/gated_act_mul.rs
new file mode 100644
index 000000000..d4605fbdb
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/gpu_types/gated_act_mul.rs
@@ -0,0 +1,7 @@
+#[repr(C)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum GatedActMulOp {
+ FullPrecision,
+ Quantize,
+ QuantizeWithGroupSums,
+}
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs b/crates/backend-uzu/src/backends/common/gpu_types/mod.rs
index 261cf7294..5c1089585 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/mod.rs
@@ -6,6 +6,7 @@
pub mod activation_transform;
pub mod activation_type;
pub mod attention;
+pub mod gated_act_mul;
pub mod gemm;
pub mod hadamard_order;
pub mod kv_cache_update;
@@ -19,6 +20,7 @@ pub mod weaver;
pub use activation_transform::*;
pub use activation_type::*;
pub use attention::*;
+pub use gated_act_mul::*;
pub use hadamard_order::*;
pub use kv_cache_update::*;
pub use matmul::*;
diff --git a/crates/backend-uzu/src/backends/common/kernel/gated_act_mul.rs b/crates/backend-uzu/src/backends/common/kernel/gated_act_mul.rs
new file mode 100644
index 000000000..198261601
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/kernel/gated_act_mul.rs
@@ -0,0 +1,185 @@
+use bitflags::bitflags;
+
+use crate::{
+ backends::common::{
+ Allocation, Backend, Encoder, Kernels,
+ gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE},
+ kernel::GatedActMulKernel,
+ },
+ data_type::DataType,
+};
+
+#[repr(u32)]
+#[derive(Clone, Copy)]
+enum GatedActMulGroupSize {
+ Size32 = 32,
+ Size64 = 64,
+ Size128 = 128,
+}
+
+impl GatedActMulGroupSize {
+ fn from_u32(value: u32) -> Self {
+ match value {
+ 32 => Self::Size32,
+ 64 => Self::Size64,
+ 128 => Self::Size128,
+ _ => panic!("unsupported activation group size: {value}"),
+ }
+ }
+}
+
+bitflags! {
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
+ struct GatedActMulOptions: u8 {
+ const INTERLEAVED = 1 << 0;
+ const HADAMARD = 1 << 1;
+ }
+}
+
+pub struct GatedActMul<B: Backend> {
+ kernel: <B::Kernels as Kernels>::GatedActMulKernel,
+ ops: GatedActMulOp,
+ options: GatedActMulOptions,
+ activation_group_size: u32,
+ sum_group_size: u32,
+}
+
+impl<B: Backend> GatedActMul<B> {
+ pub fn full_precision(
+ context: &B::Context,
+ data_type: DataType,
+ interleaved: bool,
+ use_hadamard: bool,
+ ) -> Result<Self, B::Error> {
+ let mut options = GatedActMulOptions::empty();
+ options.set(GatedActMulOptions::INTERLEAVED, interleaved);
+ options.set(GatedActMulOptions::HADAMARD, use_hadamard);
+ Self::new(
+ context,
+ data_type,
+ GatedActMulOp::FullPrecision,
+ options,
+ HADAMARD_TRANSFORM_BLOCK_SIZE,
+ HADAMARD_TRANSFORM_BLOCK_SIZE,
+ )
+ }
+
+ pub fn quantized(
+ context: &B::Context,
+ data_type: DataType,
+ activation_group_size: u32,
+ sum_group_size: Option<u32>,
+ ) -> Result<Self, B::Error> {
+ let activation_group_size = GatedActMulGroupSize::from_u32(activation_group_size);
+ let sum_group_size = sum_group_size.map(GatedActMulGroupSize::from_u32);
+ Self::new(
+ context,
+ data_type,
+ sum_group_size.map_or(GatedActMulOp::Quantize, |_| GatedActMulOp::QuantizeWithGroupSums),
+ GatedActMulOptions::INTERLEAVED | GatedActMulOptions::HADAMARD,
+ activation_group_size as u32,
+ sum_group_size.unwrap_or(activation_group_size) as u32,
+ )
+ }
+
+ fn new(
+ context: &B::Context,
+ data_type: DataType,
+ ops: GatedActMulOp,
+ options: GatedActMulOptions,
+ activation_group_size: u32,
+ sum_group_size: u32,
+ ) -> Result<Self, B::Error> {
+ let kernel = <B::Kernels as Kernels>::GatedActMulKernel::new(
+ context,
+ data_type,
+ ops,
+ options.contains(GatedActMulOptions::INTERLEAVED),
+ options.contains(GatedActMulOptions::HADAMARD),
+ activation_group_size,
+ sum_group_size,
+ )?;
+ Ok(Self {
+ kernel,
+ ops,
+ options,
+ activation_group_size,
+ sum_group_size,
+ })
+ }
+
+ pub fn encode_fp(
+ &self,
+ act_operand: &Allocation<B>,
+ value_operand: Option<&Allocation<B>>,
+ output: &mut Allocation<B>,
+ hadamard_factors: Option<&Allocation<B>>,
+ gated_dim: u32,
+ batch_dim: u32,
+ value_offset: u32,
+ value_row_stride: u32,
+ act_type: ActivationType,
+ encoder: &mut Encoder<B>,
+ ) {
+ assert_eq!(self.ops, GatedActMulOp::FullPrecision);
+ assert_eq!(self.options.contains(GatedActMulOptions::INTERLEAVED), value_operand.is_none());
+ assert_eq!(self.options.contains(GatedActMulOptions::HADAMARD), hadamard_factors.is_some());
+ assert!(
+ !self.options.contains(GatedActMulOptions::HADAMARD)
+ || gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE)
+ );
+ self.kernel.encode(
+ act_operand,
+ value_operand,
+ Some(output),
+ None::<&mut Allocation<B>>,
+ None::<&mut Allocation<B>>,
+ None::<&mut Allocation<B>>,
+ hadamard_factors,
+ gated_dim,
+ batch_dim,
+ value_offset,
+ value_row_stride,
+ act_type,
+ encoder,
+ );
+ }
+
+ pub fn encode_quantized(
+ &self,
+ act_operand: &Allocation<B>,
+ values: &mut Allocation<B>,
+ scales: &mut Allocation<B>,
+ group_sums: Option<&mut Allocation<B>>,
+ hadamard_factors: &Allocation<B>,
+ gated_dim: u32,
+ batch_dim: u32,
+ act_type: ActivationType,
+ encoder: &mut Encoder<B>,
+ ) {
+ assert!(matches!(self.ops, GatedActMulOp::Quantize | GatedActMulOp::QuantizeWithGroupSums));
+ assert!(self.options.contains(GatedActMulOptions::INTERLEAVED));
+ assert!(self.options.contains(GatedActMulOptions::HADAMARD));
+ assert!(gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));
+ assert!(gated_dim.is_multiple_of(self.activation_group_size));
+ assert_eq!(self.ops == GatedActMulOp::QuantizeWithGroupSums, group_sums.is_some());
+ if self.ops == GatedActMulOp::QuantizeWithGroupSums {
+ assert!(gated_dim.is_multiple_of(self.sum_group_size));
+ }
+ self.kernel.encode(
+ act_operand,
+ None::<&Allocation<B>>,
+ None::<&mut Allocation<B>>,
+ Some(values),
+ Some(scales),
+ group_sums,
+ Some(hadamard_factors),
+ gated_dim,
+ batch_dim,
+ 0,
+ 0,
+ act_type,
+ encoder,
+ );
+ }
+}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
index de49d1364..6ff0365f0 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
@@ -3,7 +3,7 @@ use crate::{
Backend, BufferArg, Encoder, Kernels,
kernel::matmul::{
arguments::MatmulArguments,
- routing::{MatmulPath, MatmulShape},
+ routing::{A8ActivationPlan, ActivationFormat, MatmulShape},
},
},
data_type::DataType,
@@ -25,11 +25,19 @@ pub trait MatmulKernel: Sized + Send + Sync {
encoder: &mut Encoder<Self::Backend>,
) -> Result<(), <Self::Backend as Backend>::Error>;
- fn select_path(
+ fn a8_activation_plan(
&self,
- _shape: &MatmulShape,
+ _candidate: &MatmulShape,
_context: &<Self::Backend as Backend>::Context,
- ) -> MatmulPath {
- MatmulPath::Gemm
+ ) -> Option<A8ActivationPlan> {
+ None
+ }
+
+ fn select_activation_format(
+ &self,
+ _bf16_shape: &MatmulShape,
+ _context: &<Self::Backend as Backend>::Context,
+ ) -> ActivationFormat {
+ ActivationFormat::Bf16
}
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs
index c727ec22f..13089c3fd 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs
@@ -12,4 +12,4 @@ pub use error::MatmulError;
pub use kernel::MatmulKernel;
pub use matmul_a::MatmulA;
pub use matmul_b::MatmulB;
-pub use routing::{MatmulPath, MatmulShape};
+pub use routing::{A8ActivationPlan, ActivationFormat, MatmulShape};
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/routing.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/routing.rs
index 705c03573..e0f3e3c1f 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/routing.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/routing.rs
@@ -4,6 +4,18 @@ use crate::backends::common::{
gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
};
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ActivationFormat {
+ Bf16,
+ Int8,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct A8ActivationPlan {
+ pub activation_group_size: u32,
+ pub sum_group_size: Option<u32>,
+}
+
#[derive(Clone, Copy)]
pub struct MatmulShape {
pub m: u32,
@@ -44,9 +56,3 @@ impl MatmulShape {
self.b_prologue != GemmBPrologueKind::FullPrecision
}
}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum MatmulPath {
- Gemv,
- Gemm,
-}
diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rs
index 25adfc458..78c86cca5 100644
--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs
@@ -4,10 +4,12 @@ pub mod activation_transform;
pub mod attention_gemm;
pub mod delta_net_chunked_prefill;
pub mod delta_net_tree_verify;
+pub mod gated_act_mul;
pub mod matmul;
pub mod radix_top_k_small;
pub use activation_transform::ActivationTransform;
+pub use gated_act_mul::GatedActMul;
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs
index b33ecaff9..c6e0ce5ae 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs
@@ -4,6 +4,8 @@ use crate::backends::common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE;
pub const INT8_SYMMETRIC_QUANTIZATION_MAXIMUM: f32 = 127.0;
+pub use activation_transform::quantize_transformed_row;
+
pub fn min_max_symmetric_divisor(values: &[f32]) -> f32 {
let (min, max) =
values.iter().fold((f32::INFINITY, f32::NEG_INFINITY), |(min, max), &value| (min.min(value), max.max(value)));
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs b/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
index 48a0fa013..5f65b8e24 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
@@ -2,45 +2,119 @@ use half::bf16;
use num_traits::Float;
use proc_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
+use crate::{
+ array::ArrayElement,
+ backends::{
+ common::gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE},
+ cpu::kernel::activation_transform::{hadamard_transform, quantize_transformed_row},
+ },
+};
#[kernel(GatedActMul)]
#[variants(T, f32, bf16)]
pub fn gated_act_mul<T: ArrayElement + Float>(
act_operand: *const T,
#[optional(!interleaved)] value_operand: Option<*const T>,
- output: *mut T,
+ #[optional(ops == GatedActMulOp::FullPrecision)] fp_out: Option<*mut T>,
+ #[optional(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)] q_out: Option<*mut i8>,
+ #[optional(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)] scales_out: Option<
+ *mut f32,
+ >,
+ #[optional(ops == GatedActMulOp::QuantizeWithGroupSums)] group_sums_out: Option<*mut i32>,
#[optional(use_hadamard)] hadamard_factors: Option<*const i32>,
gated_dim: u32,
batch_dim: u32,
value_offset: u32,
value_row_stride: u32,
act_type: ActivationType,
+ #[specialize] ops: GatedActMulOp,
#[specialize] interleaved: bool,
#[specialize] use_hadamard: bool,
+ #[specialize] activation_scale_group_size: u32,
+ #[specialize] sum_group_size: u32,
) {
assert_eq!(hadamard_factors.is_some(), use_hadamard);
+ let quantize = matches!(ops, GatedActMulOp::Quantize | GatedActMulOp::QuantizeWithGroupSums);
+ let emit_group_sums = ops == GatedActMulOp::QuantizeWithGroupSums;
+ assert!(!quantize || use_hadamard, "quantized gate activation requires RHT");
+ assert!(!quantize || gated_dim.is_multiple_of(activation_scale_group_size));
+ assert!(!emit_group_sums || gated_dim.is_multiple_of(sum_group_size));
+ assert!(!use_hadamard || gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));
- if use_hadamard {
- unimplemented!("not supported yet");
- }
let gated_dim = gated_dim as usize;
let batch_dim = batch_dim as usize;
- let value_offset = value_offset as usize;
- let value_row_stride = value_row_stride as usize;
+ let activation_scale_group_size = activation_scale_group_size as usize;
+ let sum_group_size = sum_group_size as usize;
for batch in 0..batch_dim {
+ let mut transformed = use_hadamard.then(|| vec![0.0f32; gated_dim]);
for gated in 0..gated_dim {
let (act_index, value) = if interleaved {
let base = batch * 2 * gated_dim;
(base + gated_dim + gated, unsafe { *act_operand.add(base + gated) })
} else {
- let value_index = batch * value_row_stride + value_offset + gated;
+ let value_index = batch * value_row_stride as usize + value_offset as usize + gated;
(batch * gated_dim + gated, unsafe { *value_operand.unwrap().add(value_index) })
};
- let activated: T = act_type.activate(unsafe { *act_operand.add(act_index) });
- let result = value.to_f32().unwrap() * activated.to_f32().unwrap();
- unsafe { *output.add(batch * gated_dim + gated) = T::from(result).unwrap() };
+ let gate = unsafe { *act_operand.add(act_index) };
+ let result = super::gated_act_mul(value, gate, act_type);
+ if let Some(transformed) = transformed.as_mut() {
+ transformed[gated] = result;
+ } else {
+ unsafe {
+ *fp_out.expect("FP gate activation requires fp_out").add(batch * gated_dim + gated) =
+ T::from(result).unwrap();
+ }
+ }
+ }
+
+ if let Some(transformed) = transformed.as_mut() {
+ let factors = hadamard_factors.expect("Hadamard factors are required");
+ for stripe_start in (0..gated_dim).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {
+ let mut stripe = [0.0f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize];
+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE as usize {
+ let gated = stripe_start + lane;
+ stripe[lane] = transformed[gated] * unsafe { *factors.add(gated) } as f32;
+ }
+ hadamard_transform(&mut stripe);
+ transformed[stripe_start..stripe_start + HADAMARD_TRANSFORM_BLOCK_SIZE as usize]
+ .copy_from_slice(&stripe);
+ }
+
+ if quantize {
+ let values = unsafe {
+ std::slice::from_raw_parts_mut(
+ q_out.expect("quantized gate activation requires q_out").add(batch * gated_dim),
+ gated_dim,
+ )
+ };
+ let scales_per_row = gated_dim / activation_scale_group_size;
+ let scales = unsafe {
+ std::slice::from_raw_parts_mut(
+ scales_out.expect("quantized gate activation requires scales_out").add(batch * scales_per_row),
+ scales_per_row,
+ )
+ };
+ let group_sums = group_sums_out.map(|output| unsafe {
+ std::slice::from_raw_parts_mut(
+ output.add(batch * gated_dim / sum_group_size),
+ gated_dim / sum_group_size,
+ )
+ });
+ quantize_transformed_row(
+ transformed,
+ activation_scale_group_size,
+ emit_group_sums.then_some(sum_group_size),
+ values,
+ scales,
+ group_sums,
+ );
+ } else {
+ let output = fp_out.expect("FP gate activation requires fp_out");
+ for (gated, &value) in transformed.iter().enumerate() {
+ unsafe { *output.add(batch * gated_dim + gated) = T::from(value).unwrap() };
+ }
+ }
}
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/mod.rs
index 95bdbdf41..ef320ee64 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/mod.rs
@@ -1 +1,14 @@
+use num_traits::Float;
+
+use crate::backends::common::gpu_types::activation_type::ActivationType;
+
+#[inline]
+pub(super) fn gated_act_mul<T: Float>(
+ value: T,
+ gate: T,
+ act_type: ActivationType,
+) -> f32 {
+ (value * act_type.activate(gate)).to_f32().unwrap()
+}
+
pub mod gated_act_mul;
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index f31b27958..9c7f350fc 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -269,9 +269,6 @@ impl Context for MetalContext {
if self.device.supports_placement_sparse_resources() {
capabilities |= DeviceCapabilities::SPARSE_BUFFERS;
}
- if self.device.supports_mxu() {
- capabilities |= DeviceCapabilities::NATIVE_INT8_MATMUL;
- }
capabilities
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/activation_transform/activation_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/activation_transform/activation_transform.metal
index 51e9e70f9..d5bfe2713 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/activation_transform/activation_transform.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/activation_transform/activation_transform.metal
@@ -2,71 +2,19 @@
#include "../common/defines.h"
#include "../common/dsl.h"
#include "../common/thread_context.h"
+#include "../common/activation_quantization.h"
#include "../generated/activation_transform.h"
#include "../hadamard_transform/hadamard_transform.h"
using namespace metal;
using namespace uzu::activation_transform;
-UZU_CONST uint ACTIVATION_TILE_SIZE = 128;
-UZU_CONST uint SIMDGROUPS_PER_THREADGROUP = ACTIVATION_TILE_SIZE / METAL_SIMD_SIZE;
-UZU_CONST float INT8_QMAX = 127.0f;
+#define NUM_SIMDGROUPS 4
+#define NUM_THREADS NUM_SIMDGROUPS* METAL_SIMD_SIZE
#define QUANTIZED (ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)
#define EMITS_GROUP_SUMS (ops == ActivationTransformOp::QuantizeWithGroupSums)
-template <typename T, typename SimdReduce, typename Combine>
-METAL_FUNC T reduce_quantization_group(
- const T value,
- const uint group_size,
- threadgroup T* partials,
- const thread ThreadContext& thread_context,
- SimdReduce simd_reduce,
- Combine combine
-) {
- const ushort lane_index = thread_context.simd_lane_id;
- const ushort simdgroup_index = thread_context.simdgroup_index;
- const T simdgroup_value = simd_reduce(value);
- if (group_size == METAL_SIMD_SIZE) {
- return simdgroup_value;
- }
- if (lane_index == 0) {
- partials[simdgroup_index] = simdgroup_value;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const uint simdgroups_per_group = group_size / METAL_SIMD_SIZE;
- const uint simdgroup_base = simdgroup_index / simdgroups_per_group * simdgroups_per_group;
- T result = partials[simdgroup_base];
- METAL_PRAGMA_UNROLL
- for (uint index = 1; index < simdgroups_per_group; ++index) {
- result = combine(result, partials[simdgroup_base + index]);
- }
- return result;
-}
-
-template <typename T>
-METAL_FUNC void write_quantization_group(
- device T* output,
- const T value,
- const uint group_size,
- const uint element_count,
- const uint tile_index,
- const uint batch_index,
- const thread ThreadContext& thread_context
-) {
- const ushort simdgroup_index = thread_context.simdgroup_index;
- const uint simdgroups_per_group = group_size / METAL_SIMD_SIZE;
- const uint simdgroup_offset = tile_index * ACTIVATION_TILE_SIZE + simdgroup_index * METAL_SIMD_SIZE;
- if (thread_context.simd_lane_id == 0 && simdgroup_offset < element_count &&
- (simdgroup_index + 1) % simdgroups_per_group == 0) {
- const uint groups_per_row = element_count / group_size;
- const uint groups_per_tile = ACTIVATION_TILE_SIZE / group_size;
- const uint group_index = tile_index * groups_per_tile + simdgroup_index / simdgroups_per_group;
- output[batch_index * groups_per_row + group_index] = value;
- }
-}
-
template <typename T>
VARIANTS(T, float, bfloat)
PUBLIC KERNEL(ActivationTransform)(
@@ -82,11 +30,11 @@ PUBLIC KERNEL(ActivationTransform)(
const bool in_place SPECIALIZE,
const uint activation_scale_group_size SPECIALIZE,
const uint sum_group_size SPECIALIZE,
- threadgroup float partial_max OPTIONAL(QUANTIZED && activation_scale_group_size > METAL_SIMD_SIZE)[4],
- threadgroup int partial_sums OPTIONAL(EMITS_GROUP_SUMS && sum_group_size > METAL_SIMD_SIZE)[4],
- uint activation_tile_index GROUPS(element_count.div_ceil(128)),
+ threadgroup float partial_max OPTIONAL(QUANTIZED && activation_scale_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],
+ threadgroup int partial_sums OPTIONAL(EMITS_GROUP_SUMS && sum_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],
+ uint activation_tile_index GROUPS(element_count.div_ceil(NUM_THREADS)),
uint batch_index GROUPS(batch_size),
- uint thread_index THREADS(128),
+ uint thread_index THREADS(NUM_THREADS),
const ThreadContext thread_context
) {
(void)thread_index;
@@ -97,7 +45,7 @@ PUBLIC KERNEL(ActivationTransform)(
const bool input_rht = ops != ActivationTransformOp::OutputRht;
const ushort lane_index = thread_context.simd_lane_id;
const ushort simdgroup_index = thread_context.simdgroup_index;
- const uint tile_offset = activation_tile_index * ACTIVATION_TILE_SIZE;
+ const uint tile_offset = activation_tile_index * ACTIVATION_QUANT_TILE_SIZE;
const uint simdgroup_offset = tile_offset + simdgroup_index * METAL_SIMD_SIZE;
const bool in_bounds = simdgroup_offset < element_count;
const uint factor_index = simdgroup_offset + lane_index;
@@ -117,7 +65,7 @@ PUBLIC KERNEL(ActivationTransform)(
return;
}
- const float maximum = reduce_quantization_group(
+ const float maximum = reduce_activation_quantization_group(
fabs(value),
activation_scale_group_size,
partial_max,
@@ -125,13 +73,14 @@ PUBLIC KERNEL(ActivationTransform)(
[](float x) { return simd_max(x); },
[](float x, float y) { return max(x, y); }
);
- const float scale = isfinite(maximum) && maximum > 0.0f ? maximum / INT8_QMAX : 1.0f;
+ const float scale = isfinite(maximum) && maximum > 0.0f ? maximum / ACTIVATION_QUANT_INT8_MAX : 1.0f;
- const int8_t code = static_cast<int8_t>(clamp(round(value / scale), -INT8_QMAX, INT8_QMAX));
+ const int8_t code =
+ static_cast<int8_t>(clamp(round(value / scale), -ACTIVATION_QUANT_INT8_MAX, ACTIVATION_QUANT_INT8_MAX));
if (in_bounds) {
q_out[element_index] = code;
}
- write_quantization_group(
+ write_activation_quantization_group(
scales_out,
scale,
activation_scale_group_size,
@@ -142,7 +91,7 @@ PUBLIC KERNEL(ActivationTransform)(
);
if (EMITS_GROUP_SUMS) {
- const int sum = reduce_quantization_group(
+ const int sum = reduce_activation_quantization_group(
int(code),
sum_group_size,
partial_sums,
@@ -150,7 +99,7 @@ PUBLIC KERNEL(ActivationTransform)(
[](int x) { return simd_sum(x); },
[](int x, int y) { return x + y; }
);
- write_quantization_group(
+ write_activation_quantization_group(
group_sums_out,
sum,
sum_group_size,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/activation_quantization.h b/crates/backend-uzu/src/backends/metal/kernel/common/activation_quantization.h
new file mode 100644
index 000000000..a391f1388
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/activation_quantization.h
@@ -0,0 +1,63 @@
+#pragma once
+
+#include <metal_stdlib>
+#include "defines.h"
+#include "thread_context.h"
+
+using namespace metal;
+
+UZU_CONST uint ACTIVATION_QUANT_TILE_SIZE = 128;
+UZU_CONST float ACTIVATION_QUANT_INT8_MAX = 127.0f;
+#define ACTIVATION_QUANT_SIMDGROUPS 4
+
+template <typename T, typename SimdReduce, typename Combine>
+METAL_FUNC T reduce_activation_quantization_group(
+ const T value,
+ const uint group_size,
+ threadgroup T* partials,
+ const thread ThreadContext& thread_context,
+ SimdReduce simd_reduce,
+ Combine combine
+) {
+ const ushort lane_index = thread_context.simd_lane_id;
+ const ushort simdgroup_index = thread_context.simdgroup_index;
+ const T simdgroup_value = simd_reduce(value);
+ if (group_size == METAL_SIMD_SIZE) {
+ return simdgroup_value;
+ }
+ if (lane_index == 0) {
+ partials[simdgroup_index] = simdgroup_value;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const uint simdgroups_per_group = group_size / METAL_SIMD_SIZE;
+ const uint simdgroup_base = simdgroup_index / simdgroups_per_group * simdgroups_per_group;
+ T result = partials[simdgroup_base];
+ METAL_PRAGMA_UNROLL
+ for (uint index = 1; index < simdgroups_per_group; ++index) {
+ result = combine(result, partials[simdgroup_base + index]);
+ }
+ return result;
+}
+
+template <typename T>
+METAL_FUNC void write_activation_quantization_group(
+ device T* output,
+ const T value,
+ const uint group_size,
+ const uint element_count,
+ const uint tile_index,
+ const uint batch_index,
+ const thread ThreadContext& thread_context
+) {
+ const ushort simdgroup_index = thread_context.simdgroup_index;
+ const uint simdgroups_per_group = group_size / METAL_SIMD_SIZE;
+ const uint simdgroup_offset = tile_index * ACTIVATION_QUANT_TILE_SIZE + simdgroup_index * METAL_SIMD_SIZE;
+ if (thread_context.simd_lane_id == 0 && simdgroup_offset < element_count &&
+ (simdgroup_index + 1) % simdgroups_per_group == 0) {
+ const uint groups_per_row = element_count / group_size;
+ const uint groups_per_tile = ACTIVATION_QUANT_TILE_SIZE / group_size;
+ const uint group_index = tile_index * groups_per_tile + simdgroup_index / simdgroups_per_group;
+ output[batch_index * groups_per_row + group_index] = value;
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/gated_act_mul.h b/crates/backend-uzu/src/backends/metal/kernel/common/gated_act_mul.h
new file mode 100644
index 000000000..cee631d64
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/gated_act_mul.h
@@ -0,0 +1,14 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../activation/activations.h"
+
+using namespace metal;
+using namespace uzu::activation_type;
+
+template <typename T>
+static METAL_FUNC float gated_act_mul(T value, T gate, ActivationType act_type) {
+ const T result = value * activate(gate, act_type);
+ return static_cast<float>(result);
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal b/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
index ef6ca3413..b4867bb7e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
@@ -1,40 +1,88 @@
#include <metal_stdlib>
-#include "../activation/activations.h"
+#include "../common/activation_quantization.h"
#include "../common/dsl.h"
+#include "../common/gated_act_mul.h"
+#include "../common/thread_context.h"
+#include "../generated/gated_act_mul.h"
#include "../hadamard_transform/hadamard_transform.h"
+using namespace metal;
using namespace uzu::activation_type;
+using namespace uzu::gated_act_mul;
+
+#define NUM_SIMDGROUPS 4
+#define NUM_THREADS NUM_SIMDGROUPS* METAL_SIMD_SIZE
+
+#define QUANTIZED (ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)
+#define EMITS_GROUP_SUMS (ops == GatedActMulOp::QuantizeWithGroupSums)
template <typename T>
VARIANTS(T, float, bfloat)
PUBLIC KERNEL(GatedActMul) (
const device T* act_operand,
const device T* value_operand OPTIONAL(!interleaved),
- device T* output,
+ device T* fp_out OPTIONAL(!QUANTIZED),
+ device int8_t* q_out OPTIONAL(QUANTIZED),
+ device float* scales_out OPTIONAL(QUANTIZED),
+ device int32_t* group_sums_out OPTIONAL(EMITS_GROUP_SUMS),
const device int32_t* hadamard_factors OPTIONAL(use_hadamard),
const constant uint& gated_dim,
const constant uint& batch_dim,
const constant uint& value_offset,
const constant uint& value_row_stride,
const constant ActivationType& act_type,
+ const GatedActMulOp ops SPECIALIZE,
const bool interleaved SPECIALIZE,
const bool use_hadamard SPECIALIZE,
- uint gated_idx AXIS(gated_dim, 64),
- uint batch_idx AXIS(batch_dim, 1)
+ const uint activation_scale_group_size SPECIALIZE,
+ const uint sum_group_size SPECIALIZE,
+ threadgroup float partial_max OPTIONAL(QUANTIZED && activation_scale_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],
+ threadgroup int partial_sums OPTIONAL(EMITS_GROUP_SUMS && sum_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],
+ uint activation_tile_index GROUPS(gated_dim.div_ceil(NUM_THREADS)),
+ uint batch_idx GROUPS(batch_dim),
+ uint thread_index THREADS(NUM_THREADS),
+ const ThreadContext thread_context
) {
- T activated;
- T value;
- if (interleaved) {
- uint base = batch_idx * (2 * gated_dim);
- value = act_operand[base + gated_idx];
- activated = activate(act_operand[base + gated_dim + gated_idx], act_type);
- } else {
- activated = activate(act_operand[batch_idx * gated_dim + gated_idx], act_type);
- value = value_operand[batch_idx * value_row_stride + value_offset + gated_idx];
+ const uint gated_idx = activation_tile_index * ACTIVATION_QUANT_TILE_SIZE + thread_index;
+ const uint simdgroup_offset =
+ activation_tile_index * ACTIVATION_QUANT_TILE_SIZE + (thread_index / METAL_SIMD_SIZE) * METAL_SIMD_SIZE;
+ const bool element_in_bounds = gated_idx < gated_dim;
+ const bool simdgroup_in_bounds = simdgroup_offset + METAL_SIMD_SIZE <= gated_dim;
+ T value = static_cast<T>(0);
+ T gate = static_cast<T>(0);
+ if (element_in_bounds) {
+ if (interleaved) {
+ const uint base = batch_idx * (2 * gated_dim);
+ value = act_operand[base + gated_idx];
+ gate = act_operand[base + gated_dim + gated_idx];
+ } else {
+ value = value_operand[batch_idx * value_row_stride + value_offset + gated_idx];
+ gate = act_operand[batch_idx * gated_dim + gated_idx];
+ }
}
- T result = value * activated;
- if (use_hadamard) {
+ if (!QUANTIZED) {
+ T result = static_cast<T>(0);
+ if (element_in_bounds && (!use_hadamard || simdgroup_in_bounds)) {
+ float gated = gated_act_mul(value, gate, act_type);
+ if (use_hadamard) {
+ gated = simdgroup_input_random_hadamard_transform(
+ static_cast<ushort>(gated_idx % METAL_SIMD_SIZE),
+ gated,
+ hadamard_factors[gated_idx]
+ );
+ }
+ result = static_cast<T>(gated);
+ }
+ if (element_in_bounds) {
+ fp_out[batch_idx * gated_dim + gated_idx] = result;
+ }
+ return;
+ }
+
+ float result = 0.0f;
+ if (simdgroup_in_bounds) {
+ result = gated_act_mul(value, gate, act_type);
result = simdgroup_input_random_hadamard_transform(
static_cast<ushort>(gated_idx % METAL_SIMD_SIZE),
result,
@@ -42,5 +90,48 @@ PUBLIC KERNEL(GatedActMul) (
);
}
- output[batch_idx * gated_dim + gated_idx] = result;
+ const float maximum = reduce_activation_quantization_group(
+ fabs(result),
+ activation_scale_group_size,
+ partial_max,
+ thread_context,
+ [](float x) { return simd_max(x); },
+ [](float x, float y) { return max(x, y); }
+ );
+ const float scale = isfinite(maximum) && maximum > 0.0f ? maximum / ACTIVATION_QUANT_INT8_MAX : 1.0f;
+ const int8_t code =
+ static_cast<int8_t>(clamp(round(result / scale), -ACTIVATION_QUANT_INT8_MAX, ACTIVATION_QUANT_INT8_MAX));
+ if (element_in_bounds) {
+ q_out[batch_idx * gated_dim + gated_idx] = code;
+ }
+
+ write_activation_quantization_group(
+ scales_out,
+ scale,
+ activation_scale_group_size,
+ gated_dim,
+ activation_tile_index,
+ batch_idx,
+ thread_context
+ );
+
+ if (EMITS_GROUP_SUMS) {
+ const int sum = reduce_activation_quantization_group(
+ static_cast<int>(code),
+ sum_group_size,
+ partial_sums,
+ thread_context,
+ [](int x) { return simd_sum(x); },
+ [](int x, int y) { return x + y; }
+ );
+ write_activation_quantization_group(
+ group_sums_out,
+ sum,
+ sum_group_size,
+ gated_dim,
+ activation_tile_index,
+ batch_idx,
+ thread_context
+ );
+ }
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gated_act_mul.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gated_act_mul.h
new file mode 100644
index 000000000..571e56256
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gated_act_mul.h
@@ -0,0 +1,13 @@
+// Auto-generated from gpu_types/gated_act_mul - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::gated_act_mul {
+enum class GatedActMulOp : uint32_t {
+ FullPrecision = 0,
+ Quantize = 1,
+ QuantizeWithGroupSums = 2,
+};
+} // namespace uzu::gated_act_mul
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index 03ade8771..e6bfdba49 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -10,8 +10,11 @@ use crate::{
backends::{
common::{
BufferArg, Encoder,
- gpu_types::gemm::GemmTiling,
- kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulPath, MatmulShape},
+ gpu_types::gemm::{GemmBPrologueKind, GemmTiling},
+ kernel::{
+ activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
+ matmul::{A8ActivationPlan, ActivationFormat, MatmulArguments, MatmulError, MatmulKernel, MatmulShape},
+ },
},
metal::{Metal, context::MetalContext, error::MetalError},
},
@@ -120,14 +123,57 @@ impl MatmulKernel for MatmulMetalKernel {
})
}
- fn select_path(
+ fn a8_activation_plan(
&self,
shape: &MatmulShape,
context: &MetalContext,
- ) -> MatmulPath {
- match self.select_dispatch(shape, context) {
- MatmulDispatch::Gemv(_) => MatmulPath::Gemv,
- MatmulDispatch::Gemm(_) => MatmulPath::Gemm,
+ ) -> Option<A8ActivationPlan> {
+ let activation_group_size = ACTIVATION_SCALE_GROUP_SIZE;
+ if !context.supports_mxu()
+ || self.input_data_type != DataType::BF16
+ || self.output_data_type != DataType::BF16
+ || shape.a_full_precision
+ || !shape.is_quant()
+ || !shape.signed_codes
+ || !shape.b_transpose
+ || shape.b_leading_dimension.is_some()
+ || !matches!(shape.b_bits, Some(4 | 8))
+ || !matches!(shape.b_group_size, Some(32 | 64 | 128))
+ || !shape.k.is_multiple_of(activation_group_size)
+ || !shape.k.is_multiple_of(shape.b_group_size.unwrap())
+ {
+ return None;
+ }
+
+ let sum_group_size = match shape.b_prologue {
+ GemmBPrologueKind::ScaleSymmetricDequant => None,
+ GemmBPrologueKind::ScaleBiasDequant | GemmBPrologueKind::ScaleZeroPointDequant => {
+ Some(shape.b_group_size.unwrap().min(activation_group_size))
+ },
+ GemmBPrologueKind::FullPrecision => return None,
+ };
+ Some(A8ActivationPlan {
+ activation_group_size,
+ sum_group_size,
+ })
+ }
+
+ fn select_activation_format(
+ &self,
+ bf16_shape: &MatmulShape,
+ context: &MetalContext,
+ ) -> ActivationFormat {
+ if matches!(self.select_dispatch(bf16_shape, context), MatmulDispatch::Gemv(_)) {
+ return ActivationFormat::Bf16;
+ }
+
+ let a8_shape = MatmulShape {
+ a_full_precision: false,
+ ..*bf16_shape
+ };
+ match self.select_dispatch(&a8_shape, context) {
+ MatmulDispatch::Gemm(plan) if plan.engine == gemm::GemmEngine::Mxu => ActivationFormat::Int8,
+ _ => ActivationFormat::Bf16,
}
}
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index e04b78329..7b352fff4 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -153,10 +153,10 @@ impl<B: Backend> Embedding<B> {
AnyWeightMatrixSpec::HybridSpec(HybridSpec {
quantization_spec,
adapter_spec: None,
- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
+ incoherence_block_size: Some(block_size),
incoherence_processing_mode: IncoherenceProcessingMode::Output,
..
- }) => {
+ }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => {
let incoherence_signs_tree = embedding_tree.subtree("incoherence_signs");
let output_hadamard_factors = Some(
incoherence_signs_tree
@@ -201,10 +201,10 @@ impl<B: Backend> Embedding<B> {
AnyWeightMatrixSpec::HybridSpec(HybridSpec {
quantization_spec,
adapter_spec: None,
- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
+ incoherence_block_size: Some(block_size),
incoherence_processing_mode: IncoherenceProcessingMode::Output,
..
- }) => {
+ }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => {
let output_hadamard_factors = Some(
input_embedding_tree
.subtree("incoherence_signs")
@@ -240,10 +240,10 @@ impl<B: Backend> Embedding<B> {
AnyWeightMatrixSpec::HybridSpec(HybridSpec {
quantization_spec,
adapter_spec: None,
- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
+ incoherence_block_size: Some(block_size),
incoherence_processing_mode: IncoherenceProcessingMode::Input,
..
- }) => {
+ }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => {
let matrix = WeightMatrix::load(
&output_embedding_tree.subtree("quantized"),
*quantization_spec,
diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
index 213a857ad..59a5e8827 100644
--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
@@ -2,17 +2,21 @@ use parking_lot::Mutex;
use thiserror::Error;
use crate::{
+ array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder,
kernel::{
Kernels,
- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulPath, MatmulShape},
+ matmul::{
+ A8ActivationPlan, ActivationFormat, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel,
+ MatmulShape,
+ },
},
},
config::weight_matrix::{AnyWeightMatrixSpec, Layout},
data_type::DataType,
encodable_block::{
- linear::Linear,
+ linear::{Linear, LinearInput},
weight_matrix::{WeightMatrix, WeightMatrixError},
},
parameters::{ParameterLoaderError, ParameterTree},
@@ -104,8 +108,15 @@ impl<B: Backend> LinearMatmul<B> {
})
}
- pub(super) fn make_codes_signed(&mut self) {
+ pub(super) fn prepare_a8(
+ &mut self,
+ context: &B::Context,
+ ) -> Option<A8ActivationPlan> {
+ let mut candidate = self.matmul_shape(1, false);
+ candidate.signed_codes = true;
+ let plan = self.kernel.lock().a8_activation_plan(&candidate, context)?;
self.matrix.make_codes_signed();
+ Some(plan)
}
pub(super) fn encode_with_a(
@@ -114,7 +125,8 @@ impl<B: Backend> LinearMatmul<B> {
batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut output = encoder.allocate_scratch_for_shape(&[batch_dim, self.output_dim], self.output_data_type)?;
+ let mut output =
+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;
self.kernel.lock().encode(
MatmulArguments {
@@ -135,25 +147,13 @@ impl<B: Backend> LinearMatmul<B> {
Ok(output)
}
- fn matmul_b(&self) -> MatmulB<'_, B> {
- self.matrix.matmul_b()
- }
-
- fn d_ops(&self) -> MatmulDOps<'_, B> {
- MatmulDOps {
- bias: self.biases.as_ref(),
- rht_factors: self.output_hadamard_factors.as_ref(),
- ..MatmulDOps::none()
- }
- }
-
- pub(super) fn select_path(
+ fn matmul_shape(
&self,
batch_dim: u32,
- context: &B::Context,
- ) -> MatmulPath {
+ a_full_precision: bool,
+ ) -> MatmulShape {
let b = self.matmul_b();
- let shape = MatmulShape {
+ MatmulShape {
m: batch_dim,
n: self.output_dim,
k: self.input_dim,
@@ -163,11 +163,34 @@ impl<B: Backend> LinearMatmul<B> {
b_bits: b.bits_per_b(),
b_group_size: b.group_size(),
signed_codes: b.signed_codes(),
- a_full_precision: true,
+ a_full_precision,
gathered: false,
d_transform: self.d_ops().mask(),
- };
- self.kernel.lock().select_path(&shape, context)
+ }
+ }
+
+ pub(super) fn select_activation_format(
+ &self,
+ batch_dim: u32,
+ context: &B::Context,
+ ) -> ActivationFormat {
+ if !self.matmul_b().signed_codes() {
+ return ActivationFormat::Bf16;
+ }
+ let bf16_shape = self.matmul_shape(batch_dim, true);
+ self.kernel.lock().select_activation_format(&bf16_shape, context)
+ }
+
+ fn matmul_b(&self) -> MatmulB<'_, B> {
+ self.matrix.matmul_b()
+ }
+
+ fn d_ops(&self) -> MatmulDOps<'_, B> {
+ MatmulDOps {
+ bias: self.biases.as_ref(),
+ rht_factors: self.output_hadamard_factors.as_ref(),
+ ..MatmulDOps::none()
+ }
}
}
@@ -193,4 +216,38 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
Ok(output)
}
+
+ fn encode_input(
+ &self,
+ input: LinearInput<B>,
+ batch_dim: u32,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ match input {
+ LinearInput::FullPrecision(input) => self.encode(input, batch_dim, encoder),
+ LinearInput::Int8Symmetric {
+ values,
+ scales,
+ group_sums,
+ group_size,
+ } => self.encode_with_a(
+ MatmulA::Int8Symmetric {
+ values: &values,
+ scales: &scales,
+ group_sums: group_sums.as_ref(),
+ group_size,
+ },
+ batch_dim,
+ encoder,
+ ),
+ }
+ }
+
+ fn select_activation_format(
+ &self,
+ batch_dim: u32,
+ context: &B::Context,
+ ) -> ActivationFormat {
+ Self::select_activation_format(self, batch_dim, context)
+ }
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rs
index 42594c9cf..333e6209a 100644
--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs
@@ -4,21 +4,20 @@ mod rht_wrapper;
pub use matmul::{LinearMatmul, LinearMatmulError};
pub use qlora_wrapper::{QLoRALinearWrapper, QLoRALinearWrapperError};
-use rht_wrapper::int8_activations_eligible;
pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};
use thiserror::Error;
use crate::{
backends::common::{
- Allocation, Backend, Encoder, gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
- kernel::activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
+ Allocation, Backend, Encoder,
+ gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ kernel::matmul::{A8ActivationPlan, ActivationFormat},
},
config::weight_matrix::{
AnyWeightMatrixSpec,
hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
},
data_type::DataType,
- encodable_block::weight_matrix::parse_spec,
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -29,14 +28,51 @@ pub trait Linear<B: Backend>: Send + Sync {
batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error>;
+
+ fn encode_input(
+ &self,
+ input: LinearInput<B>,
+ batch_dim: u32,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ match input {
+ LinearInput::FullPrecision(input) => self.encode(input, batch_dim, encoder),
+ LinearInput::Int8Symmetric {
+ ..
+ } => {
+ panic!("linear does not support pre-quantized activations")
+ },
+ }
+ }
+
+ fn select_activation_format(
+ &self,
+ _batch_dim: u32,
+ _context: &B::Context,
+ ) -> ActivationFormat {
+ ActivationFormat::Bf16
+ }
+}
+
+pub enum LinearInput<B: Backend> {
+ FullPrecision(Allocation<B>),
+ Int8Symmetric {
+ values: Allocation<B>,
+ scales: Allocation<B>,
+ group_sums: Option<Allocation<B>>,
+ group_size: u32,
+ },
+}
+
+pub struct LinearInputPreparation<B: Backend> {
+ pub input_factors: Allocation<B>,
+ pub a8_plan: Option<A8ActivationPlan>,
}
#[derive(Debug, Error)]
pub enum LinearBlockError<B: Backend> {
#[error("LinearMatmul error: {0}")]
LinearMatmulError(#[from] LinearMatmulError<B>),
- #[error("Output hadamard linear error: {0}")]
- OutputHadamardLinearError(#[from] OutputHadamardLinearError<B>),
#[error("QLoRALinearWrapper error: {0}")]
QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),
#[error("RHTLinearWrapper error: {0}")]
@@ -47,16 +83,6 @@ pub enum LinearBlockError<B: Backend> {
UnsupportedConfiguration(String),
}
-#[derive(Debug, Error)]
-pub enum OutputHadamardLinearError<B: Backend> {
- #[error("LinearMatmul error: {0}")]
- LinearMatmulError(#[from] LinearMatmulError<B>),
- #[error("Parameter loading error: {0}")]
- ParameterError(#[from] ParameterLoaderError<B>),
- #[error("Unsupported linear configuration: {0}")]
- UnsupportedConfiguration(String),
-}
-
impl<B: Backend> dyn Linear<B> {
pub fn new_mixed_precision<const N: usize>(
input_dimension: u32,
@@ -91,10 +117,10 @@ impl<B: Backend> dyn Linear<B> {
},
AnyWeightMatrixSpec::HybridSpec(HybridSpec {
adapter_spec: None,
- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
+ incoherence_block_size: Some(block_size),
incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
..
- }) => Ok(Box::new(RHTLinearWrapper::new(
+ }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => Ok(Box::new(RHTLinearWrapper::new(
context,
input_dimension,
output_dimension_sum,
@@ -154,121 +180,81 @@ impl<B: Backend> dyn Linear<B> {
)
}
- pub fn new_with_output_hadamard_mixed_precision(
- context: &B::Context,
- parameter_tree: &ParameterTree<B>,
- output_factors: Allocation<B>,
- input_dim: u32,
- output_dim: u32,
+ pub fn new_with_input_rht_mixed_precision<const N: usize>(
+ input_dimension: u32,
+ output_dimensions: [u32; N],
has_biases: bool,
+ context: &B::Context,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
- ) -> Result<Box<dyn Linear<B>>, OutputHadamardLinearError<B>> {
- let weights_tree = parameter_tree.subtree("weights").subtree("quantized");
- let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
- match spec {
- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {
- Ok(Box::new(LinearMatmul::load(
- context,
- spec,
- input_dim,
- output_dim,
- weights_data_type,
- input_data_type,
- output_data_type,
- &weights_tree,
- has_biases.then_some(parameter_tree),
- Some(output_factors),
- )?))
- },
- spec => Err(OutputHadamardLinearError::UnsupportedConfiguration(format!(
- "{spec:?} doesn't support fused output hadamard"
- ))),
+ parameter_tree: &ParameterTree<B>,
+ ) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {
+ let output_dimension_sum: u32 = output_dimensions.iter().sum();
+ if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(
+ context,
+ input_dimension,
+ output_dimension_sum,
+ has_biases,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
+ false,
+ parameter_tree,
+ )? {
+ return Ok((linear.0, linear.1.map(|preparation| preparation.input_factors)));
}
+
+ let linear = Self::new_mixed_precision(
+ input_dimension,
+ output_dimensions,
+ has_biases,
+ context,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
+ parameter_tree,
+ )?;
+ Ok((linear, None))
}
- pub fn new_extracting_input_hadamard_mixed_precision<const N: usize>(
+ pub fn new_for_fused_input<const N: usize>(
input_dimension: u32,
output_dimensions: [u32; N],
has_biases: bool,
context: &B::Context,
- weights_data_type: DataType,
- input_data_type: DataType,
- output_data_type: DataType,
+ data_type: DataType,
parameter_tree: &ParameterTree<B>,
- ) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {
+ ) -> Result<(Box<dyn Linear<B>>, Option<LinearInputPreparation<B>>), LinearBlockError<B>> {
let output_dimension_sum: u32 = output_dimensions.iter().sum();
- let weights_tree = parameter_tree.subtree("weights");
- let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
- if let AnyWeightMatrixSpec::HybridSpec(HybridSpec {
- adapter_spec: None,
- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
- incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
- ..
- }) = &spec
- {
- let quantization_spec = weights_tree.subtree("quantized").metadata::<AnyWeightMatrixSpec>("spec")?;
- let parsed = parse_spec::<B>(&quantization_spec).ok();
- if parsed.as_ref().is_some_and(|parsed| {
- int8_activations_eligible::<B>(
- context,
- parsed,
- input_dimension,
- input_data_type,
- output_data_type,
- ACTIVATION_SCALE_GROUP_SIZE,
- )
- }) {
- let linear = RHTLinearWrapper::new(
- context,
- input_dimension,
- output_dimension_sum,
- has_biases,
- weights_data_type,
- input_data_type,
- output_data_type,
- parameter_tree,
- )?;
- return Ok((Box::new(linear), None));
- }
-
- let input_factors = weights_tree
- .leaf("incoherence_signs.input_signs")?
- .validate(&[input_dimension], DataType::I32)?
- .read_allocation()?;
- let output_factors = weights_tree
- .leaf("incoherence_signs.output_signs")?
- .validate(&[output_dimension_sum], DataType::I32)?
- .read_allocation()?;
- let inner_linear = Self::new_with_output_hadamard_mixed_precision(
- context,
- parameter_tree,
- output_factors,
- input_dimension,
- output_dimension_sum,
- has_biases,
- weights_data_type,
- input_data_type,
- output_data_type,
- )?;
- Ok((inner_linear, Some(input_factors)))
- } else {
- let linear = Self::new_mixed_precision(
- input_dimension,
- output_dimensions,
- has_biases,
- context,
- weights_data_type,
- input_data_type,
- output_data_type,
- parameter_tree,
- )?;
- Ok((linear, None))
+ if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(
+ context,
+ input_dimension,
+ output_dimension_sum,
+ has_biases,
+ data_type,
+ data_type,
+ data_type,
+ true,
+ parameter_tree,
+ )? {
+ return Ok(linear);
}
+
+ let linear = Self::new_mixed_precision(
+ input_dimension,
+ output_dimensions,
+ has_biases,
+ context,
+ data_type,
+ data_type,
+ data_type,
+ parameter_tree,
+ )?;
+ Ok((linear, None))
}
- pub fn new_extracting_input_hadamard<const N: usize>(
+ pub fn new_with_input_rht<const N: usize>(
input_dimension: u32,
output_dimensions: [u32; N],
has_biases: bool,
@@ -276,7 +262,7 @@ impl<B: Backend> dyn Linear<B> {
data_type: DataType,
parameter_tree: &ParameterTree<B>,
) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {
- Self::new_extracting_input_hadamard_mixed_precision(
+ Self::new_with_input_rht_mixed_precision(
input_dimension,
output_dimensions,
has_biases,
diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
index f5866e5b2..03003245b 100644
--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
@@ -64,7 +64,11 @@ impl<B: Backend> QLoRALinearWrapper<B> {
) -> Result<Self, QLoRALinearWrapperError<B>> {
let use_incoherence_signs = match (incoherence_block_size, incoherence_processing_mode) {
(None, _) => false,
- (Some(HADAMARD_TRANSFORM_BLOCK_SIZE), IncoherenceProcessingMode::InputOutput) => true,
+ (Some(block_size), IncoherenceProcessingMode::InputOutput)
+ if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE =>
+ {
+ true
+ },
(incoherence_block_size, incoherence_processing_mode) => {
return Err(QLoRALinearWrapperError::UnsupportedConfiguration(format!(
"incoherence block_size={incoherence_block_size:?}, processing_mode={incoherence_processing_mode:?}"
diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
index 08488ef70..dcef5873a 100644
--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
@@ -1,64 +1,28 @@
use thiserror::Error;
use crate::{
+ array::size_for_shape,
backends::common::{
- Allocation, Backend, Context, DeviceCapabilities, Encoder,
- gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod},
+ Allocation, Backend, Encoder,
+ gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
kernel::{
ActivationTransform,
- activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
- matmul::{MatmulA, MatmulPath},
+ matmul::{A8ActivationPlan, ActivationFormat, MatmulA},
},
},
config::weight_matrix::{
- AnyWeightMatrixSpec, Layout,
+ AnyWeightMatrixSpec,
hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
},
data_type::DataType,
- encodable_block::{
- linear::{Linear, LinearMatmul, LinearMatmulError},
- weight_matrix::{ParsedWeightSpec, WeightMatrixError, parse_spec},
- },
+ encodable_block::linear::{Linear, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError},
parameters::{ParameterLoaderError, ParameterTree},
};
-// Temporary, for testing only: flip to false to run linear layers on the bf16
-// activation path without touching model configs.
-const INT8_ACTIVATIONS_ENABLED: bool = true;
-
-pub(super) fn int8_activations_eligible<B: Backend>(
- context: &B::Context,
- spec: &ParsedWeightSpec,
- input_dimension: u32,
- input_data_type: DataType,
- output_data_type: DataType,
- activation_group_size: u32,
-) -> bool {
- if !INT8_ACTIVATIONS_ENABLED {
- return false;
- }
- if !context.device_capabilities().contains(DeviceCapabilities::NATIVE_INT8_MATMUL) {
- return false;
- }
- if input_data_type != DataType::BF16 || output_data_type != DataType::BF16 {
- return false;
- }
- let (Some(info), Layout::OutputInput) = (spec.quantization, &spec.layout) else {
- return false;
- };
- let group_size = info.group_size;
- matches!(group_size, 32 | 64 | 128)
- && input_dimension.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE)
- && input_dimension.is_multiple_of(group_size)
- && input_dimension.is_multiple_of(activation_group_size)
-}
-
#[derive(Debug, Error)]
pub enum RHTLinearWrapperError<B: Backend> {
#[error("Inner linear error: {0}")]
InnerLinearError(#[from] LinearMatmulError<B>),
- #[error("Weight matrix error: {0}")]
- WeightMatrix(#[from] WeightMatrixError<B>),
#[error("Parameter loading error: {0}")]
ParameterError(#[from] ParameterLoaderError<B>),
#[error("Backend error: {0}")]
@@ -67,16 +31,35 @@ pub enum RHTLinearWrapperError<B: Backend> {
UnsupportedConfiguration(String),
}
+enum InputRht<B: Backend> {
+ FullPrecision(ActivationTransform<B>),
+ A8 {
+ fallback: ActivationTransform<B>,
+ a8: ActivationTransform<B>,
+ },
+}
+
pub struct RHTLinearWrapper<B: Backend> {
- input_transform: ActivationTransform<B>,
- quantize_transform: Option<ActivationTransform<B>>,
+ input_rht: InputRht<B>,
input_factors: Allocation<B>,
inner_linear: LinearMatmul<B>,
input_dimension: u32,
}
+fn has_input_output_rht(spec: &AnyWeightMatrixSpec) -> bool {
+ matches!(
+ spec,
+ AnyWeightMatrixSpec::HybridSpec(HybridSpec {
+ adapter_spec: None,
+ incoherence_block_size: Some(block_size),
+ incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
+ ..
+ }) if *block_size == HADAMARD_TRANSFORM_BLOCK_SIZE
+ )
+}
+
impl<B: Backend> RHTLinearWrapper<B> {
- pub fn new(
+ pub(super) fn new(
context: &B::Context,
input_dimension: u32,
output_dimension: u32,
@@ -88,16 +71,88 @@ impl<B: Backend> RHTLinearWrapper<B> {
) -> Result<Self, RHTLinearWrapperError<B>> {
let weights_tree = parameter_tree.subtree("weights");
let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
- let AnyWeightMatrixSpec::HybridSpec(HybridSpec {
- adapter_spec: None,
- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
- incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
- ..
- }) = &spec
- else {
+ if !has_input_output_rht(&spec) {
return Err(RHTLinearWrapperError::UnsupportedConfiguration(format!("{spec:?}")));
- };
+ }
+
+ let (input_factors, mut inner_linear) = Self::load_inner_with_output_rht(
+ context,
+ input_dimension,
+ output_dimension,
+ has_biases,
+ input_data_type,
+ output_data_type,
+ weights_data_type,
+ parameter_tree,
+ )?;
+ let a8_plan = inner_linear.prepare_a8(context);
+ Self::build_self_contained(context, input_dimension, input_data_type, input_factors, inner_linear, a8_plan)
+ }
+
+ pub(super) fn try_new_with_input_preparation(
+ context: &B::Context,
+ input_dimension: u32,
+ output_dimension: u32,
+ has_biases: bool,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
+ allow_prequantized_activation: bool,
+ parameter_tree: &ParameterTree<B>,
+ ) -> Result<Option<(Box<dyn Linear<B>>, Option<LinearInputPreparation<B>>)>, RHTLinearWrapperError<B>> {
+ let weights_tree = parameter_tree.subtree("weights");
+ let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
+ if !has_input_output_rht(&spec) {
+ return Ok(None);
+ }
+
+ let (input_factors, mut inner_linear) = Self::load_inner_with_output_rht(
+ context,
+ input_dimension,
+ output_dimension,
+ has_biases,
+ input_data_type,
+ output_data_type,
+ weights_data_type,
+ parameter_tree,
+ )?;
+ let a8_plan = inner_linear.prepare_a8(context);
+ if let Some(a8_plan) = a8_plan
+ && !allow_prequantized_activation
+ {
+ let wrapper = Self::build_self_contained(
+ context,
+ input_dimension,
+ input_data_type,
+ input_factors,
+ inner_linear,
+ Some(a8_plan),
+ )?;
+ Ok(Some((Box::new(wrapper), None)))
+ } else {
+ Ok(Some((
+ Box::new(inner_linear),
+ Some(LinearInputPreparation {
+ input_factors,
+ a8_plan,
+ }),
+ )))
+ }
+ }
+ fn load_inner_with_output_rht(
+ context: &B::Context,
+ input_dimension: u32,
+ output_dimension: u32,
+ has_biases: bool,
+ input_data_type: DataType,
+ output_data_type: DataType,
+ weights_data_type: DataType,
+ parameter_tree: &ParameterTree<B>,
+ ) -> Result<(Allocation<B>, LinearMatmul<B>), RHTLinearWrapperError<B>> {
+ let weights_tree = parameter_tree.subtree("weights");
+ let quantized_weights_tree = weights_tree.subtree("quantized");
+ let quantization_spec = quantized_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
let input_factors = weights_tree
.leaf("incoherence_signs.input_signs")?
.validate(&[input_dimension], DataType::I32)?
@@ -106,18 +161,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
.leaf("incoherence_signs.output_signs")?
.validate(&[output_dimension], DataType::I32)?
.read_allocation()?;
- let quantized_weights_tree = weights_tree.subtree("quantized");
- let quantization_spec = quantized_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
- let parsed = parse_spec::<B>(&quantization_spec)?;
- let Some(quantization) = parsed.quantization else {
- return Err(RHTLinearWrapperError::UnsupportedConfiguration("RHT requires a quantized inner spec".into()));
- };
- let activation_group_size = ACTIVATION_SCALE_GROUP_SIZE;
-
- let input_transform = ActivationTransform::input_rht(context, input_data_type, true)
- .map_err(RHTLinearWrapperError::BackendError)?;
-
- let mut inner_linear = LinearMatmul::load(
+ let inner_linear = LinearMatmul::load(
context,
quantization_spec,
input_dimension,
@@ -129,32 +173,38 @@ impl<B: Backend> RHTLinearWrapper<B> {
has_biases.then_some(parameter_tree),
Some(output_factors),
)?;
+ Ok((input_factors, inner_linear))
+ }
- let quantize_transform = if int8_activations_eligible::<B>(
- context,
- &parsed,
- input_dimension,
- input_data_type,
- output_data_type,
- activation_group_size,
- ) {
- let emit_group_sums = !matches!(quantization.method, QuantizationMethod::ScaleSymmetric);
- let transform = ActivationTransform::quantize(
- context,
- input_data_type,
- activation_group_size,
- emit_group_sums.then_some(quantization.group_size.min(activation_group_size)),
- )
+ fn build_self_contained(
+ context: &B::Context,
+ input_dimension: u32,
+ input_data_type: DataType,
+ input_factors: Allocation<B>,
+ inner_linear: LinearMatmul<B>,
+ a8_plan: Option<A8ActivationPlan>,
+ ) -> Result<Self, RHTLinearWrapperError<B>> {
+ let input_transform = ActivationTransform::input_rht(context, input_data_type, true)
.map_err(RHTLinearWrapperError::BackendError)?;
- inner_linear.make_codes_signed();
- Some(transform)
- } else {
- None
+ let input_rht = match a8_plan {
+ Some(plan) => {
+ let quantized = ActivationTransform::quantize(
+ context,
+ input_data_type,
+ plan.activation_group_size,
+ plan.sum_group_size,
+ )
+ .map_err(RHTLinearWrapperError::BackendError)?;
+ InputRht::A8 {
+ fallback: input_transform,
+ a8: quantized,
+ }
+ },
+ None => InputRht::FullPrecision(input_transform),
};
Ok(Self {
- input_transform,
- quantize_transform,
+ input_rht,
input_factors,
inner_linear,
input_dimension,
@@ -168,22 +218,46 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
input: Allocation<B>,
batch_dim: u32,
encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ self.encode_input(LinearInput::FullPrecision(input), batch_dim, encoder)
+ }
+
+ fn encode_input(
+ &self,
+ input: LinearInput<B>,
+ batch_dim: u32,
+ encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group("linear (rht)");
- if let Some(quantize_transform) = &self.quantize_transform
- && self.inner_linear.select_path(batch_dim, encoder.context()) == MatmulPath::Gemm
+ let input = match input {
+ LinearInput::FullPrecision(input) => input,
+ input => {
+ let output = self.inner_linear.encode_input(input, batch_dim, encoder);
+ encoder.pop_debug_group();
+ return output;
+ },
+ };
+
+ if let InputRht::A8 {
+ a8: quantized,
+ ..
+ } = &self.input_rht
+ && self.inner_linear.select_activation_format(batch_dim, encoder.context()) == ActivationFormat::Int8
{
- let scale_groups_per_row = self.input_dimension.div_ceil(quantize_transform.activation_group_size());
- let sum_groups_per_row =
- quantize_transform.sum_group_size().map(|group_size| self.input_dimension.div_ceil(group_size));
- let mut values = encoder.allocate_scratch_for_shape(&[batch_dim, self.input_dimension], DataType::I8)?;
- let mut scales = encoder.allocate_scratch_for_shape(&[batch_dim, scale_groups_per_row], DataType::F32)?;
- let mut group_sums = sum_groups_per_row
- .map(|groups| encoder.allocate_scratch_for_shape(&[batch_dim, groups], DataType::I32))
+ let activation_group_size = quantized.activation_group_size();
+ let scale_groups_per_row = self.input_dimension.div_ceil(activation_group_size);
+ let mut values =
+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dimension], DataType::I8))?;
+ let mut scales =
+ encoder.allocate_scratch(size_for_shape(&[batch_dim, scale_groups_per_row], DataType::F32))?;
+ let mut group_sums = quantized
+ .sum_group_size()
+ .map(|group_size| self.input_dimension.div_ceil(group_size))
+ .map(|groups| encoder.allocate_scratch(size_for_shape(&[batch_dim, groups], DataType::I32)))
.transpose()?;
- quantize_transform.encode_quantize(
+ quantized.encode_quantize(
&input,
&mut values,
&mut scales,
@@ -198,29 +272,28 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
values: &values,
scales: &scales,
group_sums: group_sums.as_ref(),
- group_size: quantize_transform.activation_group_size(),
+ group_size: activation_group_size,
},
batch_dim,
encoder,
)?;
encoder.pop_debug_group();
-
return Ok(output);
}
let mut input = input;
- self.input_transform.encode_fp_in_place(
- &mut input,
- &self.input_factors,
- batch_dim,
- self.input_dimension,
- encoder,
- );
+ let full_precision = match &self.input_rht {
+ InputRht::FullPrecision(transform)
+ | InputRht::A8 {
+ fallback: transform,
+ ..
+ } => transform,
+ };
+ full_precision.encode_fp_in_place(&mut input, &self.input_factors, batch_dim, self.input_dimension, encoder);
let output = self.inner_linear.encode(input, batch_dim, encoder)?;
encoder.pop_debug_group();
-
Ok(output)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs
index a95e84210..4d8c641fe 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs
@@ -96,7 +96,7 @@ impl<B: Backend> Attention<B> {
q_dim
};
let (qkv_projection, in_projection_input_hadamard_factors) = if !has_gate {
- <dyn Linear<B>>::new_extracting_input_hadamard(
+ <dyn Linear<B>>::new_with_input_rht(
hidden_dim,
[qkv_projection_output_dimension],
config.has_qkv_biases,
diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
index 8e8cfa965..7f4629c0e 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
@@ -191,7 +191,7 @@ impl<B: Backend> DeltaNet<B> {
let total_proj_dim = conv_dim + value_dim + config.num_heads + config.num_heads;
let (in_projection, in_projection_input_hadamard_factors) =
- <dyn Linear<B>>::new_extracting_input_hadamard_mixed_precision(
+ <dyn Linear<B>>::new_with_input_rht_mixed_precision(
hidden_dim,
[total_proj_dim],
false,
diff --git a/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs b/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs
index 01e7e0bf0..1ed109da6 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs
@@ -113,7 +113,7 @@ impl<B: Backend> Mamba2<B> {
}
let (in_projection, in_projection_input_hadamard_factors) =
- <dyn Linear<B>>::new_extracting_input_hadamard_mixed_precision(
+ <dyn Linear<B>>::new_with_input_rht_mixed_precision(
hidden_dim,
[conv_dim, inner_dim, num_heads],
config.has_in_biases,
diff --git a/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs b/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs
index bd544b76a..4e7fd4b9f 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs
@@ -117,7 +117,7 @@ impl<B: Backend> ShortConv<B> {
)));
}
- let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
+ let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(
hidden_dim,
[hidden_dim * 3],
false,
diff --git a/crates/backend-uzu/src/encodable_block/mlp/dense.rs b/crates/backend-uzu/src/encodable_block/mlp/dense.rs
index a646339f3..4ae3e82cf 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/dense.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/dense.rs
@@ -38,8 +38,9 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {
encoder.push_debug_group("mlp (dense)");
let fused_up = self.up.encode(input, batch_dim, encoder)?;
- let hidden = self.gate.encode(encoder, &fused_up, batch_dim)?;
- let output = self.down.encode(hidden, batch_dim, encoder)?;
+ let act_format = self.down.select_activation_format(batch_dim, encoder.context());
+ let down_input = self.gate.encode_for_linear(encoder, &fused_up, batch_dim, act_format)?;
+ let output = self.down.encode_input(down_input, batch_dim, encoder)?;
encoder.pop_debug_group();
diff --git a/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs b/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs
index 73799c439..bd629fe69 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs
@@ -1,15 +1,26 @@
use crate::{
- backends::common::{Allocation, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::GatedActMulKernel},
+ array::size_for_shape,
+ backends::common::{
+ Allocation, Backend, Encoder,
+ gpu_types::ActivationType,
+ kernel::{
+ GatedActMul,
+ matmul::{A8ActivationPlan, ActivationFormat},
+ },
+ },
config::activation::AnyActivation,
data_type::DataType,
+ encodable_block::linear::{LinearInput, LinearInputPreparation},
};
pub struct MlpGateActMulEncodable<B: Backend> {
- kernel: <B::Kernels as Kernels>::GatedActMulKernel,
+ fp_kernel: GatedActMul<B>,
activation: AnyActivation,
hidden_dim: u32,
data_type: DataType,
hadamard_factors: Option<Allocation<B>>,
+ a8_plan: Option<A8ActivationPlan>,
+ quantized_kernel: Option<GatedActMul<B>>,
}
impl<B: Backend> MlpGateActMulEncodable<B> {
@@ -18,46 +29,91 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
data_type: DataType,
activation: AnyActivation,
hidden_dim: u32,
- hadamard_factors: Option<Allocation<B>>,
+ input_preparation: Option<LinearInputPreparation<B>>,
) -> Result<Self, B::Error> {
- let kernel =
- <B::Kernels as Kernels>::GatedActMulKernel::new(context, data_type, true, hadamard_factors.is_some())?;
+ let (hadamard_factors, a8_plan) = input_preparation
+ .map_or((None, None), |preparation| (Some(preparation.input_factors), preparation.a8_plan));
+ let fp_kernel = GatedActMul::full_precision(context, data_type, true, hadamard_factors.is_some())?;
+ let quantized_kernel = a8_plan
+ .map(|plan| GatedActMul::quantized(context, data_type, plan.activation_group_size, plan.sum_group_size))
+ .transpose()?;
Ok(Self {
- kernel,
+ fp_kernel,
activation,
hidden_dim,
data_type,
hadamard_factors,
+ a8_plan,
+ quantized_kernel,
})
}
- pub fn encode(
+ pub fn encode_for_linear(
&self,
encoder: &mut Encoder<B>,
fused_up: &Allocation<B>,
batch_dim: u32,
- ) -> Result<Allocation<B>, B::Error> {
+ act_format: ActivationFormat,
+ ) -> Result<LinearInput<B>, B::Error> {
encoder.push_debug_group("gate act mul");
if self.activation.act_type() == ActivationType::IDENTITY {
panic!("Identity activation is not supported for kernel")
}
- let mut hidden = encoder.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;
- self.kernel.encode(
- fused_up,
- None::<&Allocation<B>>,
- &mut hidden,
- self.hadamard_factors.as_ref(),
- self.hidden_dim,
- batch_dim,
- 0,
- 0,
- self.activation.act_type(),
- encoder,
- );
+ let input = if act_format == ActivationFormat::Int8
+ && let Some(plan) = self.a8_plan
+ {
+ let kernel = self.quantized_kernel.as_ref().expect("INT8 input requires a quantized gate kernel");
+ let mut values = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], DataType::I8))?;
+ let mut scales = encoder.allocate_scratch(size_for_shape(
+ &[batch_dim, self.hidden_dim.div_ceil(plan.activation_group_size)],
+ DataType::F32,
+ ))?;
+ let mut group_sums = plan
+ .sum_group_size
+ .map(|group_size| {
+ encoder.allocate_scratch(size_for_shape(
+ &[batch_dim, self.hidden_dim.div_ceil(group_size)],
+ DataType::I32,
+ ))
+ })
+ .transpose()?;
+ kernel.encode_quantized(
+ fused_up,
+ &mut values,
+ &mut scales,
+ group_sums.as_mut(),
+ self.hadamard_factors.as_ref().expect("INT8 input requires RHT factors"),
+ self.hidden_dim,
+ batch_dim,
+ self.activation.act_type(),
+ encoder,
+ );
+ LinearInput::Int8Symmetric {
+ values,
+ scales,
+ group_sums,
+ group_size: plan.activation_group_size,
+ }
+ } else {
+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;
+ self.fp_kernel.encode_fp(
+ fused_up,
+ None,
+ &mut hidden,
+ self.hadamard_factors.as_ref(),
+ self.hidden_dim,
+ batch_dim,
+ 0,
+ 0,
+ self.activation.act_type(),
+ encoder,
+ );
+ LinearInput::FullPrecision(hidden)
+ };
encoder.pop_debug_group();
- Ok(hidden)
+ Ok(input)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/mlp/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/mod.rs
index 57230dd8c..606ad0220 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/mod.rs
@@ -45,7 +45,7 @@ impl<B: Backend> dyn Mlp<B> {
) -> Result<(Box<dyn Mlp<B>>, Option<Allocation<B>>), MlpBlockError<B>> {
match config {
AnyMLPConfig::DenseMLPConfig(dense_config) => {
- let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
+ let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(
model_dimension,
[2 * hidden_dimension],
dense_config.has_up_biases,
@@ -54,7 +54,7 @@ impl<B: Backend> dyn Mlp<B> {
&parameter_tree.subtree("up_projection"),
)?;
- let (down_projection, down_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
+ let (down_projection, down_input_preparation) = <dyn Linear<B>>::new_for_fused_input(
hidden_dimension,
[model_dimension],
dense_config.has_down_biases,
@@ -68,7 +68,7 @@ impl<B: Backend> dyn Mlp<B> {
data_type,
dense_config.activation.clone(),
hidden_dimension,
- down_input_hadamard_factors,
+ down_input_preparation,
)
.map_err(MlpBlockError::BackendError)?;
diff --git a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
index 8840eabbc..919b54abf 100644
--- a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
@@ -1,9 +1,10 @@
use thiserror::Error;
use crate::{
+ array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder,
- kernel::{GatedActMulKernel, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},
+ kernel::{GatedActMul, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},
},
config::{
activation::AnyActivation,
@@ -115,18 +116,18 @@ impl<B: Backend> PerLayerEmbedding<B> {
let total_rows = batch_dim * self.num_layers;
let total_elements = batch_dim * total_ple_dim;
- let mut token_ple = encoder.allocate_scratch_for_shape(&[batch_dim, total_ple_dim], self.data_type)?;
+ let mut token_ple = encoder.allocate_scratch(size_for_shape(&[batch_dim, total_ple_dim], self.data_type))?;
self.token_embedding.encode_lookup(token_ids, &mut token_ple, batch_dim, self.fused_token_scale, encoder);
let mut model_projection_input =
- encoder.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;
+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.model_dim], self.data_type))?;
encoder.encode_copy(inner_features, .., &mut model_projection_input, ..);
let model_projected = self.model_projection.encode(model_projection_input, batch_dim, encoder)?;
let model_normed = self.projection_norm.encode(&model_projected, 0, total_rows, None, encoder)?;
let mut per_layer_inputs =
- encoder.allocate_scratch_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type)?;
+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type))?;
self.add_scale.encode(
Some(&token_ple),
&model_normed,
@@ -147,7 +148,7 @@ pub struct PerLayerEmbeddingProjection<B: Backend> {
gate: Box<dyn Linear<B>>,
projection: Box<dyn Linear<B>>,
norm: Normalization<B>,
- gate_act_mul: <B::Kernels as Kernels>::GatedActMulKernel,
+ gate_act_mul: GatedActMul<B>,
residual_finalize: <B::Kernels as Kernels>::TensorAddBiasKernel,
residual_combine: <B::Kernels as Kernels>::TensorAddScaleKernel,
model_dim: u32,
@@ -195,7 +196,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
context,
)?;
- let gate_act_mul = <B::Kernels as Kernels>::GatedActMulKernel::new(context, data_type, false, false)
+ let gate_act_mul = GatedActMul::full_precision(context, data_type, false, false)
.map_err(PerLayerEmbeddingError::BackendError)?;
let residual_finalize = <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, data_type, true)
.map_err(PerLayerEmbeddingError::BackendError)?;
@@ -233,16 +234,16 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
self.residual_finalize.encode(None::<&Allocation<B>>, hidden, &mut *outputs, length, length, encoder);
- let mut gate_input = encoder.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;
+ let mut gate_input = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.model_dim], self.data_type))?;
encoder.encode_copy(outputs, .., &mut gate_input, ..);
let gate_out = self.gate.encode(gate_input, batch_dim, encoder)?;
- let mut activated = encoder.allocate_scratch_for_shape(&[batch_dim, self.ple_dim], self.data_type)?;
- self.gate_act_mul.encode(
+ let mut activated = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.ple_dim], self.data_type))?;
+ self.gate_act_mul.encode_fp(
&gate_out,
Some(per_layer_input),
&mut activated,
- None::<&Allocation<B>>,
+ None,
self.ple_dim,
batch_dim,
layer_index * self.ple_dim,
diff --git a/crates/backend-uzu/src/tests/matmul/quant.rs b/crates/backend-uzu/src/tests/matmul/quant.rs
index 22b46e5c0..4ff4f5ffe 100644
--- a/crates/backend-uzu/src/tests/matmul/quant.rs
+++ b/crates/backend-uzu/src/tests/matmul/quant.rs
@@ -64,15 +64,13 @@ impl<T: ArrayElement + Float> QuantInput<T> {
quant_method: QuantizationMethod,
seed: u64,
) -> Self {
- let num_groups_k = k.div_ceil(group_size) as usize;
- let (rows, inner, columns) = (m as usize, k as usize, n as usize);
+ let num_groups_k = k.div_ceil(group_size);
let mut rng = SmallRng::seed_from_u64(seed);
- let w_packed: Vec<u32> =
- (0..columns * inner * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect();
+ let w_packed: Vec<u32> = (0..(n * k * bits / 32) as usize).map(|_| rng.random_range(0..u32::MAX)).collect();
let scales: Vec<T> =
- (0..columns * num_groups_k).map(|_| T::from(rng.random_range(0.01f32..0.3f32)).unwrap()).collect();
- let x: Vec<T> = (0..rows * inner).map(|_| T::from(rng.random_range(-0.3f32..0.3f32)).unwrap()).collect();
+ (0..(n * num_groups_k) as usize).map(|_| T::from(rng.random_range(0.01f32..0.3f32)).unwrap()).collect();
+ let x: Vec<T> = (0..(m * k) as usize).map(|_| T::from(rng.random_range(-0.3f32..0.3f32)).unwrap()).collect();
let zp_stride = if bits == 4 {
num_groups_k.div_ceil(2)
@@ -83,13 +81,13 @@ impl<T: ArrayElement + Float> QuantInput<T> {
QuantizationMethod::ScaleBias => (
None,
Some(
- (0..columns * num_groups_k)
+ (0..(n * num_groups_k) as usize)
.map(|_| T::from(rng.random_range(-0.03f32..0.03f32)).unwrap())
.collect(),
),
),
QuantizationMethod::ScaleZeroPoint => {
- (Some((0..columns * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)
+ (Some((0..(n * zp_stride) as usize).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)
},
QuantizationMethod::ScaleSymmetric => (None, None),
};
@@ -131,11 +129,11 @@ impl<T: ArrayElement + Float> QuantInput<T> {
let context = <Cpu as Backend>::Context::new().expect("CPU context");
let input = alloc_allocation_with_data::<Cpu, T>(&context, &self.x);
let factors = alloc_allocation_with_data::<Cpu, i32>(&context, &vec![1; columns as usize]);
- let elements = rows as usize * columns as usize;
- let mut values = alloc_allocation::<Cpu, i8>(&context, elements);
- let mut scales = alloc_allocation::<Cpu, f32>(&context, elements / activation_group_size as usize);
- let mut group_sums =
- sum_group_size.map(|group_size| alloc_allocation::<Cpu, i32>(&context, elements / group_size as usize));
+ let element_count = rows * columns;
+ let mut values = alloc_allocation::<Cpu, i8>(&context, element_count as usize);
+ let mut scales = alloc_allocation::<Cpu, f32>(&context, (element_count / activation_group_size) as usize);
+ let mut group_sums = sum_group_size
+ .map(|group_size| alloc_allocation::<Cpu, i32>(&context, (element_count / group_size) as usize));
let transform =
ActivationTransform::<Cpu>::quantize(&context, T::data_type(), activation_group_size, sum_group_size)
.expect("CPU activation quantization transform");
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs
index f1548ae25..42e4aec4f 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs
@@ -120,20 +120,21 @@ mod quantize {
fn run<B: Backend>(
input_data: &[f32],
factors_data: &[i32],
- rows: usize,
- columns: usize,
+ rows: u32,
+ columns: u32,
activation_group_size: u32,
emit_group_sums: bool,
sum_group_size: Option<u32>,
) -> (Vec<i8>, Vec<f32>, Option<Vec<i32>>) {
- let scale_groups = columns / activation_group_size as usize;
- let sum_groups = sum_group_size.map_or(0, |group_size| columns / group_size as usize);
+ let scale_groups = columns / activation_group_size;
+ let sum_groups = sum_group_size.map_or(0, |group_size| columns / group_size);
let context = B::Context::new().expect("context");
let input = alloc_allocation_with_data::<B, f32>(context.as_ref(), input_data);
let factors = alloc_allocation_with_data::<B, i32>(context.as_ref(), factors_data);
- let mut values = alloc_allocation::<B, i8>(context.as_ref(), rows * columns);
- let mut scales = alloc_allocation::<B, f32>(context.as_ref(), rows * scale_groups);
- let mut group_sums = emit_group_sums.then(|| alloc_allocation::<B, i32>(context.as_ref(), rows * sum_groups));
+ let mut values = alloc_allocation::<B, i8>(context.as_ref(), (rows * columns) as usize);
+ let mut scales = alloc_allocation::<B, f32>(context.as_ref(), (rows * scale_groups) as usize);
+ let mut group_sums =
+ emit_group_sums.then(|| alloc_allocation::<B, i32>(context.as_ref(), (rows * sum_groups) as usize));
let kernel =
ActivationTransform::quantize(context.as_ref(), DataType::F32, activation_group_size, sum_group_size)
.expect("quantize transform");
@@ -144,8 +145,8 @@ mod quantize {
&mut scales,
group_sums.as_mut(),
&factors,
- rows as u32,
- columns as u32,
+ rows,
+ columns,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
@@ -205,10 +206,10 @@ mod quantize {
for (group_index, (&actual, &expected)) in
actual_group_sums.iter().zip(expected_group_sums.as_ref().expect("CPU group sums")).enumerate()
{
- let sum_group_size = sum_group_size.expect("correction group") as usize;
- let start = group_index * sum_group_size;
+ let sum_group_size = sum_group_size.expect("correction group");
+ let start = group_index * sum_group_size as usize;
let sum_from_actual_codes: i32 =
- actual_values[start..start + sum_group_size].iter().copied().map(i32::from).sum();
+ actual_values[start..start + sum_group_size as usize].iter().copied().map(i32::from).sum();
assert_eq!(actual, sum_from_actual_codes, "group sum {group_index}");
assert!(
(actual - expected).abs() <= sum_group_size as i32,
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/gated_act_mul_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/gated_act_mul_test.rs
index 71f7de56b..cfee331bf 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/gated_act_mul_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/gated_act_mul_test.rs
@@ -7,9 +7,7 @@ use proc_macros::uzu_test;
use crate::{
array::ArrayElement,
backends::{
- common::{
- Allocation, Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::GatedActMulKernel,
- },
+ common::{Allocation, Backend, Context, Encoder, gpu_types::ActivationType, kernel::GatedActMul},
cpu::Cpu,
},
data_type::DataType,
@@ -23,6 +21,7 @@ use crate::{
struct InterleavedInput<T: ArrayElement + Float> {
fused_up: Box<[T]>,
+ hadamard_factors: Box<[i32]>,
gated_dim: u32,
batch_dim: u32,
act_type: ActivationType,
@@ -38,28 +37,33 @@ fn interleaved_input<T: ArrayElement + Float>(act_type: ActivationType) -> Inter
}
InterleavedInput {
fused_up: fused_up.into_boxed_slice(),
+ hadamard_factors: vec![1; gated_dim as usize].into_boxed_slice(),
gated_dim,
batch_dim,
act_type,
}
}
-fn run_interleaved<T: ArrayElement + Float, B: Backend>(input: &InterleavedInput<T>) -> Vec<T> {
+fn run_interleaved<T: ArrayElement + Float, B: Backend>(
+ input: &InterleavedInput<T>,
+ use_hadamard: bool,
+) -> Vec<T> {
let context = B::Context::new().expect("create context");
- let kernel = <<B as Backend>::Kernels as Kernels>::GatedActMulKernel::new(&context, T::data_type(), true, false)
- .expect("create GatedActMulKernel");
+ let kernel =
+ GatedActMul::<B>::full_precision(&context, T::data_type(), true, use_hadamard).expect("create GatedActMul");
let fused_length = (input.batch_dim * 2 * input.gated_dim) as usize;
let output_length = (input.batch_dim * input.gated_dim) as usize;
let fused_up = alloc_allocation_with_data::<B, T>(&context, &input.fused_up[..fused_length]);
+ let hadamard_factors = alloc_allocation_with_data::<B, i32>(&context, &input.hadamard_factors);
let mut output = alloc_allocation::<B, T>(&context, output_length);
let mut encoder = Encoder::new(context.as_ref()).expect("create encoder");
- kernel.encode(
+ kernel.encode_fp(
&fused_up,
None::<&Allocation<B>>,
&mut output,
- None::<&Allocation<B>>,
+ use_hadamard.then_some(&hadamard_factors),
input.gated_dim,
input.batch_dim,
0,
@@ -79,14 +83,24 @@ fn interleaved_test<T: ArrayElement + Float + Debug + Display>(act_type: Activat
1e-5
};
let input = interleaved_input::<T>(act_type);
- let expected = run_interleaved::<T, Cpu>(&input);
+ let expected = run_interleaved::<T, Cpu>(&input, false);
for_each_non_cpu_backend!(|B| {
- let output = run_interleaved::<T, B>(&input);
+ let output = run_interleaved::<T, B>(&input, false);
let message = format!("interleaved mismatch for backend {}", std::any::type_name::<B>());
assert_eq_float::<T>(&expected, &output, eps, &message);
});
}
+#[uzu_test]
+fn test_gated_act_mul_interleaved_hadamard_bf16() {
+ let input = interleaved_input::<bf16>(ActivationType::SILU);
+ let expected = run_interleaved::<bf16, Cpu>(&input, true);
+ for_each_non_cpu_backend!(|B| {
+ let actual = run_interleaved::<bf16, B>(&input, true);
+ assert_eq_float::<bf16>(&expected, &actual, 0.02, "Hadamard gated activation mismatch");
+ });
+}
+
#[uzu_test]
fn test_gated_act_mul_interleaved_silu_f32() {
interleaved_test::<f32>(ActivationType::SILU);
@@ -147,19 +161,18 @@ fn separate_input<T: ArrayElement + Float>() -> (SeparateInput<T>, Vec<T>) {
fn run_separate<T: ArrayElement + Float, B: Backend>(input: &SeparateInput<T>) -> Vec<T> {
let context = B::Context::new().expect("create context");
- let kernel = <<B as Backend>::Kernels as Kernels>::GatedActMulKernel::new(&context, T::data_type(), false, false)
- .expect("create GatedActMulKernel");
+ let kernel = GatedActMul::<B>::full_precision(&context, T::data_type(), false, false).expect("create GatedActMul");
let gate_out = alloc_allocation_with_data::<B, T>(&context, &input.gate_out);
let per_layer_input = alloc_allocation_with_data::<B, T>(&context, &input.per_layer_input);
let mut output = alloc_allocation::<B, T>(&context, (input.batch_dim * input.gated_dim) as usize);
let mut encoder = Encoder::new(context.as_ref()).expect("create encoder");
- kernel.encode(
+ kernel.encode_fp(
&gate_out,
Some(&per_layer_input),
&mut output,
- None::<&Allocation<B>>,
+ None,
input.gated_dim,
input.batch_dim,
input.value_offset,