A high-performance inference engine for AI models
RHT Block improvements + ActivationTransform fusion (#704)
CCYehaae4e743ebf8
diff --git a/crates/backend-uzu/src/backends/common/device_capabilities.rs b/crates/backend-uzu/src/backends/common/device_capabilities.rsindex 8125a319a..cc6852156 100644--- a/crates/backend-uzu/src/backends/common/device_capabilities.rs+++ b/crates/backend-uzu/src/backends/common/device_capabilities.rs@@ -5,6 +5,5 @@ bitflags! {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct DeviceCapabilities: u32 {const SPARSE_BUFFERS = 1 << 0;- const NATIVE_INT8_MATMUL = 1 << 1;}}diff --git a/crates/backend-uzu/src/backends/common/gpu_types/gated_act_mul.rs b/crates/backend-uzu/src/backends/common/gpu_types/gated_act_mul.rsnew file mode 100644index 000000000..d4605fbdb--- /dev/null+++ b/crates/backend-uzu/src/backends/common/gpu_types/gated_act_mul.rs@@ -0,0 +1,7 @@+#[repr(C)]+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum GatedActMulOp {+ FullPrecision,+ Quantize,+ QuantizeWithGroupSums,+}diff --git a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs b/crates/backend-uzu/src/backends/common/gpu_types/mod.rsindex 261cf7294..5c1089585 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/mod.rs@@ -6,6 +6,7 @@pub mod activation_transform;pub mod activation_type;pub mod attention;+pub mod gated_act_mul;pub mod gemm;pub mod hadamard_order;pub mod kv_cache_update;@@ -19,6 +20,7 @@ pub mod weaver;pub use activation_transform::*;pub use activation_type::*;pub use attention::*;+pub use gated_act_mul::*;pub use hadamard_order::*;pub use kv_cache_update::*;pub use matmul::*;diff --git a/crates/backend-uzu/src/backends/common/kernel/gated_act_mul.rs b/crates/backend-uzu/src/backends/common/kernel/gated_act_mul.rsnew file mode 100644index 000000000..198261601--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/gated_act_mul.rs@@ -0,0 +1,185 @@+use bitflags::bitflags;++use crate::{+ backends::common::{+ Allocation, Backend, Encoder, Kernels,+ gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE},+ kernel::GatedActMulKernel,+ },+ data_type::DataType,+};++#[repr(u32)]+#[derive(Clone, Copy)]+enum GatedActMulGroupSize {+ Size32 = 32,+ Size64 = 64,+ Size128 = 128,+}++impl GatedActMulGroupSize {+ fn from_u32(value: u32) -> Self {+ match value {+ 32 => Self::Size32,+ 64 => Self::Size64,+ 128 => Self::Size128,+ _ => panic!("unsupported activation group size: {value}"),+ }+ }+}++bitflags! {+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]+ struct GatedActMulOptions: u8 {+ const INTERLEAVED = 1 << 0;+ const HADAMARD = 1 << 1;+ }+}++pub struct GatedActMul<B: Backend> {+ kernel: <B::Kernels as Kernels>::GatedActMulKernel,+ ops: GatedActMulOp,+ options: GatedActMulOptions,+ activation_group_size: u32,+ sum_group_size: u32,+}++impl<B: Backend> GatedActMul<B> {+ pub fn full_precision(+ context: &B::Context,+ data_type: DataType,+ interleaved: bool,+ use_hadamard: bool,+ ) -> Result<Self, B::Error> {+ let mut options = GatedActMulOptions::empty();+ options.set(GatedActMulOptions::INTERLEAVED, interleaved);+ options.set(GatedActMulOptions::HADAMARD, use_hadamard);+ Self::new(+ context,+ data_type,+ GatedActMulOp::FullPrecision,+ options,+ HADAMARD_TRANSFORM_BLOCK_SIZE,+ HADAMARD_TRANSFORM_BLOCK_SIZE,+ )+ }++ pub fn quantized(+ context: &B::Context,+ data_type: DataType,+ activation_group_size: u32,+ sum_group_size: Option<u32>,+ ) -> Result<Self, B::Error> {+ let activation_group_size = GatedActMulGroupSize::from_u32(activation_group_size);+ let sum_group_size = sum_group_size.map(GatedActMulGroupSize::from_u32);+ Self::new(+ context,+ data_type,+ sum_group_size.map_or(GatedActMulOp::Quantize, |_| GatedActMulOp::QuantizeWithGroupSums),+ GatedActMulOptions::INTERLEAVED | GatedActMulOptions::HADAMARD,+ activation_group_size as u32,+ sum_group_size.unwrap_or(activation_group_size) as u32,+ )+ }++ fn new(+ context: &B::Context,+ data_type: DataType,+ ops: GatedActMulOp,+ options: GatedActMulOptions,+ activation_group_size: u32,+ sum_group_size: u32,+ ) -> Result<Self, B::Error> {+ let kernel = <B::Kernels as Kernels>::GatedActMulKernel::new(+ context,+ data_type,+ ops,+ options.contains(GatedActMulOptions::INTERLEAVED),+ options.contains(GatedActMulOptions::HADAMARD),+ activation_group_size,+ sum_group_size,+ )?;+ Ok(Self {+ kernel,+ ops,+ options,+ activation_group_size,+ sum_group_size,+ })+ }++ pub fn encode_fp(+ &self,+ act_operand: &Allocation<B>,+ value_operand: Option<&Allocation<B>>,+ output: &mut Allocation<B>,+ hadamard_factors: Option<&Allocation<B>>,+ gated_dim: u32,+ batch_dim: u32,+ value_offset: u32,+ value_row_stride: u32,+ act_type: ActivationType,+ encoder: &mut Encoder<B>,+ ) {+ assert_eq!(self.ops, GatedActMulOp::FullPrecision);+ assert_eq!(self.options.contains(GatedActMulOptions::INTERLEAVED), value_operand.is_none());+ assert_eq!(self.options.contains(GatedActMulOptions::HADAMARD), hadamard_factors.is_some());+ assert!(+ !self.options.contains(GatedActMulOptions::HADAMARD)+ || gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE)+ );+ self.kernel.encode(+ act_operand,+ value_operand,+ Some(output),+ None::<&mut Allocation<B>>,+ None::<&mut Allocation<B>>,+ None::<&mut Allocation<B>>,+ hadamard_factors,+ gated_dim,+ batch_dim,+ value_offset,+ value_row_stride,+ act_type,+ encoder,+ );+ }++ pub fn encode_quantized(+ &self,+ act_operand: &Allocation<B>,+ values: &mut Allocation<B>,+ scales: &mut Allocation<B>,+ group_sums: Option<&mut Allocation<B>>,+ hadamard_factors: &Allocation<B>,+ gated_dim: u32,+ batch_dim: u32,+ act_type: ActivationType,+ encoder: &mut Encoder<B>,+ ) {+ assert!(matches!(self.ops, GatedActMulOp::Quantize | GatedActMulOp::QuantizeWithGroupSums));+ assert!(self.options.contains(GatedActMulOptions::INTERLEAVED));+ assert!(self.options.contains(GatedActMulOptions::HADAMARD));+ assert!(gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));+ assert!(gated_dim.is_multiple_of(self.activation_group_size));+ assert_eq!(self.ops == GatedActMulOp::QuantizeWithGroupSums, group_sums.is_some());+ if self.ops == GatedActMulOp::QuantizeWithGroupSums {+ assert!(gated_dim.is_multiple_of(self.sum_group_size));+ }+ self.kernel.encode(+ act_operand,+ None::<&Allocation<B>>,+ None::<&mut Allocation<B>>,+ Some(values),+ Some(scales),+ group_sums,+ Some(hadamard_factors),+ gated_dim,+ batch_dim,+ 0,+ 0,+ act_type,+ encoder,+ );+ }+}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rsindex de49d1364..6ff0365f0 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs@@ -3,7 +3,7 @@ use crate::{Backend, BufferArg, Encoder, Kernels,kernel::matmul::{arguments::MatmulArguments,- routing::{MatmulPath, MatmulShape},+ routing::{A8ActivationPlan, ActivationFormat, MatmulShape},},},data_type::DataType,@@ -25,11 +25,19 @@ pub trait MatmulKernel: Sized + Send + Sync {encoder: &mut Encoder<Self::Backend>,) -> Result<(), <Self::Backend as Backend>::Error>;- fn select_path(+ fn a8_activation_plan(&self,- _shape: &MatmulShape,+ _candidate: &MatmulShape,_context: &<Self::Backend as Backend>::Context,- ) -> MatmulPath {- MatmulPath::Gemm+ ) -> Option<A8ActivationPlan> {+ None+ }++ fn select_activation_format(+ &self,+ _bf16_shape: &MatmulShape,+ _context: &<Self::Backend as Backend>::Context,+ ) -> ActivationFormat {+ ActivationFormat::Bf16}}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rsindex c727ec22f..13089c3fd 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs@@ -12,4 +12,4 @@ pub use error::MatmulError;pub use kernel::MatmulKernel;pub use matmul_a::MatmulA;pub use matmul_b::MatmulB;-pub use routing::{MatmulPath, MatmulShape};+pub use routing::{A8ActivationPlan, ActivationFormat, MatmulShape};diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/routing.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/routing.rsindex 705c03573..e0f3e3c1f 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/routing.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/routing.rs@@ -4,6 +4,18 @@ use crate::backends::common::{gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},};+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum ActivationFormat {+ Bf16,+ Int8,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct A8ActivationPlan {+ pub activation_group_size: u32,+ pub sum_group_size: Option<u32>,+}+#[derive(Clone, Copy)]pub struct MatmulShape {pub m: u32,@@ -44,9 +56,3 @@ impl MatmulShape {self.b_prologue != GemmBPrologueKind::FullPrecision}}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum MatmulPath {- Gemv,- Gemm,-}diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rsindex 25adfc458..78c86cca5 100644--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs@@ -4,10 +4,12 @@ pub mod activation_transform;pub mod attention_gemm;pub mod delta_net_chunked_prefill;pub mod delta_net_tree_verify;+pub mod gated_act_mul;pub mod matmul;pub mod radix_top_k_small;pub use activation_transform::ActivationTransform;+pub use gated_act_mul::GatedActMul;include!(concat!(env!("OUT_DIR"), "/traits.rs"));diff --git a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rsindex b33ecaff9..c6e0ce5ae 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs@@ -4,6 +4,8 @@ use crate::backends::common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE;pub const INT8_SYMMETRIC_QUANTIZATION_MAXIMUM: f32 = 127.0;+pub use activation_transform::quantize_transformed_row;+pub fn min_max_symmetric_divisor(values: &[f32]) -> f32 {let (min, max) =values.iter().fold((f32::INFINITY, f32::NEG_INFINITY), |(min, max), &value| (min.min(value), max.max(value)));diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs b/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rsindex 48a0fa013..5f65b8e24 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs@@ -2,45 +2,119 @@ use half::bf16;use num_traits::Float;use proc_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};+use crate::{+ array::ArrayElement,+ backends::{+ common::gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE},+ cpu::kernel::activation_transform::{hadamard_transform, quantize_transformed_row},+ },+};#[kernel(GatedActMul)]#[variants(T, f32, bf16)]pub fn gated_act_mul<T: ArrayElement + Float>(act_operand: *const T,#[optional(!interleaved)] value_operand: Option<*const T>,- output: *mut T,+ #[optional(ops == GatedActMulOp::FullPrecision)] fp_out: Option<*mut T>,+ #[optional(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)] q_out: Option<*mut i8>,+ #[optional(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)] scales_out: Option<+ *mut f32,+ >,+ #[optional(ops == GatedActMulOp::QuantizeWithGroupSums)] group_sums_out: Option<*mut i32>,#[optional(use_hadamard)] hadamard_factors: Option<*const i32>,gated_dim: u32,batch_dim: u32,value_offset: u32,value_row_stride: u32,act_type: ActivationType,+ #[specialize] ops: GatedActMulOp,#[specialize] interleaved: bool,#[specialize] use_hadamard: bool,+ #[specialize] activation_scale_group_size: u32,+ #[specialize] sum_group_size: u32,) {assert_eq!(hadamard_factors.is_some(), use_hadamard);+ let quantize = matches!(ops, GatedActMulOp::Quantize | GatedActMulOp::QuantizeWithGroupSums);+ let emit_group_sums = ops == GatedActMulOp::QuantizeWithGroupSums;+ assert!(!quantize || use_hadamard, "quantized gate activation requires RHT");+ assert!(!quantize || gated_dim.is_multiple_of(activation_scale_group_size));+ assert!(!emit_group_sums || gated_dim.is_multiple_of(sum_group_size));+ assert!(!use_hadamard || gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));- if use_hadamard {- unimplemented!("not supported yet");- }let gated_dim = gated_dim as usize;let batch_dim = batch_dim as usize;- let value_offset = value_offset as usize;- let value_row_stride = value_row_stride as usize;+ let activation_scale_group_size = activation_scale_group_size as usize;+ let sum_group_size = sum_group_size as usize;for batch in 0..batch_dim {+ let mut transformed = use_hadamard.then(|| vec![0.0f32; gated_dim]);for gated in 0..gated_dim {let (act_index, value) = if interleaved {let base = batch * 2 * gated_dim;(base + gated_dim + gated, unsafe { *act_operand.add(base + gated) })} else {- let value_index = batch * value_row_stride + value_offset + gated;+ let value_index = batch * value_row_stride as usize + value_offset as usize + gated;(batch * gated_dim + gated, unsafe { *value_operand.unwrap().add(value_index) })};- let activated: T = act_type.activate(unsafe { *act_operand.add(act_index) });- let result = value.to_f32().unwrap() * activated.to_f32().unwrap();- unsafe { *output.add(batch * gated_dim + gated) = T::from(result).unwrap() };+ let gate = unsafe { *act_operand.add(act_index) };+ let result = super::gated_act_mul(value, gate, act_type);+ if let Some(transformed) = transformed.as_mut() {+ transformed[gated] = result;+ } else {+ unsafe {+ *fp_out.expect("FP gate activation requires fp_out").add(batch * gated_dim + gated) =+ T::from(result).unwrap();+ }+ }+ }++ if let Some(transformed) = transformed.as_mut() {+ let factors = hadamard_factors.expect("Hadamard factors are required");+ for stripe_start in (0..gated_dim).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {+ let mut stripe = [0.0f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize];+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE as usize {+ let gated = stripe_start + lane;+ stripe[lane] = transformed[gated] * unsafe { *factors.add(gated) } as f32;+ }+ hadamard_transform(&mut stripe);+ transformed[stripe_start..stripe_start + HADAMARD_TRANSFORM_BLOCK_SIZE as usize]+ .copy_from_slice(&stripe);+ }++ if quantize {+ let values = unsafe {+ std::slice::from_raw_parts_mut(+ q_out.expect("quantized gate activation requires q_out").add(batch * gated_dim),+ gated_dim,+ )+ };+ let scales_per_row = gated_dim / activation_scale_group_size;+ let scales = unsafe {+ std::slice::from_raw_parts_mut(+ scales_out.expect("quantized gate activation requires scales_out").add(batch * scales_per_row),+ scales_per_row,+ )+ };+ let group_sums = group_sums_out.map(|output| unsafe {+ std::slice::from_raw_parts_mut(+ output.add(batch * gated_dim / sum_group_size),+ gated_dim / sum_group_size,+ )+ });+ quantize_transformed_row(+ transformed,+ activation_scale_group_size,+ emit_group_sums.then_some(sum_group_size),+ values,+ scales,+ group_sums,+ );+ } else {+ let output = fp_out.expect("FP gate activation requires fp_out");+ for (gated, &value) in transformed.iter().enumerate() {+ unsafe { *output.add(batch * gated_dim + gated) = T::from(value).unwrap() };+ }+ }}}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/mod.rsindex 95bdbdf41..ef320ee64 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gated_act_mul/mod.rs@@ -1 +1,14 @@+use num_traits::Float;++use crate::backends::common::gpu_types::activation_type::ActivationType;++#[inline]+pub(super) fn gated_act_mul<T: Float>(+ value: T,+ gate: T,+ act_type: ActivationType,+) -> f32 {+ (value * act_type.activate(gate)).to_f32().unwrap()+}+pub mod gated_act_mul;diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rsindex f31b27958..9c7f350fc 100644--- a/crates/backend-uzu/src/backends/metal/context.rs+++ b/crates/backend-uzu/src/backends/metal/context.rs@@ -269,9 +269,6 @@ impl Context for MetalContext {if self.device.supports_placement_sparse_resources() {capabilities |= DeviceCapabilities::SPARSE_BUFFERS;}- if self.device.supports_mxu() {- capabilities |= DeviceCapabilities::NATIVE_INT8_MATMUL;- }capabilities}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/activation_transform/activation_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/activation_transform/activation_transform.metalindex 51e9e70f9..d5bfe2713 100644--- a/crates/backend-uzu/src/backends/metal/kernel/activation_transform/activation_transform.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/activation_transform/activation_transform.metal@@ -2,71 +2,19 @@#include "../common/defines.h"#include "../common/dsl.h"#include "../common/thread_context.h"+#include "../common/activation_quantization.h"#include "../generated/activation_transform.h"#include "../hadamard_transform/hadamard_transform.h"using namespace metal;using namespace uzu::activation_transform;-UZU_CONST uint ACTIVATION_TILE_SIZE = 128;-UZU_CONST uint SIMDGROUPS_PER_THREADGROUP = ACTIVATION_TILE_SIZE / METAL_SIMD_SIZE;-UZU_CONST float INT8_QMAX = 127.0f;+#define NUM_SIMDGROUPS 4+#define NUM_THREADS NUM_SIMDGROUPS* METAL_SIMD_SIZE#define QUANTIZED (ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)#define EMITS_GROUP_SUMS (ops == ActivationTransformOp::QuantizeWithGroupSums)-template <typename T, typename SimdReduce, typename Combine>-METAL_FUNC T reduce_quantization_group(- const T value,- const uint group_size,- threadgroup T* partials,- const thread ThreadContext& thread_context,- SimdReduce simd_reduce,- Combine combine-) {- const ushort lane_index = thread_context.simd_lane_id;- const ushort simdgroup_index = thread_context.simdgroup_index;- const T simdgroup_value = simd_reduce(value);- if (group_size == METAL_SIMD_SIZE) {- return simdgroup_value;- }- if (lane_index == 0) {- partials[simdgroup_index] = simdgroup_value;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const uint simdgroups_per_group = group_size / METAL_SIMD_SIZE;- const uint simdgroup_base = simdgroup_index / simdgroups_per_group * simdgroups_per_group;- T result = partials[simdgroup_base];- METAL_PRAGMA_UNROLL- for (uint index = 1; index < simdgroups_per_group; ++index) {- result = combine(result, partials[simdgroup_base + index]);- }- return result;-}--template <typename T>-METAL_FUNC void write_quantization_group(- device T* output,- const T value,- const uint group_size,- const uint element_count,- const uint tile_index,- const uint batch_index,- const thread ThreadContext& thread_context-) {- const ushort simdgroup_index = thread_context.simdgroup_index;- const uint simdgroups_per_group = group_size / METAL_SIMD_SIZE;- const uint simdgroup_offset = tile_index * ACTIVATION_TILE_SIZE + simdgroup_index * METAL_SIMD_SIZE;- if (thread_context.simd_lane_id == 0 && simdgroup_offset < element_count &&- (simdgroup_index + 1) % simdgroups_per_group == 0) {- const uint groups_per_row = element_count / group_size;- const uint groups_per_tile = ACTIVATION_TILE_SIZE / group_size;- const uint group_index = tile_index * groups_per_tile + simdgroup_index / simdgroups_per_group;- output[batch_index * groups_per_row + group_index] = value;- }-}-template <typename T>VARIANTS(T, float, bfloat)PUBLIC KERNEL(ActivationTransform)(@@ -82,11 +30,11 @@ PUBLIC KERNEL(ActivationTransform)(const bool in_place SPECIALIZE,const uint activation_scale_group_size SPECIALIZE,const uint sum_group_size SPECIALIZE,- threadgroup float partial_max OPTIONAL(QUANTIZED && activation_scale_group_size > METAL_SIMD_SIZE)[4],- threadgroup int partial_sums OPTIONAL(EMITS_GROUP_SUMS && sum_group_size > METAL_SIMD_SIZE)[4],- uint activation_tile_index GROUPS(element_count.div_ceil(128)),+ threadgroup float partial_max OPTIONAL(QUANTIZED && activation_scale_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],+ threadgroup int partial_sums OPTIONAL(EMITS_GROUP_SUMS && sum_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],+ uint activation_tile_index GROUPS(element_count.div_ceil(NUM_THREADS)),uint batch_index GROUPS(batch_size),- uint thread_index THREADS(128),+ uint thread_index THREADS(NUM_THREADS),const ThreadContext thread_context) {(void)thread_index;@@ -97,7 +45,7 @@ PUBLIC KERNEL(ActivationTransform)(const bool input_rht = ops != ActivationTransformOp::OutputRht;const ushort lane_index = thread_context.simd_lane_id;const ushort simdgroup_index = thread_context.simdgroup_index;- const uint tile_offset = activation_tile_index * ACTIVATION_TILE_SIZE;+ const uint tile_offset = activation_tile_index * ACTIVATION_QUANT_TILE_SIZE;const uint simdgroup_offset = tile_offset + simdgroup_index * METAL_SIMD_SIZE;const bool in_bounds = simdgroup_offset < element_count;const uint factor_index = simdgroup_offset + lane_index;@@ -117,7 +65,7 @@ PUBLIC KERNEL(ActivationTransform)(return;}- const float maximum = reduce_quantization_group(+ const float maximum = reduce_activation_quantization_group(fabs(value),activation_scale_group_size,partial_max,@@ -125,13 +73,14 @@ PUBLIC KERNEL(ActivationTransform)([](float x) { return simd_max(x); },[](float x, float y) { return max(x, y); });- const float scale = isfinite(maximum) && maximum > 0.0f ? maximum / INT8_QMAX : 1.0f;+ const float scale = isfinite(maximum) && maximum > 0.0f ? maximum / ACTIVATION_QUANT_INT8_MAX : 1.0f;- const int8_t code = static_cast<int8_t>(clamp(round(value / scale), -INT8_QMAX, INT8_QMAX));+ const int8_t code =+ static_cast<int8_t>(clamp(round(value / scale), -ACTIVATION_QUANT_INT8_MAX, ACTIVATION_QUANT_INT8_MAX));if (in_bounds) {q_out[element_index] = code;}- write_quantization_group(+ write_activation_quantization_group(scales_out,scale,activation_scale_group_size,@@ -142,7 +91,7 @@ PUBLIC KERNEL(ActivationTransform)();if (EMITS_GROUP_SUMS) {- const int sum = reduce_quantization_group(+ const int sum = reduce_activation_quantization_group(int(code),sum_group_size,partial_sums,@@ -150,7 +99,7 @@ PUBLIC KERNEL(ActivationTransform)([](int x) { return simd_sum(x); },[](int x, int y) { return x + y; });- write_quantization_group(+ write_activation_quantization_group(group_sums_out,sum,sum_group_size,diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/activation_quantization.h b/crates/backend-uzu/src/backends/metal/kernel/common/activation_quantization.hnew file mode 100644index 000000000..a391f1388--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/common/activation_quantization.h@@ -0,0 +1,63 @@+#pragma once++#include <metal_stdlib>+#include "defines.h"+#include "thread_context.h"++using namespace metal;++UZU_CONST uint ACTIVATION_QUANT_TILE_SIZE = 128;+UZU_CONST float ACTIVATION_QUANT_INT8_MAX = 127.0f;+#define ACTIVATION_QUANT_SIMDGROUPS 4++template <typename T, typename SimdReduce, typename Combine>+METAL_FUNC T reduce_activation_quantization_group(+ const T value,+ const uint group_size,+ threadgroup T* partials,+ const thread ThreadContext& thread_context,+ SimdReduce simd_reduce,+ Combine combine+) {+ const ushort lane_index = thread_context.simd_lane_id;+ const ushort simdgroup_index = thread_context.simdgroup_index;+ const T simdgroup_value = simd_reduce(value);+ if (group_size == METAL_SIMD_SIZE) {+ return simdgroup_value;+ }+ if (lane_index == 0) {+ partials[simdgroup_index] = simdgroup_value;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ const uint simdgroups_per_group = group_size / METAL_SIMD_SIZE;+ const uint simdgroup_base = simdgroup_index / simdgroups_per_group * simdgroups_per_group;+ T result = partials[simdgroup_base];+ METAL_PRAGMA_UNROLL+ for (uint index = 1; index < simdgroups_per_group; ++index) {+ result = combine(result, partials[simdgroup_base + index]);+ }+ return result;+}++template <typename T>+METAL_FUNC void write_activation_quantization_group(+ device T* output,+ const T value,+ const uint group_size,+ const uint element_count,+ const uint tile_index,+ const uint batch_index,+ const thread ThreadContext& thread_context+) {+ const ushort simdgroup_index = thread_context.simdgroup_index;+ const uint simdgroups_per_group = group_size / METAL_SIMD_SIZE;+ const uint simdgroup_offset = tile_index * ACTIVATION_QUANT_TILE_SIZE + simdgroup_index * METAL_SIMD_SIZE;+ if (thread_context.simd_lane_id == 0 && simdgroup_offset < element_count &&+ (simdgroup_index + 1) % simdgroups_per_group == 0) {+ const uint groups_per_row = element_count / group_size;+ const uint groups_per_tile = ACTIVATION_QUANT_TILE_SIZE / group_size;+ const uint group_index = tile_index * groups_per_tile + simdgroup_index / simdgroups_per_group;+ output[batch_index * groups_per_row + group_index] = value;+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/gated_act_mul.h b/crates/backend-uzu/src/backends/metal/kernel/common/gated_act_mul.hnew file mode 100644index 000000000..cee631d64--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/common/gated_act_mul.h@@ -0,0 +1,14 @@+#pragma once++#include <metal_stdlib>++#include "../activation/activations.h"++using namespace metal;+using namespace uzu::activation_type;++template <typename T>+static METAL_FUNC float gated_act_mul(T value, T gate, ActivationType act_type) {+ const T result = value * activate(gate, act_type);+ return static_cast<float>(result);+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal b/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metalindex ef6ca3413..b4867bb7e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal@@ -1,40 +1,88 @@#include <metal_stdlib>-#include "../activation/activations.h"+#include "../common/activation_quantization.h"#include "../common/dsl.h"+#include "../common/gated_act_mul.h"+#include "../common/thread_context.h"+#include "../generated/gated_act_mul.h"#include "../hadamard_transform/hadamard_transform.h"+using namespace metal;using namespace uzu::activation_type;+using namespace uzu::gated_act_mul;++#define NUM_SIMDGROUPS 4+#define NUM_THREADS NUM_SIMDGROUPS* METAL_SIMD_SIZE++#define QUANTIZED (ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)+#define EMITS_GROUP_SUMS (ops == GatedActMulOp::QuantizeWithGroupSums)template <typename T>VARIANTS(T, float, bfloat)PUBLIC KERNEL(GatedActMul) (const device T* act_operand,const device T* value_operand OPTIONAL(!interleaved),- device T* output,+ device T* fp_out OPTIONAL(!QUANTIZED),+ device int8_t* q_out OPTIONAL(QUANTIZED),+ device float* scales_out OPTIONAL(QUANTIZED),+ device int32_t* group_sums_out OPTIONAL(EMITS_GROUP_SUMS),const device int32_t* hadamard_factors OPTIONAL(use_hadamard),const constant uint& gated_dim,const constant uint& batch_dim,const constant uint& value_offset,const constant uint& value_row_stride,const constant ActivationType& act_type,+ const GatedActMulOp ops SPECIALIZE,const bool interleaved SPECIALIZE,const bool use_hadamard SPECIALIZE,- uint gated_idx AXIS(gated_dim, 64),- uint batch_idx AXIS(batch_dim, 1)+ const uint activation_scale_group_size SPECIALIZE,+ const uint sum_group_size SPECIALIZE,+ threadgroup float partial_max OPTIONAL(QUANTIZED && activation_scale_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],+ threadgroup int partial_sums OPTIONAL(EMITS_GROUP_SUMS && sum_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],+ uint activation_tile_index GROUPS(gated_dim.div_ceil(NUM_THREADS)),+ uint batch_idx GROUPS(batch_dim),+ uint thread_index THREADS(NUM_THREADS),+ const ThreadContext thread_context) {- T activated;- T value;- if (interleaved) {- uint base = batch_idx * (2 * gated_dim);- value = act_operand[base + gated_idx];- activated = activate(act_operand[base + gated_dim + gated_idx], act_type);- } else {- activated = activate(act_operand[batch_idx * gated_dim + gated_idx], act_type);- value = value_operand[batch_idx * value_row_stride + value_offset + gated_idx];+ const uint gated_idx = activation_tile_index * ACTIVATION_QUANT_TILE_SIZE + thread_index;+ const uint simdgroup_offset =+ activation_tile_index * ACTIVATION_QUANT_TILE_SIZE + (thread_index / METAL_SIMD_SIZE) * METAL_SIMD_SIZE;+ const bool element_in_bounds = gated_idx < gated_dim;+ const bool simdgroup_in_bounds = simdgroup_offset + METAL_SIMD_SIZE <= gated_dim;+ T value = static_cast<T>(0);+ T gate = static_cast<T>(0);+ if (element_in_bounds) {+ if (interleaved) {+ const uint base = batch_idx * (2 * gated_dim);+ value = act_operand[base + gated_idx];+ gate = act_operand[base + gated_dim + gated_idx];+ } else {+ value = value_operand[batch_idx * value_row_stride + value_offset + gated_idx];+ gate = act_operand[batch_idx * gated_dim + gated_idx];+ }}- T result = value * activated;- if (use_hadamard) {+ if (!QUANTIZED) {+ T result = static_cast<T>(0);+ if (element_in_bounds && (!use_hadamard || simdgroup_in_bounds)) {+ float gated = gated_act_mul(value, gate, act_type);+ if (use_hadamard) {+ gated = simdgroup_input_random_hadamard_transform(+ static_cast<ushort>(gated_idx % METAL_SIMD_SIZE),+ gated,+ hadamard_factors[gated_idx]+ );+ }+ result = static_cast<T>(gated);+ }+ if (element_in_bounds) {+ fp_out[batch_idx * gated_dim + gated_idx] = result;+ }+ return;+ }++ float result = 0.0f;+ if (simdgroup_in_bounds) {+ result = gated_act_mul(value, gate, act_type);result = simdgroup_input_random_hadamard_transform(static_cast<ushort>(gated_idx % METAL_SIMD_SIZE),result,@@ -42,5 +90,48 @@ PUBLIC KERNEL(GatedActMul) ();}- output[batch_idx * gated_dim + gated_idx] = result;+ const float maximum = reduce_activation_quantization_group(+ fabs(result),+ activation_scale_group_size,+ partial_max,+ thread_context,+ [](float x) { return simd_max(x); },+ [](float x, float y) { return max(x, y); }+ );+ const float scale = isfinite(maximum) && maximum > 0.0f ? maximum / ACTIVATION_QUANT_INT8_MAX : 1.0f;+ const int8_t code =+ static_cast<int8_t>(clamp(round(result / scale), -ACTIVATION_QUANT_INT8_MAX, ACTIVATION_QUANT_INT8_MAX));+ if (element_in_bounds) {+ q_out[batch_idx * gated_dim + gated_idx] = code;+ }++ write_activation_quantization_group(+ scales_out,+ scale,+ activation_scale_group_size,+ gated_dim,+ activation_tile_index,+ batch_idx,+ thread_context+ );++ if (EMITS_GROUP_SUMS) {+ const int sum = reduce_activation_quantization_group(+ static_cast<int>(code),+ sum_group_size,+ partial_sums,+ thread_context,+ [](int x) { return simd_sum(x); },+ [](int x, int y) { return x + y; }+ );+ write_activation_quantization_group(+ group_sums_out,+ sum,+ sum_group_size,+ gated_dim,+ activation_tile_index,+ batch_idx,+ thread_context+ );+ }}diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gated_act_mul.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gated_act_mul.hnew file mode 100644index 000000000..571e56256--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gated_act_mul.h@@ -0,0 +1,13 @@+// Auto-generated from gpu_types/gated_act_mul - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::gated_act_mul {+enum class GatedActMulOp : uint32_t {+ FullPrecision = 0,+ Quantize = 1,+ QuantizeWithGroupSums = 2,+};+} // namespace uzu::gated_act_muldiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex 03ade8771..e6bfdba49 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -10,8 +10,11 @@ use crate::{backends::{common::{BufferArg, Encoder,- gpu_types::gemm::GemmTiling,- kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulPath, MatmulShape},+ gpu_types::gemm::{GemmBPrologueKind, GemmTiling},+ kernel::{+ activation_transform::ACTIVATION_SCALE_GROUP_SIZE,+ matmul::{A8ActivationPlan, ActivationFormat, MatmulArguments, MatmulError, MatmulKernel, MatmulShape},+ },},metal::{Metal, context::MetalContext, error::MetalError},},@@ -120,14 +123,57 @@ impl MatmulKernel for MatmulMetalKernel {})}- fn select_path(+ fn a8_activation_plan(&self,shape: &MatmulShape,context: &MetalContext,- ) -> MatmulPath {- match self.select_dispatch(shape, context) {- MatmulDispatch::Gemv(_) => MatmulPath::Gemv,- MatmulDispatch::Gemm(_) => MatmulPath::Gemm,+ ) -> Option<A8ActivationPlan> {+ let activation_group_size = ACTIVATION_SCALE_GROUP_SIZE;+ if !context.supports_mxu()+ || self.input_data_type != DataType::BF16+ || self.output_data_type != DataType::BF16+ || shape.a_full_precision+ || !shape.is_quant()+ || !shape.signed_codes+ || !shape.b_transpose+ || shape.b_leading_dimension.is_some()+ || !matches!(shape.b_bits, Some(4 | 8))+ || !matches!(shape.b_group_size, Some(32 | 64 | 128))+ || !shape.k.is_multiple_of(activation_group_size)+ || !shape.k.is_multiple_of(shape.b_group_size.unwrap())+ {+ return None;+ }++ let sum_group_size = match shape.b_prologue {+ GemmBPrologueKind::ScaleSymmetricDequant => None,+ GemmBPrologueKind::ScaleBiasDequant | GemmBPrologueKind::ScaleZeroPointDequant => {+ Some(shape.b_group_size.unwrap().min(activation_group_size))+ },+ GemmBPrologueKind::FullPrecision => return None,+ };+ Some(A8ActivationPlan {+ activation_group_size,+ sum_group_size,+ })+ }++ fn select_activation_format(+ &self,+ bf16_shape: &MatmulShape,+ context: &MetalContext,+ ) -> ActivationFormat {+ if matches!(self.select_dispatch(bf16_shape, context), MatmulDispatch::Gemv(_)) {+ return ActivationFormat::Bf16;+ }++ let a8_shape = MatmulShape {+ a_full_precision: false,+ ..*bf16_shape+ };+ match self.select_dispatch(&a8_shape, context) {+ MatmulDispatch::Gemm(plan) if plan.engine == gemm::GemmEngine::Mxu => ActivationFormat::Int8,+ _ => ActivationFormat::Bf16,}}diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex e04b78329..7b352fff4 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -153,10 +153,10 @@ impl<B: Backend> Embedding<B> {AnyWeightMatrixSpec::HybridSpec(HybridSpec {quantization_spec,adapter_spec: None,- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),+ incoherence_block_size: Some(block_size),incoherence_processing_mode: IncoherenceProcessingMode::Output,..- }) => {+ }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => {let incoherence_signs_tree = embedding_tree.subtree("incoherence_signs");let output_hadamard_factors = Some(incoherence_signs_tree@@ -201,10 +201,10 @@ impl<B: Backend> Embedding<B> {AnyWeightMatrixSpec::HybridSpec(HybridSpec {quantization_spec,adapter_spec: None,- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),+ incoherence_block_size: Some(block_size),incoherence_processing_mode: IncoherenceProcessingMode::Output,..- }) => {+ }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => {let output_hadamard_factors = Some(input_embedding_tree.subtree("incoherence_signs")@@ -240,10 +240,10 @@ impl<B: Backend> Embedding<B> {AnyWeightMatrixSpec::HybridSpec(HybridSpec {quantization_spec,adapter_spec: None,- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),+ incoherence_block_size: Some(block_size),incoherence_processing_mode: IncoherenceProcessingMode::Input,..- }) => {+ }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => {let matrix = WeightMatrix::load(&output_embedding_tree.subtree("quantized"),*quantization_spec,diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rsindex 213a857ad..59a5e8827 100644--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs@@ -2,17 +2,21 @@ use parking_lot::Mutex;use thiserror::Error;use crate::{+ array::size_for_shape,backends::common::{Allocation, Backend, Encoder,kernel::{Kernels,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulPath, MatmulShape},+ matmul::{+ A8ActivationPlan, ActivationFormat, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel,+ MatmulShape,+ },},},config::weight_matrix::{AnyWeightMatrixSpec, Layout},data_type::DataType,encodable_block::{- linear::Linear,+ linear::{Linear, LinearInput},weight_matrix::{WeightMatrix, WeightMatrixError},},parameters::{ParameterLoaderError, ParameterTree},@@ -104,8 +108,15 @@ impl<B: Backend> LinearMatmul<B> {})}- pub(super) fn make_codes_signed(&mut self) {+ pub(super) fn prepare_a8(+ &mut self,+ context: &B::Context,+ ) -> Option<A8ActivationPlan> {+ let mut candidate = self.matmul_shape(1, false);+ candidate.signed_codes = true;+ let plan = self.kernel.lock().a8_activation_plan(&candidate, context)?;self.matrix.make_codes_signed();+ Some(plan)}pub(super) fn encode_with_a(@@ -114,7 +125,8 @@ impl<B: Backend> LinearMatmul<B> {batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let mut output = encoder.allocate_scratch_for_shape(&[batch_dim, self.output_dim], self.output_data_type)?;+ let mut output =+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;self.kernel.lock().encode(MatmulArguments {@@ -135,25 +147,13 @@ impl<B: Backend> LinearMatmul<B> {Ok(output)}- fn matmul_b(&self) -> MatmulB<'_, B> {- self.matrix.matmul_b()- }-- fn d_ops(&self) -> MatmulDOps<'_, B> {- MatmulDOps {- bias: self.biases.as_ref(),- rht_factors: self.output_hadamard_factors.as_ref(),- ..MatmulDOps::none()- }- }-- pub(super) fn select_path(+ fn matmul_shape(&self,batch_dim: u32,- context: &B::Context,- ) -> MatmulPath {+ a_full_precision: bool,+ ) -> MatmulShape {let b = self.matmul_b();- let shape = MatmulShape {+ MatmulShape {m: batch_dim,n: self.output_dim,k: self.input_dim,@@ -163,11 +163,34 @@ impl<B: Backend> LinearMatmul<B> {b_bits: b.bits_per_b(),b_group_size: b.group_size(),signed_codes: b.signed_codes(),- a_full_precision: true,+ a_full_precision,gathered: false,d_transform: self.d_ops().mask(),- };- self.kernel.lock().select_path(&shape, context)+ }+ }++ pub(super) fn select_activation_format(+ &self,+ batch_dim: u32,+ context: &B::Context,+ ) -> ActivationFormat {+ if !self.matmul_b().signed_codes() {+ return ActivationFormat::Bf16;+ }+ let bf16_shape = self.matmul_shape(batch_dim, true);+ self.kernel.lock().select_activation_format(&bf16_shape, context)+ }++ fn matmul_b(&self) -> MatmulB<'_, B> {+ self.matrix.matmul_b()+ }++ fn d_ops(&self) -> MatmulDOps<'_, B> {+ MatmulDOps {+ bias: self.biases.as_ref(),+ rht_factors: self.output_hadamard_factors.as_ref(),+ ..MatmulDOps::none()+ }}}@@ -193,4 +216,38 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {Ok(output)}++ fn encode_input(+ &self,+ input: LinearInput<B>,+ batch_dim: u32,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ match input {+ LinearInput::FullPrecision(input) => self.encode(input, batch_dim, encoder),+ LinearInput::Int8Symmetric {+ values,+ scales,+ group_sums,+ group_size,+ } => self.encode_with_a(+ MatmulA::Int8Symmetric {+ values: &values,+ scales: &scales,+ group_sums: group_sums.as_ref(),+ group_size,+ },+ batch_dim,+ encoder,+ ),+ }+ }++ fn select_activation_format(+ &self,+ batch_dim: u32,+ context: &B::Context,+ ) -> ActivationFormat {+ Self::select_activation_format(self, batch_dim, context)+ }}diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rsindex 42594c9cf..333e6209a 100644--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs@@ -4,21 +4,20 @@ mod rht_wrapper;pub use matmul::{LinearMatmul, LinearMatmulError};pub use qlora_wrapper::{QLoRALinearWrapper, QLoRALinearWrapperError};-use rht_wrapper::int8_activations_eligible;pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};use thiserror::Error;use crate::{backends::common::{- Allocation, Backend, Encoder, gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,- kernel::activation_transform::ACTIVATION_SCALE_GROUP_SIZE,+ Allocation, Backend, Encoder,+ gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,+ kernel::matmul::{A8ActivationPlan, ActivationFormat},},config::weight_matrix::{AnyWeightMatrixSpec,hybrid_spec::{HybridSpec, IncoherenceProcessingMode},},data_type::DataType,- encodable_block::weight_matrix::parse_spec,parameters::{ParameterLoaderError, ParameterTree},};@@ -29,14 +28,51 @@ pub trait Linear<B: Backend>: Send + Sync {batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error>;++ fn encode_input(+ &self,+ input: LinearInput<B>,+ batch_dim: u32,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ match input {+ LinearInput::FullPrecision(input) => self.encode(input, batch_dim, encoder),+ LinearInput::Int8Symmetric {+ ..+ } => {+ panic!("linear does not support pre-quantized activations")+ },+ }+ }++ fn select_activation_format(+ &self,+ _batch_dim: u32,+ _context: &B::Context,+ ) -> ActivationFormat {+ ActivationFormat::Bf16+ }+}++pub enum LinearInput<B: Backend> {+ FullPrecision(Allocation<B>),+ Int8Symmetric {+ values: Allocation<B>,+ scales: Allocation<B>,+ group_sums: Option<Allocation<B>>,+ group_size: u32,+ },+}++pub struct LinearInputPreparation<B: Backend> {+ pub input_factors: Allocation<B>,+ pub a8_plan: Option<A8ActivationPlan>,}#[derive(Debug, Error)]pub enum LinearBlockError<B: Backend> {#[error("LinearMatmul error: {0}")]LinearMatmulError(#[from] LinearMatmulError<B>),- #[error("Output hadamard linear error: {0}")]- OutputHadamardLinearError(#[from] OutputHadamardLinearError<B>),#[error("QLoRALinearWrapper error: {0}")]QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),#[error("RHTLinearWrapper error: {0}")]@@ -47,16 +83,6 @@ pub enum LinearBlockError<B: Backend> {UnsupportedConfiguration(String),}-#[derive(Debug, Error)]-pub enum OutputHadamardLinearError<B: Backend> {- #[error("LinearMatmul error: {0}")]- LinearMatmulError(#[from] LinearMatmulError<B>),- #[error("Parameter loading error: {0}")]- ParameterError(#[from] ParameterLoaderError<B>),- #[error("Unsupported linear configuration: {0}")]- UnsupportedConfiguration(String),-}-impl<B: Backend> dyn Linear<B> {pub fn new_mixed_precision<const N: usize>(input_dimension: u32,@@ -91,10 +117,10 @@ impl<B: Backend> dyn Linear<B> {},AnyWeightMatrixSpec::HybridSpec(HybridSpec {adapter_spec: None,- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),+ incoherence_block_size: Some(block_size),incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,..- }) => Ok(Box::new(RHTLinearWrapper::new(+ }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => Ok(Box::new(RHTLinearWrapper::new(context,input_dimension,output_dimension_sum,@@ -154,121 +180,81 @@ impl<B: Backend> dyn Linear<B> {)}- pub fn new_with_output_hadamard_mixed_precision(- context: &B::Context,- parameter_tree: &ParameterTree<B>,- output_factors: Allocation<B>,- input_dim: u32,- output_dim: u32,+ pub fn new_with_input_rht_mixed_precision<const N: usize>(+ input_dimension: u32,+ output_dimensions: [u32; N],has_biases: bool,+ context: &B::Context,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,- ) -> Result<Box<dyn Linear<B>>, OutputHadamardLinearError<B>> {- let weights_tree = parameter_tree.subtree("weights").subtree("quantized");- let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;- match spec {- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {- Ok(Box::new(LinearMatmul::load(- context,- spec,- input_dim,- output_dim,- weights_data_type,- input_data_type,- output_data_type,- &weights_tree,- has_biases.then_some(parameter_tree),- Some(output_factors),- )?))- },- spec => Err(OutputHadamardLinearError::UnsupportedConfiguration(format!(- "{spec:?} doesn't support fused output hadamard"- ))),+ parameter_tree: &ParameterTree<B>,+ ) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {+ let output_dimension_sum: u32 = output_dimensions.iter().sum();+ if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(+ context,+ input_dimension,+ output_dimension_sum,+ has_biases,+ weights_data_type,+ input_data_type,+ output_data_type,+ false,+ parameter_tree,+ )? {+ return Ok((linear.0, linear.1.map(|preparation| preparation.input_factors)));}++ let linear = Self::new_mixed_precision(+ input_dimension,+ output_dimensions,+ has_biases,+ context,+ weights_data_type,+ input_data_type,+ output_data_type,+ parameter_tree,+ )?;+ Ok((linear, None))}- pub fn new_extracting_input_hadamard_mixed_precision<const N: usize>(+ pub fn new_for_fused_input<const N: usize>(input_dimension: u32,output_dimensions: [u32; N],has_biases: bool,context: &B::Context,- weights_data_type: DataType,- input_data_type: DataType,- output_data_type: DataType,+ data_type: DataType,parameter_tree: &ParameterTree<B>,- ) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {+ ) -> Result<(Box<dyn Linear<B>>, Option<LinearInputPreparation<B>>), LinearBlockError<B>> {let output_dimension_sum: u32 = output_dimensions.iter().sum();- let weights_tree = parameter_tree.subtree("weights");- let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;- if let AnyWeightMatrixSpec::HybridSpec(HybridSpec {- adapter_spec: None,- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),- incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,- ..- }) = &spec- {- let quantization_spec = weights_tree.subtree("quantized").metadata::<AnyWeightMatrixSpec>("spec")?;- let parsed = parse_spec::<B>(&quantization_spec).ok();- if parsed.as_ref().is_some_and(|parsed| {- int8_activations_eligible::<B>(- context,- parsed,- input_dimension,- input_data_type,- output_data_type,- ACTIVATION_SCALE_GROUP_SIZE,- )- }) {- let linear = RHTLinearWrapper::new(- context,- input_dimension,- output_dimension_sum,- has_biases,- weights_data_type,- input_data_type,- output_data_type,- parameter_tree,- )?;- return Ok((Box::new(linear), None));- }-- let input_factors = weights_tree- .leaf("incoherence_signs.input_signs")?- .validate(&[input_dimension], DataType::I32)?- .read_allocation()?;- let output_factors = weights_tree- .leaf("incoherence_signs.output_signs")?- .validate(&[output_dimension_sum], DataType::I32)?- .read_allocation()?;- let inner_linear = Self::new_with_output_hadamard_mixed_precision(- context,- parameter_tree,- output_factors,- input_dimension,- output_dimension_sum,- has_biases,- weights_data_type,- input_data_type,- output_data_type,- )?;- Ok((inner_linear, Some(input_factors)))- } else {- let linear = Self::new_mixed_precision(- input_dimension,- output_dimensions,- has_biases,- context,- weights_data_type,- input_data_type,- output_data_type,- parameter_tree,- )?;- Ok((linear, None))+ if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(+ context,+ input_dimension,+ output_dimension_sum,+ has_biases,+ data_type,+ data_type,+ data_type,+ true,+ parameter_tree,+ )? {+ return Ok(linear);}++ let linear = Self::new_mixed_precision(+ input_dimension,+ output_dimensions,+ has_biases,+ context,+ data_type,+ data_type,+ data_type,+ parameter_tree,+ )?;+ Ok((linear, None))}- pub fn new_extracting_input_hadamard<const N: usize>(+ pub fn new_with_input_rht<const N: usize>(input_dimension: u32,output_dimensions: [u32; N],has_biases: bool,@@ -276,7 +262,7 @@ impl<B: Backend> dyn Linear<B> {data_type: DataType,parameter_tree: &ParameterTree<B>,) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {- Self::new_extracting_input_hadamard_mixed_precision(+ Self::new_with_input_rht_mixed_precision(input_dimension,output_dimensions,has_biases,diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rsindex f5866e5b2..03003245b 100644--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -64,7 +64,11 @@ impl<B: Backend> QLoRALinearWrapper<B> {) -> Result<Self, QLoRALinearWrapperError<B>> {let use_incoherence_signs = match (incoherence_block_size, incoherence_processing_mode) {(None, _) => false,- (Some(HADAMARD_TRANSFORM_BLOCK_SIZE), IncoherenceProcessingMode::InputOutput) => true,+ (Some(block_size), IncoherenceProcessingMode::InputOutput)+ if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE =>+ {+ true+ },(incoherence_block_size, incoherence_processing_mode) => {return Err(QLoRALinearWrapperError::UnsupportedConfiguration(format!("incoherence block_size={incoherence_block_size:?}, processing_mode={incoherence_processing_mode:?}"diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rsindex 08488ef70..dcef5873a 100644--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs@@ -1,64 +1,28 @@use thiserror::Error;use crate::{+ array::size_for_shape,backends::common::{- Allocation, Backend, Context, DeviceCapabilities, Encoder,- gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod},+ Allocation, Backend, Encoder,+ gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,kernel::{ActivationTransform,- activation_transform::ACTIVATION_SCALE_GROUP_SIZE,- matmul::{MatmulA, MatmulPath},+ matmul::{A8ActivationPlan, ActivationFormat, MatmulA},},},config::weight_matrix::{- AnyWeightMatrixSpec, Layout,+ AnyWeightMatrixSpec,hybrid_spec::{HybridSpec, IncoherenceProcessingMode},},data_type::DataType,- encodable_block::{- linear::{Linear, LinearMatmul, LinearMatmulError},- weight_matrix::{ParsedWeightSpec, WeightMatrixError, parse_spec},- },+ encodable_block::linear::{Linear, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError},parameters::{ParameterLoaderError, ParameterTree},};-// Temporary, for testing only: flip to false to run linear layers on the bf16-// activation path without touching model configs.-const INT8_ACTIVATIONS_ENABLED: bool = true;--pub(super) fn int8_activations_eligible<B: Backend>(- context: &B::Context,- spec: &ParsedWeightSpec,- input_dimension: u32,- input_data_type: DataType,- output_data_type: DataType,- activation_group_size: u32,-) -> bool {- if !INT8_ACTIVATIONS_ENABLED {- return false;- }- if !context.device_capabilities().contains(DeviceCapabilities::NATIVE_INT8_MATMUL) {- return false;- }- if input_data_type != DataType::BF16 || output_data_type != DataType::BF16 {- return false;- }- let (Some(info), Layout::OutputInput) = (spec.quantization, &spec.layout) else {- return false;- };- let group_size = info.group_size;- matches!(group_size, 32 | 64 | 128)- && input_dimension.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE)- && input_dimension.is_multiple_of(group_size)- && input_dimension.is_multiple_of(activation_group_size)-}-#[derive(Debug, Error)]pub enum RHTLinearWrapperError<B: Backend> {#[error("Inner linear error: {0}")]InnerLinearError(#[from] LinearMatmulError<B>),- #[error("Weight matrix error: {0}")]- WeightMatrix(#[from] WeightMatrixError<B>),#[error("Parameter loading error: {0}")]ParameterError(#[from] ParameterLoaderError<B>),#[error("Backend error: {0}")]@@ -67,16 +31,35 @@ pub enum RHTLinearWrapperError<B: Backend> {UnsupportedConfiguration(String),}+enum InputRht<B: Backend> {+ FullPrecision(ActivationTransform<B>),+ A8 {+ fallback: ActivationTransform<B>,+ a8: ActivationTransform<B>,+ },+}+pub struct RHTLinearWrapper<B: Backend> {- input_transform: ActivationTransform<B>,- quantize_transform: Option<ActivationTransform<B>>,+ input_rht: InputRht<B>,input_factors: Allocation<B>,inner_linear: LinearMatmul<B>,input_dimension: u32,}+fn has_input_output_rht(spec: &AnyWeightMatrixSpec) -> bool {+ matches!(+ spec,+ AnyWeightMatrixSpec::HybridSpec(HybridSpec {+ adapter_spec: None,+ incoherence_block_size: Some(block_size),+ incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,+ ..+ }) if *block_size == HADAMARD_TRANSFORM_BLOCK_SIZE+ )+}+impl<B: Backend> RHTLinearWrapper<B> {- pub fn new(+ pub(super) fn new(context: &B::Context,input_dimension: u32,output_dimension: u32,@@ -88,16 +71,88 @@ impl<B: Backend> RHTLinearWrapper<B> {) -> Result<Self, RHTLinearWrapperError<B>> {let weights_tree = parameter_tree.subtree("weights");let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;- let AnyWeightMatrixSpec::HybridSpec(HybridSpec {- adapter_spec: None,- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),- incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,- ..- }) = &spec- else {+ if !has_input_output_rht(&spec) {return Err(RHTLinearWrapperError::UnsupportedConfiguration(format!("{spec:?}")));- };+ }++ let (input_factors, mut inner_linear) = Self::load_inner_with_output_rht(+ context,+ input_dimension,+ output_dimension,+ has_biases,+ input_data_type,+ output_data_type,+ weights_data_type,+ parameter_tree,+ )?;+ let a8_plan = inner_linear.prepare_a8(context);+ Self::build_self_contained(context, input_dimension, input_data_type, input_factors, inner_linear, a8_plan)+ }++ pub(super) fn try_new_with_input_preparation(+ context: &B::Context,+ input_dimension: u32,+ output_dimension: u32,+ has_biases: bool,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,+ allow_prequantized_activation: bool,+ parameter_tree: &ParameterTree<B>,+ ) -> Result<Option<(Box<dyn Linear<B>>, Option<LinearInputPreparation<B>>)>, RHTLinearWrapperError<B>> {+ let weights_tree = parameter_tree.subtree("weights");+ let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;+ if !has_input_output_rht(&spec) {+ return Ok(None);+ }++ let (input_factors, mut inner_linear) = Self::load_inner_with_output_rht(+ context,+ input_dimension,+ output_dimension,+ has_biases,+ input_data_type,+ output_data_type,+ weights_data_type,+ parameter_tree,+ )?;+ let a8_plan = inner_linear.prepare_a8(context);+ if let Some(a8_plan) = a8_plan+ && !allow_prequantized_activation+ {+ let wrapper = Self::build_self_contained(+ context,+ input_dimension,+ input_data_type,+ input_factors,+ inner_linear,+ Some(a8_plan),+ )?;+ Ok(Some((Box::new(wrapper), None)))+ } else {+ Ok(Some((+ Box::new(inner_linear),+ Some(LinearInputPreparation {+ input_factors,+ a8_plan,+ }),+ )))+ }+ }+ fn load_inner_with_output_rht(+ context: &B::Context,+ input_dimension: u32,+ output_dimension: u32,+ has_biases: bool,+ input_data_type: DataType,+ output_data_type: DataType,+ weights_data_type: DataType,+ parameter_tree: &ParameterTree<B>,+ ) -> Result<(Allocation<B>, LinearMatmul<B>), RHTLinearWrapperError<B>> {+ let weights_tree = parameter_tree.subtree("weights");+ let quantized_weights_tree = weights_tree.subtree("quantized");+ let quantization_spec = quantized_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;let input_factors = weights_tree.leaf("incoherence_signs.input_signs")?.validate(&[input_dimension], DataType::I32)?@@ -106,18 +161,7 @@ impl<B: Backend> RHTLinearWrapper<B> {.leaf("incoherence_signs.output_signs")?.validate(&[output_dimension], DataType::I32)?.read_allocation()?;- let quantized_weights_tree = weights_tree.subtree("quantized");- let quantization_spec = quantized_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;- let parsed = parse_spec::<B>(&quantization_spec)?;- let Some(quantization) = parsed.quantization else {- return Err(RHTLinearWrapperError::UnsupportedConfiguration("RHT requires a quantized inner spec".into()));- };- let activation_group_size = ACTIVATION_SCALE_GROUP_SIZE;-- let input_transform = ActivationTransform::input_rht(context, input_data_type, true)- .map_err(RHTLinearWrapperError::BackendError)?;-- let mut inner_linear = LinearMatmul::load(+ let inner_linear = LinearMatmul::load(context,quantization_spec,input_dimension,@@ -129,32 +173,38 @@ impl<B: Backend> RHTLinearWrapper<B> {has_biases.then_some(parameter_tree),Some(output_factors),)?;+ Ok((input_factors, inner_linear))+ }- let quantize_transform = if int8_activations_eligible::<B>(- context,- &parsed,- input_dimension,- input_data_type,- output_data_type,- activation_group_size,- ) {- let emit_group_sums = !matches!(quantization.method, QuantizationMethod::ScaleSymmetric);- let transform = ActivationTransform::quantize(- context,- input_data_type,- activation_group_size,- emit_group_sums.then_some(quantization.group_size.min(activation_group_size)),- )+ fn build_self_contained(+ context: &B::Context,+ input_dimension: u32,+ input_data_type: DataType,+ input_factors: Allocation<B>,+ inner_linear: LinearMatmul<B>,+ a8_plan: Option<A8ActivationPlan>,+ ) -> Result<Self, RHTLinearWrapperError<B>> {+ let input_transform = ActivationTransform::input_rht(context, input_data_type, true).map_err(RHTLinearWrapperError::BackendError)?;- inner_linear.make_codes_signed();- Some(transform)- } else {- None+ let input_rht = match a8_plan {+ Some(plan) => {+ let quantized = ActivationTransform::quantize(+ context,+ input_data_type,+ plan.activation_group_size,+ plan.sum_group_size,+ )+ .map_err(RHTLinearWrapperError::BackendError)?;+ InputRht::A8 {+ fallback: input_transform,+ a8: quantized,+ }+ },+ None => InputRht::FullPrecision(input_transform),};Ok(Self {- input_transform,- quantize_transform,+ input_rht,input_factors,inner_linear,input_dimension,@@ -168,22 +218,46 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {input: Allocation<B>,batch_dim: u32,encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ self.encode_input(LinearInput::FullPrecision(input), batch_dim, encoder)+ }++ fn encode_input(+ &self,+ input: LinearInput<B>,+ batch_dim: u32,+ encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group("linear (rht)");- if let Some(quantize_transform) = &self.quantize_transform- && self.inner_linear.select_path(batch_dim, encoder.context()) == MatmulPath::Gemm+ let input = match input {+ LinearInput::FullPrecision(input) => input,+ input => {+ let output = self.inner_linear.encode_input(input, batch_dim, encoder);+ encoder.pop_debug_group();+ return output;+ },+ };++ if let InputRht::A8 {+ a8: quantized,+ ..+ } = &self.input_rht+ && self.inner_linear.select_activation_format(batch_dim, encoder.context()) == ActivationFormat::Int8{- let scale_groups_per_row = self.input_dimension.div_ceil(quantize_transform.activation_group_size());- let sum_groups_per_row =- quantize_transform.sum_group_size().map(|group_size| self.input_dimension.div_ceil(group_size));- let mut values = encoder.allocate_scratch_for_shape(&[batch_dim, self.input_dimension], DataType::I8)?;- let mut scales = encoder.allocate_scratch_for_shape(&[batch_dim, scale_groups_per_row], DataType::F32)?;- let mut group_sums = sum_groups_per_row- .map(|groups| encoder.allocate_scratch_for_shape(&[batch_dim, groups], DataType::I32))+ let activation_group_size = quantized.activation_group_size();+ let scale_groups_per_row = self.input_dimension.div_ceil(activation_group_size);+ let mut values =+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dimension], DataType::I8))?;+ let mut scales =+ encoder.allocate_scratch(size_for_shape(&[batch_dim, scale_groups_per_row], DataType::F32))?;+ let mut group_sums = quantized+ .sum_group_size()+ .map(|group_size| self.input_dimension.div_ceil(group_size))+ .map(|groups| encoder.allocate_scratch(size_for_shape(&[batch_dim, groups], DataType::I32))).transpose()?;- quantize_transform.encode_quantize(+ quantized.encode_quantize(&input,&mut values,&mut scales,@@ -198,29 +272,28 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {values: &values,scales: &scales,group_sums: group_sums.as_ref(),- group_size: quantize_transform.activation_group_size(),+ group_size: activation_group_size,},batch_dim,encoder,)?;encoder.pop_debug_group();-return Ok(output);}let mut input = input;- self.input_transform.encode_fp_in_place(- &mut input,- &self.input_factors,- batch_dim,- self.input_dimension,- encoder,- );+ let full_precision = match &self.input_rht {+ InputRht::FullPrecision(transform)+ | InputRht::A8 {+ fallback: transform,+ ..+ } => transform,+ };+ full_precision.encode_fp_in_place(&mut input, &self.input_factors, batch_dim, self.input_dimension, encoder);let output = self.inner_linear.encode(input, batch_dim, encoder)?;encoder.pop_debug_group();-Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rsindex a95e84210..4d8c641fe 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs@@ -96,7 +96,7 @@ impl<B: Backend> Attention<B> {q_dim};let (qkv_projection, in_projection_input_hadamard_factors) = if !has_gate {- <dyn Linear<B>>::new_extracting_input_hadamard(+ <dyn Linear<B>>::new_with_input_rht(hidden_dim,[qkv_projection_output_dimension],config.has_qkv_biases,diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rsindex 8e8cfa965..7f4629c0e 100644--- a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs@@ -191,7 +191,7 @@ impl<B: Backend> DeltaNet<B> {let total_proj_dim = conv_dim + value_dim + config.num_heads + config.num_heads;let (in_projection, in_projection_input_hadamard_factors) =- <dyn Linear<B>>::new_extracting_input_hadamard_mixed_precision(+ <dyn Linear<B>>::new_with_input_rht_mixed_precision(hidden_dim,[total_proj_dim],false,diff --git a/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs b/crates/backend-uzu/src/encodable_block/mixer/mamba2.rsindex 01e7e0bf0..1ed109da6 100644--- a/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs@@ -113,7 +113,7 @@ impl<B: Backend> Mamba2<B> {}let (in_projection, in_projection_input_hadamard_factors) =- <dyn Linear<B>>::new_extracting_input_hadamard_mixed_precision(+ <dyn Linear<B>>::new_with_input_rht_mixed_precision(hidden_dim,[conv_dim, inner_dim, num_heads],config.has_in_biases,diff --git a/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs b/crates/backend-uzu/src/encodable_block/mixer/short_conv.rsindex bd544b76a..4e7fd4b9f 100644--- a/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs@@ -117,7 +117,7 @@ impl<B: Backend> ShortConv<B> {)));}- let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(+ let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(hidden_dim,[hidden_dim * 3],false,diff --git a/crates/backend-uzu/src/encodable_block/mlp/dense.rs b/crates/backend-uzu/src/encodable_block/mlp/dense.rsindex a646339f3..4ae3e82cf 100644--- a/crates/backend-uzu/src/encodable_block/mlp/dense.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/dense.rs@@ -38,8 +38,9 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {encoder.push_debug_group("mlp (dense)");let fused_up = self.up.encode(input, batch_dim, encoder)?;- let hidden = self.gate.encode(encoder, &fused_up, batch_dim)?;- let output = self.down.encode(hidden, batch_dim, encoder)?;+ let act_format = self.down.select_activation_format(batch_dim, encoder.context());+ let down_input = self.gate.encode_for_linear(encoder, &fused_up, batch_dim, act_format)?;+ let output = self.down.encode_input(down_input, batch_dim, encoder)?;encoder.pop_debug_group();diff --git a/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs b/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rsindex 73799c439..bd629fe69 100644--- a/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs@@ -1,15 +1,26 @@use crate::{- backends::common::{Allocation, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::GatedActMulKernel},+ array::size_for_shape,+ backends::common::{+ Allocation, Backend, Encoder,+ gpu_types::ActivationType,+ kernel::{+ GatedActMul,+ matmul::{A8ActivationPlan, ActivationFormat},+ },+ },config::activation::AnyActivation,data_type::DataType,+ encodable_block::linear::{LinearInput, LinearInputPreparation},};pub struct MlpGateActMulEncodable<B: Backend> {- kernel: <B::Kernels as Kernels>::GatedActMulKernel,+ fp_kernel: GatedActMul<B>,activation: AnyActivation,hidden_dim: u32,data_type: DataType,hadamard_factors: Option<Allocation<B>>,+ a8_plan: Option<A8ActivationPlan>,+ quantized_kernel: Option<GatedActMul<B>>,}impl<B: Backend> MlpGateActMulEncodable<B> {@@ -18,46 +29,91 @@ impl<B: Backend> MlpGateActMulEncodable<B> {data_type: DataType,activation: AnyActivation,hidden_dim: u32,- hadamard_factors: Option<Allocation<B>>,+ input_preparation: Option<LinearInputPreparation<B>>,) -> Result<Self, B::Error> {- let kernel =- <B::Kernels as Kernels>::GatedActMulKernel::new(context, data_type, true, hadamard_factors.is_some())?;+ let (hadamard_factors, a8_plan) = input_preparation+ .map_or((None, None), |preparation| (Some(preparation.input_factors), preparation.a8_plan));+ let fp_kernel = GatedActMul::full_precision(context, data_type, true, hadamard_factors.is_some())?;+ let quantized_kernel = a8_plan+ .map(|plan| GatedActMul::quantized(context, data_type, plan.activation_group_size, plan.sum_group_size))+ .transpose()?;Ok(Self {- kernel,+ fp_kernel,activation,hidden_dim,data_type,hadamard_factors,+ a8_plan,+ quantized_kernel,})}- pub fn encode(+ pub fn encode_for_linear(&self,encoder: &mut Encoder<B>,fused_up: &Allocation<B>,batch_dim: u32,- ) -> Result<Allocation<B>, B::Error> {+ act_format: ActivationFormat,+ ) -> Result<LinearInput<B>, B::Error> {encoder.push_debug_group("gate act mul");if self.activation.act_type() == ActivationType::IDENTITY {panic!("Identity activation is not supported for kernel")}- let mut hidden = encoder.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;- self.kernel.encode(- fused_up,- None::<&Allocation<B>>,- &mut hidden,- self.hadamard_factors.as_ref(),- self.hidden_dim,- batch_dim,- 0,- 0,- self.activation.act_type(),- encoder,- );+ let input = if act_format == ActivationFormat::Int8+ && let Some(plan) = self.a8_plan+ {+ let kernel = self.quantized_kernel.as_ref().expect("INT8 input requires a quantized gate kernel");+ let mut values = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], DataType::I8))?;+ let mut scales = encoder.allocate_scratch(size_for_shape(+ &[batch_dim, self.hidden_dim.div_ceil(plan.activation_group_size)],+ DataType::F32,+ ))?;+ let mut group_sums = plan+ .sum_group_size+ .map(|group_size| {+ encoder.allocate_scratch(size_for_shape(+ &[batch_dim, self.hidden_dim.div_ceil(group_size)],+ DataType::I32,+ ))+ })+ .transpose()?;+ kernel.encode_quantized(+ fused_up,+ &mut values,+ &mut scales,+ group_sums.as_mut(),+ self.hadamard_factors.as_ref().expect("INT8 input requires RHT factors"),+ self.hidden_dim,+ batch_dim,+ self.activation.act_type(),+ encoder,+ );+ LinearInput::Int8Symmetric {+ values,+ scales,+ group_sums,+ group_size: plan.activation_group_size,+ }+ } else {+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;+ self.fp_kernel.encode_fp(+ fused_up,+ None,+ &mut hidden,+ self.hadamard_factors.as_ref(),+ self.hidden_dim,+ batch_dim,+ 0,+ 0,+ self.activation.act_type(),+ encoder,+ );+ LinearInput::FullPrecision(hidden)+ };encoder.pop_debug_group();- Ok(hidden)+ Ok(input)}}diff --git a/crates/backend-uzu/src/encodable_block/mlp/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/mod.rsindex 57230dd8c..606ad0220 100644--- a/crates/backend-uzu/src/encodable_block/mlp/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/mod.rs@@ -45,7 +45,7 @@ impl<B: Backend> dyn Mlp<B> {) -> Result<(Box<dyn Mlp<B>>, Option<Allocation<B>>), MlpBlockError<B>> {match config {AnyMLPConfig::DenseMLPConfig(dense_config) => {- let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(+ let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(model_dimension,[2 * hidden_dimension],dense_config.has_up_biases,@@ -54,7 +54,7 @@ impl<B: Backend> dyn Mlp<B> {¶meter_tree.subtree("up_projection"),)?;- let (down_projection, down_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(+ let (down_projection, down_input_preparation) = <dyn Linear<B>>::new_for_fused_input(hidden_dimension,[model_dimension],dense_config.has_down_biases,@@ -68,7 +68,7 @@ impl<B: Backend> dyn Mlp<B> {data_type,dense_config.activation.clone(),hidden_dimension,- down_input_hadamard_factors,+ down_input_preparation,).map_err(MlpBlockError::BackendError)?;diff --git a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rsindex 8840eabbc..919b54abf 100644--- a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs+++ b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs@@ -1,9 +1,10 @@use thiserror::Error;use crate::{+ array::size_for_shape,backends::common::{Allocation, Backend, Encoder,- kernel::{GatedActMulKernel, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},+ kernel::{GatedActMul, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},},config::{activation::AnyActivation,@@ -115,18 +116,18 @@ impl<B: Backend> PerLayerEmbedding<B> {let total_rows = batch_dim * self.num_layers;let total_elements = batch_dim * total_ple_dim;- let mut token_ple = encoder.allocate_scratch_for_shape(&[batch_dim, total_ple_dim], self.data_type)?;+ let mut token_ple = encoder.allocate_scratch(size_for_shape(&[batch_dim, total_ple_dim], self.data_type))?;self.token_embedding.encode_lookup(token_ids, &mut token_ple, batch_dim, self.fused_token_scale, encoder);let mut model_projection_input =- encoder.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.model_dim], self.data_type))?;encoder.encode_copy(inner_features, .., &mut model_projection_input, ..);let model_projected = self.model_projection.encode(model_projection_input, batch_dim, encoder)?;let model_normed = self.projection_norm.encode(&model_projected, 0, total_rows, None, encoder)?;let mut per_layer_inputs =- encoder.allocate_scratch_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type)?;+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type))?;self.add_scale.encode(Some(&token_ple),&model_normed,@@ -147,7 +148,7 @@ pub struct PerLayerEmbeddingProjection<B: Backend> {gate: Box<dyn Linear<B>>,projection: Box<dyn Linear<B>>,norm: Normalization<B>,- gate_act_mul: <B::Kernels as Kernels>::GatedActMulKernel,+ gate_act_mul: GatedActMul<B>,residual_finalize: <B::Kernels as Kernels>::TensorAddBiasKernel,residual_combine: <B::Kernels as Kernels>::TensorAddScaleKernel,model_dim: u32,@@ -195,7 +196,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {context,)?;- let gate_act_mul = <B::Kernels as Kernels>::GatedActMulKernel::new(context, data_type, false, false)+ let gate_act_mul = GatedActMul::full_precision(context, data_type, false, false).map_err(PerLayerEmbeddingError::BackendError)?;let residual_finalize = <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, data_type, true).map_err(PerLayerEmbeddingError::BackendError)?;@@ -233,16 +234,16 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {self.residual_finalize.encode(None::<&Allocation<B>>, hidden, &mut *outputs, length, length, encoder);- let mut gate_input = encoder.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;+ let mut gate_input = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.model_dim], self.data_type))?;encoder.encode_copy(outputs, .., &mut gate_input, ..);let gate_out = self.gate.encode(gate_input, batch_dim, encoder)?;- let mut activated = encoder.allocate_scratch_for_shape(&[batch_dim, self.ple_dim], self.data_type)?;- self.gate_act_mul.encode(+ let mut activated = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.ple_dim], self.data_type))?;+ self.gate_act_mul.encode_fp(&gate_out,Some(per_layer_input),&mut activated,- None::<&Allocation<B>>,+ None,self.ple_dim,batch_dim,layer_index * self.ple_dim,diff --git a/crates/backend-uzu/src/tests/matmul/quant.rs b/crates/backend-uzu/src/tests/matmul/quant.rsindex 22b46e5c0..4ff4f5ffe 100644--- a/crates/backend-uzu/src/tests/matmul/quant.rs+++ b/crates/backend-uzu/src/tests/matmul/quant.rs@@ -64,15 +64,13 @@ impl<T: ArrayElement + Float> QuantInput<T> {quant_method: QuantizationMethod,seed: u64,) -> Self {- let num_groups_k = k.div_ceil(group_size) as usize;- let (rows, inner, columns) = (m as usize, k as usize, n as usize);+ let num_groups_k = k.div_ceil(group_size);let mut rng = SmallRng::seed_from_u64(seed);- let w_packed: Vec<u32> =- (0..columns * inner * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect();+ let w_packed: Vec<u32> = (0..(n * k * bits / 32) as usize).map(|_| rng.random_range(0..u32::MAX)).collect();let scales: Vec<T> =- (0..columns * num_groups_k).map(|_| T::from(rng.random_range(0.01f32..0.3f32)).unwrap()).collect();- let x: Vec<T> = (0..rows * inner).map(|_| T::from(rng.random_range(-0.3f32..0.3f32)).unwrap()).collect();+ (0..(n * num_groups_k) as usize).map(|_| T::from(rng.random_range(0.01f32..0.3f32)).unwrap()).collect();+ let x: Vec<T> = (0..(m * k) as usize).map(|_| T::from(rng.random_range(-0.3f32..0.3f32)).unwrap()).collect();let zp_stride = if bits == 4 {num_groups_k.div_ceil(2)@@ -83,13 +81,13 @@ impl<T: ArrayElement + Float> QuantInput<T> {QuantizationMethod::ScaleBias => (None,Some(- (0..columns * num_groups_k)+ (0..(n * num_groups_k) as usize).map(|_| T::from(rng.random_range(-0.03f32..0.03f32)).unwrap()).collect(),),),QuantizationMethod::ScaleZeroPoint => {- (Some((0..columns * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)+ (Some((0..(n * zp_stride) as usize).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)},QuantizationMethod::ScaleSymmetric => (None, None),};@@ -131,11 +129,11 @@ impl<T: ArrayElement + Float> QuantInput<T> {let context = <Cpu as Backend>::Context::new().expect("CPU context");let input = alloc_allocation_with_data::<Cpu, T>(&context, &self.x);let factors = alloc_allocation_with_data::<Cpu, i32>(&context, &vec![1; columns as usize]);- let elements = rows as usize * columns as usize;- let mut values = alloc_allocation::<Cpu, i8>(&context, elements);- let mut scales = alloc_allocation::<Cpu, f32>(&context, elements / activation_group_size as usize);- let mut group_sums =- sum_group_size.map(|group_size| alloc_allocation::<Cpu, i32>(&context, elements / group_size as usize));+ let element_count = rows * columns;+ let mut values = alloc_allocation::<Cpu, i8>(&context, element_count as usize);+ let mut scales = alloc_allocation::<Cpu, f32>(&context, (element_count / activation_group_size) as usize);+ let mut group_sums = sum_group_size+ .map(|group_size| alloc_allocation::<Cpu, i32>(&context, (element_count / group_size) as usize));let transform =ActivationTransform::<Cpu>::quantize(&context, T::data_type(), activation_group_size, sum_group_size).expect("CPU activation quantization transform");diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rsindex f1548ae25..42e4aec4f 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs@@ -120,20 +120,21 @@ mod quantize {fn run<B: Backend>(input_data: &[f32],factors_data: &[i32],- rows: usize,- columns: usize,+ rows: u32,+ columns: u32,activation_group_size: u32,emit_group_sums: bool,sum_group_size: Option<u32>,) -> (Vec<i8>, Vec<f32>, Option<Vec<i32>>) {- let scale_groups = columns / activation_group_size as usize;- let sum_groups = sum_group_size.map_or(0, |group_size| columns / group_size as usize);+ let scale_groups = columns / activation_group_size;+ let sum_groups = sum_group_size.map_or(0, |group_size| columns / group_size);let context = B::Context::new().expect("context");let input = alloc_allocation_with_data::<B, f32>(context.as_ref(), input_data);let factors = alloc_allocation_with_data::<B, i32>(context.as_ref(), factors_data);- let mut values = alloc_allocation::<B, i8>(context.as_ref(), rows * columns);- let mut scales = alloc_allocation::<B, f32>(context.as_ref(), rows * scale_groups);- let mut group_sums = emit_group_sums.then(|| alloc_allocation::<B, i32>(context.as_ref(), rows * sum_groups));+ let mut values = alloc_allocation::<B, i8>(context.as_ref(), (rows * columns) as usize);+ let mut scales = alloc_allocation::<B, f32>(context.as_ref(), (rows * scale_groups) as usize);+ let mut group_sums =+ emit_group_sums.then(|| alloc_allocation::<B, i32>(context.as_ref(), (rows * sum_groups) as usize));let kernel =ActivationTransform::quantize(context.as_ref(), DataType::F32, activation_group_size, sum_group_size).expect("quantize transform");@@ -144,8 +145,8 @@ mod quantize {&mut scales,group_sums.as_mut(),&factors,- rows as u32,- columns as u32,+ rows,+ columns,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();@@ -205,10 +206,10 @@ mod quantize {for (group_index, (&actual, &expected)) inactual_group_sums.iter().zip(expected_group_sums.as_ref().expect("CPU group sums")).enumerate(){- let sum_group_size = sum_group_size.expect("correction group") as usize;- let start = group_index * sum_group_size;+ let sum_group_size = sum_group_size.expect("correction group");+ let start = group_index * sum_group_size as usize;let sum_from_actual_codes: i32 =- actual_values[start..start + sum_group_size].iter().copied().map(i32::from).sum();+ actual_values[start..start + sum_group_size as usize].iter().copied().map(i32::from).sum();assert_eq!(actual, sum_from_actual_codes, "group sum {group_index}");assert!((actual - expected).abs() <= sum_group_size as i32,diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/gated_act_mul_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/gated_act_mul_test.rsindex 71f7de56b..cfee331bf 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/gated_act_mul_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/gated_act_mul_test.rs@@ -7,9 +7,7 @@ use proc_macros::uzu_test;use crate::{array::ArrayElement,backends::{- common::{- Allocation, Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::GatedActMulKernel,- },+ common::{Allocation, Backend, Context, Encoder, gpu_types::ActivationType, kernel::GatedActMul},cpu::Cpu,},data_type::DataType,@@ -23,6 +21,7 @@ use crate::{struct InterleavedInput<T: ArrayElement + Float> {fused_up: Box<[T]>,+ hadamard_factors: Box<[i32]>,gated_dim: u32,batch_dim: u32,act_type: ActivationType,@@ -38,28 +37,33 @@ fn interleaved_input<T: ArrayElement + Float>(act_type: ActivationType) -> Inter}InterleavedInput {fused_up: fused_up.into_boxed_slice(),+ hadamard_factors: vec![1; gated_dim as usize].into_boxed_slice(),gated_dim,batch_dim,act_type,}}-fn run_interleaved<T: ArrayElement + Float, B: Backend>(input: &InterleavedInput<T>) -> Vec<T> {+fn run_interleaved<T: ArrayElement + Float, B: Backend>(+ input: &InterleavedInput<T>,+ use_hadamard: bool,+) -> Vec<T> {let context = B::Context::new().expect("create context");- let kernel = <<B as Backend>::Kernels as Kernels>::GatedActMulKernel::new(&context, T::data_type(), true, false)- .expect("create GatedActMulKernel");+ let kernel =+ GatedActMul::<B>::full_precision(&context, T::data_type(), true, use_hadamard).expect("create GatedActMul");let fused_length = (input.batch_dim * 2 * input.gated_dim) as usize;let output_length = (input.batch_dim * input.gated_dim) as usize;let fused_up = alloc_allocation_with_data::<B, T>(&context, &input.fused_up[..fused_length]);+ let hadamard_factors = alloc_allocation_with_data::<B, i32>(&context, &input.hadamard_factors);let mut output = alloc_allocation::<B, T>(&context, output_length);let mut encoder = Encoder::new(context.as_ref()).expect("create encoder");- kernel.encode(+ kernel.encode_fp(&fused_up,None::<&Allocation<B>>,&mut output,- None::<&Allocation<B>>,+ use_hadamard.then_some(&hadamard_factors),input.gated_dim,input.batch_dim,0,@@ -79,14 +83,24 @@ fn interleaved_test<T: ArrayElement + Float + Debug + Display>(act_type: Activat1e-5};let input = interleaved_input::<T>(act_type);- let expected = run_interleaved::<T, Cpu>(&input);+ let expected = run_interleaved::<T, Cpu>(&input, false);for_each_non_cpu_backend!(|B| {- let output = run_interleaved::<T, B>(&input);+ let output = run_interleaved::<T, B>(&input, false);let message = format!("interleaved mismatch for backend {}", std::any::type_name::<B>());assert_eq_float::<T>(&expected, &output, eps, &message);});}+#[uzu_test]+fn test_gated_act_mul_interleaved_hadamard_bf16() {+ let input = interleaved_input::<bf16>(ActivationType::SILU);+ let expected = run_interleaved::<bf16, Cpu>(&input, true);+ for_each_non_cpu_backend!(|B| {+ let actual = run_interleaved::<bf16, B>(&input, true);+ assert_eq_float::<bf16>(&expected, &actual, 0.02, "Hadamard gated activation mismatch");+ });+}+#[uzu_test]fn test_gated_act_mul_interleaved_silu_f32() {interleaved_test::<f32>(ActivationType::SILU);@@ -147,19 +161,18 @@ fn separate_input<T: ArrayElement + Float>() -> (SeparateInput<T>, Vec<T>) {fn run_separate<T: ArrayElement + Float, B: Backend>(input: &SeparateInput<T>) -> Vec<T> {let context = B::Context::new().expect("create context");- let kernel = <<B as Backend>::Kernels as Kernels>::GatedActMulKernel::new(&context, T::data_type(), false, false)- .expect("create GatedActMulKernel");+ let kernel = GatedActMul::<B>::full_precision(&context, T::data_type(), false, false).expect("create GatedActMul");let gate_out = alloc_allocation_with_data::<B, T>(&context, &input.gate_out);let per_layer_input = alloc_allocation_with_data::<B, T>(&context, &input.per_layer_input);let mut output = alloc_allocation::<B, T>(&context, (input.batch_dim * input.gated_dim) as usize);let mut encoder = Encoder::new(context.as_ref()).expect("create encoder");- kernel.encode(+ kernel.encode_fp(&gate_out,Some(&per_layer_input),&mut output,- None::<&Allocation<B>>,+ None,input.gated_dim,input.batch_dim,input.value_offset,