A high-performance inference engine for AI models
Untied readout uses LinearMatmul(A8W8/4) (#862)
Chen-Chen Yeh40cc9df7c91a
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rsindex 71c5bccf0..265d9e137 100644--- a/crates/uzu-engine/src/encodable_block/embedding.rs+++ b/crates/uzu-engine/src/encodable_block/embedding.rs@@ -6,21 +6,22 @@ use crate::{Allocation, Backend, Encoder, Kernels,gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,kernel::{- ActivationTransform, LogitTransformKernel,+ LogitTransformKernel,matmul::{MatmulA, MatmulArguments, MatmulDOps, MatmulKernel},},},config::{embedding::AnyEmbeddingConfig,weight_matrix::{- AnyWeightMatrixSpec, Layout,+ AnyWeightMatrixSpec,hybrid_spec::{HybridSpec, IncoherenceProcessingMode},},},data_type::DataType,encodable_block::{embedding_table::{EmbeddingTable, EmbeddingTableError},- weight_matrix::{WeightMatrix, WeightMatrixError},+ linear::{Gather, LinearMatmulError, UntiedReadout},+ weight_matrix::WeightMatrixError,},parameters::{ParameterLoaderError, ParameterTree},};@@ -37,17 +38,8 @@ pub enum EmbeddingError<B: Backend> {EmbeddingTable(#[from] EmbeddingTableError<B>),#[error("Weight matrix error: {0}")]WeightMatrix(#[from] WeightMatrixError<B>),-}--struct UntiedReadout<B: Backend> {- matrix: WeightMatrix<B>,- readout: Mutex<<B::Kernels as Kernels>::MatmulKernel>,- input_hadamard: Option<InputHadamard<B>>,-}--struct InputHadamard<B: Backend> {- factors: Allocation<B>,- kernel: ActivationTransform<B>,+ #[error(transparent)]+ LinearMatmul(#[from] LinearMatmulError<B>),}enum EmbeddingTying<B: Backend> {@@ -85,31 +77,6 @@ impl<B: Backend> Embedding<B> {self.model_dim}- fn readout_input_hadamard(&self) -> Option<&InputHadamard<B>> {- match &self.tying {- EmbeddingTying::Untied {- output,- ..- } => output.input_hadamard.as_ref(),- EmbeddingTying::Tied {- ..- } => None,- }- }-- fn readout_operands(&self) -> (&WeightMatrix<B>, &Mutex<<B::Kernels as Kernels>::MatmulKernel>) {- match &self.tying {- EmbeddingTying::Tied {- table,- readout,- } => (table.matrix(), readout),- EmbeddingTying::Untied {- output,- ..- } => (&output.matrix, &output.readout),- }- }-pub fn new(context: &B::Context,vocab_size: u32,@@ -231,59 +198,14 @@ impl<B: Backend> Embedding<B> {let output_embedding_tree = parameter_tree.subtree("output_embedding");let output_embedding_spec = output_embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;- let output = match output_embedding_spec {- AnyWeightMatrixSpec::HybridSpec(HybridSpec {- quantization_spec,- adapter_spec: None,- incoherence_block_size: Some(block_size),- incoherence_processing_mode: IncoherenceProcessingMode::Input,- ..- }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => {- let matrix = WeightMatrix::load(- &output_embedding_tree.subtree("quantized"),- *quantization_spec,- Layout::OutputInput,- vocab_size,- model_dim,- data_type,- )?;-- // Input-side incoherence is applied privately to the readout- // input: the shared hidden state must stay untransformed- // (e.g. for the speculator).- let factors = output_embedding_tree- .subtree("incoherence_signs")- .leaf("input_signs")?- .validate(&[model_dim], DataType::I32)?- .read_allocation()?;- let kernel = ActivationTransform::input_rht(context, data_type, false)- .map_err(EmbeddingError::BackendError)?;-- UntiedReadout {- matrix,- readout: readout_kernel(context, data_type)?,- input_hadamard: Some(InputHadamard {- factors,- kernel,- }),- }- },- spec => {- let matrix = WeightMatrix::load(- &output_embedding_tree,- spec,- Layout::OutputInput,- vocab_size,- model_dim,- data_type,- )?;- UntiedReadout {- matrix,- readout: readout_kernel(context, data_type)?,- input_hadamard: None,- }- },- };+ let output = UntiedReadout::load(+ context,+ &output_embedding_tree,+ output_embedding_spec,+ vocab_size,+ model_dim,+ data_type,+ )?;(EmbeddingTying::Untied {@@ -365,50 +287,48 @@ impl<B: Backend> Embedding<B> {encoder.push_debug_group("embedding readout");assert!(batch_dim > 0 && output_dim > 0, "Embedding readout requires non-empty dimensions");- let input_hadamard = self.readout_input_hadamard();- let mut output = encoder- .allocate_scratch_for_shape(&[batch_dim, output_dim], self.data_type)- .map_err(EmbeddingError::BackendError)?;-- let (matrix, readout) = self.readout_operands();- let mut rht_input: Option<Allocation<B>> = None;- let a = match input_hadamard {- Some(input_hadamard) => {- let mut transformed =- encoder.allocate_scratch(input_allocation.size()).map_err(EmbeddingError::BackendError)?;- input_hadamard.kernel.encode_fp(- input_allocation,- &mut transformed,- &input_hadamard.factors,- batch_dim,- self.model_dim,- encoder,- );- rht_input.insert(transformed)+ let mut output_allocation = match &self.tying {+ EmbeddingTying::Untied {+ output,+ ..+ } => {+ let gather = gather_indices.map(|indices| Gather {+ indices,+ output_dim,+ });+ output.encode(input_allocation, batch_dim, gather, encoder).map_err(EmbeddingError::BackendError)?},- None => input_allocation,- };- let arguments = MatmulArguments {- a: MatmulA::FullPrecision {- values: a,- offset: 0,+ EmbeddingTying::Tied {+ table,+ readout,+ } => {+ let mut output = encoder+ .allocate_scratch_for_shape(&[batch_dim, output_dim], self.data_type)+ .map_err(EmbeddingError::BackendError)?;+ let arguments = MatmulArguments {+ a: MatmulA::FullPrecision {+ values: input_allocation,+ offset: 0,+ },+ b: table.matrix().matmul_b(),+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut output,+ d_transform: MatmulDOps::none(),+ gather_indices,+ m: batch_dim,+ n: output_dim,+ k: self.model_dim,+ };+ readout.lock().encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;+ output},- b: matrix.matmul_b(),- b_leading_dimension: None,- b_transpose: true,- d: &mut output,- d_transform: MatmulDOps::none(),- gather_indices,- m: batch_dim,- n: output_dim,- k: self.model_dim,};- readout.lock().encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;if apply_logit_transform && let Some(logit_transform) = &self.logit_transform {let length = batch_dim * output_dim;logit_transform.kernel.encode(- &mut output,+ &mut output_allocation,length,logit_transform.scale,logit_transform.soft_cap.unwrap_or(0.0),@@ -418,7 +338,7 @@ impl<B: Backend> Embedding<B> {encoder.pop_debug_group();- Ok(output)+ Ok(output_allocation)}}fn readout_kernel<B: Backend>(diff --git a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs b/crates/uzu-engine/src/encodable_block/linear/input_rht.rsnew file mode 100644index 000000000..10d0111d0--- /dev/null+++ b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs@@ -0,0 +1,109 @@+use std::mem::size_of;++use super::{LinearInput, LinearInputPreparation};+use crate::{+ array::size_for_shape,+ backends::common::{+ Allocation, Backend, Encoder,+ kernel::{ActivationTransform, matmul::ActivationFormat},+ },+ data_type::DataType,+};++pub(super) struct InputRht<B: Backend> {+ rht_signs: Allocation<B>,+ rht: ActivationTransform<B>,+ quantizer: Option<ActivationTransform<B>>,+}++impl<B: Backend> InputRht<B> {+ pub(super) fn new(+ context: &B::Context,+ data_type: DataType,+ preparation: LinearInputPreparation<B>,+ in_place: bool,+ ) -> Result<Self, B::Error> {+ let LinearInputPreparation {+ rht_signs,+ a8_plan,+ } = preparation;+ let rht = ActivationTransform::input_rht(context, data_type, in_place)?;+ let quantizer = a8_plan+ .map(|plan| {+ ActivationTransform::quantize(context, data_type, plan.activation_group_size, plan.sum_group_size)+ })+ .transpose()?;++ Ok(Self {+ rht_signs,+ rht,+ quantizer,+ })+ }++ pub(super) fn prepare(+ &self,+ input: &Allocation<B>,+ batch_dim: u32,+ format: ActivationFormat,+ encoder: &mut Encoder<B>,+ ) -> Result<LinearInput<B>, B::Error> {+ if format == ActivationFormat::Int8+ && let Some(quantizer) = &self.quantizer+ {+ let input_dim = self.input_dim();+ let groups_per_row = input_dim.div_ceil(quantizer.activation_group_size());+ let mut values = encoder.allocate_scratch(size_for_shape(&[batch_dim, input_dim], DataType::I8))?;+ let mut scales = encoder.allocate_scratch(size_for_shape(&[batch_dim, groups_per_row], DataType::F32))?;+ let mut group_sums = quantizer+ .sum_group_size()+ .map(|group_size| {+ encoder+ .allocate_scratch(size_for_shape(&[batch_dim, input_dim.div_ceil(group_size)], DataType::I32))+ })+ .transpose()?;+ quantizer.encode_quantize(+ input,+ &mut values,+ &mut scales,+ group_sums.as_mut(),+ &self.rht_signs,+ batch_dim,+ input_dim,+ encoder,+ );++ return Ok(LinearInput::Int8Symmetric {+ values,+ scales,+ group_sums,+ group_size: quantizer.activation_group_size(),+ });+ }++ let input_dim = self.input_dim();+ let mut transformed = encoder.allocate_scratch(input.size())?;+ self.rht.encode_fp(input, &mut transformed, &self.rht_signs, batch_dim, input_dim, encoder);+ Ok(LinearInput::FullPrecision(transformed))+ }++ pub(super) fn prepare_in_place(+ &self,+ mut input: Allocation<B>,+ batch_dim: u32,+ format: ActivationFormat,+ encoder: &mut Encoder<B>,+ ) -> Result<LinearInput<B>, B::Error> {+ if format == ActivationFormat::Int8 && self.quantizer.is_some() {+ return self.prepare(&input, batch_dim, format, encoder);+ }++ let input_dim = self.input_dim();+ self.rht.encode_fp_in_place(&mut input, &self.rht_signs, batch_dim, input_dim, encoder);+ Ok(LinearInput::FullPrecision(input))+ }++ fn input_dim(&self) -> u32 {+ (self.rht_signs.size() / size_of::<i32>()) as u32+ }+}diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rsindex 59a5e8827..edf45acbc 100644--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs@@ -16,7 +16,7 @@ use crate::{config::weight_matrix::{AnyWeightMatrixSpec, Layout},data_type::DataType,encodable_block::{- linear::{Linear, LinearInput},+ linear::{Gather, Linear, LinearInput},weight_matrix::{WeightMatrix, WeightMatrixError},},parameters::{ParameterLoaderError, ParameterTree},@@ -123,11 +123,12 @@ impl<B: Backend> LinearMatmul<B> {&self,a: MatmulA<'_, B>,batch_dim: u32,+ gather: Option<Gather<'_, B>>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let mut output =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;-+ let (output_dim, gather_indices) =+ gather.map_or((self.output_dim, None), |gather| (gather.output_dim, Some(gather.indices)));+ let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, output_dim], self.output_data_type))?;self.kernel.lock().encode(MatmulArguments {a,@@ -136,9 +137,9 @@ impl<B: Backend> LinearMatmul<B> {b_transpose: true,d: &mut output,d_transform: self.d_ops(),- gather_indices: None,+ gather_indices,m: batch_dim,- n: self.output_dim,+ n: output_dim,k: self.input_dim,},encoder,@@ -169,18 +170,6 @@ impl<B: Backend> LinearMatmul<B> {}}- pub(super) fn select_activation_format(- &self,- batch_dim: u32,- context: &B::Context,- ) -> ActivationFormat {- if !self.matmul_b().signed_codes() {- return ActivationFormat::Bf16;- }- let bf16_shape = self.matmul_shape(batch_dim, true);- self.kernel.lock().select_activation_format(&bf16_shape, context)- }-fn matmul_b(&self) -> MatmulB<'_, B> {self.matrix.matmul_b()}@@ -209,6 +198,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {offset: 0,},batch_dim,+ None,encoder,)?;@@ -223,24 +213,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- match input {- LinearInput::FullPrecision(input) => self.encode(input, batch_dim, encoder),- LinearInput::Int8Symmetric {- values,- scales,- group_sums,- group_size,- } => self.encode_with_a(- MatmulA::Int8Symmetric {- values: &values,- scales: &scales,- group_sums: group_sums.as_ref(),- group_size,- },- batch_dim,- encoder,- ),- }+ self.encode_with_a(input.as_matmul_a(), batch_dim, None, encoder)}fn select_activation_format(@@ -248,6 +221,10 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {batch_dim: u32,context: &B::Context,) -> ActivationFormat {- Self::select_activation_format(self, batch_dim, context)+ if !self.matmul_b().signed_codes() {+ return ActivationFormat::Bf16;+ }+ let bf16_shape = self.matmul_shape(batch_dim, true);+ self.kernel.lock().select_activation_format(&bf16_shape, context)}}diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rsindex 333e6209a..dd156c573 100644--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs@@ -1,17 +1,21 @@+use thiserror::Error;++mod input_rht;mod matmul;mod qlora_wrapper;mod rht_wrapper;+mod untied_readout;pub use matmul::{LinearMatmul, LinearMatmulError};pub use qlora_wrapper::{QLoRALinearWrapper, QLoRALinearWrapperError};pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};-use thiserror::Error;+pub use untied_readout::UntiedReadout;use crate::{backends::common::{Allocation, Backend, Encoder,gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,- kernel::matmul::{A8ActivationPlan, ActivationFormat},+ kernel::matmul::{A8ActivationPlan, ActivationFormat, MatmulA},},config::weight_matrix::{AnyWeightMatrixSpec,@@ -64,8 +68,35 @@ pub enum LinearInput<B: Backend> {},}+pub struct Gather<'a, B: Backend> {+ pub indices: &'a Allocation<B>,+ pub output_dim: u32,+}++impl<B: Backend> LinearInput<B> {+ fn as_matmul_a(&self) -> MatmulA<'_, B> {+ match self {+ Self::FullPrecision(values) => MatmulA::FullPrecision {+ values,+ offset: 0,+ },+ Self::Int8Symmetric {+ values,+ scales,+ group_sums,+ group_size,+ } => MatmulA::Int8Symmetric {+ values,+ scales,+ group_sums: group_sums.as_ref(),+ group_size: *group_size,+ },+ }+ }+}+pub struct LinearInputPreparation<B: Backend> {- pub input_factors: Allocation<B>,+ pub rht_signs: Allocation<B>,pub a8_plan: Option<A8ActivationPlan>,}@@ -202,7 +233,7 @@ impl<B: Backend> dyn Linear<B> {false,parameter_tree,)? {- return Ok((linear.0, linear.1.map(|preparation| preparation.input_factors)));+ return Ok((linear.0, linear.1.map(|preparation| preparation.rht_signs)));}let linear = Self::new_mixed_precision(diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rsindex dcef5873a..df4ecb35d 100644--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs@@ -1,21 +1,15 @@use thiserror::Error;use crate::{- array::size_for_shape,- backends::common::{- Allocation, Backend, Encoder,- gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,- kernel::{- ActivationTransform,- matmul::{A8ActivationPlan, ActivationFormat, MatmulA},- },- },+ backends::common::{Allocation, Backend, Encoder, gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE},config::weight_matrix::{AnyWeightMatrixSpec,hybrid_spec::{HybridSpec, IncoherenceProcessingMode},},data_type::DataType,- encodable_block::linear::{Linear, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError},+ encodable_block::linear::{+ Linear, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,+ },parameters::{ParameterLoaderError, ParameterTree},};@@ -31,19 +25,9 @@ pub enum RHTLinearWrapperError<B: Backend> {UnsupportedConfiguration(String),}-enum InputRht<B: Backend> {- FullPrecision(ActivationTransform<B>),- A8 {- fallback: ActivationTransform<B>,- a8: ActivationTransform<B>,- },-}-pub struct RHTLinearWrapper<B: Backend> {input_rht: InputRht<B>,- input_factors: Allocation<B>,inner_linear: LinearMatmul<B>,- input_dimension: u32,}fn has_input_output_rht(spec: &AnyWeightMatrixSpec) -> bool {@@ -75,7 +59,7 @@ impl<B: Backend> RHTLinearWrapper<B> {return Err(RHTLinearWrapperError::UnsupportedConfiguration(format!("{spec:?}")));}- let (input_factors, mut inner_linear) = Self::load_inner_with_output_rht(+ let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(context,input_dimension,output_dimension,@@ -86,7 +70,15 @@ impl<B: Backend> RHTLinearWrapper<B> {parameter_tree,)?;let a8_plan = inner_linear.prepare_a8(context);- Self::build_self_contained(context, input_dimension, input_data_type, input_factors, inner_linear, a8_plan)+ Self::build_self_contained(+ context,+ input_data_type,+ LinearInputPreparation {+ rht_signs,+ a8_plan,+ },+ inner_linear,+ )}pub(super) fn try_new_with_input_preparation(@@ -106,7 +98,7 @@ impl<B: Backend> RHTLinearWrapper<B> {return Ok(None);}- let (input_factors, mut inner_linear) = Self::load_inner_with_output_rht(+ let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(context,input_dimension,output_dimension,@@ -116,27 +108,15 @@ impl<B: Backend> RHTLinearWrapper<B> {weights_data_type,parameter_tree,)?;- let a8_plan = inner_linear.prepare_a8(context);- if let Some(a8_plan) = a8_plan- && !allow_prequantized_activation- {- let wrapper = Self::build_self_contained(- context,- input_dimension,- input_data_type,- input_factors,- inner_linear,- Some(a8_plan),- )?;+ let input_preparation = LinearInputPreparation {+ rht_signs,+ a8_plan: inner_linear.prepare_a8(context),+ };+ if input_preparation.a8_plan.is_some() && !allow_prequantized_activation {+ let wrapper = Self::build_self_contained(context, input_data_type, input_preparation, inner_linear)?;Ok(Some((Box::new(wrapper), None)))} else {- Ok(Some((- Box::new(inner_linear),- Some(LinearInputPreparation {- input_factors,- a8_plan,- }),- )))+ Ok(Some((Box::new(inner_linear), Some(input_preparation))))}}@@ -153,7 +133,7 @@ impl<B: Backend> RHTLinearWrapper<B> {let weights_tree = parameter_tree.subtree("weights");let quantized_weights_tree = weights_tree.subtree("quantized");let quantization_spec = quantized_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;- let input_factors = weights_tree+ let rht_signs = weights_tree.leaf("incoherence_signs.input_signs")?.validate(&[input_dimension], DataType::I32)?.read_allocation()?;@@ -173,41 +153,21 @@ impl<B: Backend> RHTLinearWrapper<B> {has_biases.then_some(parameter_tree),Some(output_factors),)?;- Ok((input_factors, inner_linear))+ Ok((rht_signs, inner_linear))}fn build_self_contained(context: &B::Context,- input_dimension: u32,input_data_type: DataType,- input_factors: Allocation<B>,+ input_preparation: LinearInputPreparation<B>,inner_linear: LinearMatmul<B>,- a8_plan: Option<A8ActivationPlan>,) -> Result<Self, RHTLinearWrapperError<B>> {- let input_transform = ActivationTransform::input_rht(context, input_data_type, true)+ let input_rht = InputRht::new(context, input_data_type, input_preparation, true).map_err(RHTLinearWrapperError::BackendError)?;- let input_rht = match a8_plan {- Some(plan) => {- let quantized = ActivationTransform::quantize(- context,- input_data_type,- plan.activation_group_size,- plan.sum_group_size,- )- .map_err(RHTLinearWrapperError::BackendError)?;- InputRht::A8 {- fallback: input_transform,- a8: quantized,- }- },- None => InputRht::FullPrecision(input_transform),- };Ok(Self {input_rht,- input_factors,inner_linear,- input_dimension,})}}@@ -238,60 +198,9 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {return output;},};-- if let InputRht::A8 {- a8: quantized,- ..- } = &self.input_rht- && self.inner_linear.select_activation_format(batch_dim, encoder.context()) == ActivationFormat::Int8- {- let activation_group_size = quantized.activation_group_size();- let scale_groups_per_row = self.input_dimension.div_ceil(activation_group_size);- let mut values =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dimension], DataType::I8))?;- let mut scales =- encoder.allocate_scratch(size_for_shape(&[batch_dim, scale_groups_per_row], DataType::F32))?;- let mut group_sums = quantized- .sum_group_size()- .map(|group_size| self.input_dimension.div_ceil(group_size))- .map(|groups| encoder.allocate_scratch(size_for_shape(&[batch_dim, groups], DataType::I32)))- .transpose()?;-- quantized.encode_quantize(- &input,- &mut values,- &mut scales,- group_sums.as_mut(),- &self.input_factors,- batch_dim,- self.input_dimension,- encoder,- );- let output = self.inner_linear.encode_with_a(- MatmulA::Int8Symmetric {- values: &values,- scales: &scales,- group_sums: group_sums.as_ref(),- group_size: activation_group_size,- },- batch_dim,- encoder,- )?;-- encoder.pop_debug_group();- return Ok(output);- }-- let mut input = input;- let full_precision = match &self.input_rht {- InputRht::FullPrecision(transform)- | InputRht::A8 {- fallback: transform,- ..- } => transform,- };- full_precision.encode_fp_in_place(&mut input, &self.input_factors, batch_dim, self.input_dimension, encoder);- let output = self.inner_linear.encode(input, batch_dim, encoder)?;+ let format = self.inner_linear.select_activation_format(batch_dim, encoder.context());+ let input = self.input_rht.prepare_in_place(input, batch_dim, format, encoder)?;+ let output = self.inner_linear.encode_with_a(input.as_matmul_a(), batch_dim, None, encoder)?;encoder.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rsnew file mode 100644index 000000000..62e43b481--- /dev/null+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs@@ -0,0 +1,101 @@+use crate::{+ backends::common::{+ Allocation, Backend, Encoder,+ gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,+ kernel::matmul::{ActivationFormat, MatmulA},+ },+ config::weight_matrix::{+ AnyWeightMatrixSpec,+ hybrid_spec::{HybridSpec, IncoherenceProcessingMode},+ },+ data_type::DataType,+ encodable_block::linear::{+ Gather, Linear, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,+ },+ parameters::ParameterTree,+};++pub struct UntiedReadout<B: Backend> {+ linear: LinearMatmul<B>,+ input_rht: Option<InputRht<B>>,+}++impl<B: Backend> UntiedReadout<B> {+ pub fn load(+ context: &B::Context,+ tree: &ParameterTree<B>,+ spec: AnyWeightMatrixSpec,+ vocab_size: u32,+ model_dim: u32,+ data_type: DataType,+ ) -> Result<Self, LinearMatmulError<B>> {+ let AnyWeightMatrixSpec::HybridSpec(HybridSpec {+ quantization_spec,+ adapter_spec: None,+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),+ incoherence_processing_mode: IncoherenceProcessingMode::Input,+ ..+ }) = spec+ else {+ return Ok(Self {+ linear: LinearMatmul::load(+ context, spec, model_dim, vocab_size, data_type, data_type, data_type, tree, None, None,+ )?,+ input_rht: None,+ });+ };++ let mut linear = LinearMatmul::load(+ context,+ *quantization_spec,+ model_dim,+ vocab_size,+ data_type,+ data_type,+ data_type,+ &tree.subtree("quantized"),+ None,+ None,+ )?;+ let rht_signs = tree+ .subtree("incoherence_signs")+ .leaf("input_signs")?+ .validate(&[model_dim], DataType::I32)?+ .read_allocation()?;+ let preparation = LinearInputPreparation {+ rht_signs,+ a8_plan: linear.prepare_a8(context),+ };+ let input_rht = InputRht::new(context, data_type, preparation, /* in_place */ false)+ .map_err(LinearMatmulError::BackendError)?;++ Ok(Self {+ linear,+ input_rht: Some(input_rht),+ })+ }++ pub fn encode(+ &self,+ input: &Allocation<B>,+ batch_dim: u32,+ gather: Option<Gather<'_, B>>,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ let Some(input_rht) = &self.input_rht else {+ let a = MatmulA::FullPrecision {+ values: input,+ offset: 0,+ };+ return self.linear.encode_with_a(a, batch_dim, gather, encoder);+ };++ let format = if gather.is_some() {+ ActivationFormat::Bf16+ } else {+ self.linear.select_activation_format(batch_dim, encoder.context())+ };+ let input = input_rht.prepare(input, batch_dim, format, encoder)?;+ self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, encoder)+ }+}diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rsindex 8e58a3cbe..caee9d37c 100644--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs@@ -33,8 +33,8 @@ impl<B: Backend> MlpGateActMulEncodable<B> {hidden_dim: u32,input_preparation: Option<LinearInputPreparation<B>>,) -> Result<Self, B::Error> {- let (hadamard_factors, a8_plan) = input_preparation- .map_or((None, None), |preparation| (Some(preparation.input_factors), preparation.a8_plan));+ let (hadamard_factors, a8_plan) =+ input_preparation.map_or((None, None), |preparation| (Some(preparation.rht_signs), preparation.a8_plan));let settings = GatedActMulSettings {activation_alpha: activation.custom_alpha(),gate_clipping,diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rsindex b8b16e790..1a98329ec 100644--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs@@ -22,7 +22,7 @@ enum TransformOrder {Output,}-fn run<T: ArrayElement + Float, B: Backend>(+fn run<T: ArrayElement + Float + Debug, B: Backend>(data: &[T],factors: &[i32],channel_count: usize,@@ -47,6 +47,9 @@ fn run<T: ArrayElement + Float, B: Backend>(kernel.encode_fp(&input, &mut output, &factors, batch_count, channel_count as u32, &mut encoder);}encoder.end_encoding().submit().wait_until_completed().unwrap();+ if !in_place {+ assert_eq!(allocation_to_vec::<B, T>(&input), data);+ }allocation_to_vec(if in_place {&input} else {diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rsindex 3ffdf8234..82a918d9c 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs@@ -235,14 +235,18 @@ fn gemv_gather() {fp_gather_case::<f32>(soft_cap, 0.01);}// Quantized (bf16, per bits/method) — inline, since it isn't type-generic.- for (bits, method) in [- (4, QuantizationMethod::ScaleBias),- (4, QuantizationMethod::ScaleZeroPoint),- (4, QuantizationMethod::ScaleSymmetric),- (8, QuantizationMethod::ScaleZeroPoint),+ for (bits, method, signed_codes) in [+ (4, QuantizationMethod::ScaleBias, false),+ (4, QuantizationMethod::ScaleZeroPoint, false),+ (4, QuantizationMethod::ScaleZeroPoint, true),+ (4, QuantizationMethod::ScaleSymmetric, false),+ (8, QuantizationMethod::ScaleZeroPoint, false),] {let (m, k, vocab, ids_per_row, group_size) = (8usize, 128usize, 64usize, 8usize, 32u32);- let input = QuantInput::<bf16>::new(m as u32, k as u32, vocab as u32, group_size, bits, method, 0x5EED);+ let mut input = QuantInput::<bf16>::new(m as u32, k as u32, vocab as u32, group_size, bits, method, 0x5EED);+ if signed_codes {+ input = input.with_signed_weight_codes();+ }let ids: Vec<u32> = (0..m * ids_per_row).map(|i| ((i * 37 + 11) % vocab) as u32).collect();// K_SPLIT == 1 keeps k in one reduction, so gather and dense share the exact accumulation.check_gather!(m, vocab, ids, ids_per_row, 0.05, |B| {