Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Untied readout uses LinearMatmul(A8W8/4) (#862)

Chen-Chen Yeh40cc9df7c91a
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rs
index 71c5bccf0..265d9e137 100644
--- a/crates/uzu-engine/src/encodable_block/embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding.rs
@@ -6,21 +6,22 @@ use crate::{
Allocation, Backend, Encoder, Kernels,
gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
kernel::{
- ActivationTransform, LogitTransformKernel,
+ LogitTransformKernel,
matmul::{MatmulA, MatmulArguments, MatmulDOps, MatmulKernel},
},
},
config::{
embedding::AnyEmbeddingConfig,
weight_matrix::{
- AnyWeightMatrixSpec, Layout,
+ AnyWeightMatrixSpec,
hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
},
},
data_type::DataType,
encodable_block::{
embedding_table::{EmbeddingTable, EmbeddingTableError},
- weight_matrix::{WeightMatrix, WeightMatrixError},
+ linear::{Gather, LinearMatmulError, UntiedReadout},
+ weight_matrix::WeightMatrixError,
},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -37,17 +38,8 @@ pub enum EmbeddingError<B: Backend> {
EmbeddingTable(#[from] EmbeddingTableError<B>),
#[error("Weight matrix error: {0}")]
WeightMatrix(#[from] WeightMatrixError<B>),
-}
-
-struct UntiedReadout<B: Backend> {
- matrix: WeightMatrix<B>,
- readout: Mutex<<B::Kernels as Kernels>::MatmulKernel>,
- input_hadamard: Option<InputHadamard<B>>,
-}
-
-struct InputHadamard<B: Backend> {
- factors: Allocation<B>,
- kernel: ActivationTransform<B>,
+ #[error(transparent)]
+ LinearMatmul(#[from] LinearMatmulError<B>),
}
enum EmbeddingTying<B: Backend> {
@@ -85,31 +77,6 @@ impl<B: Backend> Embedding<B> {
self.model_dim
}
- fn readout_input_hadamard(&self) -> Option<&InputHadamard<B>> {
- match &self.tying {
- EmbeddingTying::Untied {
- output,
- ..
- } => output.input_hadamard.as_ref(),
- EmbeddingTying::Tied {
- ..
- } => None,
- }
- }
-
- fn readout_operands(&self) -> (&WeightMatrix<B>, &Mutex<<B::Kernels as Kernels>::MatmulKernel>) {
- match &self.tying {
- EmbeddingTying::Tied {
- table,
- readout,
- } => (table.matrix(), readout),
- EmbeddingTying::Untied {
- output,
- ..
- } => (&output.matrix, &output.readout),
- }
- }
-
pub fn new(
context: &B::Context,
vocab_size: u32,
@@ -231,59 +198,14 @@ impl<B: Backend> Embedding<B> {
let output_embedding_tree = parameter_tree.subtree("output_embedding");
let output_embedding_spec = output_embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
- let output = match output_embedding_spec {
- AnyWeightMatrixSpec::HybridSpec(HybridSpec {
- quantization_spec,
- adapter_spec: None,
- incoherence_block_size: Some(block_size),
- incoherence_processing_mode: IncoherenceProcessingMode::Input,
- ..
- }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => {
- let matrix = WeightMatrix::load(
- &output_embedding_tree.subtree("quantized"),
- *quantization_spec,
- Layout::OutputInput,
- vocab_size,
- model_dim,
- data_type,
- )?;
-
- // Input-side incoherence is applied privately to the readout
- // input: the shared hidden state must stay untransformed
- // (e.g. for the speculator).
- let factors = output_embedding_tree
- .subtree("incoherence_signs")
- .leaf("input_signs")?
- .validate(&[model_dim], DataType::I32)?
- .read_allocation()?;
- let kernel = ActivationTransform::input_rht(context, data_type, false)
- .map_err(EmbeddingError::BackendError)?;
-
- UntiedReadout {
- matrix,
- readout: readout_kernel(context, data_type)?,
- input_hadamard: Some(InputHadamard {
- factors,
- kernel,
- }),
- }
- },
- spec => {
- let matrix = WeightMatrix::load(
- &output_embedding_tree,
- spec,
- Layout::OutputInput,
- vocab_size,
- model_dim,
- data_type,
- )?;
- UntiedReadout {
- matrix,
- readout: readout_kernel(context, data_type)?,
- input_hadamard: None,
- }
- },
- };
+ let output = UntiedReadout::load(
+ context,
+ &output_embedding_tree,
+ output_embedding_spec,
+ vocab_size,
+ model_dim,
+ data_type,
+ )?;
(
EmbeddingTying::Untied {
@@ -365,50 +287,48 @@ impl<B: Backend> Embedding<B> {
encoder.push_debug_group("embedding readout");
assert!(batch_dim > 0 && output_dim > 0, "Embedding readout requires non-empty dimensions");
- let input_hadamard = self.readout_input_hadamard();
- let mut output = encoder
- .allocate_scratch_for_shape(&[batch_dim, output_dim], self.data_type)
- .map_err(EmbeddingError::BackendError)?;
-
- let (matrix, readout) = self.readout_operands();
- let mut rht_input: Option<Allocation<B>> = None;
- let a = match input_hadamard {
- Some(input_hadamard) => {
- let mut transformed =
- encoder.allocate_scratch(input_allocation.size()).map_err(EmbeddingError::BackendError)?;
- input_hadamard.kernel.encode_fp(
- input_allocation,
- &mut transformed,
- &input_hadamard.factors,
- batch_dim,
- self.model_dim,
- encoder,
- );
- rht_input.insert(transformed)
+ let mut output_allocation = match &self.tying {
+ EmbeddingTying::Untied {
+ output,
+ ..
+ } => {
+ let gather = gather_indices.map(|indices| Gather {
+ indices,
+ output_dim,
+ });
+ output.encode(input_allocation, batch_dim, gather, encoder).map_err(EmbeddingError::BackendError)?
},
- None => input_allocation,
- };
- let arguments = MatmulArguments {
- a: MatmulA::FullPrecision {
- values: a,
- offset: 0,
+ EmbeddingTying::Tied {
+ table,
+ readout,
+ } => {
+ let mut output = encoder
+ .allocate_scratch_for_shape(&[batch_dim, output_dim], self.data_type)
+ .map_err(EmbeddingError::BackendError)?;
+ let arguments = MatmulArguments {
+ a: MatmulA::FullPrecision {
+ values: input_allocation,
+ offset: 0,
+ },
+ b: table.matrix().matmul_b(),
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut output,
+ d_transform: MatmulDOps::none(),
+ gather_indices,
+ m: batch_dim,
+ n: output_dim,
+ k: self.model_dim,
+ };
+ readout.lock().encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;
+ output
},
- b: matrix.matmul_b(),
- b_leading_dimension: None,
- b_transpose: true,
- d: &mut output,
- d_transform: MatmulDOps::none(),
- gather_indices,
- m: batch_dim,
- n: output_dim,
- k: self.model_dim,
};
- readout.lock().encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;
if apply_logit_transform && let Some(logit_transform) = &self.logit_transform {
let length = batch_dim * output_dim;
logit_transform.kernel.encode(
- &mut output,
+ &mut output_allocation,
length,
logit_transform.scale,
logit_transform.soft_cap.unwrap_or(0.0),
@@ -418,7 +338,7 @@ impl<B: Backend> Embedding<B> {
encoder.pop_debug_group();
- Ok(output)
+ Ok(output_allocation)
}
}
fn readout_kernel<B: Backend>(
diff --git a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
new file mode 100644
index 000000000..10d0111d0
--- /dev/null
+++ b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
@@ -0,0 +1,109 @@
+use std::mem::size_of;
+
+use super::{LinearInput, LinearInputPreparation};
+use crate::{
+ array::size_for_shape,
+ backends::common::{
+ Allocation, Backend, Encoder,
+ kernel::{ActivationTransform, matmul::ActivationFormat},
+ },
+ data_type::DataType,
+};
+
+pub(super) struct InputRht<B: Backend> {
+ rht_signs: Allocation<B>,
+ rht: ActivationTransform<B>,
+ quantizer: Option<ActivationTransform<B>>,
+}
+
+impl<B: Backend> InputRht<B> {
+ pub(super) fn new(
+ context: &B::Context,
+ data_type: DataType,
+ preparation: LinearInputPreparation<B>,
+ in_place: bool,
+ ) -> Result<Self, B::Error> {
+ let LinearInputPreparation {
+ rht_signs,
+ a8_plan,
+ } = preparation;
+ let rht = ActivationTransform::input_rht(context, data_type, in_place)?;
+ let quantizer = a8_plan
+ .map(|plan| {
+ ActivationTransform::quantize(context, data_type, plan.activation_group_size, plan.sum_group_size)
+ })
+ .transpose()?;
+
+ Ok(Self {
+ rht_signs,
+ rht,
+ quantizer,
+ })
+ }
+
+ pub(super) fn prepare(
+ &self,
+ input: &Allocation<B>,
+ batch_dim: u32,
+ format: ActivationFormat,
+ encoder: &mut Encoder<B>,
+ ) -> Result<LinearInput<B>, B::Error> {
+ if format == ActivationFormat::Int8
+ && let Some(quantizer) = &self.quantizer
+ {
+ let input_dim = self.input_dim();
+ let groups_per_row = input_dim.div_ceil(quantizer.activation_group_size());
+ let mut values = encoder.allocate_scratch(size_for_shape(&[batch_dim, input_dim], DataType::I8))?;
+ let mut scales = encoder.allocate_scratch(size_for_shape(&[batch_dim, groups_per_row], DataType::F32))?;
+ let mut group_sums = quantizer
+ .sum_group_size()
+ .map(|group_size| {
+ encoder
+ .allocate_scratch(size_for_shape(&[batch_dim, input_dim.div_ceil(group_size)], DataType::I32))
+ })
+ .transpose()?;
+ quantizer.encode_quantize(
+ input,
+ &mut values,
+ &mut scales,
+ group_sums.as_mut(),
+ &self.rht_signs,
+ batch_dim,
+ input_dim,
+ encoder,
+ );
+
+ return Ok(LinearInput::Int8Symmetric {
+ values,
+ scales,
+ group_sums,
+ group_size: quantizer.activation_group_size(),
+ });
+ }
+
+ let input_dim = self.input_dim();
+ let mut transformed = encoder.allocate_scratch(input.size())?;
+ self.rht.encode_fp(input, &mut transformed, &self.rht_signs, batch_dim, input_dim, encoder);
+ Ok(LinearInput::FullPrecision(transformed))
+ }
+
+ pub(super) fn prepare_in_place(
+ &self,
+ mut input: Allocation<B>,
+ batch_dim: u32,
+ format: ActivationFormat,
+ encoder: &mut Encoder<B>,
+ ) -> Result<LinearInput<B>, B::Error> {
+ if format == ActivationFormat::Int8 && self.quantizer.is_some() {
+ return self.prepare(&input, batch_dim, format, encoder);
+ }
+
+ let input_dim = self.input_dim();
+ self.rht.encode_fp_in_place(&mut input, &self.rht_signs, batch_dim, input_dim, encoder);
+ Ok(LinearInput::FullPrecision(input))
+ }
+
+ fn input_dim(&self) -> u32 {
+ (self.rht_signs.size() / size_of::<i32>()) as u32
+ }
+}
diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
index 59a5e8827..edf45acbc 100644
--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
@@ -16,7 +16,7 @@ use crate::{
config::weight_matrix::{AnyWeightMatrixSpec, Layout},
data_type::DataType,
encodable_block::{
- linear::{Linear, LinearInput},
+ linear::{Gather, Linear, LinearInput},
weight_matrix::{WeightMatrix, WeightMatrixError},
},
parameters::{ParameterLoaderError, ParameterTree},
@@ -123,11 +123,12 @@ impl<B: Backend> LinearMatmul<B> {
&self,
a: MatmulA<'_, B>,
batch_dim: u32,
+ gather: Option<Gather<'_, B>>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut output =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;
-
+ let (output_dim, gather_indices) =
+ gather.map_or((self.output_dim, None), |gather| (gather.output_dim, Some(gather.indices)));
+ let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, output_dim], self.output_data_type))?;
self.kernel.lock().encode(
MatmulArguments {
a,
@@ -136,9 +137,9 @@ impl<B: Backend> LinearMatmul<B> {
b_transpose: true,
d: &mut output,
d_transform: self.d_ops(),
- gather_indices: None,
+ gather_indices,
m: batch_dim,
- n: self.output_dim,
+ n: output_dim,
k: self.input_dim,
},
encoder,
@@ -169,18 +170,6 @@ impl<B: Backend> LinearMatmul<B> {
}
}
- pub(super) fn select_activation_format(
- &self,
- batch_dim: u32,
- context: &B::Context,
- ) -> ActivationFormat {
- if !self.matmul_b().signed_codes() {
- return ActivationFormat::Bf16;
- }
- let bf16_shape = self.matmul_shape(batch_dim, true);
- self.kernel.lock().select_activation_format(&bf16_shape, context)
- }
-
fn matmul_b(&self) -> MatmulB<'_, B> {
self.matrix.matmul_b()
}
@@ -209,6 +198,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
offset: 0,
},
batch_dim,
+ None,
encoder,
)?;
@@ -223,24 +213,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- match input {
- LinearInput::FullPrecision(input) => self.encode(input, batch_dim, encoder),
- LinearInput::Int8Symmetric {
- values,
- scales,
- group_sums,
- group_size,
- } => self.encode_with_a(
- MatmulA::Int8Symmetric {
- values: &values,
- scales: &scales,
- group_sums: group_sums.as_ref(),
- group_size,
- },
- batch_dim,
- encoder,
- ),
- }
+ self.encode_with_a(input.as_matmul_a(), batch_dim, None, encoder)
}
fn select_activation_format(
@@ -248,6 +221,10 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
batch_dim: u32,
context: &B::Context,
) -> ActivationFormat {
- Self::select_activation_format(self, batch_dim, context)
+ if !self.matmul_b().signed_codes() {
+ return ActivationFormat::Bf16;
+ }
+ let bf16_shape = self.matmul_shape(batch_dim, true);
+ self.kernel.lock().select_activation_format(&bf16_shape, context)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rs
index 333e6209a..dd156c573 100644
--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs
@@ -1,17 +1,21 @@
+use thiserror::Error;
+
+mod input_rht;
mod matmul;
mod qlora_wrapper;
mod rht_wrapper;
+mod untied_readout;
pub use matmul::{LinearMatmul, LinearMatmulError};
pub use qlora_wrapper::{QLoRALinearWrapper, QLoRALinearWrapperError};
pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};
-use thiserror::Error;
+pub use untied_readout::UntiedReadout;
use crate::{
backends::common::{
Allocation, Backend, Encoder,
gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
- kernel::matmul::{A8ActivationPlan, ActivationFormat},
+ kernel::matmul::{A8ActivationPlan, ActivationFormat, MatmulA},
},
config::weight_matrix::{
AnyWeightMatrixSpec,
@@ -64,8 +68,35 @@ pub enum LinearInput<B: Backend> {
},
}
+pub struct Gather<'a, B: Backend> {
+ pub indices: &'a Allocation<B>,
+ pub output_dim: u32,
+}
+
+impl<B: Backend> LinearInput<B> {
+ fn as_matmul_a(&self) -> MatmulA<'_, B> {
+ match self {
+ Self::FullPrecision(values) => MatmulA::FullPrecision {
+ values,
+ offset: 0,
+ },
+ Self::Int8Symmetric {
+ values,
+ scales,
+ group_sums,
+ group_size,
+ } => MatmulA::Int8Symmetric {
+ values,
+ scales,
+ group_sums: group_sums.as_ref(),
+ group_size: *group_size,
+ },
+ }
+ }
+}
+
pub struct LinearInputPreparation<B: Backend> {
- pub input_factors: Allocation<B>,
+ pub rht_signs: Allocation<B>,
pub a8_plan: Option<A8ActivationPlan>,
}
@@ -202,7 +233,7 @@ impl<B: Backend> dyn Linear<B> {
false,
parameter_tree,
)? {
- return Ok((linear.0, linear.1.map(|preparation| preparation.input_factors)));
+ return Ok((linear.0, linear.1.map(|preparation| preparation.rht_signs)));
}
let linear = Self::new_mixed_precision(
diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
index dcef5873a..df4ecb35d 100644
--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
@@ -1,21 +1,15 @@
use thiserror::Error;
use crate::{
- array::size_for_shape,
- backends::common::{
- Allocation, Backend, Encoder,
- gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
- kernel::{
- ActivationTransform,
- matmul::{A8ActivationPlan, ActivationFormat, MatmulA},
- },
- },
+ backends::common::{Allocation, Backend, Encoder, gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE},
config::weight_matrix::{
AnyWeightMatrixSpec,
hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
},
data_type::DataType,
- encodable_block::linear::{Linear, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError},
+ encodable_block::linear::{
+ Linear, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,
+ },
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -31,19 +25,9 @@ pub enum RHTLinearWrapperError<B: Backend> {
UnsupportedConfiguration(String),
}
-enum InputRht<B: Backend> {
- FullPrecision(ActivationTransform<B>),
- A8 {
- fallback: ActivationTransform<B>,
- a8: ActivationTransform<B>,
- },
-}
-
pub struct RHTLinearWrapper<B: Backend> {
input_rht: InputRht<B>,
- input_factors: Allocation<B>,
inner_linear: LinearMatmul<B>,
- input_dimension: u32,
}
fn has_input_output_rht(spec: &AnyWeightMatrixSpec) -> bool {
@@ -75,7 +59,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
return Err(RHTLinearWrapperError::UnsupportedConfiguration(format!("{spec:?}")));
}
- let (input_factors, mut inner_linear) = Self::load_inner_with_output_rht(
+ let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(
context,
input_dimension,
output_dimension,
@@ -86,7 +70,15 @@ impl<B: Backend> RHTLinearWrapper<B> {
parameter_tree,
)?;
let a8_plan = inner_linear.prepare_a8(context);
- Self::build_self_contained(context, input_dimension, input_data_type, input_factors, inner_linear, a8_plan)
+ Self::build_self_contained(
+ context,
+ input_data_type,
+ LinearInputPreparation {
+ rht_signs,
+ a8_plan,
+ },
+ inner_linear,
+ )
}
pub(super) fn try_new_with_input_preparation(
@@ -106,7 +98,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
return Ok(None);
}
- let (input_factors, mut inner_linear) = Self::load_inner_with_output_rht(
+ let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(
context,
input_dimension,
output_dimension,
@@ -116,27 +108,15 @@ impl<B: Backend> RHTLinearWrapper<B> {
weights_data_type,
parameter_tree,
)?;
- let a8_plan = inner_linear.prepare_a8(context);
- if let Some(a8_plan) = a8_plan
- && !allow_prequantized_activation
- {
- let wrapper = Self::build_self_contained(
- context,
- input_dimension,
- input_data_type,
- input_factors,
- inner_linear,
- Some(a8_plan),
- )?;
+ let input_preparation = LinearInputPreparation {
+ rht_signs,
+ a8_plan: inner_linear.prepare_a8(context),
+ };
+ if input_preparation.a8_plan.is_some() && !allow_prequantized_activation {
+ let wrapper = Self::build_self_contained(context, input_data_type, input_preparation, inner_linear)?;
Ok(Some((Box::new(wrapper), None)))
} else {
- Ok(Some((
- Box::new(inner_linear),
- Some(LinearInputPreparation {
- input_factors,
- a8_plan,
- }),
- )))
+ Ok(Some((Box::new(inner_linear), Some(input_preparation))))
}
}
@@ -153,7 +133,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
let weights_tree = parameter_tree.subtree("weights");
let quantized_weights_tree = weights_tree.subtree("quantized");
let quantization_spec = quantized_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
- let input_factors = weights_tree
+ let rht_signs = weights_tree
.leaf("incoherence_signs.input_signs")?
.validate(&[input_dimension], DataType::I32)?
.read_allocation()?;
@@ -173,41 +153,21 @@ impl<B: Backend> RHTLinearWrapper<B> {
has_biases.then_some(parameter_tree),
Some(output_factors),
)?;
- Ok((input_factors, inner_linear))
+ Ok((rht_signs, inner_linear))
}
fn build_self_contained(
context: &B::Context,
- input_dimension: u32,
input_data_type: DataType,
- input_factors: Allocation<B>,
+ input_preparation: LinearInputPreparation<B>,
inner_linear: LinearMatmul<B>,
- a8_plan: Option<A8ActivationPlan>,
) -> Result<Self, RHTLinearWrapperError<B>> {
- let input_transform = ActivationTransform::input_rht(context, input_data_type, true)
+ let input_rht = InputRht::new(context, input_data_type, input_preparation, true)
.map_err(RHTLinearWrapperError::BackendError)?;
- let input_rht = match a8_plan {
- Some(plan) => {
- let quantized = ActivationTransform::quantize(
- context,
- input_data_type,
- plan.activation_group_size,
- plan.sum_group_size,
- )
- .map_err(RHTLinearWrapperError::BackendError)?;
- InputRht::A8 {
- fallback: input_transform,
- a8: quantized,
- }
- },
- None => InputRht::FullPrecision(input_transform),
- };
Ok(Self {
input_rht,
- input_factors,
inner_linear,
- input_dimension,
})
}
}
@@ -238,60 +198,9 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
return output;
},
};
-
- if let InputRht::A8 {
- a8: quantized,
- ..
- } = &self.input_rht
- && self.inner_linear.select_activation_format(batch_dim, encoder.context()) == ActivationFormat::Int8
- {
- let activation_group_size = quantized.activation_group_size();
- let scale_groups_per_row = self.input_dimension.div_ceil(activation_group_size);
- let mut values =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dimension], DataType::I8))?;
- let mut scales =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, scale_groups_per_row], DataType::F32))?;
- let mut group_sums = quantized
- .sum_group_size()
- .map(|group_size| self.input_dimension.div_ceil(group_size))
- .map(|groups| encoder.allocate_scratch(size_for_shape(&[batch_dim, groups], DataType::I32)))
- .transpose()?;
-
- quantized.encode_quantize(
- &input,
- &mut values,
- &mut scales,
- group_sums.as_mut(),
- &self.input_factors,
- batch_dim,
- self.input_dimension,
- encoder,
- );
- let output = self.inner_linear.encode_with_a(
- MatmulA::Int8Symmetric {
- values: &values,
- scales: &scales,
- group_sums: group_sums.as_ref(),
- group_size: activation_group_size,
- },
- batch_dim,
- encoder,
- )?;
-
- encoder.pop_debug_group();
- return Ok(output);
- }
-
- let mut input = input;
- let full_precision = match &self.input_rht {
- InputRht::FullPrecision(transform)
- | InputRht::A8 {
- fallback: transform,
- ..
- } => transform,
- };
- full_precision.encode_fp_in_place(&mut input, &self.input_factors, batch_dim, self.input_dimension, encoder);
- let output = self.inner_linear.encode(input, batch_dim, encoder)?;
+ let format = self.inner_linear.select_activation_format(batch_dim, encoder.context());
+ let input = self.input_rht.prepare_in_place(input, batch_dim, format, encoder)?;
+ let output = self.inner_linear.encode_with_a(input.as_matmul_a(), batch_dim, None, encoder)?;
encoder.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
new file mode 100644
index 000000000..62e43b481
--- /dev/null
+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
@@ -0,0 +1,101 @@
+use crate::{
+ backends::common::{
+ Allocation, Backend, Encoder,
+ gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ kernel::matmul::{ActivationFormat, MatmulA},
+ },
+ config::weight_matrix::{
+ AnyWeightMatrixSpec,
+ hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
+ },
+ data_type::DataType,
+ encodable_block::linear::{
+ Gather, Linear, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,
+ },
+ parameters::ParameterTree,
+};
+
+pub struct UntiedReadout<B: Backend> {
+ linear: LinearMatmul<B>,
+ input_rht: Option<InputRht<B>>,
+}
+
+impl<B: Backend> UntiedReadout<B> {
+ pub fn load(
+ context: &B::Context,
+ tree: &ParameterTree<B>,
+ spec: AnyWeightMatrixSpec,
+ vocab_size: u32,
+ model_dim: u32,
+ data_type: DataType,
+ ) -> Result<Self, LinearMatmulError<B>> {
+ let AnyWeightMatrixSpec::HybridSpec(HybridSpec {
+ quantization_spec,
+ adapter_spec: None,
+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
+ incoherence_processing_mode: IncoherenceProcessingMode::Input,
+ ..
+ }) = spec
+ else {
+ return Ok(Self {
+ linear: LinearMatmul::load(
+ context, spec, model_dim, vocab_size, data_type, data_type, data_type, tree, None, None,
+ )?,
+ input_rht: None,
+ });
+ };
+
+ let mut linear = LinearMatmul::load(
+ context,
+ *quantization_spec,
+ model_dim,
+ vocab_size,
+ data_type,
+ data_type,
+ data_type,
+ &tree.subtree("quantized"),
+ None,
+ None,
+ )?;
+ let rht_signs = tree
+ .subtree("incoherence_signs")
+ .leaf("input_signs")?
+ .validate(&[model_dim], DataType::I32)?
+ .read_allocation()?;
+ let preparation = LinearInputPreparation {
+ rht_signs,
+ a8_plan: linear.prepare_a8(context),
+ };
+ let input_rht = InputRht::new(context, data_type, preparation, /* in_place */ false)
+ .map_err(LinearMatmulError::BackendError)?;
+
+ Ok(Self {
+ linear,
+ input_rht: Some(input_rht),
+ })
+ }
+
+ pub fn encode(
+ &self,
+ input: &Allocation<B>,
+ batch_dim: u32,
+ gather: Option<Gather<'_, B>>,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ let Some(input_rht) = &self.input_rht else {
+ let a = MatmulA::FullPrecision {
+ values: input,
+ offset: 0,
+ };
+ return self.linear.encode_with_a(a, batch_dim, gather, encoder);
+ };
+
+ let format = if gather.is_some() {
+ ActivationFormat::Bf16
+ } else {
+ self.linear.select_activation_format(batch_dim, encoder.context())
+ };
+ let input = input_rht.prepare(input, batch_dim, format, encoder)?;
+ self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, encoder)
+ }
+}
diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
index 8e58a3cbe..caee9d37c 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
@@ -33,8 +33,8 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
hidden_dim: u32,
input_preparation: Option<LinearInputPreparation<B>>,
) -> Result<Self, B::Error> {
- let (hadamard_factors, a8_plan) = input_preparation
- .map_or((None, None), |preparation| (Some(preparation.input_factors), preparation.a8_plan));
+ let (hadamard_factors, a8_plan) =
+ input_preparation.map_or((None, None), |preparation| (Some(preparation.rht_signs), preparation.a8_plan));
let settings = GatedActMulSettings {
activation_alpha: activation.custom_alpha(),
gate_clipping,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
index b8b16e790..1a98329ec 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
@@ -22,7 +22,7 @@ enum TransformOrder {
Output,
}
-fn run<T: ArrayElement + Float, B: Backend>(
+fn run<T: ArrayElement + Float + Debug, B: Backend>(
data: &[T],
factors: &[i32],
channel_count: usize,
@@ -47,6 +47,9 @@ fn run<T: ArrayElement + Float, B: Backend>(
kernel.encode_fp(&input, &mut output, &factors, batch_count, channel_count as u32, &mut encoder);
}
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ if !in_place {
+ assert_eq!(allocation_to_vec::<B, T>(&input), data);
+ }
allocation_to_vec(if in_place {
&input
} else {
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
index 3ffdf8234..82a918d9c 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
@@ -235,14 +235,18 @@ fn gemv_gather() {
fp_gather_case::<f32>(soft_cap, 0.01);
}
// Quantized (bf16, per bits/method) — inline, since it isn't type-generic.
- for (bits, method) in [
- (4, QuantizationMethod::ScaleBias),
- (4, QuantizationMethod::ScaleZeroPoint),
- (4, QuantizationMethod::ScaleSymmetric),
- (8, QuantizationMethod::ScaleZeroPoint),
+ for (bits, method, signed_codes) in [
+ (4, QuantizationMethod::ScaleBias, false),
+ (4, QuantizationMethod::ScaleZeroPoint, false),
+ (4, QuantizationMethod::ScaleZeroPoint, true),
+ (4, QuantizationMethod::ScaleSymmetric, false),
+ (8, QuantizationMethod::ScaleZeroPoint, false),
] {
let (m, k, vocab, ids_per_row, group_size) = (8usize, 128usize, 64usize, 8usize, 32u32);
- let input = QuantInput::<bf16>::new(m as u32, k as u32, vocab as u32, group_size, bits, method, 0x5EED);
+ let mut input = QuantInput::<bf16>::new(m as u32, k as u32, vocab as u32, group_size, bits, method, 0x5EED);
+ if signed_codes {
+ input = input.with_signed_weight_codes();
+ }
let ids: Vec<u32> = (0..m * ids_per_row).map(|i| ((i * 37 + 11) % vocab) as u32).collect();
// K_SPLIT == 1 keeps k in one reduction, so gather and dense share the exact accumulation.
check_gather!(m, vocab, ids, ids_per_row, 0.05, |B| {