A high-performance inference engine for AI models
QLoRA (#297)
uuuvn3a39be8d0712
diff --git a/crates/uzu/src/encodable_block/linear/mod.rs b/crates/uzu/src/encodable_block/linear/mod.rsindex 7d8478fc6..de530a9e4 100644--- a/crates/uzu/src/encodable_block/linear/mod.rs+++ b/crates/uzu/src/encodable_block/linear/mod.rs@@ -1,7 +1,9 @@mod full_precision;+mod qlora_wrapper;mod quantized;pub use full_precision::{FullPrecisionLinear, FullPrecisionLinearError};+pub use qlora_wrapper::{QLoRALinearWrapper, QLoRALinearWrapperError};pub use quantized::{QuantizedLinear, QuantizedLinearError};use thiserror::Error;@@ -23,11 +25,11 @@ pub trait Linear<B: Backend> {#[derive(Debug, Error)]pub enum LinearBlockError<B: Backend> {#[error("QuantizedLinear error: {0}")]- QuantizedLinearError(#[source] QuantizedLinearError<B>),+ QuantizedLinearError(#[from] QuantizedLinearError<B>),#[error("FullPrecisionLinear error: {0}")]- FullPrecisionLinearError(#[source] FullPrecisionLinearError<B>),- #[error("QLoRA linear layer not supported")]- QLoRaNotSupported,+ FullPrecisionLinearError(#[from] FullPrecisionLinearError<B>),+ #[error("QLoRALinearWrapper error: {0}")]+ QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),}impl<B: Backend> dyn Linear<B> {@@ -52,8 +54,7 @@ impl<B: Backend> dyn Linear<B> {parameter_tree,input_array_id,output_array_id,- )- .map_err(LinearBlockError::QuantizedLinearError)?;+ )?;Ok(Box::new(block))},LinearConfig::FullPrecision {@@ -67,13 +68,27 @@ impl<B: Backend> dyn Linear<B> {parameter_tree,input_array_id,output_array_id,- )- .map_err(LinearBlockError::FullPrecisionLinearError)?;+ )?;Ok(Box::new(block))},LinearConfig::QLoRA {- ..- } => Err(LinearBlockError::QLoRaNotSupported),+ quantization,+ lora_rank,+ lora_scale,+ } => {+ let block = QLoRALinearWrapper::new(+ context,+ quantization,+ *lora_rank,+ *lora_scale,+ input_dimension,+ output_dimension_sum,+ parameter_tree,+ input_array_id,+ output_array_id,+ )?;+ Ok(Box::new(block))+ },}}}diff --git a/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rsnew file mode 100644index 000000000..3552c1564--- /dev/null+++ b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -0,0 +1,177 @@+use std::{+ cell::RefCell,+ ops::{Deref, DerefMut},+};++use thiserror::Error;++use crate::{+ DataType,+ backends::common::{+ Backend, Encoder,+ kernel::{+ ManualKernels,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},+ },+ },+ config::QuantizationConfig,+ encodable_block::{+ Linear,+ linear::{LinearBlockError, QuantizedLinear, QuantizedLinearError},+ },+ forward_pass::state::{ArrayId, ForwardPassState},+ prelude::{ParameterLeaf, ParameterLoaderError, ParameterTree},+};++#[derive(Debug, Error)]+pub enum QLoRALinearWrapperError<B: Backend> {+ #[error("Inner linear error: {0}")]+ InnerLinearError(#[from] Box<LinearBlockError<B>>),+ #[error("Quantized linear error: {0}")]+ QuantizedLinearError(#[from] QuantizedLinearError<B>),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[from] ParameterLoaderError<B>),+ #[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]+ InvalidTensor {+ shape: Box<[usize]>,+ data_type: DataType,+ expected_shape: Box<[usize]>,+ expected_data_type: DataType,+ },+ #[error("Matmul error: {0}")]+ MatmulError(#[from] MatmulError<B>),+}++pub struct QLoRALinearWrapper<B: Backend> {+ base_linear: QuantizedLinear<B>,+ adapter_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,+ adapter_down: B::Buffer,+ adapter_up: B::Buffer,+ input_dim: usize,+ output_dim: usize,+ lora_rank: usize,+ lora_scale: f32,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+}++// TODO: figure out how to make this generic over QLoRAWrapperError::InvalidTensor or make one global "Invalid Tensor" error and make this a common helper+fn validate_tensor<'file, 'context, 'leaf, B: Backend>(+ weights_leaf: &ParameterLeaf<'file, 'context, 'leaf, B::Context>,+ expected_shape: [usize; 2],+ expected_data_type: DataType,+) -> Result<(), QLoRALinearWrapperError<B>> {+ let shape = weights_leaf.shape();+ let data_type = weights_leaf.data_type();++ if (shape, data_type) != (expected_shape.as_ref(), expected_data_type) {+ return Err(QLoRALinearWrapperError::InvalidTensor {+ shape: shape.into(),+ data_type: weights_leaf.data_type(),+ expected_shape: expected_shape.into(),+ expected_data_type,+ });+ }++ Ok(())+}++impl<B: Backend> QLoRALinearWrapper<B> {+ pub fn new(+ context: &B::Context,+ quantization: &QuantizationConfig,+ lora_rank: usize,+ lora_scale: f32,+ input_dim: usize,+ output_dim: usize,+ parameter_tree: &ParameterTree<B::Context>,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ ) -> Result<Self, QLoRALinearWrapperError<B>> {+ let data_type = quantization.activation_precision.into();++ let base_linear = QuantizedLinear::new(+ context,+ quantization,+ input_dim,+ output_dim,+ parameter_tree,+ input_array_id,+ output_array_id,+ )?;+ let adapter_kernel =+ RefCell::new(<<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)?);++ let adapter_down_leaf = parameter_tree.leaf("down_weights")?;+ validate_tensor(&adapter_down_leaf, [lora_rank as usize, input_dim as usize], data_type)?;+ let adapter_down = adapter_down_leaf.read_buffer()?;++ let adapter_up_leaf = parameter_tree.leaf("up_weights")?;+ validate_tensor(&adapter_up_leaf, [output_dim, lora_rank as usize], data_type)?;+ let adapter_up = adapter_up_leaf.read_buffer()?;++ Ok(Self {+ base_linear,+ adapter_kernel,+ adapter_down,+ adapter_up,+ input_dim,+ output_dim,+ lora_rank,+ lora_scale,+ input_array_id,+ output_array_id,+ })+ }+}++impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {+ fn encode(+ &self,+ state: &mut ForwardPassState<B>,+ encoder: &mut Encoder<B>,+ ) -> Result<(), <B as Backend>::Error> {+ self.base_linear.encode(state, encoder)?;++ let mut adapter_kernel = self.adapter_kernel.borrow_mut();+ let batch_size = state.active_suffix_length();++ let intermediate_array = state.common_aux.lora_intermediate.as_ref().unwrap();+ let input_array = state.array(self.input_array_id);+ let output_array = state.array(self.output_array_id);++ adapter_kernel.encode(+ state.context(),+ MatmulArguments {+ a: input_array.buffer().borrow().deref(),+ a_offset: 0,+ b: &self.adapter_down,+ ab_scale: 1.0,+ c: MatmulArgumentC::None,+ d: intermediate_array.buffer().borrow_mut().deref_mut(),+ batch_dim: batch_size as u32,+ input_dim: self.input_dim as u32,+ output_dim: self.lora_rank as u32,+ },+ encoder,+ );++ adapter_kernel.encode(+ state.context(),+ MatmulArguments {+ a: intermediate_array.buffer().borrow().deref(),+ a_offset: 0,+ b: &self.adapter_up,+ ab_scale: self.lora_scale,+ c: MatmulArgumentC::Accumulate,+ d: output_array.buffer().borrow_mut().deref_mut(),+ batch_dim: batch_size as u32,+ input_dim: self.lora_rank as u32,+ output_dim: self.output_dim as u32,+ },+ encoder,+ );++ Ok(())+ }+}diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rsindex 789e1c252..7a46bbda5 100644--- a/crates/uzu/src/forward_pass/model_shape.rs+++ b/crates/uzu/src/forward_pass/model_shape.rs@@ -1,6 +1,6 @@use crate::{DataType,- config::{DecoderConfig, DecoderLayerType, MixerConfig},+ config::{DecoderConfig, DecoderLayerType, LinearConfig, MLPConfig, MixerConfig},};#[derive(Debug)]@@ -26,6 +26,7 @@ pub struct ModelShape {max_mamba_state_dim: usize,max_mamba_kernel_size: usize,max_rope_dim: usize,+ max_lora_intermediate: Option<usize>,has_gate: bool,}@@ -76,6 +77,7 @@ impl ModelShape {}}let mut max_rope_dim = 0usize;+ let mut max_lora_intermediate = None;let mut has_gate = false;let all_layer_configs = decoder_config@@ -85,10 +87,45 @@ impl ModelShape {.unwrap_or_else(|| vec![&decoder_config.layer_config; num_layers]);for layer_config in &all_layer_configs {- if let MixerConfig::Attention(attn) = &layer_config.mixer_config {- let hd = attn.head_dim.unwrap_or(decoder_config.head_dim);- max_rope_dim = max_rope_dim.max(attn.partial_rope_dim.unwrap_or(hd));- has_gate = has_gate || attn.has_gate;+ let mut linear_configs = Vec::new();++ match &layer_config.mixer_config {+ MixerConfig::Attention(attn) => {+ let hd = attn.head_dim.unwrap_or(decoder_config.head_dim);+ max_rope_dim = max_rope_dim.max(attn.partial_rope_dim.unwrap_or(hd));+ has_gate = has_gate || attn.has_gate;+ linear_configs.extend([&attn.qkv_projection_config, &attn.out_projection_config]);+ },+ MixerConfig::Mamba(mamba) => {+ linear_configs.extend([&mamba.in_projection_config, &mamba.out_projection_config]);+ },+ MixerConfig::ShortConv(short_conv) => {+ linear_configs.extend([&short_conv.in_projection_config, &short_conv.out_projection_config]);+ },+ MixerConfig::DeltaNet(delta_net) => {+ linear_configs.extend([&delta_net.in_proj_config, &delta_net.out_proj_config]);+ },+ };++ match &layer_config.mlp_config {+ MLPConfig::Dense(dense) => {+ linear_configs.push(&dense.linear_config);+ },+ MLPConfig::MixtureOfExperts(moe) => {+ linear_configs.extend([&moe.router_config, &moe.expert_config.linear_config]);+ },+ };++ while let Some(linear_config) = linear_configs.pop() {+ match linear_config {+ LinearConfig::QLoRA {+ lora_rank,+ ..+ } => {+ max_lora_intermediate = Some(max_lora_intermediate.unwrap_or(0).max(*lora_rank));+ },+ _ => (),+ };}}@@ -119,6 +156,7 @@ impl ModelShape {max_mamba_state_dim,max_mamba_kernel_size,max_rope_dim,+ max_lora_intermediate,has_gate,}}@@ -513,6 +551,13 @@ impl ModelShape {}}+ pub fn lora_intermediate(+ &self,+ suffix_length: usize,+ ) -> Option<[usize; 2]> {+ self.max_lora_intermediate.map(|max_lora_intermediate| [suffix_length, max_lora_intermediate])+ }+fn max_mamba_inproj_dim_internal(&self) -> Option<usize> {self.layer_types.iter()diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rsindex c36e7a4a2..f920895b8 100644--- a/crates/uzu/src/forward_pass/scratch_buffers.rs+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs@@ -25,6 +25,7 @@ pub struct ScratchBuffers<B: Backend> {pub attention_output: Array<B>,pub mlp_fused_up: Array<B>,pub mlp_hidden: Array<B>,+ pub lora_intermediate: Option<Array<B>>,pub ssm_inproj: Option<Array<B>>,pub ssm_packed: Option<Array<B>>,pub ssm_conv_padded: Option<Array<B>>,@@ -114,6 +115,9 @@ impl<B: Backend> ScratchBuffers<B> {attention_output: alloc(&model_shape.attention_output_shape(max_suffix_len), act_ty, "attention_output"),mlp_fused_up: alloc(&model_shape.mlp_fused_up_shape(max_suffix_len), act_ty, "mlp_fused_up"),mlp_hidden: alloc(&model_shape.mlp_hidden_shape(max_suffix_len), act_ty, "mlp_hidden"),+ lora_intermediate: model_shape+ .lora_intermediate(max_suffix_len)+ .map(|shape| alloc(&shape, act_ty, "lora_intermediate")),ssm_inproj: model_shape.ssm_inproj_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_inproj")),ssm_packed: model_shape.ssm_packed_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_packed")),ssm_conv_padded: model_shapediff --git a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs b/crates/uzu/src/forward_pass/state/common_aux_buffers.rsindex 696b3f57c..2124f53a6 100644--- a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs+++ b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs@@ -13,6 +13,7 @@ pub struct CommonAuxBuffers<B: Backend> {pub attention_output: Array<B>,pub mlp_fused_up: Array<B>,pub mlp_hidden: Array<B>,+ pub lora_intermediate: Option<Array<B>>,pub rotated_queries: Array<B>,pub rotated_keys: Array<B>,pub extracted_values: Array<B>,@@ -36,6 +37,11 @@ impl<B: Backend> CommonAuxBuffers<B> {attention_output: scratch.attention_output.view(&model_shape.attention_output_shape(suffix_length)),mlp_fused_up: scratch.mlp_fused_up.view(&model_shape.mlp_fused_up_shape(suffix_length)),mlp_hidden: scratch.mlp_hidden.view(&model_shape.mlp_hidden_shape(suffix_length)),+ lora_intermediate: scratch+ .lora_intermediate+ .as_ref()+ .zip(model_shape.lora_intermediate(suffix_length))+ .map(|(buf, shape)| buf.view(&shape)),rotated_queries: scratch.rotated_queries.view(&model_shape.rotated_queries_shape(suffix_length)),rotated_keys: scratch.rotated_keys.view(&model_shape.rotated_keys_shape(suffix_length)),extracted_values: scratch.extracted_values.view(&model_shape.extracted_values_shape(suffix_length)),diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex e2cf57288..d130f7865 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -58,7 +58,7 @@ pub struct ForwardPassState<B: Backend> {token_parents: Array<B>,token_bitmask: Option<Array<B>>,pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,- common_aux: CommonAuxBuffers<B>,+ pub common_aux: CommonAuxBuffers<B>,llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,mode: ForwardPassMode<B>,}