Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

QLoRA (#297)

uuuvn3a39be8d0712
diff --git a/crates/uzu/src/encodable_block/linear/mod.rs b/crates/uzu/src/encodable_block/linear/mod.rs
index 7d8478fc6..de530a9e4 100644
--- a/crates/uzu/src/encodable_block/linear/mod.rs
+++ b/crates/uzu/src/encodable_block/linear/mod.rs
@@ -1,7 +1,9 @@
mod full_precision;
+mod qlora_wrapper;
mod quantized;
pub use full_precision::{FullPrecisionLinear, FullPrecisionLinearError};
+pub use qlora_wrapper::{QLoRALinearWrapper, QLoRALinearWrapperError};
pub use quantized::{QuantizedLinear, QuantizedLinearError};
use thiserror::Error;
@@ -23,11 +25,11 @@ pub trait Linear<B: Backend> {
#[derive(Debug, Error)]
pub enum LinearBlockError<B: Backend> {
#[error("QuantizedLinear error: {0}")]
- QuantizedLinearError(#[source] QuantizedLinearError<B>),
+ QuantizedLinearError(#[from] QuantizedLinearError<B>),
#[error("FullPrecisionLinear error: {0}")]
- FullPrecisionLinearError(#[source] FullPrecisionLinearError<B>),
- #[error("QLoRA linear layer not supported")]
- QLoRaNotSupported,
+ FullPrecisionLinearError(#[from] FullPrecisionLinearError<B>),
+ #[error("QLoRALinearWrapper error: {0}")]
+ QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),
}
impl<B: Backend> dyn Linear<B> {
@@ -52,8 +54,7 @@ impl<B: Backend> dyn Linear<B> {
parameter_tree,
input_array_id,
output_array_id,
- )
- .map_err(LinearBlockError::QuantizedLinearError)?;
+ )?;
Ok(Box::new(block))
},
LinearConfig::FullPrecision {
@@ -67,13 +68,27 @@ impl<B: Backend> dyn Linear<B> {
parameter_tree,
input_array_id,
output_array_id,
- )
- .map_err(LinearBlockError::FullPrecisionLinearError)?;
+ )?;
Ok(Box::new(block))
},
LinearConfig::QLoRA {
- ..
- } => Err(LinearBlockError::QLoRaNotSupported),
+ quantization,
+ lora_rank,
+ lora_scale,
+ } => {
+ let block = QLoRALinearWrapper::new(
+ context,
+ quantization,
+ *lora_rank,
+ *lora_scale,
+ input_dimension,
+ output_dimension_sum,
+ parameter_tree,
+ input_array_id,
+ output_array_id,
+ )?;
+ Ok(Box::new(block))
+ },
}
}
}
diff --git a/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs
new file mode 100644
index 000000000..3552c1564
--- /dev/null
+++ b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs
@@ -0,0 +1,177 @@
+use std::{
+ cell::RefCell,
+ ops::{Deref, DerefMut},
+};
+
+use thiserror::Error;
+
+use crate::{
+ DataType,
+ backends::common::{
+ Backend, Encoder,
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
+ },
+ },
+ config::QuantizationConfig,
+ encodable_block::{
+ Linear,
+ linear::{LinearBlockError, QuantizedLinear, QuantizedLinearError},
+ },
+ forward_pass::state::{ArrayId, ForwardPassState},
+ prelude::{ParameterLeaf, ParameterLoaderError, ParameterTree},
+};
+
+#[derive(Debug, Error)]
+pub enum QLoRALinearWrapperError<B: Backend> {
+ #[error("Inner linear error: {0}")]
+ InnerLinearError(#[from] Box<LinearBlockError<B>>),
+ #[error("Quantized linear error: {0}")]
+ QuantizedLinearError(#[from] QuantizedLinearError<B>),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[from] ParameterLoaderError<B>),
+ #[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]
+ InvalidTensor {
+ shape: Box<[usize]>,
+ data_type: DataType,
+ expected_shape: Box<[usize]>,
+ expected_data_type: DataType,
+ },
+ #[error("Matmul error: {0}")]
+ MatmulError(#[from] MatmulError<B>),
+}
+
+pub struct QLoRALinearWrapper<B: Backend> {
+ base_linear: QuantizedLinear<B>,
+ adapter_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
+ adapter_down: B::Buffer,
+ adapter_up: B::Buffer,
+ input_dim: usize,
+ output_dim: usize,
+ lora_rank: usize,
+ lora_scale: f32,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+}
+
+// TODO: figure out how to make this generic over QLoRAWrapperError::InvalidTensor or make one global "Invalid Tensor" error and make this a common helper
+fn validate_tensor<'file, 'context, 'leaf, B: Backend>(
+ weights_leaf: &ParameterLeaf<'file, 'context, 'leaf, B::Context>,
+ expected_shape: [usize; 2],
+ expected_data_type: DataType,
+) -> Result<(), QLoRALinearWrapperError<B>> {
+ let shape = weights_leaf.shape();
+ let data_type = weights_leaf.data_type();
+
+ if (shape, data_type) != (expected_shape.as_ref(), expected_data_type) {
+ return Err(QLoRALinearWrapperError::InvalidTensor {
+ shape: shape.into(),
+ data_type: weights_leaf.data_type(),
+ expected_shape: expected_shape.into(),
+ expected_data_type,
+ });
+ }
+
+ Ok(())
+}
+
+impl<B: Backend> QLoRALinearWrapper<B> {
+ pub fn new(
+ context: &B::Context,
+ quantization: &QuantizationConfig,
+ lora_rank: usize,
+ lora_scale: f32,
+ input_dim: usize,
+ output_dim: usize,
+ parameter_tree: &ParameterTree<B::Context>,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ ) -> Result<Self, QLoRALinearWrapperError<B>> {
+ let data_type = quantization.activation_precision.into();
+
+ let base_linear = QuantizedLinear::new(
+ context,
+ quantization,
+ input_dim,
+ output_dim,
+ parameter_tree,
+ input_array_id,
+ output_array_id,
+ )?;
+ let adapter_kernel =
+ RefCell::new(<<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)?);
+
+ let adapter_down_leaf = parameter_tree.leaf("down_weights")?;
+ validate_tensor(&adapter_down_leaf, [lora_rank as usize, input_dim as usize], data_type)?;
+ let adapter_down = adapter_down_leaf.read_buffer()?;
+
+ let adapter_up_leaf = parameter_tree.leaf("up_weights")?;
+ validate_tensor(&adapter_up_leaf, [output_dim, lora_rank as usize], data_type)?;
+ let adapter_up = adapter_up_leaf.read_buffer()?;
+
+ Ok(Self {
+ base_linear,
+ adapter_kernel,
+ adapter_down,
+ adapter_up,
+ input_dim,
+ output_dim,
+ lora_rank,
+ lora_scale,
+ input_array_id,
+ output_array_id,
+ })
+ }
+}
+
+impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState<B>,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(), <B as Backend>::Error> {
+ self.base_linear.encode(state, encoder)?;
+
+ let mut adapter_kernel = self.adapter_kernel.borrow_mut();
+ let batch_size = state.active_suffix_length();
+
+ let intermediate_array = state.common_aux.lora_intermediate.as_ref().unwrap();
+ let input_array = state.array(self.input_array_id);
+ let output_array = state.array(self.output_array_id);
+
+ adapter_kernel.encode(
+ state.context(),
+ MatmulArguments {
+ a: input_array.buffer().borrow().deref(),
+ a_offset: 0,
+ b: &self.adapter_down,
+ ab_scale: 1.0,
+ c: MatmulArgumentC::None,
+ d: intermediate_array.buffer().borrow_mut().deref_mut(),
+ batch_dim: batch_size as u32,
+ input_dim: self.input_dim as u32,
+ output_dim: self.lora_rank as u32,
+ },
+ encoder,
+ );
+
+ adapter_kernel.encode(
+ state.context(),
+ MatmulArguments {
+ a: intermediate_array.buffer().borrow().deref(),
+ a_offset: 0,
+ b: &self.adapter_up,
+ ab_scale: self.lora_scale,
+ c: MatmulArgumentC::Accumulate,
+ d: output_array.buffer().borrow_mut().deref_mut(),
+ batch_dim: batch_size as u32,
+ input_dim: self.lora_rank as u32,
+ output_dim: self.output_dim as u32,
+ },
+ encoder,
+ );
+
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rs
index 789e1c252..7a46bbda5 100644
--- a/crates/uzu/src/forward_pass/model_shape.rs
+++ b/crates/uzu/src/forward_pass/model_shape.rs
@@ -1,6 +1,6 @@
use crate::{
DataType,
- config::{DecoderConfig, DecoderLayerType, MixerConfig},
+ config::{DecoderConfig, DecoderLayerType, LinearConfig, MLPConfig, MixerConfig},
};
#[derive(Debug)]
@@ -26,6 +26,7 @@ pub struct ModelShape {
max_mamba_state_dim: usize,
max_mamba_kernel_size: usize,
max_rope_dim: usize,
+ max_lora_intermediate: Option<usize>,
has_gate: bool,
}
@@ -76,6 +77,7 @@ impl ModelShape {
}
}
let mut max_rope_dim = 0usize;
+ let mut max_lora_intermediate = None;
let mut has_gate = false;
let all_layer_configs = decoder_config
@@ -85,10 +87,45 @@ impl ModelShape {
.unwrap_or_else(|| vec![&decoder_config.layer_config; num_layers]);
for layer_config in &all_layer_configs {
- if let MixerConfig::Attention(attn) = &layer_config.mixer_config {
- let hd = attn.head_dim.unwrap_or(decoder_config.head_dim);
- max_rope_dim = max_rope_dim.max(attn.partial_rope_dim.unwrap_or(hd));
- has_gate = has_gate || attn.has_gate;
+ let mut linear_configs = Vec::new();
+
+ match &layer_config.mixer_config {
+ MixerConfig::Attention(attn) => {
+ let hd = attn.head_dim.unwrap_or(decoder_config.head_dim);
+ max_rope_dim = max_rope_dim.max(attn.partial_rope_dim.unwrap_or(hd));
+ has_gate = has_gate || attn.has_gate;
+ linear_configs.extend([&attn.qkv_projection_config, &attn.out_projection_config]);
+ },
+ MixerConfig::Mamba(mamba) => {
+ linear_configs.extend([&mamba.in_projection_config, &mamba.out_projection_config]);
+ },
+ MixerConfig::ShortConv(short_conv) => {
+ linear_configs.extend([&short_conv.in_projection_config, &short_conv.out_projection_config]);
+ },
+ MixerConfig::DeltaNet(delta_net) => {
+ linear_configs.extend([&delta_net.in_proj_config, &delta_net.out_proj_config]);
+ },
+ };
+
+ match &layer_config.mlp_config {
+ MLPConfig::Dense(dense) => {
+ linear_configs.push(&dense.linear_config);
+ },
+ MLPConfig::MixtureOfExperts(moe) => {
+ linear_configs.extend([&moe.router_config, &moe.expert_config.linear_config]);
+ },
+ };
+
+ while let Some(linear_config) = linear_configs.pop() {
+ match linear_config {
+ LinearConfig::QLoRA {
+ lora_rank,
+ ..
+ } => {
+ max_lora_intermediate = Some(max_lora_intermediate.unwrap_or(0).max(*lora_rank));
+ },
+ _ => (),
+ };
}
}
@@ -119,6 +156,7 @@ impl ModelShape {
max_mamba_state_dim,
max_mamba_kernel_size,
max_rope_dim,
+ max_lora_intermediate,
has_gate,
}
}
@@ -513,6 +551,13 @@ impl ModelShape {
}
}
+ pub fn lora_intermediate(
+ &self,
+ suffix_length: usize,
+ ) -> Option<[usize; 2]> {
+ self.max_lora_intermediate.map(|max_lora_intermediate| [suffix_length, max_lora_intermediate])
+ }
+
fn max_mamba_inproj_dim_internal(&self) -> Option<usize> {
self.layer_types
.iter()
diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rs
index c36e7a4a2..f920895b8 100644
--- a/crates/uzu/src/forward_pass/scratch_buffers.rs
+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs
@@ -25,6 +25,7 @@ pub struct ScratchBuffers<B: Backend> {
pub attention_output: Array<B>,
pub mlp_fused_up: Array<B>,
pub mlp_hidden: Array<B>,
+ pub lora_intermediate: Option<Array<B>>,
pub ssm_inproj: Option<Array<B>>,
pub ssm_packed: Option<Array<B>>,
pub ssm_conv_padded: Option<Array<B>>,
@@ -114,6 +115,9 @@ impl<B: Backend> ScratchBuffers<B> {
attention_output: alloc(&model_shape.attention_output_shape(max_suffix_len), act_ty, "attention_output"),
mlp_fused_up: alloc(&model_shape.mlp_fused_up_shape(max_suffix_len), act_ty, "mlp_fused_up"),
mlp_hidden: alloc(&model_shape.mlp_hidden_shape(max_suffix_len), act_ty, "mlp_hidden"),
+ lora_intermediate: model_shape
+ .lora_intermediate(max_suffix_len)
+ .map(|shape| alloc(&shape, act_ty, "lora_intermediate")),
ssm_inproj: model_shape.ssm_inproj_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_inproj")),
ssm_packed: model_shape.ssm_packed_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_packed")),
ssm_conv_padded: model_shape
diff --git a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
index 696b3f57c..2124f53a6 100644
--- a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
+++ b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
@@ -13,6 +13,7 @@ pub struct CommonAuxBuffers<B: Backend> {
pub attention_output: Array<B>,
pub mlp_fused_up: Array<B>,
pub mlp_hidden: Array<B>,
+ pub lora_intermediate: Option<Array<B>>,
pub rotated_queries: Array<B>,
pub rotated_keys: Array<B>,
pub extracted_values: Array<B>,
@@ -36,6 +37,11 @@ impl<B: Backend> CommonAuxBuffers<B> {
attention_output: scratch.attention_output.view(&model_shape.attention_output_shape(suffix_length)),
mlp_fused_up: scratch.mlp_fused_up.view(&model_shape.mlp_fused_up_shape(suffix_length)),
mlp_hidden: scratch.mlp_hidden.view(&model_shape.mlp_hidden_shape(suffix_length)),
+ lora_intermediate: scratch
+ .lora_intermediate
+ .as_ref()
+ .zip(model_shape.lora_intermediate(suffix_length))
+ .map(|(buf, shape)| buf.view(&shape)),
rotated_queries: scratch.rotated_queries.view(&model_shape.rotated_queries_shape(suffix_length)),
rotated_keys: scratch.rotated_keys.view(&model_shape.rotated_keys_shape(suffix_length)),
extracted_values: scratch.extracted_values.view(&model_shape.extracted_values_shape(suffix_length)),
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index e2cf57288..d130f7865 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -58,7 +58,7 @@ pub struct ForwardPassState<B: Backend> {
token_parents: Array<B>,
token_bitmask: Option<Array<B>>,
pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
- common_aux: CommonAuxBuffers<B>,
+ pub common_aux: CommonAuxBuffers<B>,
llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,
mode: ForwardPassMode<B>,
}