Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Gemma4 (#600)

CCYeh03b3e0c29f66
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index 22afa6ed1..dc560853d 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -98,6 +98,10 @@ impl<B: Backend> Decoder<B> {
self.transformer.max_context_length()
}
+ pub fn prefill_cache_skips_trailing_layers(&self) -> bool {
+ self.transformer.prefill_cache_skips_trailing_layers()
+ }
+
pub fn create_empty_state(
&self,
max_context_length: Option<usize>,
diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rs
index 0b26e9266..3eb222c7c 100644
--- a/crates/backend-uzu/src/encodable_block/transformer.rs
+++ b/crates/backend-uzu/src/encodable_block/transformer.rs
@@ -160,6 +160,21 @@ impl<B: Backend> Transformer<B> {
})
}
+ pub fn prefill_cache_layer_count(&self) -> usize {
+ let num_layers = self.layers.len();
+ let Some(last_owned_kv_layer_index) =
+ self.layers.iter().rposition(|(layer, _rope_index)| layer.kv_source_layer_index.is_none())
+ else {
+ return num_layers;
+ };
+
+ last_owned_kv_layer_index + 1
+ }
+
+ pub fn prefill_cache_skips_trailing_layers(&self) -> bool {
+ self.prefill_cache_layer_count() < self.layers.len()
+ }
+
pub fn create_empty_state(
&self,
max_context_length: Option<usize>,
@@ -195,6 +210,11 @@ impl<B: Backend> Transformer<B> {
hidden_feature_layer_indices: &[usize],
) -> Result<TransformerEncodeOutput<B>, B::Error> {
let mut hidden = input;
+ let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_empty() {
+ self.prefill_cache_layer_count()
+ } else {
+ self.layers.len()
+ };
let mut shortcut = encoder.allocate_scratch(hidden.size())?;
let mut hidden_features = (0..hidden_feature_layer_indices.len()).map(|_| None).collect::<Vec<_>>();
@@ -209,7 +229,7 @@ impl<B: Backend> Transformer<B> {
.map(|rope_config| PrecalculatedRoPE::precalculate(rope_config, &token_positions, encoder))
.collect::<Result<Box<[_]>, B::Error>>()?;
- for (layer, layer_rope_index) in self.layers.iter() {
+ for (layer, layer_rope_index) in self.layers.iter().take(layer_count) {
let precalculated_rope = layer_rope_index.map(|i| &precalculated_ropes[i]);
let layer_state = if let Some(state) = &mut state {
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
index 1a017fa59..d9f6b590d 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
@@ -75,6 +75,19 @@ enum DecodingState<B: Backend> {
Invalid,
}
+fn prefill_chunk_parts(
+ input_chunk: &[u64],
+ last_batch: bool,
+ split_logits_row: bool,
+) -> [Option<(&[u64], bool)>; 2] {
+ if last_batch && split_logits_row && input_chunk.len() > 1 {
+ let (prompt_chunk, sample_chunk) = input_chunk.split_at(input_chunk.len() - 1);
+ [Some((prompt_chunk, false)), Some((sample_chunk, true))]
+ } else {
+ [Some((input_chunk, last_batch)), None]
+ }
+}
+
pub struct LanguageModelStream<'a, B: Backend> {
model: &'a LanguageModel<B>,
model_state: &'a mut LanguageModelState<B>,
@@ -158,10 +171,16 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.map_err(LanguageModelStreamError::Backend)?;
let mut output = None;
-
- for (batch_idx, input_chunk) in input.chunks(max_batch_size).enumerate() {
- let last_batch = batch_idx == number_of_batches - 1;
-
+ let split_logits_row = model.decoder.prefill_cache_skips_trailing_layers();
+
+ for (input_chunk, sample_last) in input
+ .chunks(max_batch_size)
+ .enumerate()
+ .flat_map(|(batch_idx, input_chunk)| {
+ prefill_chunk_parts(input_chunk, batch_idx == number_of_batches - 1, split_logits_row)
+ })
+ .flatten()
+ {
let input_trie = TrieNode::flat(model_state.tokens.len(), input_chunk, &model_state.prng);
let input_flat_trie = input_trie.linearize();
@@ -178,14 +197,14 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.encode(
&token_ids,
&batch_dim,
- last_batch.then(|| (input_chunk.len() - 1)..input_chunk.len()),
+ sample_last.then(|| (input_chunk.len() - 1)..input_chunk.len()),
&mut model_state.transformer_state,
&mut encoder,
&[],
)?
.logits;
- if last_batch {
+ if sample_last {
let logits = logits.unwrap();
let seeds = if matches!(options.sampling_method, SamplingMethod::Stochastic { .. }) {