A high-performance inference engine for AI models
Gemma4 (#600)
CCYeh03b3e0c29f66
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rsindex 22afa6ed1..dc560853d 100644--- a/crates/backend-uzu/src/encodable_block/decoder.rs+++ b/crates/backend-uzu/src/encodable_block/decoder.rs@@ -98,6 +98,10 @@ impl<B: Backend> Decoder<B> {self.transformer.max_context_length()}+ pub fn prefill_cache_skips_trailing_layers(&self) -> bool {+ self.transformer.prefill_cache_skips_trailing_layers()+ }+pub fn create_empty_state(&self,max_context_length: Option<usize>,diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rsindex 0b26e9266..3eb222c7c 100644--- a/crates/backend-uzu/src/encodable_block/transformer.rs+++ b/crates/backend-uzu/src/encodable_block/transformer.rs@@ -160,6 +160,21 @@ impl<B: Backend> Transformer<B> {})}+ pub fn prefill_cache_layer_count(&self) -> usize {+ let num_layers = self.layers.len();+ let Some(last_owned_kv_layer_index) =+ self.layers.iter().rposition(|(layer, _rope_index)| layer.kv_source_layer_index.is_none())+ else {+ return num_layers;+ };++ last_owned_kv_layer_index + 1+ }++ pub fn prefill_cache_skips_trailing_layers(&self) -> bool {+ self.prefill_cache_layer_count() < self.layers.len()+ }+pub fn create_empty_state(&self,max_context_length: Option<usize>,@@ -195,6 +210,11 @@ impl<B: Backend> Transformer<B> {hidden_feature_layer_indices: &[usize],) -> Result<TransformerEncodeOutput<B>, B::Error> {let mut hidden = input;+ let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_empty() {+ self.prefill_cache_layer_count()+ } else {+ self.layers.len()+ };let mut shortcut = encoder.allocate_scratch(hidden.size())?;let mut hidden_features = (0..hidden_feature_layer_indices.len()).map(|_| None).collect::<Vec<_>>();@@ -209,7 +229,7 @@ impl<B: Backend> Transformer<B> {.map(|rope_config| PrecalculatedRoPE::precalculate(rope_config, &token_positions, encoder)).collect::<Result<Box<[_]>, B::Error>>()?;- for (layer, layer_rope_index) in self.layers.iter() {+ for (layer, layer_rope_index) in self.layers.iter().take(layer_count) {let precalculated_rope = layer_rope_index.map(|i| &precalculated_ropes[i]);let layer_state = if let Some(state) = &mut state {diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rsindex 1a017fa59..d9f6b590d 100644--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs@@ -75,6 +75,19 @@ enum DecodingState<B: Backend> {Invalid,}+fn prefill_chunk_parts(+ input_chunk: &[u64],+ last_batch: bool,+ split_logits_row: bool,+) -> [Option<(&[u64], bool)>; 2] {+ if last_batch && split_logits_row && input_chunk.len() > 1 {+ let (prompt_chunk, sample_chunk) = input_chunk.split_at(input_chunk.len() - 1);+ [Some((prompt_chunk, false)), Some((sample_chunk, true))]+ } else {+ [Some((input_chunk, last_batch)), None]+ }+}+pub struct LanguageModelStream<'a, B: Backend> {model: &'a LanguageModel<B>,model_state: &'a mut LanguageModelState<B>,@@ -158,10 +171,16 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.map_err(LanguageModelStreamError::Backend)?;let mut output = None;-- for (batch_idx, input_chunk) in input.chunks(max_batch_size).enumerate() {- let last_batch = batch_idx == number_of_batches - 1;-+ let split_logits_row = model.decoder.prefill_cache_skips_trailing_layers();++ for (input_chunk, sample_last) in input+ .chunks(max_batch_size)+ .enumerate()+ .flat_map(|(batch_idx, input_chunk)| {+ prefill_chunk_parts(input_chunk, batch_idx == number_of_batches - 1, split_logits_row)+ })+ .flatten()+ {let input_trie = TrieNode::flat(model_state.tokens.len(), input_chunk, &model_state.prng);let input_flat_trie = input_trie.linearize();@@ -178,14 +197,14 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.encode(&token_ids,&batch_dim,- last_batch.then(|| (input_chunk.len() - 1)..input_chunk.len()),+ sample_last.then(|| (input_chunk.len() - 1)..input_chunk.len()),&mut model_state.transformer_state,&mut encoder,&[],)?.logits;- if last_batch {+ if sample_last {let logits = logits.unwrap();let seeds = if matches!(options.sampling_method, SamplingMethod::Stochastic { .. }) {