A high-performance inference engine for AI models
Support hidden capture (#581)
CCYeh6af43c3a2e5e
diff --git a/crates/backend-uzu/src/encodable_block/classifier.rs b/crates/backend-uzu/src/encodable_block/classifier.rsindex 19db1bbcc..aab36146c 100644--- a/crates/backend-uzu/src/encodable_block/classifier.rs+++ b/crates/backend-uzu/src/encodable_block/classifier.rs@@ -132,8 +132,9 @@ impl<B: Backend> Classifier<B> {.collect::<Box<[TrieNode]>>();let hidden = self.transformer- .encode(hidden, None, &BatchTopology::new(&nodes, true), Some(0..batch_dim), None, encoder)+ .encode(hidden, None, &BatchTopology::new(&nodes, true), Some(0..batch_dim), None, encoder, &[]).map_err(ClassifierError::Backend)?+ .output.unwrap();let mut pooled = encoderdiff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rsindex 11764264b..22afa6ed1 100644--- a/crates/backend-uzu/src/encodable_block/decoder.rs+++ b/crates/backend-uzu/src/encodable_block/decoder.rs@@ -35,6 +35,13 @@ pub struct Decoder<B: Backend> {transformer: Transformer<B>,}+pub struct DecoderEncodeOutput<B: Backend> {+ pub logits: Option<Allocation<B>>,+ // TODO: remove after DFlash wiring+ #[allow(dead_code)]+ pub hidden_features: Box<[Allocation<B>]>,+}+impl<B: Backend> Decoder<B> {pub fn new(context: &B::Context,@@ -106,7 +113,8 @@ impl<B: Backend> Decoder<B> {output_range: Option<Range<usize>>,state: &mut TransformerState<B>,encoder: &mut Encoder<B>,- ) -> Result<Option<Allocation<B>>, DecoderError<B>> {+ hidden_feature_layer_indices: &[usize],+ ) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {let embedded = self.embedding.encode_lookup(token_ids, batch_dim.size(), encoder)?;let per_layer_inputs = if let Some(per_layer_embedding) = &self.per_layer_embedding {@@ -119,17 +127,28 @@ impl<B: Backend> Decoder<B> {None};- let hidden = self+ let transformer_output = self.transformer- .encode(embedded, per_layer_inputs.as_ref(), batch_dim, output_range.clone(), Some(state), encoder)+ .encode(+ embedded,+ per_layer_inputs.as_ref(),+ batch_dim,+ output_range.clone(),+ Some(state),+ encoder,+ hidden_feature_layer_indices,+ ).map_err(DecoderError::Backend)?;- let Some(output_range) = output_range else {- return Ok(None);+ let logits = if let Some(output_range) = output_range {+ Some(self.embedding.encode_readout(output_range.len(), &transformer_output.output.unwrap(), encoder)?)+ } else {+ None};- let logits = self.embedding.encode_readout(output_range.len(), &hidden.unwrap(), encoder)?;-- Ok(Some(logits))+ Ok(DecoderEncodeOutput {+ logits,+ hidden_features: transformer_output.hidden_features,+ })}}diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rsindex 5e30554e0..0b26e9266 100644--- a/crates/backend-uzu/src/encodable_block/transformer.rs+++ b/crates/backend-uzu/src/encodable_block/transformer.rs@@ -26,6 +26,11 @@ pub struct TransformerState<B: Backend> {context_length: usize,}+pub struct TransformerEncodeOutput<B: Backend> {+ pub output: Option<Allocation<B>>,+ pub hidden_features: Box<[Allocation<B>]>,+}+impl<B: Backend> TransformerState<B> {pub fn context_length(&self) -> usize {self.context_length@@ -187,10 +192,12 @@ impl<B: Backend> Transformer<B> {output_range: Option<Range<usize>>,mut state: Option<&mut TransformerState<B>>,encoder: &mut Encoder<B>,- ) -> Result<Option<Allocation<B>>, B::Error> {+ hidden_feature_layer_indices: &[usize],+ ) -> Result<TransformerEncodeOutput<B>, B::Error> {let mut hidden = input;let mut shortcut = encoder.allocate_scratch(hidden.size())?;+ let mut hidden_features = (0..hidden_feature_layer_indices.len()).map(|_| None).collect::<Vec<_>>();let context_length = state.as_ref().map(|state| state.context_length).unwrap_or(0);let token_positions =@@ -229,15 +236,39 @@ impl<B: Backend> Transformer<B> {layer_state,encoder,)?;++ for (feature_index, &layer_index) in hidden_feature_layer_indices.iter().enumerate() {+ if layer_index == layer.layer_index {+ let mut feature = encoder.allocate_scratch(hidden.size())?;+ encoder.encode_copy(&hidden, .., &mut feature, ..);+ hidden_features[feature_index] = Some(feature);+ }+ }}+ let hidden_features: Box<[Allocation<B>]> = hidden_features+ .into_iter()+ .enumerate()+ .map(|(feature_index, feature)| {+ feature.unwrap_or_else(|| {+ panic!("requested hidden feature for missing layer {}", hidden_feature_layer_indices[feature_index])+ })+ })+ .collect();+let Some(output_range) = output_range else {- return Ok(None);+ return Ok(TransformerEncodeOutput {+ output: None,+ hidden_features,+ });};let output_normalized =self.output_norm.encode(&hidden, output_range.start, output_range.len(), Some(&mut shortcut), encoder)?;- Ok(Some(output_normalized))+ Ok(TransformerEncodeOutput {+ output: Some(output_normalized),+ hidden_features,+ })}}diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rsindex 7b52d31e1..03b25a95d 100644--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs@@ -168,13 +168,17 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let input_flat_trie_nodes = input_flat_trie.token_subtrie_ranges().collect::<Box<[GpuTrieNode]>>();let batch_dim = BatchTopology::new(&input_flat_trie_nodes, true);- let logits = model.decoder.encode(- &token_ids,- &batch_dim,- last_batch.then(|| (input_chunk.len() - 1)..input_chunk.len()),- &mut model_state.transformer_state,- &mut encoder,- )?;+ let logits = model+ .decoder+ .encode(+ &token_ids,+ &batch_dim,+ last_batch.then(|| (input_chunk.len() - 1)..input_chunk.len()),+ &mut model_state.transformer_state,+ &mut encoder,+ &[],+ )?+ .logits;if last_batch {let logits = logits.unwrap();@@ -418,7 +422,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {Some(0..batch_dim.size()),&mut self.model_state.transformer_state,&mut encoder,+ &[],)?+ .logits.unwrap();let (bitmask, mut encoder) = if let Some(grammar) = self.options.grammar.as_deref_mut() {