Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Support hidden capture (#581)

CCYeh6af43c3a2e5e
diff --git a/crates/backend-uzu/src/encodable_block/classifier.rs b/crates/backend-uzu/src/encodable_block/classifier.rs
index 19db1bbcc..aab36146c 100644
--- a/crates/backend-uzu/src/encodable_block/classifier.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier.rs
@@ -132,8 +132,9 @@ impl<B: Backend> Classifier<B> {
.collect::<Box<[TrieNode]>>();
let hidden = self
.transformer
- .encode(hidden, None, &BatchTopology::new(&nodes, true), Some(0..batch_dim), None, encoder)
+ .encode(hidden, None, &BatchTopology::new(&nodes, true), Some(0..batch_dim), None, encoder, &[])
.map_err(ClassifierError::Backend)?
+ .output
.unwrap();
let mut pooled = encoder
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index 11764264b..22afa6ed1 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -35,6 +35,13 @@ pub struct Decoder<B: Backend> {
transformer: Transformer<B>,
}
+pub struct DecoderEncodeOutput<B: Backend> {
+ pub logits: Option<Allocation<B>>,
+ // TODO: remove after DFlash wiring
+ #[allow(dead_code)]
+ pub hidden_features: Box<[Allocation<B>]>,
+}
+
impl<B: Backend> Decoder<B> {
pub fn new(
context: &B::Context,
@@ -106,7 +113,8 @@ impl<B: Backend> Decoder<B> {
output_range: Option<Range<usize>>,
state: &mut TransformerState<B>,
encoder: &mut Encoder<B>,
- ) -> Result<Option<Allocation<B>>, DecoderError<B>> {
+ hidden_feature_layer_indices: &[usize],
+ ) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {
let embedded = self.embedding.encode_lookup(token_ids, batch_dim.size(), encoder)?;
let per_layer_inputs = if let Some(per_layer_embedding) = &self.per_layer_embedding {
@@ -119,17 +127,28 @@ impl<B: Backend> Decoder<B> {
None
};
- let hidden = self
+ let transformer_output = self
.transformer
- .encode(embedded, per_layer_inputs.as_ref(), batch_dim, output_range.clone(), Some(state), encoder)
+ .encode(
+ embedded,
+ per_layer_inputs.as_ref(),
+ batch_dim,
+ output_range.clone(),
+ Some(state),
+ encoder,
+ hidden_feature_layer_indices,
+ )
.map_err(DecoderError::Backend)?;
- let Some(output_range) = output_range else {
- return Ok(None);
+ let logits = if let Some(output_range) = output_range {
+ Some(self.embedding.encode_readout(output_range.len(), &transformer_output.output.unwrap(), encoder)?)
+ } else {
+ None
};
- let logits = self.embedding.encode_readout(output_range.len(), &hidden.unwrap(), encoder)?;
-
- Ok(Some(logits))
+ Ok(DecoderEncodeOutput {
+ logits,
+ hidden_features: transformer_output.hidden_features,
+ })
}
}
diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rs
index 5e30554e0..0b26e9266 100644
--- a/crates/backend-uzu/src/encodable_block/transformer.rs
+++ b/crates/backend-uzu/src/encodable_block/transformer.rs
@@ -26,6 +26,11 @@ pub struct TransformerState<B: Backend> {
context_length: usize,
}
+pub struct TransformerEncodeOutput<B: Backend> {
+ pub output: Option<Allocation<B>>,
+ pub hidden_features: Box<[Allocation<B>]>,
+}
+
impl<B: Backend> TransformerState<B> {
pub fn context_length(&self) -> usize {
self.context_length
@@ -187,10 +192,12 @@ impl<B: Backend> Transformer<B> {
output_range: Option<Range<usize>>,
mut state: Option<&mut TransformerState<B>>,
encoder: &mut Encoder<B>,
- ) -> Result<Option<Allocation<B>>, B::Error> {
+ hidden_feature_layer_indices: &[usize],
+ ) -> Result<TransformerEncodeOutput<B>, B::Error> {
let mut hidden = input;
let mut shortcut = encoder.allocate_scratch(hidden.size())?;
+ let mut hidden_features = (0..hidden_feature_layer_indices.len()).map(|_| None).collect::<Vec<_>>();
let context_length = state.as_ref().map(|state| state.context_length).unwrap_or(0);
let token_positions =
@@ -229,15 +236,39 @@ impl<B: Backend> Transformer<B> {
layer_state,
encoder,
)?;
+
+ for (feature_index, &layer_index) in hidden_feature_layer_indices.iter().enumerate() {
+ if layer_index == layer.layer_index {
+ let mut feature = encoder.allocate_scratch(hidden.size())?;
+ encoder.encode_copy(&hidden, .., &mut feature, ..);
+ hidden_features[feature_index] = Some(feature);
+ }
+ }
}
+ let hidden_features: Box<[Allocation<B>]> = hidden_features
+ .into_iter()
+ .enumerate()
+ .map(|(feature_index, feature)| {
+ feature.unwrap_or_else(|| {
+ panic!("requested hidden feature for missing layer {}", hidden_feature_layer_indices[feature_index])
+ })
+ })
+ .collect();
+
let Some(output_range) = output_range else {
- return Ok(None);
+ return Ok(TransformerEncodeOutput {
+ output: None,
+ hidden_features,
+ });
};
let output_normalized =
self.output_norm.encode(&hidden, output_range.start, output_range.len(), Some(&mut shortcut), encoder)?;
- Ok(Some(output_normalized))
+ Ok(TransformerEncodeOutput {
+ output: Some(output_normalized),
+ hidden_features,
+ })
}
}
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
index 7b52d31e1..03b25a95d 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
@@ -168,13 +168,17 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let input_flat_trie_nodes = input_flat_trie.token_subtrie_ranges().collect::<Box<[GpuTrieNode]>>();
let batch_dim = BatchTopology::new(&input_flat_trie_nodes, true);
- let logits = model.decoder.encode(
- &token_ids,
- &batch_dim,
- last_batch.then(|| (input_chunk.len() - 1)..input_chunk.len()),
- &mut model_state.transformer_state,
- &mut encoder,
- )?;
+ let logits = model
+ .decoder
+ .encode(
+ &token_ids,
+ &batch_dim,
+ last_batch.then(|| (input_chunk.len() - 1)..input_chunk.len()),
+ &mut model_state.transformer_state,
+ &mut encoder,
+ &[],
+ )?
+ .logits;
if last_batch {
let logits = logits.unwrap();
@@ -418,7 +422,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
Some(0..batch_dim.size()),
&mut self.model_state.transformer_state,
&mut encoder,
+ &[],
)?
+ .logits
.unwrap();
let (bitmask, mut encoder) = if let Some(grammar) = self.options.grammar.as_deref_mut() {