Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Simplify decode loop (#1078)

odilf4a386eba7cf8
diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rs
index 629aa716c..1f6c90cbc 100644
--- a/crates/uzu-engine/src/backends/common/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs
@@ -86,6 +86,7 @@ pub trait CommandBufferEncoding: Send {
fn with_timestamps(self) -> Self;
+ #[track_caller]
fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;
}
diff --git a/crates/uzu-engine/src/encodable_block/batch_topology.rs b/crates/uzu-engine/src/encodable_block/batch_topology.rs
index 4606f8562..52b844f9b 100644
--- a/crates/uzu-engine/src/encodable_block/batch_topology.rs
+++ b/crates/uzu-engine/src/encodable_block/batch_topology.rs
@@ -1,15 +1,15 @@
use crate::backends::common::gpu_types::trie::TrieNode;
-pub struct BatchTopology<'a> {
- nodes: &'a [TrieNode],
+pub struct BatchTopology {
+ nodes: Box<[TrieNode]>,
parents: Box<[i32]>,
is_flat: bool,
full_accept: bool,
}
-impl<'a> BatchTopology<'a> {
+impl BatchTopology {
pub fn new(
- nodes: &'a [TrieNode],
+ nodes: Box<[TrieNode]>,
full_accept: bool,
) -> Self {
let mut stack: Vec<usize> = Vec::new();
@@ -25,7 +25,6 @@ impl<'a> BatchTopology<'a> {
is_flat &= node.height == index;
}
let parents = unsafe { parents.assume_init() };
-
assert!(is_flat || !full_accept, "full_accept requires trie to be flat");
Self {
@@ -36,8 +35,19 @@ impl<'a> BatchTopology<'a> {
}
}
- pub fn nodes(&self) -> &'a [TrieNode] {
- self.nodes
+ pub fn flat(
+ size: u32,
+ full_accept: bool,
+ ) -> Self {
+ let nodes = (0..size)
+ .map(|index| TrieNode {
+ trie_start: index,
+ trie_end: size - 1,
+ height: index,
+ })
+ .collect();
+
+ Self::new(nodes, full_accept)
}
pub fn size(&self) -> u32 {
@@ -52,11 +62,15 @@ impl<'a> BatchTopology<'a> {
self.full_accept
}
- pub fn heights(&self) -> impl Iterator<Item = u32> + 'a {
- self.nodes.iter().map(|trie| trie.height)
+ pub fn nodes(&self) -> &[TrieNode] {
+ &self.nodes
}
pub fn parents(&self) -> &[i32] {
&self.parents
}
+
+ pub fn heights(&self) -> impl Iterator<Item = u32> {
+ self.nodes().iter().map(|t| t.height)
+ }
}
diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rs
index 98aced9bb..684b09799 100644
--- a/crates/uzu-engine/src/encodable_block/classifier.rs
+++ b/crates/uzu-engine/src/encodable_block/classifier.rs
@@ -2,10 +2,7 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- backends::common::{
- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels, gpu_types::trie::TrieNode,
- kernel::PoolingMeanKernel,
- },
+ backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels, kernel::PoolingMeanKernel},
config::classifier::{ClassifierConfig, PoolingType},
data_type::DataType,
encodable_block::{
@@ -135,19 +132,12 @@ impl<B: Backend> Classifier<B> {
.encode(&embedded, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)
.map_err(ClassifierError::Backend)?;
- let nodes = (0..batch_dim)
- .map(|index| TrieNode {
- trie_start: index,
- trie_end: batch_dim - 1,
- height: index,
- })
- .collect::<Box<[TrieNode]>>();
let hidden = self
.transformer
.encode(
hidden,
None::<&B::ScratchBuffer>,
- &BatchTopology::new(&nodes, true),
+ &BatchTopology::flat(batch_dim, true),
Some((0..batch_dim).into()),
None,
None,
diff --git a/crates/uzu-engine/src/encodable_block/decision.rs b/crates/uzu-engine/src/encodable_block/decision.rs
index 55c7d131c..cf193ad87 100644
--- a/crates/uzu-engine/src/encodable_block/decision.rs
+++ b/crates/uzu-engine/src/encodable_block/decision.rs
@@ -2,7 +2,7 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding, gpu_types::trie::TrieNode},
+ backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding},
config::model::decision_model::DecisionModelConfig,
data_type::DataType,
encodable_block::{
@@ -114,20 +114,12 @@ impl<B: Backend> Decision<B> {
.map_err(DecisionError::Backend)?;
self.input_embedding.encode_lookup(token_ids, &mut embedded, token_count, self.input_scale, command_buffer);
- let nodes = (0..token_count)
- .map(|index| TrieNode {
- trie_start: index,
- trie_end: token_count - 1,
- height: index,
- })
- .collect::<Box<[TrieNode]>>();
-
let hidden = self
.transformer
.encode(
embedded,
None::<&B::ScratchBuffer>,
- &BatchTopology::new(&nodes, true),
+ &BatchTopology::flat(token_count, true),
Some((token_count - 1..token_count).into()),
None,
None,
diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rs
index b48d7b03c..ff4b42bd7 100644
--- a/crates/uzu-engine/src/encodable_block/dflash.rs
+++ b/crates/uzu-engine/src/encodable_block/dflash.rs
@@ -4,9 +4,7 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- backends::common::{
- Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, gpu_types::trie::TrieNode,
- },
+ backends::common::{Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding},
config::{dflash::DFlashDraftConfig, rope::AnyRoPEConfig, token_mixer::AnyTokenMixerConfig},
data_type::DataType,
encodable_block::{
@@ -310,14 +308,7 @@ impl<B: Backend> DFlash<B> {
let token_embeddings = target_embedding.encode_lookup(&token_ids, batch_size, command_buffer)?;
- let nodes = (0..batch_size)
- .map(|index| TrieNode {
- trie_start: index,
- trie_end: batch_size - 1,
- height: index,
- })
- .collect::<Box<[_]>>();
- let batch_topology = BatchTopology::new(&nodes, true);
+ let batch_topology = BatchTopology::flat(batch_size, true);
let token_positions = (state.context_length..state.context_length + batch_size).collect::<Box<[_]>>();
let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, command_buffer)
.map_err(DFlashEncodeError::Backend)?;
diff --git a/crates/uzu-engine/src/engine/language_model/state.rs b/crates/uzu-engine/src/engine/language_model/state.rs
index c7d755635..d11a4c1e0 100644
--- a/crates/uzu-engine/src/engine/language_model/state.rs
+++ b/crates/uzu-engine/src/engine/language_model/state.rs
@@ -9,7 +9,6 @@ use crate::{
pub struct LanguageModelState<B: Backend> {
pub(super) tokens: Vec<u64>,
- pub(super) last_output_token: Option<u64>, // TODO: this leaks previous LanguageModelStreamOptions
pub(super) prng: PRng,
pub(super) transformer_state: TransformerState<B>,
pub(super) speculator_state: Option<DFlashState<B>>,
@@ -42,7 +41,6 @@ impl<B: Backend> LanguageModel<B> {
sampling_seed: u64,
) -> Result<LanguageModelState<B>, LanguageModelCreateEmptyStateError<B>> {
let tokens = Vec::new();
- let last_output_token = None;
let prng = PRng::new(sampling_seed);
@@ -62,7 +60,6 @@ impl<B: Backend> LanguageModel<B> {
Ok(LanguageModelState {
tokens,
- last_output_token,
prng,
transformer_state,
speculator_state,
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index 1c51ce5aa..ed6af073a 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -1,8 +1,8 @@
use std::{
iter::{once, repeat_n},
- mem::replace,
+ mem,
range::Range,
- sync::{Arc, mpsc::Sender},
+ sync::Arc,
};
use futures::executor::block_on;
@@ -11,16 +11,14 @@ use shoji::traits::backend::chat_token::TokenStreamMetrics;
use crate::{
backends::common::{
Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, CpuBuffer,
- DeviceCapabilities, GlobalBufferCpuGpu, GpuBuffer, TimestampSpan,
+ DeviceCapabilities, GlobalBufferCpuGpu, GpuBuffer,
capture_matcher::{CAPTURE_MATCHER, CaptureMatcherSpan},
- gpu_types::trie::TrieNode as GpuTrieNode,
kernel::ContextRingUpdateKernel,
},
data_type::DataType,
encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},
engine::language_model::{
LanguageModel,
- grammar::Grammar,
state::LanguageModelState,
stream::{LanguageModelStreamError, LanguageModelStreamOptions},
},
@@ -31,131 +29,79 @@ use crate::{
},
};
-enum ForwardPassChaining<B: Backend> {
- Constant {
- token: u64,
- output_norm: Option<B::ScratchBuffer>,
- },
- InFlight(DecodingStatePending<B>),
-}
-
-impl<B: Backend> ForwardPassChaining<B> {
- async fn resolve<'a>(
- &'a mut self,
- readback: &mut Option<impl Future<Output = Result<Vec<u32>, B::Error>> + Send>,
- tokens: &mut Vec<u64>,
- timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
- grammar: Option<&mut Grammar>,
- ) -> Result<(u64, Option<&'a B::ScratchBuffer>), LanguageModelStreamError<B>> {
- match self {
- Self::Constant {
- token,
- output_norm,
- } => Ok((*token, output_norm.as_ref())),
- Self::InFlight(in_flight) => {
- assert!(in_flight.full_accept);
- for pending in replace(&mut in_flight.pending, Box::new([])) {
- wait(pending, timestamps).await.map_err(LanguageModelStreamError::Backend)?;
- }
- let output_tokens = readback
- .take()
- .expect("in-flight output readback")
- .await
- .map_err(LanguageModelStreamError::Backend)?;
- assert_eq!(output_tokens.len(), 1);
- let token_id = output_tokens[0] as u64;
- let output_norm = in_flight.output_norm.take();
- *self = Self::Constant {
- token: token_id,
- output_norm,
- };
- tokens.push(token_id);
- if let Some(grammar) = grammar {
- grammar.accept_token(token_id)?;
- }
- let Self::Constant {
- output_norm,
- ..
- } = self
- else {
- unreachable!()
- };
- Ok((token_id, output_norm.as_ref()))
- },
- }
- }
-
- fn drop_output_norm(&mut self) {
- match self {
- Self::Constant {
- output_norm,
- ..
- } => drop(output_norm.take()),
- Self::InFlight(pending) => drop(pending.output_norm.take()),
- }
- }
+pub struct LanguageModelStream<'a, B: Backend> {
+ submitter: StreamEnqueuer<'a, B>,
+ job: Option<GenerationJob<B>>,
+ emitting: Option<(ResolvedJob<B>, usize)>,
}
-struct DecodingStatePending<B: Backend> {
- input_trie: TrieNode,
- full_accept: bool,
- pending: Box<[<B::CommandBuffer as CommandBuffer>::Pending]>,
- capture_span: Option<CaptureMatcherSpan>,
- hidden_features: Option<Box<[B::ScratchBuffer]>>,
- output_norm: Option<B::ScratchBuffer>,
- output_tokens: Arc<B::GlobalGpuBuffer>,
- _input_token_source: Option<Arc<B::GlobalGpuBuffer>>,
+struct StreamEnqueuer<'a, B: Backend> {
+ model: &'a LanguageModel<B>,
+ model_state: &'a mut LanguageModelState<B>,
+ options: LanguageModelStreamOptions,
+ allocation_pool: Arc<B::AllocationPool>,
+ context_ring: Option<B::GlobalGpuBuffer>,
+ metrics: TokenStreamMetrics,
+ propose_trie: bool,
}
-enum DecodingState<B: Backend> {
- Seeded {
- seed_token: u64,
- },
- ForwardPassPending(DecodingStatePending<B>),
- Accepting {
- full: Box<[(usize, u64, u64)]>,
- num_accepted: usize,
+enum GenerationJob<B: Backend> {
+ Pending {
+ output_norm: Option<B::ScratchBuffer>,
+ output_tokens: B::GlobalGpuBuffer,
+ batch_dim: BatchTopology,
+ trie: Option<TrieNode>,
hidden_features: Option<Box<[B::ScratchBuffer]>>,
- output_norm: Option<(B::ScratchBuffer, usize)>,
capture_span: Option<CaptureMatcherSpan>,
+ pendings: Box<[<<B as Backend>::CommandBuffer as CommandBuffer>::Pending]>,
+ _input_token_source: Option<B::GlobalGpuBuffer>,
},
- Halted,
+ Resolved(ResolvedJob<B>),
Invalid,
- Closed,
}
-fn prefill_chunk_parts(
- input_chunk: &[u64],
- last_batch: bool,
- split_logits_row: bool,
-) -> [Option<(&[u64], bool)>; 2] {
- if last_batch && split_logits_row && input_chunk.len() > 1 {
- let (prompt_chunk, sample_chunk) = input_chunk.split_at(input_chunk.len() - 1);
- [Some((prompt_chunk, false)), Some((sample_chunk, true))]
- } else {
- [Some((input_chunk, last_batch)), None]
- }
+struct ResolvedJob<B: Backend> {
+ output_tokens: Box<[u64]>,
+ output_norm_buffer: Option<B::ScratchBuffer>,
+ output_tokens_buffer: Option<B::GlobalGpuBuffer>,
+ batch_dim: BatchTopology,
+ hidden_features: Option<Box<[B::ScratchBuffer]>>,
+ accepted_indices_and_input_tokens: Option<(Box<[u32]>, Box<[u64]>)>,
+ capture_span: Option<CaptureMatcherSpan>,
}
-pub struct LanguageModelStream<'a, B: Backend> {
- model: &'a LanguageModel<B>,
- model_state: &'a mut LanguageModelState<B>,
- options: LanguageModelStreamOptions,
- allocation_pool: Arc<B::AllocationPool>,
- context_ring: Option<B::GlobalGpuBuffer>,
- decoding_state: DecodingState<B>,
- metrics: TokenStreamMetrics,
+impl<B: Backend> GenerationJob<B> {
+ fn batch_dim(&self) -> &BatchTopology {
+ match self {
+ Self::Pending {
+ batch_dim,
+ ..
+ } => batch_dim,
+ Self::Resolved(resolved) => &resolved.batch_dim,
+ Self::Invalid => unreachable!(),
+ }
+ }
+
+ fn output_tokens_buffer(&self) -> &B::GlobalGpuBuffer {
+ match self {
+ Self::Pending {
+ output_tokens,
+ ..
+ } => output_tokens,
+ Self::Resolved(resolved) => resolved.output_tokens_buffer.as_ref().unwrap(),
+ Self::Invalid => unreachable!(),
+ }
+ }
}
-impl<'a, B: Backend> LanguageModelStream<'a, B> {
- pub fn new(
+impl<'a, B: Backend> StreamEnqueuer<'a, B> {
+ fn new(
model: &'a LanguageModel<B>,
input: &[u64],
model_state: &'a mut LanguageModelState<B>,
options: LanguageModelStreamOptions,
) -> Result<Self, LanguageModelStreamError<B>> {
- let mut options = options;
- if model_state.tokens.is_empty() && input.is_empty() {
+ if input.is_empty() {
return Err(LanguageModelStreamError::NoSeedToken);
};
@@ -168,9 +114,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let allocation_pool = model.engine.context.create_allocation_pool();
- let mut context_ring = if let Some(suffix_repetition_length) =
- options.sampling_method.suffix_repetition_length()
- {
+ let context_ring = if let Some(suffix_repetition_length) = options.sampling_method.suffix_repetition_length() {
let mut context_ring = model
.engine
.context
@@ -195,382 +139,320 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
None
};
- let mut metrics = TokenStreamMetrics::default();
-
- let decoding_state = if !input.is_empty() {
- let _capture_span = CAPTURE_MATCHER.capture::<B>(&model.engine.context, "prefill")?;
- model_state.last_output_token.take();
-
- // NOTE: this is required for attention correctness (hardcoded suffix 1024). This is really bad design, attention should be rewritten to allow on-demand suffix length
- let max_batch_size = 1024;
- let number_of_batches = input.len().div_ceil(max_batch_size);
-
- model_state
- .transformer_state
- .prepare(
- model_state.transformer_state.context_length() + ((number_of_batches - 1) * max_batch_size) as u32,
- usize::min(max_batch_size, input.len()) as u32,
- )
- .map_err(LanguageModelStreamError::Backend)?;
-
- let command_buffer = model
- .engine
- .context
- .create_command_buffer(Some("prefill"), Some(allocation_pool.clone()))
- .map_err(LanguageModelStreamError::Backend)?;
- let mut command_buffer = record(command_buffer, options.timestamps.as_ref());
-
- command_buffer.debug_point(&format!(
- "prefill at context {}, suffix {} ({} batches)",
- model_state.transformer_state.context_length(),
- input.len(),
- number_of_batches
- ));
-
- let mut output_tokens = None;
- let mut output_norm = None;
- let split_logits_row = model.decoder.prefill_cache_skips_trailing_layers();
- let hidden_feature_layer_indices =
- model.speculator.as_ref().map(|speculator| speculator.hidden_feature_layer_indices());
-
- for (input_chunk, sample_last) in input
- .chunks(max_batch_size)
- .enumerate()
- .flat_map(|(batch_idx, input_chunk)| {
- prefill_chunk_parts(input_chunk, batch_idx == number_of_batches - 1, split_logits_row)
- })
- .flatten()
- {
- let input_trie = TrieNode::flat(model_state.tokens.len(), input_chunk, &model_state.prng);
- let input_flat_trie = input_trie.linearize();
-
- let token_ids = command_buffer
- .allocate_constant_from_slice(
- &input_chunk.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>(),
- )
- .map_err(LanguageModelStreamError::Backend)?;
-
- let input_flat_trie_nodes = input_flat_trie.token_subtrie_ranges().collect::<Box<[GpuTrieNode]>>();
- let batch_dim = BatchTopology::new(&input_flat_trie_nodes, true);
-
- let decoder_output = model.decoder.encode(
- &token_ids,
- &batch_dim,
- sample_last.then(|| (input_chunk.len() as u32 - 1..input_chunk.len() as u32).into()),
- hidden_feature_layer_indices,
- &mut model_state.transformer_state,
- &mut command_buffer,
- )?;
- let logits = decoder_output.logits;
-
- if sample_last {
- let logits = logits.unwrap();
-
- let seeds = if matches!(options.sampling_method, SamplingMethod::Stochastic { .. }) {
- Some(
- command_buffer
- .allocate_constant_from_slice(&[model_state
- .prng
- .derive((model_state.tokens.len() + input_chunk.len() - 1) as u64)])
- .map_err(LanguageModelStreamError::Backend)?,
- )
- } else {
- None
- };
-
- let bitmask = if let Some(grammar) = options.grammar.as_mut() {
- let mut bitmask = command_buffer
- .allocate_constant(
- model.vocab_size.div_ceil(DataType::U32.size_in_bits()) * DataType::U32.size_in_bytes(),
- )
- .map_err(LanguageModelStreamError::Backend)?;
-
- if grammar.next_bitmask(bitmask.as_slice_mut()) {
- Some(bitmask)
- } else {
- None
- }
- } else {
- None
- };
-
- output_norm = decoder_output.final_hidden;
- let sampled_row = batch_dim.size() - 1;
- output_tokens = Some(
- model
- .sampling
- .encode(
- &logits,
- seeds.as_ref(),
- bitmask.as_ref(),
- context_ring.as_ref(),
- Some(&token_ids),
- &batch_dim,
- (sampled_row..sampled_row + 1).into(),
- &options.sampling_method,
- &mut command_buffer,
- )
- .map_err(LanguageModelStreamError::Backend)?,
- );
- }
-
- model_state
- .transformer_state
- .encode_accept(&(0..input_chunk.len() as u32).collect::<Box<[u32]>>(), &mut command_buffer)
- .map_err(LanguageModelStreamError::Backend)?;
-
- if let Some(speculator) = model.speculator.as_ref() {
- let speculator_state = model_state.speculator_state.as_mut().unwrap();
- speculator.encode_accept(
- speculator_state,
- decoder_output.hidden_features.as_ref().unwrap().iter(),
- &(0..input_chunk.len() as u32).collect::<Box<[u32]>>(),
- &mut command_buffer,
- )?;
- }
+ let metrics = TokenStreamMetrics::default();
- if let Some(suffix_repetition_length) = options.sampling_method.suffix_repetition_length() {
- model.context_ring_update.encode(
- &token_ids,
- context_ring.as_mut().unwrap(),
- suffix_repetition_length,
- input_chunk.len() as u32,
- &mut command_buffer,
- );
- }
-
- model_state.tokens.extend(input_chunk);
- }
-
- let pending = Box::new([command_buffer.end_encoding().submit()]);
-
- metrics.num_prefill_forward_passes += 1;
- metrics.num_tokens_prefilled += input.len();
- metrics.num_tokens_proposed += 1;
- metrics.num_tokens_accepted += 1;
-
- DecodingState::ForwardPassPending(DecodingStatePending {
- input_trie: TrieNode::new(0, 0, 0.0),
- full_accept: true,
- pending,
- capture_span: None,
- hidden_features: None,
- output_norm,
- output_tokens: Arc::new(output_tokens.unwrap()),
- _input_token_source: None,
- })
- } else {
- // TODO: this leaks previous LanguageModelStreamOptions
- DecodingState::Seeded {
- seed_token: model_state.last_output_token.take().unwrap(),
- }
- };
-
- Ok(LanguageModelStream {
+ Ok(StreamEnqueuer {
model,
model_state,
options,
allocation_pool,
context_ring,
- decoding_state,
metrics,
+ propose_trie: true,
})
}
- pub async fn generate(&mut self) -> Result<Option<u64>, LanguageModelStreamError<B>> {
- let context = &self.model.engine.context;
- let (mut prev_output, acceptance): (
- ForwardPassChaining<B>,
- Option<(<B::CommandBuffer as CommandBuffer>::Encoding, Option<CaptureMatcherSpan>)>,
- ) = match replace(&mut self.decoding_state, DecodingState::Invalid) {
- DecodingState::Seeded {
- seed_token,
- } => {
- self.model_state.tokens.push(seed_token);
- if let Some(grammar) = self.options.grammar.as_mut() {
- let _ = grammar.accept_token(seed_token); // TODO: this should not be ignored
- }
- self.metrics.num_tokens_returned += 1;
- (
- ForwardPassChaining::Constant {
- token: seed_token,
- output_norm: None,
- },
- None,
- )
- },
- DecodingState::ForwardPassPending(forward_pass_pending) => {
- if forward_pass_pending.full_accept {
- self.metrics.num_tokens_returned += 1;
- (ForwardPassChaining::InFlight(forward_pass_pending), None)
- } else {
- for pending in forward_pass_pending.pending {
- wait(pending, self.options.timestamps.as_ref())
- .await
- .map_err(LanguageModelStreamError::Backend)?;
- }
- let sampled_tokens = context
- .copyout::<u32>(&*forward_pass_pending.output_tokens)
- .await
- .map_err(LanguageModelStreamError::Backend)?;
- let sampled_tokens = sampled_tokens.iter().map(|token| u64::from(*token)).collect::<Box<[u64]>>();
- let flat_trie = forward_pass_pending.input_trie.linearize();
- let full = flat_trie.accept(&sampled_tokens, self.options.grammar.as_mut())?;
- let output_norm = forward_pass_pending.output_norm.map(|norm| {
- let row_bytes = norm.size() / flat_trie.len();
- (norm, row_bytes)
- });
- self.metrics.num_tokens_accepted += full.len();
- self.decoding_state = DecodingState::Accepting {
- full,
- num_accepted: 0,
- hidden_features: forward_pass_pending.hidden_features,
- output_norm,
- capture_span: forward_pass_pending.capture_span,
- };
- return Box::pin(self.generate()).await;
- }
- },
- DecodingState::Accepting {
- full,
- num_accepted,
+ async fn resolve<'b>(
+ &mut self,
+ job: &'b mut GenerationJob<B>,
+ ) -> Result<&'b ResolvedJob<B>, LanguageModelStreamError<B>> {
+ let owned = mem::replace(job, GenerationJob::Invalid);
+ let resolved = match owned {
+ GenerationJob::Pending {
+ output_norm: output_norm_buffer,
+ output_tokens: output_tokens_buffer,
+ _input_token_source,
+ batch_dim,
+ trie,
hidden_features,
- output_norm,
capture_span,
+ pendings,
} => {
- let output_token_id = full[num_accepted].2;
-
- self.metrics.num_tokens_returned += 1;
-
- if num_accepted < full.len() - 1 {
- self.decoding_state = DecodingState::Accepting {
- full,
- num_accepted: num_accepted + 1,
- hidden_features,
- output_norm,
- capture_span,
- };
- return Ok(Some(output_token_id));
- } else {
+ for pending in pendings {
+ wait(pending, self.options.timestamps.as_ref()).await.map_err(LanguageModelStreamError::Backend)?;
+ }
+ let gpu_tokens = self
+ .model
+ .engine
+ .context
+ .copyout::<u32>(&output_tokens_buffer)
+ .await
+ .map_err(LanguageModelStreamError::Backend)?;
+
+ let mut output_tokens = gpu_tokens.iter().map(|&t| t as u64).collect::<Box<[_]>>();
+
+ let accepted_indices_and_input_tokens = if let Some(trie) = trie {
+ let flat = trie.linearize();
+ let full = flat.accept(&output_tokens, self.options.grammar.as_mut())?;
+
let accepted_token_indicies = full.iter().map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();
let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();
let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();
- let command_buffer = self
- .model
- .engine
- .context
- .create_command_buffer(
- Some(&format!("decode {} (accept)", self.metrics.num_decode_forward_passes - 1)),
- Some(self.allocation_pool.clone()),
- )
- .map_err(LanguageModelStreamError::Backend)?;
- let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
- self.model_state
- .transformer_state
- .encode_accept(&accepted_token_indicies, &mut command_buffer)
- .map_err(LanguageModelStreamError::Backend)?;
- if let Some(speculator) = self.model.speculator.as_ref() {
- speculator.encode_accept(
- self.model_state.speculator_state.as_mut().unwrap(),
- hidden_features.as_ref().unwrap().iter(),
- &accepted_token_indicies,
- &mut command_buffer,
- )?;
- }
- let output_norm = if let Some((final_hidden, row_bytes)) = output_norm {
- let row = full.last().unwrap().0;
- let mut norm =
- command_buffer.allocate_scratch(row_bytes).map_err(LanguageModelStreamError::Backend)?;
- command_buffer
- .encode_copy(final_hidden.subrange(row * row_bytes..(row + 1) * row_bytes), &mut norm);
- Some(norm)
- } else {
- None
- };
- if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {
- let accepted_input_token_ids_const = command_buffer
- .allocate_constant_from_slice(
- &accepted_input_token_ids
- .iter()
- .map(|token_id| *token_id as u32)
- .collect::<Box<[u32]>>(),
- )
- .map_err(LanguageModelStreamError::Backend)?;
- self.model.context_ring_update.encode(
- &accepted_input_token_ids_const,
- self.context_ring.as_mut().unwrap(),
- suffix_repetition_length,
- full.len() as u32,
- &mut command_buffer,
- );
+ output_tokens = accepted_output_token_ids;
+
+ Some((accepted_token_indicies, accepted_input_token_ids))
+ } else {
+ if let Some(grammar) = self.options.grammar.as_mut() {
+ for &token_id in &output_tokens {
+ grammar.accept_token(token_id)?
+ }
}
- self.model_state.tokens.extend(accepted_output_token_ids);
- (
- ForwardPassChaining::Constant {
- token: output_token_id,
- output_norm,
- },
- Some((command_buffer, capture_span)),
- )
+ None
+ };
+
+ ResolvedJob {
+ output_tokens,
+ output_norm_buffer,
+ output_tokens_buffer: Some(output_tokens_buffer),
+ batch_dim,
+ hidden_features,
+ accepted_indices_and_input_tokens,
+ capture_span,
}
},
- DecodingState::Halted => return Ok(None),
- DecodingState::Invalid => unreachable!(),
- DecodingState::Closed => return Ok(None),
+ GenerationJob::Resolved(resolved) => resolved,
+ GenerationJob::Invalid => unreachable!(),
};
- let readback_buffer = match &prev_output {
- ForwardPassChaining::InFlight(pending) => Some(pending.output_tokens.clone()),
- _ => None,
+ *job = GenerationJob::Resolved(resolved);
+ let GenerationJob::Resolved(resolved) = &*job else {
+ unreachable!()
};
- let mut output_readback = readback_buffer.as_deref().map(|buffer| context.copyout::<u32>(buffer));
- let context_length = self.model_state.transformer_state.context_length();
+ Ok(resolved)
+ }
+
+ async fn resolve_take(
+ &mut self,
+ mut job: GenerationJob<B>,
+ ) -> Result<ResolvedJob<B>, LanguageModelStreamError<B>> {
+ self.resolve(&mut job).await?;
+ let GenerationJob::Resolved(resolved) = job else {
+ unreachable!()
+ };
+
+ Ok(resolved)
+ }
+
+ pub fn prefill(
+ &mut self,
+ input: &[u64],
+ ) -> Result<GenerationJob<B>, LanguageModelStreamError<B>> {
+ let _capture_span = CAPTURE_MATCHER.capture::<B>(&self.model.engine.context, "prefill")?;
+
+ // NOTE: this is required for attention correctness (hardcoded suffix
+ // 1024). This is really bad design, attention should be rewritten to
+ // allow on-demand suffix length
+ let max_batch_size = 1024;
+ let number_of_batches = input.len().div_ceil(max_batch_size);
+
+ self.model_state
+ .transformer_state
+ .prepare(
+ self.model_state.transformer_state.context_length() + ((number_of_batches - 1) * max_batch_size) as u32,
+ usize::min(max_batch_size, input.len()) as u32,
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
+
+ let command_buffer = self
+ .model
+ .engine
+ .context
+ .create_command_buffer(Some("prefill"), Some(self.allocation_pool.clone()))
+ .map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
+
+ command_buffer.debug_point(&format!(
+ "prefill at context {}, suffix {} ({} batches)",
+ self.model_state.transformer_state.context_length(),
+ input.len(),
+ number_of_batches
+ ));
+
+ let hidden_feature_layer_indices =
+ self.model.speculator.as_ref().map(|speculator| speculator.hidden_feature_layer_indices());
+
+ let mut encode_batch = |input_chunk: &[u64], sample: bool| -> Result<_, LanguageModelStreamError<B>> {
+ let token_ids = command_buffer
+ .allocate_constant_from_slice(
+ &input_chunk.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>(),
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
- if self.model_state.max_context_length.is_some_and(|max_context_length| context_length >= max_context_length) {
- self.decoding_state = DecodingState::Halted;
- return Ok(Some(
- prev_output
- .resolve(
- &mut output_readback,
- &mut self.model_state.tokens,
- self.options.timestamps.as_ref(),
- self.options.grammar.as_mut(),
- )
- .await?
- .0,
- ));
+ let batch_dim = BatchTopology::flat(input_chunk.len() as u32, true);
+
+ let decoder_output = self.model.decoder.encode(
+ &token_ids,
+ &batch_dim,
+ sample.then(|| (input_chunk.len() as u32 - 1..input_chunk.len() as u32).into()),
+ hidden_feature_layer_indices,
+ &mut self.model_state.transformer_state,
+ &mut command_buffer,
+ )?;
+
+ self.model_state
+ .transformer_state
+ .encode_accept(&(0..input_chunk.len() as u32).collect::<Box<[u32]>>(), &mut command_buffer)
+ .map_err(LanguageModelStreamError::Backend)?;
+
+ if let Some(speculator) = self.model.speculator.as_ref() {
+ let speculator_state = self.model_state.speculator_state.as_mut().unwrap();
+ speculator.encode_accept(
+ speculator_state,
+ decoder_output.hidden_features.as_ref().unwrap().iter(),
+ &(0..input_chunk.len() as u32).collect::<Box<[u32]>>(),
+ &mut command_buffer,
+ )?;
+ }
+
+ if !sample && let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {
+ self.model.context_ring_update.encode(
+ &token_ids,
+ self.context_ring.as_mut().unwrap(),
+ suffix_repetition_length,
+ input_chunk.len() as u32,
+ &mut command_buffer,
+ );
+ }
+
+ self.model_state.tokens.extend(input_chunk);
+
+ Ok((decoder_output, token_ids, batch_dim))
+ };
+
+ let mut token_chunks = input.chunks(max_batch_size);
+ for input_chunk in token_chunks.by_ref().take(number_of_batches - 1) {
+ encode_batch(input_chunk, false)?;
}
- let mut pending = Vec::new();
- if let Some((accept_command_buffer, _capture_span)) = acceptance {
- pending.push(accept_command_buffer.end_encoding().submit());
+ // `token_chunks` should have exactly one element left.
+ let final_chunk = token_chunks.next().unwrap();
+
+ let split_last_logits_row = self.model.decoder.prefill_cache_skips_trailing_layers();
+ let (decoder_output, token_ids, batch_dim) = if split_last_logits_row && final_chunk.len() > 1 {
+ let (start, end) = final_chunk.split_at(final_chunk.len() - 1);
+ encode_batch(start, false)?;
+ encode_batch(end, true)?
+ } else {
+ encode_batch(final_chunk, true)?
+ };
+
+ let logits = decoder_output.logits.unwrap();
+
+ let seeds = if matches!(self.options.sampling_method, SamplingMethod::Stochastic { .. }) {
+ Some(
+ command_buffer
+ .allocate_constant_from_slice(&[self
+ .model_state
+ .prng
+ .derive(self.model_state.tokens.len() as u64 - 1)])
+ .map_err(LanguageModelStreamError::Backend)?,
+ )
+ } else {
+ None
+ };
+
+ let bitmask = if let Some(grammar) = self.options.grammar.as_mut() {
+ let mut bitmask = command_buffer
+ .allocate_constant(
+ self.model.vocab_size.div_ceil(DataType::U32.size_in_bits()) * DataType::U32.size_in_bytes(),
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
+
+ if grammar.next_bitmask(bitmask.as_slice_mut()) {
+ Some(bitmask)
+ } else {
+ None
+ }
+ } else {
+ None
+ };
+
+ let output_norm = decoder_output.final_hidden;
+ let sampled_row = batch_dim.size() - 1;
+ let output_token = self
+ .model
+ .sampling
+ .encode(
+ &logits,
+ seeds.as_ref(),
+ bitmask.as_ref(),
+ self.context_ring.as_ref(),
+ Some(&token_ids),
+ &batch_dim,
+ (sampled_row..sampled_row + 1).into(),
+ &self.options.sampling_method,
+ &mut command_buffer,
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
+
+ if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {
+ self.model.context_ring_update.encode(
+ &token_ids,
+ self.context_ring.as_mut().unwrap(),
+ suffix_repetition_length,
+ batch_dim.size(),
+ &mut command_buffer,
+ );
}
+ drop(seeds);
+ drop(bitmask);
+ drop(token_ids);
+
+ self.metrics.num_prefill_forward_passes += 1;
+ self.metrics.num_tokens_prefilled += input.len();
+ self.metrics.num_tokens_proposed += 1;
+
+ Ok(GenerationJob::Pending {
+ output_norm,
+ output_tokens: output_token,
+ _input_token_source: None,
+ batch_dim,
+ trie: None,
+ hidden_features: decoder_output.hidden_features,
+ capture_span: None,
+ pendings: Box::new([command_buffer.end_encoding().submit()]),
+ })
+ }
+
+ pub async fn get_and_queue_pass(
+ &mut self,
+ mut last_job: GenerationJob<B>,
+ ) -> Result<(ResolvedJob<B>, GenerationJob<B>), LanguageModelStreamError<B>> {
let mut capture_span = CAPTURE_MATCHER
.capture::<B>(&self.model.engine.context, &format!("decode:{}", self.metrics.num_decode_forward_passes))?;
- #[allow(clippy::let_and_return)]
- let (input_trie, chain_copy, full_accept) = if let Some(speculator) = &self.model.speculator
+ // Speculator pass
+ let context_length = self.model_state.transformer_state.context_length();
+ let trie = if self.propose_trie
+ && let Some(speculator) = &self.model.speculator
&& let Some(shape) = speculator.make_shape(
self.model_state.max_context_length.map(|max_context_length| max_context_length - context_length),
- )
- && let (root_token, Some(output_norm)) = {
- let resolved = prev_output
- .resolve(
- &mut output_readback,
- &mut self.model_state.tokens,
- self.options.timestamps.as_ref(),
- self.options.grammar.as_mut(),
- )
- .await?;
- resolved
- } {
+ ) {
+ // TODO: CPU-bubble
+ let resolved = self.resolve(&mut last_job).await?;
+ let tokens = &resolved.output_tokens;
+ let batch_dim = &resolved.batch_dim;
+ let final_hidden = resolved.output_norm_buffer.as_ref().unwrap();
+
+ let last_norm_buffer =
+ if let Some((accepted_indices, _)) = resolved.accepted_indices_and_input_tokens.as_ref() {
+ let row = (*accepted_indices.last().unwrap()) as usize;
+ let row_bytes = final_hidden.size() / batch_dim.size() as usize;
+ final_hidden.subrange(row * row_bytes..(row + 1) * row_bytes)
+ } else {
+ final_hidden.subrange(..)
+ };
+
+ // TODO: CPU-bubble: for next `batch_dim`. To remove we need to
+ // contruct the trie nodes GPU-side and have `batch_dim` carry a
+ // GPU representation.
let trie = speculator
.propose_tree(
self.model_state.speculator_state.as_mut().unwrap(),
- output_norm,
- root_token as u32,
+ last_norm_buffer,
+ tokens[tokens.len() - 1] as u32,
self.model.decoder.embedding(),
shape,
&self.options.sampling_method,
@@ -581,66 +463,61 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.options.timestamps.as_ref(),
)
.await?;
- prev_output.drop_output_norm();
- (trie, None, false)
+ Some(trie)
} else {
- prev_output.drop_output_norm();
- let (token, chain_copy) = match &prev_output {
- ForwardPassChaining::Constant {
- token,
- ..
- } => (*token, None),
- ForwardPassChaining::InFlight(pending) => (0, Some(pending.output_tokens.clone())),
- };
- (TrieNode::new(token, self.model_state.prng.derive(context_length as u64), 0.0), chain_copy, true)
+ None
};
- let input_flat_trie = input_trie.linearize();
-
- self.allocation_pool = self.model.engine.context.create_allocation_pool();
+ // Main (verifier) forward pass
let command_buffer = self
.model
.engine
.context
.create_command_buffer(
- Some(&format!("decode {} (forward)", self.metrics.num_decode_forward_passes)),
+ Some(&format!("decode {}", self.metrics.num_decode_forward_passes)),
Some(self.allocation_pool.clone()),
)
.map_err(LanguageModelStreamError::Backend)?;
- let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
- command_buffer.debug_point(&format!(
- "decode round {} at context {} suffix {}",
- self.metrics.num_decode_forward_passes,
- context_length,
- input_flat_trie.len()
- ));
+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
- let scratch_token_ids;
let mut constant_token_ids = None;
- let token_ids: &dyn GpuBuffer<Backend = B> = if let Some(chain_copy) = chain_copy.as_deref() {
- let mut token_ids = command_buffer
- .allocate_scratch(DataType::U32.size_in_bytes())
+ let mut needs_last_job_input = false;
+ let flat_trie = trie.as_ref().map(|trie| trie.linearize());
+ let (token_ids, batch_dim): (&dyn GpuBuffer<Backend = B>, _) = if let Some(flat_trie) = flat_trie.as_ref() {
+ let batch_dim = BatchTopology::new(flat_trie.token_subtrie_ranges().collect::<Box<[_]>>(), false);
+
+ let token_ids = command_buffer
+ .allocate_constant_from_slice(
+ &flat_trie.token_ids().map(|token_id| token_id as u32).collect::<Box<[u32]>>(),
+ )
.map_err(LanguageModelStreamError::Backend)?;
- command_buffer.encode_copy(chain_copy, &mut token_ids);
- scratch_token_ids = token_ids;
- &scratch_token_ids
+
+ constant_token_ids = Some(token_ids);
+ (constant_token_ids.as_ref().unwrap(), batch_dim)
+ } else if last_job.batch_dim().size() == 1 {
+ needs_last_job_input = true;
+ (last_job.output_tokens_buffer(), BatchTopology::flat(1, true))
} else {
- constant_token_ids.insert(
- command_buffer
- .allocate_constant_from_slice(
- &input_flat_trie.token_ids().map(|token_id| token_id as u32).collect::<Box<[u32]>>(),
- )
- .map_err(LanguageModelStreamError::Backend)?,
- )
+ let resolved = self.resolve(&mut last_job).await?;
+ let token = *resolved.output_tokens.last().unwrap() as u32;
+ let token_ids =
+ command_buffer.allocate_constant_from_slice(&[token]).map_err(LanguageModelStreamError::Backend)?;
+
+ constant_token_ids = Some(token_ids);
+ (constant_token_ids.as_ref().unwrap(), BatchTopology::flat(1, true))
};
- let input_flat_trie_nodes = input_flat_trie.token_subtrie_ranges().collect::<Box<[GpuTrieNode]>>();
- let batch_dim = BatchTopology::new(&input_flat_trie_nodes, full_accept);
+ command_buffer.debug_point(&format!(
+ "decode round {} at context {} suffix {}",
+ self.metrics.num_decode_forward_passes,
+ context_length,
+ batch_dim.size()
+ ));
self.model_state
.transformer_state
- .prepare(self.model_state.transformer_state.context_length(), batch_dim.size())
+ .prepare(context_length, batch_dim.size())
.map_err(LanguageModelStreamError::Backend)?;
let hidden_feature_layer_indices =
@@ -654,67 +531,60 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
&mut self.model_state.transformer_state,
&mut command_buffer,
)?;
- let logits = decoder_output.logits.unwrap();
- let (bitmask, mut command_buffer) = if let Some(grammar) = self.options.grammar.as_mut() {
- if chain_copy.is_some() {
- pending.push(command_buffer.end_encoding().submit());
+ let mut pendings = Vec::new();
- let command_buffer = self
- .model
- .engine
- .context
- .create_command_buffer(
- Some(&format!("decode {} (sample and accept)", self.metrics.num_decode_forward_passes)),
- Some(self.allocation_pool.clone()),
- )
- .map_err(LanguageModelStreamError::Backend)?;
- let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
+ // sampling
+ let (mut command_buffer, token_ids): (_, &dyn GpuBuffer<Backend = B>) =
+ if needs_last_job_input && self.options.grammar.is_some() {
+ pendings.push(command_buffer.end_encoding().submit());
+ let command_buffer = record(
+ self.model
+ .engine
+ .context
+ .create_command_buffer(
+ Some(&format!("decode {} (sample and accept)", self.metrics.num_decode_forward_passes)),
+ Some(self.allocation_pool.clone()),
+ )
+ .map_err(LanguageModelStreamError::Backend)?,
+ self.options.timestamps.as_ref(),
+ );
- let mut bitmask = command_buffer
- .allocate_constant(
- self.model.vocab_size.div_ceil(DataType::U32.size_in_bits()) * DataType::U32.size_in_bytes(),
- )
- .map_err(LanguageModelStreamError::Backend)?;
+ // NOTE: Not a CPU bubble, forward pass is overlapped.
+ self.resolve(&mut last_job).await?;
+ (command_buffer, last_job.output_tokens_buffer())
+ } else {
+ (command_buffer, token_ids)
+ };
- prev_output
- .resolve(
- &mut output_readback,
- &mut self.model_state.tokens,
- self.options.timestamps.as_ref(),
- Some(grammar),
- )
- .await?;
- if grammar.next_bitmask(bitmask.as_slice_mut()) {
- (Some(bitmask), command_buffer)
- } else {
- (None, command_buffer)
- }
+ let bitmask = if let Some(grammar) = self.options.grammar.as_mut() {
+ let mut bitmask = command_buffer
+ .allocate_constant(
+ self.model.vocab_size.div_ceil(DataType::U32.size_in_bits())
+ * DataType::U32.size_in_bytes()
+ * batch_dim.size() as usize,
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
+
+ let was_filled = if let Some(flat_trie) = flat_trie.as_ref() {
+ flat_trie.fill_bitmasks(bitmask.as_slice_mut(), self.model.vocab_size, grammar)
} else {
- let mut bitmasks = command_buffer
- .allocate_constant(
- input_flat_trie.len()
- * self.model.vocab_size.div_ceil(DataType::U32.size_in_bits())
- * DataType::U32.size_in_bytes(),
- )
- .map_err(LanguageModelStreamError::Backend)?;
+ grammar.next_bitmask(bitmask.as_slice_mut())
+ };
- if input_flat_trie.fill_bitmasks(bitmasks.as_slice_mut(), self.model.vocab_size, grammar) {
- (Some(bitmasks), command_buffer)
- } else {
- (None, command_buffer)
- }
- }
+ was_filled.then_some(bitmask)
} else {
- (None, command_buffer)
+ None
};
let seeds = if matches!(self.options.sampling_method, SamplingMethod::Stochastic { .. }) {
- Some(
- command_buffer
- .allocate_constant_from_slice(&input_flat_trie.token_seeds().collect::<Box<[u64]>>())
- .map_err(LanguageModelStreamError::Backend)?,
- )
+ let seeds = if let Some(flat_trie) = flat_trie.as_ref() {
+ flat_trie.token_seeds().collect::<Box<[u64]>>()
+ } else {
+ Box::new([self.model_state.prng.derive(context_length as u64)])
+ };
+
+ Some(command_buffer.allocate_constant_from_slice(&seeds).map_err(LanguageModelStreamError::Backend)?)
} else {
None
};
@@ -723,7 +593,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.model
.sampling
.encode(
- &logits,
+ &decoder_output.logits.unwrap(),
seeds.as_ref(),
bitmask.as_ref(),
self.context_ring.as_ref(),
@@ -737,9 +607,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
drop(seeds);
drop(bitmask);
- drop(logits);
- if full_accept {
+ // new job and statistics
+ self.metrics.num_decode_forward_passes += 1;
+ self.metrics.num_tokens_proposed += batch_dim.size() as usize;
+
+ if batch_dim.full_accept() {
self.model_state
.transformer_state
.encode_accept(&(0..batch_dim.size()).collect::<Box<[u32]>>(), &mut command_buffer)
@@ -767,145 +640,226 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
}
drop(constant_token_ids);
- pending.push(command_buffer.end_encoding().submit());
- if full_accept {
+ pendings.push(command_buffer.end_encoding().submit());
+
+ if batch_dim.full_accept() {
capture_span.take();
}
- self.metrics.num_decode_forward_passes += 1;
- self.metrics.num_tokens_proposed += input_flat_trie.len();
- if full_accept {
- self.metrics.num_tokens_accepted += input_flat_trie.len();
+ let mut new_job = GenerationJob::Pending {
+ output_norm: decoder_output.final_hidden,
+ output_tokens,
+ _input_token_source: None,
+ batch_dim,
+ trie,
+ hidden_features: decoder_output.hidden_features,
+ capture_span,
+ pendings: pendings.into_boxed_slice(),
+ };
+
+ // Return results
+ // NOTE: not a CPU bubble. The next job has already been scheduled.
+ let mut resolved_job = self.resolve_take(last_job).await?;
+
+ if needs_last_job_input {
+ // Keep previous input alive if necessary.
+ let GenerationJob::Pending {
+ _input_token_source,
+ ..
+ } = &mut new_job
+ else {
+ unreachable!()
+ };
+ *_input_token_source = resolved_job.output_tokens_buffer.take();
}
- self.decoding_state = DecodingState::ForwardPassPending(DecodingStatePending {
- input_trie,
- full_accept,
- pending: pending.into_boxed_slice(),
- capture_span,
- hidden_features: if full_accept {
- None
- } else {
- decoder_output.hidden_features
- },
- output_norm: decoder_output.final_hidden,
- output_tokens: Arc::new(output_tokens),
- _input_token_source: chain_copy,
- });
-
- Ok(Some(
- prev_output
- .resolve(
- &mut output_readback,
- &mut self.model_state.tokens,
- self.options.timestamps.as_ref(),
- self.options.grammar.as_mut(),
+ Ok((resolved_job, new_job))
+ }
+
+ fn encode_accept(
+ &mut self,
+ resolved: &mut ResolvedJob<B>,
+ accept_len: usize,
+ ) -> Result<Option<<<B as Backend>::CommandBuffer as CommandBuffer>::Pending>, LanguageModelStreamError<B>> {
+ let batch_dim = &resolved.batch_dim;
+
+ if batch_dim.full_accept() {
+ return Ok(None);
+ }
+
+ let command_buffer = self
+ .model
+ .engine
+ .context
+ .create_command_buffer(Some("accept"), Some(self.allocation_pool.clone()))
+ .map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
+
+ let (accepted_token_indices, accepted_input_token_ids) =
+ resolved.accepted_indices_and_input_tokens.as_ref().unwrap();
+ let accepted_token_indices = &accepted_token_indices[..accept_len];
+ let accepted_input_token_ids = &accepted_input_token_ids[..accept_len];
+
+ self.model_state
+ .transformer_state
+ .encode_accept(accepted_token_indices, &mut command_buffer)
+ .map_err(LanguageModelStreamError::Backend)?;
+
+ if let Some(speculator) = self.model.speculator.as_ref() {
+ speculator.encode_accept(
+ self.model_state.speculator_state.as_mut().unwrap(),
+ resolved.hidden_features.as_ref().unwrap().iter(),
+ accepted_token_indices,
+ &mut command_buffer,
+ )?;
+ }
+
+ if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {
+ // TODO: This can be provided by a `verify_trie` kernel.
+ let accepted_input_token_ids_const = command_buffer
+ .allocate_constant_from_slice(
+ &accepted_input_token_ids.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>(),
)
- .await?
- .0,
- ))
+ .map_err(LanguageModelStreamError::Backend)?;
+
+ self.model.context_ring_update.encode(
+ &accepted_input_token_ids_const,
+ self.context_ring.as_mut().unwrap(),
+ suffix_repetition_length,
+ accepted_token_indices.len() as u32,
+ &mut command_buffer,
+ );
+ }
+
+ let pending = command_buffer.end_encoding().submit();
+ resolved.capture_span.take();
+
+ Ok(Some(pending))
}
+}
- pub fn metrics(&self) -> &TokenStreamMetrics {
- &self.metrics
+impl<'a, B: Backend> LanguageModelStream<'a, B> {
+ pub fn new(
+ model: &'a LanguageModel<B>,
+ input: &[u64],
+ model_state: &'a mut LanguageModelState<B>,
+ options: LanguageModelStreamOptions,
+ ) -> Result<Self, LanguageModelStreamError<B>> {
+ let mut submitter = StreamEnqueuer::new(model, input, model_state, options)?;
+ let job = submitter.prefill(input)?;
+
+ Ok(Self {
+ submitter,
+ emitting: None,
+ job: Some(job),
+ })
}
- pub async fn close(&mut self) -> Result<(), LanguageModelStreamError<B>> {
- let last_output_token = match replace(&mut self.decoding_state, DecodingState::Closed) {
- DecodingState::Seeded {
- seed_token,
- } => Some(seed_token),
- DecodingState::ForwardPassPending(in_flight) => {
- for pending in in_flight.pending {
- wait(pending, self.options.timestamps.as_ref()).await.map_err(LanguageModelStreamError::Backend)?;
- }
+ pub fn closed(&self) -> bool {
+ self.job.is_none() && self.emitting.is_none()
+ }
- if !in_flight.full_accept {
- let command_buffer = self
- .model
- .engine
- .context
- .create_command_buffer(
- Some(&format!("decode {} (accept on drop)", self.metrics.num_decode_forward_passes - 1)),
- Some(self.allocation_pool.clone()),
- )
- .map_err(LanguageModelStreamError::Backend)?;
- let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
- self.model_state
- .transformer_state
- .encode_accept(&[0], &mut command_buffer)
- .map_err(LanguageModelStreamError::Backend)?;
- if let Some(speculator) = self.model.speculator.as_ref() {
- speculator.encode_accept(
- self.model_state.speculator_state.as_mut().unwrap(),
- in_flight.hidden_features.as_ref().unwrap().iter(),
- &[0],
- &mut command_buffer,
- )?;
- }
- wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref())
- .await
- .map_err(LanguageModelStreamError::Backend)?;
- }
+ fn emit_token(&mut self) -> Option<u64> {
+ let (resolved, i) = self.emitting.as_mut()?;
+ if *i >= resolved.output_tokens.len() {
+ return None;
+ }
- let context = &self.model.engine.context;
- Some(
- context
- .copyout::<u32>(in_flight.output_tokens.subrange(..size_of::<u32>()))
- .await
- .map_err(LanguageModelStreamError::Backend)?[0] as u64,
- )
- },
- DecodingState::Accepting {
- full,
- num_accepted,
- hidden_features,
- output_norm: _,
- capture_span,
- } => {
- assert!(num_accepted > 0 && num_accepted < full.len());
+ let token = resolved.output_tokens[*i];
+ *i += 1;
- let command_buffer = self
- .model
- .engine
- .context
- .create_command_buffer(
- Some(&format!("decode {} (accept on drop)", self.metrics.num_decode_forward_passes - 1)),
- Some(self.allocation_pool.clone()),
- )
- .map_err(LanguageModelStreamError::Backend)?;
- let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
- let accepted_token_indicies =
- full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();
- self.model_state
- .transformer_state
- .encode_accept(&accepted_token_indicies, &mut command_buffer)
- .map_err(LanguageModelStreamError::Backend)?;
- if let Some(speculator) = self.model.speculator.as_ref() {
- speculator.encode_accept(
- self.model_state.speculator_state.as_mut().unwrap(),
- hidden_features.as_ref().unwrap().iter(),
- &accepted_token_indicies,
- &mut command_buffer,
- )?;
- }
- wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref())
+ self.submitter.model_state.tokens.push(token);
+ self.submitter.metrics.num_tokens_accepted += 1;
+ self.submitter.metrics.num_tokens_returned += 1;
+ Some(token)
+ }
+
+ pub async fn generate(&mut self) -> Result<Option<u64>, LanguageModelStreamError<B>> {
+ if let Some(token) = self.emit_token() {
+ return Ok(Some(token));
+ }
+
+ if let Some(job) = self.job.take() {
+ let context_full = self.submitter.model_state.max_context_length.is_some_and(|max_context_length| {
+ self.submitter.model_state.transformer_state.context_length() >= max_context_length
+ });
+
+ if job.batch_dim().full_accept() && !context_full {
+ let (last_job, new_job) = self.submitter.get_and_queue_pass(job).await?;
+ self.job = Some(new_job);
+ self.emitting = Some((last_job, 0));
+ } else {
+ let resolved = self.submitter.resolve_take(job).await?;
+ self.emitting = Some((resolved, 0))
+ }
+ } else if let Some((mut queued, emitted_len)) = self.emitting.take() {
+ let accept_pending = self.submitter.encode_accept(&mut queued, emitted_len)?;
+ if let Some(pending) = accept_pending {
+ wait(pending, self.submitter.options.timestamps.as_ref())
.await
.map_err(LanguageModelStreamError::Backend)?;
+ }
+
+ let context_full = self.submitter.model_state.max_context_length.is_some_and(|max_context_length| {
+ self.submitter.model_state.transformer_state.context_length() >= max_context_length
+ });
+
+ if !context_full {
+ let (_, new_job) = self.submitter.get_and_queue_pass(GenerationJob::Resolved(queued)).await?;
+ let resolved_new = self.submitter.resolve_take(new_job).await?;
+ self.emitting = Some((resolved_new, 0))
+ } else {
+ assert!(self.emit_token().is_none());
+ }
+ }
- drop(capture_span);
+ Ok(self.emit_token())
+ }
- self.model_state.tokens.extend(full.iter().take(num_accepted).map(|(_, _, t)| *t));
+ pub fn metrics(&self) -> &TokenStreamMetrics {
+ &self.submitter.metrics
+ }
- Some(full[num_accepted].2)
- },
- DecodingState::Halted => None,
- DecodingState::Invalid => None, // TODO: proper error handling
- DecodingState::Closed => self.model_state.last_output_token,
+ pub async fn close(&mut self) -> Result<(), LanguageModelStreamError<B>> {
+ if let Some((mut last_resolved, emitted_len)) = self.emitting.take() {
+ let output_len = last_resolved.output_tokens.len();
+ let accept_len = output_len.min(emitted_len + 1);
+ let accept_pending = self.submitter.encode_accept(&mut last_resolved, accept_len)?;
+ if let Some(pending) = accept_pending {
+ wait(pending, self.submitter.options.timestamps.as_ref())
+ .await
+ .map_err(LanguageModelStreamError::Backend)?;
+ }
+
+ let context_full = self.submitter.model_state.max_context_length.is_some_and(|max_context_length| {
+ self.submitter.model_state.transformer_state.context_length() >= max_context_length
+ });
+
+ // Make sure last token is accepted.
+ if emitted_len == output_len && !context_full {
+ let next_job = match self.job.take() {
+ Some(job) => job,
+ None => {
+ self.submitter.propose_trie = false;
+ let queued = self.submitter.get_and_queue_pass(GenerationJob::Resolved(last_resolved)).await;
+ self.submitter.propose_trie = true;
+ queued?.1
+ },
+ };
+ let mut next_resolved = self.submitter.resolve_take(next_job).await?;
+ let accept_pending = self.submitter.encode_accept(&mut next_resolved, 1)?;
+ if let Some(pending) = accept_pending {
+ wait(pending, self.submitter.options.timestamps.as_ref())
+ .await
+ .map_err(LanguageModelStreamError::Backend)?;
+ }
+ }
};
- self.model_state.last_output_token = last_output_token;
+ if let Some(job) = self.job.take() {
+ self.submitter.resolve_take(job).await?;
+ }
Ok(())
}
@@ -913,8 +867,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
fn drop(&mut self) {
- if !matches!(self.decoding_state, DecodingState::Closed)
+ if !self.closed()
&& self
+ .submitter
.model
.engine
.context
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index ebb165fc7..fccbe2d7c 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -13,7 +13,6 @@ pub use crate::encodable_block::dflash::DFlashState;
use crate::{
backends::common::{
Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, TimestampSpan,
- gpu_types::trie::TrieNode as GpuTrieNode,
},
config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},
data_type::DataType,
@@ -278,14 +277,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
state,
&mut command_buffer,
)?;
- let topology_nodes = (0..chain_length)
- .map(|index| GpuTrieNode {
- trie_start: index,
- trie_end: chain_length - 1,
- height: index,
- })
- .collect::<Box<[_]>>();
- let batch_topology = BatchTopology::new(&topology_nodes, true);
+ let batch_topology = BatchTopology::flat(chain_length, true);
let sampled = self
.sampling
.encode(
diff --git a/crates/uzu-engine/src/trie.rs b/crates/uzu-engine/src/trie.rs
index 42abaaff2..f55bb5956 100644
--- a/crates/uzu-engine/src/trie.rs
+++ b/crates/uzu-engine/src/trie.rs
@@ -4,7 +4,6 @@ use thiserror::Error;
use crate::{
backends::common::gpu_types::trie::TrieNode as GpuTrieNode,
data_type::DataType,
- encodable_block::sampling::PRng,
engine::language_model::grammar::{Grammar, GrammarError},
};
@@ -135,24 +134,6 @@ impl TrieNode {
prune(self, &kept, &mut 0);
}
- pub fn flat(
- prefix_length: usize,
- tokens: &[u64],
- prng: &PRng,
- ) -> Self {
- assert!(!tokens.is_empty(), "need seed node");
-
- let mut root = TrieNode::new(tokens[0], prng.derive(prefix_length as u64), 0.0);
- let mut leaf = &mut root;
-
- for (index, token) in tokens.iter().copied().enumerate().skip(1) {
- leaf.add(TrieNode::new(token, prng.derive((prefix_length + index) as u64), 0.0)).unwrap();
- leaf = &mut leaf.next[0];
- }
-
- root
- }
-
pub fn linearize(&self) -> FlatTrie<'_> {
let mut tokens = vec![FlatTrieNode::new(self, (0, 0), 0)];
@@ -197,6 +178,7 @@ impl<'a> FlatTrie<'a> {
}
}
+ #[cfg(test)]
pub fn len(&self) -> usize {
self.tokens.len()
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/repetition_penalty_test.rs b/crates/uzu-engine/unit/backends/common/kernel/repetition_penalty_test.rs
index 4af430b4e..eed5038af 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/repetition_penalty_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/repetition_penalty_test.rs
@@ -146,7 +146,7 @@ fn check_sampling<B: Backend, T: DataTypeRepr + Float + Debug>() {
};
for flat in [false, true] {
let nodes = nodes(flat);
- let topology = BatchTopology::new(&nodes, flat);
+ let topology = BatchTopology::new(nodes.into_boxed_slice(), flat);
for range in [0..9, 3..5, 7..9] {
let mut logits = vec![T::from(-16.0).unwrap(); range.len() * VOCAB_SIZE];
for row in logits.as_chunks_mut::<VOCAB_SIZE>().0 {
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
index 80e45225f..b30c6e377 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
@@ -28,6 +28,22 @@ use crate::{
type AttentionShape = (usize, usize, usize, usize, usize, bool);
+fn flat_trie(
+ prefix_length: usize,
+ tokens: &[u64],
+ prng: &PRng,
+) -> TrieNode {
+ assert!(!tokens.is_empty(), "need seed node");
+
+ let mut node = TrieNode::new(tokens[0], prng.derive(prefix_length as u64), 0.0);
+ for (index, token) in tokens.iter().copied().enumerate().skip(1).rev() {
+ let mut parent = TrieNode::new(token, prng.derive((prefix_length + index) as u64), 0.0);
+ parent.add(node).unwrap();
+ node = parent;
+ }
+ node
+}
+
pub(super) fn fill_attention(
count: usize,
phase: f32,
@@ -85,7 +101,7 @@ fn attention_matches_cpu() {
])
.to_vec()
} else {
- TrieNode::flat(0, &tokens, &PRng::new(0)).linearize().token_subtrie_ranges().collect()
+ flat_trie(0, &tokens, &PRng::new(0)).linearize().token_subtrie_ranges().collect()
};
let (shape, nodes) =
((head_dim, q_heads, kv_heads, suffix, prefix, true), use_trie.then_some(trie.as_slice()));
diff --git a/crates/uzu-engine/unit/encodable_block/sampling_test.rs b/crates/uzu-engine/unit/encodable_block/sampling_test.rs
index db5534c79..6358db14c 100644
--- a/crates/uzu-engine/unit/encodable_block/sampling_test.rs
+++ b/crates/uzu-engine/unit/encodable_block/sampling_test.rs
@@ -8,10 +8,7 @@ use rand_distr::Normal;
use uzu_engine_macros::uzu_test;
use crate::{
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
- gpu_types::trie::TrieNode,
- },
+ backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},
data_type::{DataType, DataTypeRepr},
encodable_block::{
batch_topology::BatchTopology,
@@ -51,14 +48,7 @@ fn do_sampling_backend<B: Backend, T: DataTypeRepr + Float>(
let seeds_buffer = seeds.map(|seeds| command_buffer.allocate_constant_from_slice(seeds).unwrap());
let bitmask_buffer = bitmask.map(|bitmask| command_buffer.allocate_constant_from_slice(bitmask).unwrap());
- let nodes = (0..batch_size)
- .map(|index| TrieNode {
- trie_start: index,
- trie_end: batch_size - 1,
- height: index,
- })
- .collect::<Box<[_]>>();
- let batch_topology = BatchTopology::new(&nodes, true);
+ let batch_topology = BatchTopology::flat(batch_size, true);
let sampled_buffer = sampling
.encode(