A high-performance inference engine for AI models
Simplify decode loop (#1078)
odilf4a386eba7cf8
diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rsindex 629aa716c..1f6c90cbc 100644--- a/crates/uzu-engine/src/backends/common/command_buffer.rs+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs@@ -86,6 +86,7 @@ pub trait CommandBufferEncoding: Send {fn with_timestamps(self) -> Self;+ #[track_caller]fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;}diff --git a/crates/uzu-engine/src/encodable_block/batch_topology.rs b/crates/uzu-engine/src/encodable_block/batch_topology.rsindex 4606f8562..52b844f9b 100644--- a/crates/uzu-engine/src/encodable_block/batch_topology.rs+++ b/crates/uzu-engine/src/encodable_block/batch_topology.rs@@ -1,15 +1,15 @@use crate::backends::common::gpu_types::trie::TrieNode;-pub struct BatchTopology<'a> {- nodes: &'a [TrieNode],+pub struct BatchTopology {+ nodes: Box<[TrieNode]>,parents: Box<[i32]>,is_flat: bool,full_accept: bool,}-impl<'a> BatchTopology<'a> {+impl BatchTopology {pub fn new(- nodes: &'a [TrieNode],+ nodes: Box<[TrieNode]>,full_accept: bool,) -> Self {let mut stack: Vec<usize> = Vec::new();@@ -25,7 +25,6 @@ impl<'a> BatchTopology<'a> {is_flat &= node.height == index;}let parents = unsafe { parents.assume_init() };-assert!(is_flat || !full_accept, "full_accept requires trie to be flat");Self {@@ -36,8 +35,19 @@ impl<'a> BatchTopology<'a> {}}- pub fn nodes(&self) -> &'a [TrieNode] {- self.nodes+ pub fn flat(+ size: u32,+ full_accept: bool,+ ) -> Self {+ let nodes = (0..size)+ .map(|index| TrieNode {+ trie_start: index,+ trie_end: size - 1,+ height: index,+ })+ .collect();++ Self::new(nodes, full_accept)}pub fn size(&self) -> u32 {@@ -52,11 +62,15 @@ impl<'a> BatchTopology<'a> {self.full_accept}- pub fn heights(&self) -> impl Iterator<Item = u32> + 'a {- self.nodes.iter().map(|trie| trie.height)+ pub fn nodes(&self) -> &[TrieNode] {+ &self.nodes}pub fn parents(&self) -> &[i32] {&self.parents}++ pub fn heights(&self) -> impl Iterator<Item = u32> {+ self.nodes().iter().map(|t| t.height)+ }}diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rsindex 98aced9bb..684b09799 100644--- a/crates/uzu-engine/src/encodable_block/classifier.rs+++ b/crates/uzu-engine/src/encodable_block/classifier.rs@@ -2,10 +2,7 @@ use derive_more::Debug;use thiserror::Error;use crate::{- backends::common::{- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels, gpu_types::trie::TrieNode,- kernel::PoolingMeanKernel,- },+ backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels, kernel::PoolingMeanKernel},config::classifier::{ClassifierConfig, PoolingType},data_type::DataType,encodable_block::{@@ -135,19 +132,12 @@ impl<B: Backend> Classifier<B> {.encode(&embedded, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer).map_err(ClassifierError::Backend)?;- let nodes = (0..batch_dim)- .map(|index| TrieNode {- trie_start: index,- trie_end: batch_dim - 1,- height: index,- })- .collect::<Box<[TrieNode]>>();let hidden = self.transformer.encode(hidden,None::<&B::ScratchBuffer>,- &BatchTopology::new(&nodes, true),+ &BatchTopology::flat(batch_dim, true),Some((0..batch_dim).into()),None,None,diff --git a/crates/uzu-engine/src/encodable_block/decision.rs b/crates/uzu-engine/src/encodable_block/decision.rsindex 55c7d131c..cf193ad87 100644--- a/crates/uzu-engine/src/encodable_block/decision.rs+++ b/crates/uzu-engine/src/encodable_block/decision.rs@@ -2,7 +2,7 @@ use derive_more::Debug;use thiserror::Error;use crate::{- backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding, gpu_types::trie::TrieNode},+ backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding},config::model::decision_model::DecisionModelConfig,data_type::DataType,encodable_block::{@@ -114,20 +114,12 @@ impl<B: Backend> Decision<B> {.map_err(DecisionError::Backend)?;self.input_embedding.encode_lookup(token_ids, &mut embedded, token_count, self.input_scale, command_buffer);- let nodes = (0..token_count)- .map(|index| TrieNode {- trie_start: index,- trie_end: token_count - 1,- height: index,- })- .collect::<Box<[TrieNode]>>();-let hidden = self.transformer.encode(embedded,None::<&B::ScratchBuffer>,- &BatchTopology::new(&nodes, true),+ &BatchTopology::flat(token_count, true),Some((token_count - 1..token_count).into()),None,None,diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rsindex b48d7b03c..ff4b42bd7 100644--- a/crates/uzu-engine/src/encodable_block/dflash.rs+++ b/crates/uzu-engine/src/encodable_block/dflash.rs@@ -4,9 +4,7 @@ use derive_more::Debug;use thiserror::Error;use crate::{- backends::common::{- Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, gpu_types::trie::TrieNode,- },+ backends::common::{Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding},config::{dflash::DFlashDraftConfig, rope::AnyRoPEConfig, token_mixer::AnyTokenMixerConfig},data_type::DataType,encodable_block::{@@ -310,14 +308,7 @@ impl<B: Backend> DFlash<B> {let token_embeddings = target_embedding.encode_lookup(&token_ids, batch_size, command_buffer)?;- let nodes = (0..batch_size)- .map(|index| TrieNode {- trie_start: index,- trie_end: batch_size - 1,- height: index,- })- .collect::<Box<[_]>>();- let batch_topology = BatchTopology::new(&nodes, true);+ let batch_topology = BatchTopology::flat(batch_size, true);let token_positions = (state.context_length..state.context_length + batch_size).collect::<Box<[_]>>();let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, command_buffer).map_err(DFlashEncodeError::Backend)?;diff --git a/crates/uzu-engine/src/engine/language_model/state.rs b/crates/uzu-engine/src/engine/language_model/state.rsindex c7d755635..d11a4c1e0 100644--- a/crates/uzu-engine/src/engine/language_model/state.rs+++ b/crates/uzu-engine/src/engine/language_model/state.rs@@ -9,7 +9,6 @@ use crate::{pub struct LanguageModelState<B: Backend> {pub(super) tokens: Vec<u64>,- pub(super) last_output_token: Option<u64>, // TODO: this leaks previous LanguageModelStreamOptionspub(super) prng: PRng,pub(super) transformer_state: TransformerState<B>,pub(super) speculator_state: Option<DFlashState<B>>,@@ -42,7 +41,6 @@ impl<B: Backend> LanguageModel<B> {sampling_seed: u64,) -> Result<LanguageModelState<B>, LanguageModelCreateEmptyStateError<B>> {let tokens = Vec::new();- let last_output_token = None;let prng = PRng::new(sampling_seed);@@ -62,7 +60,6 @@ impl<B: Backend> LanguageModel<B> {Ok(LanguageModelState {tokens,- last_output_token,prng,transformer_state,speculator_state,diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex 1c51ce5aa..ed6af073a 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -1,8 +1,8 @@use std::{iter::{once, repeat_n},- mem::replace,+ mem,range::Range,- sync::{Arc, mpsc::Sender},+ sync::Arc,};use futures::executor::block_on;@@ -11,16 +11,14 @@ use shoji::traits::backend::chat_token::TokenStreamMetrics;use crate::{backends::common::{Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, CpuBuffer,- DeviceCapabilities, GlobalBufferCpuGpu, GpuBuffer, TimestampSpan,+ DeviceCapabilities, GlobalBufferCpuGpu, GpuBuffer,capture_matcher::{CAPTURE_MATCHER, CaptureMatcherSpan},- gpu_types::trie::TrieNode as GpuTrieNode,kernel::ContextRingUpdateKernel,},data_type::DataType,encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},engine::language_model::{LanguageModel,- grammar::Grammar,state::LanguageModelState,stream::{LanguageModelStreamError, LanguageModelStreamOptions},},@@ -31,131 +29,79 @@ use crate::{},};-enum ForwardPassChaining<B: Backend> {- Constant {- token: u64,- output_norm: Option<B::ScratchBuffer>,- },- InFlight(DecodingStatePending<B>),-}--impl<B: Backend> ForwardPassChaining<B> {- async fn resolve<'a>(- &'a mut self,- readback: &mut Option<impl Future<Output = Result<Vec<u32>, B::Error>> + Send>,- tokens: &mut Vec<u64>,- timestamps: Option<&Sender<Box<[TimestampSpan]>>>,- grammar: Option<&mut Grammar>,- ) -> Result<(u64, Option<&'a B::ScratchBuffer>), LanguageModelStreamError<B>> {- match self {- Self::Constant {- token,- output_norm,- } => Ok((*token, output_norm.as_ref())),- Self::InFlight(in_flight) => {- assert!(in_flight.full_accept);- for pending in replace(&mut in_flight.pending, Box::new([])) {- wait(pending, timestamps).await.map_err(LanguageModelStreamError::Backend)?;- }- let output_tokens = readback- .take()- .expect("in-flight output readback")- .await- .map_err(LanguageModelStreamError::Backend)?;- assert_eq!(output_tokens.len(), 1);- let token_id = output_tokens[0] as u64;- let output_norm = in_flight.output_norm.take();- *self = Self::Constant {- token: token_id,- output_norm,- };- tokens.push(token_id);- if let Some(grammar) = grammar {- grammar.accept_token(token_id)?;- }- let Self::Constant {- output_norm,- ..- } = self- else {- unreachable!()- };- Ok((token_id, output_norm.as_ref()))- },- }- }-- fn drop_output_norm(&mut self) {- match self {- Self::Constant {- output_norm,- ..- } => drop(output_norm.take()),- Self::InFlight(pending) => drop(pending.output_norm.take()),- }- }+pub struct LanguageModelStream<'a, B: Backend> {+ submitter: StreamEnqueuer<'a, B>,+ job: Option<GenerationJob<B>>,+ emitting: Option<(ResolvedJob<B>, usize)>,}-struct DecodingStatePending<B: Backend> {- input_trie: TrieNode,- full_accept: bool,- pending: Box<[<B::CommandBuffer as CommandBuffer>::Pending]>,- capture_span: Option<CaptureMatcherSpan>,- hidden_features: Option<Box<[B::ScratchBuffer]>>,- output_norm: Option<B::ScratchBuffer>,- output_tokens: Arc<B::GlobalGpuBuffer>,- _input_token_source: Option<Arc<B::GlobalGpuBuffer>>,+struct StreamEnqueuer<'a, B: Backend> {+ model: &'a LanguageModel<B>,+ model_state: &'a mut LanguageModelState<B>,+ options: LanguageModelStreamOptions,+ allocation_pool: Arc<B::AllocationPool>,+ context_ring: Option<B::GlobalGpuBuffer>,+ metrics: TokenStreamMetrics,+ propose_trie: bool,}-enum DecodingState<B: Backend> {- Seeded {- seed_token: u64,- },- ForwardPassPending(DecodingStatePending<B>),- Accepting {- full: Box<[(usize, u64, u64)]>,- num_accepted: usize,+enum GenerationJob<B: Backend> {+ Pending {+ output_norm: Option<B::ScratchBuffer>,+ output_tokens: B::GlobalGpuBuffer,+ batch_dim: BatchTopology,+ trie: Option<TrieNode>,hidden_features: Option<Box<[B::ScratchBuffer]>>,- output_norm: Option<(B::ScratchBuffer, usize)>,capture_span: Option<CaptureMatcherSpan>,+ pendings: Box<[<<B as Backend>::CommandBuffer as CommandBuffer>::Pending]>,+ _input_token_source: Option<B::GlobalGpuBuffer>,},- Halted,+ Resolved(ResolvedJob<B>),Invalid,- Closed,}-fn prefill_chunk_parts(- input_chunk: &[u64],- last_batch: bool,- split_logits_row: bool,-) -> [Option<(&[u64], bool)>; 2] {- if last_batch && split_logits_row && input_chunk.len() > 1 {- let (prompt_chunk, sample_chunk) = input_chunk.split_at(input_chunk.len() - 1);- [Some((prompt_chunk, false)), Some((sample_chunk, true))]- } else {- [Some((input_chunk, last_batch)), None]- }+struct ResolvedJob<B: Backend> {+ output_tokens: Box<[u64]>,+ output_norm_buffer: Option<B::ScratchBuffer>,+ output_tokens_buffer: Option<B::GlobalGpuBuffer>,+ batch_dim: BatchTopology,+ hidden_features: Option<Box<[B::ScratchBuffer]>>,+ accepted_indices_and_input_tokens: Option<(Box<[u32]>, Box<[u64]>)>,+ capture_span: Option<CaptureMatcherSpan>,}-pub struct LanguageModelStream<'a, B: Backend> {- model: &'a LanguageModel<B>,- model_state: &'a mut LanguageModelState<B>,- options: LanguageModelStreamOptions,- allocation_pool: Arc<B::AllocationPool>,- context_ring: Option<B::GlobalGpuBuffer>,- decoding_state: DecodingState<B>,- metrics: TokenStreamMetrics,+impl<B: Backend> GenerationJob<B> {+ fn batch_dim(&self) -> &BatchTopology {+ match self {+ Self::Pending {+ batch_dim,+ ..+ } => batch_dim,+ Self::Resolved(resolved) => &resolved.batch_dim,+ Self::Invalid => unreachable!(),+ }+ }++ fn output_tokens_buffer(&self) -> &B::GlobalGpuBuffer {+ match self {+ Self::Pending {+ output_tokens,+ ..+ } => output_tokens,+ Self::Resolved(resolved) => resolved.output_tokens_buffer.as_ref().unwrap(),+ Self::Invalid => unreachable!(),+ }+ }}-impl<'a, B: Backend> LanguageModelStream<'a, B> {- pub fn new(+impl<'a, B: Backend> StreamEnqueuer<'a, B> {+ fn new(model: &'a LanguageModel<B>,input: &[u64],model_state: &'a mut LanguageModelState<B>,options: LanguageModelStreamOptions,) -> Result<Self, LanguageModelStreamError<B>> {- let mut options = options;- if model_state.tokens.is_empty() && input.is_empty() {+ if input.is_empty() {return Err(LanguageModelStreamError::NoSeedToken);};@@ -168,9 +114,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let allocation_pool = model.engine.context.create_allocation_pool();- let mut context_ring = if let Some(suffix_repetition_length) =- options.sampling_method.suffix_repetition_length()- {+ let context_ring = if let Some(suffix_repetition_length) = options.sampling_method.suffix_repetition_length() {let mut context_ring = model.engine.context@@ -195,382 +139,320 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {None};- let mut metrics = TokenStreamMetrics::default();-- let decoding_state = if !input.is_empty() {- let _capture_span = CAPTURE_MATCHER.capture::<B>(&model.engine.context, "prefill")?;- model_state.last_output_token.take();-- // NOTE: this is required for attention correctness (hardcoded suffix 1024). This is really bad design, attention should be rewritten to allow on-demand suffix length- let max_batch_size = 1024;- let number_of_batches = input.len().div_ceil(max_batch_size);-- model_state- .transformer_state- .prepare(- model_state.transformer_state.context_length() + ((number_of_batches - 1) * max_batch_size) as u32,- usize::min(max_batch_size, input.len()) as u32,- )- .map_err(LanguageModelStreamError::Backend)?;-- let command_buffer = model- .engine- .context- .create_command_buffer(Some("prefill"), Some(allocation_pool.clone()))- .map_err(LanguageModelStreamError::Backend)?;- let mut command_buffer = record(command_buffer, options.timestamps.as_ref());-- command_buffer.debug_point(&format!(- "prefill at context {}, suffix {} ({} batches)",- model_state.transformer_state.context_length(),- input.len(),- number_of_batches- ));-- let mut output_tokens = None;- let mut output_norm = None;- let split_logits_row = model.decoder.prefill_cache_skips_trailing_layers();- let hidden_feature_layer_indices =- model.speculator.as_ref().map(|speculator| speculator.hidden_feature_layer_indices());-- for (input_chunk, sample_last) in input- .chunks(max_batch_size)- .enumerate()- .flat_map(|(batch_idx, input_chunk)| {- prefill_chunk_parts(input_chunk, batch_idx == number_of_batches - 1, split_logits_row)- })- .flatten()- {- let input_trie = TrieNode::flat(model_state.tokens.len(), input_chunk, &model_state.prng);- let input_flat_trie = input_trie.linearize();-- let token_ids = command_buffer- .allocate_constant_from_slice(- &input_chunk.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>(),- )- .map_err(LanguageModelStreamError::Backend)?;-- let input_flat_trie_nodes = input_flat_trie.token_subtrie_ranges().collect::<Box<[GpuTrieNode]>>();- let batch_dim = BatchTopology::new(&input_flat_trie_nodes, true);-- let decoder_output = model.decoder.encode(- &token_ids,- &batch_dim,- sample_last.then(|| (input_chunk.len() as u32 - 1..input_chunk.len() as u32).into()),- hidden_feature_layer_indices,- &mut model_state.transformer_state,- &mut command_buffer,- )?;- let logits = decoder_output.logits;-- if sample_last {- let logits = logits.unwrap();-- let seeds = if matches!(options.sampling_method, SamplingMethod::Stochastic { .. }) {- Some(- command_buffer- .allocate_constant_from_slice(&[model_state- .prng- .derive((model_state.tokens.len() + input_chunk.len() - 1) as u64)])- .map_err(LanguageModelStreamError::Backend)?,- )- } else {- None- };-- let bitmask = if let Some(grammar) = options.grammar.as_mut() {- let mut bitmask = command_buffer- .allocate_constant(- model.vocab_size.div_ceil(DataType::U32.size_in_bits()) * DataType::U32.size_in_bytes(),- )- .map_err(LanguageModelStreamError::Backend)?;-- if grammar.next_bitmask(bitmask.as_slice_mut()) {- Some(bitmask)- } else {- None- }- } else {- None- };-- output_norm = decoder_output.final_hidden;- let sampled_row = batch_dim.size() - 1;- output_tokens = Some(- model- .sampling- .encode(- &logits,- seeds.as_ref(),- bitmask.as_ref(),- context_ring.as_ref(),- Some(&token_ids),- &batch_dim,- (sampled_row..sampled_row + 1).into(),- &options.sampling_method,- &mut command_buffer,- )- .map_err(LanguageModelStreamError::Backend)?,- );- }-- model_state- .transformer_state- .encode_accept(&(0..input_chunk.len() as u32).collect::<Box<[u32]>>(), &mut command_buffer)- .map_err(LanguageModelStreamError::Backend)?;-- if let Some(speculator) = model.speculator.as_ref() {- let speculator_state = model_state.speculator_state.as_mut().unwrap();- speculator.encode_accept(- speculator_state,- decoder_output.hidden_features.as_ref().unwrap().iter(),- &(0..input_chunk.len() as u32).collect::<Box<[u32]>>(),- &mut command_buffer,- )?;- }+ let metrics = TokenStreamMetrics::default();- if let Some(suffix_repetition_length) = options.sampling_method.suffix_repetition_length() {- model.context_ring_update.encode(- &token_ids,- context_ring.as_mut().unwrap(),- suffix_repetition_length,- input_chunk.len() as u32,- &mut command_buffer,- );- }-- model_state.tokens.extend(input_chunk);- }-- let pending = Box::new([command_buffer.end_encoding().submit()]);-- metrics.num_prefill_forward_passes += 1;- metrics.num_tokens_prefilled += input.len();- metrics.num_tokens_proposed += 1;- metrics.num_tokens_accepted += 1;-- DecodingState::ForwardPassPending(DecodingStatePending {- input_trie: TrieNode::new(0, 0, 0.0),- full_accept: true,- pending,- capture_span: None,- hidden_features: None,- output_norm,- output_tokens: Arc::new(output_tokens.unwrap()),- _input_token_source: None,- })- } else {- // TODO: this leaks previous LanguageModelStreamOptions- DecodingState::Seeded {- seed_token: model_state.last_output_token.take().unwrap(),- }- };-- Ok(LanguageModelStream {+ Ok(StreamEnqueuer {model,model_state,options,allocation_pool,context_ring,- decoding_state,metrics,+ propose_trie: true,})}- pub async fn generate(&mut self) -> Result<Option<u64>, LanguageModelStreamError<B>> {- let context = &self.model.engine.context;- let (mut prev_output, acceptance): (- ForwardPassChaining<B>,- Option<(<B::CommandBuffer as CommandBuffer>::Encoding, Option<CaptureMatcherSpan>)>,- ) = match replace(&mut self.decoding_state, DecodingState::Invalid) {- DecodingState::Seeded {- seed_token,- } => {- self.model_state.tokens.push(seed_token);- if let Some(grammar) = self.options.grammar.as_mut() {- let _ = grammar.accept_token(seed_token); // TODO: this should not be ignored- }- self.metrics.num_tokens_returned += 1;- (- ForwardPassChaining::Constant {- token: seed_token,- output_norm: None,- },- None,- )- },- DecodingState::ForwardPassPending(forward_pass_pending) => {- if forward_pass_pending.full_accept {- self.metrics.num_tokens_returned += 1;- (ForwardPassChaining::InFlight(forward_pass_pending), None)- } else {- for pending in forward_pass_pending.pending {- wait(pending, self.options.timestamps.as_ref())- .await- .map_err(LanguageModelStreamError::Backend)?;- }- let sampled_tokens = context- .copyout::<u32>(&*forward_pass_pending.output_tokens)- .await- .map_err(LanguageModelStreamError::Backend)?;- let sampled_tokens = sampled_tokens.iter().map(|token| u64::from(*token)).collect::<Box<[u64]>>();- let flat_trie = forward_pass_pending.input_trie.linearize();- let full = flat_trie.accept(&sampled_tokens, self.options.grammar.as_mut())?;- let output_norm = forward_pass_pending.output_norm.map(|norm| {- let row_bytes = norm.size() / flat_trie.len();- (norm, row_bytes)- });- self.metrics.num_tokens_accepted += full.len();- self.decoding_state = DecodingState::Accepting {- full,- num_accepted: 0,- hidden_features: forward_pass_pending.hidden_features,- output_norm,- capture_span: forward_pass_pending.capture_span,- };- return Box::pin(self.generate()).await;- }- },- DecodingState::Accepting {- full,- num_accepted,+ async fn resolve<'b>(+ &mut self,+ job: &'b mut GenerationJob<B>,+ ) -> Result<&'b ResolvedJob<B>, LanguageModelStreamError<B>> {+ let owned = mem::replace(job, GenerationJob::Invalid);+ let resolved = match owned {+ GenerationJob::Pending {+ output_norm: output_norm_buffer,+ output_tokens: output_tokens_buffer,+ _input_token_source,+ batch_dim,+ trie,hidden_features,- output_norm,capture_span,+ pendings,} => {- let output_token_id = full[num_accepted].2;-- self.metrics.num_tokens_returned += 1;-- if num_accepted < full.len() - 1 {- self.decoding_state = DecodingState::Accepting {- full,- num_accepted: num_accepted + 1,- hidden_features,- output_norm,- capture_span,- };- return Ok(Some(output_token_id));- } else {+ for pending in pendings {+ wait(pending, self.options.timestamps.as_ref()).await.map_err(LanguageModelStreamError::Backend)?;+ }+ let gpu_tokens = self+ .model+ .engine+ .context+ .copyout::<u32>(&output_tokens_buffer)+ .await+ .map_err(LanguageModelStreamError::Backend)?;++ let mut output_tokens = gpu_tokens.iter().map(|&t| t as u64).collect::<Box<[_]>>();++ let accepted_indices_and_input_tokens = if let Some(trie) = trie {+ let flat = trie.linearize();+ let full = flat.accept(&output_tokens, self.options.grammar.as_mut())?;+let accepted_token_indicies = full.iter().map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();- let command_buffer = self- .model- .engine- .context- .create_command_buffer(- Some(&format!("decode {} (accept)", self.metrics.num_decode_forward_passes - 1)),- Some(self.allocation_pool.clone()),- )- .map_err(LanguageModelStreamError::Backend)?;- let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());- self.model_state- .transformer_state- .encode_accept(&accepted_token_indicies, &mut command_buffer)- .map_err(LanguageModelStreamError::Backend)?;- if let Some(speculator) = self.model.speculator.as_ref() {- speculator.encode_accept(- self.model_state.speculator_state.as_mut().unwrap(),- hidden_features.as_ref().unwrap().iter(),- &accepted_token_indicies,- &mut command_buffer,- )?;- }- let output_norm = if let Some((final_hidden, row_bytes)) = output_norm {- let row = full.last().unwrap().0;- let mut norm =- command_buffer.allocate_scratch(row_bytes).map_err(LanguageModelStreamError::Backend)?;- command_buffer- .encode_copy(final_hidden.subrange(row * row_bytes..(row + 1) * row_bytes), &mut norm);- Some(norm)- } else {- None- };- if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {- let accepted_input_token_ids_const = command_buffer- .allocate_constant_from_slice(- &accepted_input_token_ids- .iter()- .map(|token_id| *token_id as u32)- .collect::<Box<[u32]>>(),- )- .map_err(LanguageModelStreamError::Backend)?;- self.model.context_ring_update.encode(- &accepted_input_token_ids_const,- self.context_ring.as_mut().unwrap(),- suffix_repetition_length,- full.len() as u32,- &mut command_buffer,- );+ output_tokens = accepted_output_token_ids;++ Some((accepted_token_indicies, accepted_input_token_ids))+ } else {+ if let Some(grammar) = self.options.grammar.as_mut() {+ for &token_id in &output_tokens {+ grammar.accept_token(token_id)?+ }}- self.model_state.tokens.extend(accepted_output_token_ids);- (- ForwardPassChaining::Constant {- token: output_token_id,- output_norm,- },- Some((command_buffer, capture_span)),- )+ None+ };++ ResolvedJob {+ output_tokens,+ output_norm_buffer,+ output_tokens_buffer: Some(output_tokens_buffer),+ batch_dim,+ hidden_features,+ accepted_indices_and_input_tokens,+ capture_span,}},- DecodingState::Halted => return Ok(None),- DecodingState::Invalid => unreachable!(),- DecodingState::Closed => return Ok(None),+ GenerationJob::Resolved(resolved) => resolved,+ GenerationJob::Invalid => unreachable!(),};- let readback_buffer = match &prev_output {- ForwardPassChaining::InFlight(pending) => Some(pending.output_tokens.clone()),- _ => None,+ *job = GenerationJob::Resolved(resolved);+ let GenerationJob::Resolved(resolved) = &*job else {+ unreachable!()};- let mut output_readback = readback_buffer.as_deref().map(|buffer| context.copyout::<u32>(buffer));- let context_length = self.model_state.transformer_state.context_length();+ Ok(resolved)+ }++ async fn resolve_take(+ &mut self,+ mut job: GenerationJob<B>,+ ) -> Result<ResolvedJob<B>, LanguageModelStreamError<B>> {+ self.resolve(&mut job).await?;+ let GenerationJob::Resolved(resolved) = job else {+ unreachable!()+ };++ Ok(resolved)+ }++ pub fn prefill(+ &mut self,+ input: &[u64],+ ) -> Result<GenerationJob<B>, LanguageModelStreamError<B>> {+ let _capture_span = CAPTURE_MATCHER.capture::<B>(&self.model.engine.context, "prefill")?;++ // NOTE: this is required for attention correctness (hardcoded suffix+ // 1024). This is really bad design, attention should be rewritten to+ // allow on-demand suffix length+ let max_batch_size = 1024;+ let number_of_batches = input.len().div_ceil(max_batch_size);++ self.model_state+ .transformer_state+ .prepare(+ self.model_state.transformer_state.context_length() + ((number_of_batches - 1) * max_batch_size) as u32,+ usize::min(max_batch_size, input.len()) as u32,+ )+ .map_err(LanguageModelStreamError::Backend)?;++ let command_buffer = self+ .model+ .engine+ .context+ .create_command_buffer(Some("prefill"), Some(self.allocation_pool.clone()))+ .map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());++ command_buffer.debug_point(&format!(+ "prefill at context {}, suffix {} ({} batches)",+ self.model_state.transformer_state.context_length(),+ input.len(),+ number_of_batches+ ));++ let hidden_feature_layer_indices =+ self.model.speculator.as_ref().map(|speculator| speculator.hidden_feature_layer_indices());++ let mut encode_batch = |input_chunk: &[u64], sample: bool| -> Result<_, LanguageModelStreamError<B>> {+ let token_ids = command_buffer+ .allocate_constant_from_slice(+ &input_chunk.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>(),+ )+ .map_err(LanguageModelStreamError::Backend)?;- if self.model_state.max_context_length.is_some_and(|max_context_length| context_length >= max_context_length) {- self.decoding_state = DecodingState::Halted;- return Ok(Some(- prev_output- .resolve(- &mut output_readback,- &mut self.model_state.tokens,- self.options.timestamps.as_ref(),- self.options.grammar.as_mut(),- )- .await?- .0,- ));+ let batch_dim = BatchTopology::flat(input_chunk.len() as u32, true);++ let decoder_output = self.model.decoder.encode(+ &token_ids,+ &batch_dim,+ sample.then(|| (input_chunk.len() as u32 - 1..input_chunk.len() as u32).into()),+ hidden_feature_layer_indices,+ &mut self.model_state.transformer_state,+ &mut command_buffer,+ )?;++ self.model_state+ .transformer_state+ .encode_accept(&(0..input_chunk.len() as u32).collect::<Box<[u32]>>(), &mut command_buffer)+ .map_err(LanguageModelStreamError::Backend)?;++ if let Some(speculator) = self.model.speculator.as_ref() {+ let speculator_state = self.model_state.speculator_state.as_mut().unwrap();+ speculator.encode_accept(+ speculator_state,+ decoder_output.hidden_features.as_ref().unwrap().iter(),+ &(0..input_chunk.len() as u32).collect::<Box<[u32]>>(),+ &mut command_buffer,+ )?;+ }++ if !sample && let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {+ self.model.context_ring_update.encode(+ &token_ids,+ self.context_ring.as_mut().unwrap(),+ suffix_repetition_length,+ input_chunk.len() as u32,+ &mut command_buffer,+ );+ }++ self.model_state.tokens.extend(input_chunk);++ Ok((decoder_output, token_ids, batch_dim))+ };++ let mut token_chunks = input.chunks(max_batch_size);+ for input_chunk in token_chunks.by_ref().take(number_of_batches - 1) {+ encode_batch(input_chunk, false)?;}- let mut pending = Vec::new();- if let Some((accept_command_buffer, _capture_span)) = acceptance {- pending.push(accept_command_buffer.end_encoding().submit());+ // `token_chunks` should have exactly one element left.+ let final_chunk = token_chunks.next().unwrap();++ let split_last_logits_row = self.model.decoder.prefill_cache_skips_trailing_layers();+ let (decoder_output, token_ids, batch_dim) = if split_last_logits_row && final_chunk.len() > 1 {+ let (start, end) = final_chunk.split_at(final_chunk.len() - 1);+ encode_batch(start, false)?;+ encode_batch(end, true)?+ } else {+ encode_batch(final_chunk, true)?+ };++ let logits = decoder_output.logits.unwrap();++ let seeds = if matches!(self.options.sampling_method, SamplingMethod::Stochastic { .. }) {+ Some(+ command_buffer+ .allocate_constant_from_slice(&[self+ .model_state+ .prng+ .derive(self.model_state.tokens.len() as u64 - 1)])+ .map_err(LanguageModelStreamError::Backend)?,+ )+ } else {+ None+ };++ let bitmask = if let Some(grammar) = self.options.grammar.as_mut() {+ let mut bitmask = command_buffer+ .allocate_constant(+ self.model.vocab_size.div_ceil(DataType::U32.size_in_bits()) * DataType::U32.size_in_bytes(),+ )+ .map_err(LanguageModelStreamError::Backend)?;++ if grammar.next_bitmask(bitmask.as_slice_mut()) {+ Some(bitmask)+ } else {+ None+ }+ } else {+ None+ };++ let output_norm = decoder_output.final_hidden;+ let sampled_row = batch_dim.size() - 1;+ let output_token = self+ .model+ .sampling+ .encode(+ &logits,+ seeds.as_ref(),+ bitmask.as_ref(),+ self.context_ring.as_ref(),+ Some(&token_ids),+ &batch_dim,+ (sampled_row..sampled_row + 1).into(),+ &self.options.sampling_method,+ &mut command_buffer,+ )+ .map_err(LanguageModelStreamError::Backend)?;++ if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {+ self.model.context_ring_update.encode(+ &token_ids,+ self.context_ring.as_mut().unwrap(),+ suffix_repetition_length,+ batch_dim.size(),+ &mut command_buffer,+ );}+ drop(seeds);+ drop(bitmask);+ drop(token_ids);++ self.metrics.num_prefill_forward_passes += 1;+ self.metrics.num_tokens_prefilled += input.len();+ self.metrics.num_tokens_proposed += 1;++ Ok(GenerationJob::Pending {+ output_norm,+ output_tokens: output_token,+ _input_token_source: None,+ batch_dim,+ trie: None,+ hidden_features: decoder_output.hidden_features,+ capture_span: None,+ pendings: Box::new([command_buffer.end_encoding().submit()]),+ })+ }++ pub async fn get_and_queue_pass(+ &mut self,+ mut last_job: GenerationJob<B>,+ ) -> Result<(ResolvedJob<B>, GenerationJob<B>), LanguageModelStreamError<B>> {let mut capture_span = CAPTURE_MATCHER.capture::<B>(&self.model.engine.context, &format!("decode:{}", self.metrics.num_decode_forward_passes))?;- #[allow(clippy::let_and_return)]- let (input_trie, chain_copy, full_accept) = if let Some(speculator) = &self.model.speculator+ // Speculator pass+ let context_length = self.model_state.transformer_state.context_length();+ let trie = if self.propose_trie+ && let Some(speculator) = &self.model.speculator&& let Some(shape) = speculator.make_shape(self.model_state.max_context_length.map(|max_context_length| max_context_length - context_length),- )- && let (root_token, Some(output_norm)) = {- let resolved = prev_output- .resolve(- &mut output_readback,- &mut self.model_state.tokens,- self.options.timestamps.as_ref(),- self.options.grammar.as_mut(),- )- .await?;- resolved- } {+ ) {+ // TODO: CPU-bubble+ let resolved = self.resolve(&mut last_job).await?;+ let tokens = &resolved.output_tokens;+ let batch_dim = &resolved.batch_dim;+ let final_hidden = resolved.output_norm_buffer.as_ref().unwrap();++ let last_norm_buffer =+ if let Some((accepted_indices, _)) = resolved.accepted_indices_and_input_tokens.as_ref() {+ let row = (*accepted_indices.last().unwrap()) as usize;+ let row_bytes = final_hidden.size() / batch_dim.size() as usize;+ final_hidden.subrange(row * row_bytes..(row + 1) * row_bytes)+ } else {+ final_hidden.subrange(..)+ };++ // TODO: CPU-bubble: for next `batch_dim`. To remove we need to+ // contruct the trie nodes GPU-side and have `batch_dim` carry a+ // GPU representation.let trie = speculator.propose_tree(self.model_state.speculator_state.as_mut().unwrap(),- output_norm,- root_token as u32,+ last_norm_buffer,+ tokens[tokens.len() - 1] as u32,self.model.decoder.embedding(),shape,&self.options.sampling_method,@@ -581,66 +463,61 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {self.options.timestamps.as_ref(),).await?;- prev_output.drop_output_norm();- (trie, None, false)+ Some(trie)} else {- prev_output.drop_output_norm();- let (token, chain_copy) = match &prev_output {- ForwardPassChaining::Constant {- token,- ..- } => (*token, None),- ForwardPassChaining::InFlight(pending) => (0, Some(pending.output_tokens.clone())),- };- (TrieNode::new(token, self.model_state.prng.derive(context_length as u64), 0.0), chain_copy, true)+ None};- let input_flat_trie = input_trie.linearize();-- self.allocation_pool = self.model.engine.context.create_allocation_pool();+ // Main (verifier) forward passlet command_buffer = self.model.engine.context.create_command_buffer(- Some(&format!("decode {} (forward)", self.metrics.num_decode_forward_passes)),+ Some(&format!("decode {}", self.metrics.num_decode_forward_passes)),Some(self.allocation_pool.clone()),).map_err(LanguageModelStreamError::Backend)?;- let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());- command_buffer.debug_point(&format!(- "decode round {} at context {} suffix {}",- self.metrics.num_decode_forward_passes,- context_length,- input_flat_trie.len()- ));+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());- let scratch_token_ids;let mut constant_token_ids = None;- let token_ids: &dyn GpuBuffer<Backend = B> = if let Some(chain_copy) = chain_copy.as_deref() {- let mut token_ids = command_buffer- .allocate_scratch(DataType::U32.size_in_bytes())+ let mut needs_last_job_input = false;+ let flat_trie = trie.as_ref().map(|trie| trie.linearize());+ let (token_ids, batch_dim): (&dyn GpuBuffer<Backend = B>, _) = if let Some(flat_trie) = flat_trie.as_ref() {+ let batch_dim = BatchTopology::new(flat_trie.token_subtrie_ranges().collect::<Box<[_]>>(), false);++ let token_ids = command_buffer+ .allocate_constant_from_slice(+ &flat_trie.token_ids().map(|token_id| token_id as u32).collect::<Box<[u32]>>(),+ ).map_err(LanguageModelStreamError::Backend)?;- command_buffer.encode_copy(chain_copy, &mut token_ids);- scratch_token_ids = token_ids;- &scratch_token_ids++ constant_token_ids = Some(token_ids);+ (constant_token_ids.as_ref().unwrap(), batch_dim)+ } else if last_job.batch_dim().size() == 1 {+ needs_last_job_input = true;+ (last_job.output_tokens_buffer(), BatchTopology::flat(1, true))} else {- constant_token_ids.insert(- command_buffer- .allocate_constant_from_slice(- &input_flat_trie.token_ids().map(|token_id| token_id as u32).collect::<Box<[u32]>>(),- )- .map_err(LanguageModelStreamError::Backend)?,- )+ let resolved = self.resolve(&mut last_job).await?;+ let token = *resolved.output_tokens.last().unwrap() as u32;+ let token_ids =+ command_buffer.allocate_constant_from_slice(&[token]).map_err(LanguageModelStreamError::Backend)?;++ constant_token_ids = Some(token_ids);+ (constant_token_ids.as_ref().unwrap(), BatchTopology::flat(1, true))};- let input_flat_trie_nodes = input_flat_trie.token_subtrie_ranges().collect::<Box<[GpuTrieNode]>>();- let batch_dim = BatchTopology::new(&input_flat_trie_nodes, full_accept);+ command_buffer.debug_point(&format!(+ "decode round {} at context {} suffix {}",+ self.metrics.num_decode_forward_passes,+ context_length,+ batch_dim.size()+ ));self.model_state.transformer_state- .prepare(self.model_state.transformer_state.context_length(), batch_dim.size())+ .prepare(context_length, batch_dim.size()).map_err(LanguageModelStreamError::Backend)?;let hidden_feature_layer_indices =@@ -654,67 +531,60 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {&mut self.model_state.transformer_state,&mut command_buffer,)?;- let logits = decoder_output.logits.unwrap();- let (bitmask, mut command_buffer) = if let Some(grammar) = self.options.grammar.as_mut() {- if chain_copy.is_some() {- pending.push(command_buffer.end_encoding().submit());+ let mut pendings = Vec::new();- let command_buffer = self- .model- .engine- .context- .create_command_buffer(- Some(&format!("decode {} (sample and accept)", self.metrics.num_decode_forward_passes)),- Some(self.allocation_pool.clone()),- )- .map_err(LanguageModelStreamError::Backend)?;- let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());+ // sampling+ let (mut command_buffer, token_ids): (_, &dyn GpuBuffer<Backend = B>) =+ if needs_last_job_input && self.options.grammar.is_some() {+ pendings.push(command_buffer.end_encoding().submit());+ let command_buffer = record(+ self.model+ .engine+ .context+ .create_command_buffer(+ Some(&format!("decode {} (sample and accept)", self.metrics.num_decode_forward_passes)),+ Some(self.allocation_pool.clone()),+ )+ .map_err(LanguageModelStreamError::Backend)?,+ self.options.timestamps.as_ref(),+ );- let mut bitmask = command_buffer- .allocate_constant(- self.model.vocab_size.div_ceil(DataType::U32.size_in_bits()) * DataType::U32.size_in_bytes(),- )- .map_err(LanguageModelStreamError::Backend)?;+ // NOTE: Not a CPU bubble, forward pass is overlapped.+ self.resolve(&mut last_job).await?;+ (command_buffer, last_job.output_tokens_buffer())+ } else {+ (command_buffer, token_ids)+ };- prev_output- .resolve(- &mut output_readback,- &mut self.model_state.tokens,- self.options.timestamps.as_ref(),- Some(grammar),- )- .await?;- if grammar.next_bitmask(bitmask.as_slice_mut()) {- (Some(bitmask), command_buffer)- } else {- (None, command_buffer)- }+ let bitmask = if let Some(grammar) = self.options.grammar.as_mut() {+ let mut bitmask = command_buffer+ .allocate_constant(+ self.model.vocab_size.div_ceil(DataType::U32.size_in_bits())+ * DataType::U32.size_in_bytes()+ * batch_dim.size() as usize,+ )+ .map_err(LanguageModelStreamError::Backend)?;++ let was_filled = if let Some(flat_trie) = flat_trie.as_ref() {+ flat_trie.fill_bitmasks(bitmask.as_slice_mut(), self.model.vocab_size, grammar)} else {- let mut bitmasks = command_buffer- .allocate_constant(- input_flat_trie.len()- * self.model.vocab_size.div_ceil(DataType::U32.size_in_bits())- * DataType::U32.size_in_bytes(),- )- .map_err(LanguageModelStreamError::Backend)?;+ grammar.next_bitmask(bitmask.as_slice_mut())+ };- if input_flat_trie.fill_bitmasks(bitmasks.as_slice_mut(), self.model.vocab_size, grammar) {- (Some(bitmasks), command_buffer)- } else {- (None, command_buffer)- }- }+ was_filled.then_some(bitmask)} else {- (None, command_buffer)+ None};let seeds = if matches!(self.options.sampling_method, SamplingMethod::Stochastic { .. }) {- Some(- command_buffer- .allocate_constant_from_slice(&input_flat_trie.token_seeds().collect::<Box<[u64]>>())- .map_err(LanguageModelStreamError::Backend)?,- )+ let seeds = if let Some(flat_trie) = flat_trie.as_ref() {+ flat_trie.token_seeds().collect::<Box<[u64]>>()+ } else {+ Box::new([self.model_state.prng.derive(context_length as u64)])+ };++ Some(command_buffer.allocate_constant_from_slice(&seeds).map_err(LanguageModelStreamError::Backend)?)} else {None};@@ -723,7 +593,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.model.sampling.encode(- &logits,+ &decoder_output.logits.unwrap(),seeds.as_ref(),bitmask.as_ref(),self.context_ring.as_ref(),@@ -737,9 +607,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {drop(seeds);drop(bitmask);- drop(logits);- if full_accept {+ // new job and statistics+ self.metrics.num_decode_forward_passes += 1;+ self.metrics.num_tokens_proposed += batch_dim.size() as usize;++ if batch_dim.full_accept() {self.model_state.transformer_state.encode_accept(&(0..batch_dim.size()).collect::<Box<[u32]>>(), &mut command_buffer)@@ -767,145 +640,226 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {}drop(constant_token_ids);- pending.push(command_buffer.end_encoding().submit());- if full_accept {+ pendings.push(command_buffer.end_encoding().submit());++ if batch_dim.full_accept() {capture_span.take();}- self.metrics.num_decode_forward_passes += 1;- self.metrics.num_tokens_proposed += input_flat_trie.len();- if full_accept {- self.metrics.num_tokens_accepted += input_flat_trie.len();+ let mut new_job = GenerationJob::Pending {+ output_norm: decoder_output.final_hidden,+ output_tokens,+ _input_token_source: None,+ batch_dim,+ trie,+ hidden_features: decoder_output.hidden_features,+ capture_span,+ pendings: pendings.into_boxed_slice(),+ };++ // Return results+ // NOTE: not a CPU bubble. The next job has already been scheduled.+ let mut resolved_job = self.resolve_take(last_job).await?;++ if needs_last_job_input {+ // Keep previous input alive if necessary.+ let GenerationJob::Pending {+ _input_token_source,+ ..+ } = &mut new_job+ else {+ unreachable!()+ };+ *_input_token_source = resolved_job.output_tokens_buffer.take();}- self.decoding_state = DecodingState::ForwardPassPending(DecodingStatePending {- input_trie,- full_accept,- pending: pending.into_boxed_slice(),- capture_span,- hidden_features: if full_accept {- None- } else {- decoder_output.hidden_features- },- output_norm: decoder_output.final_hidden,- output_tokens: Arc::new(output_tokens),- _input_token_source: chain_copy,- });-- Ok(Some(- prev_output- .resolve(- &mut output_readback,- &mut self.model_state.tokens,- self.options.timestamps.as_ref(),- self.options.grammar.as_mut(),+ Ok((resolved_job, new_job))+ }++ fn encode_accept(+ &mut self,+ resolved: &mut ResolvedJob<B>,+ accept_len: usize,+ ) -> Result<Option<<<B as Backend>::CommandBuffer as CommandBuffer>::Pending>, LanguageModelStreamError<B>> {+ let batch_dim = &resolved.batch_dim;++ if batch_dim.full_accept() {+ return Ok(None);+ }++ let command_buffer = self+ .model+ .engine+ .context+ .create_command_buffer(Some("accept"), Some(self.allocation_pool.clone()))+ .map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());++ let (accepted_token_indices, accepted_input_token_ids) =+ resolved.accepted_indices_and_input_tokens.as_ref().unwrap();+ let accepted_token_indices = &accepted_token_indices[..accept_len];+ let accepted_input_token_ids = &accepted_input_token_ids[..accept_len];++ self.model_state+ .transformer_state+ .encode_accept(accepted_token_indices, &mut command_buffer)+ .map_err(LanguageModelStreamError::Backend)?;++ if let Some(speculator) = self.model.speculator.as_ref() {+ speculator.encode_accept(+ self.model_state.speculator_state.as_mut().unwrap(),+ resolved.hidden_features.as_ref().unwrap().iter(),+ accepted_token_indices,+ &mut command_buffer,+ )?;+ }++ if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {+ // TODO: This can be provided by a `verify_trie` kernel.+ let accepted_input_token_ids_const = command_buffer+ .allocate_constant_from_slice(+ &accepted_input_token_ids.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>(),)- .await?- .0,- ))+ .map_err(LanguageModelStreamError::Backend)?;++ self.model.context_ring_update.encode(+ &accepted_input_token_ids_const,+ self.context_ring.as_mut().unwrap(),+ suffix_repetition_length,+ accepted_token_indices.len() as u32,+ &mut command_buffer,+ );+ }++ let pending = command_buffer.end_encoding().submit();+ resolved.capture_span.take();++ Ok(Some(pending))}+}- pub fn metrics(&self) -> &TokenStreamMetrics {- &self.metrics+impl<'a, B: Backend> LanguageModelStream<'a, B> {+ pub fn new(+ model: &'a LanguageModel<B>,+ input: &[u64],+ model_state: &'a mut LanguageModelState<B>,+ options: LanguageModelStreamOptions,+ ) -> Result<Self, LanguageModelStreamError<B>> {+ let mut submitter = StreamEnqueuer::new(model, input, model_state, options)?;+ let job = submitter.prefill(input)?;++ Ok(Self {+ submitter,+ emitting: None,+ job: Some(job),+ })}- pub async fn close(&mut self) -> Result<(), LanguageModelStreamError<B>> {- let last_output_token = match replace(&mut self.decoding_state, DecodingState::Closed) {- DecodingState::Seeded {- seed_token,- } => Some(seed_token),- DecodingState::ForwardPassPending(in_flight) => {- for pending in in_flight.pending {- wait(pending, self.options.timestamps.as_ref()).await.map_err(LanguageModelStreamError::Backend)?;- }+ pub fn closed(&self) -> bool {+ self.job.is_none() && self.emitting.is_none()+ }- if !in_flight.full_accept {- let command_buffer = self- .model- .engine- .context- .create_command_buffer(- Some(&format!("decode {} (accept on drop)", self.metrics.num_decode_forward_passes - 1)),- Some(self.allocation_pool.clone()),- )- .map_err(LanguageModelStreamError::Backend)?;- let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());- self.model_state- .transformer_state- .encode_accept(&[0], &mut command_buffer)- .map_err(LanguageModelStreamError::Backend)?;- if let Some(speculator) = self.model.speculator.as_ref() {- speculator.encode_accept(- self.model_state.speculator_state.as_mut().unwrap(),- in_flight.hidden_features.as_ref().unwrap().iter(),- &[0],- &mut command_buffer,- )?;- }- wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref())- .await- .map_err(LanguageModelStreamError::Backend)?;- }+ fn emit_token(&mut self) -> Option<u64> {+ let (resolved, i) = self.emitting.as_mut()?;+ if *i >= resolved.output_tokens.len() {+ return None;+ }- let context = &self.model.engine.context;- Some(- context- .copyout::<u32>(in_flight.output_tokens.subrange(..size_of::<u32>()))- .await- .map_err(LanguageModelStreamError::Backend)?[0] as u64,- )- },- DecodingState::Accepting {- full,- num_accepted,- hidden_features,- output_norm: _,- capture_span,- } => {- assert!(num_accepted > 0 && num_accepted < full.len());+ let token = resolved.output_tokens[*i];+ *i += 1;- let command_buffer = self- .model- .engine- .context- .create_command_buffer(- Some(&format!("decode {} (accept on drop)", self.metrics.num_decode_forward_passes - 1)),- Some(self.allocation_pool.clone()),- )- .map_err(LanguageModelStreamError::Backend)?;- let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());- let accepted_token_indicies =- full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();- self.model_state- .transformer_state- .encode_accept(&accepted_token_indicies, &mut command_buffer)- .map_err(LanguageModelStreamError::Backend)?;- if let Some(speculator) = self.model.speculator.as_ref() {- speculator.encode_accept(- self.model_state.speculator_state.as_mut().unwrap(),- hidden_features.as_ref().unwrap().iter(),- &accepted_token_indicies,- &mut command_buffer,- )?;- }- wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref())+ self.submitter.model_state.tokens.push(token);+ self.submitter.metrics.num_tokens_accepted += 1;+ self.submitter.metrics.num_tokens_returned += 1;+ Some(token)+ }++ pub async fn generate(&mut self) -> Result<Option<u64>, LanguageModelStreamError<B>> {+ if let Some(token) = self.emit_token() {+ return Ok(Some(token));+ }++ if let Some(job) = self.job.take() {+ let context_full = self.submitter.model_state.max_context_length.is_some_and(|max_context_length| {+ self.submitter.model_state.transformer_state.context_length() >= max_context_length+ });++ if job.batch_dim().full_accept() && !context_full {+ let (last_job, new_job) = self.submitter.get_and_queue_pass(job).await?;+ self.job = Some(new_job);+ self.emitting = Some((last_job, 0));+ } else {+ let resolved = self.submitter.resolve_take(job).await?;+ self.emitting = Some((resolved, 0))+ }+ } else if let Some((mut queued, emitted_len)) = self.emitting.take() {+ let accept_pending = self.submitter.encode_accept(&mut queued, emitted_len)?;+ if let Some(pending) = accept_pending {+ wait(pending, self.submitter.options.timestamps.as_ref()).await.map_err(LanguageModelStreamError::Backend)?;+ }++ let context_full = self.submitter.model_state.max_context_length.is_some_and(|max_context_length| {+ self.submitter.model_state.transformer_state.context_length() >= max_context_length+ });++ if !context_full {+ let (_, new_job) = self.submitter.get_and_queue_pass(GenerationJob::Resolved(queued)).await?;+ let resolved_new = self.submitter.resolve_take(new_job).await?;+ self.emitting = Some((resolved_new, 0))+ } else {+ assert!(self.emit_token().is_none());+ }+ }- drop(capture_span);+ Ok(self.emit_token())+ }- self.model_state.tokens.extend(full.iter().take(num_accepted).map(|(_, _, t)| *t));+ pub fn metrics(&self) -> &TokenStreamMetrics {+ &self.submitter.metrics+ }- Some(full[num_accepted].2)- },- DecodingState::Halted => None,- DecodingState::Invalid => None, // TODO: proper error handling- DecodingState::Closed => self.model_state.last_output_token,+ pub async fn close(&mut self) -> Result<(), LanguageModelStreamError<B>> {+ if let Some((mut last_resolved, emitted_len)) = self.emitting.take() {+ let output_len = last_resolved.output_tokens.len();+ let accept_len = output_len.min(emitted_len + 1);+ let accept_pending = self.submitter.encode_accept(&mut last_resolved, accept_len)?;+ if let Some(pending) = accept_pending {+ wait(pending, self.submitter.options.timestamps.as_ref())+ .await+ .map_err(LanguageModelStreamError::Backend)?;+ }++ let context_full = self.submitter.model_state.max_context_length.is_some_and(|max_context_length| {+ self.submitter.model_state.transformer_state.context_length() >= max_context_length+ });++ // Make sure last token is accepted.+ if emitted_len == output_len && !context_full {+ let next_job = match self.job.take() {+ Some(job) => job,+ None => {+ self.submitter.propose_trie = false;+ let queued = self.submitter.get_and_queue_pass(GenerationJob::Resolved(last_resolved)).await;+ self.submitter.propose_trie = true;+ queued?.1+ },+ };+ let mut next_resolved = self.submitter.resolve_take(next_job).await?;+ let accept_pending = self.submitter.encode_accept(&mut next_resolved, 1)?;+ if let Some(pending) = accept_pending {+ wait(pending, self.submitter.options.timestamps.as_ref())+ .await+ .map_err(LanguageModelStreamError::Backend)?;+ }+ }};- self.model_state.last_output_token = last_output_token;+ if let Some(job) = self.job.take() {+ self.submitter.resolve_take(job).await?;+ }Ok(())}@@ -913,8 +867,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {fn drop(&mut self) {- if !matches!(self.decoding_state, DecodingState::Closed)+ if !self.closed()&& self+ .submitter.model.engine.contextdiff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex ebb165fc7..fccbe2d7c 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -13,7 +13,6 @@ pub use crate::encodable_block::dflash::DFlashState;use crate::{backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, TimestampSpan,- gpu_types::trie::TrieNode as GpuTrieNode,},config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},data_type::DataType,@@ -278,14 +277,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {state,&mut command_buffer,)?;- let topology_nodes = (0..chain_length)- .map(|index| GpuTrieNode {- trie_start: index,- trie_end: chain_length - 1,- height: index,- })- .collect::<Box<[_]>>();- let batch_topology = BatchTopology::new(&topology_nodes, true);+ let batch_topology = BatchTopology::flat(chain_length, true);let sampled = self.sampling.encode(diff --git a/crates/uzu-engine/src/trie.rs b/crates/uzu-engine/src/trie.rsindex 42abaaff2..f55bb5956 100644--- a/crates/uzu-engine/src/trie.rs+++ b/crates/uzu-engine/src/trie.rs@@ -4,7 +4,6 @@ use thiserror::Error;use crate::{backends::common::gpu_types::trie::TrieNode as GpuTrieNode,data_type::DataType,- encodable_block::sampling::PRng,engine::language_model::grammar::{Grammar, GrammarError},};@@ -135,24 +134,6 @@ impl TrieNode {prune(self, &kept, &mut 0);}- pub fn flat(- prefix_length: usize,- tokens: &[u64],- prng: &PRng,- ) -> Self {- assert!(!tokens.is_empty(), "need seed node");-- let mut root = TrieNode::new(tokens[0], prng.derive(prefix_length as u64), 0.0);- let mut leaf = &mut root;-- for (index, token) in tokens.iter().copied().enumerate().skip(1) {- leaf.add(TrieNode::new(token, prng.derive((prefix_length + index) as u64), 0.0)).unwrap();- leaf = &mut leaf.next[0];- }-- root- }-pub fn linearize(&self) -> FlatTrie<'_> {let mut tokens = vec![FlatTrieNode::new(self, (0, 0), 0)];@@ -197,6 +178,7 @@ impl<'a> FlatTrie<'a> {}}+ #[cfg(test)]pub fn len(&self) -> usize {self.tokens.len()}diff --git a/crates/uzu-engine/unit/backends/common/kernel/repetition_penalty_test.rs b/crates/uzu-engine/unit/backends/common/kernel/repetition_penalty_test.rsindex 4af430b4e..eed5038af 100644--- a/crates/uzu-engine/unit/backends/common/kernel/repetition_penalty_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/repetition_penalty_test.rs@@ -146,7 +146,7 @@ fn check_sampling<B: Backend, T: DataTypeRepr + Float + Debug>() {};for flat in [false, true] {let nodes = nodes(flat);- let topology = BatchTopology::new(&nodes, flat);+ let topology = BatchTopology::new(nodes.into_boxed_slice(), flat);for range in [0..9, 3..5, 7..9] {let mut logits = vec![T::from(-16.0).unwrap(); range.len() * VOCAB_SIZE];for row in logits.as_chunks_mut::<VOCAB_SIZE>().0 {diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rsindex 80e45225f..b30c6e377 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs@@ -28,6 +28,22 @@ use crate::{type AttentionShape = (usize, usize, usize, usize, usize, bool);+fn flat_trie(+ prefix_length: usize,+ tokens: &[u64],+ prng: &PRng,+) -> TrieNode {+ assert!(!tokens.is_empty(), "need seed node");++ let mut node = TrieNode::new(tokens[0], prng.derive(prefix_length as u64), 0.0);+ for (index, token) in tokens.iter().copied().enumerate().skip(1).rev() {+ let mut parent = TrieNode::new(token, prng.derive((prefix_length + index) as u64), 0.0);+ parent.add(node).unwrap();+ node = parent;+ }+ node+}+pub(super) fn fill_attention(count: usize,phase: f32,@@ -85,7 +101,7 @@ fn attention_matches_cpu() {]).to_vec()} else {- TrieNode::flat(0, &tokens, &PRng::new(0)).linearize().token_subtrie_ranges().collect()+ flat_trie(0, &tokens, &PRng::new(0)).linearize().token_subtrie_ranges().collect()};let (shape, nodes) =((head_dim, q_heads, kv_heads, suffix, prefix, true), use_trie.then_some(trie.as_slice()));diff --git a/crates/uzu-engine/unit/encodable_block/sampling_test.rs b/crates/uzu-engine/unit/encodable_block/sampling_test.rsindex db5534c79..6358db14c 100644--- a/crates/uzu-engine/unit/encodable_block/sampling_test.rs+++ b/crates/uzu-engine/unit/encodable_block/sampling_test.rs@@ -8,10 +8,7 @@ use rand_distr::Normal;use uzu_engine_macros::uzu_test;use crate::{- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,- gpu_types::trie::TrieNode,- },+ backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},data_type::{DataType, DataTypeRepr},encodable_block::{batch_topology::BatchTopology,@@ -51,14 +48,7 @@ fn do_sampling_backend<B: Backend, T: DataTypeRepr + Float>(let seeds_buffer = seeds.map(|seeds| command_buffer.allocate_constant_from_slice(seeds).unwrap());let bitmask_buffer = bitmask.map(|bitmask| command_buffer.allocate_constant_from_slice(bitmask).unwrap());- let nodes = (0..batch_size)- .map(|index| TrieNode {- trie_start: index,- trie_end: batch_size - 1,- height: index,- })- .collect::<Box<[_]>>();- let batch_topology = BatchTopology::new(&nodes, true);+ let batch_topology = BatchTopology::flat(batch_size, true);let sampled_buffer = sampling.encode(