A high-performance inference engine for AI models
xgrammar fix
eugeneaf5febdd1f6e
diff --git a/crates/uzu/src/backends/metal/encodable_block/sampling.rs b/crates/uzu/src/backends/metal/encodable_block/sampling.rsindex 269a8c1cd..7b1d21648 100644--- a/crates/uzu/src/backends/metal/encodable_block/sampling.rs+++ b/crates/uzu/src/backends/metal/encodable_block/sampling.rs@@ -82,12 +82,18 @@ impl EncodableBlock for Sampling {state.sampling_output().unwrap().borrow_mut();let sampling_method = state.sampling_method().unwrap();+ let seeds_offset = seeds.buffer_offset();let root_command_buffer =command_buffer.root_command_buffer().to_owned();+ let bitmask_buffer = state+ .token_bitmask()+ .map(|cell| unsafe { cell.borrow_mut().mtl_buffer().clone() });if let Err(e) = self.kernel.encode(unsafe { &logits.mtl_buffer() },unsafe { Some(&seeds.mtl_buffer()) },+ seeds_offset,+ bitmask_buffer.as_ref(),unsafe { &output_buffer_ref.mtl_buffer() },sampling_method,batch_size,diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rsindex 64e6d1c08..2468fd782 100644--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs@@ -162,6 +162,9 @@ impl ForwardPassState {DataType::U32,)};+ if let Ok(dst) = bitmask_array.as_slice_mut::<u32>() {+ dst.fill(0);+ }context.copy_from_view(&mut bitmask_array, bitmask.into());RefCell::new(bitmask_array)});diff --git a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metalindex 27e7c0780..c1571bc2f 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal@@ -16,6 +16,7 @@ void batched_bitmask(uint batch_idx,uint thread_idx) {+ uint bitmask_size = (vocab_size + (BITS_IN_U32 - 1)) / BITS_IN_U32;uint base_idx =batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;uint batch_end = batch_idx * vocab_size + vocab_size;@@ -24,8 +25,10 @@ void batched_bitmask(for (uint i = 0; i < GRAIN_SIZE; i++) {uint global_idx = base_idx + i * BLOCK_SIZE;if (global_idx < batch_end) {- bool mask = (bitmask_data[global_idx / BITS_IN_U32] >>- (global_idx % BITS_IN_U32)) &+ uint token_idx = global_idx - batch_idx * vocab_size;+ uint bitmask_idx = batch_idx * bitmask_size + (token_idx / BITS_IN_U32);+ bool mask = (bitmask_data[bitmask_idx] >>+ (token_idx % BITS_IN_U32)) &0b1;processed_logits[global_idx] =select(T(-INFINITY), logits_data[global_idx], mask);@@ -81,5 +84,5 @@ generateBitmaskKernels(batched_bitmask)#undef outerArguments#undef innerArguments-#undef generatebitmaskKernel-#undef generatebitmaskKernels+#undef generateBitmaskKernel+#undef generateBitmaskKernelsdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rsindex 1e599cc6c..aedbd34bd 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs@@ -51,14 +51,16 @@ enum ArgmaxImplementation {}pub struct SamplingKernel {+ bitmask_pipeline: MTLComputePipelineState,+ bitmask_applied_logits: MTLBuffer,temperature_pipeline: MTLComputePipelineState,- partial_temperature_buffer: MTLBuffer,+ temperature_applied_logits: MTLBuffer,topk_pipeline: MTLComputePipelineState,- partial_topk_buffer: MTLBuffer,+ topk_applied_logits: MTLBuffer,topp_pipeline: MTLComputePipelineState,- partial_topp_buffer: MTLBuffer,+ topp_applied_logits: MTLBuffer,gumbel_pipeline: MTLComputePipelineState,- partial_gumbel_buffer: MTLBuffer,+ gumbel_applied_logits: MTLBuffer,argmax_implementation: ArgmaxImplementation,max_batch_size: usize,max_vocab_size: usize,@@ -104,6 +106,20 @@ impl SamplingKernel {let data_suffix = data_type.function_name_suffix();let max_elements = max_batch_size * max_vocab_size;+ let bitmask_pipeline = context+ .compute_pipeline_state_with_reflection(+ &format!("batched_bitmask_{}", data_suffix),+ None,+ )+ .map(|(pipeline, _)| pipeline)+ .map_err(SamplingError::MetalError)?;++ let bitmask_applied_logits = context.device.new_buffer(+ (max_elements * Into::<DataType>::into(data_type).size_in_bytes())+ as u64,+ MTLResourceOptions::StorageModeShared,+ );+let temperature_pipeline = context.compute_pipeline_state_with_reflection(&format!("batched_temperature_{}", data_suffix),@@ -112,7 +128,7 @@ impl SamplingKernel {.map(|(pipeline, _)| pipeline).map_err(SamplingError::MetalError)?;- let partial_temperature_buffer = context.device.new_buffer(+ let temperature_applied_logits = context.device.new_buffer((max_elements * Into::<DataType>::into(data_type).size_in_bytes())as u64,MTLResourceOptions::StorageModeShared,@@ -126,7 +142,7 @@ impl SamplingKernel {.map(|(pipeline, _)| pipeline).map_err(SamplingError::MetalError)?;- let partial_topk_buffer = context.device.new_buffer(+ let topk_applied_logits = context.device.new_buffer((max_elements * Into::<DataType>::into(data_type).size_in_bytes())as u64,MTLResourceOptions::StorageModeShared,@@ -140,7 +156,7 @@ impl SamplingKernel {.map(|(pipeline, _)| pipeline).map_err(SamplingError::MetalError)?;- let partial_topp_buffer = context.device.new_buffer(+ let topp_applied_logits = context.device.new_buffer((max_elements * Into::<DataType>::into(data_type).size_in_bytes())as u64,MTLResourceOptions::StorageModeShared,@@ -154,7 +170,7 @@ impl SamplingKernel {.map(|(pipeline, _)| pipeline).map_err(SamplingError::MetalError)?;- let partial_gumbel_buffer = context.device.new_buffer(+ let gumbel_applied_logits = context.device.new_buffer((max_elements * Into::<DataType>::into(data_type).size_in_bytes())as u64,MTLResourceOptions::StorageModeShared,@@ -212,14 +228,16 @@ impl SamplingKernel {};Ok(Self {+ bitmask_pipeline,+ bitmask_applied_logits,temperature_pipeline,- partial_temperature_buffer,+ temperature_applied_logits,topk_pipeline,- partial_topk_buffer,+ topk_applied_logits,topp_pipeline,- partial_topp_buffer,+ topp_applied_logits,gumbel_pipeline,- partial_gumbel_buffer,+ gumbel_applied_logits,argmax_implementation,max_batch_size,max_vocab_size,@@ -230,6 +248,8 @@ impl SamplingKernel {&self,logits_buffer: &MTLBuffer,seeds_buffer: Option<&MTLBuffer>,+ seeds_offset: usize,+ bitmask_buffer: Option<&MTLBuffer>,sampled_tokens_buffer: &MTLBuffer,sampling_method: SamplingMethod,batch_size: usize,@@ -240,6 +260,8 @@ impl SamplingKernel {self.encode_with_encoder(logits_buffer,seeds_buffer,+ seeds_offset,+ bitmask_buffer,sampled_tokens_buffer,sampling_method,batch_size,@@ -254,6 +276,8 @@ impl SamplingKernel {&self,logits_buffer: &MTLBuffer,seeds_buffer: Option<&MTLBuffer>,+ seeds_offset: usize,+ bitmask_buffer: Option<&MTLBuffer>,sampled_tokens_buffer: &MTLBuffer,sampling_method: SamplingMethod,batch_size: usize,@@ -275,6 +299,18 @@ impl SamplingKernel {let mut last_logits_buffer = logits_buffer;+ if let Some(bitmask_buffer) = bitmask_buffer {+ self.encode_bitmask(+ last_logits_buffer,+ bitmask_buffer,+ &self.bitmask_applied_logits,+ batch_size as u32,+ vocab_size as u32,+ compute_encoder,+ )?;+ last_logits_buffer = &self.bitmask_applied_logits;+ }+if let SamplingMethod::Stochastic {temperature,top_k,@@ -284,48 +320,49 @@ impl SamplingKernel {if let Some(temperature) = temperature {self.encode_temperature(last_logits_buffer,- &self.partial_temperature_buffer,+ &self.temperature_applied_logits,batch_size as u32,vocab_size as u32,temperature,compute_encoder,)?;- last_logits_buffer = &self.partial_temperature_buffer;+ last_logits_buffer = &self.temperature_applied_logits;}if let Some(top_k) = top_k {self.encode_topk(last_logits_buffer,- &self.partial_topk_buffer,+ &self.topk_applied_logits,batch_size as u32,vocab_size as u32,top_k,compute_encoder,)?;- last_logits_buffer = &self.partial_topk_buffer;+ last_logits_buffer = &self.topk_applied_logits;}if let Some(top_p) = top_p {self.encode_topp(last_logits_buffer,- &self.partial_topp_buffer,+ &self.topp_applied_logits,batch_size as u32,vocab_size as u32,top_p,compute_encoder,)?;- last_logits_buffer = &self.partial_topp_buffer;+ last_logits_buffer = &self.topp_applied_logits;}self.encode_gumbel(last_logits_buffer,seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,- &self.partial_gumbel_buffer,+ seeds_offset,+ &self.gumbel_applied_logits,batch_size as u32,vocab_size as u32,compute_encoder,)?;- last_logits_buffer = &self.partial_gumbel_buffer;+ last_logits_buffer = &self.gumbel_applied_logits;}match &self.argmax_implementation {@@ -356,6 +393,38 @@ impl SamplingKernel {}}+ pub fn encode_bitmask(+ &self,+ logits_buffer: &MTLBuffer,+ bitmask_buffer: &MTLBuffer,+ processed_logits_buffer: &MTLBuffer,+ batch_size: u32,+ vocab_size: u32,+ compute_encoder: &ComputeCommandEncoderRef,+ ) -> Result<(), SamplingError> {+ compute_encoder.set_compute_pipeline_state(&self.bitmask_pipeline);++ compute_encoder.set_buffer(0, Some(logits_buffer), 0);+ compute_encoder.set_buffer(1, Some(bitmask_buffer), 0);+ compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);+ compute_encoder.set_bytes(+ 3,+ size_of::<u32>() as u64,+ &vocab_size as *const u32 as *const std::ffi::c_void,+ );++ let elements_in_group = BLOCK_SIZE * ELEMENTWISE_GRAIN_SIZE;+ let groups = (vocab_size + (elements_in_group as u32 - 1))+ / elements_in_group as u32;++ compute_encoder.dispatch_thread_groups(+ MTLSize::new(groups as u64, batch_size as u64, 1),+ MTLSize::new(BLOCK_SIZE as u64, 1, 1),+ );++ Ok(())+ }+pub fn encode_temperature(&self,logits_buffer: &MTLBuffer,@@ -460,6 +529,7 @@ impl SamplingKernel {&self,logits_buffer: &MTLBuffer,seeds_buffer: &MTLBuffer,+ seeds_offset: usize,processed_logits_buffer: &MTLBuffer,batch_size: u32,vocab_size: u32,@@ -468,7 +538,7 @@ impl SamplingKernel {compute_encoder.set_compute_pipeline_state(&self.gumbel_pipeline);compute_encoder.set_buffer(0, Some(logits_buffer), 0);- compute_encoder.set_buffer(1, Some(seeds_buffer), 0);+ compute_encoder.set_buffer(1, Some(seeds_buffer), seeds_offset as u64);compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);compute_encoder.set_bytes(3,diff --git a/crates/uzu/src/language_model/grammar.rs b/crates/uzu/src/language_model/grammar.rsindex d2ede8835..9d6d841fe 100644--- a/crates/uzu/src/language_model/grammar.rs+++ b/crates/uzu/src/language_model/grammar.rs@@ -177,12 +177,19 @@ impl CompiledGrammar {pub fn accept_token(&mut self,token_id: u64,- ) {+ ) -> Result<(), Error> {if self.engagement_state.is_engaged() {+ if (token_id as usize) >= self.vocab_size {+ return Err(Error::TokenOutOfGrammarRange(+ token_id,+ self.vocab_size,+ ));+ }self.matcher.accept_token(token_id as i32);}self.engagement_state.accept_token(token_id);+ Ok(())}pub fn rollback(@@ -200,4 +207,8 @@ impl CompiledGrammar {self.matcher.reset();self.engagement_state.reset();}++ pub fn is_terminated(&self) -> bool {+ self.matcher.is_terminated()+ }}diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 3fb445fb3..5a0af9763 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -156,7 +156,16 @@ impl LanguageModelGenerator {step_token_positions.collect::<Box<[usize]>>();let step_token_bitmask = step_token_bitmask.map(|mask| match mask {- Some(mask) => Either::Left(mask.iter().copied()),+ Some(mask) => Either::Left(+ mask.iter()+ .copied()+ .take(single_token_bitmask_size)+ .chain(repeat_n(+ 0u32,+ single_token_bitmask_size+ .saturating_sub(mask.len()),+ )),+ ),None => Either::Right(repeat_n(u32::MAX,single_token_bitmask_size,@@ -269,7 +278,7 @@ impl LanguageModelGenerator {if let Some(compiled_grammar) = compiled_grammar.as_deref_mut() {for &token in &accepted_tokens {- compiled_grammar.accept_token(token);+ compiled_grammar.accept_token(token)?;}}@@ -322,7 +331,15 @@ impl LanguageModelGenerator {.token_masks().chain(repeat_n(None, suffix_length - active_suffix_length)).map(|mask| match mask {- Some(mask) => Either::Left(mask.iter().copied()),+ Some(mask) => Either::Left(+ mask.iter().copied().take(single_token_bitmask_size).chain(+ repeat_n(+ 0u32,+ single_token_bitmask_size+ .saturating_sub(mask.len()),+ ),+ ),+ ),None => {Either::Right(repeat_n(u32::MAX, single_token_bitmask_size))},@@ -369,7 +386,7 @@ impl LanguageModelGenerator {if let Some(compiled_grammar) = compiled_grammar.as_deref_mut() {for &token in &accepted_tokens {- compiled_grammar.accept_token(token);+ compiled_grammar.accept_token(token)?;}}diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex 23b19607f..4de7b809f 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -121,6 +121,11 @@ impl ChatSession {}}+ let language_model_config = model_metadata+ .model_config+ .as_language_model()+ .ok_or(Error::UnableToLoadConfig)?;+let tokenizer_path = model_path.join("tokenizer.json");if !tokenizer_path.exists() {return Err(Error::UnableToLoadTokenizer);@@ -128,15 +133,17 @@ impl ChatSession {let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;- // Create TokenizerInfo for grammar support- let tokenizer_info =- TokenizerInfo::from_huggingface(&tokenizer, None, None);-- // Extract language model config- let language_model_config = model_metadata- .model_config- .as_language_model()- .ok_or(Error::UnableToLoadConfig)?;+ let stop_token_ids: Vec<i32> = language_model_config+ .generation_config+ .stop_token_ids+ .iter()+ .map(|&x| x as i32)+ .collect();+ let tokenizer_info = TokenizerInfo::from_huggingface(+ &tokenizer,+ None,+ Some(&stop_token_ids),+ );let input_processor = InputProcessorDefault::new(language_model_config.message_processor_config.clone(),@@ -183,11 +190,11 @@ impl ChatSession {match &context_mode {ContextMode::None => {- let llm = self+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- llm.reset_state();+ language_model_generator.reset_state();},ContextMode::Static {input,@@ -201,7 +208,7 @@ impl ChatSession {}let tmp = self.static_context.take();if let Some(ref ctx) = tmp {- self.reconfigure_llm(Some(ctx))?;+ self.reconfigure_language_model_generator(Some(ctx))?;}self.static_context = tmp;},@@ -215,11 +222,11 @@ impl ChatSession {| ContextMode::Static {..} => {- let llm = self+ let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- llm.reset_state();+ language_model_generator.reset_state();},ContextMode::Dynamic => {},}@@ -227,9 +234,9 @@ impl ChatSession {}pub fn reset(&mut self) -> Result<(), Error> {- let llm =+ let language_model_generator =self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- llm.reset_state();+ language_model_generator.reset_state();Ok(())}}@@ -241,13 +248,13 @@ impl ChatSession {context: Option<&Context>,config: RunConfig,) -> Result<(Output, Context), Error> {- self.reconfigure_llm(context)?;+ self.reconfigure_language_model_generator(context)?;let output =self.run_internal(input, config, None::<fn(Output) -> bool>)?;- let new_context = self.build_context_from_llm()?;- let llm =+ let new_context = self.build_context_from_language_model_generator()?;+ let language_model_generator =self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- llm.reset_state();+ language_model_generator.reset_state();Ok((output, new_context))}@@ -278,15 +285,17 @@ impl ChatSession {.as_language_model().ok_or(Error::UnableToLoadConfig)?;- let llm =+ let language_model_generator =self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- let context_length =- llm.decoding_config.context_length.resolve(language_model_config);+ let context_length = language_model_generator+ .decoding_config+ .context_length+ .resolve(language_model_config);if tokens.len() >= context_length {return Err(Error::ContextLengthExceeded);}- let prefix_offset = llm.tokens.len();+ let prefix_offset = language_model_generator.tokens.len();let prefix_len_before = prefix_offset.saturating_sub(1);let eos_tokens: Vec<u64> = language_model_config@@ -312,7 +321,7 @@ impl ChatSession {let prefill_start = Instant::now();let sample_suffix = config.tokens_limit > 0;- let prefill_result = llm.prefill(+ let prefill_result = language_model_generator.prefill(tokens.clone(),compiled_grammar.as_mut(),sampling_method,@@ -321,9 +330,9 @@ impl ChatSession {)?;let prefill_tokens = prefill_result.tokens.clone();let prefill_duration = prefill_start.elapsed().as_secs_f64();- llm.clear_cache();+ language_model_generator.clear_cache();- let prefill_suffix_length = llm+ let prefill_suffix_length = language_model_generator.decoding_config.prefill_step_size.resolve(language_model_config);@@ -340,14 +349,27 @@ impl ChatSession {run_start,};- let prefill_finish_reason =- Self::check_finish_reason(&run_context, llm, &prefill_tokens);+ let grammar_terminated = config.stop_on_grammar_complete+ && compiled_grammar+ .as_ref()+ .map(|g| g.is_terminated())+ .unwrap_or(false);++ let prefill_finish_reason = if grammar_terminated {+ Some(FinishReason::Stop)+ } else {+ Self::check_finish_reason(+ &run_context,+ language_model_generator,+ &prefill_tokens,+ )+ };let prefill_output = Self::build_output(&self.model_metadata,&self.tokenizer,&self.output_parser,&run_context,- llm,+ language_model_generator,&[],&[],prefill_finish_reason.clone(),@@ -368,19 +390,23 @@ impl ChatSession {}}- let can_use_async = llm.decoding_config.generate_suffix_length() == 1- && !llm.has_attention_layers()- && compiled_grammar.is_none();+ let can_use_async =+ language_model_generator.decoding_config.generate_suffix_length()+ == 1+ && !language_model_generator.has_attention_layers()+ && compiled_grammar.is_none();let generate_output = if can_use_async {- let batch_size =- llm.decoding_config.async_batch_size.resolve(&self.model_path);+ let batch_size = language_model_generator+ .decoding_config+ .async_batch_size+ .resolve(&self.model_path);Self::run_async_batch(&self.model_metadata,&self.tokenizer,&self.output_parser,&run_context,- llm,+ language_model_generator,sampling_method,&progress,batch_size,@@ -391,14 +417,15 @@ impl ChatSession {&self.tokenizer,&self.output_parser,&run_context,- llm,+ language_model_generator,compiled_grammar.as_mut(),sampling_method,+ config.stop_on_grammar_complete,&progress,)?};- llm.clear_cache();+ language_model_generator.clear_cache();Ok(generate_output.clone_with_duration(run_start.elapsed().as_secs_f64()))}@@ -408,9 +435,10 @@ impl ChatSession {tokenizer: &Tokenizer,output_parser: &OutputParser,run_context: &RunContext,- llm: &mut LanguageModelGenerator,+ language_model_generator: &mut LanguageModelGenerator,compiled_grammar: Option<&mut CompiledGrammar>,sampling_method: super::parameter::SamplingMethod,+ stop_on_grammar_complete: bool,progress: &Option<F>,) -> Result<Output, Error>where@@ -422,7 +450,7 @@ impl ChatSession {loop {let generate_start = Instant::now();- let generate_result = llm.generate(+ let generate_result = language_model_generator.generate(compiled_grammar_mut.as_deref_mut(),sampling_method,)?;@@ -431,14 +459,27 @@ impl ChatSession {generate_results.push(generate_result);generate_durations.push(generate_duration);- let generate_finish_reason =- Self::check_finish_reason(run_context, llm, &generate_tokens);+ let grammar_terminated = stop_on_grammar_complete+ && compiled_grammar_mut+ .as_ref()+ .map(|g| g.is_terminated())+ .unwrap_or(false);++ let generate_finish_reason = if grammar_terminated {+ Some(FinishReason::Stop)+ } else {+ Self::check_finish_reason(+ run_context,+ language_model_generator,+ &generate_tokens,+ )+ };let generate_output = Self::build_output(model_metadata,tokenizer,output_parser,run_context,- llm,+ language_model_generator,&generate_results,&generate_durations,generate_finish_reason.clone(),@@ -589,15 +630,16 @@ impl ChatSession {)}- fn reconfigure_llm(+ fn reconfigure_language_model_generator(&mut self,context: Option<&Context>,) -> Result<(), Error> {- let llm =+ let language_model_generator =self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- llm.reset_state();+ language_model_generator.reset_state();if let Some(ctx) = context {- let mut llm_state = llm.context.cache_layers.borrow_mut();+ let mut llm_state =+ language_model_generator.context.cache_layers.borrow_mut();for (ctx_layer, gen_layer) inctx.cache_layers.data.iter().zip(llm_state.data.iter_mut()){@@ -655,20 +697,25 @@ impl ChatSession {}drop(llm_state);- llm.tokens = ctx.tokens.clone();+ language_model_generator.tokens = ctx.tokens.clone();}Ok(())}- fn build_context_from_llm(&self) -> Result<Context, Error> {- let llm =+ fn build_context_from_language_model_generator(+ &self+ ) -> Result<Context, Error> {+ let language_model_generator =self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- let cache_layers =- llm.context.cache_layers.borrow().clone(&llm.context.mtl_context);+ let cache_layers = language_model_generator+ .context+ .cache_layers+ .borrow()+ .clone(&language_model_generator.context.mtl_context);let context = Context::new(- llm.tokens.clone(),+ language_model_generator.tokens.clone(),cache_layers,- llm.decoding_config.clone(),+ language_model_generator.decoding_config.clone(),);Ok(context)}@@ -677,15 +724,17 @@ impl ChatSession {impl ChatSession {fn check_finish_reason(run_context: &RunContext,- llm: &LanguageModelGenerator,+ language_model_generator: &LanguageModelGenerator,new_tokens: &[u64],) -> Option<FinishReason> {let start_idx =run_context.prefix_len_before + run_context.input_tokens_len;- let total_new_tokens = llm.tokens.len().saturating_sub(start_idx);+ let total_new_tokens =+ language_model_generator.tokens.len().saturating_sub(start_idx);let has_eos =new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));- let context_limit = llm.tokens.len() >= run_context.context_length;+ let context_limit =+ language_model_generator.tokens.len() >= run_context.context_length;if has_eos {Some(FinishReason::Stop)@@ -700,13 +749,16 @@ impl ChatSession {fn decode_generated_tokens(tokenizer: &Tokenizer,- llm: &LanguageModelGenerator,+ language_model_generator: &LanguageModelGenerator,run_context: &RunContext,) -> Result<String, Error> {let start_idx =run_context.prefix_len_before + run_context.input_tokens_len;- let generated_tokens: Vec<u32> =- llm.tokens[start_idx..].iter().map(|&v| v as u32).collect();+ let generated_tokens: Vec<u32> = language_model_generator.tokens+ [start_idx..]+ .iter()+ .map(|&v| v as u32)+ .collect();tokenizer.decode(&generated_tokens, true).map_err(|_| Error::UnableToDecodeText)@@ -717,16 +769,21 @@ impl ChatSession {tokenizer: &Tokenizer,output_parser: &OutputParser,run_context: &RunContext,- llm: &LanguageModelGenerator,+ language_model_generator: &LanguageModelGenerator,generate_results: &[GenerateResult],generate_durations: &[f64],finish_reason: Option<FinishReason>,) -> Result<Output, Error> {- let text = Self::decode_generated_tokens(tokenizer, llm, run_context)?;+ let text = Self::decode_generated_tokens(+ tokenizer,+ language_model_generator,+ run_context,+ )?;let parsed = output_parser.parse(text);let start_idx =run_context.prefix_len_before + run_context.input_tokens_len;- let output_tokens = llm.tokens.len().saturating_sub(start_idx);+ let output_tokens =+ language_model_generator.tokens.len().saturating_sub(start_idx);Ok(Output {text: parsed,@@ -737,7 +794,9 @@ impl ChatSession {run_context.prefill_suffix_length,generate_results.to_vec(),generate_durations.to_vec(),- llm.decoding_config.generate_suffix_length(),+ language_model_generator+ .decoding_config+ .generate_suffix_length(),run_context.run_start.elapsed().as_secs_f64(),run_context.input_tokens_len,output_tokens,diff --git a/crates/uzu/src/session/config/run_config.rs b/crates/uzu/src/session/config/run_config.rsindex ad0a3f9de..ecbe720ac 100644--- a/crates/uzu/src/session/config/run_config.rs+++ b/crates/uzu/src/session/config/run_config.rs@@ -6,6 +6,7 @@ pub struct RunConfig {pub enable_thinking: bool,pub sampling_policy: SamplingPolicy,pub grammar_config: Option<GrammarConfig>,+ pub stop_on_grammar_complete: bool,}impl RunConfig {@@ -19,6 +20,7 @@ impl RunConfig {enable_thinking,sampling_policy,grammar_config: None,+ stop_on_grammar_complete: false,}}@@ -53,6 +55,14 @@ impl RunConfig {self.grammar_config = Some(grammar_config);self}++ pub fn stop_on_grammar_complete(+ mut self,+ stop_on_grammar_complete: bool,+ ) -> Self {+ self.stop_on_grammar_complete = stop_on_grammar_complete;+ self+ }}impl Default for RunConfig {diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rsindex c75b0a14d..273d3e4ce 100644--- a/crates/uzu/src/session/types/error.rs+++ b/crates/uzu/src/session/types/error.rs@@ -42,6 +42,8 @@ pub enum Error {SamplingFailed,#[error("Grammar error")]GrammarError,+ #[error("Token {0} out of grammar vocabulary range (0..{1})")]+ TokenOutOfGrammarRange(u64, usize),#[error("Classifier error: {0}")]Classifier(#[from] ClassifierError),}diff --git a/crates/uzu/src/trie.rs b/crates/uzu/src/trie.rsindex 5faf525b8..0fcdbca88 100644--- a/crates/uzu/src/trie.rs+++ b/crates/uzu/src/trie.rs@@ -123,7 +123,14 @@ impl TrieNode {let mask = if let Some(compiled_grammar) =compiled_grammar.as_deref_mut(){- compiled_grammar.accept_token(next_speculated_token);+ if compiled_grammar+ .accept_token(next_speculated_token)+ .is_err()+ {+ cur_node_speculator_weights+ .remove(&next_speculated_token);+ continue;+ }let next_bitmask = compiled_grammar.next_bitmask().unwrap();compiled_grammar.rollback(1);@@ -154,7 +161,9 @@ impl TrieNode {speculated_suffix.push(next_node_token);if let Some(compiled_grammar) = compiled_grammar.as_deref_mut(){- compiled_grammar.accept_token(next_node_token);+ if compiled_grammar.accept_token(next_node_token).is_err() {+ break;+ }grammar_accept_count += 1;}cur_node = cur_node.get_mut(next_node_token).unwrap();