Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

xgrammar fix

eugeneaf5febdd1f6e
diff --git a/crates/uzu/src/backends/metal/encodable_block/sampling.rs b/crates/uzu/src/backends/metal/encodable_block/sampling.rs
index 269a8c1cd..7b1d21648 100644
--- a/crates/uzu/src/backends/metal/encodable_block/sampling.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/sampling.rs
@@ -82,12 +82,18 @@ impl EncodableBlock for Sampling {
state.sampling_output().unwrap().borrow_mut();
let sampling_method = state.sampling_method().unwrap();
+ let seeds_offset = seeds.buffer_offset();
let root_command_buffer =
command_buffer.root_command_buffer().to_owned();
+ let bitmask_buffer = state
+ .token_bitmask()
+ .map(|cell| unsafe { cell.borrow_mut().mtl_buffer().clone() });
if let Err(e) = self.kernel.encode(
unsafe { &logits.mtl_buffer() },
unsafe { Some(&seeds.mtl_buffer()) },
+ seeds_offset,
+ bitmask_buffer.as_ref(),
unsafe { &output_buffer_ref.mtl_buffer() },
sampling_method,
batch_size,
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
index 64e6d1c08..2468fd782 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
@@ -162,6 +162,9 @@ impl ForwardPassState {
DataType::U32,
)
};
+ if let Ok(dst) = bitmask_array.as_slice_mut::<u32>() {
+ dst.fill(0);
+ }
context.copy_from_view(&mut bitmask_array, bitmask.into());
RefCell::new(bitmask_array)
});
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
index 27e7c0780..c1571bc2f 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
@@ -16,6 +16,7 @@ void batched_bitmask(
uint batch_idx,
uint thread_idx
) {
+ uint bitmask_size = (vocab_size + (BITS_IN_U32 - 1)) / BITS_IN_U32;
uint base_idx =
batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
uint batch_end = batch_idx * vocab_size + vocab_size;
@@ -24,8 +25,10 @@ void batched_bitmask(
for (uint i = 0; i < GRAIN_SIZE; i++) {
uint global_idx = base_idx + i * BLOCK_SIZE;
if (global_idx < batch_end) {
- bool mask = (bitmask_data[global_idx / BITS_IN_U32] >>
- (global_idx % BITS_IN_U32)) &
+ uint token_idx = global_idx - batch_idx * vocab_size;
+ uint bitmask_idx = batch_idx * bitmask_size + (token_idx / BITS_IN_U32);
+ bool mask = (bitmask_data[bitmask_idx] >>
+ (token_idx % BITS_IN_U32)) &
0b1;
processed_logits[global_idx] =
select(T(-INFINITY), logits_data[global_idx], mask);
@@ -81,5 +84,5 @@ generateBitmaskKernels(batched_bitmask)
#undef outerArguments
#undef innerArguments
-#undef generatebitmaskKernel
-#undef generatebitmaskKernels
+#undef generateBitmaskKernel
+#undef generateBitmaskKernels
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
index 1e599cc6c..aedbd34bd 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
@@ -51,14 +51,16 @@ enum ArgmaxImplementation {
}
pub struct SamplingKernel {
+ bitmask_pipeline: MTLComputePipelineState,
+ bitmask_applied_logits: MTLBuffer,
temperature_pipeline: MTLComputePipelineState,
- partial_temperature_buffer: MTLBuffer,
+ temperature_applied_logits: MTLBuffer,
topk_pipeline: MTLComputePipelineState,
- partial_topk_buffer: MTLBuffer,
+ topk_applied_logits: MTLBuffer,
topp_pipeline: MTLComputePipelineState,
- partial_topp_buffer: MTLBuffer,
+ topp_applied_logits: MTLBuffer,
gumbel_pipeline: MTLComputePipelineState,
- partial_gumbel_buffer: MTLBuffer,
+ gumbel_applied_logits: MTLBuffer,
argmax_implementation: ArgmaxImplementation,
max_batch_size: usize,
max_vocab_size: usize,
@@ -104,6 +106,20 @@ impl SamplingKernel {
let data_suffix = data_type.function_name_suffix();
let max_elements = max_batch_size * max_vocab_size;
+ let bitmask_pipeline = context
+ .compute_pipeline_state_with_reflection(
+ &format!("batched_bitmask_{}", data_suffix),
+ None,
+ )
+ .map(|(pipeline, _)| pipeline)
+ .map_err(SamplingError::MetalError)?;
+
+ let bitmask_applied_logits = context.device.new_buffer(
+ (max_elements * Into::<DataType>::into(data_type).size_in_bytes())
+ as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
let temperature_pipeline = context
.compute_pipeline_state_with_reflection(
&format!("batched_temperature_{}", data_suffix),
@@ -112,7 +128,7 @@ impl SamplingKernel {
.map(|(pipeline, _)| pipeline)
.map_err(SamplingError::MetalError)?;
- let partial_temperature_buffer = context.device.new_buffer(
+ let temperature_applied_logits = context.device.new_buffer(
(max_elements * Into::<DataType>::into(data_type).size_in_bytes())
as u64,
MTLResourceOptions::StorageModeShared,
@@ -126,7 +142,7 @@ impl SamplingKernel {
.map(|(pipeline, _)| pipeline)
.map_err(SamplingError::MetalError)?;
- let partial_topk_buffer = context.device.new_buffer(
+ let topk_applied_logits = context.device.new_buffer(
(max_elements * Into::<DataType>::into(data_type).size_in_bytes())
as u64,
MTLResourceOptions::StorageModeShared,
@@ -140,7 +156,7 @@ impl SamplingKernel {
.map(|(pipeline, _)| pipeline)
.map_err(SamplingError::MetalError)?;
- let partial_topp_buffer = context.device.new_buffer(
+ let topp_applied_logits = context.device.new_buffer(
(max_elements * Into::<DataType>::into(data_type).size_in_bytes())
as u64,
MTLResourceOptions::StorageModeShared,
@@ -154,7 +170,7 @@ impl SamplingKernel {
.map(|(pipeline, _)| pipeline)
.map_err(SamplingError::MetalError)?;
- let partial_gumbel_buffer = context.device.new_buffer(
+ let gumbel_applied_logits = context.device.new_buffer(
(max_elements * Into::<DataType>::into(data_type).size_in_bytes())
as u64,
MTLResourceOptions::StorageModeShared,
@@ -212,14 +228,16 @@ impl SamplingKernel {
};
Ok(Self {
+ bitmask_pipeline,
+ bitmask_applied_logits,
temperature_pipeline,
- partial_temperature_buffer,
+ temperature_applied_logits,
topk_pipeline,
- partial_topk_buffer,
+ topk_applied_logits,
topp_pipeline,
- partial_topp_buffer,
+ topp_applied_logits,
gumbel_pipeline,
- partial_gumbel_buffer,
+ gumbel_applied_logits,
argmax_implementation,
max_batch_size,
max_vocab_size,
@@ -230,6 +248,8 @@ impl SamplingKernel {
&self,
logits_buffer: &MTLBuffer,
seeds_buffer: Option<&MTLBuffer>,
+ seeds_offset: usize,
+ bitmask_buffer: Option<&MTLBuffer>,
sampled_tokens_buffer: &MTLBuffer,
sampling_method: SamplingMethod,
batch_size: usize,
@@ -240,6 +260,8 @@ impl SamplingKernel {
self.encode_with_encoder(
logits_buffer,
seeds_buffer,
+ seeds_offset,
+ bitmask_buffer,
sampled_tokens_buffer,
sampling_method,
batch_size,
@@ -254,6 +276,8 @@ impl SamplingKernel {
&self,
logits_buffer: &MTLBuffer,
seeds_buffer: Option<&MTLBuffer>,
+ seeds_offset: usize,
+ bitmask_buffer: Option<&MTLBuffer>,
sampled_tokens_buffer: &MTLBuffer,
sampling_method: SamplingMethod,
batch_size: usize,
@@ -275,6 +299,18 @@ impl SamplingKernel {
let mut last_logits_buffer = logits_buffer;
+ if let Some(bitmask_buffer) = bitmask_buffer {
+ self.encode_bitmask(
+ last_logits_buffer,
+ bitmask_buffer,
+ &self.bitmask_applied_logits,
+ batch_size as u32,
+ vocab_size as u32,
+ compute_encoder,
+ )?;
+ last_logits_buffer = &self.bitmask_applied_logits;
+ }
+
if let SamplingMethod::Stochastic {
temperature,
top_k,
@@ -284,48 +320,49 @@ impl SamplingKernel {
if let Some(temperature) = temperature {
self.encode_temperature(
last_logits_buffer,
- &self.partial_temperature_buffer,
+ &self.temperature_applied_logits,
batch_size as u32,
vocab_size as u32,
temperature,
compute_encoder,
)?;
- last_logits_buffer = &self.partial_temperature_buffer;
+ last_logits_buffer = &self.temperature_applied_logits;
}
if let Some(top_k) = top_k {
self.encode_topk(
last_logits_buffer,
- &self.partial_topk_buffer,
+ &self.topk_applied_logits,
batch_size as u32,
vocab_size as u32,
top_k,
compute_encoder,
)?;
- last_logits_buffer = &self.partial_topk_buffer;
+ last_logits_buffer = &self.topk_applied_logits;
}
if let Some(top_p) = top_p {
self.encode_topp(
last_logits_buffer,
- &self.partial_topp_buffer,
+ &self.topp_applied_logits,
batch_size as u32,
vocab_size as u32,
top_p,
compute_encoder,
)?;
- last_logits_buffer = &self.partial_topp_buffer;
+ last_logits_buffer = &self.topp_applied_logits;
}
self.encode_gumbel(
last_logits_buffer,
seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,
- &self.partial_gumbel_buffer,
+ seeds_offset,
+ &self.gumbel_applied_logits,
batch_size as u32,
vocab_size as u32,
compute_encoder,
)?;
- last_logits_buffer = &self.partial_gumbel_buffer;
+ last_logits_buffer = &self.gumbel_applied_logits;
}
match &self.argmax_implementation {
@@ -356,6 +393,38 @@ impl SamplingKernel {
}
}
+ pub fn encode_bitmask(
+ &self,
+ logits_buffer: &MTLBuffer,
+ bitmask_buffer: &MTLBuffer,
+ processed_logits_buffer: &MTLBuffer,
+ batch_size: u32,
+ vocab_size: u32,
+ compute_encoder: &ComputeCommandEncoderRef,
+ ) -> Result<(), SamplingError> {
+ compute_encoder.set_compute_pipeline_state(&self.bitmask_pipeline);
+
+ compute_encoder.set_buffer(0, Some(logits_buffer), 0);
+ compute_encoder.set_buffer(1, Some(bitmask_buffer), 0);
+ compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);
+ compute_encoder.set_bytes(
+ 3,
+ size_of::<u32>() as u64,
+ &vocab_size as *const u32 as *const std::ffi::c_void,
+ );
+
+ let elements_in_group = BLOCK_SIZE * ELEMENTWISE_GRAIN_SIZE;
+ let groups = (vocab_size + (elements_in_group as u32 - 1))
+ / elements_in_group as u32;
+
+ compute_encoder.dispatch_thread_groups(
+ MTLSize::new(groups as u64, batch_size as u64, 1),
+ MTLSize::new(BLOCK_SIZE as u64, 1, 1),
+ );
+
+ Ok(())
+ }
+
pub fn encode_temperature(
&self,
logits_buffer: &MTLBuffer,
@@ -460,6 +529,7 @@ impl SamplingKernel {
&self,
logits_buffer: &MTLBuffer,
seeds_buffer: &MTLBuffer,
+ seeds_offset: usize,
processed_logits_buffer: &MTLBuffer,
batch_size: u32,
vocab_size: u32,
@@ -468,7 +538,7 @@ impl SamplingKernel {
compute_encoder.set_compute_pipeline_state(&self.gumbel_pipeline);
compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(seeds_buffer), 0);
+ compute_encoder.set_buffer(1, Some(seeds_buffer), seeds_offset as u64);
compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);
compute_encoder.set_bytes(
3,
diff --git a/crates/uzu/src/language_model/grammar.rs b/crates/uzu/src/language_model/grammar.rs
index d2ede8835..9d6d841fe 100644
--- a/crates/uzu/src/language_model/grammar.rs
+++ b/crates/uzu/src/language_model/grammar.rs
@@ -177,12 +177,19 @@ impl CompiledGrammar {
pub fn accept_token(
&mut self,
token_id: u64,
- ) {
+ ) -> Result<(), Error> {
if self.engagement_state.is_engaged() {
+ if (token_id as usize) >= self.vocab_size {
+ return Err(Error::TokenOutOfGrammarRange(
+ token_id,
+ self.vocab_size,
+ ));
+ }
self.matcher.accept_token(token_id as i32);
}
self.engagement_state.accept_token(token_id);
+ Ok(())
}
pub fn rollback(
@@ -200,4 +207,8 @@ impl CompiledGrammar {
self.matcher.reset();
self.engagement_state.reset();
}
+
+ pub fn is_terminated(&self) -> bool {
+ self.matcher.is_terminated()
+ }
}
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 3fb445fb3..5a0af9763 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -156,7 +156,16 @@ impl LanguageModelGenerator {
step_token_positions.collect::<Box<[usize]>>();
let step_token_bitmask = step_token_bitmask
.map(|mask| match mask {
- Some(mask) => Either::Left(mask.iter().copied()),
+ Some(mask) => Either::Left(
+ mask.iter()
+ .copied()
+ .take(single_token_bitmask_size)
+ .chain(repeat_n(
+ 0u32,
+ single_token_bitmask_size
+ .saturating_sub(mask.len()),
+ )),
+ ),
None => Either::Right(repeat_n(
u32::MAX,
single_token_bitmask_size,
@@ -269,7 +278,7 @@ impl LanguageModelGenerator {
if let Some(compiled_grammar) = compiled_grammar.as_deref_mut() {
for &token in &accepted_tokens {
- compiled_grammar.accept_token(token);
+ compiled_grammar.accept_token(token)?;
}
}
@@ -322,7 +331,15 @@ impl LanguageModelGenerator {
.token_masks()
.chain(repeat_n(None, suffix_length - active_suffix_length))
.map(|mask| match mask {
- Some(mask) => Either::Left(mask.iter().copied()),
+ Some(mask) => Either::Left(
+ mask.iter().copied().take(single_token_bitmask_size).chain(
+ repeat_n(
+ 0u32,
+ single_token_bitmask_size
+ .saturating_sub(mask.len()),
+ ),
+ ),
+ ),
None => {
Either::Right(repeat_n(u32::MAX, single_token_bitmask_size))
},
@@ -369,7 +386,7 @@ impl LanguageModelGenerator {
if let Some(compiled_grammar) = compiled_grammar.as_deref_mut() {
for &token in &accepted_tokens {
- compiled_grammar.accept_token(token);
+ compiled_grammar.accept_token(token)?;
}
}
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index 23b19607f..4de7b809f 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -121,6 +121,11 @@ impl ChatSession {
}
}
+ let language_model_config = model_metadata
+ .model_config
+ .as_language_model()
+ .ok_or(Error::UnableToLoadConfig)?;
+
let tokenizer_path = model_path.join("tokenizer.json");
if !tokenizer_path.exists() {
return Err(Error::UnableToLoadTokenizer);
@@ -128,15 +133,17 @@ impl ChatSession {
let tokenizer = Tokenizer::from_file(&tokenizer_path)
.map_err(|_| Error::UnableToLoadTokenizer)?;
- // Create TokenizerInfo for grammar support
- let tokenizer_info =
- TokenizerInfo::from_huggingface(&tokenizer, None, None);
-
- // Extract language model config
- let language_model_config = model_metadata
- .model_config
- .as_language_model()
- .ok_or(Error::UnableToLoadConfig)?;
+ let stop_token_ids: Vec<i32> = language_model_config
+ .generation_config
+ .stop_token_ids
+ .iter()
+ .map(|&x| x as i32)
+ .collect();
+ let tokenizer_info = TokenizerInfo::from_huggingface(
+ &tokenizer,
+ None,
+ Some(&stop_token_ids),
+ );
let input_processor = InputProcessorDefault::new(
language_model_config.message_processor_config.clone(),
@@ -183,11 +190,11 @@ impl ChatSession {
match &context_mode {
ContextMode::None => {
- let llm = self
+ let language_model_generator = self
.llm
.as_mut()
.ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- llm.reset_state();
+ language_model_generator.reset_state();
},
ContextMode::Static {
input,
@@ -201,7 +208,7 @@ impl ChatSession {
}
let tmp = self.static_context.take();
if let Some(ref ctx) = tmp {
- self.reconfigure_llm(Some(ctx))?;
+ self.reconfigure_language_model_generator(Some(ctx))?;
}
self.static_context = tmp;
},
@@ -215,11 +222,11 @@ impl ChatSession {
| ContextMode::Static {
..
} => {
- let llm = self
+ let language_model_generator = self
.llm
.as_mut()
.ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- llm.reset_state();
+ language_model_generator.reset_state();
},
ContextMode::Dynamic => {},
}
@@ -227,9 +234,9 @@ impl ChatSession {
}
pub fn reset(&mut self) -> Result<(), Error> {
- let llm =
+ let language_model_generator =
self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- llm.reset_state();
+ language_model_generator.reset_state();
Ok(())
}
}
@@ -241,13 +248,13 @@ impl ChatSession {
context: Option<&Context>,
config: RunConfig,
) -> Result<(Output, Context), Error> {
- self.reconfigure_llm(context)?;
+ self.reconfigure_language_model_generator(context)?;
let output =
self.run_internal(input, config, None::<fn(Output) -> bool>)?;
- let new_context = self.build_context_from_llm()?;
- let llm =
+ let new_context = self.build_context_from_language_model_generator()?;
+ let language_model_generator =
self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- llm.reset_state();
+ language_model_generator.reset_state();
Ok((output, new_context))
}
@@ -278,15 +285,17 @@ impl ChatSession {
.as_language_model()
.ok_or(Error::UnableToLoadConfig)?;
- let llm =
+ let language_model_generator =
self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- let context_length =
- llm.decoding_config.context_length.resolve(language_model_config);
+ let context_length = language_model_generator
+ .decoding_config
+ .context_length
+ .resolve(language_model_config);
if tokens.len() >= context_length {
return Err(Error::ContextLengthExceeded);
}
- let prefix_offset = llm.tokens.len();
+ let prefix_offset = language_model_generator.tokens.len();
let prefix_len_before = prefix_offset.saturating_sub(1);
let eos_tokens: Vec<u64> = language_model_config
@@ -312,7 +321,7 @@ impl ChatSession {
let prefill_start = Instant::now();
let sample_suffix = config.tokens_limit > 0;
- let prefill_result = llm.prefill(
+ let prefill_result = language_model_generator.prefill(
tokens.clone(),
compiled_grammar.as_mut(),
sampling_method,
@@ -321,9 +330,9 @@ impl ChatSession {
)?;
let prefill_tokens = prefill_result.tokens.clone();
let prefill_duration = prefill_start.elapsed().as_secs_f64();
- llm.clear_cache();
+ language_model_generator.clear_cache();
- let prefill_suffix_length = llm
+ let prefill_suffix_length = language_model_generator
.decoding_config
.prefill_step_size
.resolve(language_model_config);
@@ -340,14 +349,27 @@ impl ChatSession {
run_start,
};
- let prefill_finish_reason =
- Self::check_finish_reason(&run_context, llm, &prefill_tokens);
+ let grammar_terminated = config.stop_on_grammar_complete
+ && compiled_grammar
+ .as_ref()
+ .map(|g| g.is_terminated())
+ .unwrap_or(false);
+
+ let prefill_finish_reason = if grammar_terminated {
+ Some(FinishReason::Stop)
+ } else {
+ Self::check_finish_reason(
+ &run_context,
+ language_model_generator,
+ &prefill_tokens,
+ )
+ };
let prefill_output = Self::build_output(
&self.model_metadata,
&self.tokenizer,
&self.output_parser,
&run_context,
- llm,
+ language_model_generator,
&[],
&[],
prefill_finish_reason.clone(),
@@ -368,19 +390,23 @@ impl ChatSession {
}
}
- let can_use_async = llm.decoding_config.generate_suffix_length() == 1
- && !llm.has_attention_layers()
- && compiled_grammar.is_none();
+ let can_use_async =
+ language_model_generator.decoding_config.generate_suffix_length()
+ == 1
+ && !language_model_generator.has_attention_layers()
+ && compiled_grammar.is_none();
let generate_output = if can_use_async {
- let batch_size =
- llm.decoding_config.async_batch_size.resolve(&self.model_path);
+ let batch_size = language_model_generator
+ .decoding_config
+ .async_batch_size
+ .resolve(&self.model_path);
Self::run_async_batch(
&self.model_metadata,
&self.tokenizer,
&self.output_parser,
&run_context,
- llm,
+ language_model_generator,
sampling_method,
&progress,
batch_size,
@@ -391,14 +417,15 @@ impl ChatSession {
&self.tokenizer,
&self.output_parser,
&run_context,
- llm,
+ language_model_generator,
compiled_grammar.as_mut(),
sampling_method,
+ config.stop_on_grammar_complete,
&progress,
)?
};
- llm.clear_cache();
+ language_model_generator.clear_cache();
Ok(generate_output
.clone_with_duration(run_start.elapsed().as_secs_f64()))
}
@@ -408,9 +435,10 @@ impl ChatSession {
tokenizer: &Tokenizer,
output_parser: &OutputParser,
run_context: &RunContext,
- llm: &mut LanguageModelGenerator,
+ language_model_generator: &mut LanguageModelGenerator,
compiled_grammar: Option<&mut CompiledGrammar>,
sampling_method: super::parameter::SamplingMethod,
+ stop_on_grammar_complete: bool,
progress: &Option<F>,
) -> Result<Output, Error>
where
@@ -422,7 +450,7 @@ impl ChatSession {
loop {
let generate_start = Instant::now();
- let generate_result = llm.generate(
+ let generate_result = language_model_generator.generate(
compiled_grammar_mut.as_deref_mut(),
sampling_method,
)?;
@@ -431,14 +459,27 @@ impl ChatSession {
generate_results.push(generate_result);
generate_durations.push(generate_duration);
- let generate_finish_reason =
- Self::check_finish_reason(run_context, llm, &generate_tokens);
+ let grammar_terminated = stop_on_grammar_complete
+ && compiled_grammar_mut
+ .as_ref()
+ .map(|g| g.is_terminated())
+ .unwrap_or(false);
+
+ let generate_finish_reason = if grammar_terminated {
+ Some(FinishReason::Stop)
+ } else {
+ Self::check_finish_reason(
+ run_context,
+ language_model_generator,
+ &generate_tokens,
+ )
+ };
let generate_output = Self::build_output(
model_metadata,
tokenizer,
output_parser,
run_context,
- llm,
+ language_model_generator,
&generate_results,
&generate_durations,
generate_finish_reason.clone(),
@@ -589,15 +630,16 @@ impl ChatSession {
)
}
- fn reconfigure_llm(
+ fn reconfigure_language_model_generator(
&mut self,
context: Option<&Context>,
) -> Result<(), Error> {
- let llm =
+ let language_model_generator =
self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- llm.reset_state();
+ language_model_generator.reset_state();
if let Some(ctx) = context {
- let mut llm_state = llm.context.cache_layers.borrow_mut();
+ let mut llm_state =
+ language_model_generator.context.cache_layers.borrow_mut();
for (ctx_layer, gen_layer) in
ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut())
{
@@ -655,20 +697,25 @@ impl ChatSession {
}
drop(llm_state);
- llm.tokens = ctx.tokens.clone();
+ language_model_generator.tokens = ctx.tokens.clone();
}
Ok(())
}
- fn build_context_from_llm(&self) -> Result<Context, Error> {
- let llm =
+ fn build_context_from_language_model_generator(
+ &self
+ ) -> Result<Context, Error> {
+ let language_model_generator =
self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- let cache_layers =
- llm.context.cache_layers.borrow().clone(&llm.context.mtl_context);
+ let cache_layers = language_model_generator
+ .context
+ .cache_layers
+ .borrow()
+ .clone(&language_model_generator.context.mtl_context);
let context = Context::new(
- llm.tokens.clone(),
+ language_model_generator.tokens.clone(),
cache_layers,
- llm.decoding_config.clone(),
+ language_model_generator.decoding_config.clone(),
);
Ok(context)
}
@@ -677,15 +724,17 @@ impl ChatSession {
impl ChatSession {
fn check_finish_reason(
run_context: &RunContext,
- llm: &LanguageModelGenerator,
+ language_model_generator: &LanguageModelGenerator,
new_tokens: &[u64],
) -> Option<FinishReason> {
let start_idx =
run_context.prefix_len_before + run_context.input_tokens_len;
- let total_new_tokens = llm.tokens.len().saturating_sub(start_idx);
+ let total_new_tokens =
+ language_model_generator.tokens.len().saturating_sub(start_idx);
let has_eos =
new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));
- let context_limit = llm.tokens.len() >= run_context.context_length;
+ let context_limit =
+ language_model_generator.tokens.len() >= run_context.context_length;
if has_eos {
Some(FinishReason::Stop)
@@ -700,13 +749,16 @@ impl ChatSession {
fn decode_generated_tokens(
tokenizer: &Tokenizer,
- llm: &LanguageModelGenerator,
+ language_model_generator: &LanguageModelGenerator,
run_context: &RunContext,
) -> Result<String, Error> {
let start_idx =
run_context.prefix_len_before + run_context.input_tokens_len;
- let generated_tokens: Vec<u32> =
- llm.tokens[start_idx..].iter().map(|&v| v as u32).collect();
+ let generated_tokens: Vec<u32> = language_model_generator.tokens
+ [start_idx..]
+ .iter()
+ .map(|&v| v as u32)
+ .collect();
tokenizer
.decode(&generated_tokens, true)
.map_err(|_| Error::UnableToDecodeText)
@@ -717,16 +769,21 @@ impl ChatSession {
tokenizer: &Tokenizer,
output_parser: &OutputParser,
run_context: &RunContext,
- llm: &LanguageModelGenerator,
+ language_model_generator: &LanguageModelGenerator,
generate_results: &[GenerateResult],
generate_durations: &[f64],
finish_reason: Option<FinishReason>,
) -> Result<Output, Error> {
- let text = Self::decode_generated_tokens(tokenizer, llm, run_context)?;
+ let text = Self::decode_generated_tokens(
+ tokenizer,
+ language_model_generator,
+ run_context,
+ )?;
let parsed = output_parser.parse(text);
let start_idx =
run_context.prefix_len_before + run_context.input_tokens_len;
- let output_tokens = llm.tokens.len().saturating_sub(start_idx);
+ let output_tokens =
+ language_model_generator.tokens.len().saturating_sub(start_idx);
Ok(Output {
text: parsed,
@@ -737,7 +794,9 @@ impl ChatSession {
run_context.prefill_suffix_length,
generate_results.to_vec(),
generate_durations.to_vec(),
- llm.decoding_config.generate_suffix_length(),
+ language_model_generator
+ .decoding_config
+ .generate_suffix_length(),
run_context.run_start.elapsed().as_secs_f64(),
run_context.input_tokens_len,
output_tokens,
diff --git a/crates/uzu/src/session/config/run_config.rs b/crates/uzu/src/session/config/run_config.rs
index ad0a3f9de..ecbe720ac 100644
--- a/crates/uzu/src/session/config/run_config.rs
+++ b/crates/uzu/src/session/config/run_config.rs
@@ -6,6 +6,7 @@ pub struct RunConfig {
pub enable_thinking: bool,
pub sampling_policy: SamplingPolicy,
pub grammar_config: Option<GrammarConfig>,
+ pub stop_on_grammar_complete: bool,
}
impl RunConfig {
@@ -19,6 +20,7 @@ impl RunConfig {
enable_thinking,
sampling_policy,
grammar_config: None,
+ stop_on_grammar_complete: false,
}
}
@@ -53,6 +55,14 @@ impl RunConfig {
self.grammar_config = Some(grammar_config);
self
}
+
+ pub fn stop_on_grammar_complete(
+ mut self,
+ stop_on_grammar_complete: bool,
+ ) -> Self {
+ self.stop_on_grammar_complete = stop_on_grammar_complete;
+ self
+ }
}
impl Default for RunConfig {
diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rs
index c75b0a14d..273d3e4ce 100644
--- a/crates/uzu/src/session/types/error.rs
+++ b/crates/uzu/src/session/types/error.rs
@@ -42,6 +42,8 @@ pub enum Error {
SamplingFailed,
#[error("Grammar error")]
GrammarError,
+ #[error("Token {0} out of grammar vocabulary range (0..{1})")]
+ TokenOutOfGrammarRange(u64, usize),
#[error("Classifier error: {0}")]
Classifier(#[from] ClassifierError),
}
diff --git a/crates/uzu/src/trie.rs b/crates/uzu/src/trie.rs
index 5faf525b8..0fcdbca88 100644
--- a/crates/uzu/src/trie.rs
+++ b/crates/uzu/src/trie.rs
@@ -123,7 +123,14 @@ impl TrieNode {
let mask = if let Some(compiled_grammar) =
compiled_grammar.as_deref_mut()
{
- compiled_grammar.accept_token(next_speculated_token);
+ if compiled_grammar
+ .accept_token(next_speculated_token)
+ .is_err()
+ {
+ cur_node_speculator_weights
+ .remove(&next_speculated_token);
+ continue;
+ }
let next_bitmask = compiled_grammar.next_bitmask().unwrap();
compiled_grammar.rollback(1);
@@ -154,7 +161,9 @@ impl TrieNode {
speculated_suffix.push(next_node_token);
if let Some(compiled_grammar) = compiled_grammar.as_deref_mut()
{
- compiled_grammar.accept_token(next_node_token);
+ if compiled_grammar.accept_token(next_node_token).is_err() {
+ break;
+ }
grammar_accept_count += 1;
}
cur_node = cur_node.get_mut(next_node_token).unwrap();