A high-performance inference engine for AI models
Speculation rate metric (#244)
Alexander Verbitsky3ec7bb77ae26
diff --git a/crates/cli/src/handlers/run.rs b/crates/cli/src/handlers/run.rsindex 491117fc5..cbcfe95d1 100644--- a/crates/cli/src/handlers/run.rs+++ b/crates/cli/src/handlers/run.rs@@ -23,8 +23,27 @@ fn format_output(output: Output) -> String {};let style_stats = Style::new().bold();- let stats_info = style_stats.apply_to(format!("{:.3}s, {:.3}t/s", stats.total_stats.duration, tokens_per_second,));+ let acceptance_info = if let Some(generate_stats) = &stats.generate_stats {+ if generate_stats.speculator_proposed > 0 {+ let acceptance_rate = generate_stats.speculator_accepted as f64 / generate_stats.speculator_proposed as f64;+ format!(+ ", speculation-rate: {:.0}/{:.0} ({:.0}%)",+ generate_stats.speculator_accepted,+ generate_stats.speculator_proposed,+ acceptance_rate * 100.0+ )+ } else {+ String::new()+ }+ } else {+ String::new()+ };++ let stats_info = style_stats.apply_to(format!(+ "time: {:.3}s, speed: {:.3}t/s, tokens: {}{}",+ stats.total_stats.duration, tokens_per_second, stats.total_stats.tokens_count_output, acceptance_info+ ));let result = format!("{}\n\n{}", output.text.original, stats_info,);result}@@ -36,6 +55,7 @@ pub fn handle_run(seed: Option<u64>,speculator: Option<String>,mut message: Option<String>,+ no_thinking: bool,) {let mut session = load_session(model_path, prefill_step_size, seed, speculator);@@ -96,7 +116,7 @@ pub fn handle_run(let session_output = match session.run(Input::Text(input.to_string()),- RunConfig::new(tokens_limit as u64, true, SamplingPolicy::Default, None),+ RunConfig::new(tokens_limit as u64, !no_thinking, SamplingPolicy::Default, None),Some(session_progress),) {Ok(output) => output,diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rsindex 8fc6ac6c6..e46a42a65 100644--- a/crates/cli/src/main.rs+++ b/crates/cli/src/main.rs@@ -24,6 +24,8 @@ enum Commands {/// Non-interactive mode: run a single message and exit#[arg(long, short)]message: Option<String>,+ #[arg(long, short)]+ no_thinking: bool,},/// Start a server with the specified model pathServe {@@ -53,8 +55,9 @@ fn main() {seed,speculator,message,+ no_thinking,}) => {- handle_run(model_path, 2048, prefill_step_size, seed, speculator, message);+ handle_run(model_path, 2048, prefill_step_size, seed, speculator, message, no_thinking);},Some(Commands::Serve {model_path,diff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rsindex b24a6b765..26c9a2146 100644--- a/crates/cli/src/server/chat_completions.rs+++ b/crates/cli/src/server/chat_completions.rs@@ -130,6 +130,8 @@ fn zero_stats() -> Stats {average_duration: 0.0,},run: None,+ speculator_proposed: 0,+ speculator_accepted: 0,},generate_stats: None,total_stats: TotalStats {diff --git a/crates/cli/tests/chat_completions.rs b/crates/cli/tests/chat_completions.rsindex 6a36c049a..82e61036e 100644--- a/crates/cli/tests/chat_completions.rs+++ b/crates/cli/tests/chat_completions.rs@@ -43,6 +43,8 @@ impl RunSession for MockSession {average_duration: 0.0,},run: None,+ speculator_proposed: 0,+ speculator_accepted: 0,},generate_stats: None,total_stats: TotalStats {diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 87e31a7e7..dcc57048d 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -396,6 +396,8 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let (accepted_tokens, accepted_token_indices) =flat_trie.accept(&sampled_tokens, compiled_grammar.as_deref_mut());+ let speculator_proposed = active_suffix_length.saturating_sub(1);+ let speculator_accepted = accepted_tokens.len().saturating_sub(1);self.update_cache_layers(&accepted_token_indices, None, false)?;@@ -405,6 +407,8 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {Ok(GenerateResult {tokens: accepted_tokens,forwardpass_duration: run_time,+ speculator_proposed,+ speculator_accepted,})}diff --git a/crates/uzu/src/language_model/result.rs b/crates/uzu/src/language_model/result.rsindex 6bfd7cd2c..6a6db174b 100644--- a/crates/uzu/src/language_model/result.rs+++ b/crates/uzu/src/language_model/result.rs@@ -8,4 +8,6 @@ pub struct PrefillResult {pub struct GenerateResult {pub tokens: Vec<u64>,pub forwardpass_duration: f64,+ pub speculator_proposed: usize,+ pub speculator_accepted: usize,}diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex 586c336e8..0e4eef3f5 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -470,6 +470,8 @@ impl ChatSession {results.push(GenerateResult {tokens: vec![token],forwardpass_duration: duration,+ speculator_proposed: 0,+ speculator_accepted: 0,});durations.push(duration);@@ -634,14 +636,15 @@ impl ChatSession {average_duration: model_run_average_duration,},run: None,+ speculator_proposed: 0,+ speculator_accepted: 0,}};let generate_stats: Option<StepStats>;if generate_results.len() != 0 {let duration = generate_durations.iter().sum::<f64>();- let tokens_count =- generate_results.iter().flat_map(|result| result.tokens.clone()).collect::<Vec<u64>>().len();+ let tokens_count: usize = generate_results.iter().map(|result| result.tokens.len()).sum();let tokens_per_second: f64 = tokens_count as f64 / duration;let model_run_count = generate_results.len();@@ -651,6 +654,9 @@ impl ChatSession {let run_count = generate_durations.len();let run_average_duration = generate_durations.iter().sum::<f64>() / run_count as f64;+ let speculator_proposed: usize = generate_results.iter().map(|result| result.speculator_proposed).sum();+ let speculator_accepted: usize = generate_results.iter().map(|result| result.speculator_accepted).sum();+generate_stats = Some(StepStats {duration,suffix_length: generate_suffix_length as u64,@@ -665,6 +671,8 @@ impl ChatSession {count: run_count as u64,average_duration: run_average_duration,}),+ speculator_proposed: speculator_proposed as u64,+ speculator_accepted: speculator_accepted as u64,});} else {generate_stats = None;diff --git a/crates/uzu/src/session/types/stats.rs b/crates/uzu/src/session/types/stats.rsindex 31368e2e7..cace999a3 100644--- a/crates/uzu/src/session/types/stats.rs+++ b/crates/uzu/src/session/types/stats.rs@@ -15,6 +15,8 @@ pub struct StepStats {pub processed_tokens_per_second: f64,pub model_run: RunStats,pub run: Option<RunStats>,+ pub speculator_proposed: u64,+ pub speculator_accepted: u64,}#[derive(Debug, Clone, Deserialize, Serialize)]