Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Speculation rate metric (#244)

Alexander Verbitsky3ec7bb77ae26
diff --git a/crates/cli/src/handlers/run.rs b/crates/cli/src/handlers/run.rs
index 491117fc5..cbcfe95d1 100644
--- a/crates/cli/src/handlers/run.rs
+++ b/crates/cli/src/handlers/run.rs
@@ -23,8 +23,27 @@ fn format_output(output: Output) -> String {
};
let style_stats = Style::new().bold();
- let stats_info = style_stats.apply_to(format!("{:.3}s, {:.3}t/s", stats.total_stats.duration, tokens_per_second,));
+ let acceptance_info = if let Some(generate_stats) = &stats.generate_stats {
+ if generate_stats.speculator_proposed > 0 {
+ let acceptance_rate = generate_stats.speculator_accepted as f64 / generate_stats.speculator_proposed as f64;
+ format!(
+ ", speculation-rate: {:.0}/{:.0} ({:.0}%)",
+ generate_stats.speculator_accepted,
+ generate_stats.speculator_proposed,
+ acceptance_rate * 100.0
+ )
+ } else {
+ String::new()
+ }
+ } else {
+ String::new()
+ };
+
+ let stats_info = style_stats.apply_to(format!(
+ "time: {:.3}s, speed: {:.3}t/s, tokens: {}{}",
+ stats.total_stats.duration, tokens_per_second, stats.total_stats.tokens_count_output, acceptance_info
+ ));
let result = format!("{}\n\n{}", output.text.original, stats_info,);
result
}
@@ -36,6 +55,7 @@ pub fn handle_run(
seed: Option<u64>,
speculator: Option<String>,
mut message: Option<String>,
+ no_thinking: bool,
) {
let mut session = load_session(model_path, prefill_step_size, seed, speculator);
@@ -96,7 +116,7 @@ pub fn handle_run(
let session_output = match session.run(
Input::Text(input.to_string()),
- RunConfig::new(tokens_limit as u64, true, SamplingPolicy::Default, None),
+ RunConfig::new(tokens_limit as u64, !no_thinking, SamplingPolicy::Default, None),
Some(session_progress),
) {
Ok(output) => output,
diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs
index 8fc6ac6c6..e46a42a65 100644
--- a/crates/cli/src/main.rs
+++ b/crates/cli/src/main.rs
@@ -24,6 +24,8 @@ enum Commands {
/// Non-interactive mode: run a single message and exit
#[arg(long, short)]
message: Option<String>,
+ #[arg(long, short)]
+ no_thinking: bool,
},
/// Start a server with the specified model path
Serve {
@@ -53,8 +55,9 @@ fn main() {
seed,
speculator,
message,
+ no_thinking,
}) => {
- handle_run(model_path, 2048, prefill_step_size, seed, speculator, message);
+ handle_run(model_path, 2048, prefill_step_size, seed, speculator, message, no_thinking);
},
Some(Commands::Serve {
model_path,
diff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rs
index b24a6b765..26c9a2146 100644
--- a/crates/cli/src/server/chat_completions.rs
+++ b/crates/cli/src/server/chat_completions.rs
@@ -130,6 +130,8 @@ fn zero_stats() -> Stats {
average_duration: 0.0,
},
run: None,
+ speculator_proposed: 0,
+ speculator_accepted: 0,
},
generate_stats: None,
total_stats: TotalStats {
diff --git a/crates/cli/tests/chat_completions.rs b/crates/cli/tests/chat_completions.rs
index 6a36c049a..82e61036e 100644
--- a/crates/cli/tests/chat_completions.rs
+++ b/crates/cli/tests/chat_completions.rs
@@ -43,6 +43,8 @@ impl RunSession for MockSession {
average_duration: 0.0,
},
run: None,
+ speculator_proposed: 0,
+ speculator_accepted: 0,
},
generate_stats: None,
total_stats: TotalStats {
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 87e31a7e7..dcc57048d 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -396,6 +396,8 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let (accepted_tokens, accepted_token_indices) =
flat_trie.accept(&sampled_tokens, compiled_grammar.as_deref_mut());
+ let speculator_proposed = active_suffix_length.saturating_sub(1);
+ let speculator_accepted = accepted_tokens.len().saturating_sub(1);
self.update_cache_layers(&accepted_token_indices, None, false)?;
@@ -405,6 +407,8 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
Ok(GenerateResult {
tokens: accepted_tokens,
forwardpass_duration: run_time,
+ speculator_proposed,
+ speculator_accepted,
})
}
diff --git a/crates/uzu/src/language_model/result.rs b/crates/uzu/src/language_model/result.rs
index 6bfd7cd2c..6a6db174b 100644
--- a/crates/uzu/src/language_model/result.rs
+++ b/crates/uzu/src/language_model/result.rs
@@ -8,4 +8,6 @@ pub struct PrefillResult {
pub struct GenerateResult {
pub tokens: Vec<u64>,
pub forwardpass_duration: f64,
+ pub speculator_proposed: usize,
+ pub speculator_accepted: usize,
}
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index 586c336e8..0e4eef3f5 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -470,6 +470,8 @@ impl ChatSession {
results.push(GenerateResult {
tokens: vec![token],
forwardpass_duration: duration,
+ speculator_proposed: 0,
+ speculator_accepted: 0,
});
durations.push(duration);
@@ -634,14 +636,15 @@ impl ChatSession {
average_duration: model_run_average_duration,
},
run: None,
+ speculator_proposed: 0,
+ speculator_accepted: 0,
}
};
let generate_stats: Option<StepStats>;
if generate_results.len() != 0 {
let duration = generate_durations.iter().sum::<f64>();
- let tokens_count =
- generate_results.iter().flat_map(|result| result.tokens.clone()).collect::<Vec<u64>>().len();
+ let tokens_count: usize = generate_results.iter().map(|result| result.tokens.len()).sum();
let tokens_per_second: f64 = tokens_count as f64 / duration;
let model_run_count = generate_results.len();
@@ -651,6 +654,9 @@ impl ChatSession {
let run_count = generate_durations.len();
let run_average_duration = generate_durations.iter().sum::<f64>() / run_count as f64;
+ let speculator_proposed: usize = generate_results.iter().map(|result| result.speculator_proposed).sum();
+ let speculator_accepted: usize = generate_results.iter().map(|result| result.speculator_accepted).sum();
+
generate_stats = Some(StepStats {
duration,
suffix_length: generate_suffix_length as u64,
@@ -665,6 +671,8 @@ impl ChatSession {
count: run_count as u64,
average_duration: run_average_duration,
}),
+ speculator_proposed: speculator_proposed as u64,
+ speculator_accepted: speculator_accepted as u64,
});
} else {
generate_stats = None;
diff --git a/crates/uzu/src/session/types/stats.rs b/crates/uzu/src/session/types/stats.rs
index 31368e2e7..cace999a3 100644
--- a/crates/uzu/src/session/types/stats.rs
+++ b/crates/uzu/src/session/types/stats.rs
@@ -15,6 +15,8 @@ pub struct StepStats {
pub processed_tokens_per_second: f64,
pub model_run: RunStats,
pub run: Option<RunStats>,
+ pub speculator_proposed: u64,
+ pub speculator_accepted: u64,
}
#[derive(Debug, Clone, Deserialize, Serialize)]