Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Unify processed_tokens_per_second calculations

Aleksei Savin5c6ee90fffb1
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index ee658d11f..df5e14a4c 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -836,18 +836,7 @@ impl ChatSession {
let tokens_count = prefill_result.tokens.len();
let tokens_per_second = tokens_count as f64 / prefill_duration;
- // TODO: Unify after removing MPS blocks
- let processed_tokens = if model_metadata.quantization.is_some() {
- let result = tokens_count_input + tokens_count;
- result
- } else {
- let number_of_prefill_steps =
- (tokens_count_input as f32 / prefill_suffix_length as f32)
- .ceil() as usize;
- let result = prefill_suffix_length * number_of_prefill_steps
- + tokens_count;
- result
- };
+ let processed_tokens = tokens_count_input + tokens_count;
let processed_tokens_per_second =
processed_tokens as f64 / prefill_duration;