Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Cli fixes (#702)

Aleksandr Golokozbbe264702aa6
diff --git a/crates/cli/src/interactive/components/application.rs b/crates/cli/src/interactive/components/application.rs
index a701e4382..d4f12fbd3 100644
--- a/crates/cli/src/interactive/components/application.rs
+++ b/crates/cli/src/interactive/components/application.rs
@@ -164,16 +164,12 @@ pub fn Application(
state.write().flow = Some(Box::new(ModelRegistriesFlow));
return;
}
- let summary = format!("Model: {}", model.name());
state.write().model_state = Some(ModelState {
model,
download_state: DownloadState::not_downloaded(0),
session_state: None,
capabilities: ModelCapabilities::default(),
});
- state.write().history.push(HistoryCellType::CommandResult {
- result: summary,
- });
},
Ok(None) => {
state.write().history.push(HistoryCellType::CommandResult {
diff --git a/crates/cli/src/interactive/components/history_cell.rs b/crates/cli/src/interactive/components/history_cell.rs
index 211e49f9a..bde546e57 100644
--- a/crates/cli/src/interactive/components/history_cell.rs
+++ b/crates/cli/src/interactive/components/history_cell.rs
@@ -352,7 +352,7 @@ fn chat_reply_stats_component(
color: subtitle_color,
)
Text(
- content: format!("total power: {power}"),
+ content: format!("average power: {power}"),
color: subtitle_color,
)
Text(
diff --git a/crates/nagare/src/chat/mod.rs b/crates/nagare/src/chat/mod.rs
index 8a169e037..f59cb20f9 100644
--- a/crates/nagare/src/chat/mod.rs
+++ b/crates/nagare/src/chat/mod.rs
@@ -21,7 +21,8 @@ use shoji::{
basic::{CancelToken, ToolCall, ToolDescription, ToolNamespace, Value},
model::{Model, ModelSpecialization},
session::chat::{
- ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyFinishReason, ChatRole,
+ ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyFinishReason,
+ ChatReplyPowerStats, ChatReplySpeculatorStats, ChatReplyStats, ChatRole,
},
},
};
@@ -243,6 +244,7 @@ impl ChatSession {
input: Vec<ChatMessage>,
config: ChatReplyConfig,
cancel_token: CancellationToken,
+ completed_stats: &mut Vec<ChatReplyStats>,
) -> Option<Vec<ChatReply>> {
// prepare all messages
let all_messages = {
@@ -268,6 +270,7 @@ impl ChatSession {
let mut error_value: Option<serde_json::Value> = None;
let mut interrupted = false;
let mut generated_tool_call_identifiers: Vec<Option<String>> = Vec::new();
+ let mut latest_stats: Option<ChatReplyStats> = None;
let mut instance = self.instance.lock().await;
let mut stream = match &mut *instance {
@@ -282,9 +285,10 @@ impl ChatSession {
let finish_reason = backend_output.finish_reason;
let output = ChatReply {
message: message.clone(),
- stats: backend_output.stats.clone(),
+ stats: aggregate_stats(completed_stats, &backend_output.stats),
finish_reason: finish_reason.clone(),
};
+ latest_stats = Some(backend_output.stats);
// add output to messages list
let mut messages_guard = self.messages.lock().await;
@@ -321,13 +325,17 @@ impl ChatSession {
self.telemetry.report(TelemetryEvent::ModelInferenceFailed {
error,
});
- } else if let Some(last) = outputs.values().last() {
+ } else if let Some(stats) = latest_stats.as_ref() {
self.telemetry.report(TelemetryEvent::ModelInferenceFinished {
model_id: self.model_id.clone(),
- stats: last.stats.clone().into(),
+ stats: stats.clone().into(),
});
}
+ if !interrupted && let Some(stats) = latest_stats {
+ completed_stats.push(stats);
+ }
+
(!interrupted).then(|| outputs.into_values().collect())
}
@@ -347,11 +355,13 @@ impl ChatSession {
let tool_turn_limit = config.tool_turn_limit.unwrap_or(DEFAULT_TOOL_TURN_LIMIT);
let mut tool_turns: u32 = 0;
let mut next_input = input;
+ let mut completed_stats = Vec::new();
loop {
// send message
let turn_input = std::mem::take(&mut next_input);
- let Some(turn_replies) = self.send_input(&sender, turn_input, config.clone(), cancel_token.clone()).await
+ let Some(turn_replies) =
+ self.send_input(&sender, turn_input, config.clone(), cancel_token.clone(), &mut completed_stats).await
else {
break;
};
@@ -652,6 +662,100 @@ fn build_message(
message
}
+fn aggregate_stats(
+ completed: &[ChatReplyStats],
+ current: &ChatReplyStats,
+) -> ChatReplyStats {
+ let stats = completed.iter().chain(std::iter::once(current)).collect::<Vec<_>>();
+ let speculator_stats = aggregate_speculator_stats(&stats);
+
+ ChatReplyStats {
+ duration: stats.iter().map(|stats| stats.duration).sum(),
+ time_to_first_token: stats.iter().find_map(|stats| stats.time_to_first_token),
+ prefill_tokens_per_second: stats.iter().find_map(|stats| stats.prefill_tokens_per_second),
+ generate_tokens_per_second: aggregate_generate_rate(&stats),
+ tokens_count_input: sum_optional_u32(&stats, |stats| stats.tokens_count_input),
+ tokens_count_output: sum_optional_u32(&stats, |stats| stats.tokens_count_output),
+ memory_used_bytes: stats.iter().filter_map(|stats| stats.memory_used_bytes).max(),
+ speculator_stats,
+ power_stats: aggregate_power_stats(&stats),
+ }
+}
+
+fn aggregate_speculator_stats(stats: &[&ChatReplyStats]) -> Option<ChatReplySpeculatorStats> {
+ let (num_decode_tokens, num_decode_forward_passes) = stats
+ .iter()
+ .filter_map(|stats| stats.speculator_stats.as_ref())
+ .fold((0.0, 0_u32), |(tokens, passes), stats| {
+ (
+ tokens + stats.tokens_per_forward_pass * f64::from(stats.num_decode_forward_passes),
+ passes + stats.num_decode_forward_passes,
+ )
+ });
+
+ (num_decode_forward_passes > 0).then(|| ChatReplySpeculatorStats {
+ tokens_per_forward_pass: num_decode_tokens / f64::from(num_decode_forward_passes),
+ num_decode_forward_passes,
+ })
+}
+
+fn aggregate_generate_rate(stats: &[&ChatReplyStats]) -> Option<f64> {
+ if let [stats] = stats {
+ return stats.generate_tokens_per_second;
+ }
+
+ let (token_intervals, duration) = stats.iter().try_fold((0_u64, 0.0), |(token_intervals, duration), stats| {
+ let tokens = stats.tokens_count_output?;
+ if tokens < 2 {
+ return Some((token_intervals, duration));
+ }
+ let generate_rate = stats.generate_tokens_per_second?;
+ if !generate_rate.is_finite() || generate_rate <= 0.0 {
+ return None;
+ }
+ let intervals = u64::from(tokens - 1);
+ Some((token_intervals + intervals, duration + intervals as f64 / generate_rate))
+ })?;
+ (token_intervals > 0 && duration > 0.0).then(|| token_intervals as f64 / duration)
+}
+
+fn sum_optional_u32(
+ stats: &[&ChatReplyStats],
+ value: impl Fn(&ChatReplyStats) -> Option<u32>,
+) -> Option<u32> {
+ stats.iter().try_fold(0_u32, |sum, stats| sum.checked_add(value(stats)?))
+}
+
+fn aggregate_power_stats(stats: &[&ChatReplyStats]) -> Option<ChatReplyPowerStats> {
+ let stats =
+ stats.iter().filter_map(|stats| stats.power_stats.as_ref().map(|power| (*stats, power))).collect::<Vec<_>>();
+ if stats.is_empty() {
+ return None;
+ }
+
+ Some(ChatReplyPowerStats {
+ samples_count: stats.iter().map(|(_, power)| power.samples_count).sum(),
+ average_cpu_watts: duration_weighted_power(&stats, |power| power.average_cpu_watts),
+ average_gpu_watts: duration_weighted_power(&stats, |power| power.average_gpu_watts),
+ average_ane_watts: duration_weighted_power(&stats, |power| power.average_ane_watts),
+ average_ram_watts: duration_weighted_power(&stats, |power| power.average_ram_watts),
+ average_total_watts: duration_weighted_power(&stats, |power| power.average_total_watts),
+ energy_joules: stats.iter().map(|(_, power)| power.energy_joules).sum(),
+ })
+}
+
+fn duration_weighted_power(
+ stats: &[(&ChatReplyStats, &ChatReplyPowerStats)],
+ value: impl Fn(&ChatReplyPowerStats) -> f64,
+) -> f64 {
+ let duration = stats.iter().map(|(stats, _)| stats.duration).sum::<f64>();
+ if duration > 0.0 {
+ stats.iter().map(|(stats, power)| value(power) * stats.duration).sum::<f64>() / duration
+ } else {
+ stats.iter().map(|(_, power)| value(power)).sum::<f64>() / stats.len() as f64
+ }
+}
+
fn find_tools_definitions(messages: &mut [ChatMessage]) -> Option<&mut Vec<ToolNamespace>> {
messages.iter_mut().filter(|msg| msg.role == ChatRole::Developer {}).find_map(|msg| {
msg.content.iter_mut().find_map(|content| match content {