A high-performance inference engine for AI models
Cli fixes (#702)
Aleksandr Golokozbbe264702aa6
diff --git a/crates/cli/src/interactive/components/application.rs b/crates/cli/src/interactive/components/application.rsindex a701e4382..d4f12fbd3 100644--- a/crates/cli/src/interactive/components/application.rs+++ b/crates/cli/src/interactive/components/application.rs@@ -164,16 +164,12 @@ pub fn Application(state.write().flow = Some(Box::new(ModelRegistriesFlow));return;}- let summary = format!("Model: {}", model.name());state.write().model_state = Some(ModelState {model,download_state: DownloadState::not_downloaded(0),session_state: None,capabilities: ModelCapabilities::default(),});- state.write().history.push(HistoryCellType::CommandResult {- result: summary,- });},Ok(None) => {state.write().history.push(HistoryCellType::CommandResult {diff --git a/crates/cli/src/interactive/components/history_cell.rs b/crates/cli/src/interactive/components/history_cell.rsindex 211e49f9a..bde546e57 100644--- a/crates/cli/src/interactive/components/history_cell.rs+++ b/crates/cli/src/interactive/components/history_cell.rs@@ -352,7 +352,7 @@ fn chat_reply_stats_component(color: subtitle_color,)Text(- content: format!("total power: {power}"),+ content: format!("average power: {power}"),color: subtitle_color,)Text(diff --git a/crates/nagare/src/chat/mod.rs b/crates/nagare/src/chat/mod.rsindex 8a169e037..f59cb20f9 100644--- a/crates/nagare/src/chat/mod.rs+++ b/crates/nagare/src/chat/mod.rs@@ -21,7 +21,8 @@ use shoji::{basic::{CancelToken, ToolCall, ToolDescription, ToolNamespace, Value},model::{Model, ModelSpecialization},session::chat::{- ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyFinishReason, ChatRole,+ ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyFinishReason,+ ChatReplyPowerStats, ChatReplySpeculatorStats, ChatReplyStats, ChatRole,},},};@@ -243,6 +244,7 @@ impl ChatSession {input: Vec<ChatMessage>,config: ChatReplyConfig,cancel_token: CancellationToken,+ completed_stats: &mut Vec<ChatReplyStats>,) -> Option<Vec<ChatReply>> {// prepare all messageslet all_messages = {@@ -268,6 +270,7 @@ impl ChatSession {let mut error_value: Option<serde_json::Value> = None;let mut interrupted = false;let mut generated_tool_call_identifiers: Vec<Option<String>> = Vec::new();+ let mut latest_stats: Option<ChatReplyStats> = None;let mut instance = self.instance.lock().await;let mut stream = match &mut *instance {@@ -282,9 +285,10 @@ impl ChatSession {let finish_reason = backend_output.finish_reason;let output = ChatReply {message: message.clone(),- stats: backend_output.stats.clone(),+ stats: aggregate_stats(completed_stats, &backend_output.stats),finish_reason: finish_reason.clone(),};+ latest_stats = Some(backend_output.stats);// add output to messages listlet mut messages_guard = self.messages.lock().await;@@ -321,13 +325,17 @@ impl ChatSession {self.telemetry.report(TelemetryEvent::ModelInferenceFailed {error,});- } else if let Some(last) = outputs.values().last() {+ } else if let Some(stats) = latest_stats.as_ref() {self.telemetry.report(TelemetryEvent::ModelInferenceFinished {model_id: self.model_id.clone(),- stats: last.stats.clone().into(),+ stats: stats.clone().into(),});}+ if !interrupted && let Some(stats) = latest_stats {+ completed_stats.push(stats);+ }+(!interrupted).then(|| outputs.into_values().collect())}@@ -347,11 +355,13 @@ impl ChatSession {let tool_turn_limit = config.tool_turn_limit.unwrap_or(DEFAULT_TOOL_TURN_LIMIT);let mut tool_turns: u32 = 0;let mut next_input = input;+ let mut completed_stats = Vec::new();loop {// send messagelet turn_input = std::mem::take(&mut next_input);- let Some(turn_replies) = self.send_input(&sender, turn_input, config.clone(), cancel_token.clone()).await+ let Some(turn_replies) =+ self.send_input(&sender, turn_input, config.clone(), cancel_token.clone(), &mut completed_stats).awaitelse {break;};@@ -652,6 +662,100 @@ fn build_message(message}+fn aggregate_stats(+ completed: &[ChatReplyStats],+ current: &ChatReplyStats,+) -> ChatReplyStats {+ let stats = completed.iter().chain(std::iter::once(current)).collect::<Vec<_>>();+ let speculator_stats = aggregate_speculator_stats(&stats);++ ChatReplyStats {+ duration: stats.iter().map(|stats| stats.duration).sum(),+ time_to_first_token: stats.iter().find_map(|stats| stats.time_to_first_token),+ prefill_tokens_per_second: stats.iter().find_map(|stats| stats.prefill_tokens_per_second),+ generate_tokens_per_second: aggregate_generate_rate(&stats),+ tokens_count_input: sum_optional_u32(&stats, |stats| stats.tokens_count_input),+ tokens_count_output: sum_optional_u32(&stats, |stats| stats.tokens_count_output),+ memory_used_bytes: stats.iter().filter_map(|stats| stats.memory_used_bytes).max(),+ speculator_stats,+ power_stats: aggregate_power_stats(&stats),+ }+}++fn aggregate_speculator_stats(stats: &[&ChatReplyStats]) -> Option<ChatReplySpeculatorStats> {+ let (num_decode_tokens, num_decode_forward_passes) = stats+ .iter()+ .filter_map(|stats| stats.speculator_stats.as_ref())+ .fold((0.0, 0_u32), |(tokens, passes), stats| {+ (+ tokens + stats.tokens_per_forward_pass * f64::from(stats.num_decode_forward_passes),+ passes + stats.num_decode_forward_passes,+ )+ });++ (num_decode_forward_passes > 0).then(|| ChatReplySpeculatorStats {+ tokens_per_forward_pass: num_decode_tokens / f64::from(num_decode_forward_passes),+ num_decode_forward_passes,+ })+}++fn aggregate_generate_rate(stats: &[&ChatReplyStats]) -> Option<f64> {+ if let [stats] = stats {+ return stats.generate_tokens_per_second;+ }++ let (token_intervals, duration) = stats.iter().try_fold((0_u64, 0.0), |(token_intervals, duration), stats| {+ let tokens = stats.tokens_count_output?;+ if tokens < 2 {+ return Some((token_intervals, duration));+ }+ let generate_rate = stats.generate_tokens_per_second?;+ if !generate_rate.is_finite() || generate_rate <= 0.0 {+ return None;+ }+ let intervals = u64::from(tokens - 1);+ Some((token_intervals + intervals, duration + intervals as f64 / generate_rate))+ })?;+ (token_intervals > 0 && duration > 0.0).then(|| token_intervals as f64 / duration)+}++fn sum_optional_u32(+ stats: &[&ChatReplyStats],+ value: impl Fn(&ChatReplyStats) -> Option<u32>,+) -> Option<u32> {+ stats.iter().try_fold(0_u32, |sum, stats| sum.checked_add(value(stats)?))+}++fn aggregate_power_stats(stats: &[&ChatReplyStats]) -> Option<ChatReplyPowerStats> {+ let stats =+ stats.iter().filter_map(|stats| stats.power_stats.as_ref().map(|power| (*stats, power))).collect::<Vec<_>>();+ if stats.is_empty() {+ return None;+ }++ Some(ChatReplyPowerStats {+ samples_count: stats.iter().map(|(_, power)| power.samples_count).sum(),+ average_cpu_watts: duration_weighted_power(&stats, |power| power.average_cpu_watts),+ average_gpu_watts: duration_weighted_power(&stats, |power| power.average_gpu_watts),+ average_ane_watts: duration_weighted_power(&stats, |power| power.average_ane_watts),+ average_ram_watts: duration_weighted_power(&stats, |power| power.average_ram_watts),+ average_total_watts: duration_weighted_power(&stats, |power| power.average_total_watts),+ energy_joules: stats.iter().map(|(_, power)| power.energy_joules).sum(),+ })+}++fn duration_weighted_power(+ stats: &[(&ChatReplyStats, &ChatReplyPowerStats)],+ value: impl Fn(&ChatReplyPowerStats) -> f64,+) -> f64 {+ let duration = stats.iter().map(|(stats, _)| stats.duration).sum::<f64>();+ if duration > 0.0 {+ stats.iter().map(|(stats, power)| value(power) * stats.duration).sum::<f64>() / duration+ } else {+ stats.iter().map(|(_, power)| value(power)).sum::<f64>() / stats.len() as f64+ }+}+fn find_tools_definitions(messages: &mut [ChatMessage]) -> Option<&mut Vec<ToolNamespace>> {messages.iter_mut().filter(|msg| msg.role == ChatRole::Developer {}).find_map(|msg| {msg.content.iter_mut().find_map(|content| match content {