A high-performance inference engine for AI models
add joules per token stat (#668)
eugeneb2460bb6cac5
diff --git a/apps/playground/Sources/Extensions/ReplyStatsFormat.swift b/apps/playground/Sources/Extensions/ReplyStatsFormat.swiftindex 5bd086c63..b52af55e0 100644--- a/apps/playground/Sources/Extensions/ReplyStatsFormat.swift+++ b/apps/playground/Sources/Extensions/ReplyStatsFormat.swift@@ -24,8 +24,8 @@ enum ReplyStatsFormat {measurement.string(from: Measurement(value: joules, unit: UnitEnergy.joules))}- static func energyPerToken(joules: Double, tokens: Int) -> String {- let value = number.string(from: NSNumber(value: joules / Double(tokens))) ?? placeholder+ static func energyPerToken(_ joulesPerToken: Double) -> String {+ let value = number.string(from: NSNumber(value: joulesPerToken)) ?? placeholderreturn "\(value) J/tok"}diff --git a/apps/playground/Sources/ViewModels/ReplyStats.swift b/apps/playground/Sources/ViewModels/ReplyStats.swiftindex 2802518f1..0812cd096 100644--- a/apps/playground/Sources/ViewModels/ReplyStats.swift+++ b/apps/playground/Sources/ViewModels/ReplyStats.swift@@ -7,7 +7,7 @@ struct ReplyStats: Equatable {let memoryUsedBytes: Int64?let totalPower: Double?let totalEnergy: Double?- let tokensCount: Int?+ let joulesPerToken: Double?let totalTime: Doubleinit(stats: ChatReplyStats) {@@ -16,11 +16,7 @@ struct ReplyStats: Equatable {memoryUsedBytes = stats.memoryUsedBytestotalPower = stats.powerStats?.averageTotalWattstotalEnergy = stats.powerStats?.energyJoules- if let input = stats.tokensCountInput, let output = stats.tokensCountOutput {- tokensCount = Int(input) + Int(output)- } else {- tokensCount = nil- }+ joulesPerToken = stats.joulesPerToken()totalTime = stats.duration}}diff --git a/apps/playground/Sources/Views/ReplyStatsView.swift b/apps/playground/Sources/Views/ReplyStatsView.swiftindex 00876be80..c6c4ebc20 100644--- a/apps/playground/Sources/Views/ReplyStatsView.swift+++ b/apps/playground/Sources/Views/ReplyStatsView.swift@@ -22,8 +22,8 @@ struct ReplyStatsView: View {if let energy = stats.totalEnergy {metricRow(label: "Energy:", value: ReplyStatsFormat.energy(energy))}- if let energy = stats.totalEnergy, let tokens = stats.tokensCount, tokens > 0 {- metricRow(label: "Energy / token:", value: ReplyStatsFormat.energyPerToken(joules: energy, tokens: tokens))+ if let joulesPerToken = stats.joulesPerToken {+ metricRow(label: "Energy / token:", value: ReplyStatsFormat.energyPerToken(joulesPerToken))}metricRow(label: "Total time:", value: String(format: "%.3f s", stats.totalTime))}diff --git a/bindings/python/uzu/__init__.pyi b/bindings/python/uzu/__init__.pyiindex 9f9d16587..ca788b061 100644--- a/bindings/python/uzu/__init__.pyi+++ b/bindings/python/uzu/__init__.pyi@@ -378,6 +378,8 @@ class ChatReplyStats:def power_stats(self) -> typing.Optional[ChatReplyPowerStats]: ...@propertydef tokens_count(self) -> typing.Optional[builtins.int]: ...+ @property+ def joules_per_token(self) -> typing.Optional[builtins.float]: ...def __new__(cls, duration: builtins.float, time_to_first_token: typing.Optional[builtins.float], prefill_tokens_per_second: typing.Optional[builtins.float], generate_tokens_per_second: typing.Optional[builtins.float], tokens_count_input: typing.Optional[builtins.int], tokens_count_output: typing.Optional[builtins.int], memory_used_bytes: typing.Optional[builtins.int], speculator_stats: typing.Optional[ChatReplySpeculatorStats], power_stats: typing.Optional[ChatReplyPowerStats]) -> ChatReplyStats: ...class ChatRole:diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swiftindex f0ae80bb4..a4ffb55ad 100644--- a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift+++ b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift@@ -1327,6 +1327,17 @@ public struct ChatReplyStats: Equatable, Hashable, Codable {}+ /**+ * Energy spent per processed token, counting both input and output tokens.+ */+public func joulesPerToken() -> Double? {+ return try! FfiConverterOptionDouble.lift(try! rustCall() {+ uniffi_shoji_fn_method_chatreplystats_joules_per_token(+ FfiConverterTypeChatReplyStats_lower(self),$0+ )+})+}+public func tokensCount() -> UInt32? {return try! FfiConverterOptionUInt32.lift(try! rustCall() {uniffi_shoji_fn_method_chatreplystats_tokens_count(diff --git a/bindings/typescript/src/napi/index.d.mts b/bindings/typescript/src/napi/index.d.mtsindex db5e78aff..a81b20e34 100644--- a/bindings/typescript/src/napi/index.d.mts+++ b/bindings/typescript/src/napi/index.d.mts@@ -305,6 +305,8 @@ export declare class ChatReplyStats {powerStats?: ChatReplyPowerStatsconstructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)get tokensCount(): number | null+ /** Energy spent per processed token, counting both input and output tokens. */+ get joulesPerToken(): number | null}export declare class ChatRoleAssistant {diff --git a/bindings/typescript/src/napi/index.d.ts b/bindings/typescript/src/napi/index.d.tsindex db5e78aff..a81b20e34 100644--- a/bindings/typescript/src/napi/index.d.ts+++ b/bindings/typescript/src/napi/index.d.ts@@ -305,6 +305,8 @@ export declare class ChatReplyStats {powerStats?: ChatReplyPowerStatsconstructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)get tokensCount(): number | null+ /** Energy spent per processed token, counting both input and output tokens. */+ get joulesPerToken(): number | null}export declare class ChatRoleAssistant {diff --git a/crates/nagare/src/chat/mod.rs b/crates/nagare/src/chat/mod.rsindex 7af075585..7d0c6e878 100644--- a/crates/nagare/src/chat/mod.rs+++ b/crates/nagare/src/chat/mod.rs@@ -324,7 +324,7 @@ impl ChatSession {} else if let Some(last) = outputs.values().last() {self.telemetry.report(TelemetryEvent::ModelInferenceFinished {model_id: self.model_id.clone(),- stats: last.stats.clone(),+ stats: last.stats.clone().into(),});}diff --git a/crates/nagare/src/telemetry/event.rs b/crates/nagare/src/telemetry/event.rsindex b694d1202..5fa9a2642 100644--- a/crates/nagare/src/telemetry/event.rs+++ b/crates/nagare/src/telemetry/event.rs@@ -2,6 +2,24 @@ use serde::{Deserialize, Serialize};use serde_json::Value;use shoji::types::session::chat::ChatReplyStats;+/// Reply stats plus the metrics `ChatReplyStats` exposes as getters, which serde+/// would otherwise leave out of the reported payload.+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+pub struct TelemetryStats {+ #[serde(flatten)]+ pub stats: ChatReplyStats,+ pub joules_per_token: Option<f64>,+}++impl From<ChatReplyStats> for TelemetryStats {+ fn from(stats: ChatReplyStats) -> Self {+ Self {+ joules_per_token: stats.joules_per_token(),+ stats,+ }+ }+}+#[derive(Debug, Clone, Serialize, Deserialize)]#[serde(tag = "event_name", content = "payload", rename_all = "snake_case")]pub enum TelemetryEvent {@@ -16,7 +34,7 @@ pub enum TelemetryEvent {},ModelInferenceFinished {model_id: String,- stats: ChatReplyStats,+ stats: TelemetryStats,},ModelInferenceFailed {error: Value,diff --git a/crates/nagare/src/telemetry/mod.rs b/crates/nagare/src/telemetry/mod.rsindex 18b557577..a73b97e2b 100644--- a/crates/nagare/src/telemetry/mod.rs+++ b/crates/nagare/src/telemetry/mod.rs@@ -8,5 +8,5 @@ mod telemetry;mod worker;pub use context::{TelemetryContext, TelemetryDevice};-pub use event::TelemetryEvent;+pub use event::{TelemetryEvent, TelemetryStats};pub use telemetry::Telemetry;diff --git a/crates/shoji/src/types/session/chat/reply/stats.rs b/crates/shoji/src/types/session/chat/reply/stats.rsindex 164422d4d..1e07d797e 100644--- a/crates/shoji/src/types/session/chat/reply/stats.rs+++ b/crates/shoji/src/types/session/chat/reply/stats.rs@@ -38,4 +38,12 @@ impl ChatReplyStats {pub fn tokens_count(&self) -> Option<u32> {self.tokens_count_input.and_then(|input| self.tokens_count_output.map(|output| input + output))}++ /// Energy spent per processed token, counting both input and output tokens.+ #[bindings::export(Method(Getter))]+ pub fn joules_per_token(&self) -> Option<f64> {+ let energy_joules = self.power_stats.as_ref()?.energy_joules;+ let tokens_count = self.tokens_count()?;+ (tokens_count > 0).then(|| energy_joules / f64::from(tokens_count))+ }}diff --git a/crates/uzu/src/cli/components/history_cell.rs b/crates/uzu/src/cli/components/history_cell.rsindex f238a100e..95e15e266 100644--- a/crates/uzu/src/cli/components/history_cell.rs+++ b/crates/uzu/src/cli/components/history_cell.rs@@ -208,6 +208,10 @@ fn chat_reply_stats_component(.as_ref().map(|power| format!("{:.2} J", power.energy_joules)).unwrap_or_else(|| SYMBOL_LONG_DASH.to_string());+ let energy_per_token = stats+ .joules_per_token()+ .map(|joules_per_token| format!("{joules_per_token:.3} J/tok"))+ .unwrap_or_else(|| SYMBOL_LONG_DASH.to_string());let duration = format!("{:.2} s", stats.duration);element! {@@ -243,6 +247,10 @@ fn chat_reply_stats_component(content: format!("total energy: {energy}"),color: subtitle_color,)+ Text(+ content: format!("energy per token: {energy_per_token}"),+ color: subtitle_color,+ )Text(content: format!("duration: {duration}"),color: subtitle_color,