Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add joules per token stat (#668)

eugeneb2460bb6cac5
diff --git a/apps/playground/Sources/Extensions/ReplyStatsFormat.swift b/apps/playground/Sources/Extensions/ReplyStatsFormat.swift
index 5bd086c63..b52af55e0 100644
--- a/apps/playground/Sources/Extensions/ReplyStatsFormat.swift
+++ b/apps/playground/Sources/Extensions/ReplyStatsFormat.swift
@@ -24,8 +24,8 @@ enum ReplyStatsFormat {
measurement.string(from: Measurement(value: joules, unit: UnitEnergy.joules))
}
- static func energyPerToken(joules: Double, tokens: Int) -> String {
- let value = number.string(from: NSNumber(value: joules / Double(tokens))) ?? placeholder
+ static func energyPerToken(_ joulesPerToken: Double) -> String {
+ let value = number.string(from: NSNumber(value: joulesPerToken)) ?? placeholder
return "\(value) J/tok"
}
diff --git a/apps/playground/Sources/ViewModels/ReplyStats.swift b/apps/playground/Sources/ViewModels/ReplyStats.swift
index 2802518f1..0812cd096 100644
--- a/apps/playground/Sources/ViewModels/ReplyStats.swift
+++ b/apps/playground/Sources/ViewModels/ReplyStats.swift
@@ -7,7 +7,7 @@ struct ReplyStats: Equatable {
let memoryUsedBytes: Int64?
let totalPower: Double?
let totalEnergy: Double?
- let tokensCount: Int?
+ let joulesPerToken: Double?
let totalTime: Double
init(stats: ChatReplyStats) {
@@ -16,11 +16,7 @@ struct ReplyStats: Equatable {
memoryUsedBytes = stats.memoryUsedBytes
totalPower = stats.powerStats?.averageTotalWatts
totalEnergy = stats.powerStats?.energyJoules
- if let input = stats.tokensCountInput, let output = stats.tokensCountOutput {
- tokensCount = Int(input) + Int(output)
- } else {
- tokensCount = nil
- }
+ joulesPerToken = stats.joulesPerToken()
totalTime = stats.duration
}
}
diff --git a/apps/playground/Sources/Views/ReplyStatsView.swift b/apps/playground/Sources/Views/ReplyStatsView.swift
index 00876be80..c6c4ebc20 100644
--- a/apps/playground/Sources/Views/ReplyStatsView.swift
+++ b/apps/playground/Sources/Views/ReplyStatsView.swift
@@ -22,8 +22,8 @@ struct ReplyStatsView: View {
if let energy = stats.totalEnergy {
metricRow(label: "Energy:", value: ReplyStatsFormat.energy(energy))
}
- if let energy = stats.totalEnergy, let tokens = stats.tokensCount, tokens > 0 {
- metricRow(label: "Energy / token:", value: ReplyStatsFormat.energyPerToken(joules: energy, tokens: tokens))
+ if let joulesPerToken = stats.joulesPerToken {
+ metricRow(label: "Energy / token:", value: ReplyStatsFormat.energyPerToken(joulesPerToken))
}
metricRow(label: "Total time:", value: String(format: "%.3f s", stats.totalTime))
}
diff --git a/bindings/python/uzu/__init__.pyi b/bindings/python/uzu/__init__.pyi
index 9f9d16587..ca788b061 100644
--- a/bindings/python/uzu/__init__.pyi
+++ b/bindings/python/uzu/__init__.pyi
@@ -378,6 +378,8 @@ class ChatReplyStats:
def power_stats(self) -> typing.Optional[ChatReplyPowerStats]: ...
@property
def tokens_count(self) -> typing.Optional[builtins.int]: ...
+ @property
+ def joules_per_token(self) -> typing.Optional[builtins.float]: ...
def __new__(cls, duration: builtins.float, time_to_first_token: typing.Optional[builtins.float], prefill_tokens_per_second: typing.Optional[builtins.float], generate_tokens_per_second: typing.Optional[builtins.float], tokens_count_input: typing.Optional[builtins.int], tokens_count_output: typing.Optional[builtins.int], memory_used_bytes: typing.Optional[builtins.int], speculator_stats: typing.Optional[ChatReplySpeculatorStats], power_stats: typing.Optional[ChatReplyPowerStats]) -> ChatReplyStats: ...
class ChatRole:
diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
index f0ae80bb4..a4ffb55ad 100644
--- a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
+++ b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
@@ -1327,6 +1327,17 @@ public struct ChatReplyStats: Equatable, Hashable, Codable {
}
+ /**
+ * Energy spent per processed token, counting both input and output tokens.
+ */
+public func joulesPerToken() -> Double? {
+ return try! FfiConverterOptionDouble.lift(try! rustCall() {
+ uniffi_shoji_fn_method_chatreplystats_joules_per_token(
+ FfiConverterTypeChatReplyStats_lower(self),$0
+ )
+})
+}
+
public func tokensCount() -> UInt32? {
return try! FfiConverterOptionUInt32.lift(try! rustCall() {
uniffi_shoji_fn_method_chatreplystats_tokens_count(
diff --git a/bindings/typescript/src/napi/index.d.mts b/bindings/typescript/src/napi/index.d.mts
index db5e78aff..a81b20e34 100644
--- a/bindings/typescript/src/napi/index.d.mts
+++ b/bindings/typescript/src/napi/index.d.mts
@@ -305,6 +305,8 @@ export declare class ChatReplyStats {
powerStats?: ChatReplyPowerStats
constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)
get tokensCount(): number | null
+ /** Energy spent per processed token, counting both input and output tokens. */
+ get joulesPerToken(): number | null
}
export declare class ChatRoleAssistant {
diff --git a/bindings/typescript/src/napi/index.d.ts b/bindings/typescript/src/napi/index.d.ts
index db5e78aff..a81b20e34 100644
--- a/bindings/typescript/src/napi/index.d.ts
+++ b/bindings/typescript/src/napi/index.d.ts
@@ -305,6 +305,8 @@ export declare class ChatReplyStats {
powerStats?: ChatReplyPowerStats
constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)
get tokensCount(): number | null
+ /** Energy spent per processed token, counting both input and output tokens. */
+ get joulesPerToken(): number | null
}
export declare class ChatRoleAssistant {
diff --git a/crates/nagare/src/chat/mod.rs b/crates/nagare/src/chat/mod.rs
index 7af075585..7d0c6e878 100644
--- a/crates/nagare/src/chat/mod.rs
+++ b/crates/nagare/src/chat/mod.rs
@@ -324,7 +324,7 @@ impl ChatSession {
} else if let Some(last) = outputs.values().last() {
self.telemetry.report(TelemetryEvent::ModelInferenceFinished {
model_id: self.model_id.clone(),
- stats: last.stats.clone(),
+ stats: last.stats.clone().into(),
});
}
diff --git a/crates/nagare/src/telemetry/event.rs b/crates/nagare/src/telemetry/event.rs
index b694d1202..5fa9a2642 100644
--- a/crates/nagare/src/telemetry/event.rs
+++ b/crates/nagare/src/telemetry/event.rs
@@ -2,6 +2,24 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use shoji::types::session::chat::ChatReplyStats;
+/// Reply stats plus the metrics `ChatReplyStats` exposes as getters, which serde
+/// would otherwise leave out of the reported payload.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct TelemetryStats {
+ #[serde(flatten)]
+ pub stats: ChatReplyStats,
+ pub joules_per_token: Option<f64>,
+}
+
+impl From<ChatReplyStats> for TelemetryStats {
+ fn from(stats: ChatReplyStats) -> Self {
+ Self {
+ joules_per_token: stats.joules_per_token(),
+ stats,
+ }
+ }
+}
+
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "event_name", content = "payload", rename_all = "snake_case")]
pub enum TelemetryEvent {
@@ -16,7 +34,7 @@ pub enum TelemetryEvent {
},
ModelInferenceFinished {
model_id: String,
- stats: ChatReplyStats,
+ stats: TelemetryStats,
},
ModelInferenceFailed {
error: Value,
diff --git a/crates/nagare/src/telemetry/mod.rs b/crates/nagare/src/telemetry/mod.rs
index 18b557577..a73b97e2b 100644
--- a/crates/nagare/src/telemetry/mod.rs
+++ b/crates/nagare/src/telemetry/mod.rs
@@ -8,5 +8,5 @@ mod telemetry;
mod worker;
pub use context::{TelemetryContext, TelemetryDevice};
-pub use event::TelemetryEvent;
+pub use event::{TelemetryEvent, TelemetryStats};
pub use telemetry::Telemetry;
diff --git a/crates/shoji/src/types/session/chat/reply/stats.rs b/crates/shoji/src/types/session/chat/reply/stats.rs
index 164422d4d..1e07d797e 100644
--- a/crates/shoji/src/types/session/chat/reply/stats.rs
+++ b/crates/shoji/src/types/session/chat/reply/stats.rs
@@ -38,4 +38,12 @@ impl ChatReplyStats {
pub fn tokens_count(&self) -> Option<u32> {
self.tokens_count_input.and_then(|input| self.tokens_count_output.map(|output| input + output))
}
+
+ /// Energy spent per processed token, counting both input and output tokens.
+ #[bindings::export(Method(Getter))]
+ pub fn joules_per_token(&self) -> Option<f64> {
+ let energy_joules = self.power_stats.as_ref()?.energy_joules;
+ let tokens_count = self.tokens_count()?;
+ (tokens_count > 0).then(|| energy_joules / f64::from(tokens_count))
+ }
}
diff --git a/crates/uzu/src/cli/components/history_cell.rs b/crates/uzu/src/cli/components/history_cell.rs
index f238a100e..95e15e266 100644
--- a/crates/uzu/src/cli/components/history_cell.rs
+++ b/crates/uzu/src/cli/components/history_cell.rs
@@ -208,6 +208,10 @@ fn chat_reply_stats_component(
.as_ref()
.map(|power| format!("{:.2} J", power.energy_joules))
.unwrap_or_else(|| SYMBOL_LONG_DASH.to_string());
+ let energy_per_token = stats
+ .joules_per_token()
+ .map(|joules_per_token| format!("{joules_per_token:.3} J/tok"))
+ .unwrap_or_else(|| SYMBOL_LONG_DASH.to_string());
let duration = format!("{:.2} s", stats.duration);
element! {
@@ -243,6 +247,10 @@ fn chat_reply_stats_component(
content: format!("total energy: {energy}"),
color: subtitle_color,
)
+ Text(
+ content: format!("energy per token: {energy_per_token}"),
+ color: subtitle_color,
+ )
Text(
content: format!("duration: {duration}"),
color: subtitle_color,