A high-performance inference engine for AI models
Report number of forward passes in oai server (#710)
uuuvn0a7597471ae1
diff --git a/bindings/python/uzu/__init__.pyi b/bindings/python/uzu/__init__.pyiindex c91c65562..dd63e625e 100644--- a/bindings/python/uzu/__init__.pyi+++ b/bindings/python/uzu/__init__.pyi@@ -353,7 +353,9 @@ class ChatReplyPowerStats:class ChatReplySpeculatorStats:@propertydef tokens_per_forward_pass(self) -> builtins.float: ...- def __new__(cls, tokens_per_forward_pass: builtins.float) -> ChatReplySpeculatorStats: ...+ @property+ def num_decode_forward_passes(self) -> builtins.int: ...+ def __new__(cls, tokens_per_forward_pass: builtins.float, num_decode_forward_passes: builtins.int) -> ChatReplySpeculatorStats: ...@typing.finalclass ChatReplyStats:diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swiftindex a4ffb55ad..7c51f5ee9 100644--- a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift+++ b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift@@ -1253,11 +1253,13 @@ public func FfiConverterTypeChatReplyPowerStats_lower(_ value: ChatReplyPowerStapublic struct ChatReplySpeculatorStats: Equatable, Hashable, Codable {public var tokensPerForwardPass: Double+ public var numDecodeForwardPasses: UInt32// Default memberwise initializers are never public by default, so we// declare one manually.- public init(tokensPerForwardPass: Double) {+ public init(tokensPerForwardPass: Double, numDecodeForwardPasses: UInt32) {self.tokensPerForwardPass = tokensPerForwardPass+ self.numDecodeForwardPasses = numDecodeForwardPasses}@@ -1276,12 +1278,14 @@ public struct FfiConverterTypeChatReplySpeculatorStats: FfiConverterRustBuffer {public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ChatReplySpeculatorStats {returntry ChatReplySpeculatorStats(- tokensPerForwardPass: FfiConverterDouble.read(from: &buf)+ tokensPerForwardPass: FfiConverterDouble.read(from: &buf),+ numDecodeForwardPasses: FfiConverterUInt32.read(from: &buf))}public static func write(_ value: ChatReplySpeculatorStats, into buf: inout [UInt8]) {FfiConverterDouble.write(value.tokensPerForwardPass, into: &buf)+ FfiConverterUInt32.write(value.numDecodeForwardPasses, into: &buf)}}diff --git a/bindings/typescript/src/napi/index.d.mts b/bindings/typescript/src/napi/index.d.mtsindex 94a9dabcd..857eb7914 100644--- a/bindings/typescript/src/napi/index.d.mts+++ b/bindings/typescript/src/napi/index.d.mts@@ -291,7 +291,8 @@ export declare class ChatReplyPowerStats {export declare class ChatReplySpeculatorStats {tokensPerForwardPass: number- constructor(tokensPerForwardPass: number)+ numDecodeForwardPasses: number+ constructor(tokensPerForwardPass: number, numDecodeForwardPasses: number)}export declare class ChatReplyStats {diff --git a/bindings/typescript/src/napi/index.d.ts b/bindings/typescript/src/napi/index.d.tsindex 94a9dabcd..857eb7914 100644--- a/bindings/typescript/src/napi/index.d.ts+++ b/bindings/typescript/src/napi/index.d.ts@@ -291,7 +291,8 @@ export declare class ChatReplyPowerStats {export declare class ChatReplySpeculatorStats {tokensPerForwardPass: number- constructor(tokensPerForwardPass: number)+ numDecodeForwardPasses: number+ constructor(tokensPerForwardPass: number, numDecodeForwardPasses: number)}export declare class ChatReplyStats {diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rsindex 85c02707f..6d0fb2de5 100644--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs@@ -326,7 +326,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let pending = Box::new([encoder.end_encoding().submit()]);- metrics.num_forward_passes += 1;+ metrics.num_prefill_forward_passes += 1;metrics.num_tokens_prefilled += input.len();metrics.num_tokens_proposed += 1;metrics.num_tokens_accepted += 1;@@ -746,7 +746,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {pending.push(encoder.end_encoding().submit());- self.metrics.num_forward_passes += 1;+ self.metrics.num_decode_forward_passes += 1;self.metrics.num_tokens_proposed += input_flat_trie.len();if full_accept {self.metrics.num_tokens_accepted += input_flat_trie.len();diff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rsindex 7bda510fe..50755dc2b 100644--- a/crates/cli/src/server/chat_completions.rs+++ b/crates/cli/src/server/chat_completions.rs@@ -117,6 +117,8 @@ pub struct ChatCompletionUsage {pub prompt_tokens: u32,pub completion_tokens: u32,pub total_tokens: u32,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub spec_verify_ct: Option<u32>,}#[derive(Serialize, Clone)]@@ -485,6 +487,10 @@ fn usage_from_stats(stats: &ChatReplyStats) -> ChatCompletionUsage {prompt_tokens,completion_tokens,total_tokens: prompt_tokens + completion_tokens,+ spec_verify_ct: stats+ .speculator_stats+ .as_ref()+ .map(|speculator_stats| speculator_stats.num_decode_forward_passes),}}diff --git a/crates/nagare/src/chat/token.rs b/crates/nagare/src/chat/token.rsindex 626494736..33a59e7ee 100644--- a/crates/nagare/src/chat/token.rs+++ b/crates/nagare/src/chat/token.rs@@ -379,10 +379,11 @@ impl StreamingState<'_> {last_stat: bool,) -> ChatReplyStats {let speculator_stats = if let Some(metrics) = self.metrics.as_ref()- && metrics.num_forward_passes > 0+ && metrics.num_decode_forward_passes > 0{Some(ChatReplySpeculatorStats {- tokens_per_forward_pass: metrics.num_tokens_accepted as f64 / metrics.num_forward_passes as f64,+ tokens_per_forward_pass: metrics.num_tokens_accepted as f64 / metrics.num_decode_forward_passes as f64,+ num_decode_forward_passes: metrics.num_decode_forward_passes as u32,})} else {Nonediff --git a/crates/shoji/src/traits/backend/chat_token.rs b/crates/shoji/src/traits/backend/chat_token.rsindex 6901c4f9d..5aa3ce9cf 100644--- a/crates/shoji/src/traits/backend/chat_token.rs+++ b/crates/shoji/src/traits/backend/chat_token.rs@@ -14,7 +14,8 @@ pub enum TokenStreamOutput {#[derive(Debug, Clone, Default)]pub struct TokenStreamMetrics {- pub num_forward_passes: usize,+ pub num_prefill_forward_passes: usize,+ pub num_decode_forward_passes: usize,pub num_tokens_prefilled: usize,pub num_tokens_proposed: usize,pub num_tokens_accepted: usize,diff --git a/crates/shoji/src/types/session/chat/reply/stats.rs b/crates/shoji/src/types/session/chat/reply/stats.rsindex 1e07d797e..136716dd7 100644--- a/crates/shoji/src/types/session/chat/reply/stats.rs+++ b/crates/shoji/src/types/session/chat/reply/stats.rs@@ -16,6 +16,7 @@ pub struct ChatReplyPowerStats {#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]pub struct ChatReplySpeculatorStats {pub tokens_per_forward_pass: f64,+ pub num_decode_forward_passes: u32,}#[bindings::export(Structure(Class))]