Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Report number of forward passes in oai server (#710)

uuuvn0a7597471ae1
diff --git a/bindings/python/uzu/__init__.pyi b/bindings/python/uzu/__init__.pyi
index c91c65562..dd63e625e 100644
--- a/bindings/python/uzu/__init__.pyi
+++ b/bindings/python/uzu/__init__.pyi
@@ -353,7 +353,9 @@ class ChatReplyPowerStats:
class ChatReplySpeculatorStats:
@property
def tokens_per_forward_pass(self) -> builtins.float: ...
- def __new__(cls, tokens_per_forward_pass: builtins.float) -> ChatReplySpeculatorStats: ...
+ @property
+ def num_decode_forward_passes(self) -> builtins.int: ...
+ def __new__(cls, tokens_per_forward_pass: builtins.float, num_decode_forward_passes: builtins.int) -> ChatReplySpeculatorStats: ...
@typing.final
class ChatReplyStats:
diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
index a4ffb55ad..7c51f5ee9 100644
--- a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
+++ b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
@@ -1253,11 +1253,13 @@ public func FfiConverterTypeChatReplyPowerStats_lower(_ value: ChatReplyPowerSta
public struct ChatReplySpeculatorStats: Equatable, Hashable, Codable {
public var tokensPerForwardPass: Double
+ public var numDecodeForwardPasses: UInt32
// Default memberwise initializers are never public by default, so we
// declare one manually.
- public init(tokensPerForwardPass: Double) {
+ public init(tokensPerForwardPass: Double, numDecodeForwardPasses: UInt32) {
self.tokensPerForwardPass = tokensPerForwardPass
+ self.numDecodeForwardPasses = numDecodeForwardPasses
}
@@ -1276,12 +1278,14 @@ public struct FfiConverterTypeChatReplySpeculatorStats: FfiConverterRustBuffer {
public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ChatReplySpeculatorStats {
return
try ChatReplySpeculatorStats(
- tokensPerForwardPass: FfiConverterDouble.read(from: &buf)
+ tokensPerForwardPass: FfiConverterDouble.read(from: &buf),
+ numDecodeForwardPasses: FfiConverterUInt32.read(from: &buf)
)
}
public static func write(_ value: ChatReplySpeculatorStats, into buf: inout [UInt8]) {
FfiConverterDouble.write(value.tokensPerForwardPass, into: &buf)
+ FfiConverterUInt32.write(value.numDecodeForwardPasses, into: &buf)
}
}
diff --git a/bindings/typescript/src/napi/index.d.mts b/bindings/typescript/src/napi/index.d.mts
index 94a9dabcd..857eb7914 100644
--- a/bindings/typescript/src/napi/index.d.mts
+++ b/bindings/typescript/src/napi/index.d.mts
@@ -291,7 +291,8 @@ export declare class ChatReplyPowerStats {
export declare class ChatReplySpeculatorStats {
tokensPerForwardPass: number
- constructor(tokensPerForwardPass: number)
+ numDecodeForwardPasses: number
+ constructor(tokensPerForwardPass: number, numDecodeForwardPasses: number)
}
export declare class ChatReplyStats {
diff --git a/bindings/typescript/src/napi/index.d.ts b/bindings/typescript/src/napi/index.d.ts
index 94a9dabcd..857eb7914 100644
--- a/bindings/typescript/src/napi/index.d.ts
+++ b/bindings/typescript/src/napi/index.d.ts
@@ -291,7 +291,8 @@ export declare class ChatReplyPowerStats {
export declare class ChatReplySpeculatorStats {
tokensPerForwardPass: number
- constructor(tokensPerForwardPass: number)
+ numDecodeForwardPasses: number
+ constructor(tokensPerForwardPass: number, numDecodeForwardPasses: number)
}
export declare class ChatReplyStats {
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
index 85c02707f..6d0fb2de5 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
@@ -326,7 +326,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let pending = Box::new([encoder.end_encoding().submit()]);
- metrics.num_forward_passes += 1;
+ metrics.num_prefill_forward_passes += 1;
metrics.num_tokens_prefilled += input.len();
metrics.num_tokens_proposed += 1;
metrics.num_tokens_accepted += 1;
@@ -746,7 +746,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
pending.push(encoder.end_encoding().submit());
- self.metrics.num_forward_passes += 1;
+ self.metrics.num_decode_forward_passes += 1;
self.metrics.num_tokens_proposed += input_flat_trie.len();
if full_accept {
self.metrics.num_tokens_accepted += input_flat_trie.len();
diff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rs
index 7bda510fe..50755dc2b 100644
--- a/crates/cli/src/server/chat_completions.rs
+++ b/crates/cli/src/server/chat_completions.rs
@@ -117,6 +117,8 @@ pub struct ChatCompletionUsage {
pub prompt_tokens: u32,
pub completion_tokens: u32,
pub total_tokens: u32,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub spec_verify_ct: Option<u32>,
}
#[derive(Serialize, Clone)]
@@ -485,6 +487,10 @@ fn usage_from_stats(stats: &ChatReplyStats) -> ChatCompletionUsage {
prompt_tokens,
completion_tokens,
total_tokens: prompt_tokens + completion_tokens,
+ spec_verify_ct: stats
+ .speculator_stats
+ .as_ref()
+ .map(|speculator_stats| speculator_stats.num_decode_forward_passes),
}
}
diff --git a/crates/nagare/src/chat/token.rs b/crates/nagare/src/chat/token.rs
index 626494736..33a59e7ee 100644
--- a/crates/nagare/src/chat/token.rs
+++ b/crates/nagare/src/chat/token.rs
@@ -379,10 +379,11 @@ impl StreamingState<'_> {
last_stat: bool,
) -> ChatReplyStats {
let speculator_stats = if let Some(metrics) = self.metrics.as_ref()
- && metrics.num_forward_passes > 0
+ && metrics.num_decode_forward_passes > 0
{
Some(ChatReplySpeculatorStats {
- tokens_per_forward_pass: metrics.num_tokens_accepted as f64 / metrics.num_forward_passes as f64,
+ tokens_per_forward_pass: metrics.num_tokens_accepted as f64 / metrics.num_decode_forward_passes as f64,
+ num_decode_forward_passes: metrics.num_decode_forward_passes as u32,
})
} else {
None
diff --git a/crates/shoji/src/traits/backend/chat_token.rs b/crates/shoji/src/traits/backend/chat_token.rs
index 6901c4f9d..5aa3ce9cf 100644
--- a/crates/shoji/src/traits/backend/chat_token.rs
+++ b/crates/shoji/src/traits/backend/chat_token.rs
@@ -14,7 +14,8 @@ pub enum TokenStreamOutput {
#[derive(Debug, Clone, Default)]
pub struct TokenStreamMetrics {
- pub num_forward_passes: usize,
+ pub num_prefill_forward_passes: usize,
+ pub num_decode_forward_passes: usize,
pub num_tokens_prefilled: usize,
pub num_tokens_proposed: usize,
pub num_tokens_accepted: usize,
diff --git a/crates/shoji/src/types/session/chat/reply/stats.rs b/crates/shoji/src/types/session/chat/reply/stats.rs
index 1e07d797e..136716dd7 100644
--- a/crates/shoji/src/types/session/chat/reply/stats.rs
+++ b/crates/shoji/src/types/session/chat/reply/stats.rs
@@ -16,6 +16,7 @@ pub struct ChatReplyPowerStats {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
pub struct ChatReplySpeculatorStats {
pub tokens_per_forward_pass: f64,
+ pub num_decode_forward_passes: u32,
}
#[bindings::export(Structure(Class))]