Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

2.4 KB
Raw Blame
use std::time::Instant;

use uzu::types::session::chat::ChatReplyStats;

/// Per-request tracing: one event when the request arrives, one when it ends,
/// correlated by a short tag derived from the request id.
pub struct RequestLog {
    started: Instant,
}

impl RequestLog {
    pub fn start(
        stream: bool,
        messages: usize,
        tools: usize,
        reasoning_effort: Option<&str>,
    ) -> Self {
        let stream = if stream {
            "stream"
        } else {
            "blocking"
        };
        tracing::info!(
            "received: {messages} messages, {}, {tools} tools, reasoning_effort={}",
            stream,
            reasoning_effort.unwrap_or("default"),
        );
        Self {
            started: Instant::now(),
        }
    }

    pub fn finish(
        &self,
        finish_reason: &str,
        stats: Option<&ChatReplyStats>,
        notes: Vec<String>,
    ) {
        let mut parts =
            vec![format!("completed in {:.2}s, finish={finish_reason}", self.started.elapsed().as_secs_f64())];
        if let Some(stats) = stats {
            let cached = stats.tokens_count_input_cached.unwrap_or(0);
            let prefilled = stats.tokens_count_input.unwrap_or(0);
            parts.push(format!(
                "prompt {} tok ({} cached, {} prefilled @ {} tok/s)",
                cached + prefilled,
                cached,
                prefilled,
                stats.prefill_tokens_per_second.map_or("-".to_string(), |tps| format!("{tps:.1}")),
            ));
            if let Some(ttft) = stats.time_to_first_token {
                parts.push(format!("ttft {ttft:.2}s"));
            }
            if let Some(generated) = stats.tokens_count_output {
                parts.push(format!(
                    "decode {generated} tok @ {} tok/s",
                    stats.generate_tokens_per_second.map_or("-".to_string(), |tps| format!("{tps:.1}")),
                ));
            }
            if let Some(speculator) = &stats.speculator_stats {
                parts.push(format!(
                    "spec {:.2} tok/pass ({} passes)",
                    speculator.tokens_per_forward_pass, speculator.num_decode_forward_passes
                ));
            }
        }
        parts.extend(notes);
        tracing::info!("{}", parts.join(", "));
    }

    pub fn fail(
        &self,
        error: &str,
    ) {
        tracing::info!("failed in {:.2}s: {error}", self.started.elapsed().as_secs_f64());
    }
}