Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Added cli logs (#10)

LuckyIYI1c3851831fd5
diff --git a/cli/src/server/chat_completions.rs b/cli/src/server/chat_completions.rs
index 89e175527..9da6a6b5b 100644
--- a/cli/src/server/chat_completions.rs
+++ b/cli/src/server/chat_completions.rs
@@ -40,10 +40,23 @@ pub fn handle_chat_completions(
) -> Json<ChatCompletionsResponse> {
let id = Uuid::new_v4().to_string();
- let tokens_limit = request.max_completion_tokens.unwrap_or(128) as u64;
+ println!("πŸ“¨ [{}] Incoming chat completion request:", id);
+ println!(" Messages: {} message(s)", request.messages.len());
+ println!(" Max tokens: {}", request.max_completion_tokens.unwrap_or(512));
+ for (i, msg) in request.messages.iter().enumerate() {
+ let content_preview = if msg.content.len() > 10000 {
+ format!("{}...", &msg.content[..10000])
+ } else {
+ msg.content.clone()
+ };
+ println!(" Message {}: {:?} - {}", i + 1, msg.role, content_preview);
+ }
+
+ let tokens_limit = request.max_completion_tokens.unwrap_or(512) as u64;
let input = SessionInput::Messages(request.messages.clone());
let run_config = SessionRunConfig::new(tokens_limit);
+ let start_time = std::time::Instant::now();
let mut session = state.session_wrapper.lock();
let output = session.run(
input,
@@ -52,23 +65,56 @@ pub fn handle_chat_completions(
return true;
}),
);
+ let processing_time = start_time.elapsed();
+
+ println!("output: {:?}", &output);
+
+ let SessionOutput {
+ text,
+ stats,
+ finish_reason,
+ ..
+ } = output;
+
+ let finish_reason_val =
+ finish_reason.unwrap_or(SessionOutputFinishReason::Cancelled);
+
+ println!("πŸ“€ [{}] Sending response:", id);
+ println!(" Response length: {} chars", text.len());
+ println!(" Finish reason: {:?}", finish_reason_val);
+ println!(" Processing time: {:.3}s", processing_time.as_secs_f64());
+ println!(
+ " Tokens/second: {:.1}",
+ stats
+ .generate_stats
+ .as_ref()
+ .map(|s| s.tokens_per_second)
+ .unwrap_or(0.0)
+ );
+
+ print!(" Response preview: ");
+ if text.len() > 3000 {
+ print!("{}", &text[..3000]);
+ println!("...");
+ } else {
+ println!("{}", &text);
+ }
+ println!();
let choice = ChatCompletionsResponseChoice {
index: 0,
message: SessionMessage {
role: SessionMessageRole::Assistant,
- content: output.text,
+ content: text,
},
- finish_reason: output
- .finish_reason
- .unwrap_or(SessionOutputFinishReason::Cancelled),
+ finish_reason: finish_reason_val,
};
let response = ChatCompletionsResponse {
- id: id,
+ id: id.clone(),
model: state.model_name.clone(),
choices: vec![choice],
- stats: output.stats,
+ stats,
};
Json(response)
diff --git a/cli/src/server/main.rs b/cli/src/server/main.rs
index 95286ebcb..356109563 100644
--- a/cli/src/server/main.rs
+++ b/cli/src/server/main.rs
@@ -1,12 +1,35 @@
use std::path::PathBuf;
-use rocket::{Config, config::LogLevel, routes};
+use log::LevelFilter;
+use rocket::{Config, config::LogLevel, log::private as log, routes};
use crate::server::{
SessionState, SessionWrapper, handle_chat_completions, load_session,
};
+struct SilentLogger;
+static SILENT_LOGGER: SilentLogger = SilentLogger;
+impl log::Log for SilentLogger {
+ fn enabled(
+ &self,
+ _: &log::Metadata<'_>,
+ ) -> bool {
+ false
+ }
+ fn log(
+ &self,
+ _record: &log::Record<'_>,
+ ) {
+ }
+ fn flush(&self) {}
+}
+// -------------------------------------------------------------------------------
+
pub async fn run_server(model_path: String) {
+ // Install the silent logger **before** Rocket initializes its own logger.
+ let _ = log::set_logger(&SILENT_LOGGER)
+ .map(|_| log::set_max_level(LevelFilter::Off));
+
let config = Config {
workers: 1,
log_level: LogLevel::Off,
@@ -19,6 +42,17 @@ pub async fn run_server(model_path: String) {
.to_str()
.unwrap()
.to_string();
+
+ println!("πŸš€ Starting server with model: {}", model_name);
+ println!("πŸ“‚ Model path: {}", model_path);
+ println!(
+ "🌐 Server will be available at: http://localhost:{}",
+ config.port
+ );
+ println!(
+ "πŸ“ Endpoints:\n POST /chat/completions - Chat completions API\n"
+ );
+
let session = load_session(model_path);
let state = SessionState {
model_name,