A high-performance inference engine for AI models
Added cli logs (#10)
LuckyIYI1c3851831fd5
diff --git a/cli/src/server/chat_completions.rs b/cli/src/server/chat_completions.rsindex 89e175527..9da6a6b5b 100644--- a/cli/src/server/chat_completions.rs+++ b/cli/src/server/chat_completions.rs@@ -40,10 +40,23 @@ pub fn handle_chat_completions() -> Json<ChatCompletionsResponse> {let id = Uuid::new_v4().to_string();- let tokens_limit = request.max_completion_tokens.unwrap_or(128) as u64;+ println!("π¨ [{}] Incoming chat completion request:", id);+ println!(" Messages: {} message(s)", request.messages.len());+ println!(" Max tokens: {}", request.max_completion_tokens.unwrap_or(512));+ for (i, msg) in request.messages.iter().enumerate() {+ let content_preview = if msg.content.len() > 10000 {+ format!("{}...", &msg.content[..10000])+ } else {+ msg.content.clone()+ };+ println!(" Message {}: {:?} - {}", i + 1, msg.role, content_preview);+ }++ let tokens_limit = request.max_completion_tokens.unwrap_or(512) as u64;let input = SessionInput::Messages(request.messages.clone());let run_config = SessionRunConfig::new(tokens_limit);+ let start_time = std::time::Instant::now();let mut session = state.session_wrapper.lock();let output = session.run(input,@@ -52,23 +65,56 @@ pub fn handle_chat_completions(return true;}),);+ let processing_time = start_time.elapsed();++ println!("output: {:?}", &output);++ let SessionOutput {+ text,+ stats,+ finish_reason,+ ..+ } = output;++ let finish_reason_val =+ finish_reason.unwrap_or(SessionOutputFinishReason::Cancelled);++ println!("π€ [{}] Sending response:", id);+ println!(" Response length: {} chars", text.len());+ println!(" Finish reason: {:?}", finish_reason_val);+ println!(" Processing time: {:.3}s", processing_time.as_secs_f64());+ println!(+ " Tokens/second: {:.1}",+ stats+ .generate_stats+ .as_ref()+ .map(|s| s.tokens_per_second)+ .unwrap_or(0.0)+ );++ print!(" Response preview: ");+ if text.len() > 3000 {+ print!("{}", &text[..3000]);+ println!("...");+ } else {+ println!("{}", &text);+ }+ println!();let choice = ChatCompletionsResponseChoice {index: 0,message: SessionMessage {role: SessionMessageRole::Assistant,- content: output.text,+ content: text,},- finish_reason: output- .finish_reason- .unwrap_or(SessionOutputFinishReason::Cancelled),+ finish_reason: finish_reason_val,};let response = ChatCompletionsResponse {- id: id,+ id: id.clone(),model: state.model_name.clone(),choices: vec![choice],- stats: output.stats,+ stats,};Json(response)diff --git a/cli/src/server/main.rs b/cli/src/server/main.rsindex 95286ebcb..356109563 100644--- a/cli/src/server/main.rs+++ b/cli/src/server/main.rs@@ -1,12 +1,35 @@use std::path::PathBuf;-use rocket::{Config, config::LogLevel, routes};+use log::LevelFilter;+use rocket::{Config, config::LogLevel, log::private as log, routes};use crate::server::{SessionState, SessionWrapper, handle_chat_completions, load_session,};+struct SilentLogger;+static SILENT_LOGGER: SilentLogger = SilentLogger;+impl log::Log for SilentLogger {+ fn enabled(+ &self,+ _: &log::Metadata<'_>,+ ) -> bool {+ false+ }+ fn log(+ &self,+ _record: &log::Record<'_>,+ ) {+ }+ fn flush(&self) {}+}+// -------------------------------------------------------------------------------+pub async fn run_server(model_path: String) {+ // Install the silent logger **before** Rocket initializes its own logger.+ let _ = log::set_logger(&SILENT_LOGGER)+ .map(|_| log::set_max_level(LevelFilter::Off));+let config = Config {workers: 1,log_level: LogLevel::Off,@@ -19,6 +42,17 @@ pub async fn run_server(model_path: String) {.to_str().unwrap().to_string();++ println!("π Starting server with model: {}", model_name);+ println!("π Model path: {}", model_path);+ println!(+ "π Server will be available at: http://localhost:{}",+ config.port+ );+ println!(+ "π Endpoints:\n POST /chat/completions - Chat completions API\n"+ );+let session = load_session(model_path);let state = SessionState {model_name,