A high-performance inference engine for AI models
Cli request logs (#822)
Aleksandr Golokoze88f395bb2e3
diff --git a/Cargo.lock b/Cargo.lockindex fdbb11fd0..8d20bfd21 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -948,6 +948,7 @@ dependencies = ["tokio-stream","toml 1.1.2+spec-1.1.0","tracing",+ "tracing-subscriber","unicode-segmentation","unicode-width 0.2.2","uuid",@@ -2422,10 +2423,8 @@ dependencies = [name = "hanashi"version = "0.5.26"dependencies = [- "fancy-regex 0.17.0","indexmap 2.14.0","libc",- "llm_json","minijinja","minijinja-contrib","openai-harmony",@@ -2436,6 +2435,7 @@ dependencies = ["thiserror 2.0.18","token-stream-parser","tokenizers",+ "tracing",][[package]]diff --git a/crates/legacy/cli/Cargo.toml b/crates/legacy/cli/Cargo.tomlindex cd9a8daf9..273054341 100644--- a/crates/legacy/cli/Cargo.toml+++ b/crates/legacy/cli/Cargo.toml@@ -47,6 +47,7 @@ tokio = { workspace = true, features = ["full"] }tokio-stream.workspace = truetoml.workspace = truetracing.workspace = true+tracing-subscriber.workspace = trueunicode-segmentation.workspace = trueunicode-width.workspace = trueuuid.workspace = truediff --git a/crates/legacy/cli/src/main.rs b/crates/legacy/cli/src/main.rsindex ac57345c1..1630eb4c7 100644--- a/crates/legacy/cli/src/main.rs+++ b/crates/legacy/cli/src/main.rs@@ -9,7 +9,7 @@ mod server;mod storage;#[derive(Parser)]-#[command(name = "cli", bin_name = "cli")]+#[command(name = "cli", bin_name = "cli", version)]struct Cli {/// Identifier of the model to start with (e.g. "alibaba:qwen3.5:0.8b:mirai:mirai-m:4").#[arg(long, value_name = "MODEL")]@@ -48,6 +48,9 @@ enum Commands {/// Reset the session on every request, disabling prefix cache reuse.#[arg(long = "no-prefix-cache", action = clap::ArgAction::SetTrue)]no_prefix_cache: bool,+ /// Print verbose information into the file log+ #[arg(long)]+ verbose_file_log: bool,},Storage {#[arg(long, value_enum, default_value_t = storage::DownloadManagerCliType::default())]@@ -75,7 +78,8 @@ async fn main() -> Result<()> {host,prefix_cache,no_prefix_cache,- }) => server::run_server(model, host, port, prefix_cache || !no_prefix_cache).await?,+ verbose_file_log,+ }) => server::run_server(model, host, port, prefix_cache || !no_prefix_cache, verbose_file_log).await?,Some(Commands::Storage {download_manager,}) => storage::run(download_manager).await?,diff --git a/crates/legacy/cli/src/server/chat_completions.rs b/crates/legacy/cli/src/server/chat_completions.rsindex 70de7d578..04da77cc4 100644--- a/crates/legacy/cli/src/server/chat_completions.rs+++ b/crates/legacy/cli/src/server/chat_completions.rs@@ -1,10 +1,4 @@-use std::{- collections::HashMap,- pin::Pin,- str::FromStr,- sync::Arc,- time::{SystemTime, UNIX_EPOCH},-};+use std::{collections::HashMap, pin::Pin, str::FromStr, sync::Arc};use rocket::{Request, State,@@ -21,7 +15,7 @@ use rocket::{use serde::{Deserialize, Serialize};use tokio::sync::{Mutex, mpsc};use tokio_stream::wrappers::UnboundedReceiverStream;-use uuid::Uuid;+use tracing::Instrument;use uzu::{session::chat::{ChatSession, ChatSessionStream, ChatSessionStreamChunk, UNPARSED_ARGUMENTS_KEY},types::{@@ -41,6 +35,7 @@ use crate::{coerce_tool_call, insert_tools_message, oai_tool_call, parse_scalar_text, reply_tool_calls, to_tool_call,tool_call_result_block, withhold_stream_text,},+ request_info::RequestInfo,request_log::RequestLog,},};@@ -242,10 +237,6 @@ impl<'r> Responder<'r, 'r> for ChatCompletionResult {}}-fn now_unix() -> i64 {- SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0)-}-fn to_chat_messages(messages: &[OaiMessage]) -> Vec<ChatMessage> {messages.iter()@@ -1150,9 +1141,12 @@ async fn run_stream(#[allow(private_interfaces)]#[post("/chat/completions", format = "json", data = "<body>")]+#[tracing::instrument(skip_all, parent = &request_info.span)]pub async fn handle_chat_completions(body: Data<'_>,state: &State<ServerState>,+ request_info: &RequestInfo,+ content_type: &ContentType,) -> ChatCompletionResult {let body = match body.open(ByteUnit::Mebibyte(64)).into_string().await {Ok(body) if body.is_complete() => body.into_inner(),@@ -1163,10 +1157,17 @@ pub async fn handle_chat_completions(return invalid_request_response("body", "invalid_body", format!("failed to read request body: {error}"));},};+ let log_body = if content_type.is_json() {+ serde_json::from_str::<serde_json::Value>(&body).map(|value| value.to_string()).unwrap_or_else(|_| body.clone())+ } else {+ body.clone()+ };+ tracing::debug!("--> {} {} body={}", request_info.method, request_info.uri, log_body);+let request = match serde_json::from_str::<ChatCompletionRequest>(&body) {Ok(request) => request,Err(error) => {- RequestLog::rejected(&format!("failed to parse chat completion request: {error}"));+ tracing::info!("rejected: {error}");return invalid_request_response("body","invalid_request",@@ -1174,12 +1175,11 @@ pub async fn handle_chat_completions();},};- let id = format!("chatcmpl-{}", Uuid::new_v4().simple());- let created = now_unix();+let model = state.model_name.clone();+ let created = request_info.created_at;let is_stream = request.stream.unwrap_or(false);let log = RequestLog::start(- &id,is_stream,request.messages.len(),request.tools.as_ref().map_or(0, Vec::len),@@ -1212,38 +1212,34 @@ pub async fn handle_chat_completions(// the types when replies cross back into the OpenAI wire format.let parameter_types = ToolParameterTypes::from_tools(request.tools.as_deref());+ let id = request_info.id.as_str().to_owned();if is_stream {let session = Arc::clone(&state.session);let (sender, receiver) = mpsc::unbounded_channel::<Event>();- rocket::tokio::spawn(run_stream(- session,- messages,- config,- id,- model,- created,- state.prefix_cache,- parameter_types,- sender,- log,- ));+ tokio::spawn(+ run_stream(session, messages, config, id, model, created, state.prefix_cache, parameter_types, sender, log)+ .in_current_span(),+ );let body: Pin<Box<dyn Stream<Item = Event> + Send>> = Box::pin(UnboundedReceiverStream::new(receiver));ChatCompletionResult::Stream(EventStream::from(body))} else {let session = Arc::clone(&state.session);let (sender, receiver) = mpsc::unbounded_channel::<Vec<u8>>();- rocket::tokio::spawn(run_blocking(- session,- messages,- config,- id,- model,- created,- state.prefix_cache,- parameter_types,- sender,- log,- ));+ tokio::spawn(+ run_blocking(+ session,+ messages,+ config,+ id,+ model,+ created,+ state.prefix_cache,+ parameter_types,+ sender,+ log,+ )+ .in_current_span(),+ );let body: Pin<Box<dyn Stream<Item = Vec<u8>> + Send>> = Box::pin(UnboundedReceiverStream::new(receiver));// Hand the body to Rocket immediately so it can write keepalives and// drop the receiver when the client disconnects. Any backend errordiff --git a/crates/legacy/cli/src/server/log/file_logger.rs b/crates/legacy/cli/src/server/log/file_logger.rsnew file mode 100644index 000000000..634558e47--- /dev/null+++ b/crates/legacy/cli/src/server/log/file_logger.rs@@ -0,0 +1,79 @@+use std::{+ fs::{self},+ io::{self},+ path::{Path, PathBuf},+};++use chrono::{Local, NaiveDateTime, TimeDelta};+use tracing_subscriber::fmt::writer::MakeWriter;+use uuid::Uuid;++use crate::server::log::log_file::LogFile;++const LOG_FILE_DATE_FORMAT: &str = "%Y-%m-%d-%H-%M-%S";+const LOG_FILE_EXTENSION: &str = "log";+const LOG_FILE_RETENTION: TimeDelta = TimeDelta::weeks(1);++#[derive(Clone)]+pub struct FileLogger {+ file: LogFile,+}++impl FileLogger {+ pub fn new(logs_dir_path: PathBuf) -> io::Result<Self> {+ fs::create_dir_all(&logs_dir_path)?;+ Self::remove_expired_log_files(&logs_dir_path)?;++ let date = Local::now().format(LOG_FILE_DATE_FORMAT);+ let session_id = Uuid::new_v4().simple().to_string();+ let file_name = format!("{}-{}.{}", date, session_id, LOG_FILE_EXTENSION);+ let file = LogFile::new(logs_dir_path.join(file_name))?;+ Ok(Self {+ file,+ })+ }++ pub fn file_path(&self) -> &PathBuf {+ self.file.path()+ }++ fn remove_expired_log_files(logs_dir_path: &Path) -> io::Result<()> {+ let cutoff = Local::now().naive_local() - LOG_FILE_RETENTION;++ for entry in fs::read_dir(logs_dir_path)? {+ let entry = entry?;+ if !entry.file_type()?.is_file() {+ continue;+ }++ let path = entry.path();+ if path.extension().is_none_or(|extension| extension != LOG_FILE_EXTENSION) {+ continue;+ }++ let Some(file_stem) = path.file_stem().and_then(|file_stem| file_stem.to_str()) else {+ continue;+ };+ let Some((date, _session_id)) = file_stem.rsplit_once('-') else {+ continue;+ };+ let Ok(created_at) = NaiveDateTime::parse_from_str(date, LOG_FILE_DATE_FORMAT) else {+ continue;+ };++ if created_at < cutoff {+ fs::remove_file(path)?;+ }+ }++ Ok(())+ }+}++impl<'a> MakeWriter<'a> for FileLogger {+ type Writer = <LogFile as MakeWriter<'a>>::Writer;++ fn make_writer(&'a self) -> Self::Writer {+ self.file.make_writer()+ }+}diff --git a/crates/legacy/cli/src/server/log/log_file.rs b/crates/legacy/cli/src/server/log/log_file.rsnew file mode 100644index 000000000..87984ad0a--- /dev/null+++ b/crates/legacy/cli/src/server/log/log_file.rs@@ -0,0 +1,43 @@+use std::{+ fs::{File, OpenOptions},+ io,+ path::{Path, PathBuf},+ sync::{Arc, Mutex},+};++use tracing_subscriber::fmt::writer::{MakeWriter, MutexGuardWriter};++#[derive(Clone)]+pub struct LogFile {+ file: Arc<Mutex<File>>,+ path: PathBuf,+}++impl LogFile {+ pub fn new(path: impl AsRef<Path>) -> io::Result<Self> {+ let path = path.as_ref();+ if let Some(parent) = path.parent()+ && !parent.as_os_str().is_empty()+ {+ std::fs::create_dir_all(parent)?;+ }++ let file = OpenOptions::new().create(true).append(true).open(path)?;+ Ok(Self {+ file: Arc::new(Mutex::new(file)),+ path: path.into(),+ })+ }++ pub fn path(&self) -> &PathBuf {+ &self.path+ }+}++impl<'a> MakeWriter<'a> for LogFile {+ type Writer = MutexGuardWriter<'a, File>;++ fn make_writer(&'a self) -> Self::Writer {+ self.file.make_writer()+ }+}diff --git a/crates/legacy/cli/src/server/log/mod.rs b/crates/legacy/cli/src/server/log/mod.rsnew file mode 100644index 000000000..c4dcc0643--- /dev/null+++ b/crates/legacy/cli/src/server/log/mod.rs@@ -0,0 +1,29 @@+mod file_logger;+mod log_file;+mod request_trace;++use std::io;++pub use file_logger::FileLogger;+use tracing_subscriber::{EnvFilter, fmt::writer::MakeWriterExt, layer::SubscriberExt, util::SubscriberInitExt};++use self::request_trace::{RequestContextLayer, RequestFormatter};++pub fn init(+ stdout_max_level: tracing::Level,+ file_logger: Option<FileLogger>,+ file_max_level: tracing::Level,+) -> anyhow::Result<()> {+ let subscriber = tracing_subscriber::registry()+ .with(EnvFilter::new("off,cli=debug,hanashi=debug,nagare::chat::token=debug"))+ .with(RequestContextLayer);+ let format = tracing_subscriber::fmt::layer().event_format(RequestFormatter).with_ansi(false);++ let console = io::stdout.with_max_level(stdout_max_level);+ let result = if let Some(file_logger) = file_logger {+ subscriber.with(format.with_writer(console.and(file_logger.with_max_level(file_max_level)))).try_init()+ } else {+ subscriber.with(format.with_writer(console)).try_init()+ };+ result.map_err(|error| anyhow::anyhow!("Failed to initialize server tracing: {error}"))+}diff --git a/crates/legacy/cli/src/server/log/request_trace.rs b/crates/legacy/cli/src/server/log/request_trace.rsnew file mode 100644index 000000000..26ab6fad9--- /dev/null+++ b/crates/legacy/cli/src/server/log/request_trace.rs@@ -0,0 +1,88 @@+use std::fmt;++use tracing::{+ Event, Subscriber,+ field::{Field, Visit},+ span::{Attributes, Id},+};+use tracing_subscriber::{+ Layer,+ fmt::{FmtContext, FormatEvent, FormatFields, format::Writer},+ layer::Context,+ registry::LookupSpan,+};++#[derive(Default)]+struct RequestId(Option<String>);++impl Visit for RequestId {+ fn record_str(+ &mut self,+ field: &Field,+ value: &str,+ ) {+ if field.name() == "request_id" {+ self.0 = Some(value.to_owned());+ }+ }++ fn record_debug(+ &mut self,+ field: &Field,+ value: &dyn fmt::Debug,+ ) {+ if field.name() == "request_id" {+ self.0 = Some(format!("{value:?}"));+ }+ }+}++/// Store the request ID separately from the span's human-readable fields.+pub(super) struct RequestContextLayer;++impl<S> Layer<S> for RequestContextLayer+where+ S: Subscriber + for<'a> LookupSpan<'a>,+{+ fn on_new_span(+ &self,+ attrs: &Attributes<'_>,+ id: &Id,+ ctx: Context<'_, S>,+ ) {+ let mut request_id = RequestId::default();+ attrs.record(&mut request_id);+ if request_id.0.is_some()+ && let Some(span) = ctx.span(id)+ {+ span.extensions_mut().insert(request_id);+ }+ }+}++pub(super) struct RequestFormatter;++impl<S, N> FormatEvent<S, N> for RequestFormatter+where+ S: Subscriber + for<'a> LookupSpan<'a>,+ N: for<'a> FormatFields<'a> + 'static,+{+ fn format_event(+ &self,+ ctx: &FmtContext<'_, S, N>,+ mut writer: Writer<'_>,+ event: &Event<'_>,+ ) -> fmt::Result {+ if let Some(scope) = ctx.event_scope() {+ for span in scope {+ let extensions = span.extensions();+ if let Some(RequestId(Some(id))) = extensions.get::<RequestId>() {+ write!(writer, "[{id}] ")?;+ break;+ }+ }+ }+ ctx.field_format().format_fields(writer.by_ref(), event)?;+ writeln!(writer)+ }+}diff --git a/crates/legacy/cli/src/server/mod.rs b/crates/legacy/cli/src/server/mod.rsindex 8a09adf69..ed97ac331 100644--- a/crates/legacy/cli/src/server/mod.rs+++ b/crates/legacy/cli/src/server/mod.rs@@ -1,3 +1,6 @@+mod log;+mod request_info;+pub mod chat_completions;pub mod chat_tool_calls;pub mod models;diff --git a/crates/legacy/cli/src/server/models.rs b/crates/legacy/cli/src/server/models.rsindex 1db75c6b3..74212a2db 100644--- a/crates/legacy/cli/src/server/models.rs+++ b/crates/legacy/cli/src/server/models.rs@@ -1,7 +1,7 @@use rocket::{State, get, serde::json::Json};use serde::{Deserialize, Serialize};-use crate::server::ServerState;+use crate::server::{ServerState, request_info::RequestInfo};#[derive(Serialize, Deserialize)]pub struct Model {@@ -18,7 +18,11 @@ pub struct ModelsResponse {}#[get("/models")]-pub fn handle_models(state: &State<ServerState>) -> Json<ModelsResponse> {+#[tracing::instrument(skip_all, parent = &request_info.span)]+pub fn handle_models(+ state: &State<ServerState>,+ request_info: &RequestInfo,+) -> Json<ModelsResponse> {Json(ModelsResponse {object: "list".to_string(),data: vec![Model {diff --git a/crates/legacy/cli/src/server/request_info.rs b/crates/legacy/cli/src/server/request_info.rsnew file mode 100644index 000000000..f4e36209d--- /dev/null+++ b/crates/legacy/cli/src/server/request_info.rs@@ -0,0 +1,48 @@+use std::{+ convert::Infallible,+ time::{SystemTime, UNIX_EPOCH},+};++use rocket::{+ Request,+ http::Method,+ request::{FromRequest, Outcome},+};+use uuid::Uuid;++pub struct RequestInfo {+ pub method: Method,+ pub uri: String,+ pub id: String,+ pub created_at: i64,+ pub span: tracing::Span,+}++impl RequestInfo {+ pub fn new(+ method: Method,+ uri: String,+ ) -> Self {+ let id = Uuid::new_v4().simple().to_string();+ let span = tracing::info_span!(parent: None, "request", request_id = &id[..8]);+ Self {+ method,+ uri,+ id,+ span,+ created_at: SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .map(|duration| duration.as_secs() as i64)+ .unwrap_or(0),+ }+ }+}++#[rocket::async_trait]+impl<'a> FromRequest<'a> for &'a RequestInfo {+ type Error = Infallible;++ async fn from_request(request: &'a Request<'_>) -> Outcome<Self, Self::Error> {+ Outcome::Success(request.local_cache(|| RequestInfo::new(request.method(), request.uri().to_string())))+ }+}diff --git a/crates/legacy/cli/src/server/request_log.rs b/crates/legacy/cli/src/server/request_log.rsindex e1aa7fac0..322d5173f 100644--- a/crates/legacy/cli/src/server/request_log.rs+++ b/crates/legacy/cli/src/server/request_log.rs@@ -1,51 +1,35 @@use std::time::Instant;-use uuid::Uuid;use uzu::types::session::chat::ChatReplyStats;-/// Per-request console logging: one line when the request arrives, one when it-/// ends, correlated by a short tag derived from the request id. Each line is a-/// single `println!`, which holds the stdout lock for the whole write, so lines-/// from concurrent requests never interleave.+/// Per-request tracing: one event when the request arrives, one when it ends,+/// correlated by a short tag derived from the request id.pub struct RequestLog {- tag: String,started: Instant,}-fn short_tag(id: &str) -> String {- let id = id.strip_prefix("chatcmpl-").unwrap_or(id);- id.chars().take(8).collect()-}-impl RequestLog {pub fn start(- id: &str,stream: bool,messages: usize,tools: usize,reasoning_effort: Option<&str>,) -> Self {- let tag = short_tag(id);- println!(- "[req {tag}] received: {messages} messages, {}, {tools} tools, reasoning_effort={}",- if stream {- "stream"- } else {- "blocking"- },+ let stream = if stream {+ "stream"+ } else {+ "blocking"+ };+ tracing::info!(+ "received: {messages} messages, {}, {tools} tools, reasoning_effort={}",+ stream,reasoning_effort.unwrap_or("default"),);Self {- tag,started: Instant::now(),}}- /// For requests rejected before an id could be assigned to them.- pub fn rejected(error: &str) {- println!("[req {}] rejected: {error}", short_tag(&Uuid::new_v4().simple().to_string()));- }-pub fn finish(&self,finish_reason: &str,@@ -81,13 +65,13 @@ impl RequestLog {}}parts.extend(notes);- println!("[req {}] {}", self.tag, parts.join(", "));+ tracing::info!("{}", parts.join(", "));}pub fn fail(&self,error: &str,) {- println!("[req {}] failed in {:.2}s: {error}", self.tag, self.started.elapsed().as_secs_f64());+ tracing::info!("failed in {:.2}s: {error}", self.started.elapsed().as_secs_f64());}}diff --git a/crates/legacy/cli/src/server/runner.rs b/crates/legacy/cli/src/server/runner.rsindex afc0755ff..185448289 100644--- a/crates/legacy/cli/src/server/runner.rs+++ b/crates/legacy/cli/src/server/runner.rs@@ -1,9 +1,19 @@-use std::{net::IpAddr, sync::Arc, time::Duration};+use std::{io, net::IpAddr, path::PathBuf, pin::Pin, sync::Arc, task::Poll, time::Duration};use anyhow::{Context, Result, bail};use indicatif::{ProgressBar, ProgressStyle};-use rocket::{Config, config::LogLevel, routes};-use tokio::sync::Mutex;+use rocket::{+ Config, Request, Response,+ config::LogLevel,+ data::Data,+ fairing::{Fairing, Info, Kind},+ response::Body,+ routes,+};+use tokio::{+ io::{AsyncRead, ReadBuf},+ sync::Mutex,+};use uzu::{engine::{Engine, EngineConfig},types::session::chat::ChatConfig,@@ -11,15 +21,156 @@ use uzu::{use crate::{common::thinking::ThinkingSupport,- server::{ServerState, handle_chat_completions, handle_models},+ server::{+ ServerState, handle_chat_completions, handle_models,+ log::{self, FileLogger},+ request_info::RequestInfo,+ },};+pub struct ResponseBodyLogger<'r> {+ body: Body<'r>,+ prefix: String,+ bytes: Vec<u8>,+ is_json: bool,+ logged: bool,+ span: tracing::Span,+}++impl<'r> ResponseBodyLogger<'r> {+ pub fn new(+ body: Body<'r>,+ prefix: String,+ is_json: bool,+ span: tracing::Span,+ ) -> Self {+ Self {+ body,+ prefix,+ bytes: Vec::new(),+ is_json,+ logged: false,+ span,+ }+ }++ fn log(+ &mut self,+ suffix: &str,+ ) {+ if self.logged {+ return;+ }+ self.logged = true;++ let body = if self.is_json {+ serde_json::from_slice::<serde_json::Value>(&self.bytes)+ .map(|value| value.to_string())+ .unwrap_or_else(|_| format!("{:?}", String::from_utf8_lossy(&self.bytes)))+ } else {+ format!("{:?}", String::from_utf8_lossy(&self.bytes))+ };+ self.span.in_scope(|| tracing::debug!("{} body={}{}\n", self.prefix, body, suffix));+ }+}++impl AsyncRead for ResponseBodyLogger<'_> {+ fn poll_read(+ self: Pin<&mut Self>,+ cx: &mut std::task::Context<'_>,+ buffer: &mut ReadBuf<'_>,+ ) -> Poll<io::Result<()>> {+ let this = self.get_mut();+ let span = this.span.clone();+ let _entered = span.enter();+ let filled_before = buffer.filled().len();++ match Pin::new(&mut this.body).poll_read(cx, buffer) {+ Poll::Ready(Ok(())) => {+ let filled_after = buffer.filled().len();+ if filled_after == filled_before {+ this.log("");+ } else {+ this.bytes.extend_from_slice(&buffer.filled()[filled_before..filled_after]);+ }+ Poll::Ready(Ok(()))+ },+ Poll::Ready(Err(error)) => {+ this.log(" [read error]");+ Poll::Ready(Err(error))+ },+ Poll::Pending => Poll::Pending,+ }+ }+}++impl Drop for ResponseBodyLogger<'_> {+ fn drop(&mut self) {+ self.log(" [incomplete]");+ }+}++struct RequestLoggingFairing;++#[rocket::async_trait]+impl Fairing for RequestLoggingFairing {+ fn info(&self) -> Info {+ Info {+ name: "Request and response logger",+ kind: Kind::Request | Kind::Response,+ }+ }++ async fn on_request(+ &self,+ request: &mut Request<'_>,+ _data: &mut Data<'_>,+ ) {+ let _ = request.local_cache(|| RequestInfo::new(request.method(), request.uri().to_string()));+ }++ async fn on_response<'r>(+ &self,+ request: &'r Request<'_>,+ response: &mut Response<'r>,+ ) {+ if matches!(request.uri().path().as_str(), "/logs") {+ return;+ }++ let request_info = request.local_cache(|| RequestInfo::new(request.method(), request.uri().to_string()));+ let _entered = request_info.span.enter();+ let prefix = format!("<-- {} {}", response.status(), request.uri());+ if response.body().is_none() {+ tracing::debug!("{prefix} body=<empty>\n");+ } else {+ let is_json = response.content_type().is_some_and(|content_type| content_type.is_json());+ let body = response.body_mut().take();+ response.set_streamed_body(ResponseBodyLogger::new(body, prefix, is_json, request_info.span.clone()));+ }+ }+}+pub async fn run_server(model: String,host: String,port: u16,prefix_cache: bool,+ verbose_file_logs: bool,) -> Result<()> {+ let file_logger = if verbose_file_logs {+ let cache_dir = std::env::var_os("XDG_CACHE_HOME")+ .map(PathBuf::from)+ .filter(|path| path.is_absolute())+ .or_else(|| dirs::home_dir().map(|path| path.join(".cache")))+ .context("Failed to resolve cache directory")?;+ let logs_dir_path = cache_dir.join("mirai").join("server").join("logs");+ Some(FileLogger::new(logs_dir_path)?)+ } else {+ None+ };+ log::init(tracing::Level::INFO, file_logger.clone(), tracing::Level::DEBUG)?;+let engine_config = EngineConfig::default().with_application_identifier("com.trymirai.cli".to_string());let engine = Engine::new(engine_config).await.context("Failed to create engine")?;@@ -57,22 +208,33 @@ pub async fn run_server(..Config::default()};- println!("🚀 OpenAI-compatible server for model: {model_name}");- println!("🌐 Available at: http://{host}:{port}");- println!(- "🗄️ Prefix cache: {}",- if prefix_cache {- "enabled"- } else {- "disabled"- }+ let prefix_cache_enabled = if prefix_cache {+ "enabled"+ } else {+ "disabled"+ };+ if let Some(ref file_logger) = file_logger {+ tracing::info!("🗃️ Logs will be written to file: {}", file_logger.file_path().display())+ }+ tracing::info!(+ concat!(+ "🚀 OpenAI-compatible server v{version} for model: {model_name}\n",+ "🌐 Available at: http://{host}:{port}\n",+ "🗄️ Prefix cache: {prefix_cache_enabled}\n",+ "📝 Endpoints:\n",+ " POST /v1/chat/completions (or /chat/completions)\n",+ " GET /v1/models (or /models)\n",+ ),+ version = Engine::version(),+ model_name = model_name,+ host = host,+ port = port,+ prefix_cache_enabled = prefix_cache_enabled,);- println!("📝 Endpoints:");- println!(" POST /v1/chat/completions (or /chat/completions)");- println!(" GET /v1/models (or /models)");let rocket = rocket::custom(config).manage(state)+ .attach(RequestLoggingFairing).mount("/", routes![handle_chat_completions, handle_models]).mount("/v1", routes![handle_chat_completions, handle_models]);diff --git a/crates/legacy/hanashi/Cargo.toml b/crates/legacy/hanashi/Cargo.tomlindex bde07dbbe..a8f7a1719 100644--- a/crates/legacy/hanashi/Cargo.toml+++ b/crates/legacy/hanashi/Cargo.toml@@ -19,10 +19,9 @@ thiserror.workspace = trueminijinja.workspace = trueminijinja-contrib.workspace = trueregex.workspace = true-fancy-regex.workspace = true-llm_json.workspace = truelibc.workspace = trueindexmap.workspace = true+tracing.workspace = true[build-dependencies]serde_json.workspace = truediff --git a/crates/legacy/hanashi/src/chat/hanashi/mod.rs b/crates/legacy/hanashi/src/chat/hanashi/mod.rsindex 6bcb94e3d..c528b6272 100644--- a/crates/legacy/hanashi/src/chat/hanashi/mod.rs+++ b/crates/legacy/hanashi/src/chat/hanashi/mod.rs@@ -114,6 +114,7 @@ impl EncodingTrait for HanashiEncodingImpl {let eos_token = self.config.tokens.eos_token_id.and_then(|token_id| self.resolve_token(token_id, false).ok());let text = self.renderer.render(&messages, true, bos_token, eos_token, None)?;let text_encoding = self.tokenizer.encode(text, false).map_err(|_| Error::UnableToEncodeText)?;+ tracing::debug!("Encoded tokens: {:?}", text_encoding.get_ids());for token_id in text_encoding.get_ids() {let token = self.resolve_token(*token_id, true)?;self.push_token_to_parser(&token, true)?;diff --git a/crates/legacy/nagare/src/chat/mod.rs b/crates/legacy/nagare/src/chat/mod.rsindex 75f0c0168..8245abeea 100644--- a/crates/legacy/nagare/src/chat/mod.rs+++ b/crates/legacy/nagare/src/chat/mod.rs@@ -30,6 +30,7 @@ use shoji::{};use tokio::sync::{Mutex, mpsc, mpsc::UnboundedSender};use tokio_util::sync::CancellationToken;+use tracing::Instrument;use uuid::Uuid;#[cfg(feature = "bindings-uniffi")]@@ -638,7 +639,7 @@ impl ChatSession {let cancel_token = cancel_token_to_return.inner().clone();let (sender, receiver) = mpsc::unbounded_channel::<Result<Vec<ChatReply>, ChatSessionError>>();let session = self.clone();- let turn = async move { session.execute_turn(sender, input, config, cancel_token).await };+ let turn = async move { session.execute_turn(sender, input, config, cancel_token).await }.in_current_span();#[cfg(feature = "bindings-pyo3")]bindings_pyo3::spawn_with_current_task_locals(turn);#[cfg(not(feature = "bindings-pyo3"))]diff --git a/crates/legacy/nagare/src/chat/token.rs b/crates/legacy/nagare/src/chat/token.rsindex dcffa4770..47450d30e 100644--- a/crates/legacy/nagare/src/chat/token.rs+++ b/crates/legacy/nagare/src/chat/token.rs@@ -23,7 +23,7 @@ use shoji::{},},types::{- basic::{SamplingParameters, TokenId},+ basic::{SamplingParameters, Token, TokenId},model::Model,session::chat::{ChatConfig, ChatContentBlock, ChatMessage, ChatReplyConfig, ChatReplyEnergy, ChatReplyFinishReason,@@ -162,6 +162,10 @@ impl Session {if let Err(err) = self.state_reset().await {return error_stream(err);}+ tracing::debug!(+ "Reprefill: {}",+ describe_reprefill(&curr_all_tokens, &self.encoding.state().tokens, &curr_text, &new_text)+ );new_all_tokens} else {match self.encoding.tokenize(&new_text[curr_text.len()..]) {@@ -183,11 +187,13 @@ impl Session {}let time_prefill_start = Instant::now();let stream = instance.stream(&self.input_tokens, self.state.as_mut(), config.clone(), cancel_token.clone());+ let output_tokens_start = self.encoding.state().tokens.len();let stream_state = StreamingState {config: config.clone(),cancel_token,encoding: &mut self.encoding,+ output_tokens_start,max_context_length: self.instance.max_context_length(),stop_token_ids: self.stop_token_ids.clone(),#[cfg(any(target_os = "macos", target_os = "ios"))]@@ -358,10 +364,55 @@ impl Session {}}+fn describe_reprefill(+ cached: &[Token],+ rendered: &[Token],+ cached_text: &str,+ rendered_text: &str,+) -> String {+ let text_difference = cached_text.bytes().zip(rendered_text.bytes()).take_while(|(a, b)| a == b).count();+ let token_difference = cached.iter().zip(rendered).take_while(|(a, b)| a.id == b.id).count();+ let reason = if cached_text.starts_with(rendered_text) {+ "rendered prompt is shorter than cached text"+ } else {+ "rendered text changed"+ };+ let token_difference_description = if token_difference == cached.len() && token_difference == rendered.len() {+ "none (token IDs match)".to_string()+ } else {+ let describe = |token: Option<&Token>| {+ token.map_or_else(|| "<end>".to_string(), |token| format!("{} ({:?})", token.id, token.value))+ };+ format!(+ "{token_difference} (0-based), cached={}, rendered={}",+ describe(cached.get(token_difference)),+ describe(rendered.get(token_difference)),+ )+ };+ let context = |label: &str, tokens: &[Token]| {+ let start = token_difference.saturating_sub(3);+ let end = (token_difference + 4).min(tokens.len());+ let window: Vec<_> = tokens[start..end].iter().map(|token| (token.id, token.value.as_str())).collect();+ format!("{label}[{start}..{end}]={window:?}")+ };+ format!(+ "{reason}; discarding {} cached tokens, prefilling {} tokens; \+ first_text_difference_byte={text_difference} (0-based), cached_bytes={}, rendered_bytes={}; \+ first_token_difference={token_difference_description}; {}; {}",+ cached.len(),+ rendered.len(),+ cached_text.len(),+ rendered_text.len(),+ context("cached_tokens", cached),+ context("rendered_tokens", rendered),+ )+}+struct StreamingState<'a> {config: ChatReplyConfig,cancel_token: CancellationToken,encoding: &'a mut Encoding,+ output_tokens_start: usize,max_context_length: Option<usize>,stop_token_ids: Box<[u64]>,#[cfg(any(target_os = "macos", target_os = "ios"))]@@ -379,6 +430,16 @@ struct StreamingState<'a> {metrics: Option<TokenStreamMetrics>,}+impl Drop for StreamingState<'_> {+ fn drop(&mut self) {+ // Emit once per generated message, including streams stopped early.+ tracing::debug!(+ "Decoded tokens: {:?}",+ self.encoding.state().tokens[self.output_tokens_start..].iter().map(|token| token.id).collect::<Vec<_>>()+ );+ }+}+impl StreamingState<'_> {fn get_finish_reason(&self,diff --git a/crates/legacy/uzu/src/engine/mod.rs b/crates/legacy/uzu/src/engine/mod.rsindex 8fa794e2e..e9ca1ad56 100644--- a/crates/legacy/uzu/src/engine/mod.rs+++ b/crates/legacy/uzu/src/engine/mod.rs@@ -76,8 +76,8 @@ impl Engine {let telemetry = SharedAccess::new({let context = TelemetryContext::new(- env!("CARGO_PKG_VERSION").to_string(),- uzu_engine::TOOLCHAIN_VERSION.to_string(),+ Self::version(),+ Self::toolchain_version(),TelemetryDevice {os_name: device.os_name.clone(),cpu_name: device.cpu_name.clone(),@@ -183,6 +183,14 @@ impl Engine {Ok(engine)}++ pub fn toolchain_version() -> String {+ uzu_engine::TOOLCHAIN_VERSION.to_string()+ }++ pub fn version() -> String {+ uzu_engine::VERSION.to_string()+ }}#[bindings::export(Implementation)]diff --git a/crates/legacy/uzu/src/logs/mod.rs b/crates/legacy/uzu/src/logs/mod.rsindex c2ff29ce9..c3ac43d6d 100644--- a/crates/legacy/uzu/src/logs/mod.rs+++ b/crates/legacy/uzu/src/logs/mod.rs@@ -20,6 +20,11 @@ pub fn start(name: &str,stdout: bool,) {+ // Applications can install their own tracing subscriber before creating an engine.+ if tracing::dispatcher::has_been_set() {+ return;+ }+INIT.call_once(|| {init(path, name, MAX_LOG_SIZE_BYTES, KEEP_LINES, stdout);});