Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Cli request logs (#822)

Aleksandr Golokoze88f395bb2e3
diff --git a/Cargo.lock b/Cargo.lock
index fdbb11fd0..8d20bfd21 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -948,6 +948,7 @@ dependencies = [
"tokio-stream",
"toml 1.1.2+spec-1.1.0",
"tracing",
+ "tracing-subscriber",
"unicode-segmentation",
"unicode-width 0.2.2",
"uuid",
@@ -2422,10 +2423,8 @@ dependencies = [
name = "hanashi"
version = "0.5.26"
dependencies = [
- "fancy-regex 0.17.0",
"indexmap 2.14.0",
"libc",
- "llm_json",
"minijinja",
"minijinja-contrib",
"openai-harmony",
@@ -2436,6 +2435,7 @@ dependencies = [
"thiserror 2.0.18",
"token-stream-parser",
"tokenizers",
+ "tracing",
]
[[package]]
diff --git a/crates/legacy/cli/Cargo.toml b/crates/legacy/cli/Cargo.toml
index cd9a8daf9..273054341 100644
--- a/crates/legacy/cli/Cargo.toml
+++ b/crates/legacy/cli/Cargo.toml
@@ -47,6 +47,7 @@ tokio = { workspace = true, features = ["full"] }
tokio-stream.workspace = true
toml.workspace = true
tracing.workspace = true
+tracing-subscriber.workspace = true
unicode-segmentation.workspace = true
unicode-width.workspace = true
uuid.workspace = true
diff --git a/crates/legacy/cli/src/main.rs b/crates/legacy/cli/src/main.rs
index ac57345c1..1630eb4c7 100644
--- a/crates/legacy/cli/src/main.rs
+++ b/crates/legacy/cli/src/main.rs
@@ -9,7 +9,7 @@ mod server;
mod storage;
#[derive(Parser)]
-#[command(name = "cli", bin_name = "cli")]
+#[command(name = "cli", bin_name = "cli", version)]
struct Cli {
/// Identifier of the model to start with (e.g. "alibaba:qwen3.5:0.8b:mirai:mirai-m:4").
#[arg(long, value_name = "MODEL")]
@@ -48,6 +48,9 @@ enum Commands {
/// Reset the session on every request, disabling prefix cache reuse.
#[arg(long = "no-prefix-cache", action = clap::ArgAction::SetTrue)]
no_prefix_cache: bool,
+ /// Print verbose information into the file log
+ #[arg(long)]
+ verbose_file_log: bool,
},
Storage {
#[arg(long, value_enum, default_value_t = storage::DownloadManagerCliType::default())]
@@ -75,7 +78,8 @@ async fn main() -> Result<()> {
host,
prefix_cache,
no_prefix_cache,
- }) => server::run_server(model, host, port, prefix_cache || !no_prefix_cache).await?,
+ verbose_file_log,
+ }) => server::run_server(model, host, port, prefix_cache || !no_prefix_cache, verbose_file_log).await?,
Some(Commands::Storage {
download_manager,
}) => storage::run(download_manager).await?,
diff --git a/crates/legacy/cli/src/server/chat_completions.rs b/crates/legacy/cli/src/server/chat_completions.rs
index 70de7d578..04da77cc4 100644
--- a/crates/legacy/cli/src/server/chat_completions.rs
+++ b/crates/legacy/cli/src/server/chat_completions.rs
@@ -1,10 +1,4 @@
-use std::{
- collections::HashMap,
- pin::Pin,
- str::FromStr,
- sync::Arc,
- time::{SystemTime, UNIX_EPOCH},
-};
+use std::{collections::HashMap, pin::Pin, str::FromStr, sync::Arc};
use rocket::{
Request, State,
@@ -21,7 +15,7 @@ use rocket::{
use serde::{Deserialize, Serialize};
use tokio::sync::{Mutex, mpsc};
use tokio_stream::wrappers::UnboundedReceiverStream;
-use uuid::Uuid;
+use tracing::Instrument;
use uzu::{
session::chat::{ChatSession, ChatSessionStream, ChatSessionStreamChunk, UNPARSED_ARGUMENTS_KEY},
types::{
@@ -41,6 +35,7 @@ use crate::{
coerce_tool_call, insert_tools_message, oai_tool_call, parse_scalar_text, reply_tool_calls, to_tool_call,
tool_call_result_block, withhold_stream_text,
},
+ request_info::RequestInfo,
request_log::RequestLog,
},
};
@@ -242,10 +237,6 @@ impl<'r> Responder<'r, 'r> for ChatCompletionResult {
}
}
-fn now_unix() -> i64 {
- SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0)
-}
-
fn to_chat_messages(messages: &[OaiMessage]) -> Vec<ChatMessage> {
messages
.iter()
@@ -1150,9 +1141,12 @@ async fn run_stream(
#[allow(private_interfaces)]
#[post("/chat/completions", format = "json", data = "<body>")]
+#[tracing::instrument(skip_all, parent = &request_info.span)]
pub async fn handle_chat_completions(
body: Data<'_>,
state: &State<ServerState>,
+ request_info: &RequestInfo,
+ content_type: &ContentType,
) -> ChatCompletionResult {
let body = match body.open(ByteUnit::Mebibyte(64)).into_string().await {
Ok(body) if body.is_complete() => body.into_inner(),
@@ -1163,10 +1157,17 @@ pub async fn handle_chat_completions(
return invalid_request_response("body", "invalid_body", format!("failed to read request body: {error}"));
},
};
+ let log_body = if content_type.is_json() {
+ serde_json::from_str::<serde_json::Value>(&body).map(|value| value.to_string()).unwrap_or_else(|_| body.clone())
+ } else {
+ body.clone()
+ };
+ tracing::debug!("--> {} {} body={}", request_info.method, request_info.uri, log_body);
+
let request = match serde_json::from_str::<ChatCompletionRequest>(&body) {
Ok(request) => request,
Err(error) => {
- RequestLog::rejected(&format!("failed to parse chat completion request: {error}"));
+ tracing::info!("rejected: {error}");
return invalid_request_response(
"body",
"invalid_request",
@@ -1174,12 +1175,11 @@ pub async fn handle_chat_completions(
);
},
};
- let id = format!("chatcmpl-{}", Uuid::new_v4().simple());
- let created = now_unix();
+
let model = state.model_name.clone();
+ let created = request_info.created_at;
let is_stream = request.stream.unwrap_or(false);
let log = RequestLog::start(
- &id,
is_stream,
request.messages.len(),
request.tools.as_ref().map_or(0, Vec::len),
@@ -1212,38 +1212,34 @@ pub async fn handle_chat_completions(
// the types when replies cross back into the OpenAI wire format.
let parameter_types = ToolParameterTypes::from_tools(request.tools.as_deref());
+ let id = request_info.id.as_str().to_owned();
if is_stream {
let session = Arc::clone(&state.session);
let (sender, receiver) = mpsc::unbounded_channel::<Event>();
- rocket::tokio::spawn(run_stream(
- session,
- messages,
- config,
- id,
- model,
- created,
- state.prefix_cache,
- parameter_types,
- sender,
- log,
- ));
+ tokio::spawn(
+ run_stream(session, messages, config, id, model, created, state.prefix_cache, parameter_types, sender, log)
+ .in_current_span(),
+ );
let body: Pin<Box<dyn Stream<Item = Event> + Send>> = Box::pin(UnboundedReceiverStream::new(receiver));
ChatCompletionResult::Stream(EventStream::from(body))
} else {
let session = Arc::clone(&state.session);
let (sender, receiver) = mpsc::unbounded_channel::<Vec<u8>>();
- rocket::tokio::spawn(run_blocking(
- session,
- messages,
- config,
- id,
- model,
- created,
- state.prefix_cache,
- parameter_types,
- sender,
- log,
- ));
+ tokio::spawn(
+ run_blocking(
+ session,
+ messages,
+ config,
+ id,
+ model,
+ created,
+ state.prefix_cache,
+ parameter_types,
+ sender,
+ log,
+ )
+ .in_current_span(),
+ );
let body: Pin<Box<dyn Stream<Item = Vec<u8>> + Send>> = Box::pin(UnboundedReceiverStream::new(receiver));
// Hand the body to Rocket immediately so it can write keepalives and
// drop the receiver when the client disconnects. Any backend error
diff --git a/crates/legacy/cli/src/server/log/file_logger.rs b/crates/legacy/cli/src/server/log/file_logger.rs
new file mode 100644
index 000000000..634558e47
--- /dev/null
+++ b/crates/legacy/cli/src/server/log/file_logger.rs
@@ -0,0 +1,79 @@
+use std::{
+ fs::{self},
+ io::{self},
+ path::{Path, PathBuf},
+};
+
+use chrono::{Local, NaiveDateTime, TimeDelta};
+use tracing_subscriber::fmt::writer::MakeWriter;
+use uuid::Uuid;
+
+use crate::server::log::log_file::LogFile;
+
+const LOG_FILE_DATE_FORMAT: &str = "%Y-%m-%d-%H-%M-%S";
+const LOG_FILE_EXTENSION: &str = "log";
+const LOG_FILE_RETENTION: TimeDelta = TimeDelta::weeks(1);
+
+#[derive(Clone)]
+pub struct FileLogger {
+ file: LogFile,
+}
+
+impl FileLogger {
+ pub fn new(logs_dir_path: PathBuf) -> io::Result<Self> {
+ fs::create_dir_all(&logs_dir_path)?;
+ Self::remove_expired_log_files(&logs_dir_path)?;
+
+ let date = Local::now().format(LOG_FILE_DATE_FORMAT);
+ let session_id = Uuid::new_v4().simple().to_string();
+ let file_name = format!("{}-{}.{}", date, session_id, LOG_FILE_EXTENSION);
+ let file = LogFile::new(logs_dir_path.join(file_name))?;
+ Ok(Self {
+ file,
+ })
+ }
+
+ pub fn file_path(&self) -> &PathBuf {
+ self.file.path()
+ }
+
+ fn remove_expired_log_files(logs_dir_path: &Path) -> io::Result<()> {
+ let cutoff = Local::now().naive_local() - LOG_FILE_RETENTION;
+
+ for entry in fs::read_dir(logs_dir_path)? {
+ let entry = entry?;
+ if !entry.file_type()?.is_file() {
+ continue;
+ }
+
+ let path = entry.path();
+ if path.extension().is_none_or(|extension| extension != LOG_FILE_EXTENSION) {
+ continue;
+ }
+
+ let Some(file_stem) = path.file_stem().and_then(|file_stem| file_stem.to_str()) else {
+ continue;
+ };
+ let Some((date, _session_id)) = file_stem.rsplit_once('-') else {
+ continue;
+ };
+ let Ok(created_at) = NaiveDateTime::parse_from_str(date, LOG_FILE_DATE_FORMAT) else {
+ continue;
+ };
+
+ if created_at < cutoff {
+ fs::remove_file(path)?;
+ }
+ }
+
+ Ok(())
+ }
+}
+
+impl<'a> MakeWriter<'a> for FileLogger {
+ type Writer = <LogFile as MakeWriter<'a>>::Writer;
+
+ fn make_writer(&'a self) -> Self::Writer {
+ self.file.make_writer()
+ }
+}
diff --git a/crates/legacy/cli/src/server/log/log_file.rs b/crates/legacy/cli/src/server/log/log_file.rs
new file mode 100644
index 000000000..87984ad0a
--- /dev/null
+++ b/crates/legacy/cli/src/server/log/log_file.rs
@@ -0,0 +1,43 @@
+use std::{
+ fs::{File, OpenOptions},
+ io,
+ path::{Path, PathBuf},
+ sync::{Arc, Mutex},
+};
+
+use tracing_subscriber::fmt::writer::{MakeWriter, MutexGuardWriter};
+
+#[derive(Clone)]
+pub struct LogFile {
+ file: Arc<Mutex<File>>,
+ path: PathBuf,
+}
+
+impl LogFile {
+ pub fn new(path: impl AsRef<Path>) -> io::Result<Self> {
+ let path = path.as_ref();
+ if let Some(parent) = path.parent()
+ && !parent.as_os_str().is_empty()
+ {
+ std::fs::create_dir_all(parent)?;
+ }
+
+ let file = OpenOptions::new().create(true).append(true).open(path)?;
+ Ok(Self {
+ file: Arc::new(Mutex::new(file)),
+ path: path.into(),
+ })
+ }
+
+ pub fn path(&self) -> &PathBuf {
+ &self.path
+ }
+}
+
+impl<'a> MakeWriter<'a> for LogFile {
+ type Writer = MutexGuardWriter<'a, File>;
+
+ fn make_writer(&'a self) -> Self::Writer {
+ self.file.make_writer()
+ }
+}
diff --git a/crates/legacy/cli/src/server/log/mod.rs b/crates/legacy/cli/src/server/log/mod.rs
new file mode 100644
index 000000000..c4dcc0643
--- /dev/null
+++ b/crates/legacy/cli/src/server/log/mod.rs
@@ -0,0 +1,29 @@
+mod file_logger;
+mod log_file;
+mod request_trace;
+
+use std::io;
+
+pub use file_logger::FileLogger;
+use tracing_subscriber::{EnvFilter, fmt::writer::MakeWriterExt, layer::SubscriberExt, util::SubscriberInitExt};
+
+use self::request_trace::{RequestContextLayer, RequestFormatter};
+
+pub fn init(
+ stdout_max_level: tracing::Level,
+ file_logger: Option<FileLogger>,
+ file_max_level: tracing::Level,
+) -> anyhow::Result<()> {
+ let subscriber = tracing_subscriber::registry()
+ .with(EnvFilter::new("off,cli=debug,hanashi=debug,nagare::chat::token=debug"))
+ .with(RequestContextLayer);
+ let format = tracing_subscriber::fmt::layer().event_format(RequestFormatter).with_ansi(false);
+
+ let console = io::stdout.with_max_level(stdout_max_level);
+ let result = if let Some(file_logger) = file_logger {
+ subscriber.with(format.with_writer(console.and(file_logger.with_max_level(file_max_level)))).try_init()
+ } else {
+ subscriber.with(format.with_writer(console)).try_init()
+ };
+ result.map_err(|error| anyhow::anyhow!("Failed to initialize server tracing: {error}"))
+}
diff --git a/crates/legacy/cli/src/server/log/request_trace.rs b/crates/legacy/cli/src/server/log/request_trace.rs
new file mode 100644
index 000000000..26ab6fad9
--- /dev/null
+++ b/crates/legacy/cli/src/server/log/request_trace.rs
@@ -0,0 +1,88 @@
+use std::fmt;
+
+use tracing::{
+ Event, Subscriber,
+ field::{Field, Visit},
+ span::{Attributes, Id},
+};
+use tracing_subscriber::{
+ Layer,
+ fmt::{FmtContext, FormatEvent, FormatFields, format::Writer},
+ layer::Context,
+ registry::LookupSpan,
+};
+
+#[derive(Default)]
+struct RequestId(Option<String>);
+
+impl Visit for RequestId {
+ fn record_str(
+ &mut self,
+ field: &Field,
+ value: &str,
+ ) {
+ if field.name() == "request_id" {
+ self.0 = Some(value.to_owned());
+ }
+ }
+
+ fn record_debug(
+ &mut self,
+ field: &Field,
+ value: &dyn fmt::Debug,
+ ) {
+ if field.name() == "request_id" {
+ self.0 = Some(format!("{value:?}"));
+ }
+ }
+}
+
+/// Store the request ID separately from the span's human-readable fields.
+pub(super) struct RequestContextLayer;
+
+impl<S> Layer<S> for RequestContextLayer
+where
+ S: Subscriber + for<'a> LookupSpan<'a>,
+{
+ fn on_new_span(
+ &self,
+ attrs: &Attributes<'_>,
+ id: &Id,
+ ctx: Context<'_, S>,
+ ) {
+ let mut request_id = RequestId::default();
+ attrs.record(&mut request_id);
+ if request_id.0.is_some()
+ && let Some(span) = ctx.span(id)
+ {
+ span.extensions_mut().insert(request_id);
+ }
+ }
+}
+
+pub(super) struct RequestFormatter;
+
+impl<S, N> FormatEvent<S, N> for RequestFormatter
+where
+ S: Subscriber + for<'a> LookupSpan<'a>,
+ N: for<'a> FormatFields<'a> + 'static,
+{
+ fn format_event(
+ &self,
+ ctx: &FmtContext<'_, S, N>,
+ mut writer: Writer<'_>,
+ event: &Event<'_>,
+ ) -> fmt::Result {
+ if let Some(scope) = ctx.event_scope() {
+ for span in scope {
+ let extensions = span.extensions();
+ if let Some(RequestId(Some(id))) = extensions.get::<RequestId>() {
+ write!(writer, "[{id}] ")?;
+ break;
+ }
+ }
+ }
+ ctx.field_format().format_fields(writer.by_ref(), event)?;
+ writeln!(writer)
+ }
+}
diff --git a/crates/legacy/cli/src/server/mod.rs b/crates/legacy/cli/src/server/mod.rs
index 8a09adf69..ed97ac331 100644
--- a/crates/legacy/cli/src/server/mod.rs
+++ b/crates/legacy/cli/src/server/mod.rs
@@ -1,3 +1,6 @@
+mod log;
+mod request_info;
+
pub mod chat_completions;
pub mod chat_tool_calls;
pub mod models;
diff --git a/crates/legacy/cli/src/server/models.rs b/crates/legacy/cli/src/server/models.rs
index 1db75c6b3..74212a2db 100644
--- a/crates/legacy/cli/src/server/models.rs
+++ b/crates/legacy/cli/src/server/models.rs
@@ -1,7 +1,7 @@
use rocket::{State, get, serde::json::Json};
use serde::{Deserialize, Serialize};
-use crate::server::ServerState;
+use crate::server::{ServerState, request_info::RequestInfo};
#[derive(Serialize, Deserialize)]
pub struct Model {
@@ -18,7 +18,11 @@ pub struct ModelsResponse {
}
#[get("/models")]
-pub fn handle_models(state: &State<ServerState>) -> Json<ModelsResponse> {
+#[tracing::instrument(skip_all, parent = &request_info.span)]
+pub fn handle_models(
+ state: &State<ServerState>,
+ request_info: &RequestInfo,
+) -> Json<ModelsResponse> {
Json(ModelsResponse {
object: "list".to_string(),
data: vec![Model {
diff --git a/crates/legacy/cli/src/server/request_info.rs b/crates/legacy/cli/src/server/request_info.rs
new file mode 100644
index 000000000..f4e36209d
--- /dev/null
+++ b/crates/legacy/cli/src/server/request_info.rs
@@ -0,0 +1,48 @@
+use std::{
+ convert::Infallible,
+ time::{SystemTime, UNIX_EPOCH},
+};
+
+use rocket::{
+ Request,
+ http::Method,
+ request::{FromRequest, Outcome},
+};
+use uuid::Uuid;
+
+pub struct RequestInfo {
+ pub method: Method,
+ pub uri: String,
+ pub id: String,
+ pub created_at: i64,
+ pub span: tracing::Span,
+}
+
+impl RequestInfo {
+ pub fn new(
+ method: Method,
+ uri: String,
+ ) -> Self {
+ let id = Uuid::new_v4().simple().to_string();
+ let span = tracing::info_span!(parent: None, "request", request_id = &id[..8]);
+ Self {
+ method,
+ uri,
+ id,
+ span,
+ created_at: SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map(|duration| duration.as_secs() as i64)
+ .unwrap_or(0),
+ }
+ }
+}
+
+#[rocket::async_trait]
+impl<'a> FromRequest<'a> for &'a RequestInfo {
+ type Error = Infallible;
+
+ async fn from_request(request: &'a Request<'_>) -> Outcome<Self, Self::Error> {
+ Outcome::Success(request.local_cache(|| RequestInfo::new(request.method(), request.uri().to_string())))
+ }
+}
diff --git a/crates/legacy/cli/src/server/request_log.rs b/crates/legacy/cli/src/server/request_log.rs
index e1aa7fac0..322d5173f 100644
--- a/crates/legacy/cli/src/server/request_log.rs
+++ b/crates/legacy/cli/src/server/request_log.rs
@@ -1,51 +1,35 @@
use std::time::Instant;
-use uuid::Uuid;
use uzu::types::session::chat::ChatReplyStats;
-/// Per-request console logging: one line when the request arrives, one when it
-/// ends, correlated by a short tag derived from the request id. Each line is a
-/// single `println!`, which holds the stdout lock for the whole write, so lines
-/// from concurrent requests never interleave.
+/// Per-request tracing: one event when the request arrives, one when it ends,
+/// correlated by a short tag derived from the request id.
pub struct RequestLog {
- tag: String,
started: Instant,
}
-fn short_tag(id: &str) -> String {
- let id = id.strip_prefix("chatcmpl-").unwrap_or(id);
- id.chars().take(8).collect()
-}
-
impl RequestLog {
pub fn start(
- id: &str,
stream: bool,
messages: usize,
tools: usize,
reasoning_effort: Option<&str>,
) -> Self {
- let tag = short_tag(id);
- println!(
- "[req {tag}] received: {messages} messages, {}, {tools} tools, reasoning_effort={}",
- if stream {
- "stream"
- } else {
- "blocking"
- },
+ let stream = if stream {
+ "stream"
+ } else {
+ "blocking"
+ };
+ tracing::info!(
+ "received: {messages} messages, {}, {tools} tools, reasoning_effort={}",
+ stream,
reasoning_effort.unwrap_or("default"),
);
Self {
- tag,
started: Instant::now(),
}
}
- /// For requests rejected before an id could be assigned to them.
- pub fn rejected(error: &str) {
- println!("[req {}] rejected: {error}", short_tag(&Uuid::new_v4().simple().to_string()));
- }
-
pub fn finish(
&self,
finish_reason: &str,
@@ -81,13 +65,13 @@ impl RequestLog {
}
}
parts.extend(notes);
- println!("[req {}] {}", self.tag, parts.join(", "));
+ tracing::info!("{}", parts.join(", "));
}
pub fn fail(
&self,
error: &str,
) {
- println!("[req {}] failed in {:.2}s: {error}", self.tag, self.started.elapsed().as_secs_f64());
+ tracing::info!("failed in {:.2}s: {error}", self.started.elapsed().as_secs_f64());
}
}
diff --git a/crates/legacy/cli/src/server/runner.rs b/crates/legacy/cli/src/server/runner.rs
index afc0755ff..185448289 100644
--- a/crates/legacy/cli/src/server/runner.rs
+++ b/crates/legacy/cli/src/server/runner.rs
@@ -1,9 +1,19 @@
-use std::{net::IpAddr, sync::Arc, time::Duration};
+use std::{io, net::IpAddr, path::PathBuf, pin::Pin, sync::Arc, task::Poll, time::Duration};
use anyhow::{Context, Result, bail};
use indicatif::{ProgressBar, ProgressStyle};
-use rocket::{Config, config::LogLevel, routes};
-use tokio::sync::Mutex;
+use rocket::{
+ Config, Request, Response,
+ config::LogLevel,
+ data::Data,
+ fairing::{Fairing, Info, Kind},
+ response::Body,
+ routes,
+};
+use tokio::{
+ io::{AsyncRead, ReadBuf},
+ sync::Mutex,
+};
use uzu::{
engine::{Engine, EngineConfig},
types::session::chat::ChatConfig,
@@ -11,15 +21,156 @@ use uzu::{
use crate::{
common::thinking::ThinkingSupport,
- server::{ServerState, handle_chat_completions, handle_models},
+ server::{
+ ServerState, handle_chat_completions, handle_models,
+ log::{self, FileLogger},
+ request_info::RequestInfo,
+ },
};
+pub struct ResponseBodyLogger<'r> {
+ body: Body<'r>,
+ prefix: String,
+ bytes: Vec<u8>,
+ is_json: bool,
+ logged: bool,
+ span: tracing::Span,
+}
+
+impl<'r> ResponseBodyLogger<'r> {
+ pub fn new(
+ body: Body<'r>,
+ prefix: String,
+ is_json: bool,
+ span: tracing::Span,
+ ) -> Self {
+ Self {
+ body,
+ prefix,
+ bytes: Vec::new(),
+ is_json,
+ logged: false,
+ span,
+ }
+ }
+
+ fn log(
+ &mut self,
+ suffix: &str,
+ ) {
+ if self.logged {
+ return;
+ }
+ self.logged = true;
+
+ let body = if self.is_json {
+ serde_json::from_slice::<serde_json::Value>(&self.bytes)
+ .map(|value| value.to_string())
+ .unwrap_or_else(|_| format!("{:?}", String::from_utf8_lossy(&self.bytes)))
+ } else {
+ format!("{:?}", String::from_utf8_lossy(&self.bytes))
+ };
+ self.span.in_scope(|| tracing::debug!("{} body={}{}\n", self.prefix, body, suffix));
+ }
+}
+
+impl AsyncRead for ResponseBodyLogger<'_> {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ cx: &mut std::task::Context<'_>,
+ buffer: &mut ReadBuf<'_>,
+ ) -> Poll<io::Result<()>> {
+ let this = self.get_mut();
+ let span = this.span.clone();
+ let _entered = span.enter();
+ let filled_before = buffer.filled().len();
+
+ match Pin::new(&mut this.body).poll_read(cx, buffer) {
+ Poll::Ready(Ok(())) => {
+ let filled_after = buffer.filled().len();
+ if filled_after == filled_before {
+ this.log("");
+ } else {
+ this.bytes.extend_from_slice(&buffer.filled()[filled_before..filled_after]);
+ }
+ Poll::Ready(Ok(()))
+ },
+ Poll::Ready(Err(error)) => {
+ this.log(" [read error]");
+ Poll::Ready(Err(error))
+ },
+ Poll::Pending => Poll::Pending,
+ }
+ }
+}
+
+impl Drop for ResponseBodyLogger<'_> {
+ fn drop(&mut self) {
+ self.log(" [incomplete]");
+ }
+}
+
+struct RequestLoggingFairing;
+
+#[rocket::async_trait]
+impl Fairing for RequestLoggingFairing {
+ fn info(&self) -> Info {
+ Info {
+ name: "Request and response logger",
+ kind: Kind::Request | Kind::Response,
+ }
+ }
+
+ async fn on_request(
+ &self,
+ request: &mut Request<'_>,
+ _data: &mut Data<'_>,
+ ) {
+ let _ = request.local_cache(|| RequestInfo::new(request.method(), request.uri().to_string()));
+ }
+
+ async fn on_response<'r>(
+ &self,
+ request: &'r Request<'_>,
+ response: &mut Response<'r>,
+ ) {
+ if matches!(request.uri().path().as_str(), "/logs") {
+ return;
+ }
+
+ let request_info = request.local_cache(|| RequestInfo::new(request.method(), request.uri().to_string()));
+ let _entered = request_info.span.enter();
+ let prefix = format!("<-- {} {}", response.status(), request.uri());
+ if response.body().is_none() {
+ tracing::debug!("{prefix} body=<empty>\n");
+ } else {
+ let is_json = response.content_type().is_some_and(|content_type| content_type.is_json());
+ let body = response.body_mut().take();
+ response.set_streamed_body(ResponseBodyLogger::new(body, prefix, is_json, request_info.span.clone()));
+ }
+ }
+}
+
pub async fn run_server(
model: String,
host: String,
port: u16,
prefix_cache: bool,
+ verbose_file_logs: bool,
) -> Result<()> {
+ let file_logger = if verbose_file_logs {
+ let cache_dir = std::env::var_os("XDG_CACHE_HOME")
+ .map(PathBuf::from)
+ .filter(|path| path.is_absolute())
+ .or_else(|| dirs::home_dir().map(|path| path.join(".cache")))
+ .context("Failed to resolve cache directory")?;
+ let logs_dir_path = cache_dir.join("mirai").join("server").join("logs");
+ Some(FileLogger::new(logs_dir_path)?)
+ } else {
+ None
+ };
+ log::init(tracing::Level::INFO, file_logger.clone(), tracing::Level::DEBUG)?;
+
let engine_config = EngineConfig::default().with_application_identifier("com.trymirai.cli".to_string());
let engine = Engine::new(engine_config).await.context("Failed to create engine")?;
@@ -57,22 +208,33 @@ pub async fn run_server(
..Config::default()
};
- println!("🚀 OpenAI-compatible server for model: {model_name}");
- println!("🌐 Available at: http://{host}:{port}");
- println!(
- "🗄️ Prefix cache: {}",
- if prefix_cache {
- "enabled"
- } else {
- "disabled"
- }
+ let prefix_cache_enabled = if prefix_cache {
+ "enabled"
+ } else {
+ "disabled"
+ };
+ if let Some(ref file_logger) = file_logger {
+ tracing::info!("🗃️ Logs will be written to file: {}", file_logger.file_path().display())
+ }
+ tracing::info!(
+ concat!(
+ "🚀 OpenAI-compatible server v{version} for model: {model_name}\n",
+ "🌐 Available at: http://{host}:{port}\n",
+ "🗄️ Prefix cache: {prefix_cache_enabled}\n",
+ "📝 Endpoints:\n",
+ " POST /v1/chat/completions (or /chat/completions)\n",
+ " GET /v1/models (or /models)\n",
+ ),
+ version = Engine::version(),
+ model_name = model_name,
+ host = host,
+ port = port,
+ prefix_cache_enabled = prefix_cache_enabled,
);
- println!("📝 Endpoints:");
- println!(" POST /v1/chat/completions (or /chat/completions)");
- println!(" GET /v1/models (or /models)");
let rocket = rocket::custom(config)
.manage(state)
+ .attach(RequestLoggingFairing)
.mount("/", routes![handle_chat_completions, handle_models])
.mount("/v1", routes![handle_chat_completions, handle_models]);
diff --git a/crates/legacy/hanashi/Cargo.toml b/crates/legacy/hanashi/Cargo.toml
index bde07dbbe..a8f7a1719 100644
--- a/crates/legacy/hanashi/Cargo.toml
+++ b/crates/legacy/hanashi/Cargo.toml
@@ -19,10 +19,9 @@ thiserror.workspace = true
minijinja.workspace = true
minijinja-contrib.workspace = true
regex.workspace = true
-fancy-regex.workspace = true
-llm_json.workspace = true
libc.workspace = true
indexmap.workspace = true
+tracing.workspace = true
[build-dependencies]
serde_json.workspace = true
diff --git a/crates/legacy/hanashi/src/chat/hanashi/mod.rs b/crates/legacy/hanashi/src/chat/hanashi/mod.rs
index 6bcb94e3d..c528b6272 100644
--- a/crates/legacy/hanashi/src/chat/hanashi/mod.rs
+++ b/crates/legacy/hanashi/src/chat/hanashi/mod.rs
@@ -114,6 +114,7 @@ impl EncodingTrait for HanashiEncodingImpl {
let eos_token = self.config.tokens.eos_token_id.and_then(|token_id| self.resolve_token(token_id, false).ok());
let text = self.renderer.render(&messages, true, bos_token, eos_token, None)?;
let text_encoding = self.tokenizer.encode(text, false).map_err(|_| Error::UnableToEncodeText)?;
+ tracing::debug!("Encoded tokens: {:?}", text_encoding.get_ids());
for token_id in text_encoding.get_ids() {
let token = self.resolve_token(*token_id, true)?;
self.push_token_to_parser(&token, true)?;
diff --git a/crates/legacy/nagare/src/chat/mod.rs b/crates/legacy/nagare/src/chat/mod.rs
index 75f0c0168..8245abeea 100644
--- a/crates/legacy/nagare/src/chat/mod.rs
+++ b/crates/legacy/nagare/src/chat/mod.rs
@@ -30,6 +30,7 @@ use shoji::{
};
use tokio::sync::{Mutex, mpsc, mpsc::UnboundedSender};
use tokio_util::sync::CancellationToken;
+use tracing::Instrument;
use uuid::Uuid;
#[cfg(feature = "bindings-uniffi")]
@@ -638,7 +639,7 @@ impl ChatSession {
let cancel_token = cancel_token_to_return.inner().clone();
let (sender, receiver) = mpsc::unbounded_channel::<Result<Vec<ChatReply>, ChatSessionError>>();
let session = self.clone();
- let turn = async move { session.execute_turn(sender, input, config, cancel_token).await };
+ let turn = async move { session.execute_turn(sender, input, config, cancel_token).await }.in_current_span();
#[cfg(feature = "bindings-pyo3")]
bindings_pyo3::spawn_with_current_task_locals(turn);
#[cfg(not(feature = "bindings-pyo3"))]
diff --git a/crates/legacy/nagare/src/chat/token.rs b/crates/legacy/nagare/src/chat/token.rs
index dcffa4770..47450d30e 100644
--- a/crates/legacy/nagare/src/chat/token.rs
+++ b/crates/legacy/nagare/src/chat/token.rs
@@ -23,7 +23,7 @@ use shoji::{
},
},
types::{
- basic::{SamplingParameters, TokenId},
+ basic::{SamplingParameters, Token, TokenId},
model::Model,
session::chat::{
ChatConfig, ChatContentBlock, ChatMessage, ChatReplyConfig, ChatReplyEnergy, ChatReplyFinishReason,
@@ -162,6 +162,10 @@ impl Session {
if let Err(err) = self.state_reset().await {
return error_stream(err);
}
+ tracing::debug!(
+ "Reprefill: {}",
+ describe_reprefill(&curr_all_tokens, &self.encoding.state().tokens, &curr_text, &new_text)
+ );
new_all_tokens
} else {
match self.encoding.tokenize(&new_text[curr_text.len()..]) {
@@ -183,11 +187,13 @@ impl Session {
}
let time_prefill_start = Instant::now();
let stream = instance.stream(&self.input_tokens, self.state.as_mut(), config.clone(), cancel_token.clone());
+ let output_tokens_start = self.encoding.state().tokens.len();
let stream_state = StreamingState {
config: config.clone(),
cancel_token,
encoding: &mut self.encoding,
+ output_tokens_start,
max_context_length: self.instance.max_context_length(),
stop_token_ids: self.stop_token_ids.clone(),
#[cfg(any(target_os = "macos", target_os = "ios"))]
@@ -358,10 +364,55 @@ impl Session {
}
}
+fn describe_reprefill(
+ cached: &[Token],
+ rendered: &[Token],
+ cached_text: &str,
+ rendered_text: &str,
+) -> String {
+ let text_difference = cached_text.bytes().zip(rendered_text.bytes()).take_while(|(a, b)| a == b).count();
+ let token_difference = cached.iter().zip(rendered).take_while(|(a, b)| a.id == b.id).count();
+ let reason = if cached_text.starts_with(rendered_text) {
+ "rendered prompt is shorter than cached text"
+ } else {
+ "rendered text changed"
+ };
+ let token_difference_description = if token_difference == cached.len() && token_difference == rendered.len() {
+ "none (token IDs match)".to_string()
+ } else {
+ let describe = |token: Option<&Token>| {
+ token.map_or_else(|| "<end>".to_string(), |token| format!("{} ({:?})", token.id, token.value))
+ };
+ format!(
+ "{token_difference} (0-based), cached={}, rendered={}",
+ describe(cached.get(token_difference)),
+ describe(rendered.get(token_difference)),
+ )
+ };
+ let context = |label: &str, tokens: &[Token]| {
+ let start = token_difference.saturating_sub(3);
+ let end = (token_difference + 4).min(tokens.len());
+ let window: Vec<_> = tokens[start..end].iter().map(|token| (token.id, token.value.as_str())).collect();
+ format!("{label}[{start}..{end}]={window:?}")
+ };
+ format!(
+ "{reason}; discarding {} cached tokens, prefilling {} tokens; \
+ first_text_difference_byte={text_difference} (0-based), cached_bytes={}, rendered_bytes={}; \
+ first_token_difference={token_difference_description}; {}; {}",
+ cached.len(),
+ rendered.len(),
+ cached_text.len(),
+ rendered_text.len(),
+ context("cached_tokens", cached),
+ context("rendered_tokens", rendered),
+ )
+}
+
struct StreamingState<'a> {
config: ChatReplyConfig,
cancel_token: CancellationToken,
encoding: &'a mut Encoding,
+ output_tokens_start: usize,
max_context_length: Option<usize>,
stop_token_ids: Box<[u64]>,
#[cfg(any(target_os = "macos", target_os = "ios"))]
@@ -379,6 +430,16 @@ struct StreamingState<'a> {
metrics: Option<TokenStreamMetrics>,
}
+impl Drop for StreamingState<'_> {
+ fn drop(&mut self) {
+ // Emit once per generated message, including streams stopped early.
+ tracing::debug!(
+ "Decoded tokens: {:?}",
+ self.encoding.state().tokens[self.output_tokens_start..].iter().map(|token| token.id).collect::<Vec<_>>()
+ );
+ }
+}
+
impl StreamingState<'_> {
fn get_finish_reason(
&self,
diff --git a/crates/legacy/uzu/src/engine/mod.rs b/crates/legacy/uzu/src/engine/mod.rs
index 8fa794e2e..e9ca1ad56 100644
--- a/crates/legacy/uzu/src/engine/mod.rs
+++ b/crates/legacy/uzu/src/engine/mod.rs
@@ -76,8 +76,8 @@ impl Engine {
let telemetry = SharedAccess::new({
let context = TelemetryContext::new(
- env!("CARGO_PKG_VERSION").to_string(),
- uzu_engine::TOOLCHAIN_VERSION.to_string(),
+ Self::version(),
+ Self::toolchain_version(),
TelemetryDevice {
os_name: device.os_name.clone(),
cpu_name: device.cpu_name.clone(),
@@ -183,6 +183,14 @@ impl Engine {
Ok(engine)
}
+
+ pub fn toolchain_version() -> String {
+ uzu_engine::TOOLCHAIN_VERSION.to_string()
+ }
+
+ pub fn version() -> String {
+ uzu_engine::VERSION.to_string()
+ }
}
#[bindings::export(Implementation)]
diff --git a/crates/legacy/uzu/src/logs/mod.rs b/crates/legacy/uzu/src/logs/mod.rs
index c2ff29ce9..c3ac43d6d 100644
--- a/crates/legacy/uzu/src/logs/mod.rs
+++ b/crates/legacy/uzu/src/logs/mod.rs
@@ -20,6 +20,11 @@ pub fn start(
name: &str,
stdout: bool,
) {
+ // Applications can install their own tracing subscriber before creating an engine.
+ if tracing::dispatcher::has_been_set() {
+ return;
+ }
+
INIT.call_once(|| {
init(path, name, MAX_LOG_SIZE_BYTES, KEEP_LINES, stdout);
});