Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Add chat streaming (#219)

Alexander Verbitskyb20e0b3fd4e9
diff --git a/Cargo.lock b/Cargo.lock
index e725d5171..e66a95c0d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -568,6 +568,7 @@ dependencies = [
"comfy-table",
"console 0.16.2",
"ctrlc",
+ "futures",
"indicatif 0.18.3",
"inquire",
"rocket",
diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml
index 21c8a693f..95f7bf3ba 100644
--- a/crates/cli/Cargo.toml
+++ b/crates/cli/Cargo.toml
@@ -21,3 +21,4 @@ serde.workspace = true
serde_json.workspace = true
uuid.workspace = true
comfy-table.workspace = true
+futures.workspace = true
diff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rs
index 3972aaa5a..b24a6b765 100644
--- a/crates/cli/src/server/chat_completions.rs
+++ b/crates/cli/src/server/chat_completions.rs
@@ -1,7 +1,20 @@
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::{
+ cell::Cell,
+ pin::Pin,
+ sync::Arc,
+ task::{Context, Poll},
+ time::{SystemTime, UNIX_EPOCH},
+};
-use rocket::{State, post, serde::json::Json};
+use rocket::{
+ Request, State,
+ http::{ContentType, Header},
+ post,
+ response::{Responder, Response},
+ serde::json::Json,
+};
use serde::{Deserialize, Serialize};
+use tokio::io::{AsyncRead, ReadBuf};
use uuid::Uuid;
use uzu::session::{
config::RunConfig,
@@ -9,7 +22,10 @@ use uzu::session::{
types::{FinishReason, Input, Message, Output, Role, RunStats, Stats, StepStats, TotalStats},
};
-use crate::server::SessionState;
+use crate::server::{SessionState, SessionWrapper};
+
+const LOG_PREVIEW_RESPONSE_BYTES: usize = 3000;
+const LOG_PREVIEW_REQUEST_BYTES: usize = 10000;
#[derive(Serialize, Deserialize, Clone)]
pub struct ChatMessage {
@@ -21,6 +37,7 @@ pub struct ChatMessage {
pub struct ChatCompletionRequest {
pub messages: Vec<ChatMessage>,
pub max_completion_tokens: Option<u64>,
+ pub stream: Option<bool>,
}
#[derive(Serialize, Deserialize, Clone)]
@@ -30,6 +47,99 @@ pub struct ChatCompletionChoice {
pub finish_reason: String,
}
+#[derive(Serialize, Deserialize, Clone)]
+pub struct ChatCompletionUsage {
+ pub prompt_tokens: u64,
+ pub completion_tokens: u64,
+ pub total_tokens: u64,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct ChatCompletionTimings {
+ pub prompt_n: u64,
+ pub prompt_ms: f64,
+ pub prompt_per_token_ms: f64,
+ pub prompt_per_second: f64,
+ pub predicted_n: u64,
+ pub predicted_ms: f64,
+ pub predicted_per_token_ms: f64,
+ pub predicted_per_second: f64,
+}
+
+fn make_usage_and_timings(stats: &Stats) -> (ChatCompletionUsage, ChatCompletionTimings) {
+ let prompt_tokens = stats.total_stats.tokens_count_input;
+ let completion_tokens = stats.total_stats.tokens_count_output;
+ let prompt_ms = stats.prefill_stats.duration * 1000.0;
+ let predicted_ms = stats.generate_stats.as_ref().map(|s| s.duration * 1000.0).unwrap_or(0.0);
+ let usage = ChatCompletionUsage {
+ prompt_tokens,
+ completion_tokens,
+ total_tokens: prompt_tokens + completion_tokens,
+ };
+ let timings = ChatCompletionTimings {
+ prompt_n: prompt_tokens,
+ prompt_ms,
+ prompt_per_token_ms: if prompt_tokens > 0 {
+ prompt_ms / prompt_tokens as f64
+ } else {
+ 0.0
+ },
+ prompt_per_second: stats.prefill_stats.tokens_per_second,
+ predicted_n: completion_tokens,
+ predicted_ms,
+ predicted_per_token_ms: if completion_tokens > 0 {
+ predicted_ms / completion_tokens as f64
+ } else {
+ 0.0
+ },
+ predicted_per_second: stats.generate_stats.as_ref().map(|s| s.tokens_per_second).unwrap_or(0.0),
+ };
+ (usage, timings)
+}
+
+fn zero_usage_and_timings() -> (ChatCompletionUsage, ChatCompletionTimings) {
+ (
+ ChatCompletionUsage {
+ prompt_tokens: 0,
+ completion_tokens: 0,
+ total_tokens: 0,
+ },
+ ChatCompletionTimings {
+ prompt_n: 0,
+ prompt_ms: 0.0,
+ prompt_per_token_ms: 0.0,
+ prompt_per_second: 0.0,
+ predicted_n: 0,
+ predicted_ms: 0.0,
+ predicted_per_token_ms: 0.0,
+ predicted_per_second: 0.0,
+ },
+ )
+}
+
+fn zero_stats() -> Stats {
+ Stats {
+ prefill_stats: StepStats {
+ duration: 0.0,
+ suffix_length: 0,
+ tokens_count: 0,
+ tokens_per_second: 0.0,
+ processed_tokens_per_second: 0.0,
+ model_run: RunStats {
+ count: 0,
+ average_duration: 0.0,
+ },
+ run: None,
+ },
+ generate_stats: None,
+ total_stats: TotalStats {
+ duration: 0.0,
+ tokens_count_input: 0,
+ tokens_count_output: 0,
+ },
+ }
+}
+
#[derive(Serialize, Deserialize, Clone)]
pub struct ChatCompletionResponse {
pub id: String,
@@ -37,57 +147,245 @@ pub struct ChatCompletionResponse {
pub created: i64,
pub model: String,
pub choices: Vec<ChatCompletionChoice>,
+ pub usage: ChatCompletionUsage,
+ pub timings: ChatCompletionTimings,
pub stats: Stats,
}
-#[post("/chat/completions", format = "json", data = "<request>")]
-pub fn handle_chat_completions(
- request: Json<ChatCompletionRequest>,
- state: &State<SessionState>,
-) -> Json<ChatCompletionResponse> {
- let request = request.into_inner();
- let id = Uuid::new_v4().to_string();
+#[derive(Serialize)]
+struct StreamDelta {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ content: Option<String>,
+}
- println!("πŸ“¨ [{}] Incoming chat completion request:", id);
- println!(" Messages: {} message(s)", request.messages.len());
- println!(" Max tokens: {}", request.max_completion_tokens.unwrap_or(2048));
+#[derive(Serialize)]
+struct StreamChoice {
+ index: u32,
+ delta: StreamDelta,
+ finish_reason: Option<String>,
+}
- for (i, msg) in request.messages.iter().enumerate() {
- let content_preview = if msg.content.len() > 10000 {
- format!("{}...", &msg.content[..10000])
- } else {
- msg.content.clone()
- };
- println!(" Message {}: {:?} - {}", i + 1, msg.role, content_preview);
+#[derive(Serialize)]
+struct ChatCompletionChunk {
+ id: String,
+ object: String,
+ created: i64,
+ model: String,
+ choices: Vec<StreamChoice>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ usage: Option<ChatCompletionUsage>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ timings: Option<ChatCompletionTimings>,
+}
+
+struct StreamBody {
+ rx: tokio::sync::mpsc::Receiver<String>,
+ pending: Vec<u8>,
+}
+
+impl StreamBody {
+ fn new(rx: tokio::sync::mpsc::Receiver<String>) -> Self {
+ Self {
+ rx,
+ pending: Vec::new(),
+ }
}
+}
- let model_name = state.model_name.clone();
- let tokens_limit = request.max_completion_tokens.unwrap_or(2048);
- let messages: Vec<Message> = request
- .messages
- .into_iter()
- .map(|m| Message {
- content: m.content,
- role: m.role,
- reasoning_content: None,
- })
- .collect();
- let input = Input::Messages(messages);
- let run_config = RunConfig::new(tokens_limit, true, SamplingPolicy::Default, None);
+impl AsyncRead for StreamBody {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ out: &mut ReadBuf<'_>,
+ ) -> Poll<std::io::Result<()>> {
+ let this = self.get_mut();
+
+ // Drain any leftover bytes from the previous message first.
+ if !this.pending.is_empty() {
+ let n = this.pending.len().min(out.remaining());
+ out.put_slice(&this.pending[..n]);
+ this.pending.drain(..n);
+ return Poll::Ready(Ok(()));
+ }
+
+ // Try to pull the next event string from the channel.
+ match this.rx.poll_recv(cx) {
+ Poll::Ready(Some(data)) => {
+ let formatted = format!("data: {}\n\n", data);
+ let bytes = formatted.into_bytes();
+ let n = bytes.len().min(out.remaining());
+ out.put_slice(&bytes[..n]);
+ if n < bytes.len() {
+ this.pending = bytes[n..].to_vec();
+ }
+ Poll::Ready(Ok(()))
+ },
+ Poll::Ready(None) => Poll::Ready(Ok(())), // channel closed β†’ EOF
+ Poll::Pending => Poll::Pending,
+ }
+ }
+}
+
+struct AnyResponse(Response<'static>);
+
+impl<'r> Responder<'r, 'static> for AnyResponse {
+ fn respond_to(
+ self,
+ _: &'r Request<'_>,
+ ) -> rocket::response::Result<'static> {
+ Ok(self.0)
+ }
+}
+enum ChatCompletionResult {
+ Json(Json<ChatCompletionResponse>),
+ Stream(AnyResponse),
+}
+
+impl<'r> Responder<'r, 'static> for ChatCompletionResult {
+ fn respond_to(
+ self,
+ req: &'r Request<'_>,
+ ) -> rocket::response::Result<'static> {
+ match self {
+ ChatCompletionResult::Json(j) => j.respond_to(req),
+ ChatCompletionResult::Stream(a) => a.respond_to(req),
+ }
+ }
+}
+
+fn respond_stream(
+ id: String,
+ model_name: String,
+ input: Input,
+ run_config: RunConfig,
+ created: i64,
+ session_arc: Arc<SessionWrapper>,
+ tx: tokio::sync::mpsc::Sender<String>,
+) {
+ let id_for_cb = id.clone();
+ let model_for_cb = model_name.clone();
+ let tx_for_cb = tx.clone();
+ let prev_len = Cell::new(0usize);
+
+ let result = {
+ let mut session = session_arc.lock();
+ session.run(
+ input,
+ run_config,
+ Some(Box::new(move |output: Output| {
+ let text = &output.text.original;
+ let len = prev_len.get();
+ let char_start = (len..=text.len()).find(|&i| text.is_char_boundary(i)).unwrap_or(text.len());
+ if text.len() > char_start {
+ let delta = text[char_start..].to_string();
+ prev_len.set(text.len());
+ let chunk = ChatCompletionChunk {
+ id: id_for_cb.clone(),
+ object: "chat.completion.chunk".to_string(),
+ created,
+ model: model_for_cb.clone(),
+ choices: vec![StreamChoice {
+ index: 0,
+ delta: StreamDelta {
+ content: Some(delta),
+ },
+ finish_reason: None,
+ }],
+ usage: None,
+ timings: None,
+ };
+ if let Ok(json) = serde_json::to_string(&chunk) {
+ if tx_for_cb.blocking_send(json).is_err() {
+ return false; // client disconnected, stop generation
+ }
+ }
+ }
+ true
+ })),
+ )
+ };
+
+ let (finish_reason, final_usage, final_timings) = match &result {
+ Ok(output) => {
+ let fr = output
+ .finish_reason
+ .as_ref()
+ .map(|fr| format!("{:?}", fr).to_lowercase())
+ .unwrap_or_else(|| "stop".to_string());
+ let (usage, timings) = make_usage_and_timings(&output.stats);
+ (fr, Some(usage), Some(timings))
+ },
+ Err(e) => {
+ eprintln!("❌ [{}] Stream session run error: {}", id, e);
+ let error_chunk = ChatCompletionChunk {
+ id: id.clone(),
+ object: "chat.completion.chunk".to_string(),
+ created,
+ model: model_name.clone(),
+ choices: vec![StreamChoice {
+ index: 0,
+ delta: StreamDelta {
+ content: Some(format!("Error: {}", e)),
+ },
+ finish_reason: None,
+ }],
+ usage: None,
+ timings: None,
+ };
+ if let Ok(json) = serde_json::to_string(&error_chunk) {
+ tx.blocking_send(json).ok();
+ }
+ let (usage, timings) = zero_usage_and_timings();
+ ("error".to_string(), Some(usage), Some(timings))
+ },
+ };
+
+ println!("πŸ“€ [{}] Stream finished, finish_reason={}", id, finish_reason);
+
+ let final_chunk = ChatCompletionChunk {
+ id,
+ object: "chat.completion.chunk".to_string(),
+ created,
+ model: model_name,
+ choices: vec![StreamChoice {
+ index: 0,
+ delta: StreamDelta {
+ content: None,
+ },
+ finish_reason: Some(finish_reason),
+ }],
+ usage: final_usage,
+ timings: final_timings,
+ };
+ if let Ok(json) = serde_json::to_string(&final_chunk) {
+ tx.blocking_send(json).ok();
+ }
+ tx.blocking_send("[DONE]".to_string()).ok();
+ // tx dropped here β†’ channel closed β†’ StreamBody reaches EOF
+}
+
+fn respond_blocking(
+ id: String,
+ model_name: String,
+ input: Input,
+ run_config: RunConfig,
+ created: i64,
+ session_wrapper: &SessionWrapper,
+) -> ChatCompletionResult {
let start_time = std::time::Instant::now();
- let mut session = state.session_wrapper.lock();
+ let mut session = session_wrapper.lock();
- let output = match session.run(input, run_config, None::<fn(Output) -> bool>) {
+ let output = match session.run(input, run_config, None) {
Ok(output) => output,
Err(e) => {
eprintln!("❌ [{}] Session run error: {}", id, e);
- // Return error response
+ let (usage, timings) = zero_usage_and_timings();
let error_response = ChatCompletionResponse {
- id: id.clone(),
+ id,
object: "chat.completion".to_string(),
- created: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64,
- model: model_name.clone(),
+ created,
+ model: model_name,
choices: vec![ChatCompletionChoice {
index: 0,
message: ChatMessage {
@@ -96,28 +394,11 @@ pub fn handle_chat_completions(
},
finish_reason: "error".to_string(),
}],
- stats: Stats {
- prefill_stats: StepStats {
- duration: 0.0,
- suffix_length: 0,
- tokens_count: 0,
- tokens_per_second: 0.0,
- processed_tokens_per_second: 0.0,
- model_run: RunStats {
- count: 0,
- average_duration: 0.0,
- },
- run: None,
- },
- generate_stats: None,
- total_stats: TotalStats {
- duration: 0.0,
- tokens_count_input: 0,
- tokens_count_output: 0,
- },
- },
+ usage,
+ timings,
+ stats: zero_stats(),
};
- return Json(error_response);
+ return ChatCompletionResult::Json(Json(error_response));
},
};
@@ -131,7 +412,6 @@ pub fn handle_chat_completions(
finish_reason,
..
} = output;
-
let finish_reason_val = finish_reason.unwrap_or(FinishReason::Cancelled);
println!("πŸ“€ [{}] Sending response:", id);
@@ -140,20 +420,21 @@ pub fn handle_chat_completions(
println!(" Processing time: {:.3}s", processing_time.as_secs_f64());
println!(" Tokens/second: {:.1}", stats.generate_stats.as_ref().map(|s| s.tokens_per_second).unwrap_or(0.0));
+ let text = text.original;
print!(" Response preview: ");
- let text = text.original.clone();
- if text.len() > 3000 {
- print!("{}", &text[..3000]);
+ if text.len() > LOG_PREVIEW_RESPONSE_BYTES {
+ print!("{}", &text[..text.floor_char_boundary(LOG_PREVIEW_RESPONSE_BYTES)]);
println!("...");
} else {
println!("{}", &text);
}
println!();
+ let (usage, timings) = make_usage_and_timings(&stats);
let response = ChatCompletionResponse {
- id: id.clone(),
+ id,
object: "chat.completion".to_string(),
- created: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64,
+ created,
model: model_name,
choices: vec![ChatCompletionChoice {
index: 0,
@@ -163,8 +444,89 @@ pub fn handle_chat_completions(
},
finish_reason: format!("{:?}", finish_reason_val).to_lowercase(),
}],
+ usage,
+ timings,
stats,
};
- Json(response)
+ ChatCompletionResult::Json(Json(response))
+}
+
+#[allow(private_interfaces)]
+#[post("/chat/completions", format = "json", data = "<request>")]
+pub async fn handle_chat_completions(
+ request: Json<ChatCompletionRequest>,
+ state: &State<SessionState>,
+) -> ChatCompletionResult {
+ let request = request.into_inner();
+ let id = Uuid::new_v4().to_string();
+ let is_stream = request.stream.unwrap_or(false);
+
+ println!("πŸ“¨ [{}] Incoming chat completion request (stream={}):", id, is_stream);
+ println!(" Messages: {} message(s)", request.messages.len());
+ println!(" Max tokens: {}", request.max_completion_tokens.unwrap_or(2048));
+
+ for (i, msg) in request.messages.iter().enumerate() {
+ let content_preview = if msg.content.len() > LOG_PREVIEW_REQUEST_BYTES {
+ format!("{}...", &msg.content[..msg.content.floor_char_boundary(LOG_PREVIEW_REQUEST_BYTES)])
+ } else {
+ msg.content.clone()
+ };
+ println!(" Message {}: {:?} - {}", i + 1, msg.role, content_preview);
+ }
+
+ let model_name = state.model_name.clone();
+ let tokens_limit = request.max_completion_tokens.unwrap_or(2048);
+ let input = Input::Messages(
+ request
+ .messages
+ .into_iter()
+ .map(|m| Message {
+ content: m.content,
+ role: m.role,
+ reasoning_content: None,
+ })
+ .collect(),
+ );
+ let run_config = RunConfig::new(tokens_limit, true, SamplingPolicy::Default, None);
+ let created = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64;
+
+ let session_arc = Arc::clone(&state.session_wrapper);
+
+ if is_stream {
+ let (tx, rx) = tokio::sync::mpsc::channel::<String>(32);
+ let tx_for_panic = tx.clone();
+ tokio::task::spawn_blocking(move || {
+ let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+ respond_stream(id, model_name, input, run_config, created, session_arc, tx);
+ }));
+ if result.is_err() {
+ // Ensure stream closes cleanly even after panic
+ tx_for_panic.blocking_send("[DONE]".to_string()).ok();
+ }
+ });
+ let response: Response<'static> = Response::build()
+ .header(ContentType::new("text", "event-stream"))
+ .header(Header::new("Cache-Control", "no-cache"))
+ .header(Header::new("X-Accel-Buffering", "no"))
+ .streamed_body(StreamBody::new(rx))
+ .finalize();
+ ChatCompletionResult::Stream(AnyResponse(response))
+ } else {
+ tokio::task::spawn_blocking(move || respond_blocking(id, model_name, input, run_config, created, &session_arc))
+ .await
+ .unwrap_or_else(|_| {
+ let (usage, timings) = zero_usage_and_timings();
+ ChatCompletionResult::Json(Json(ChatCompletionResponse {
+ id: "error".to_string(),
+ object: "chat.completion".to_string(),
+ created,
+ model: state.model_name.clone(),
+ choices: vec![],
+ usage,
+ timings,
+ stats: zero_stats(),
+ }))
+ })
+ }
}
diff --git a/crates/cli/src/server/main.rs b/crates/cli/src/server/main.rs
index 1b2ddc7dc..96ab9f75a 100644
--- a/crates/cli/src/server/main.rs
+++ b/crates/cli/src/server/main.rs
@@ -46,7 +46,7 @@ pub async fn run_server(
let session = load_session(model_path, prefill_step_size, None);
let state = SessionState {
model_name,
- session_wrapper: SessionWrapper::new(session),
+ session_wrapper: std::sync::Arc::new(SessionWrapper::new(session)),
};
let _ =
diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rs
index 117abcae2..e40a9e440 100644
--- a/crates/cli/src/server/state.rs
+++ b/crates/cli/src/server/state.rs
@@ -1,33 +1,60 @@
-use std::{path::PathBuf, sync::Mutex};
+use std::{
+ path::PathBuf,
+ sync::{Arc, Mutex},
+};
use console::Style;
use indicatif::{ProgressBar, ProgressStyle};
use uzu::{
prelude::SamplingSeed,
- session::{Session, config::DecodingConfig, parameter::PrefillStepSize},
+ session::{
+ Session,
+ config::{DecodingConfig, RunConfig},
+ parameter::PrefillStepSize,
+ types::{Error, Input, Output},
+ },
};
-pub struct SessionWrapper(Mutex<Session>);
+pub trait RunSession {
+ fn run(
+ &mut self,
+ input: Input,
+ config: RunConfig,
+ progress: Option<Box<dyn Fn(Output) -> bool>>,
+ ) -> Result<Output, Error>;
+}
+
+impl RunSession for Session {
+ fn run(
+ &mut self,
+ input: Input,
+ config: RunConfig,
+ progress: Option<Box<dyn Fn(Output) -> bool>>,
+ ) -> Result<Output, Error> {
+ Session::run(self, input, config, progress)
+ }
+}
+
+pub struct SessionWrapper(Mutex<Box<dyn RunSession>>);
+
unsafe impl Send for SessionWrapper {}
unsafe impl Sync for SessionWrapper {}
+
impl SessionWrapper {
- pub fn new(session: Session) -> Self {
- Self(Mutex::new(session))
+ pub fn new(runner: impl RunSession + 'static) -> Self {
+ Self(Mutex::new(Box::new(runner)))
}
- pub fn lock(&self) -> std::sync::MutexGuard<'_, Session> {
+ pub fn lock(&self) -> std::sync::MutexGuard<'_, Box<dyn RunSession>> {
self.0.lock().unwrap()
}
}
pub struct SessionState {
pub model_name: String,
- pub session_wrapper: SessionWrapper,
+ pub session_wrapper: Arc<SessionWrapper>,
}
-unsafe impl Send for SessionState {}
-unsafe impl Sync for SessionState {}
-
pub fn load_session(
model_path: String,
prefill_step_size: Option<usize>,
diff --git a/crates/cli/tests/chat_completions.rs b/crates/cli/tests/chat_completions.rs
new file mode 100644
index 000000000..6a36c049a
--- /dev/null
+++ b/crates/cli/tests/chat_completions.rs
@@ -0,0 +1,237 @@
+use std::sync::Arc;
+
+use cli::server::{
+ handle_chat_completions,
+ state::{RunSession, SessionState, SessionWrapper},
+};
+use rocket::{http::Status, local::asynchronous::Client, routes};
+use serde_json::{Value, json};
+use uzu::session::{
+ config::RunConfig,
+ types::{Error, FinishReason, Input, Output, ParsedText, RunStats, Stats, StepStats, Text, TotalStats},
+};
+
+struct MockSession {
+ result: Result<String, Error>,
+}
+
+impl RunSession for MockSession {
+ fn run(
+ &mut self,
+ _input: Input,
+ _config: RunConfig,
+ _progress: Option<Box<dyn Fn(Output) -> bool>>,
+ ) -> Result<Output, Error> {
+ self.result.as_ref().map_err(|_| Error::GenerateFailed).map(|text| Output {
+ text: Text {
+ original: text.clone(),
+ parsed: ParsedText {
+ chain_of_thought: None,
+ response: Some(text.clone()),
+ },
+ },
+ finish_reason: Some(FinishReason::Stop),
+ stats: Stats {
+ prefill_stats: StepStats {
+ duration: 0.0,
+ suffix_length: 0,
+ tokens_count: 0,
+ tokens_per_second: 0.0,
+ processed_tokens_per_second: 0.0,
+ model_run: RunStats {
+ count: 0,
+ average_duration: 0.0,
+ },
+ run: None,
+ },
+ generate_stats: None,
+ total_stats: TotalStats {
+ duration: 0.0,
+ tokens_count_input: 0,
+ tokens_count_output: 0,
+ },
+ },
+ })
+ }
+}
+
+async fn make_client(mock: MockSession) -> Client {
+ let state = SessionState {
+ model_name: "test-model".to_string(),
+ session_wrapper: Arc::new(SessionWrapper::new(mock)),
+ };
+ let rocket = rocket::build().manage(state).mount("/v1", routes![handle_chat_completions]);
+ Client::tracked(rocket).await.expect("valid rocket instance")
+}
+
+#[rocket::async_test]
+async fn test_non_streaming_success() {
+ let client = make_client(MockSession {
+ result: Ok("Hello, world!".to_string()),
+ })
+ .await;
+
+ let response = client
+ .post("/v1/chat/completions")
+ .json(&json!({
+ "messages": [{"role": "user", "content": "Hi"}],
+ "stream": false
+ }))
+ .dispatch()
+ .await;
+
+ assert_eq!(response.status(), Status::Ok);
+ let body: Value = response.into_json().await.expect("valid json");
+ assert_eq!(body["choices"][0]["message"]["content"], "Hello, world!");
+ assert_eq!(body["choices"][0]["finish_reason"], "stop");
+ assert_eq!(body["model"], "test-model");
+}
+
+#[rocket::async_test]
+async fn test_non_streaming_session_error() {
+ let client = make_client(MockSession {
+ result: Err(Error::GenerateFailed),
+ })
+ .await;
+
+ let response = client
+ .post("/v1/chat/completions")
+ .json(&json!({
+ "messages": [{"role": "user", "content": "Hi"}],
+ "stream": false
+ }))
+ .dispatch()
+ .await;
+
+ assert_eq!(response.status(), Status::Ok);
+ let body: Value = response.into_json().await.expect("valid json");
+ assert_eq!(body["choices"][0]["finish_reason"], "error");
+ assert!(
+ body["choices"][0]["message"]["content"].as_str().unwrap().contains("Generate failed"),
+ "expected error text in content"
+ );
+}
+
+#[rocket::async_test]
+async fn test_streaming_content_type() {
+ let client = make_client(MockSession {
+ result: Ok("Stream me".to_string()),
+ })
+ .await;
+
+ let response = client
+ .post("/v1/chat/completions")
+ .json(&json!({
+ "messages": [{"role": "user", "content": "Hi"}],
+ "stream": true
+ }))
+ .dispatch()
+ .await;
+
+ assert_eq!(response.status(), Status::Ok);
+ assert_eq!(response.content_type().map(|ct| ct.to_string()), Some("text/event-stream".to_string()));
+}
+
+#[rocket::async_test]
+async fn test_streaming_done_sentinel() {
+ let client = make_client(MockSession {
+ result: Ok("Stream me".to_string()),
+ })
+ .await;
+
+ let body = client
+ .post("/v1/chat/completions")
+ .json(&json!({
+ "messages": [{"role": "user", "content": "Hi"}],
+ "stream": true
+ }))
+ .dispatch()
+ .await
+ .into_string()
+ .await
+ .expect("body");
+
+ assert!(body.contains("data: [DONE]"), "missing [DONE] sentinel:\n{body}");
+
+ for line in body.lines().filter(|l| !l.is_empty()) {
+ assert!(line.starts_with("data: "), "unexpected SSE line: {line}");
+ }
+}
+
+#[rocket::async_test]
+async fn test_streaming_error_in_stream() {
+ let client = make_client(MockSession {
+ result: Err(Error::GenerateFailed),
+ })
+ .await;
+
+ let body = client
+ .post("/v1/chat/completions")
+ .json(&json!({
+ "messages": [{"role": "user", "content": "Hi"}],
+ "stream": true
+ }))
+ .dispatch()
+ .await
+ .into_string()
+ .await
+ .expect("body");
+
+ assert!(body.contains("Generate failed"), "expected error text in stream:\n{body}");
+ assert!(body.contains(r#""finish_reason":"error""#), "expected finish_reason error:\n{body}");
+ assert!(body.contains("data: [DONE]"), "stream must still terminate:\n{body}");
+}
+
+/// Builds a string of exactly `prefix_bytes` ASCII bytes followed by a
+/// 2-byte UTF-8 character (Cyrillic 'Π°', U+0430), so that a byte slice at
+/// `prefix_bytes` would cut the character in half.
+fn utf8_trap(prefix_bytes: usize) -> String {
+ let mut s = "a".repeat(prefix_bytes);
+ s.push('Π°'); // U+0430: 0xD0 0xB0 β€” 2 bytes
+ s
+}
+
+#[rocket::async_test]
+async fn test_request_content_utf8_boundary_does_not_panic() {
+ // Content is 10001 bytes; byte 10000 is the second byte of 'Π°'.
+ // &content[..10000] will panic with a byte-boundary assertion.
+ let content = utf8_trap(9999);
+ assert_eq!(content.len(), 10001);
+
+ let client = make_client(MockSession {
+ result: Ok("ok".to_string()),
+ })
+ .await;
+ let response = client
+ .post("/v1/chat/completions")
+ .json(&json!({
+ "messages": [{"role": "user", "content": content}],
+ "stream": false
+ }))
+ .dispatch()
+ .await;
+
+ assert_eq!(response.status(), Status::Ok, "handler must not panic on multibyte content");
+}
+
+#[rocket::async_test]
+async fn test_response_text_utf8_boundary_does_not_panic() {
+ // Session returns 3001-byte text where byte 3000 is mid-char.
+ let text = utf8_trap(2999);
+ assert_eq!(text.len(), 3001);
+
+ let client = make_client(MockSession {
+ result: Ok(text),
+ })
+ .await;
+ let response = client
+ .post("/v1/chat/completions")
+ .json(&json!({
+ "messages": [{"role": "user", "content": "Hi"}],
+ "stream": false
+ }))
+ .dispatch()
+ .await;
+
+ assert_eq!(response.status(), Status::Ok, "handler must not panic on multibyte response");
+}