A high-performance inference engine for AI models
Add chat streaming (#219)
Alexander Verbitskyb20e0b3fd4e9
diff --git a/Cargo.lock b/Cargo.lockindex e725d5171..e66a95c0d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -568,6 +568,7 @@ dependencies = ["comfy-table","console 0.16.2","ctrlc",+ "futures","indicatif 0.18.3","inquire","rocket",diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.tomlindex 21c8a693f..95f7bf3ba 100644--- a/crates/cli/Cargo.toml+++ b/crates/cli/Cargo.toml@@ -21,3 +21,4 @@ serde.workspace = trueserde_json.workspace = trueuuid.workspace = truecomfy-table.workspace = true+futures.workspace = truediff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rsindex 3972aaa5a..b24a6b765 100644--- a/crates/cli/src/server/chat_completions.rs+++ b/crates/cli/src/server/chat_completions.rs@@ -1,7 +1,20 @@-use std::time::{SystemTime, UNIX_EPOCH};+use std::{+ cell::Cell,+ pin::Pin,+ sync::Arc,+ task::{Context, Poll},+ time::{SystemTime, UNIX_EPOCH},+};-use rocket::{State, post, serde::json::Json};+use rocket::{+ Request, State,+ http::{ContentType, Header},+ post,+ response::{Responder, Response},+ serde::json::Json,+};use serde::{Deserialize, Serialize};+use tokio::io::{AsyncRead, ReadBuf};use uuid::Uuid;use uzu::session::{config::RunConfig,@@ -9,7 +22,10 @@ use uzu::session::{types::{FinishReason, Input, Message, Output, Role, RunStats, Stats, StepStats, TotalStats},};-use crate::server::SessionState;+use crate::server::{SessionState, SessionWrapper};++const LOG_PREVIEW_RESPONSE_BYTES: usize = 3000;+const LOG_PREVIEW_REQUEST_BYTES: usize = 10000;#[derive(Serialize, Deserialize, Clone)]pub struct ChatMessage {@@ -21,6 +37,7 @@ pub struct ChatMessage {pub struct ChatCompletionRequest {pub messages: Vec<ChatMessage>,pub max_completion_tokens: Option<u64>,+ pub stream: Option<bool>,}#[derive(Serialize, Deserialize, Clone)]@@ -30,6 +47,99 @@ pub struct ChatCompletionChoice {pub finish_reason: String,}+#[derive(Serialize, Deserialize, Clone)]+pub struct ChatCompletionUsage {+ pub prompt_tokens: u64,+ pub completion_tokens: u64,+ pub total_tokens: u64,+}++#[derive(Serialize, Deserialize, Clone)]+pub struct ChatCompletionTimings {+ pub prompt_n: u64,+ pub prompt_ms: f64,+ pub prompt_per_token_ms: f64,+ pub prompt_per_second: f64,+ pub predicted_n: u64,+ pub predicted_ms: f64,+ pub predicted_per_token_ms: f64,+ pub predicted_per_second: f64,+}++fn make_usage_and_timings(stats: &Stats) -> (ChatCompletionUsage, ChatCompletionTimings) {+ let prompt_tokens = stats.total_stats.tokens_count_input;+ let completion_tokens = stats.total_stats.tokens_count_output;+ let prompt_ms = stats.prefill_stats.duration * 1000.0;+ let predicted_ms = stats.generate_stats.as_ref().map(|s| s.duration * 1000.0).unwrap_or(0.0);+ let usage = ChatCompletionUsage {+ prompt_tokens,+ completion_tokens,+ total_tokens: prompt_tokens + completion_tokens,+ };+ let timings = ChatCompletionTimings {+ prompt_n: prompt_tokens,+ prompt_ms,+ prompt_per_token_ms: if prompt_tokens > 0 {+ prompt_ms / prompt_tokens as f64+ } else {+ 0.0+ },+ prompt_per_second: stats.prefill_stats.tokens_per_second,+ predicted_n: completion_tokens,+ predicted_ms,+ predicted_per_token_ms: if completion_tokens > 0 {+ predicted_ms / completion_tokens as f64+ } else {+ 0.0+ },+ predicted_per_second: stats.generate_stats.as_ref().map(|s| s.tokens_per_second).unwrap_or(0.0),+ };+ (usage, timings)+}++fn zero_usage_and_timings() -> (ChatCompletionUsage, ChatCompletionTimings) {+ (+ ChatCompletionUsage {+ prompt_tokens: 0,+ completion_tokens: 0,+ total_tokens: 0,+ },+ ChatCompletionTimings {+ prompt_n: 0,+ prompt_ms: 0.0,+ prompt_per_token_ms: 0.0,+ prompt_per_second: 0.0,+ predicted_n: 0,+ predicted_ms: 0.0,+ predicted_per_token_ms: 0.0,+ predicted_per_second: 0.0,+ },+ )+}++fn zero_stats() -> Stats {+ Stats {+ prefill_stats: StepStats {+ duration: 0.0,+ suffix_length: 0,+ tokens_count: 0,+ tokens_per_second: 0.0,+ processed_tokens_per_second: 0.0,+ model_run: RunStats {+ count: 0,+ average_duration: 0.0,+ },+ run: None,+ },+ generate_stats: None,+ total_stats: TotalStats {+ duration: 0.0,+ tokens_count_input: 0,+ tokens_count_output: 0,+ },+ }+}+#[derive(Serialize, Deserialize, Clone)]pub struct ChatCompletionResponse {pub id: String,@@ -37,57 +147,245 @@ pub struct ChatCompletionResponse {pub created: i64,pub model: String,pub choices: Vec<ChatCompletionChoice>,+ pub usage: ChatCompletionUsage,+ pub timings: ChatCompletionTimings,pub stats: Stats,}-#[post("/chat/completions", format = "json", data = "<request>")]-pub fn handle_chat_completions(- request: Json<ChatCompletionRequest>,- state: &State<SessionState>,-) -> Json<ChatCompletionResponse> {- let request = request.into_inner();- let id = Uuid::new_v4().to_string();+#[derive(Serialize)]+struct StreamDelta {+ #[serde(skip_serializing_if = "Option::is_none")]+ content: Option<String>,+}- println!("π¨ [{}] Incoming chat completion request:", id);- println!(" Messages: {} message(s)", request.messages.len());- println!(" Max tokens: {}", request.max_completion_tokens.unwrap_or(2048));+#[derive(Serialize)]+struct StreamChoice {+ index: u32,+ delta: StreamDelta,+ finish_reason: Option<String>,+}- for (i, msg) in request.messages.iter().enumerate() {- let content_preview = if msg.content.len() > 10000 {- format!("{}...", &msg.content[..10000])- } else {- msg.content.clone()- };- println!(" Message {}: {:?} - {}", i + 1, msg.role, content_preview);+#[derive(Serialize)]+struct ChatCompletionChunk {+ id: String,+ object: String,+ created: i64,+ model: String,+ choices: Vec<StreamChoice>,+ #[serde(skip_serializing_if = "Option::is_none")]+ usage: Option<ChatCompletionUsage>,+ #[serde(skip_serializing_if = "Option::is_none")]+ timings: Option<ChatCompletionTimings>,+}++struct StreamBody {+ rx: tokio::sync::mpsc::Receiver<String>,+ pending: Vec<u8>,+}++impl StreamBody {+ fn new(rx: tokio::sync::mpsc::Receiver<String>) -> Self {+ Self {+ rx,+ pending: Vec::new(),+ }}+}- let model_name = state.model_name.clone();- let tokens_limit = request.max_completion_tokens.unwrap_or(2048);- let messages: Vec<Message> = request- .messages- .into_iter()- .map(|m| Message {- content: m.content,- role: m.role,- reasoning_content: None,- })- .collect();- let input = Input::Messages(messages);- let run_config = RunConfig::new(tokens_limit, true, SamplingPolicy::Default, None);+impl AsyncRead for StreamBody {+ fn poll_read(+ self: Pin<&mut Self>,+ cx: &mut Context<'_>,+ out: &mut ReadBuf<'_>,+ ) -> Poll<std::io::Result<()>> {+ let this = self.get_mut();++ // Drain any leftover bytes from the previous message first.+ if !this.pending.is_empty() {+ let n = this.pending.len().min(out.remaining());+ out.put_slice(&this.pending[..n]);+ this.pending.drain(..n);+ return Poll::Ready(Ok(()));+ }++ // Try to pull the next event string from the channel.+ match this.rx.poll_recv(cx) {+ Poll::Ready(Some(data)) => {+ let formatted = format!("data: {}\n\n", data);+ let bytes = formatted.into_bytes();+ let n = bytes.len().min(out.remaining());+ out.put_slice(&bytes[..n]);+ if n < bytes.len() {+ this.pending = bytes[n..].to_vec();+ }+ Poll::Ready(Ok(()))+ },+ Poll::Ready(None) => Poll::Ready(Ok(())), // channel closed β EOF+ Poll::Pending => Poll::Pending,+ }+ }+}++struct AnyResponse(Response<'static>);++impl<'r> Responder<'r, 'static> for AnyResponse {+ fn respond_to(+ self,+ _: &'r Request<'_>,+ ) -> rocket::response::Result<'static> {+ Ok(self.0)+ }+}+enum ChatCompletionResult {+ Json(Json<ChatCompletionResponse>),+ Stream(AnyResponse),+}++impl<'r> Responder<'r, 'static> for ChatCompletionResult {+ fn respond_to(+ self,+ req: &'r Request<'_>,+ ) -> rocket::response::Result<'static> {+ match self {+ ChatCompletionResult::Json(j) => j.respond_to(req),+ ChatCompletionResult::Stream(a) => a.respond_to(req),+ }+ }+}++fn respond_stream(+ id: String,+ model_name: String,+ input: Input,+ run_config: RunConfig,+ created: i64,+ session_arc: Arc<SessionWrapper>,+ tx: tokio::sync::mpsc::Sender<String>,+) {+ let id_for_cb = id.clone();+ let model_for_cb = model_name.clone();+ let tx_for_cb = tx.clone();+ let prev_len = Cell::new(0usize);++ let result = {+ let mut session = session_arc.lock();+ session.run(+ input,+ run_config,+ Some(Box::new(move |output: Output| {+ let text = &output.text.original;+ let len = prev_len.get();+ let char_start = (len..=text.len()).find(|&i| text.is_char_boundary(i)).unwrap_or(text.len());+ if text.len() > char_start {+ let delta = text[char_start..].to_string();+ prev_len.set(text.len());+ let chunk = ChatCompletionChunk {+ id: id_for_cb.clone(),+ object: "chat.completion.chunk".to_string(),+ created,+ model: model_for_cb.clone(),+ choices: vec![StreamChoice {+ index: 0,+ delta: StreamDelta {+ content: Some(delta),+ },+ finish_reason: None,+ }],+ usage: None,+ timings: None,+ };+ if let Ok(json) = serde_json::to_string(&chunk) {+ if tx_for_cb.blocking_send(json).is_err() {+ return false; // client disconnected, stop generation+ }+ }+ }+ true+ })),+ )+ };++ let (finish_reason, final_usage, final_timings) = match &result {+ Ok(output) => {+ let fr = output+ .finish_reason+ .as_ref()+ .map(|fr| format!("{:?}", fr).to_lowercase())+ .unwrap_or_else(|| "stop".to_string());+ let (usage, timings) = make_usage_and_timings(&output.stats);+ (fr, Some(usage), Some(timings))+ },+ Err(e) => {+ eprintln!("β [{}] Stream session run error: {}", id, e);+ let error_chunk = ChatCompletionChunk {+ id: id.clone(),+ object: "chat.completion.chunk".to_string(),+ created,+ model: model_name.clone(),+ choices: vec![StreamChoice {+ index: 0,+ delta: StreamDelta {+ content: Some(format!("Error: {}", e)),+ },+ finish_reason: None,+ }],+ usage: None,+ timings: None,+ };+ if let Ok(json) = serde_json::to_string(&error_chunk) {+ tx.blocking_send(json).ok();+ }+ let (usage, timings) = zero_usage_and_timings();+ ("error".to_string(), Some(usage), Some(timings))+ },+ };++ println!("π€ [{}] Stream finished, finish_reason={}", id, finish_reason);++ let final_chunk = ChatCompletionChunk {+ id,+ object: "chat.completion.chunk".to_string(),+ created,+ model: model_name,+ choices: vec![StreamChoice {+ index: 0,+ delta: StreamDelta {+ content: None,+ },+ finish_reason: Some(finish_reason),+ }],+ usage: final_usage,+ timings: final_timings,+ };+ if let Ok(json) = serde_json::to_string(&final_chunk) {+ tx.blocking_send(json).ok();+ }+ tx.blocking_send("[DONE]".to_string()).ok();+ // tx dropped here β channel closed β StreamBody reaches EOF+}++fn respond_blocking(+ id: String,+ model_name: String,+ input: Input,+ run_config: RunConfig,+ created: i64,+ session_wrapper: &SessionWrapper,+) -> ChatCompletionResult {let start_time = std::time::Instant::now();- let mut session = state.session_wrapper.lock();+ let mut session = session_wrapper.lock();- let output = match session.run(input, run_config, None::<fn(Output) -> bool>) {+ let output = match session.run(input, run_config, None) {Ok(output) => output,Err(e) => {eprintln!("β [{}] Session run error: {}", id, e);- // Return error response+ let (usage, timings) = zero_usage_and_timings();let error_response = ChatCompletionResponse {- id: id.clone(),+ id,object: "chat.completion".to_string(),- created: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64,- model: model_name.clone(),+ created,+ model: model_name,choices: vec![ChatCompletionChoice {index: 0,message: ChatMessage {@@ -96,28 +394,11 @@ pub fn handle_chat_completions(},finish_reason: "error".to_string(),}],- stats: Stats {- prefill_stats: StepStats {- duration: 0.0,- suffix_length: 0,- tokens_count: 0,- tokens_per_second: 0.0,- processed_tokens_per_second: 0.0,- model_run: RunStats {- count: 0,- average_duration: 0.0,- },- run: None,- },- generate_stats: None,- total_stats: TotalStats {- duration: 0.0,- tokens_count_input: 0,- tokens_count_output: 0,- },- },+ usage,+ timings,+ stats: zero_stats(),};- return Json(error_response);+ return ChatCompletionResult::Json(Json(error_response));},};@@ -131,7 +412,6 @@ pub fn handle_chat_completions(finish_reason,..} = output;-let finish_reason_val = finish_reason.unwrap_or(FinishReason::Cancelled);println!("π€ [{}] Sending response:", id);@@ -140,20 +420,21 @@ pub fn handle_chat_completions(println!(" Processing time: {:.3}s", processing_time.as_secs_f64());println!(" Tokens/second: {:.1}", stats.generate_stats.as_ref().map(|s| s.tokens_per_second).unwrap_or(0.0));+ let text = text.original;print!(" Response preview: ");- let text = text.original.clone();- if text.len() > 3000 {- print!("{}", &text[..3000]);+ if text.len() > LOG_PREVIEW_RESPONSE_BYTES {+ print!("{}", &text[..text.floor_char_boundary(LOG_PREVIEW_RESPONSE_BYTES)]);println!("...");} else {println!("{}", &text);}println!();+ let (usage, timings) = make_usage_and_timings(&stats);let response = ChatCompletionResponse {- id: id.clone(),+ id,object: "chat.completion".to_string(),- created: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64,+ created,model: model_name,choices: vec![ChatCompletionChoice {index: 0,@@ -163,8 +444,89 @@ pub fn handle_chat_completions(},finish_reason: format!("{:?}", finish_reason_val).to_lowercase(),}],+ usage,+ timings,stats,};- Json(response)+ ChatCompletionResult::Json(Json(response))+}++#[allow(private_interfaces)]+#[post("/chat/completions", format = "json", data = "<request>")]+pub async fn handle_chat_completions(+ request: Json<ChatCompletionRequest>,+ state: &State<SessionState>,+) -> ChatCompletionResult {+ let request = request.into_inner();+ let id = Uuid::new_v4().to_string();+ let is_stream = request.stream.unwrap_or(false);++ println!("π¨ [{}] Incoming chat completion request (stream={}):", id, is_stream);+ println!(" Messages: {} message(s)", request.messages.len());+ println!(" Max tokens: {}", request.max_completion_tokens.unwrap_or(2048));++ for (i, msg) in request.messages.iter().enumerate() {+ let content_preview = if msg.content.len() > LOG_PREVIEW_REQUEST_BYTES {+ format!("{}...", &msg.content[..msg.content.floor_char_boundary(LOG_PREVIEW_REQUEST_BYTES)])+ } else {+ msg.content.clone()+ };+ println!(" Message {}: {:?} - {}", i + 1, msg.role, content_preview);+ }++ let model_name = state.model_name.clone();+ let tokens_limit = request.max_completion_tokens.unwrap_or(2048);+ let input = Input::Messages(+ request+ .messages+ .into_iter()+ .map(|m| Message {+ content: m.content,+ role: m.role,+ reasoning_content: None,+ })+ .collect(),+ );+ let run_config = RunConfig::new(tokens_limit, true, SamplingPolicy::Default, None);+ let created = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64;++ let session_arc = Arc::clone(&state.session_wrapper);++ if is_stream {+ let (tx, rx) = tokio::sync::mpsc::channel::<String>(32);+ let tx_for_panic = tx.clone();+ tokio::task::spawn_blocking(move || {+ let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {+ respond_stream(id, model_name, input, run_config, created, session_arc, tx);+ }));+ if result.is_err() {+ // Ensure stream closes cleanly even after panic+ tx_for_panic.blocking_send("[DONE]".to_string()).ok();+ }+ });+ let response: Response<'static> = Response::build()+ .header(ContentType::new("text", "event-stream"))+ .header(Header::new("Cache-Control", "no-cache"))+ .header(Header::new("X-Accel-Buffering", "no"))+ .streamed_body(StreamBody::new(rx))+ .finalize();+ ChatCompletionResult::Stream(AnyResponse(response))+ } else {+ tokio::task::spawn_blocking(move || respond_blocking(id, model_name, input, run_config, created, &session_arc))+ .await+ .unwrap_or_else(|_| {+ let (usage, timings) = zero_usage_and_timings();+ ChatCompletionResult::Json(Json(ChatCompletionResponse {+ id: "error".to_string(),+ object: "chat.completion".to_string(),+ created,+ model: state.model_name.clone(),+ choices: vec![],+ usage,+ timings,+ stats: zero_stats(),+ }))+ })+ }}diff --git a/crates/cli/src/server/main.rs b/crates/cli/src/server/main.rsindex 1b2ddc7dc..96ab9f75a 100644--- a/crates/cli/src/server/main.rs+++ b/crates/cli/src/server/main.rs@@ -46,7 +46,7 @@ pub async fn run_server(let session = load_session(model_path, prefill_step_size, None);let state = SessionState {model_name,- session_wrapper: SessionWrapper::new(session),+ session_wrapper: std::sync::Arc::new(SessionWrapper::new(session)),};let _ =diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rsindex 117abcae2..e40a9e440 100644--- a/crates/cli/src/server/state.rs+++ b/crates/cli/src/server/state.rs@@ -1,33 +1,60 @@-use std::{path::PathBuf, sync::Mutex};+use std::{+ path::PathBuf,+ sync::{Arc, Mutex},+};use console::Style;use indicatif::{ProgressBar, ProgressStyle};use uzu::{prelude::SamplingSeed,- session::{Session, config::DecodingConfig, parameter::PrefillStepSize},+ session::{+ Session,+ config::{DecodingConfig, RunConfig},+ parameter::PrefillStepSize,+ types::{Error, Input, Output},+ },};-pub struct SessionWrapper(Mutex<Session>);+pub trait RunSession {+ fn run(+ &mut self,+ input: Input,+ config: RunConfig,+ progress: Option<Box<dyn Fn(Output) -> bool>>,+ ) -> Result<Output, Error>;+}++impl RunSession for Session {+ fn run(+ &mut self,+ input: Input,+ config: RunConfig,+ progress: Option<Box<dyn Fn(Output) -> bool>>,+ ) -> Result<Output, Error> {+ Session::run(self, input, config, progress)+ }+}++pub struct SessionWrapper(Mutex<Box<dyn RunSession>>);+unsafe impl Send for SessionWrapper {}unsafe impl Sync for SessionWrapper {}+impl SessionWrapper {- pub fn new(session: Session) -> Self {- Self(Mutex::new(session))+ pub fn new(runner: impl RunSession + 'static) -> Self {+ Self(Mutex::new(Box::new(runner)))}- pub fn lock(&self) -> std::sync::MutexGuard<'_, Session> {+ pub fn lock(&self) -> std::sync::MutexGuard<'_, Box<dyn RunSession>> {self.0.lock().unwrap()}}pub struct SessionState {pub model_name: String,- pub session_wrapper: SessionWrapper,+ pub session_wrapper: Arc<SessionWrapper>,}-unsafe impl Send for SessionState {}-unsafe impl Sync for SessionState {}-pub fn load_session(model_path: String,prefill_step_size: Option<usize>,diff --git a/crates/cli/tests/chat_completions.rs b/crates/cli/tests/chat_completions.rsnew file mode 100644index 000000000..6a36c049a--- /dev/null+++ b/crates/cli/tests/chat_completions.rs@@ -0,0 +1,237 @@+use std::sync::Arc;++use cli::server::{+ handle_chat_completions,+ state::{RunSession, SessionState, SessionWrapper},+};+use rocket::{http::Status, local::asynchronous::Client, routes};+use serde_json::{Value, json};+use uzu::session::{+ config::RunConfig,+ types::{Error, FinishReason, Input, Output, ParsedText, RunStats, Stats, StepStats, Text, TotalStats},+};++struct MockSession {+ result: Result<String, Error>,+}++impl RunSession for MockSession {+ fn run(+ &mut self,+ _input: Input,+ _config: RunConfig,+ _progress: Option<Box<dyn Fn(Output) -> bool>>,+ ) -> Result<Output, Error> {+ self.result.as_ref().map_err(|_| Error::GenerateFailed).map(|text| Output {+ text: Text {+ original: text.clone(),+ parsed: ParsedText {+ chain_of_thought: None,+ response: Some(text.clone()),+ },+ },+ finish_reason: Some(FinishReason::Stop),+ stats: Stats {+ prefill_stats: StepStats {+ duration: 0.0,+ suffix_length: 0,+ tokens_count: 0,+ tokens_per_second: 0.0,+ processed_tokens_per_second: 0.0,+ model_run: RunStats {+ count: 0,+ average_duration: 0.0,+ },+ run: None,+ },+ generate_stats: None,+ total_stats: TotalStats {+ duration: 0.0,+ tokens_count_input: 0,+ tokens_count_output: 0,+ },+ },+ })+ }+}++async fn make_client(mock: MockSession) -> Client {+ let state = SessionState {+ model_name: "test-model".to_string(),+ session_wrapper: Arc::new(SessionWrapper::new(mock)),+ };+ let rocket = rocket::build().manage(state).mount("/v1", routes![handle_chat_completions]);+ Client::tracked(rocket).await.expect("valid rocket instance")+}++#[rocket::async_test]+async fn test_non_streaming_success() {+ let client = make_client(MockSession {+ result: Ok("Hello, world!".to_string()),+ })+ .await;++ let response = client+ .post("/v1/chat/completions")+ .json(&json!({+ "messages": [{"role": "user", "content": "Hi"}],+ "stream": false+ }))+ .dispatch()+ .await;++ assert_eq!(response.status(), Status::Ok);+ let body: Value = response.into_json().await.expect("valid json");+ assert_eq!(body["choices"][0]["message"]["content"], "Hello, world!");+ assert_eq!(body["choices"][0]["finish_reason"], "stop");+ assert_eq!(body["model"], "test-model");+}++#[rocket::async_test]+async fn test_non_streaming_session_error() {+ let client = make_client(MockSession {+ result: Err(Error::GenerateFailed),+ })+ .await;++ let response = client+ .post("/v1/chat/completions")+ .json(&json!({+ "messages": [{"role": "user", "content": "Hi"}],+ "stream": false+ }))+ .dispatch()+ .await;++ assert_eq!(response.status(), Status::Ok);+ let body: Value = response.into_json().await.expect("valid json");+ assert_eq!(body["choices"][0]["finish_reason"], "error");+ assert!(+ body["choices"][0]["message"]["content"].as_str().unwrap().contains("Generate failed"),+ "expected error text in content"+ );+}++#[rocket::async_test]+async fn test_streaming_content_type() {+ let client = make_client(MockSession {+ result: Ok("Stream me".to_string()),+ })+ .await;++ let response = client+ .post("/v1/chat/completions")+ .json(&json!({+ "messages": [{"role": "user", "content": "Hi"}],+ "stream": true+ }))+ .dispatch()+ .await;++ assert_eq!(response.status(), Status::Ok);+ assert_eq!(response.content_type().map(|ct| ct.to_string()), Some("text/event-stream".to_string()));+}++#[rocket::async_test]+async fn test_streaming_done_sentinel() {+ let client = make_client(MockSession {+ result: Ok("Stream me".to_string()),+ })+ .await;++ let body = client+ .post("/v1/chat/completions")+ .json(&json!({+ "messages": [{"role": "user", "content": "Hi"}],+ "stream": true+ }))+ .dispatch()+ .await+ .into_string()+ .await+ .expect("body");++ assert!(body.contains("data: [DONE]"), "missing [DONE] sentinel:\n{body}");++ for line in body.lines().filter(|l| !l.is_empty()) {+ assert!(line.starts_with("data: "), "unexpected SSE line: {line}");+ }+}++#[rocket::async_test]+async fn test_streaming_error_in_stream() {+ let client = make_client(MockSession {+ result: Err(Error::GenerateFailed),+ })+ .await;++ let body = client+ .post("/v1/chat/completions")+ .json(&json!({+ "messages": [{"role": "user", "content": "Hi"}],+ "stream": true+ }))+ .dispatch()+ .await+ .into_string()+ .await+ .expect("body");++ assert!(body.contains("Generate failed"), "expected error text in stream:\n{body}");+ assert!(body.contains(r#""finish_reason":"error""#), "expected finish_reason error:\n{body}");+ assert!(body.contains("data: [DONE]"), "stream must still terminate:\n{body}");+}++/// Builds a string of exactly `prefix_bytes` ASCII bytes followed by a+/// 2-byte UTF-8 character (Cyrillic 'Π°', U+0430), so that a byte slice at+/// `prefix_bytes` would cut the character in half.+fn utf8_trap(prefix_bytes: usize) -> String {+ let mut s = "a".repeat(prefix_bytes);+ s.push('Π°'); // U+0430: 0xD0 0xB0 β 2 bytes+ s+}++#[rocket::async_test]+async fn test_request_content_utf8_boundary_does_not_panic() {+ // Content is 10001 bytes; byte 10000 is the second byte of 'Π°'.+ // &content[..10000] will panic with a byte-boundary assertion.+ let content = utf8_trap(9999);+ assert_eq!(content.len(), 10001);++ let client = make_client(MockSession {+ result: Ok("ok".to_string()),+ })+ .await;+ let response = client+ .post("/v1/chat/completions")+ .json(&json!({+ "messages": [{"role": "user", "content": content}],+ "stream": false+ }))+ .dispatch()+ .await;++ assert_eq!(response.status(), Status::Ok, "handler must not panic on multibyte content");+}++#[rocket::async_test]+async fn test_response_text_utf8_boundary_does_not_panic() {+ // Session returns 3001-byte text where byte 3000 is mid-char.+ let text = utf8_trap(2999);+ assert_eq!(text.len(), 3001);++ let client = make_client(MockSession {+ result: Ok(text),+ })+ .await;+ let response = client+ .post("/v1/chat/completions")+ .json(&json!({+ "messages": [{"role": "user", "content": "Hi"}],+ "stream": false+ }))+ .dispatch()+ .await;++ assert_eq!(response.status(), Status::Ok, "handler must not panic on multibyte response");+}