World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
An empty reply is asked again before it becomes an error (#631)
WaylandYangee4c2f090b3f
diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rsindex d582b516..4d8fce97 100644--- a/crates/utopia-server/src/api/chat.rs+++ b/crates/utopia-server/src/api/chat.rs@@ -50,6 +50,18 @@ const STALL_NUDGE: &str = "(system) Your last message ended the turn without calplan. Reply with the single word DONE only if the question was not about the user's data \at all: a greeting, a question about this transcript, or a refusal.";+/// 模型回了一个**空回复**(没有正文、也没调工具)时重问的那句。+///+/// 空回复不是一个结论,只是没说话。从前它直接变成一帧 `error`:台子上 48 轮里出过+/// 2 次,都在工具已经跑完之后——用户看见三四步查证,接着一句「Model returned an+/// empty answer」,查到的东西全白费。重问一次,第二次还空才报错:只给一次,所以一个+/// 始终不说话的端点不会让循环空转。+///+/// 措辞两头都留门:工具跑过的,就着查到的答;一步没查的(首轮就回空),缺证据+/// 就去调工具。它不说「答完了就回 DONE」——#509 那个出口正是被模型滥用的那一个。+const EMPTY_REPLY_RETRY: &str = "(system) Your previous reply was empty. Reply to the user now: \+ answer from the evidence gathered above, or call a tool if you still need evidence.";+/// 追问后仍不查时补在答案末尾的话。承诺已经流给用户了,收不回来;能做的是/// 让文字和空白的轨迹不再互相矛盾——对一个把「每个回答可追溯」当卖点的产品,/// 一句叙述了从未发生的查证的回答比「不知道」更糟。@@ -786,6 +798,8 @@ pub async fn chat(let mut rounds = 0usize;// #509 的追问只给一次let mut nudged = false;+ // 空回复的重问也只给一次(见 EMPTY_REPLY_RETRY)+ let mut asked_again = false;loop {if rounds >= MAX_ROUNDS {// 弹药耗尽:命令模型就现有证据作答(流式)@@ -895,8 +909,20 @@ pub async fn chat(if turn.tool_calls.is_empty() {if answer_acc.is_empty() {- yield error_event("Model returned an empty answer");- return;+ if asked_again {+ yield error_event("Model returned an empty answer");+ return;+ }+ asked_again = true;+ tracing::warn!(+ model = settings.chat_model.as_deref().unwrap_or_default(),+ rounds,+ "模型回了空,重问一次"+ );+ // 空的那一轮本身不进 msgs:一条既无正文又无调用的 assistant 消息,+ // 有的端点直接拒收。回到循环顶上重来,不占工具轮数+ msgs.push(json!({ "role": "user", "content": EMPTY_REPLY_RETRY }));+ continue;}// **没调工具的一轮不一定是答完了,也可能是停住了**(#509):正文说// 「我去查」,然后轮次就结束。循环分不出这两种,靠一次追问让模型自己@@ -1414,3 +1440,7 @@ mod stall_tests {}}}++#[cfg(test)]+#[path = "chat_empty_reply_tests.rs"]+mod chat_empty_reply_tests;diff --git a/crates/utopia-server/src/api/chat_empty_reply_tests.rs b/crates/utopia-server/src/api/chat_empty_reply_tests.rsnew file mode 100644index 00000000..c0edb7c4--- /dev/null+++ b/crates/utopia-server/src/api/chat_empty_reply_tests.rs@@ -0,0 +1,306 @@+//! 空回复重问一次(`EMPTY_REPLY_RETRY`)。+//!+//! 走整条对话:直接调 `chat` 处理函数,把它返回的 SSE 收完读帧。模型由 wiremock 按脚本+//! 假扮成 OpenAI 兼容的 `/chat/completions`——第几次请求回什么事先写好,于是「模型回空」+//! 这件真模型上几十轮才碰一次的事,在这里每次都发生。+//!+//! 1. **查完之后空一次,重问后答出来。** 这是台子上碰到的真实形状:工具先跑完,模型接着+//! 回空。用户看到的是查证步骤接着答案,不是报错;重问只发了一次,它的末尾正是那句+//! 重问;落库的助手消息就是重问后的答案。+//! (脚本若让模型一步不查就答,#509 的守卫会因为「答案什么都不站在上面」再追问一次——+//! 那是另一件事,所以这里先调一个工具,与真实失败一致。)+//! 2. **一直空,报错,而且只重问一次。** 两次请求之后是 `error` 帧——不是第三次、第四次:+//! 一个始终不说话的端点不能让循环空转。+//!+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。++use super::*;+use axum::response::IntoResponse;+use std::sync::{Arc, Mutex};+use utopia_core::models::User;+use wiremock::{+ matchers::{method, path},+ Mock, MockServer, Request, Respond, ResponseTemplate,+};++/// 假模型的一次回复+#[derive(Clone, Copy)]+enum Reply {+ /// 没有正文,也不调工具+ Empty,+ Text(&'static str),+ /// 调一个工具:(名字, 参数 JSON)+ Tool(&'static str, &'static str),+}++/// 按脚本回话的假模型。`replies[i]` 是第 i+1 次请求的回复,脚本读完之后一律回空。+/// 每次请求的正文都记下来,好查重问那一次问了什么+#[derive(Clone)]+struct Scripted {+ replies: Arc<Vec<Reply>>,+ seen: Arc<Mutex<Vec<serde_json::Value>>>,+}++impl Scripted {+ fn new(replies: Vec<Reply>) -> Self {+ Self {+ replies: Arc::new(replies),+ seen: Arc::new(Mutex::new(Vec::new())),+ }+ }+ fn requests(&self) -> Vec<serde_json::Value> {+ self.seen.lock().expect("seen lock").clone()+ }+}++impl Respond for Scripted {+ fn respond(&self, request: &Request) -> ResponseTemplate {+ let body: serde_json::Value = request.body_json().expect("chat request is JSON");+ let n = {+ let mut seen = self.seen.lock().expect("seen lock");+ seen.push(body);+ seen.len()+ };+ let frame = match self.replies.get(n - 1).copied().unwrap_or(Reply::Empty) {+ Reply::Empty => None,+ Reply::Text(text) => {+ Some(serde_json::json!({ "choices": [{ "delta": { "content": text } }] }))+ }+ Reply::Tool(name, args) => Some(serde_json::json!({ "choices": [{ "delta": {+ "tool_calls": [{+ "index": 0,+ "id": format!("call_{n}"),+ "function": { "name": name, "arguments": args }+ }]+ } }] })),+ };+ let mut sse = String::new();+ if let Some(frame) = frame {+ sse.push_str(&format!("data: {frame}\n\n"));+ }+ // 空回复就是只有这一帧:没有正文增量,也没有工具调用+ sse.push_str("data: [DONE]\n\n");+ ResponseTemplate::new(200)+ .insert_header("content-type", "text/event-stream")+ .set_body_string(sse)+ }+}++struct Fx {+ state: AppState,+ pool: sqlx::PgPool,+ org: Uuid,+ kb: Uuid,+ user: User,+ fake: Scripted,+ _server: MockServer,+ dir: std::path::PathBuf,+}++async fn fixture(fake: Scripted) -> anyhow::Result<Option<Fx>> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(None);+ };+ let pool = sqlx::PgPool::connect(&url).await?;+ let (org, ws, kb, uid) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'empty-reply-test')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'empty-reply-test')")+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'empty-reply-test')",+ )+ .bind(kb)+ .bind(ws)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO users(id,org_id,email,password_hash,display_name,is_admin)+ VALUES($1,$2,$3,'','Empty Reply',TRUE)",+ )+ .bind(uid)+ .bind(org)+ .bind(format!("empty-reply-{uid}@test.local"))+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO kb_members(kb_id,user_id,role) VALUES($1,$2,'viewer')")+ .bind(kb)+ .bind(uid)+ .execute(&pool)+ .await?;++ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/chat/completions"))+ .respond_with(fake.clone())+ .mount(&server)+ .await;+ utopia_store::settings::upsert(+ &pool,+ ws,+ Some(&server.uri()),+ None,+ Some("scripted-chat"),+ None,+ None,+ None,+ None,+ )+ .await?;++ let dir = std::env::temp_dir().join(format!("utopia-empty-reply-{kb}"));+ let cfg = utopia_core::config::AppConfig {+ data_dir: dir.to_string_lossy().into_owned(),+ ..Default::default()+ };+ let search = Arc::new(utopia_search::SearchIndex::open(&dir.join("search"))?);+ let state = AppState::new(pool.clone(), &cfg, search, "test-only".into());+ let user = sqlx::query_as::<_, User>("SELECT * FROM users WHERE id = $1")+ .bind(uid)+ .fetch_one(&pool)+ .await?;+ Ok(Some(Fx {+ state,+ pool,+ org,+ kb,+ user,+ fake,+ _server: server,+ dir,+ }))+}++impl Fx {+ /// 问一句,把整条 SSE 收成文本+ async fn ask(&self, message: &str) -> anyhow::Result<String> {+ let sse = chat(+ State(self.state.clone()),+ AuthUser(self.user.clone()),+ Path(self.kb),+ Json(ChatReq {+ conversation_id: None,+ message: message.into(),+ }),+ )+ .await+ .map_err(|_| anyhow::anyhow!("chat handler refused the request"))?;+ let body = axum::body::to_bytes(sse.into_response().into_body(), 4 * 1024 * 1024).await?;+ Ok(String::from_utf8_lossy(&body).into_owned())+ }++ async fn stored_answer(&self) -> anyhow::Result<Option<String>> {+ Ok(sqlx::query_scalar(+ "SELECT m.content FROM conversation_messages m+ JOIN conversations c ON c.id = m.conversation_id+ WHERE c.kb_id = $1 AND m.role = 'assistant'+ ORDER BY m.created_at DESC LIMIT 1",+ )+ .bind(self.kb)+ .fetch_optional(&self.pool)+ .await?)+ }++ async fn cleanup(self) -> anyhow::Result<()> {+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(self.org)+ .execute(&self.pool)+ .await?;+ let _ = std::fs::remove_dir_all(&self.dir);+ Ok(())+ }+}++/// 发给模型的请求里,最后一条消息的正文+fn last_message(request: &serde_json::Value) -> String {+ request["messages"]+ .as_array()+ .and_then(|m| m.last())+ .and_then(|m| m["content"].as_str())+ .unwrap_or_default()+ .to_string()+}++#[tokio::test]+async fn an_empty_reply_after_the_tools_is_asked_again_and_the_answer_arrives() -> anyhow::Result<()>+{+ const ANSWER: &str = "Answered on the second ask.";+ let Some(f) = fixture(Scripted::new(vec![+ Reply::Tool("find_entities", r#"{"name":"Acme"}"#),+ Reply::Empty,+ Reply::Text(ANSWER),+ ]))+ .await?+ else {+ return Ok(());+ };++ let sse = f.ask("What changed at Acme last quarter?").await?;++ assert!(sse.contains("event: step"), "the tool ran first:\n{sse}");+ assert!(+ !sse.contains("event: error"),+ "an empty reply must not reach the user as an error:\n{sse}"+ );+ assert!(sse.contains("event: done"), "the turn finishes:\n{sse}");+ assert!(+ sse.contains(ANSWER),+ "the reply after the retry is streamed:\n{sse}"+ );++ let requests = f.fake.requests();+ assert_eq!(requests.len(), 3, "tool, empty, then one retry");+ assert_eq!(+ requests+ .iter()+ .filter(|r| last_message(r) == EMPTY_REPLY_RETRY)+ .count(),+ 1,+ "the retry is sent exactly once"+ );+ assert_eq!(+ last_message(&requests[2]),+ EMPTY_REPLY_RETRY,+ "the request after the empty reply ends with the retry"+ );+ assert_eq!(+ f.stored_answer().await?.as_deref(),+ Some(ANSWER),+ "the stored answer is the one the retry produced"+ );+ f.cleanup().await+}++#[tokio::test]+async fn a_reply_that_stays_empty_is_an_error_after_one_retry() -> anyhow::Result<()> {+ let Some(f) = fixture(Scripted::new(vec![Reply::Empty; 4])).await? else {+ return Ok(());+ };++ let sse = f.ask("What changed last quarter?").await?;++ assert!(+ sse.contains("event: error"),+ "still empty is an error:\n{sse}"+ );+ assert!(+ sse.contains("Model returned an empty answer"),+ "with the same message as before:\n{sse}"+ );+ assert_eq!(+ f.fake.requests().len(),+ 2,+ "one retry, not a loop: a silent endpoint must not be asked again and again"+ );+ f.cleanup().await+}