World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Keep rejected retries out of live streams and identify mapping retrieval failures (#1027)
Daniel79176cda39b2
diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rsindex 668cb519..c9a27acb 100644--- a/crates/utopia-server/src/api/chat.rs+++ b/crates/utopia-server/src/api/chat.rs@@ -811,7 +811,7 @@ pub async fn chat(Ok(mappings) => mappings,Err(error) => {tracing::warn!(%error, "Could not retrieve conversation mappings");- yield ProducerEvent::Outcome(Err(Failure::new("search_failed", "Could not retrieve relevant mappings.")));+ yield ProducerEvent::Outcome(Err(Failure::new("mapping_search_failed", "Could not retrieve relevant mappings.")));return;}}@@ -1199,6 +1199,7 @@ pub async fn chat(// 生成登记在案,然后**这条连接也只是去「接上」它**——与刷新之后// 那条重连走的是同一段代码。两条路分开写的话,迟早只有一条是对的+ handle.start().await;let attached = live.attach(conversation_id).await;tokio::spawn(generation::run(save_pool,diff --git a/crates/utopia-server/src/api/chat_empty_reply_tests.rs b/crates/utopia-server/src/api/chat_empty_reply_tests.rsindex 6cc889f5..e1af7c10 100644--- a/crates/utopia-server/src/api/chat_empty_reply_tests.rs+++ b/crates/utopia-server/src/api/chat_empty_reply_tests.rs@@ -399,6 +399,8 @@ mod registry_tests;mod resolved_tests;#[path = "chat_restated_tests.rs"]mod restated_tests;+#[path = "chat_retrieval_failure_tests.rs"]+mod retrieval_failure_tests;#[path = "chat_retry_tests.rs"]mod retry_tests;#[path = "chat_sources_tests.rs"]diff --git a/crates/utopia-server/src/api/chat_restated_tests.rs b/crates/utopia-server/src/api/chat_restated_tests.rsindex 1940a830..e35021ac 100644--- a/crates/utopia-server/src/api/chat_restated_tests.rs+++ b/crates/utopia-server/src/api/chat_restated_tests.rs@@ -119,6 +119,7 @@ async fn a_stopped_restatement_keeps_its_sources_live_and_after_reload() -> anyhassert!(!expected_sources.as_array().unwrap().is_empty());let handle = f.state.live.begin(id).await?;+ handle.start().await;let cancellation = handle.cancellation();let response = sse_from(f.state.live.attach(id).await);utopia_store::conversations::append_message(diff --git a/crates/utopia-server/src/api/chat_retrieval_failure_tests.rs b/crates/utopia-server/src/api/chat_retrieval_failure_tests.rsnew file mode 100644index 00000000..17d9872f--- /dev/null+++ b/crates/utopia-server/src/api/chat_retrieval_failure_tests.rs@@ -0,0 +1,98 @@+//! 两种检索失败要各说各的原因,底层诊断只能留在服务端。+use super::*;++#[tokio::test]+async fn mapping_and_document_retrieval_failures_have_distinct_safe_codes() -> anyhow::Result<()> {+ for (table, code, message) in [+ (+ "concept_mappings",+ "mapping_search_failed",+ "Could not retrieve relevant mappings.",+ ),+ ("chunks", "search_failed", "Could not search the documents."),+ ] {+ let Some(mut f) = fixture(Scripted::new(vec![Reply::Http(400), Reply::Http(400)])).await?+ else {+ return Ok(());+ };+ let mapping = table == "concept_mappings";+ let source = if mapping {+ let id = utopia_store::datasources::create(+ &f.pool,+ &format!("mapping failure {}", f.kb),+ "postgres",+ "postgres://unused",+ f.user.id,+ )+ .await?;+ utopia_store::datasources::mount(&f.pool, f.kb, id).await?;+ Some(id)+ } else {+ // 真正走向量检索,让分块查询失败,不能只验证手工构造的错误帧。+ sqlx::query(+ "UPDATE llm_settings SET embed_base_url=chat_base_url, embed_model='test-embed'+ WHERE workspace_id=(SELECT workspace_id FROM knowledge_bases WHERE id=$1)",+ )+ .bind(f.kb)+ .execute(&f.pool)+ .await?;+ Mock::given(method("POST"))+ .and(path("/embeddings"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "data": [{"index": 0, "embedding": [0.1, 0.2, 0.3]}]+ })))+ .mount(&f._server)+ .await;+ None+ };+ // 只给本夹具的连接遮住对应表,让真实查询因缺列而失败。+ // 不改共享表、权限或生产查询,也不会干扰并行测试。+ let pool = sqlx::postgres::PgPoolOptions::new()+ .after_connect(move |connection, _| {+ Box::pin(async move {+ sqlx::query(&format!(+ "CREATE TEMP TABLE {table} (unrelated_column text)"+ ))+ .execute(connection)+ .await?;+ Ok(())+ })+ })+ .connect_with((*f.pool.connect_options()).clone())+ .await?;+ f.state.pool = pool.clone();+ let result = async {+ let sse = f.ask("What is the revenue?").await?;+ assert_eq!(sse.matches("event: error").count(), 1, "{sse}");+ assert!(!sse.contains("event: done"), "{sse}");+ let error = sse+ .split("\n\n")+ .find(|frame| frame.starts_with("event: error"))+ .and_then(|frame| frame.lines().find_map(|line| line.strip_prefix("data: ")))+ .expect("an error frame carries data");+ assert_eq!(+ serde_json::from_str::<serde_json::Value>(error)?,+ json!({"code":code, "error":message})+ );+ assert!(!sse.contains("does not exist"), "{sse}");+ let id: Uuid = sqlx::query_scalar("SELECT id FROM conversations WHERE kb_id=$1")+ .bind(f.kb)+ .fetch_one(&f.pool)+ .await?;+ let messages = utopia_store::conversations::messages(&f.pool, id).await?;+ assert_eq!(messages.len(), 1, "only the question is stored");+ assert_eq!(messages[0].role, "user");+ assert!(f.state.live.attach(id).await.is_none());+ assert_eq!(f.requests().len(), if mapping { 0 } else { 2 });+ Ok::<_, anyhow::Error>(())+ }+ .await;+ pool.close().await;+ if let Some(id) = source {+ utopia_store::datasources::delete(&f.pool, id).await?;+ }+ f.cleanup().await?;+ result?;+ }+ Ok(())+}diff --git a/crates/utopia-server/src/api/chat_retry_tests.rs b/crates/utopia-server/src/api/chat_retry_tests.rsindex c37c8bbd..50490f05 100644--- a/crates/utopia-server/src/api/chat_retry_tests.rs+++ b/crates/utopia-server/src/api/chat_retry_tests.rs@@ -204,6 +204,7 @@ async fn a_question_is_not_answered_twice_at_once() -> anyhow::Result<()> {utopia_store::conversations::create(&f.pool, f.kb, f.user.id, QUESTION).await?;let question = append(&f, conversation_id, "user", QUESTION).await?;let running = f.state.live.begin(conversation_id).await.unwrap();+ running.start().await;let Outcome::Refused(status, body) = post(&f, retry(conversation_id, question)).await? else {panic!("a retry started while an answer was running");};@@ -282,3 +283,97 @@ async fn every_question_tells_the_interface_its_stored_id() -> anyhow::Result<());f.cleanup().await}++/// 重试资格尚未读完时,另一标签页不能订阅这次随后会被拒绝的占位。+#[tokio::test]+async fn a_rejected_retry_does_not_publish_a_stream_to_another_tab() -> anyhow::Result<()> {+ let Some(f) = fixture(Scripted::new(vec![])).await? else {+ return Ok(());+ };+ let application_name = format!("retry-preparation-{}", f.kb.simple());+ let retry_pool = sqlx::postgres::PgPoolOptions::new()+ .max_connections(1)+ .connect_with(+ (*f.pool.connect_options())+ .clone()+ .application_name(&application_name),+ )+ .await?;+ let mut retry_state = f.state.clone();+ retry_state.pool = retry_pool.clone();+ let result = tokio::time::timeout(std::time::Duration::from_secs(20), async {+ let id = utopia_store::conversations::create(&f.pool, f.kb, f.user.id, QUESTION).await?;+ let question = append(&f, id, "user", QUESTION).await?;+ append(&f, id, "assistant", "Already answered.").await?;+ let before = stored(&f, id).await?;++ // 只在资格查询上制造等待;它之前的权限检查读其他表,begin 已经占位。+ let mut lock = f.pool.begin().await?;+ sqlx::query("LOCK TABLE conversation_messages IN ACCESS EXCLUSIVE MODE")+ .execute(&mut *lock)+ .await?;+ let blocker: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")+ .fetch_one(&mut *lock)+ .await?;+ let pending = chat(+ State(retry_state),+ AuthUser(f.user.clone()),+ Path(f.kb),+ Json(ChatReq {+ conversation_id: Some(id),+ message: QUESTION.into(),+ retry_message_id: Some(question),+ }),+ );+ tokio::pin!(pending);+ let waiting = async {+ while !sqlx::query_scalar::<_, bool>(+ "SELECT EXISTS(SELECT 1 FROM pg_stat_activity+ WHERE application_name = $1 AND $2 = ANY(pg_blocking_pids(pid)))",+ )+ .bind(&application_name)+ .bind(blocker)+ .fetch_one(&f.pool)+ .await?+ {}+ Ok::<_, anyhow::Error>(())+ };+ tokio::select! {+ _ = &mut pending => anyhow::bail!("retry must wait for its qualification query"),+ result = waiting => result?,+ }+ // 直接进入重接的订阅边界,避免单次 poll 只停在 HTTP 权限查询上。+ let other_tab = f.state.live.attach(id);+ tokio::pin!(other_tab);+ anyhow::ensure!(futures_util::poll!(&mut other_tab).is_pending());++ lock.rollback().await?;+ let refusal = pending+ .await+ .err()+ .ok_or_else(|| anyhow::anyhow!("an answered question was retried"))?+ .into_response();+ anyhow::ensure!(refusal.status().as_u16() == 409);+ let body = axum::body::to_bytes(refusal.into_body(), 65536).await?;+ anyhow::ensure!(+ serde_json::from_slice::<serde_json::Value>(&body)?["code"] == "retry_answered"+ );+ let other_tab = sse_from(other_tab.await);+ let body = axum::body::to_bytes(other_tab.into_response().into_body(), 65536).await?;+ let sse = String::from_utf8_lossy(&body);+ anyhow::ensure!(sse.contains("event: idle"), "{sse}");+ anyhow::ensure!(+ !sse.contains("event: snapshot") && !sse.contains("event: error"),+ "{sse}"+ );+ anyhow::ensure!(stored(&f, id).await? == before);+ anyhow::ensure!(f.requests().is_empty());+ drop(f.state.live.begin(id).await?);+ Ok::<_, anyhow::Error>(())+ })+ .await;+ retry_pool.close().await;+ f.cleanup().await?;+ result??;+ Ok(())+}diff --git a/crates/utopia-server/src/api/chat_stream_tests.rs b/crates/utopia-server/src/api/chat_stream_tests.rsindex 29f5fc35..e2c71c4a 100644--- a/crates/utopia-server/src/api/chat_stream_tests.rs+++ b/crates/utopia-server/src/api/chat_stream_tests.rs@@ -2,12 +2,50 @@ use super::*;use axum::response::IntoResponse;use std::sync::Arc;+#[tokio::test]+async fn a_busy_refusal_preserves_both_subscribers_and_the_terminal() {+ for stopped in [false, true] {+ let registry = Arc::new(crate::live::Registry::default());+ let id = Uuid::now_v7();+ let handle = registry.begin(id).await.unwrap();+ handle.start().await;+ let first = sse_from(registry.attach(id).await);+ handle.emit(delta_event("partial")).await;+ let second = sse_from(registry.attach(id).await);++ assert!(matches!(+ registry.begin(id).await,+ Err(AppError::CodedConflict {+ code: "answer_running",+ ..+ })+ ));+ handle.emit(delta_event(" tail")).await;+ handle+ .complete(Frame::new("done", json!({"stopped": stopped}).to_string()))+ .await;++ for response in [first, second] {+ let body = axum::body::to_bytes(response.into_response().into_body(), 65536)+ .await+ .unwrap();+ let text = String::from_utf8_lossy(&body);+ assert!(text.contains("partial") && text.contains(" tail"), "{text}");+ assert_eq!(text.matches("event: done").count(), 1, "{text}");+ assert!(text.contains(&format!("\"stopped\":{stopped}")), "{text}");+ assert!(!text.contains("event: error"), "{text}");+ }+ let _next = registry.begin(id).await.unwrap();+ }+}+#[tokio::test]async fn reattachment_preserves_terminal_frames_on_both_sides_of_emit() {for (event, data) in [("done", r#"{"stopped":true}"#), ("error", "safe outcome")] {let registry = Arc::new(crate::live::Registry::default());let id = Uuid::now_v7();let handle = registry.begin(id).await.unwrap();+ handle.start().await;handle.emit(delta_event("partial")).await;let before = sse_from(registry.attach(id).await);handle.emit(Frame::new(event, data.into())).await;@@ -44,6 +82,7 @@ async fn lagged_subscribers_receive_an_error_not_done() {let registry = Arc::new(crate::live::Registry::default());let id = Uuid::now_v7();let handle = registry.begin(id).await.unwrap();+ handle.start().await;let stream = sse_from(registry.attach(id).await);for _ in 0..300 {handle.emit(delta_event("x")).await;@@ -61,6 +100,7 @@ async fn producer_disappearing_without_an_outcome_ends_in_one_error() {let registry = Arc::new(crate::live::Registry::default());let id = Uuid::now_v7();let handle = registry.begin(id).await.unwrap();+ handle.start().await;let stream = sse_from(registry.attach(id).await);handle.emit(delta_event("partial")).await;drop(handle);@@ -83,6 +123,7 @@ async fn first_terminal_freezes_the_snapshot_and_broadcast() {let registry = Arc::new(crate::live::Registry::default());let id = Uuid::now_v7();let handle = registry.begin(id).await.unwrap();+ handle.start().await;handle.emit(delta_event("kept")).await;handle.emit(error_event("answer_failed", "original error"))diff --git a/crates/utopia-server/src/live.rs b/crates/utopia-server/src/live.rsindex f92deedb..79f430b0 100644--- a/crates/utopia-server/src/live.rs+++ b/crates/utopia-server/src/live.rs@@ -94,6 +94,7 @@ struct Entry {tx: broadcast::Sender<Frame>,snap: Arc<RwLock<Snapshot>>,cancellation: Cancellation,+ ready: watch::Sender<bool>,}/// 保留取消信号:Stop 可能早于生成器开始等待。@@ -133,6 +134,7 @@ pub struct Handle {tx: broadcast::Sender<Frame>,snap: Arc<RwLock<Snapshot>>,cancellation: Cancellation,+ ready: watch::Sender<bool>,registry: Arc<Registry>,}@@ -149,6 +151,11 @@ impl Handle {self.snap.read().await.clone()}+ /// 准备成功后才开放订阅;被拒的重试只释放占位,不会留下等待终态的客户端。+ pub async fn start(&self) {+ self.ready.send_replace(true);+ }+/// 发一个事件:记进快照,然后广播。////// **广播时仍然握着快照的写锁**,这一点是必需的。只保证「先写后发」@@ -216,6 +223,7 @@ impl Registry {..Snapshot::default()}));let cancellation = Cancellation::default();+ let (ready, _) = watch::channel(false);entries.insert(conversation_id,Entry {@@ -223,6 +231,7 @@ impl Registry {tx: tx.clone(),snap: snap.clone(),cancellation: cancellation.clone(),+ ready: ready.clone(),},);Ok(Handle {@@ -231,6 +240,7 @@ impl Registry {tx,snap,cancellation,+ ready,registry: self.clone(),})}@@ -253,11 +263,18 @@ impl Registry {&self,conversation_id: Uuid,) -> Option<(Snapshot, broadcast::Receiver<Frame>)> {- let (snap, tx) = {+ let (snap, tx, mut ready) = {let map = self.0.lock().expect("live registry lock");let entry = map.get(&conversation_id)?;- (entry.snap.clone(), entry.tx.clone())+ (+ entry.snap.clone(),+ entry.tx.clone(),+ entry.ready.subscribe(),+ )};+ // 不持有 ready 的发送端:准备失败释放占位时,等待者会醒来并回到 idle。+ // 准备成功则接上同一轮,避免过早返回 idle 后错过整个回答。+ ready.wait_for(|started| *started).await.ok()?;// **握着快照的读锁再订阅。** `emit` 是握着写锁广播的,所以这一段// 与任何一次 emit 互斥:拿到的快照与订阅起点严丝合缝,// 中间那一小段既不会漏、也不会重@@ -280,9 +297,11 @@ mod tests {let registry = Arc::new(Registry::default());let id = Uuid::now_v7();let old = registry.begin(id).await.unwrap();+ old.start().await;old.emit(delta("old")).await;old.retire();let current = registry.begin(id).await.unwrap();+ current.start().await;current.emit(delta("new")).await;let (snapshot, mut rx) = registry.attach(id).await.unwrap();assert_eq!(snapshot.content, "new");@@ -317,10 +336,12 @@ mod tests {let id = Uuid::now_v7();let other_id = Uuid::now_v7();let other = registry.begin(other_id).await.unwrap();+ other.start().await;other.emit(delta("unrelated")).await;let mut handles = Vec::new();for index in 0..3 {let handle = registry.begin(id).await.unwrap();+ handle.start().await;if index < 2 {handle.retire();}@@ -350,6 +371,7 @@ mod tests {let registry = Arc::new(Registry::default());let id = Uuid::now_v7();let handle = registry.begin(id).await.unwrap();+ handle.start().await;// 两种拿锁顺序都合法:每个增量必须恰好出现在快照或订阅中一次,// 既不能重复,也不能遗漏。for index in 0..64 {@@ -372,6 +394,46 @@ mod tests {drop(handle);}+ #[tokio::test]+ async fn a_preparing_reservation_is_exclusive_but_not_subscribable() {+ let registry = Arc::new(Registry::default());+ let id = Uuid::now_v7();+ let handle = registry.begin(id).await.unwrap();+ assert!(matches!(+ registry.begin(id).await,+ Err(AppError::CodedConflict {+ code: "answer_running",+ ..+ })+ ));+ let attached = registry.attach(id);+ tokio::pin!(attached);+ assert!(futures_util::poll!(&mut attached).is_pending());+ drop(handle);+ assert!(attached.await.is_none());+ let _next = registry.begin(id).await.unwrap();+ }++ #[tokio::test]+ async fn an_attachment_waiting_for_preparation_joins_the_started_generation() {+ let registry = Arc::new(Registry::default());+ let id = Uuid::now_v7();+ let handle = registry.begin(id).await.unwrap();+ let attached = registry.attach(id);+ tokio::pin!(attached);+ assert!(futures_util::poll!(&mut attached).is_pending());++ handle.start().await;+ let (snapshot, mut receiver) = attached.await.unwrap();+ assert_eq!(snapshot.generation_id, handle.generation_id());+ handle.emit(delta("answer")).await;+ assert_eq!(receiver.recv().await.unwrap().data, delta("answer").data);+ handle+ .complete(Frame::new("done", json!({"stopped":false}).to_string()))+ .await;+ assert_eq!(receiver.recv().await.unwrap().event, "done");+ }+#[tokio::test]async fn admission_is_exclusive_and_dropping_a_reservation_releases_it() {let registry = Arc::new(Registry::default());@@ -409,6 +471,7 @@ mod tests {let registry = Arc::new(Registry::default());let id = Uuid::now_v7();let handle = registry.begin(id).await.unwrap();+ handle.start().await;let (_, mut receiver) = registry.attach(id).await.unwrap();handle.complete(Frame::new("done", json!({"stopped":true}).to_string()))diff --git a/web/src/chatStream.test.ts b/web/src/chatStream.test.tsindex 2f89b36a..9ecdcce9 100644--- a/web/src/chatStream.test.ts+++ b/web/src/chatStream.test.ts@@ -1,4 +1,7 @@import { afterEach, describe, expect, it, vi } from "vitest";+import { S } from "./i18n";+import { en } from "./i18n/en";+import { zh } from "./i18n/zh";import { ApiError, reattachChat, streamChat, type ChatHandlers } from "./api";vi.mock("./i18n", () => ({S: {@@ -91,6 +94,21 @@ describe("a chat stream that fails", () => {expect(await replay(text)).toEqual([["error", "Worded: start a new conversation"]]);expect(await replay(text, true)).toEqual([["error", "Worded: start a new conversation"]]);});+ it.each([en, zh])("distinguishes mapping retrieval from document search in either stream", async (strings) => {+ const original = S.err;+ S.err = strings.err;+ try {+ for (const code of ["mapping_search_failed", "search_failed"] as const) {+ const text = failed({ error: "Could not retrieve relevant mappings.", code });+ const expected = [["error", strings.err[code]]];+ expect(await replay(text)).toEqual(expected);+ expect(await replay(text, true)).toEqual(expected);+ }+ expect(strings.err.mapping_search_failed).not.toBe(strings.err.search_failed);+ } finally {+ S.err = original;+ }+ });it("keeps the server's sentence for a code the table does not have", async () => {expect(await replay(failed({ error: "Some new failure", code: "unlisted_code" }))).toEqual([["error", "Some new failure"]]);diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex c6ad224f..1d09c166 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -63,6 +63,7 @@ export const en = {answer_failed: "The answer could not be completed. Ask again.",answer_not_saved: "The answer could not be saved. Try again later.",search_failed: "The documents could not be searched.",+ mapping_search_failed: "Relevant data mappings could not be retrieved.",stream_ended:"The answer stream ended unexpectedly. Reopen the conversation to check its status.",stream_lagged: "This page fell behind the answer stream. Reopen the conversation.",diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 2edb4076..166a0efb 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -63,6 +63,7 @@ export const zh: Strings = {answer_failed: "这次没能答完,请再问一次。",answer_not_saved: "回答没能保存,请稍后再试。",search_failed: "没能检索文档。",+ mapping_search_failed: "没能检索相关数据映射。",stream_ended: "回答的连接意外中断了。重新打开这个对话,看看结果如何。",stream_lagged: "这个页面跟不上回答的速度,请重新打开这个对话。",model_out_of_credit: "模型账号付不起这次请求。去给账号充值,或者在「管理 → 模型」里换一个能用的端点。",