Skip to content

World's first open-source enterprise world model.

Keep rejected retries out of live streams and identify mapping retrieval failures (#1027)

Daniel79176cda39b2
diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rs
index 668cb519..c9a27acb 100644
--- a/crates/utopia-server/src/api/chat.rs
+++ b/crates/utopia-server/src/api/chat.rs
@@ -811,7 +811,7 @@ pub async fn chat(
Ok(mappings) => mappings,
Err(error) => {
tracing::warn!(%error, "Could not retrieve conversation mappings");
- yield ProducerEvent::Outcome(Err(Failure::new("search_failed", "Could not retrieve relevant mappings.")));
+ yield ProducerEvent::Outcome(Err(Failure::new("mapping_search_failed", "Could not retrieve relevant mappings.")));
return;
}
}
@@ -1199,6 +1199,7 @@ pub async fn chat(
// 生成登记在案,然后**这条连接也只是去「接上」它**——与刷新之后
// 那条重连走的是同一段代码。两条路分开写的话,迟早只有一条是对的
+ handle.start().await;
let attached = live.attach(conversation_id).await;
tokio::spawn(generation::run(
save_pool,
diff --git a/crates/utopia-server/src/api/chat_empty_reply_tests.rs b/crates/utopia-server/src/api/chat_empty_reply_tests.rs
index 6cc889f5..e1af7c10 100644
--- a/crates/utopia-server/src/api/chat_empty_reply_tests.rs
+++ b/crates/utopia-server/src/api/chat_empty_reply_tests.rs
@@ -399,6 +399,8 @@ mod registry_tests;
mod resolved_tests;
#[path = "chat_restated_tests.rs"]
mod restated_tests;
+#[path = "chat_retrieval_failure_tests.rs"]
+mod retrieval_failure_tests;
#[path = "chat_retry_tests.rs"]
mod retry_tests;
#[path = "chat_sources_tests.rs"]
diff --git a/crates/utopia-server/src/api/chat_restated_tests.rs b/crates/utopia-server/src/api/chat_restated_tests.rs
index 1940a830..e35021ac 100644
--- a/crates/utopia-server/src/api/chat_restated_tests.rs
+++ b/crates/utopia-server/src/api/chat_restated_tests.rs
@@ -119,6 +119,7 @@ async fn a_stopped_restatement_keeps_its_sources_live_and_after_reload() -> anyh
assert!(!expected_sources.as_array().unwrap().is_empty());
let handle = f.state.live.begin(id).await?;
+ handle.start().await;
let cancellation = handle.cancellation();
let response = sse_from(f.state.live.attach(id).await);
utopia_store::conversations::append_message(
diff --git a/crates/utopia-server/src/api/chat_retrieval_failure_tests.rs b/crates/utopia-server/src/api/chat_retrieval_failure_tests.rs
new file mode 100644
index 00000000..17d9872f
--- /dev/null
+++ b/crates/utopia-server/src/api/chat_retrieval_failure_tests.rs
@@ -0,0 +1,98 @@
+//! 两种检索失败要各说各的原因,底层诊断只能留在服务端。
+use super::*;
+
+#[tokio::test]
+async fn mapping_and_document_retrieval_failures_have_distinct_safe_codes() -> anyhow::Result<()> {
+ for (table, code, message) in [
+ (
+ "concept_mappings",
+ "mapping_search_failed",
+ "Could not retrieve relevant mappings.",
+ ),
+ ("chunks", "search_failed", "Could not search the documents."),
+ ] {
+ let Some(mut f) = fixture(Scripted::new(vec![Reply::Http(400), Reply::Http(400)])).await?
+ else {
+ return Ok(());
+ };
+ let mapping = table == "concept_mappings";
+ let source = if mapping {
+ let id = utopia_store::datasources::create(
+ &f.pool,
+ &format!("mapping failure {}", f.kb),
+ "postgres",
+ "postgres://unused",
+ f.user.id,
+ )
+ .await?;
+ utopia_store::datasources::mount(&f.pool, f.kb, id).await?;
+ Some(id)
+ } else {
+ // 真正走向量检索,让分块查询失败,不能只验证手工构造的错误帧。
+ sqlx::query(
+ "UPDATE llm_settings SET embed_base_url=chat_base_url, embed_model='test-embed'
+ WHERE workspace_id=(SELECT workspace_id FROM knowledge_bases WHERE id=$1)",
+ )
+ .bind(f.kb)
+ .execute(&f.pool)
+ .await?;
+ Mock::given(method("POST"))
+ .and(path("/embeddings"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "data": [{"index": 0, "embedding": [0.1, 0.2, 0.3]}]
+ })))
+ .mount(&f._server)
+ .await;
+ None
+ };
+ // 只给本夹具的连接遮住对应表,让真实查询因缺列而失败。
+ // 不改共享表、权限或生产查询,也不会干扰并行测试。
+ let pool = sqlx::postgres::PgPoolOptions::new()
+ .after_connect(move |connection, _| {
+ Box::pin(async move {
+ sqlx::query(&format!(
+ "CREATE TEMP TABLE {table} (unrelated_column text)"
+ ))
+ .execute(connection)
+ .await?;
+ Ok(())
+ })
+ })
+ .connect_with((*f.pool.connect_options()).clone())
+ .await?;
+ f.state.pool = pool.clone();
+ let result = async {
+ let sse = f.ask("What is the revenue?").await?;
+ assert_eq!(sse.matches("event: error").count(), 1, "{sse}");
+ assert!(!sse.contains("event: done"), "{sse}");
+ let error = sse
+ .split("\n\n")
+ .find(|frame| frame.starts_with("event: error"))
+ .and_then(|frame| frame.lines().find_map(|line| line.strip_prefix("data: ")))
+ .expect("an error frame carries data");
+ assert_eq!(
+ serde_json::from_str::<serde_json::Value>(error)?,
+ json!({"code":code, "error":message})
+ );
+ assert!(!sse.contains("does not exist"), "{sse}");
+ let id: Uuid = sqlx::query_scalar("SELECT id FROM conversations WHERE kb_id=$1")
+ .bind(f.kb)
+ .fetch_one(&f.pool)
+ .await?;
+ let messages = utopia_store::conversations::messages(&f.pool, id).await?;
+ assert_eq!(messages.len(), 1, "only the question is stored");
+ assert_eq!(messages[0].role, "user");
+ assert!(f.state.live.attach(id).await.is_none());
+ assert_eq!(f.requests().len(), if mapping { 0 } else { 2 });
+ Ok::<_, anyhow::Error>(())
+ }
+ .await;
+ pool.close().await;
+ if let Some(id) = source {
+ utopia_store::datasources::delete(&f.pool, id).await?;
+ }
+ f.cleanup().await?;
+ result?;
+ }
+ Ok(())
+}
diff --git a/crates/utopia-server/src/api/chat_retry_tests.rs b/crates/utopia-server/src/api/chat_retry_tests.rs
index c37c8bbd..50490f05 100644
--- a/crates/utopia-server/src/api/chat_retry_tests.rs
+++ b/crates/utopia-server/src/api/chat_retry_tests.rs
@@ -204,6 +204,7 @@ async fn a_question_is_not_answered_twice_at_once() -> anyhow::Result<()> {
utopia_store::conversations::create(&f.pool, f.kb, f.user.id, QUESTION).await?;
let question = append(&f, conversation_id, "user", QUESTION).await?;
let running = f.state.live.begin(conversation_id).await.unwrap();
+ running.start().await;
let Outcome::Refused(status, body) = post(&f, retry(conversation_id, question)).await? else {
panic!("a retry started while an answer was running");
};
@@ -282,3 +283,97 @@ async fn every_question_tells_the_interface_its_stored_id() -> anyhow::Result<()
);
f.cleanup().await
}
+
+/// 重试资格尚未读完时,另一标签页不能订阅这次随后会被拒绝的占位。
+#[tokio::test]
+async fn a_rejected_retry_does_not_publish_a_stream_to_another_tab() -> anyhow::Result<()> {
+ let Some(f) = fixture(Scripted::new(vec![])).await? else {
+ return Ok(());
+ };
+ let application_name = format!("retry-preparation-{}", f.kb.simple());
+ let retry_pool = sqlx::postgres::PgPoolOptions::new()
+ .max_connections(1)
+ .connect_with(
+ (*f.pool.connect_options())
+ .clone()
+ .application_name(&application_name),
+ )
+ .await?;
+ let mut retry_state = f.state.clone();
+ retry_state.pool = retry_pool.clone();
+ let result = tokio::time::timeout(std::time::Duration::from_secs(20), async {
+ let id = utopia_store::conversations::create(&f.pool, f.kb, f.user.id, QUESTION).await?;
+ let question = append(&f, id, "user", QUESTION).await?;
+ append(&f, id, "assistant", "Already answered.").await?;
+ let before = stored(&f, id).await?;
+
+ // 只在资格查询上制造等待;它之前的权限检查读其他表,begin 已经占位。
+ let mut lock = f.pool.begin().await?;
+ sqlx::query("LOCK TABLE conversation_messages IN ACCESS EXCLUSIVE MODE")
+ .execute(&mut *lock)
+ .await?;
+ let blocker: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
+ .fetch_one(&mut *lock)
+ .await?;
+ let pending = chat(
+ State(retry_state),
+ AuthUser(f.user.clone()),
+ Path(f.kb),
+ Json(ChatReq {
+ conversation_id: Some(id),
+ message: QUESTION.into(),
+ retry_message_id: Some(question),
+ }),
+ );
+ tokio::pin!(pending);
+ let waiting = async {
+ while !sqlx::query_scalar::<_, bool>(
+ "SELECT EXISTS(SELECT 1 FROM pg_stat_activity
+ WHERE application_name = $1 AND $2 = ANY(pg_blocking_pids(pid)))",
+ )
+ .bind(&application_name)
+ .bind(blocker)
+ .fetch_one(&f.pool)
+ .await?
+ {}
+ Ok::<_, anyhow::Error>(())
+ };
+ tokio::select! {
+ _ = &mut pending => anyhow::bail!("retry must wait for its qualification query"),
+ result = waiting => result?,
+ }
+ // 直接进入重接的订阅边界,避免单次 poll 只停在 HTTP 权限查询上。
+ let other_tab = f.state.live.attach(id);
+ tokio::pin!(other_tab);
+ anyhow::ensure!(futures_util::poll!(&mut other_tab).is_pending());
+
+ lock.rollback().await?;
+ let refusal = pending
+ .await
+ .err()
+ .ok_or_else(|| anyhow::anyhow!("an answered question was retried"))?
+ .into_response();
+ anyhow::ensure!(refusal.status().as_u16() == 409);
+ let body = axum::body::to_bytes(refusal.into_body(), 65536).await?;
+ anyhow::ensure!(
+ serde_json::from_slice::<serde_json::Value>(&body)?["code"] == "retry_answered"
+ );
+ let other_tab = sse_from(other_tab.await);
+ let body = axum::body::to_bytes(other_tab.into_response().into_body(), 65536).await?;
+ let sse = String::from_utf8_lossy(&body);
+ anyhow::ensure!(sse.contains("event: idle"), "{sse}");
+ anyhow::ensure!(
+ !sse.contains("event: snapshot") && !sse.contains("event: error"),
+ "{sse}"
+ );
+ anyhow::ensure!(stored(&f, id).await? == before);
+ anyhow::ensure!(f.requests().is_empty());
+ drop(f.state.live.begin(id).await?);
+ Ok::<_, anyhow::Error>(())
+ })
+ .await;
+ retry_pool.close().await;
+ f.cleanup().await?;
+ result??;
+ Ok(())
+}
diff --git a/crates/utopia-server/src/api/chat_stream_tests.rs b/crates/utopia-server/src/api/chat_stream_tests.rs
index 29f5fc35..e2c71c4a 100644
--- a/crates/utopia-server/src/api/chat_stream_tests.rs
+++ b/crates/utopia-server/src/api/chat_stream_tests.rs
@@ -2,12 +2,50 @@ use super::*;
use axum::response::IntoResponse;
use std::sync::Arc;
+#[tokio::test]
+async fn a_busy_refusal_preserves_both_subscribers_and_the_terminal() {
+ for stopped in [false, true] {
+ let registry = Arc::new(crate::live::Registry::default());
+ let id = Uuid::now_v7();
+ let handle = registry.begin(id).await.unwrap();
+ handle.start().await;
+ let first = sse_from(registry.attach(id).await);
+ handle.emit(delta_event("partial")).await;
+ let second = sse_from(registry.attach(id).await);
+
+ assert!(matches!(
+ registry.begin(id).await,
+ Err(AppError::CodedConflict {
+ code: "answer_running",
+ ..
+ })
+ ));
+ handle.emit(delta_event(" tail")).await;
+ handle
+ .complete(Frame::new("done", json!({"stopped": stopped}).to_string()))
+ .await;
+
+ for response in [first, second] {
+ let body = axum::body::to_bytes(response.into_response().into_body(), 65536)
+ .await
+ .unwrap();
+ let text = String::from_utf8_lossy(&body);
+ assert!(text.contains("partial") && text.contains(" tail"), "{text}");
+ assert_eq!(text.matches("event: done").count(), 1, "{text}");
+ assert!(text.contains(&format!("\"stopped\":{stopped}")), "{text}");
+ assert!(!text.contains("event: error"), "{text}");
+ }
+ let _next = registry.begin(id).await.unwrap();
+ }
+}
+
#[tokio::test]
async fn reattachment_preserves_terminal_frames_on_both_sides_of_emit() {
for (event, data) in [("done", r#"{"stopped":true}"#), ("error", "safe outcome")] {
let registry = Arc::new(crate::live::Registry::default());
let id = Uuid::now_v7();
let handle = registry.begin(id).await.unwrap();
+ handle.start().await;
handle.emit(delta_event("partial")).await;
let before = sse_from(registry.attach(id).await);
handle.emit(Frame::new(event, data.into())).await;
@@ -44,6 +82,7 @@ async fn lagged_subscribers_receive_an_error_not_done() {
let registry = Arc::new(crate::live::Registry::default());
let id = Uuid::now_v7();
let handle = registry.begin(id).await.unwrap();
+ handle.start().await;
let stream = sse_from(registry.attach(id).await);
for _ in 0..300 {
handle.emit(delta_event("x")).await;
@@ -61,6 +100,7 @@ async fn producer_disappearing_without_an_outcome_ends_in_one_error() {
let registry = Arc::new(crate::live::Registry::default());
let id = Uuid::now_v7();
let handle = registry.begin(id).await.unwrap();
+ handle.start().await;
let stream = sse_from(registry.attach(id).await);
handle.emit(delta_event("partial")).await;
drop(handle);
@@ -83,6 +123,7 @@ async fn first_terminal_freezes_the_snapshot_and_broadcast() {
let registry = Arc::new(crate::live::Registry::default());
let id = Uuid::now_v7();
let handle = registry.begin(id).await.unwrap();
+ handle.start().await;
handle.emit(delta_event("kept")).await;
handle
.emit(error_event("answer_failed", "original error"))
diff --git a/crates/utopia-server/src/live.rs b/crates/utopia-server/src/live.rs
index f92deedb..79f430b0 100644
--- a/crates/utopia-server/src/live.rs
+++ b/crates/utopia-server/src/live.rs
@@ -94,6 +94,7 @@ struct Entry {
tx: broadcast::Sender<Frame>,
snap: Arc<RwLock<Snapshot>>,
cancellation: Cancellation,
+ ready: watch::Sender<bool>,
}
/// 保留取消信号:Stop 可能早于生成器开始等待。
@@ -133,6 +134,7 @@ pub struct Handle {
tx: broadcast::Sender<Frame>,
snap: Arc<RwLock<Snapshot>>,
cancellation: Cancellation,
+ ready: watch::Sender<bool>,
registry: Arc<Registry>,
}
@@ -149,6 +151,11 @@ impl Handle {
self.snap.read().await.clone()
}
+ /// 准备成功后才开放订阅;被拒的重试只释放占位,不会留下等待终态的客户端。
+ pub async fn start(&self) {
+ self.ready.send_replace(true);
+ }
+
/// 发一个事件:记进快照,然后广播。
///
/// **广播时仍然握着快照的写锁**,这一点是必需的。只保证「先写后发」
@@ -216,6 +223,7 @@ impl Registry {
..Snapshot::default()
}));
let cancellation = Cancellation::default();
+ let (ready, _) = watch::channel(false);
entries.insert(
conversation_id,
Entry {
@@ -223,6 +231,7 @@ impl Registry {
tx: tx.clone(),
snap: snap.clone(),
cancellation: cancellation.clone(),
+ ready: ready.clone(),
},
);
Ok(Handle {
@@ -231,6 +240,7 @@ impl Registry {
tx,
snap,
cancellation,
+ ready,
registry: self.clone(),
})
}
@@ -253,11 +263,18 @@ impl Registry {
&self,
conversation_id: Uuid,
) -> Option<(Snapshot, broadcast::Receiver<Frame>)> {
- let (snap, tx) = {
+ let (snap, tx, mut ready) = {
let map = self.0.lock().expect("live registry lock");
let entry = map.get(&conversation_id)?;
- (entry.snap.clone(), entry.tx.clone())
+ (
+ entry.snap.clone(),
+ entry.tx.clone(),
+ entry.ready.subscribe(),
+ )
};
+ // 不持有 ready 的发送端:准备失败释放占位时,等待者会醒来并回到 idle。
+ // 准备成功则接上同一轮,避免过早返回 idle 后错过整个回答。
+ ready.wait_for(|started| *started).await.ok()?;
// **握着快照的读锁再订阅。** `emit` 是握着写锁广播的,所以这一段
// 与任何一次 emit 互斥:拿到的快照与订阅起点严丝合缝,
// 中间那一小段既不会漏、也不会重
@@ -280,9 +297,11 @@ mod tests {
let registry = Arc::new(Registry::default());
let id = Uuid::now_v7();
let old = registry.begin(id).await.unwrap();
+ old.start().await;
old.emit(delta("old")).await;
old.retire();
let current = registry.begin(id).await.unwrap();
+ current.start().await;
current.emit(delta("new")).await;
let (snapshot, mut rx) = registry.attach(id).await.unwrap();
assert_eq!(snapshot.content, "new");
@@ -317,10 +336,12 @@ mod tests {
let id = Uuid::now_v7();
let other_id = Uuid::now_v7();
let other = registry.begin(other_id).await.unwrap();
+ other.start().await;
other.emit(delta("unrelated")).await;
let mut handles = Vec::new();
for index in 0..3 {
let handle = registry.begin(id).await.unwrap();
+ handle.start().await;
if index < 2 {
handle.retire();
}
@@ -350,6 +371,7 @@ mod tests {
let registry = Arc::new(Registry::default());
let id = Uuid::now_v7();
let handle = registry.begin(id).await.unwrap();
+ handle.start().await;
// 两种拿锁顺序都合法:每个增量必须恰好出现在快照或订阅中一次,
// 既不能重复,也不能遗漏。
for index in 0..64 {
@@ -372,6 +394,46 @@ mod tests {
drop(handle);
}
+ #[tokio::test]
+ async fn a_preparing_reservation_is_exclusive_but_not_subscribable() {
+ let registry = Arc::new(Registry::default());
+ let id = Uuid::now_v7();
+ let handle = registry.begin(id).await.unwrap();
+ assert!(matches!(
+ registry.begin(id).await,
+ Err(AppError::CodedConflict {
+ code: "answer_running",
+ ..
+ })
+ ));
+ let attached = registry.attach(id);
+ tokio::pin!(attached);
+ assert!(futures_util::poll!(&mut attached).is_pending());
+ drop(handle);
+ assert!(attached.await.is_none());
+ let _next = registry.begin(id).await.unwrap();
+ }
+
+ #[tokio::test]
+ async fn an_attachment_waiting_for_preparation_joins_the_started_generation() {
+ let registry = Arc::new(Registry::default());
+ let id = Uuid::now_v7();
+ let handle = registry.begin(id).await.unwrap();
+ let attached = registry.attach(id);
+ tokio::pin!(attached);
+ assert!(futures_util::poll!(&mut attached).is_pending());
+
+ handle.start().await;
+ let (snapshot, mut receiver) = attached.await.unwrap();
+ assert_eq!(snapshot.generation_id, handle.generation_id());
+ handle.emit(delta("answer")).await;
+ assert_eq!(receiver.recv().await.unwrap().data, delta("answer").data);
+ handle
+ .complete(Frame::new("done", json!({"stopped":false}).to_string()))
+ .await;
+ assert_eq!(receiver.recv().await.unwrap().event, "done");
+ }
+
#[tokio::test]
async fn admission_is_exclusive_and_dropping_a_reservation_releases_it() {
let registry = Arc::new(Registry::default());
@@ -409,6 +471,7 @@ mod tests {
let registry = Arc::new(Registry::default());
let id = Uuid::now_v7();
let handle = registry.begin(id).await.unwrap();
+ handle.start().await;
let (_, mut receiver) = registry.attach(id).await.unwrap();
handle
.complete(Frame::new("done", json!({"stopped":true}).to_string()))
diff --git a/web/src/chatStream.test.ts b/web/src/chatStream.test.ts
index 2f89b36a..9ecdcce9 100644
--- a/web/src/chatStream.test.ts
+++ b/web/src/chatStream.test.ts
@@ -1,4 +1,7 @@
import { afterEach, describe, expect, it, vi } from "vitest";
+import { S } from "./i18n";
+import { en } from "./i18n/en";
+import { zh } from "./i18n/zh";
import { ApiError, reattachChat, streamChat, type ChatHandlers } from "./api";
vi.mock("./i18n", () => ({
S: {
@@ -91,6 +94,21 @@ describe("a chat stream that fails", () => {
expect(await replay(text)).toEqual([["error", "Worded: start a new conversation"]]);
expect(await replay(text, true)).toEqual([["error", "Worded: start a new conversation"]]);
});
+ it.each([en, zh])("distinguishes mapping retrieval from document search in either stream", async (strings) => {
+ const original = S.err;
+ S.err = strings.err;
+ try {
+ for (const code of ["mapping_search_failed", "search_failed"] as const) {
+ const text = failed({ error: "Could not retrieve relevant mappings.", code });
+ const expected = [["error", strings.err[code]]];
+ expect(await replay(text)).toEqual(expected);
+ expect(await replay(text, true)).toEqual(expected);
+ }
+ expect(strings.err.mapping_search_failed).not.toBe(strings.err.search_failed);
+ } finally {
+ S.err = original;
+ }
+ });
it("keeps the server's sentence for a code the table does not have", async () => {
expect(await replay(failed({ error: "Some new failure", code: "unlisted_code" })))
.toEqual([["error", "Some new failure"]]);
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index c6ad224f..1d09c166 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -63,6 +63,7 @@ export const en = {
answer_failed: "The answer could not be completed. Ask again.",
answer_not_saved: "The answer could not be saved. Try again later.",
search_failed: "The documents could not be searched.",
+ mapping_search_failed: "Relevant data mappings could not be retrieved.",
stream_ended:
"The answer stream ended unexpectedly. Reopen the conversation to check its status.",
stream_lagged: "This page fell behind the answer stream. Reopen the conversation.",
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index 2edb4076..166a0efb 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -63,6 +63,7 @@ export const zh: Strings = {
answer_failed: "这次没能答完,请再问一次。",
answer_not_saved: "回答没能保存,请稍后再试。",
search_failed: "没能检索文档。",
+ mapping_search_failed: "没能检索相关数据映射。",
stream_ended: "回答的连接意外中断了。重新打开这个对话,看看结果如何。",
stream_lagged: "这个页面跟不上回答的速度,请重新打开这个对话。",
model_out_of_credit: "模型账号付不起这次请求。去给账号充值,或者在「管理 → 模型」里换一个能用的端点。",