Skip to content

World's first open-source enterprise world model.

Keep fallback retrieval failures distinct from empty results (#850)

Maya Kidf6ce402a7155
diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rs
index 7e6a81bb..d7e7cc0c 100644
--- a/crates/utopia-server/src/api/chat.rs
+++ b/crates/utopia-server/src/api/chat.rs
@@ -926,9 +926,14 @@ fn legacy_rag(
client: utopia_llm::LlmClient,
) -> impl Stream<Item = Frame> {
async_stream::stream! {
- let chunks = retrieval::hybrid(&state, kb_id, workspace_id, &query, 8, None)
- .await
- .unwrap_or_default();
+ let chunks = match retrieval::hybrid(&state, kb_id, workspace_id, &query, 8, None).await {
+ Ok(chunks) => chunks,
+ Err(error) => {
+ tracing::warn!(%error, "fallback document retrieval failed");
+ yield error_event("Could not search the documents.");
+ return;
+ }
+ };
let legacy_sources: Vec<serde_json::Value> = chunks
.iter()
.enumerate()
diff --git a/crates/utopia-server/src/api/chat_empty_reply_tests.rs b/crates/utopia-server/src/api/chat_empty_reply_tests.rs
index 423d81dc..b53b4945 100644
--- a/crates/utopia-server/src/api/chat_empty_reply_tests.rs
+++ b/crates/utopia-server/src/api/chat_empty_reply_tests.rs
@@ -306,6 +306,8 @@ async fn a_reply_that_stays_empty_is_an_error_after_one_retry() -> anyhow::Resul
f.cleanup().await
}
+#[path = "chat_fallback_tests.rs"]
+mod fallback_tests;
#[path = "chat_registry_tests.rs"]
mod registry_tests;
#[path = "chat_sources_tests.rs"]
diff --git a/crates/utopia-server/src/api/chat_fallback_tests.rs b/crates/utopia-server/src/api/chat_fallback_tests.rs
new file mode 100644
index 00000000..8631e1b5
--- /dev/null
+++ b/crates/utopia-server/src/api/chat_fallback_tests.rs
@@ -0,0 +1,159 @@
+//! Provider rejection reaches real retrieval before any plain RAG request.
+use super::*;
+use axum::http::StatusCode;
+
+#[derive(Clone)]
+struct FallbackModel {
+ pool: sqlx::PgPool,
+ break_retrieval: bool,
+ status: u16,
+ seen: Arc<Mutex<Vec<serde_json::Value>>>,
+}
+
+async fn respond(
+ State(m): State<FallbackModel>,
+ Json(body): Json<serde_json::Value>,
+) -> axum::response::Response {
+ let tools = body.get("tools").is_some();
+ m.seen.lock().unwrap().push(body);
+ if tools {
+ // Authentication, settings and user persistence have already succeeded.
+ // Only the ensuing retrieval sees the closed database pool.
+ if m.break_retrieval {
+ m.pool.close().await;
+ }
+ return (
+ StatusCode::from_u16(m.status).unwrap(),
+ Json(json!({"error":{"message":"tools unsupported"}})),
+ )
+ .into_response();
+ }
+ ([ ("content-type", "text/event-stream") ], "data: {\"choices\":[{\"delta\":{\"content\":\"Fallback answer.\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n").into_response()
+}
+
+async fn exercise(status: u16, break_retrieval: bool, hit: bool) -> anyhow::Result<()> {
+ let Some(mut f) = fixture(Scripted::new(vec![])).await? else {
+ return Ok(());
+ };
+ if hit {
+ let doc = utopia_store::documents::create(
+ &f.pool,
+ f.kb,
+ "fallback.md",
+ "text/plain",
+ 20,
+ "fallback-test",
+ None,
+ None,
+ None,
+ )
+ .await?;
+ let chunks = utopia_store::documents::replace_chunks(
+ &f.pool,
+ f.kb,
+ doc.id,
+ &[utopia_ingest::ChunkPiece {
+ seq: 0,
+ text: "orchard evidence".into(),
+ char_start: 0,
+ char_end: 16,
+ heading: None,
+ provenance: utopia_ingest::Provenance::stated(),
+ }],
+ )
+ .await?;
+ f.state
+ .search
+ .reindex_document(&f.kb.to_string(), &doc.id.to_string(), &chunks)?;
+ }
+ let model = FallbackModel {
+ pool: f.pool.clone(),
+ break_retrieval,
+ status,
+ seen: Default::default(),
+ };
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
+ let base = format!("http://{}", listener.local_addr()?);
+ let app = axum::Router::new()
+ .route("/chat/completions", axum::routing::post(respond))
+ .route(
+ "/embeddings",
+ axum::routing::post(|| async { StatusCode::INTERNAL_SERVER_ERROR }),
+ )
+ .with_state(model.clone());
+ let server = tokio::spawn(async move { axum::serve(listener, app).await });
+ sqlx::query("UPDATE llm_settings SET chat_base_url=$1,embed_base_url=$1,embed_model=$2 WHERE workspace_id=(SELECT workspace_id FROM knowledge_bases WHERE id=$3)")
+ .bind(base).bind(if hit {Some("broken-embedding")} else {None}).bind(f.kb).execute(&f.pool).await?;
+ let response = tokio::time::timeout(std::time::Duration::from_secs(20), f.ask("orchard")).await;
+ // Reconnect solely to read the result and clean the isolated fixture.
+ if break_retrieval {
+ f.pool = sqlx::PgPool::connect(&utopia_store::test_db::url().unwrap()).await?;
+ }
+ let result = async {
+ let sse = response??;
+ let seen = model.seen.lock().unwrap().clone();
+ let plain: Vec<_> = seen.iter().filter(|r| r.get("tools").is_none()).collect();
+ let rejected = status == 400 || status == 422;
+ if break_retrieval || !rejected {
+ anyhow::ensure!(
+ sse.contains("event: error") && !sse.contains("event: done"),
+ "{sse}"
+ );
+ anyhow::ensure!(
+ plain.is_empty(),
+ "retrieval/provider failure must not request a RAG answer: {seen:?}"
+ );
+ anyhow::ensure!(f.stored_answer().await?.is_none());
+ if break_retrieval {
+ anyhow::ensure!(sse.contains("Could not search the documents."), "{sse}");
+ anyhow::ensure!(!sse.contains("pool closed") && !sse.contains("postgres"));
+ }
+ } else {
+ anyhow::ensure!(
+ sse.contains("event: done") && sse.contains("Fallback answer."),
+ "{sse}"
+ );
+ anyhow::ensure!(plain.len() == 1);
+ anyhow::ensure!(f.stored_answer().await?.as_deref() == Some("Fallback answer."));
+ if hit {
+ anyhow::ensure!(
+ plain[0].to_string().contains("orchard evidence")
+ && sse.contains("fallback.md")
+ );
+ } else {
+ anyhow::ensure!(sse.contains("data: []"));
+ }
+ }
+ anyhow::ensure!(
+ seen.iter().filter(|r| r.get("tools").is_some()).count()
+ == if rejected { 2 } else { 1 },
+ "compatibility retries: {seen:?}"
+ );
+ Ok::<_, anyhow::Error>(())
+ }
+ .await;
+ server.abort();
+ let _ = server.await;
+ f.cleanup().await?;
+ result
+}
+
+#[tokio::test]
+async fn fallback_retrieval_failure_is_not_an_empty_result() -> anyhow::Result<()> {
+ exercise(400, true, false).await
+}
+#[tokio::test]
+async fn fallback_empty_search_still_answers() -> anyhow::Result<()> {
+ exercise(400, false, false).await
+}
+#[tokio::test]
+async fn fallback_embedding_failure_keeps_bm25_evidence() -> anyhow::Result<()> {
+ exercise(422, false, true).await
+}
+#[tokio::test]
+async fn other_provider_failures_do_not_enter_fallback() -> anyhow::Result<()> {
+ for status in [401, 402, 429, 500] {
+ exercise(status, false, false).await?;
+ }
+ Ok(())
+}