World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Keep fallback retrieval failures distinct from empty results (#850)
Maya Kidf6ce402a7155
diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rsindex 7e6a81bb..d7e7cc0c 100644--- a/crates/utopia-server/src/api/chat.rs+++ b/crates/utopia-server/src/api/chat.rs@@ -926,9 +926,14 @@ fn legacy_rag(client: utopia_llm::LlmClient,) -> impl Stream<Item = Frame> {async_stream::stream! {- let chunks = retrieval::hybrid(&state, kb_id, workspace_id, &query, 8, None)- .await- .unwrap_or_default();+ let chunks = match retrieval::hybrid(&state, kb_id, workspace_id, &query, 8, None).await {+ Ok(chunks) => chunks,+ Err(error) => {+ tracing::warn!(%error, "fallback document retrieval failed");+ yield error_event("Could not search the documents.");+ return;+ }+ };let legacy_sources: Vec<serde_json::Value> = chunks.iter().enumerate()diff --git a/crates/utopia-server/src/api/chat_empty_reply_tests.rs b/crates/utopia-server/src/api/chat_empty_reply_tests.rsindex 423d81dc..b53b4945 100644--- a/crates/utopia-server/src/api/chat_empty_reply_tests.rs+++ b/crates/utopia-server/src/api/chat_empty_reply_tests.rs@@ -306,6 +306,8 @@ async fn a_reply_that_stays_empty_is_an_error_after_one_retry() -> anyhow::Resulf.cleanup().await}+#[path = "chat_fallback_tests.rs"]+mod fallback_tests;#[path = "chat_registry_tests.rs"]mod registry_tests;#[path = "chat_sources_tests.rs"]diff --git a/crates/utopia-server/src/api/chat_fallback_tests.rs b/crates/utopia-server/src/api/chat_fallback_tests.rsnew file mode 100644index 00000000..8631e1b5--- /dev/null+++ b/crates/utopia-server/src/api/chat_fallback_tests.rs@@ -0,0 +1,159 @@+//! Provider rejection reaches real retrieval before any plain RAG request.+use super::*;+use axum::http::StatusCode;++#[derive(Clone)]+struct FallbackModel {+ pool: sqlx::PgPool,+ break_retrieval: bool,+ status: u16,+ seen: Arc<Mutex<Vec<serde_json::Value>>>,+}++async fn respond(+ State(m): State<FallbackModel>,+ Json(body): Json<serde_json::Value>,+) -> axum::response::Response {+ let tools = body.get("tools").is_some();+ m.seen.lock().unwrap().push(body);+ if tools {+ // Authentication, settings and user persistence have already succeeded.+ // Only the ensuing retrieval sees the closed database pool.+ if m.break_retrieval {+ m.pool.close().await;+ }+ return (+ StatusCode::from_u16(m.status).unwrap(),+ Json(json!({"error":{"message":"tools unsupported"}})),+ )+ .into_response();+ }+ ([ ("content-type", "text/event-stream") ], "data: {\"choices\":[{\"delta\":{\"content\":\"Fallback answer.\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n").into_response()+}++async fn exercise(status: u16, break_retrieval: bool, hit: bool) -> anyhow::Result<()> {+ let Some(mut f) = fixture(Scripted::new(vec![])).await? else {+ return Ok(());+ };+ if hit {+ let doc = utopia_store::documents::create(+ &f.pool,+ f.kb,+ "fallback.md",+ "text/plain",+ 20,+ "fallback-test",+ None,+ None,+ None,+ )+ .await?;+ let chunks = utopia_store::documents::replace_chunks(+ &f.pool,+ f.kb,+ doc.id,+ &[utopia_ingest::ChunkPiece {+ seq: 0,+ text: "orchard evidence".into(),+ char_start: 0,+ char_end: 16,+ heading: None,+ provenance: utopia_ingest::Provenance::stated(),+ }],+ )+ .await?;+ f.state+ .search+ .reindex_document(&f.kb.to_string(), &doc.id.to_string(), &chunks)?;+ }+ let model = FallbackModel {+ pool: f.pool.clone(),+ break_retrieval,+ status,+ seen: Default::default(),+ };+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;+ let base = format!("http://{}", listener.local_addr()?);+ let app = axum::Router::new()+ .route("/chat/completions", axum::routing::post(respond))+ .route(+ "/embeddings",+ axum::routing::post(|| async { StatusCode::INTERNAL_SERVER_ERROR }),+ )+ .with_state(model.clone());+ let server = tokio::spawn(async move { axum::serve(listener, app).await });+ sqlx::query("UPDATE llm_settings SET chat_base_url=$1,embed_base_url=$1,embed_model=$2 WHERE workspace_id=(SELECT workspace_id FROM knowledge_bases WHERE id=$3)")+ .bind(base).bind(if hit {Some("broken-embedding")} else {None}).bind(f.kb).execute(&f.pool).await?;+ let response = tokio::time::timeout(std::time::Duration::from_secs(20), f.ask("orchard")).await;+ // Reconnect solely to read the result and clean the isolated fixture.+ if break_retrieval {+ f.pool = sqlx::PgPool::connect(&utopia_store::test_db::url().unwrap()).await?;+ }+ let result = async {+ let sse = response??;+ let seen = model.seen.lock().unwrap().clone();+ let plain: Vec<_> = seen.iter().filter(|r| r.get("tools").is_none()).collect();+ let rejected = status == 400 || status == 422;+ if break_retrieval || !rejected {+ anyhow::ensure!(+ sse.contains("event: error") && !sse.contains("event: done"),+ "{sse}"+ );+ anyhow::ensure!(+ plain.is_empty(),+ "retrieval/provider failure must not request a RAG answer: {seen:?}"+ );+ anyhow::ensure!(f.stored_answer().await?.is_none());+ if break_retrieval {+ anyhow::ensure!(sse.contains("Could not search the documents."), "{sse}");+ anyhow::ensure!(!sse.contains("pool closed") && !sse.contains("postgres"));+ }+ } else {+ anyhow::ensure!(+ sse.contains("event: done") && sse.contains("Fallback answer."),+ "{sse}"+ );+ anyhow::ensure!(plain.len() == 1);+ anyhow::ensure!(f.stored_answer().await?.as_deref() == Some("Fallback answer."));+ if hit {+ anyhow::ensure!(+ plain[0].to_string().contains("orchard evidence")+ && sse.contains("fallback.md")+ );+ } else {+ anyhow::ensure!(sse.contains("data: []"));+ }+ }+ anyhow::ensure!(+ seen.iter().filter(|r| r.get("tools").is_some()).count()+ == if rejected { 2 } else { 1 },+ "compatibility retries: {seen:?}"+ );+ Ok::<_, anyhow::Error>(())+ }+ .await;+ server.abort();+ let _ = server.await;+ f.cleanup().await?;+ result+}++#[tokio::test]+async fn fallback_retrieval_failure_is_not_an_empty_result() -> anyhow::Result<()> {+ exercise(400, true, false).await+}+#[tokio::test]+async fn fallback_empty_search_still_answers() -> anyhow::Result<()> {+ exercise(400, false, false).await+}+#[tokio::test]+async fn fallback_embedding_failure_keeps_bm25_evidence() -> anyhow::Result<()> {+ exercise(422, false, true).await+}+#[tokio::test]+async fn other_provider_failures_do_not_enter_fallback() -> anyhow::Result<()> {+ for status in [401, 402, 429, 500] {+ exercise(status, false, false).await?;+ }+ Ok(())+}