World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
An empty balance stops open extraction at the refused chunk, so the job fails once and raises the alert (#1118)
Asaf Shitriteee837b95b39
diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rsindex adf877e2..86c0fbb4 100644--- a/crates/utopia-server/src/extraction_open.rs+++ b/crates/utopia-server/src/extraction_open.rs@@ -49,7 +49,10 @@ struct ChunkParsed {/// 要按分块顺序。`pushed`(0054)的块本身就是契约,不调模型,直接解析。////// `Ok(None)` 是这一块跳过了:调用失败或回复解析不出,原因已经记进 `drop_signal` 和-/// `unextracted`,主循环接着走下一块。`Err` 只有一种:对话模型没配。+/// `unextracted`,主循环接着走下一块。+///+/// `Err` has two causes: no chat model, or an empty balance (#1095). An empty balance+/// stops the document, and the error keeps its chain for `alerting::hopeless`.////// 被接管的检查(epoch)留在主循环顶上,不在这里:它要在调模型之前#[allow(clippy::too_many_arguments)]@@ -88,6 +91,7 @@ async fn call_and_parse_one_chunk(.await{Ok(r) => r,+ Err(e) if utopia_llm::out_of_credit(&e).is_some() => return Err(e),Err(e) => {tracing::warn!(%document_id, seq = chunk.seq, error = %e, "开放抽取调用失败,跳过该分块");drop_signal(@@ -1191,6 +1195,10 @@ pub(crate) async fn run_open(#[path = "extraction_open_known_flow_tests.rs"]mod known_flow_tests;+#[cfg(test)]+#[path = "extraction_open_out_of_credit_tests.rs"]+mod out_of_credit_tests;+#[cfg(test)]mod tests {use super::*;diff --git a/crates/utopia-server/src/extraction_open_out_of_credit_tests.rs b/crates/utopia-server/src/extraction_open_out_of_credit_tests.rsnew file mode 100644index 00000000..dd5a53fe--- /dev/null+++ b/crates/utopia-server/src/extraction_open_out_of_credit_tests.rs@@ -0,0 +1,139 @@+//! An empty balance stops open extraction, fails the job once and raises the alert (#1095).+//! Skipped without `UTOPIA_DATABASE_URL`.+use super::*;+use axum::{extract::State, http::StatusCode, response::IntoResponse, routing::post, Router};+use serde_json::json;+use std::sync::{Arc, Mutex};+use uuid::Uuid;++/// `Some` is a reply, `None` is the empty-balance 429 from OpenAI.+type Script = Arc<Mutex<Vec<Option<String>>>>;++async fn reply(State(script): State<Script>) -> axum::response::Response {+ let Some(text) = script.lock().unwrap().remove(0) else {+ let body = json!({"error": {"message": "You exceeded your current quota.",+ "type": "insufficient_quota", "code": "credit_balance_exhausted"}});+ return (StatusCode::TOO_MANY_REQUESTS, axum::Json(body)).into_response();+ };+ let frame = json!({"choices":[{"delta":{"content":text},"finish_reason":"stop"}]});+ (+ [("content-type", "text/event-stream")],+ format!("data: {frame}\n\ndata: [DONE]\n\n"),+ )+ .into_response()+}++#[tokio::test]+async fn an_empty_balance_stops_the_document_and_keeps_what_was_applied() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = sqlx::PgPool::connect(&url).await?;+ utopia_store::db::migrate(&pool).await?;+ let [org, ws, kb, doc] = [(); 4].map(|_| Uuid::now_v7());+ sqlx::raw_sql(&format!(+ "INSERT INTO organizations(id,name) VALUES ('{org}','open-out-of-credit');+ INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','open-out-of-credit');+ INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','open-out-of-credit');+ INSERT INTO documents(id,kb_id,filename,sha256) VALUES ('{doc}','{kb}','acme.txt','x');+ INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES+ (gen_random_uuid(),'{kb}','{doc}',0,'Acme is based in London.'),+ (gen_random_uuid(),'{kb}','{doc}',1,'Acme runs the harbour facility.'),+ (gen_random_uuid(),'{kb}','{doc}',2,'Acme was founded in 1990.');"+ ))+ .execute(&pool)+ .await?;+ let first = json!({+ "e": [["Acme", "organization", 1]],+ "s": [["Acme is based in London.", "Acme", "based in", null, "London", null, null, null]],+ "n": []+ });+ let script: Script = Arc::new(Mutex::new(vec![Some(first.to_string()), None, None]));+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;+ let endpoint = format!("http://{}", listener.local_addr()?);+ let router = Router::new()+ .route("/chat/completions", post(reply))+ .with_state(script.clone());+ tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });+ utopia_store::settings::upsert(+ &pool,+ ws,+ Some(&endpoint),+ None,+ Some("scripted"),+ None,+ None,+ None,+ None,+ )+ .await?;+ let dir = tempfile::tempdir()?;+ let cfg = utopia_core::config::AppConfig {+ data_dir: dir.path().to_string_lossy().into_owned(),+ ..Default::default()+ };+ let search = Arc::new(utopia_search::SearchIndex::open(+ &dir.path().join("search"),+ )?);+ let state = AppState::new(pool.clone(), &cfg, search, "test-only".into());+ let alerts_before: Vec<Uuid> = sqlx::query_scalar("SELECT id FROM alerts")+ .fetch_all(&pool)+ .await?;++ let run = async {+ let Err(err) = crate::extraction::extract_document(&state, doc, Proposer::default()).await+ else {+ anyhow::bail!("extraction succeeded with no balance");+ };+ // As `main.rs` does for a hopeless error+ anyhow::ensure!(crate::alerting::hopeless(&err), "not hopeless: {err:#}");+ let err = err.context(utopia_core::Terminal);+ let job = utopia_store::jobs::Job {+ id: 0,+ kind: "extract_document".into(),+ payload: json!({}),+ attempts: 1,+ max_attempts: 3,+ };+ crate::alerting::observe_job_failure(&state, &job, &err).await;+ let alerts: Vec<String> = sqlx::query_scalar(+ "DELETE FROM alerts WHERE NOT (id = ANY($1)) AND detail->>'job' = 'extract_document'+ RETURNING kind",+ )+ .bind(&alerts_before)+ .fetch_all(&pool)+ .await?;+ let (facts, extracted, status): (i64, Vec<bool>, String) = sqlx::query_as(+ "SELECT (SELECT count(*) FROM facts WHERE kb_id=$1 AND layer='open'),+ (SELECT array_agg(extracted_at IS NOT NULL ORDER BY seq)+ FROM chunks WHERE document_id=$2),+ graph_status+ FROM documents WHERE id=$2",+ )+ .bind(kb)+ .bind(doc)+ .fetch_one(&pool)+ .await?;+ Ok((alerts, facts, extracted, status))+ }+ .await;+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&pool)+ .await?;+ let (alerts, facts, extracted, status) = run?;+ assert_eq!(+ script.lock().unwrap().len(),+ 1,+ "the third chunk is not called"+ );+ assert_eq!(alerts, vec![utopia_store::alerts::kind::LLM_OUT_OF_CREDIT]);+ assert_eq!(facts, 1, "chunk 0 stays in the graph");+ assert_eq!(+ extracted,+ vec![true, false, false],+ "the next run resumes at chunk 1"+ );+ assert_eq!(status, "failed");+ Ok(())+}