World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
An empty balance is final, so the queue should stop asking (#201)
WaylandYang2721942bc7a8
diff --git a/crates/utopia-core/src/error.rs b/crates/utopia-core/src/error.rsindex 4d897bbd..7690d22f 100644--- a/crates/utopia-core/src/error.rs+++ b/crates/utopia-core/src/error.rs@@ -50,3 +50,33 @@ impl AppError {}pub type AppResult<T> = Result<T, AppError>;++/// 标在一个**不会因为重试而变好**的失败上(见 issue #195)。+///+/// 队列的默认假设是「再等一会儿也许就好了」,多数失败确实如此:端点抖一下、+/// 数据库忙一瞬、限流一分钟就过去。余额耗尽不是——三次重试隔着 30 秒、2 分钟、+/// 4 分半,七分钟里没有人会去充值,重试只是把同一句错误重说三遍,而运维需要+/// 看见的那条「失败」被推迟了七分钟才出现。+///+/// **判据留在处理器那一侧,不在队列里。** 什么算没救跟领域有关——+/// `utopia-store` 看不见 `utopia-llm` 的错误类型,也不该看见。处理器把这个标记+/// 挂上去(`err.context(Terminal)`),队列只问「挂了没有」。+///+/// 挂上它不影响别的:告警照报(`observe_job_failure` 一并认这个标记,+/// 否则失败得更快反而没人被告知),`last_error` 照写。+#[derive(Debug, Clone, Copy)]+pub struct Terminal;++impl std::fmt::Display for Terminal {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str("will not recover by retrying")+ }+}++impl std::error::Error for Terminal {}++/// 这次失败被标成不必重试了吗。**沿整条 context 链找**——处理器挂上标记之后,+/// 上层还会继续 `context(...)`,只看最外层就等于没看+pub fn is_terminal(err: &anyhow::Error) -> bool {+ err.chain().any(|e| e.is::<Terminal>())+}diff --git a/crates/utopia-core/src/lib.rs b/crates/utopia-core/src/lib.rsindex 761be44c..d55bdcdc 100644--- a/crates/utopia-core/src/lib.rs+++ b/crates/utopia-core/src/lib.rs@@ -4,4 +4,4 @@ pub mod config;pub mod error;pub mod models;-pub use error::{AppError, AppResult};+pub use error::{is_terminal, AppError, AppResult, Terminal};diff --git a/crates/utopia-server/src/alerting.rs b/crates/utopia-server/src/alerting.rsindex 89f36fdb..3327f342 100644--- a/crates/utopia-server/src/alerting.rs+++ b/crates/utopia-server/src/alerting.rs@@ -15,9 +15,10 @@ const RETAIN_DAYS: i32 = 30;/// 任务失败时看一眼是不是模型端点不可用。///-/// **只在任务最终放弃时报**(`attempts >= max_attempts`)。中间那几次重试报了-/// 就是同一件事发三遍——重试本来就是为了不惊动人,报出去等于把它的意义抵消掉。-/// 一个任务至多一条告警,不需要任何去重状态。+/// **只在任务最终放弃时报**——次数用完,或者这次失败被标成了不必重试+/// ([`hopeless`],#195)。中间那几次重试报了就是同一件事发三遍:重试本来就是+/// 为了不惊动人,报出去等于把它的意义抵消掉。一个任务至多一条告警,+/// 不需要任何去重状态。////// 判据是错误的**类型**,不匹配错误文本——调用链上任何一层加一句 context/// 都会改文本。分类本身抽在 [`alert_for`]。@@ -26,7 +27,9 @@ pub async fn observe_job_failure(job: &utopia_store::jobs::Job,err: &anyhow::Error,) {- if job.attempts < job.max_attempts {+ // 标成不必重试的那些**这一次就是最后一次**,所以现在就报(#195)。+ // 漏掉这一句的后果是失败得更快、却反而没人被告知——比不改还糟+ if job.attempts < job.max_attempts && !utopia_core::is_terminal(err) {return;}let Some((kind, severity)) = alert_for(err) else {@@ -53,6 +56,17 @@ pub async fn observe_job_failure(state.emit_alert();}+/// 这次失败还值不值得重试。+///+/// **只有欠费是没救的。** 限流会自己恢复,端点不可达可能是重启中的服务,+/// 别的错误更没有理由一次定生死——只有余额不会在七分钟里自己长回来(#195)。+///+/// 判据与 [`alert_for`] 同源(`utopia_llm::out_of_credit`,类型不是文本),+/// 于是「报成 error 那一类」与「不再重试那一类」永远是同一类,不会分叉。+pub fn hopeless(err: &anyhow::Error) -> bool {+ utopia_llm::out_of_credit(err).is_some()+}+/// 一次任务失败该报哪一类告警,报不报。////// **抽成纯函数是为了能测**:`observe_job_failure` 要 `AppState` 和一个真数据库,@@ -212,4 +226,33 @@ mod tests {let err = anyhow::anyhow!("LLM request failed (401 Unauthorized): bad key");assert_eq!(alert_for(&err), None);}++ /// **余额不会在七分钟里自己长回来**,所以那三次退避重试只是把同一句错误+ /// 重说三遍,还把该看见的失败推迟了七分钟(#195)。+ #[test]+ fn an_empty_balance_is_not_worth_retrying() {+ let err = anyhow::Error::new(OutOfCredit {+ status: 402,+ detail: "insufficient balance".into(),+ })+ .context("抽取失败");+ assert!(super::hopeless(&err));+ }++ /// 而限流正是这套退避的服务对象——两类分开(#176)的意义就在这里。+ #[test]+ fn a_rate_limit_still_gets_its_retries() {+ let err = anyhow::Error::new(RateLimited {+ status: 429,+ retry_after: Some(std::time::Duration::from_secs(30)),+ detail: "TPM limit reached".into(),+ });+ assert!(!super::hopeless(&err));+ }++ /// 反面:普通失败照常重试。一次网络抖动不该一次定生死。+ #[test]+ fn an_ordinary_failure_still_gets_its_retries() {+ assert!(!super::hopeless(&anyhow::anyhow!("结果解析失败")));+ }}diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex d23cabeb..56a1a500 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -131,7 +131,15 @@ async fn main() -> anyhow::Result<()> {async move {// 任务失败时看一眼是不是模型端点连不上——那是系统级故障,// 而现在它只会留在 jobs.last_error 里,没有任何界面看得到- let result = dispatch(&st, &job).await;+ //+ // 没救的失败在这里挂上标记,队列据此不再退避重试(#195):+ // 判据在这一侧,因为 `utopia-store` 看不见 LLM 的错误类型+ let result = dispatch(&st, &job)+ .await+ .map_err(|e| match alerting::hopeless(&e) {+ true => e.context(utopia_core::Terminal),+ false => e,+ });if let Err(e) = &result {alerting::observe_job_failure(&st, &job, e).await;}diff --git a/crates/utopia-store/src/jobs.rs b/crates/utopia-store/src/jobs.rsindex 77d3bfe2..0a9d689f 100644--- a/crates/utopia-store/src/jobs.rs+++ b/crates/utopia-store/src/jobs.rs@@ -1,5 +1,6 @@//! 任务队列:Postgres `FOR UPDATE SKIP LOCKED` 消费。-//! worker 与 API 同进程(tokio task),失败按 30s * attempts² 退避重试。+//! worker 与 API 同进程(tokio task),失败按 30s * attempts² 退避重试——+//! 除非处理器把它标成了 `utopia_core::Terminal`,那种一次就到此为止(见 [`retry_delay`])。//! 并发消费:调度循环按"运行中 < 目标数"续派,任务在独立 task 执行;//! 目标数经 AtomicUsize 热读——系统设置里改并发即时生效,无需重启。@@ -61,29 +62,51 @@ async fn mark_done(pool: &PgPool, id: i64) -> AppResult<()> {Ok(())}-async fn mark_failed(pool: &PgPool, job: &Job, err: &str) -> AppResult<()> {- if job.attempts >= job.max_attempts {+/// 下一次重试等多久;`None` = 到此为止。+///+/// 两个理由到此为止:**次数用完了**,或者**处理器说了这次不会因为重试而变好**+/// (`utopia_core::Terminal`,见 issue #195)。后者以前不存在,于是余额耗尽的+/// 任务照样把三次退避走完——七分钟里余额不会自己长回来,那三次只是把同一句+/// 错误重说三遍,还把运维该看见的「失败」推迟了七分钟。+///+/// 限流相反,它正是这套退避的服务对象:配额会自己恢复。#176 把两类分开就是+/// 为了让它们各走各的路,而重试策略当时没跟上。+///+/// 抽成纯函数是为了能测——决定在这里,写库只是把决定落下去。+fn retry_delay(attempts: i32, max_attempts: i32, terminal: bool) -> Option<i64> {+ if terminal || attempts >= max_attempts {+ return None;+ }+ Some(30i64 * i64::from(attempts) * i64::from(attempts))+}++async fn mark_failed(pool: &PgPool, job: &Job, err: &anyhow::Error) -> AppResult<()> {+ let text = format!("{err:#}");+ let Some(backoff_secs) = retry_delay(+ job.attempts,+ job.max_attempts,+ utopia_core::is_terminal(err),+ ) else {sqlx::query("UPDATE jobs SET status = 'failed', last_error = $2, updated_at = now() WHERE id = $1",).bind(job.id)- .bind(err)- .execute(pool)- .await?;- } else {- let backoff_secs = 30i64 * i64::from(job.attempts) * i64::from(job.attempts);- sqlx::query(- "UPDATE jobs SET status = 'queued', last_error = $2,- run_at = now() + make_interval(secs => $3::float8),- updated_at = now()- WHERE id = $1",- )- .bind(job.id)- .bind(err)- .bind(backoff_secs as f64)+ .bind(&text).execute(pool).await?;- }+ return Ok(());+ };+ sqlx::query(+ "UPDATE jobs SET status = 'queued', last_error = $2,+ run_at = now() + make_interval(secs => $3::float8),+ updated_at = now()+ WHERE id = $1",+ )+ .bind(job.id)+ .bind(&text)+ .bind(backoff_secs as f64)+ .execute(pool)+ .await?;Ok(())}@@ -137,7 +160,7 @@ whereOk(()) => mark_done(&pool, job.id).await,Err(e) => {tracing::warn!(job_id = job.id, kind = %job.kind, error = %e, "任务执行失败");- mark_failed(&pool, &job, &e.to_string()).await+ mark_failed(&pool, &job, &e).await}};if let Err(e) = outcome {@@ -154,3 +177,23 @@ where}}}++#[cfg(test)]+mod tests {+ use super::retry_delay;++ /// 退避照旧:30s、120s、270s,第三次之后放弃。+ #[test]+ fn an_ordinary_failure_backs_off_and_then_gives_up() {+ assert_eq!(retry_delay(1, 3, false), Some(30));+ assert_eq!(retry_delay(2, 3, false), Some(120));+ assert_eq!(retry_delay(3, 3, false), None);+ }++ /// 标成没救的**第一次就到此为止**——那三次退避加起来是七分钟,+ /// 而余额不会在七分钟里自己长回来(#195)。+ #[test]+ fn a_terminal_failure_does_not_spend_the_budget() {+ assert_eq!(retry_delay(1, 3, true), None);+ }+}