World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Extraction waits in the queue, not in the worker (#640)
Royce125e18e203e2
diff --git a/crates/utopia-core/src/error.rs b/crates/utopia-core/src/error.rsindex ce9dcf36..4247219f 100644--- a/crates/utopia-core/src/error.rs+++ b/crates/utopia-core/src/error.rs@@ -86,10 +86,100 @@ pub fn is_terminal(err: &anyhow::Error) -> bool {err.is::<Terminal>()}+/// 这次失败不是错了,是**等一下再试**——把任务挂回 `queued`,不烧预算(#526)。+///+/// 跟 [`Terminal`] 是一对:一个是「不会变好,别重试了」,一个是「现在做不了,+/// 等一会儿再做」。两者都是领域判断——抽取器知道本体向量还没补齐,队列看不见+/// 那张图。处理器挂标记(`err.context(Deferred::new(Duration::from_secs(30)))`),+/// `mark_failed` 认这个标记并把 `run_at` 推到未来、把 `attempts` 退回去。+///+/// 跟默认退避的区别:默认的 `30s × attempts²` 是错的——它把「再试一次值得」+/// 的失败按次数指数延后,而 `Deferred` 的语义是「这个时间点过了再来」,+/// 与失败次数无关。两次都因为同一个等待挂回队列,下次 `run_at` 都是同一个+/// 偏移,不会有 60s、120s 的递增。+///+/// **与 `Terminal` 同挂时 `Terminal` 赢**:`mark_failed` 先问 [`is_terminal`]。+/// 等待也有期限——从第一次挂回去算起超过 `jobs::DEFER_WINDOW_SECS` 还在等,就不再算等待,+/// 按普通失败退避、烧预算,免得一个永远补不齐的索引让任务永远排着。+#[derive(Debug, Clone, Copy)]+pub struct Deferred {+ pub retry_in: std::time::Duration,+}++impl Deferred {+ pub fn new(retry_in: std::time::Duration) -> Self {+ Self { retry_in }+ }+}++impl std::fmt::Display for Deferred {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ // 落进 `last_error` 的话要让人能直接读懂,所以写明秒数+ write!(f, "deferred; retry in {:?}", self.retry_in)+ }+}++impl std::error::Error for Deferred {}++/// 挂着的 `Deferred` 要等多久。与 [`is_terminal`] 同理用 anyhow 自己的 downcast,+/// 挂成 context 还是作为根都认得+pub fn is_deferred(err: &anyhow::Error) -> Option<std::time::Duration> {+ err.downcast_ref::<Deferred>().map(|d| d.retry_in)+}+#[cfg(test)]mod tests {use super::*;use anyhow::Context;+ use std::time::Duration;++ /// 标记作为根、外面套一句说明(`rss_full_content.rs` 的写法)+ #[test]+ fn terminal_is_recognised_through_context() {+ let err: anyhow::Error = anyhow::Error::new(Terminal).context("balance gone: third retry");+ assert!(is_terminal(&err));+ assert!(is_deferred(&err).is_none());+ }++ #[test]+ fn deferred_is_recognised_through_context() {+ let err: anyhow::Error = anyhow::Error::new(Deferred::new(Duration::from_secs(30)))+ .context("waiting on ontology index");+ assert!(!is_terminal(&err));+ assert_eq!(is_deferred(&err), Some(Duration::from_secs(30)));+ }++ /// 标记挂成 context(`main.rs` 与 `extraction::run` 的写法),外面再套一层也认得+ #[test]+ fn marker_attached_via_context_method_works() {+ let marked = anyhow::Error::msg("waiting")+ .context(Deferred::new(Duration::from_secs(7)))+ .context("job 42");+ assert_eq!(is_deferred(&marked), Some(Duration::from_secs(7)));++ let marked_t = anyhow::Error::msg("balance gone")+ .context(Terminal)+ .context("job 42");+ assert!(is_terminal(&marked_t));+ }++ /// 两个都挂时两个都认得;谁赢由 `mark_failed` 的提问顺序定(`Terminal` 先问)+ #[test]+ fn both_marks_are_seen() {+ let err = anyhow::Error::msg("balance gone after waiting")+ .context(Deferred::new(Duration::from_secs(10)))+ .context(Terminal);+ assert!(is_terminal(&err));+ assert_eq!(is_deferred(&err), Some(Duration::from_secs(10)));+ }++ /// 一个普通错误不该被认成 `Deferred` 或 `Terminal`+ #[test]+ fn plain_error_is_neither() {+ let err: anyhow::Error = anyhow::Error::msg("network blip");+ assert!(!is_terminal(&err));+ assert!(is_deferred(&err).is_none());+ }#[test]fn terminal_as_the_context() {diff --git a/crates/utopia-core/src/lib.rs b/crates/utopia-core/src/lib.rsindex 789d21cb..9296c035 100644--- a/crates/utopia-core/src/lib.rs+++ b/crates/utopia-core/src/lib.rs@@ -5,4 +5,4 @@ pub mod error;pub mod models;pub mod secrets;-pub use error::{is_terminal, AppError, AppResult, Terminal};+pub use error::{is_deferred, is_terminal, AppError, AppResult, Deferred, Terminal};diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rsindex 99083c7e..fbb4e16c 100644--- a/crates/utopia-server/src/extraction.rs+++ b/crates/utopia-server/src/extraction.rs@@ -3,6 +3,7 @@//! 消解灰区只入审核队列并触发独立的攒批裁决任务——LLM 裁决永不阻塞本任务。use crate::llm_util;+use crate::ontology_index;use crate::predicate_match::PredicateIndex;use crate::state::AppState;use sqlx::PgPool;@@ -870,6 +871,30 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:tracing::info!(document = %document_id, "skipping a deleted document");return Ok(());}+ // **本体向量门控(#526)——把等待从 worker 里搬回队列。**+ //+ // 在这之前 `extraction::run` 直接调 `ontology_index::refresh` 等到补齐才动。+ // 那次等待有三个坏处:它把 worker 槽占住,让同一批次的其余文档和别的库的+ // 任务全卡在锁上;它让文档在这段时间里挂着 `extracting`,用户看见 32 篇+ // 全在「抽取中」却没一个事实落库;它用一份可能没补齐的索引作依据,抽出来+ // 的图是基于半个本体写的,再也不会被重抽。+ //+ // 换成队列内门控:本体超出提示词预算且需要嵌入时,先把 `embed_ontology`+ // 排上、把这次抽取挂回 `queued` 等 30s,让 worker 槽立刻空出来——同一批+ // 其余文档和其它库的任务都能继续认领。下次轮到这个文档时本体可能已就绪,+ // 也可能还没,那就再等一轮。**不是同一个文档在等,是同一个抽取器在等**,+ // 而等候归队列管,attempts 不烧。+ //+ // 没配嵌入模型的库不等,照旧送完整本体——那种部署本来就没有检索。+ // 等也有期限(`jobs::DEFER_WINDOW_SECS`),补齐任务一直失败时这篇按失败处理。+ if ontology_index::gate_required(state, doc.kb_id).await? {+ // gate_required 已经把 `embed_ontology` 入队过了(如果应该入队的话)。+ // 这里只挂等待时长,不重复 enqueue。attempts 会被 mark_failed 退回去——+ // 同一个等待条件两次排队不应该消耗两次预算。+ let err = anyhow::Error::msg("waiting for ontology index to be embedded")+ .context(utopia_core::Deferred::new(Duration::from_secs(30)));+ return Err(err);+ }let kb = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;let settings = utopia_store::settings::get(&state.pool, kb.workspace_id).await?@@ -972,24 +997,13 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:classes = etypes.len(),"本体超出提示词预算,改为按分块检索候选");- // **这一篇要靠检索,那就先把检索用的向量补齐——不要跟补齐任务赛跑。**- //- // 建库装包、随手上传,是产品里最自然的一条路,而它踩的正是这个坑:- // `embed_ontology` 与 `extract_document` 同时排队,向量没就绪时按块检索- // 取不到候选,退回全量本体——实测每块 109k tokens(正文只占 0.2%),- // 且**多给的那些类会吃掉实体**(build_lists 上面那段注释量过:25 → 18)。- // 更糟的是连锁:109k × 并发把端点打到限流,退避五次仍失败就整块跳过,- // 一轮 13 块抽出来的同时 24 块被丢掉;抽取又占着端点,向量补得更慢。- //- // `refresh` 是幂等的、按库串行的(ontology_index::PER_KB),所以:- // 第一篇文档把向量补出来,同时到的其余文档在锁上等一下,进来时已经没事可做。- // 没配嵌入模型时它直接返回 0,退回全量那条路原样保留——那种部署本来就没有检索。+ // 走到这里说明本体超出预算且按预算逻辑需要按块检索——但本任务的等待+ // 早就在 `gate_required` 里完成(要么已经嵌好,要么通过 `Deferred` 挂回+ // 队列)。剩下的就是按块检索本身,不再有「顺便 refresh」这一步:+ // 那一步是在 worker 里等 PER_KB 锁,正是 #526 想消除的副作用。//- // 代价是新库的第一篇要多等几分钟。**换来的是它不会被一份烂抽取永久写坏**:- // 事实一旦落库,没有人会回头发现「这篇当初是在本体看不见的时候抽的」。- if let Err(e) = crate::ontology_index::refresh(state, doc.kb_id).await {- tracing::warn!(%document_id, error = %e, "本体向量补齐失败,这一篇按全量本体抽");- }+ // 留一个注释方便日后回看:若 budget 在 `gate_required` 与此处之间被改小,+ // 本文档会按全量本体抽——比 #526 之前的「等几分钟」更接近「对的那一边」。}// 内置类恒在:检索漏掉的分块仍要有地方落脚,否则模型无类可选let seed_classes: HashSet<Uuid> = etypes.iter().filter(|t| t.builtin).map(|t| t.id).collect();@@ -2715,6 +2729,19 @@ impl PromptLists {}}+/// 把当前本体在提示词里的字符数算出来。**空铺**(不筛类/关系)——这就是+/// `extract_document` 用的「全铺」档,也是判断「要不要按块检索」的标准。+///+/// 抽成独立函数是因为 `ontology_index::gate_required`(#526)要在加载抽取器+/// 之前问一次预算——那时 `build_lists` 还没被调用。两个路径必须用同一个判据,+/// 否则 gate 的判定会和实际的「全铺」走分。+pub(crate) fn full_ontology_chars(+ etypes: &[utopia_core::models::EntityType],+ rtypes: &[utopia_core::models::RelationType],+) -> usize {+ build_lists(etypes, rtypes, None, None).chars()+}+/// 从一个**选择集**排出三段清单。`None` = 全给(本体小于预算时的老路)。////// 三处细节都是选择带来的,全给时它们不会触发:diff --git a/crates/utopia-server/src/ontology_index.rs b/crates/utopia-server/src/ontology_index.rsindex 7b1ef762..0c92c882 100644--- a/crates/utopia-server/src/ontology_index.rs+++ b/crates/utopia-server/src/ontology_index.rs@@ -64,6 +64,62 @@ pub async fn refresh(state: &AppState, kb_id: Uuid) -> anyhow::Result<usize> {refresh_scoped(state, kb_id, None).await}+/// 抽取前的门控(#526)。**走这条路径的抽取器必须先问这个再继续**——+/// 不问而直接抽,要么抽出来的图是基于半个本体的、要么抽取 worker 在+/// `refresh` 锁上空转把 32 个并发槽占死。+///+/// 返回值语义:+/// - `Ok(false)`:本体小,全铺就行;本体大但已经在向量里了;或者没配嵌入模型+/// (照旧全铺)。继续抽。+/// - `Err(anyhow::Error::context(Deferred{30s}))`:本体超出预算且向量还没补齐,+/// 调用方应该把这个错误原样往上抛——`jobs::mark_failed` 认 `Deferred`,把任务+/// 挂回 `queued` 等 30s。worker 槽立刻空出来,下一轮再试。+pub async fn gate_required(state: &AppState, kb_id: Uuid) -> anyhow::Result<bool> {+ let etypes = utopia_store::graph::entity_types(&state.pool, kb_id).await?;+ let rtypes = utopia_store::graph::relation_types(&state.pool, kb_id).await?;+ let budget = utopia_store::access::ontology_prompt_budget(&state.pool).await?;+ let chars = crate::extraction::full_ontology_chars(&etypes, &rtypes);+ if chars <= budget {+ return Ok(false);+ }+ // 超出预算:需要检索候选。检索候选要先有向量——向量是否就绪取决于+ // `types_needing_embedding`,而它要求一个具体的嵌入模型名。+ //+ // KB 拿不到就不再静默当成「没事」——`run()` 后面还会再读一次同样的字段,+ // 让那次错误冒上来但归因不清。`Err` 直接归到这里,运维日志一眼能找到。+ let kb = utopia_store::kbs::get(&state.pool, kb_id).await?;+ let Some(settings) = utopia_store::settings::get(&state.pool, kb.workspace_id).await? else {+ // 没设模型就走全铺那条路——和原行为一致。配置阶段不该在这里就报错+ return Ok(false);+ };+ // 与 `refresh_scoped` 同一个判据:客户端与模型名缺一个,补齐就什么都不做,+ // 等下去只会空等到期限+ let (Some(_), Some(embed_model)) = (+ llm_util::embed_client(&settings),+ settings.embed_model.clone(),+ ) else {+ // 超预算 + 没配嵌入模型:照旧按全量本体抽。那种部署本来就没有检索,+ // 没有什么可等;判成失败会让只配了对话模型的库一篇都抽不出来+ return Ok(false);+ };+ let stale =+ utopia_store::ontology::types_needing_embedding(&state.pool, kb_id, &embed_model, None)+ .await?;+ if stale.is_empty() {+ // 超预算但所有类型都已嵌好——这个组合很罕见,意味着库被改大后又改小了,+ // 但代码路径是合法的。继续抽+ return Ok(false);+ }+ // 入队补齐任务——同库已排着的不重复(`enqueue_unless_queued` 的契约)+ utopia_store::jobs::enqueue_unless_queued(+ &state.pool,+ "embed_ontology",+ serde_json::json!({ "kb_id": kb_id }),+ )+ .await?;+ Ok(true)+}+/// 只补一半。调用方清楚自己要哪一半时用它——类型消解只用类,/// 等关系嵌完是白等。补漏的那一半有后台任务兜着。pub async fn refresh_scoped(diff --git a/crates/utopia-store/src/jobs.rs b/crates/utopia-store/src/jobs.rsindex 7f3e6779..6973d486 100644--- a/crates/utopia-store/src/jobs.rs+++ b/crates/utopia-store/src/jobs.rs@@ -273,13 +273,80 @@ fn retry_delay(attempts: i32, max_attempts: i32, terminal: bool) -> Option<i64>Some(30i64 * i64::from(attempts) * i64::from(attempts))}-async fn mark_failed(pool: &PgPool, job: &Job, err: &anyhow::Error) -> AppResult<()> {+/// 处理器挂上 `Deferred { retry_in }` 时下一次重试的等待秒数。**与失败次数无关**——+/// 这是 `Deferred` 跟默认退避的关键区别:默认的 `30s × attempts²` 是「再试一次+/// 也许能好」的递增,而 `Deferred` 是「现在条件不满足,`retry_in` 之后再试」,+/// 与第几次没有关系。同一个等待条件挂回队列两次,两次都得到同一个 `run_at`+/// 偏移,没有 60s、120s 的递增(#526)。+///+/// 一条任务最多连续等多久(见 `mark_failed`)。一小时够一个大本体在远端嵌入模型上+/// 补齐;过了还没好,多半是补齐任务自己在失败,该让这条抽取按失败处理、被人看见+pub const DEFER_WINDOW_SECS: i64 = 60 * 60;++fn deferred_retry_secs(retry_in: std::time::Duration) -> i64 {+ // 截断到秒:底层 `run_at` 是 timestamptz,亚秒精度存不住,而几十毫秒也不值得+ // 一行浮点换算。向上取整——少等一秒比早跑一秒好,前者无害,后者会把还在跑的+ // embedding job 撞回锁上。+ let secs = retry_in.as_secs_f64().ceil();+ if !secs.is_finite() || secs < 1.0 {+ 1+ } else {+ secs as i64+ }+}++/// `pub` 给集成测试用——主流程仍然由 `run_worker` 内的私有 caller 调用,+/// 不会从这里出。`#[doc(hidden)]` 是因为它属于内部契约,不进公开 API。+#[doc(hidden)]+pub async fn mark_failed(pool: &PgPool, job: &Job, err: &anyhow::Error) -> AppResult<()> {let text = format!("{err:#}");- let Some(backoff_secs) = retry_delay(- job.attempts,- job.max_attempts,- utopia_core::is_terminal(err),- ) else {+ // `Terminal` 优先:处理器最后改主意说「这次不算了」就该走 `failed` 路径,+ // 不该被 `Deferred` 覆盖。两个都挂时由调用方决定——`is_terminal` 写在前面。+ if utopia_core::is_terminal(err) {+ let res = sqlx::query(+ "UPDATE jobs SET status = 'failed', last_error = $2, updated_at = now() WHERE id = $1",+ )+ .bind(job.id)+ .bind(&text)+ .execute(pool)+ .await?;+ let _ = res.rows_affected();+ return Ok(());+ }+ // `Deferred`(#526):把任务挂回 `queued`,把 `attempts` 退回去,不烧预算。+ // 第一次走到这里时 `claim_one` 已经把 `attempts` 加 1,写回时要 -1,+ // 否则同一次等待会让 `attempts` 慢慢爬到 `max_attempts`,最后那条+ // `failed` 是我们最不想看见的——ontology 还差一秒就绪,文档却先死了。+ //+ // **等也有期限。** 从第一次挂回去算起(记在 payload 的 `deferred_since`,不用+ // `created_at`:一批上传排队几小时是常态,那不算在等)超过 [`DEFER_WINDOW_SECS`]+ // 还在等,就不再挂回去,落到下面的普通退避、烧预算。等的那件事(比如+ // `embed_ontology`)自己一直失败时,不设期限这条任务会每 30 秒醒一次、永远排着,+ // 却没有一次被记成失败+ if let Some(retry_in) = utopia_core::is_deferred(err) {+ let secs = deferred_retry_secs(retry_in);+ let res = sqlx::query(+ "UPDATE jobs SET status = 'queued', last_error = $2,+ attempts = GREATEST(0, attempts - 1),+ run_at = now() + make_interval(secs => $3::float8),+ payload = payload || jsonb_build_object('deferred_since',+ COALESCE(payload->>'deferred_since', now()::text)),+ updated_at = now()+ WHERE id = $1+ AND COALESCE((payload->>'deferred_since')::timestamptz, now())+ > now() - make_interval(secs => $4::float8)",+ )+ .bind(job.id)+ .bind(&text)+ .bind(secs as f64)+ .bind(DEFER_WINDOW_SECS as f64)+ .execute(pool)+ .await?;+ if res.rows_affected() > 0 {+ return Ok(());+ }+ }+ let Some(backoff_secs) = retry_delay(job.attempts, job.max_attempts, false) else {sqlx::query("UPDATE jobs SET status = 'failed', last_error = $2, updated_at = now() WHERE id = $1",)@@ -391,7 +458,8 @@ where#[cfg(test)]mod tests {- use super::retry_delay;+ use super::{deferred_retry_secs, retry_delay};+ use std::time::Duration;/// 退避照旧:30s、120s、270s,第三次之后放弃。#[test]@@ -407,4 +475,19 @@ mod tests {fn a_terminal_failure_does_not_spend_the_budget() {assert_eq!(retry_delay(1, 3, true), None);}++ /// `Deferred` 不随失败次数递增——同一等待条件两次排队得到的 `run_at`+ /// 偏移相同,不会出现 60s、120s 的递增(#526)。+ #[test]+ fn deferred_retry_is_independent_of_attempts() {+ assert_eq!(+ deferred_retry_secs(Duration::from_secs(30)),+ deferred_retry_secs(Duration::from_secs(30))+ );+ // 截断到秒,向上取整:29.5s → 30s+ assert_eq!(deferred_retry_secs(Duration::from_millis(29_500)), 30);+ // 0/负数/NaN 都给 1s 下界——至少等一秒,比立刻重试的轮询间隔还短就是浪费+ assert_eq!(deferred_retry_secs(Duration::ZERO), 1);+ assert_eq!(deferred_retry_secs(Duration::from_nanos(500)), 1);+ }}diff --git a/crates/utopia-store/tests/a_deferred_job_does_not_spend_its_budget.rs b/crates/utopia-store/tests/a_deferred_job_does_not_spend_its_budget.rsnew file mode 100644index 00000000..3c375c7d--- /dev/null+++ b/crates/utopia-store/tests/a_deferred_job_does_not_spend_its_budget.rs@@ -0,0 +1,202 @@+//! `#526` 的 `Deferred` 标记走通的端到端验证。+//!+//! 仅测两个边界:+//!+//! 1. 同一个等待条件两次走 `Deferred`,`attempts` **不会**爬到 `max_attempts`——+//! 它每次被退回去,**预算不烧**。这是「让它等本体向量补齐」与「让它重试+//! 失败调用」之间的区别:前者是同一个时钟;后者应该退避递增。+//! 2. `run_at` 真的被推到未来——而不是「立刻再试一次」,那会跟正在跑的+//! `embed_ontology` 任务抢同一把库级锁。+//!+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败(见 `utopia_store::test_db`)。++use std::time::Duration;++use anyhow::anyhow;+use sqlx::PgPool;+use utopia_core::Deferred;+use utopia_store::jobs;++#[tokio::test]+async fn a_deferred_job_does_not_spend_its_budget() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;++ // 直接插一行「正在跑」的任务——attempts 已经由 `claim_one` 加过一次,+ // 这一关的语义是 `mark_failed(Deferred)` 写回时把它退回去。+ let (id,): (i64,) = sqlx::query_as(+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)+ VALUES ('extract_document', '{\"document_id\":\"00000000-0000-0000-0000-000000000000\"}',+ 'running', 2, 3)+ RETURNING id",+ )+ .fetch_one(&pool)+ .await?;++ // 第一次等待:本体向量补齐,30s。+ let err = anyhow!("waiting on ontology index").context(Deferred::new(Duration::from_secs(30)));+ let job = jobs::Job {+ id,+ kind: "extract_document".into(),+ payload: serde_json::json!({}),+ attempts: 2,+ max_attempts: 3,+ };+ jobs::mark_failed(&pool, &job, &err).await?;++ let (status, attempts, last_error): (String, i32, String) =+ sqlx::query_as("SELECT status, attempts, last_error FROM jobs WHERE id = $1")+ .bind(id)+ .fetch_one(&pool)+ .await?;+ assert_eq!(status, "queued", "Deferred 应该让任务挂回 queued");+ assert_eq!(attempts, 1, "attempts 应该被退回去:原 2 减 1 等于 1");+ assert!(+ last_error.contains("waiting on ontology index"),+ "last_error 应当是让运维一眼知道在等什么的措辞;实际是:{last_error:?}"+ );++ // `run_at` 落在未来——不是「立刻又来抢锁」。+ let run_at: chrono::DateTime<chrono::Utc> =+ sqlx::query_scalar("SELECT run_at FROM jobs WHERE id = $1")+ .bind(id)+ .fetch_one(&pool)+ .await?;+ let now = chrono::Utc::now();+ let ahead = (run_at - now).num_seconds();+ assert!(+ (25..=35).contains(&ahead),+ "run_at 应该比 now 晚约 30s;实际 {ahead}s"+ );++ // 第二次等待:同一个任务、同一段等待。**预算不烧**——+ // 这是与「限流重试」的关键区别。+ let err2 = anyhow!("waiting on ontology index").context(Deferred::new(Duration::from_secs(30)));+ let job2 = jobs::Job {+ attempts: 1, // 第一次之后的状态+ ..job.clone()+ };+ jobs::mark_failed(&pool, &job2, &err2).await?;+ let (status2, attempts2, _): (String, i32, String) =+ sqlx::query_as("SELECT status, attempts, last_error FROM jobs WHERE id = $1")+ .bind(id)+ .fetch_one(&pool)+ .await?;+ assert_eq!(status2, "queued");+ assert_eq!(attempts2, 0, "两次 Deferred 之后 attempts 应该退到 0");++ // 收尾:避免污染下一次跑(其它测试共享同一个库)+ sqlx::query("DELETE FROM jobs WHERE id = $1")+ .bind(id)+ .execute(&pool)+ .await?;+ Ok(())+}++/// `Terminal` 优先于 `Deferred`:挂两个 marker 时,结果由 `Terminal` 决定。+/// 这是 `mark_failed` 里 `is_terminal` 写在前面、`is_deferred` 写在后面的语义保证。+#[tokio::test]+async fn terminal_wins_over_deferred_when_both_attached() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;++ let (id,): (i64,) = sqlx::query_as(+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)+ VALUES ('extract_document', '{\"document_id\":\"00000000-0000-0000-0000-000000000000\"}',+ 'running', 1, 3)+ RETURNING id",+ )+ .fetch_one(&pool)+ .await?;++ // 先挂 Deferred,再挂 Terminal。Terminal 是「最后一次改主意」的那一个。+ let err = anyhow!("balance gone after waiting").context(Deferred::new(Duration::from_secs(30)));+ let err = err.context(utopia_core::Terminal);+ let job = jobs::Job {+ id,+ kind: "extract_document".into(),+ payload: serde_json::json!({}),+ attempts: 1,+ max_attempts: 3,+ };+ jobs::mark_failed(&pool, &job, &err).await?;++ let (status, attempts): (String, i32) =+ sqlx::query_as("SELECT status, attempts FROM jobs WHERE id = $1")+ .bind(id)+ .fetch_one(&pool)+ .await?;+ assert_eq!(status, "failed", "Terminal 应该让任务直接进 failed");+ // attempts 不会被退回去——`failed` 走的是另一条 SQL 路径。+ assert_eq!(attempts, 1);++ sqlx::query("DELETE FROM jobs WHERE id = $1")+ .bind(id)+ .execute(&pool)+ .await?;+ Ok(())+}++/// 等也有期限:从第一次挂回去算起超过 `DEFER_WINDOW_SECS` 还在等,就按普通失败+/// 退避、烧预算。等的那件事自己一直失败时,任务不该每 30 秒醒一次、永远排着+#[tokio::test]+async fn a_wait_past_its_window_is_a_failure() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;++ // 第一次挂回去时记下开始等的时刻+ let (fresh,): (i64,) = sqlx::query_as(+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)+ VALUES ('extract_document', '{}', 'running', 1, 3) RETURNING id",+ )+ .fetch_one(&pool)+ .await?;+ let err = anyhow!("waiting on ontology index").context(Deferred::new(Duration::from_secs(30)));+ let job = jobs::Job {+ id: fresh,+ kind: "extract_document".into(),+ payload: serde_json::json!({}),+ attempts: 1,+ max_attempts: 3,+ };+ jobs::mark_failed(&pool, &job, &err).await?;+ let since: Option<String> =+ sqlx::query_scalar("SELECT payload->>'deferred_since' FROM jobs WHERE id = $1")+ .bind(fresh)+ .fetch_one(&pool)+ .await?;+ assert!(since.is_some(), "第一次等待应当记下 deferred_since");++ // 已经等过了期限:不再退回 attempts,走普通退避+ let (stale,): (i64,) = sqlx::query_as(+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)+ VALUES ('extract_document',+ jsonb_build_object('deferred_since', (now() - make_interval(secs => $1::float8))::text),+ 'running', 1, 3)+ RETURNING id",+ )+ .bind((jobs::DEFER_WINDOW_SECS + 60) as f64)+ .fetch_one(&pool)+ .await?;+ let job = jobs::Job { id: stale, ..job };+ jobs::mark_failed(&pool, &job, &err).await?;+ let (status, attempts): (String, i32) =+ sqlx::query_as("SELECT status, attempts FROM jobs WHERE id = $1")+ .bind(stale)+ .fetch_one(&pool)+ .await?;+ assert_eq!(status, "queued", "还有预算,按退避重排");+ assert_eq!(attempts, 1, "过了期限的等待要烧预算,attempts 不再退回");++ sqlx::query("DELETE FROM jobs WHERE id = ANY($1)")+ .bind(vec![fresh, stale])+ .execute(&pool)+ .await?;+ Ok(())+}