World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Retry computed job outcomes without rerunning their handlers (#1137)
Danielb7f756e1806f
diff --git a/crates/utopia-server/src/extraction_open_out_of_credit_tests.rs b/crates/utopia-server/src/extraction_open_out_of_credit_tests.rsindex 7585ea71..c53d855c 100644--- a/crates/utopia-server/src/extraction_open_out_of_credit_tests.rs+++ b/crates/utopia-server/src/extraction_open_out_of_credit_tests.rs@@ -94,6 +94,7 @@ async fn an_empty_balance_stops_the_document_and_keeps_what_was_applied() -> anypayload: json!({}),attempts: 1,max_attempts: 3,+ locked_at: chrono::Utc::now(),};crate::alerting::observe_job_failure(&state, &job, &err).await;let alerts: Vec<String> = sqlx::query_scalar(diff --git a/crates/utopia-store/src/jobs.rs b/crates/utopia-store/src/jobs.rsindex 8206f8a7..12d2553d 100644--- a/crates/utopia-store/src/jobs.rs+++ b/crates/utopia-store/src/jobs.rs@@ -21,6 +21,7 @@ pub struct Job {pub payload: serde_json::Value,pub attempts: i32,pub max_attempts: i32,+ pub locked_at: DateTime<Utc>,}/// 同种任务、同样载荷已经排着就不再排:一批文档各自抽完都想触发同一个库级任务@@ -352,27 +353,13 @@ async fn claim_one(pool: &PgPool) -> AppResult<Option<Job>> {FOR UPDATE SKIP LOCKEDLIMIT 1)- RETURNING id, kind, payload, attempts, max_attempts",+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",).fetch_optional(pool).await?;Ok(job)}-async fn mark_done(pool: &PgPool, id: i64) -> AppResult<()> {- // **成功要把上一次的错清掉。** 重试成功后 last_error 仍留着失败那次的原文,- // 于是任务表里出现 status='done' 配着一条错误信息——查问题的人读到的是- // 一个已经不成立的原因。实测就这么误导过一次:bootstrap 明明跑成了,- // 表上还挂着 "column relation_type does not exist"。- sqlx::query(- "UPDATE jobs SET status = 'done', last_error = NULL, updated_at = now() WHERE id = $1",- )- .bind(id)- .execute(pool)- .await?;- Ok(())-}-/// 下一次重试等多久;`None` = 到此为止。////// 两个理由到此为止:**次数用完了**,或者**处理器说了这次不会因为重试而变好**@@ -413,81 +400,160 @@ fn deferred_retry_secs(retry_in: std::time::Duration) -> i64 {}}-/// `pub` 给集成测试用——主流程仍然由 `run_worker` 内的私有 caller 调用,-/// 不会从这里出。`#[doc(hidden)]` 是因为它属于内部契约,不进公开 API。-#[doc(hidden)]-pub async fn mark_failed(pool: &PgPool, job: &Job, err: &anyhow::Error) -> AppResult<()> {- let text = format!("{err:#}");- // `Terminal` 优先:处理器最后改主意说「这次不算了」就该走 `failed` 路径,- // 不该被 `Deferred` 覆盖。两个都挂时由调用方决定——`is_terminal` 写在前面。- if utopia_core::is_terminal(err) {+enum JobOutcome {+ Done,+ Failed {+ error: String,+ backoff_secs: Option<i64>,+ deferred_secs: Option<i64>,+ },+}++impl JobOutcome {+ fn failed(job: &Job, err: &anyhow::Error) -> Self {+ let terminal = utopia_core::is_terminal(err);+ // Terminal 优先;结果写回重试沿用同一分类和完整错误原文。+ let deferred_secs = if terminal {+ None+ } else {+ utopia_core::is_deferred(err).map(deferred_retry_secs)+ };+ Self::Failed {+ error: format!("{err:#}"),+ backoff_secs: retry_delay(job.attempts, job.max_attempts, terminal),+ deferred_secs,+ }+ }+}++/// true = 本次写入;false = 已结束或领取失效。两者都结束这次写回。+async fn persist_outcome(+ pool: &PgPool,+ job: &Job,+ outcome: &JobOutcome,+ completed_at: DateTime<Utc>,+) -> AppResult<bool> {+ let JobOutcome::Failed {+ error,+ backoff_secs,+ deferred_secs,+ } = outcome+ else {+ // 成功要清掉前一次错误,避免任务显示 done 却仍带着失败原因。let res = sqlx::query(- "UPDATE jobs SET status = 'failed', last_error = $2, updated_at = now() WHERE id = $1",+ "UPDATE jobs SET status = 'done', last_error = NULL, updated_at = now()+ WHERE id = $1 AND status = 'running' AND locked_at = $2",).bind(job.id)- .bind(&text)+ .bind(job.locked_at).execute(pool).await?;- let _ = res.rows_affected();- return Ok(());- }- // `Deferred`(#526):把任务挂回 `queued`,把 `attempts` 退回去,不烧预算。- // 第一次走到这里时 `claim_one` 已经把 `attempts` 加 1,写回时要 -1,- // 否则同一次等待会让 `attempts` 慢慢爬到 `max_attempts`,最后那条- // `failed` 是我们最不想看见的——ontology 还差一秒就绪,文档却先死了。- //- // **等也有期限。** 从第一次挂回去算起(记在 payload 的 `deferred_since`,不用- // `created_at`:一批上传排队几小时是常态,那不算在等)超过 [`DEFER_WINDOW_SECS`]- // 还在等,就不再挂回去,落到下面的普通退避、烧预算。等的那件事(比如- // `embed_ontology`)自己一直失败时,不设期限这条任务会每 30 秒醒一次、永远排着,- // 却没有一次被记成失败- if let Some(retry_in) = utopia_core::is_deferred(err) {- let secs = deferred_retry_secs(retry_in);+ return Ok(res.rows_affected() > 0);+ };++ if let Some(secs) = deferred_secs {+ // Deferred 退回认领消耗的次数;等待窗口耗尽后才走普通失败预算。+ // 等待窗口和下次运行时刻固定在 handler 完成时,不随写回重试滑动。let res = sqlx::query("UPDATE jobs SET status = 'queued', last_error = $2,attempts = GREATEST(0, attempts - 1),- run_at = now() + make_interval(secs => $3::float8),+ run_at = $4::timestamptz + make_interval(secs => $5::float8),payload = payload || jsonb_build_object('deferred_since',- COALESCE(payload->>'deferred_since', now()::text)),+ COALESCE(payload->>'deferred_since', $4::timestamptz::text)),updated_at = now()- WHERE id = $1- AND COALESCE((payload->>'deferred_since')::timestamptz, now())- > now() - make_interval(secs => $4::float8)",+ WHERE id = $1 AND status = 'running' AND locked_at = $3+ AND COALESCE((payload->>'deferred_since')::timestamptz, $4)+ > $4 - make_interval(secs => $6::float8)",).bind(job.id)- .bind(&text)- .bind(secs as f64)+ .bind(error)+ .bind(job.locked_at)+ .bind(completed_at)+ .bind(*secs as f64).bind(DEFER_WINDOW_SECS as f64).execute(pool).await?;if res.rows_affected() > 0 {- return Ok(());+ return Ok(true);}+ // 0 行也可能是旧领取;普通失败分支保留相同保护,不能覆盖新执行。}- let Some(backoff_secs) = retry_delay(job.attempts, job.max_attempts, false) else {- sqlx::query(- "UPDATE jobs SET status = 'failed', last_error = $2, updated_at = now() WHERE id = $1",- )- .bind(job.id)- .bind(&text)- .execute(pool)- .await?;- return Ok(());++ let status = if backoff_secs.is_some() {+ "queued"+ } else {+ "failed"};- sqlx::query(- "UPDATE jobs SET status = 'queued', last_error = $2,- run_at = now() + make_interval(secs => $3::float8),+ let res = sqlx::query(+ "UPDATE jobs SET status = $3, last_error = $4,+ run_at = CASE WHEN $5::float8 IS NULL THEN run_at+ ELSE $6::timestamptz + make_interval(secs => $5::float8) END,updated_at = now()- WHERE id = $1",+ WHERE id = $1 AND status = 'running' AND locked_at = $2",).bind(job.id)- .bind(&text)- .bind(backoff_secs as f64)+ .bind(job.locked_at)+ .bind(status)+ .bind(error)+ .bind(backoff_secs.map(|secs| secs as f64))+ .bind(completed_at).execute(pool).await?;+ Ok(res.rows_affected() > 0)+}++/// 集成测试的单次写回入口;worker 保存结果后由 persist_with_retry 写回。+#[doc(hidden)]+pub async fn mark_failed(pool: &PgPool, job: &Job, err: &anyhow::Error) -> AppResult<()> {+ persist_outcome(pool, job, &JobOutcome::failed(job, err), Utc::now()).await?;Ok(())}+async fn persist_with_retry(+ pool: &PgPool,+ job: &Job,+ outcome: &JobOutcome,+ completed_at: DateTime<Utc>,+) {+ let mut delay = Duration::from_secs(1);+ loop {+ if pool.is_closed() {+ tracing::warn!(+ job_id = job.id,+ "数据库连接池已关闭,未确认的任务留待启动回收"+ );+ return;+ }+ let result = tokio::select! {+ _ = pool.close_event() => {+ tracing::warn!(job_id = job.id, "数据库连接池已关闭,未确认的任务留待启动回收");+ return;+ }+ result = persist_outcome(pool, job, outcome, completed_at) => result,+ };+ match result {+ Ok(updated) => {+ if !updated {+ tracing::debug!(+ job_id = job.id,+ claimed_at = %job.locked_at,+ "任务已结束或领取已失效,停止写回旧结果"+ );+ }+ return;+ }+ Err(e) => {+ tracing::error!(job_id = job.id, error = %e, retry_in_secs = delay.as_secs(), "任务状态写回失败,将重试同一结果");+ }+ }+ tokio::select! {+ _ = pool.close_event() => {}+ _ = tokio::time::sleep(delay) => {}+ }+ delay = (delay * 2).min(Duration::from_secs(30));+ }+}+/// worker 调度循环:运行中任务数低于目标并发就继续认领(有活立即续派),/// 空闲时等入队通知、2s 轮询兜底(#517);每个任务在独立 tokio task 中执行,长抽取不再阻塞同步。/// `concurrency` 每轮热读——系统设置里改并发数即时生效。@@ -522,6 +588,9 @@ where);let mut listener = listen_for_jobs(&pool).await;loop {+ if pool.is_closed() {+ return;+ }let cap = concurrency.load(Ordering::Relaxed).max(1);if running.load(Ordering::Relaxed) >= cap {tokio::time::sleep(Duration::from_millis(200)).await;@@ -550,16 +619,15 @@ whereErr(anyhow::anyhow!("任务处理器 panic(详情见 stderr):{join}"))}};+ let completed_at = Utc::now();let outcome = match result {- Ok(()) => mark_done(&pool, job.id).await,+ Ok(()) => JobOutcome::Done,Err(e) => {tracing::warn!(job_id = job.id, kind = %job.kind, error = %e, "任务执行失败");- mark_failed(&pool, &job, &e).await+ JobOutcome::failed(&job, &e)}};- if let Err(e) = outcome {- tracing::error!(job_id = job.id, error = %e, "任务状态写回失败");- }+ persist_with_retry(&pool, &job, &outcome, completed_at).await;running.fetch_sub(1, Ordering::Relaxed);});}@@ -574,6 +642,10 @@ where}}+#[cfg(test)]+#[path = "jobs_writeback_tests.rs"]+mod writeback_tests;+#[cfg(test)]mod tests {use super::{deferred_retry_secs, retry_delay};diff --git a/crates/utopia-store/src/jobs_writeback_tests.rs b/crates/utopia-store/src/jobs_writeback_tests.rsnew file mode 100644index 00000000..824060f9--- /dev/null+++ b/crates/utopia-store/src/jobs_writeback_tests.rs@@ -0,0 +1,364 @@+//! 写回测试只操作自己的任务行,不启动会认领全库任务的 worker。++use super::{persist_outcome, persist_with_retry, Job, JobOutcome, DEFER_WINDOW_SECS};+use chrono::{DateTime, Utc};+use serde_json::{json, Value};+use sqlx::PgPool;+use std::time::Duration;+use utopia_core::Deferred;+use uuid::Uuid;++async fn running_job(pool: &PgPool, attempts: i32) -> anyhow::Result<Job> {+ Ok(sqlx::query_as(+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts,+ locked_at, last_error)+ VALUES ($1, '{}', 'running', $2, 3, now(), 'previous failure')+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",+ )+ .bind(format!("test_writeback_{}", Uuid::now_v7()))+ .bind(attempts)+ .fetch_one(pool)+ .await?)+}++async fn claim_job(pool: &PgPool, id: i64) -> anyhow::Result<Job> {+ Ok(sqlx::query_as(+ "UPDATE jobs SET status = 'running', attempts = attempts + 1,+ locked_at = now(), updated_at = now()+ WHERE id = $1 AND status = 'queued'+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",+ )+ .bind(id)+ .fetch_one(pool)+ .await?)+}++#[derive(Debug, PartialEq, sqlx::FromRow)]+struct JobState {+ status: String,+ attempts: i32,+ locked_at: Option<DateTime<Utc>>,+ last_error: Option<String>,+ run_at: DateTime<Utc>,+ updated_at: DateTime<Utc>,+ payload: Value,+}++async fn state(pool: &PgPool, id: i64) -> anyhow::Result<JobState> {+ Ok(sqlx::query_as(+ "SELECT status, attempts, locked_at, last_error, run_at, updated_at, payload+ FROM jobs WHERE id = $1",+ )+ .bind(id)+ .fetch_one(pool)+ .await?)+}++async fn remove(pool: &PgPool, id: i64) -> anyhow::Result<()> {+ sqlx::query("DELETE FROM jobs WHERE id = $1")+ .bind(id)+ .execute(pool)+ .await?;+ Ok(())+}++#[tokio::test]+async fn a_refused_write_retries_the_computed_result() -> anyhow::Result<()> {+ let Some(url) = crate::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ let job = running_job(&pool, 1).await?;+ let object_name = format!("test_writeback_retry_{}", Uuid::now_v7().simple());++ let result: anyhow::Result<()> = async {+ // sequence 不随拒写事务回滚,计数能证明第一次写入确实失败。+ sqlx::raw_sql(&format!(+ "CREATE SEQUENCE {object_name};+ CREATE FUNCTION {object_name}() RETURNS trigger LANGUAGE plpgsql AS $$+ BEGIN+ IF nextval('{object_name}') = 1 THEN+ RAISE EXCEPTION 'injected result writeback failure';+ END IF;+ RETURN NEW;+ END; $$;+ CREATE TRIGGER {object_name} BEFORE UPDATE ON jobs+ FOR EACH ROW WHEN (OLD.id = {id} AND OLD.kind = '{kind}'+ AND OLD.status = 'running' AND NEW.status = 'done')+ EXECUTE FUNCTION {object_name}();",+ id = job.id,+ kind = job.kind,+ ))+ .execute(&pool)+ .await?;++ tokio::time::timeout(+ Duration::from_secs(10),+ persist_with_retry(&pool, &job, &JobOutcome::Done, Utc::now()),+ )+ .await?;+ let writes: i64 = sqlx::query_scalar(&format!("SELECT last_value FROM {object_name}"))+ .fetch_one(&pool)+ .await?;+ anyhow::ensure!(+ writes == 2,+ "expected one refused write and one successful retry"+ );+ let written = state(&pool, job.id).await?;+ anyhow::ensure!(written.status == "done" && written.last_error.is_none());+ anyhow::ensure!(written.attempts == job.attempts);+ anyhow::ensure!(written.locked_at == Some(job.locked_at));+ Ok(())+ }+ .await;++ let dropped = sqlx::raw_sql(&format!(+ "DROP TRIGGER IF EXISTS {object_name} ON jobs;+ DROP FUNCTION IF EXISTS {object_name}();+ DROP SEQUENCE IF EXISTS {object_name};"+ ))+ .execute(&pool)+ .await;+ let removed = remove(&pool, job.id).await;+ pool.close().await;+ result?;+ dropped?;+ removed?;+ Ok(())+}++#[tokio::test]+async fn an_old_claim_cannot_overwrite_any_new_running_result() -> anyhow::Result<()> {+ let Some(url) = crate::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ let completed_at = DateTime::<Utc>::from_timestamp(1_700_000_000, 0).unwrap();+ let ordinary = anyhow::anyhow!("network failure").context("provider request");+ let deferred = anyhow::anyhow!("index busy").context(Deferred::new(Duration::from_secs(17)));++ // 窗口内和窗口外的旧 Deferred 都不能把新认领写成失败。+ for expired in [false, true] {+ let job = running_job(&pool, 1).await?;+ let payload = if expired {+ json!({"deferred_since": (completed_at - chrono::Duration::hours(2)).to_rfc3339()})+ } else {+ json!({})+ };+ // 先提交重排,再从新事务认领;不伪造认领时间,也不认领其他测试的任务。+ sqlx::query("UPDATE jobs SET status = 'queued', attempts = 0, payload = $2 WHERE id = $1")+ .bind(job.id)+ .bind(payload)+ .execute(&pool)+ .await?;+ let new_job = claim_job(&pool, job.id).await?;+ assert_eq!(new_job.attempts, job.attempts);+ assert_ne!(new_job.locked_at, job.locked_at);+ let new_execution = state(&pool, job.id).await?;++ for outcome in [+ JobOutcome::Done,+ JobOutcome::failed(&job, &ordinary),+ JobOutcome::failed(&job, &deferred),+ ] {+ assert!(!persist_outcome(&pool, &job, &outcome, completed_at).await?);+ assert_eq!(state(&pool, job.id).await?, new_execution);+ }+ remove(&pool, job.id).await?;+ }+ pool.close().await;+ Ok(())+}++#[tokio::test]+async fn a_repeated_ack_preserves_the_first_writeback() -> anyhow::Result<()> {+ let Some(url) = crate::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ let completed_at = DateTime::<Utc>::from_timestamp(1_700_000_000, 0).unwrap();+ let deferred = anyhow::anyhow!("index busy").context(Deferred::new(Duration::from_secs(17)));+ let ordinary =+ anyhow::anyhow!("provider detail: request rejected").context("outer request context");++ for (error, starts_deferral) in [+ (None, false),+ (Some(&ordinary), false),+ (Some(&deferred), true),+ ] {+ let job = running_job(&pool, 2).await?;+ let outcome = match error {+ Some(error) => JobOutcome::failed(&job, error),+ None => JobOutcome::Done,+ };+ assert!(persist_outcome(&pool, &job, &outcome, completed_at).await?);+ let first_ack = state(&pool, job.id).await?;+ if starts_deferral {+ let since: DateTime<Utc> = sqlx::query_scalar("SELECT $1::text::timestamptz")+ .bind(first_ack.payload["deferred_since"].as_str())+ .fetch_one(&pool)+ .await?;+ assert_eq!(since, completed_at);+ }+ let duplicate_ack = persist_outcome(+ &pool,+ &job,+ &outcome,+ completed_at + chrono::Duration::hours(1),+ )+ .await?;+ assert!(!duplicate_ack);+ assert_eq!(state(&pool, job.id).await?, first_ack);+ remove(&pool, job.id).await?;+ }+ pool.close().await;+ Ok(())+}++#[tokio::test]+async fn deferred_window_uses_completion_time_and_current_database_payload() -> anyhow::Result<()> {+ let Some(url) = crate::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ // 故意用早于当前时间的完成时刻,写回等待不能令有效的 Deferred 过期。+ let completed_at = DateTime::<Utc>::from_timestamp(1_700_000_000, 0).unwrap();+ let error = anyhow::anyhow!("index busy").context(Deferred::new(Duration::from_secs(17)));+ for seconds_inside_window in [1, 0] {+ let job = running_job(&pool, 2).await?;+ let since =+ completed_at - chrono::Duration::seconds(DEFER_WINDOW_SECS - seconds_inside_window);+ let payload = json!({"deferred_since": since.to_rfc3339(), "kept": "value"});+ // handler 可以在认领后更新 payload,窗口起点以已写入数据库的值为准。+ sqlx::query("UPDATE jobs SET payload = $2 WHERE id = $1")+ .bind(job.id)+ .bind(&payload)+ .execute(&pool)+ .await?;+ let outcome = JobOutcome::failed(&job, &error);+ assert!(persist_outcome(&pool, &job, &outcome, completed_at).await?);+ let written = state(&pool, job.id).await?;+ assert_eq!(written.status, "queued");+ assert_eq!(written.payload, payload);+ if seconds_inside_window == 1 {+ assert_eq!(written.attempts, 1, "valid Deferred does not spend budget");+ assert_eq!(written.run_at, completed_at + chrono::Duration::seconds(17));+ } else {+ assert_eq!(+ written.attempts, 2,+ "expired Deferred spends ordinary budget"+ );+ assert_eq!(+ written.run_at,+ completed_at + chrono::Duration::seconds(120)+ );+ }+ remove(&pool, job.id).await?;+ }+ pool.close().await;+ Ok(())+}++#[tokio::test]+async fn a_closed_pool_does_not_keep_writeback_retry_alive() -> anyhow::Result<()> {+ // 已关闭的惰性池不会建立连接,此测试不依赖数据库。+ let pool = PgPool::connect_lazy("postgres://test:[email protected]:1/test")?;+ pool.close().await;+ let job = Job {+ id: 0,+ kind: "closed_pool_writeback".into(),+ payload: json!({}),+ attempts: 1,+ max_attempts: 3,+ locked_at: Utc::now(),+ };+ tokio::time::timeout(+ Duration::from_secs(1),+ super::persist_with_retry(&pool, &job, &JobOutcome::Done, Utc::now()),+ )+ .await?;+ Ok(())+}++#[tokio::test]+async fn closing_the_pool_cancels_writeback_while_it_waits_for_a_row_lock() -> anyhow::Result<()> {+ let Some(url) = crate::test_db::url() else {+ return Ok(());+ };+ let control = sqlx::postgres::PgPoolOptions::new()+ .max_connections(2)+ .connect(&url)+ .await?;+ let job = running_job(&control, 1).await?;+ let before = state(&control, job.id).await?;+ let mut locked = control.begin().await?;+ let locker_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")+ .fetch_one(&mut *locked)+ .await?;+ sqlx::query("SELECT id FROM jobs WHERE id = $1 FOR UPDATE")+ .bind(job.id)+ .execute(&mut *locked)+ .await?;++ let writer_pool = sqlx::postgres::PgPoolOptions::new()+ .max_connections(1)+ .connect(&url)+ .await?;+ let writer_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")+ .fetch_one(&writer_pool)+ .await?;+ let mut writer = tokio::spawn({+ let pool = writer_pool.clone();+ let job = job.clone();+ async move {+ super::persist_with_retry(&pool, &job, &JobOutcome::Done, Utc::now()).await;+ }+ });++ let result: anyhow::Result<()> = async {+ // 观测自己的后端确实在等这把锁,再关闭池,不靠固定睡眠猜测执行进度。+ tokio::time::timeout(Duration::from_secs(2), async {+ loop {+ let waiting: bool = sqlx::query_scalar(+ "SELECT EXISTS (SELECT 1 FROM pg_stat_activity+ WHERE pid = $1 AND wait_event_type = 'Lock'+ AND $2 = ANY(pg_blocking_pids(pid)))",+ )+ .bind(writer_pid)+ .bind(locker_pid)+ .fetch_one(&control)+ .await?;+ if waiting {+ return anyhow::Ok(());+ }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await??;++ let (finished, ()) = tokio::time::timeout(Duration::from_secs(2), async {+ tokio::join!(&mut writer, writer_pool.close())+ })+ .await?;+ finished?;+ let unchanged = state(&control, job.id).await?;+ anyhow::ensure!(+ unchanged == before,+ "pool closure changed the locked job: {unchanged:?}"+ );+ anyhow::Ok(())+ }+ .await;++ // 失败路径也撤销自己的行锁并终止测试 task,不能把挂起的写回留给其他测试。+ writer.abort();+ let unlocked = locked.rollback().await;+ let closed = tokio::time::timeout(Duration::from_secs(2), writer_pool.close()).await;+ let removed = remove(&control, job.id).await;+ control.close().await;+ result?;+ unlocked?;+ closed?;+ removed?;+ Ok(())+}diff --git a/crates/utopia-store/src/materialize_delivery_tests.rs b/crates/utopia-store/src/materialize_delivery_tests.rsindex 97b154a4..8924c7cb 100644--- a/crates/utopia-store/src/materialize_delivery_tests.rs+++ b/crates/utopia-store/src/materialize_delivery_tests.rs@@ -70,7 +70,7 @@ async fn accept(}async fn claim(pool: &PgPool, id: i64) -> anyhow::Result<jobs::Job> {// Restrict the production claim SQL to this test's job, never steal work.- Ok(sqlx::query_as("UPDATE jobs SET status='running', attempts=attempts+1, locked_at=now() WHERE id=$1 AND status='queued' RETURNING id,kind,payload,attempts,max_attempts")+ Ok(sqlx::query_as("UPDATE jobs SET status='running', attempts=attempts+1, locked_at=now() WHERE id=$1 AND status='queued' RETURNING id,kind,payload,attempts,max_attempts,locked_at").bind(id).fetch_one(pool).await?)}async fn handle(pool: &PgPool, kb: Uuid, job: &jobs::Job) -> anyhow::Result<()> {diff --git a/crates/utopia-store/tests/human_phrase_materialization_delivery.rs b/crates/utopia-store/tests/human_phrase_materialization_delivery.rsindex 6f32429d..e9f0952f 100644--- a/crates/utopia-store/tests/human_phrase_materialization_delivery.rs+++ b/crates/utopia-store/tests/human_phrase_materialization_delivery.rs@@ -43,7 +43,7 @@ async fn accept(}async fn claim(pool: &PgPool, id: i64) -> anyhow::Result<jobs::Job> {// Restrict the production claim SQL to this test's job, never steal work.- Ok(sqlx::query_as("UPDATE jobs SET status='running', attempts=attempts+1, locked_at=now() WHERE id=$1 AND status='queued' RETURNING id,kind,payload,attempts,max_attempts")+ Ok(sqlx::query_as("UPDATE jobs SET status='running', attempts=attempts+1, locked_at=now() WHERE id=$1 AND status='queued' RETURNING id,kind,payload,attempts,max_attempts,locked_at").bind(id).fetch_one(pool).await?)}async fn handle(pool: &PgPool, kb: Uuid, job: &jobs::Job) -> anyhow::Result<()> {diff --git a/crates/utopia-store/tests/store/a_deferred_job_does_not_spend_its_budget.rs b/crates/utopia-store/tests/store/a_deferred_job_does_not_spend_its_budget.rsindex 3c375c7d..0ecc4c08 100644--- a/crates/utopia-store/tests/store/a_deferred_job_does_not_spend_its_budget.rs+++ b/crates/utopia-store/tests/store/a_deferred_job_does_not_spend_its_budget.rs@@ -26,24 +26,18 @@ async fn a_deferred_job_does_not_spend_its_budget() -> anyhow::Result<()> {// 直接插一行「正在跑」的任务——attempts 已经由 `claim_one` 加过一次,// 这一关的语义是 `mark_failed(Deferred)` 写回时把它退回去。- let (id,): (i64,) = sqlx::query_as(- "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)+ let job: jobs::Job = sqlx::query_as(+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts, locked_at)VALUES ('extract_document', '{\"document_id\":\"00000000-0000-0000-0000-000000000000\"}',- 'running', 2, 3)- RETURNING id",+ 'running', 2, 3, now())+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",).fetch_one(&pool).await?;+ let id = job.id;// 第一次等待:本体向量补齐,30s。let err = anyhow!("waiting on ontology index").context(Deferred::new(Duration::from_secs(30)));- let job = jobs::Job {- id,- kind: "extract_document".into(),- payload: serde_json::json!({}),- attempts: 2,- max_attempts: 3,- };jobs::mark_failed(&pool, &job, &err).await?;let (status, attempts, last_error): (String, i32, String) =@@ -74,10 +68,14 @@ async fn a_deferred_job_does_not_spend_its_budget() -> anyhow::Result<()> {// 第二次等待:同一个任务、同一段等待。**预算不烧**——// 这是与「限流重试」的关键区别。let err2 = anyhow!("waiting on ontology index").context(Deferred::new(Duration::from_secs(30)));- let job2 = jobs::Job {- attempts: 1, // 第一次之后的状态- ..job.clone()- };+ let job2: jobs::Job = sqlx::query_as(+ "UPDATE jobs SET status='running', attempts=attempts+1, locked_at=now()+ WHERE id=$1 AND status='queued'+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",+ )+ .bind(id)+ .fetch_one(&pool)+ .await?;jobs::mark_failed(&pool, &job2, &err2).await?;let (status2, attempts2, _): (String, i32, String) =sqlx::query_as("SELECT status, attempts, last_error FROM jobs WHERE id = $1")@@ -85,7 +83,7 @@ async fn a_deferred_job_does_not_spend_its_budget() -> anyhow::Result<()> {.fetch_one(&pool).await?;assert_eq!(status2, "queued");- assert_eq!(attempts2, 0, "两次 Deferred 之后 attempts 应该退到 0");+ assert_eq!(attempts2, 1, "第二次认领再 Deferred 不应消耗重试预算");// 收尾:避免污染下一次跑(其它测试共享同一个库)sqlx::query("DELETE FROM jobs WHERE id = $1")@@ -104,25 +102,19 @@ async fn terminal_wins_over_deferred_when_both_attached() -> anyhow::Result<()>};let pool = PgPool::connect(&url).await?;- let (id,): (i64,) = sqlx::query_as(- "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)+ let job: jobs::Job = sqlx::query_as(+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts, locked_at)VALUES ('extract_document', '{\"document_id\":\"00000000-0000-0000-0000-000000000000\"}',- 'running', 1, 3)- RETURNING id",+ 'running', 1, 3, now())+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",).fetch_one(&pool).await?;+ let id = job.id;// 先挂 Deferred,再挂 Terminal。Terminal 是「最后一次改主意」的那一个。let err = anyhow!("balance gone after waiting").context(Deferred::new(Duration::from_secs(30)));let err = err.context(utopia_core::Terminal);- let job = jobs::Job {- id,- kind: "extract_document".into(),- payload: serde_json::json!({}),- attempts: 1,- max_attempts: 3,- };jobs::mark_failed(&pool, &job, &err).await?;let (status, attempts): (String, i32) =@@ -151,20 +143,15 @@ async fn a_wait_past_its_window_is_a_failure() -> anyhow::Result<()> {let pool = PgPool::connect(&url).await?;// 第一次挂回去时记下开始等的时刻- let (fresh,): (i64,) = sqlx::query_as(- "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)- VALUES ('extract_document', '{}', 'running', 1, 3) RETURNING id",+ let job: jobs::Job = sqlx::query_as(+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts, locked_at)+ VALUES ('extract_document', '{}', 'running', 1, 3, now())+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",).fetch_one(&pool).await?;+ let fresh = job.id;let err = anyhow!("waiting on ontology index").context(Deferred::new(Duration::from_secs(30)));- let job = jobs::Job {- id: fresh,- kind: "extract_document".into(),- payload: serde_json::json!({}),- attempts: 1,- max_attempts: 3,- };jobs::mark_failed(&pool, &job, &err).await?;let since: Option<String> =sqlx::query_scalar("SELECT payload->>'deferred_since' FROM jobs WHERE id = $1")@@ -174,17 +161,17 @@ async fn a_wait_past_its_window_is_a_failure() -> anyhow::Result<()> {assert!(since.is_some(), "第一次等待应当记下 deferred_since");// 已经等过了期限:不再退回 attempts,走普通退避- let (stale,): (i64,) = sqlx::query_as(- "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)+ let job: jobs::Job = sqlx::query_as(+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts, locked_at)VALUES ('extract_document',jsonb_build_object('deferred_since', (now() - make_interval(secs => $1::float8))::text),- 'running', 1, 3)- RETURNING id",+ 'running', 1, 3, now())+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",).bind((jobs::DEFER_WINDOW_SECS + 60) as f64).fetch_one(&pool).await?;- let job = jobs::Job { id: stale, ..job };+ let stale = job.id;jobs::mark_failed(&pool, &job, &err).await?;let (status, attempts): (String, i32) =sqlx::query_as("SELECT status, attempts FROM jobs WHERE id = $1")diff --git a/docs/decisions/0051-a-human-phrase-decision-carries-its-materialization-work.md b/docs/decisions/0051-a-human-phrase-decision-carries-its-materialization-work.mdindex 7623a6f5..525557c7 100644--- a/docs/decisions/0051-a-human-phrase-decision-carries-its-materialization-work.md+++ b/docs/decisions/0051-a-human-phrase-decision-carries-its-materialization-work.md@@ -1,6 +1,6 @@# 0051 · A human phrase decision carries its materialization work-- **Status**: Implemented 2026-09-23 (#876)+- **Status**: Implemented · 2026-09-23 (#876) · revised 2026-10-12 ([#1137](https://github.com/deeplethe/utopia/pull/1137))- **Written**: 2026-09-21- **Related**: [0044](0044-the-ontology-is-a-view-over-what-documents-say.md); [PR #841](https://github.com/deeplethe/utopia/pull/841).@@ -81,6 +81,17 @@ notification-loss polling, old-aligner/new-handler overlap, and production routepermissions/status reads/UI E2E remain explicit acceptance work. Existing temporaland evidence tests must pass after any extraction; the experiment is not a substitute.+**Revision 2026-10-12 (job result writeback, [#1137](https://github.com/deeplethe/utopia/pull/1137)).**+While the worker lives, this supersedes the failed-ack limitation above (#1106).+Guard every outcome update by ID, `status = 'running'`, and the exact database+`locked_at` returned by `claim_one`, including the expired Deferred fallback.+Freeze the result, full error, retry decision and completion time; repeated writes+cannot spend another business attempt, move the schedule or extend deferral.+Retry persistence in the same task after 1, 2, 4, 8, 16, then at most 30 seconds;+retain the worker slot, release connections before waiting, and stop on pool closure.+Outcomes stay in memory: process exit may rerun business work, and persistent+write failures retain slots. No schema change or multi-instance guarantee is added.+After contract approval, wire the route's existing authorization and binding lookupto a same-transaction decision+job function; register the pure handler in main;add job status authorization and completion/failure events; update Review and both