Skip to content

World's first open-source enterprise world model.

Retry computed job outcomes without rerunning their handlers (#1137)

Danielb7f756e1806f
diff --git a/crates/utopia-server/src/extraction_open_out_of_credit_tests.rs b/crates/utopia-server/src/extraction_open_out_of_credit_tests.rs
index 7585ea71..c53d855c 100644
--- a/crates/utopia-server/src/extraction_open_out_of_credit_tests.rs
+++ b/crates/utopia-server/src/extraction_open_out_of_credit_tests.rs
@@ -94,6 +94,7 @@ async fn an_empty_balance_stops_the_document_and_keeps_what_was_applied() -> any
payload: json!({}),
attempts: 1,
max_attempts: 3,
+ locked_at: chrono::Utc::now(),
};
crate::alerting::observe_job_failure(&state, &job, &err).await;
let alerts: Vec<String> = sqlx::query_scalar(
diff --git a/crates/utopia-store/src/jobs.rs b/crates/utopia-store/src/jobs.rs
index 8206f8a7..12d2553d 100644
--- a/crates/utopia-store/src/jobs.rs
+++ b/crates/utopia-store/src/jobs.rs
@@ -21,6 +21,7 @@ pub struct Job {
pub payload: serde_json::Value,
pub attempts: i32,
pub max_attempts: i32,
+ pub locked_at: DateTime<Utc>,
}
/// 同种任务、同样载荷已经排着就不再排:一批文档各自抽完都想触发同一个库级任务
@@ -352,27 +353,13 @@ async fn claim_one(pool: &PgPool) -> AppResult<Option<Job>> {
FOR UPDATE SKIP LOCKED
LIMIT 1
)
- RETURNING id, kind, payload, attempts, max_attempts",
+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",
)
.fetch_optional(pool)
.await?;
Ok(job)
}
-async fn mark_done(pool: &PgPool, id: i64) -> AppResult<()> {
- // **成功要把上一次的错清掉。** 重试成功后 last_error 仍留着失败那次的原文,
- // 于是任务表里出现 status='done' 配着一条错误信息——查问题的人读到的是
- // 一个已经不成立的原因。实测就这么误导过一次:bootstrap 明明跑成了,
- // 表上还挂着 "column relation_type does not exist"。
- sqlx::query(
- "UPDATE jobs SET status = 'done', last_error = NULL, updated_at = now() WHERE id = $1",
- )
- .bind(id)
- .execute(pool)
- .await?;
- Ok(())
-}
-
/// 下一次重试等多久;`None` = 到此为止。
///
/// 两个理由到此为止:**次数用完了**,或者**处理器说了这次不会因为重试而变好**
@@ -413,81 +400,160 @@ fn deferred_retry_secs(retry_in: std::time::Duration) -> i64 {
}
}
-/// `pub` 给集成测试用——主流程仍然由 `run_worker` 内的私有 caller 调用,
-/// 不会从这里出。`#[doc(hidden)]` 是因为它属于内部契约,不进公开 API。
-#[doc(hidden)]
-pub async fn mark_failed(pool: &PgPool, job: &Job, err: &anyhow::Error) -> AppResult<()> {
- let text = format!("{err:#}");
- // `Terminal` 优先:处理器最后改主意说「这次不算了」就该走 `failed` 路径,
- // 不该被 `Deferred` 覆盖。两个都挂时由调用方决定——`is_terminal` 写在前面。
- if utopia_core::is_terminal(err) {
+enum JobOutcome {
+ Done,
+ Failed {
+ error: String,
+ backoff_secs: Option<i64>,
+ deferred_secs: Option<i64>,
+ },
+}
+
+impl JobOutcome {
+ fn failed(job: &Job, err: &anyhow::Error) -> Self {
+ let terminal = utopia_core::is_terminal(err);
+ // Terminal 优先;结果写回重试沿用同一分类和完整错误原文。
+ let deferred_secs = if terminal {
+ None
+ } else {
+ utopia_core::is_deferred(err).map(deferred_retry_secs)
+ };
+ Self::Failed {
+ error: format!("{err:#}"),
+ backoff_secs: retry_delay(job.attempts, job.max_attempts, terminal),
+ deferred_secs,
+ }
+ }
+}
+
+/// true = 本次写入;false = 已结束或领取失效。两者都结束这次写回。
+async fn persist_outcome(
+ pool: &PgPool,
+ job: &Job,
+ outcome: &JobOutcome,
+ completed_at: DateTime<Utc>,
+) -> AppResult<bool> {
+ let JobOutcome::Failed {
+ error,
+ backoff_secs,
+ deferred_secs,
+ } = outcome
+ else {
+ // 成功要清掉前一次错误,避免任务显示 done 却仍带着失败原因。
let res = sqlx::query(
- "UPDATE jobs SET status = 'failed', last_error = $2, updated_at = now() WHERE id = $1",
+ "UPDATE jobs SET status = 'done', last_error = NULL, updated_at = now()
+ WHERE id = $1 AND status = 'running' AND locked_at = $2",
)
.bind(job.id)
- .bind(&text)
+ .bind(job.locked_at)
.execute(pool)
.await?;
- let _ = res.rows_affected();
- return Ok(());
- }
- // `Deferred`(#526):把任务挂回 `queued`,把 `attempts` 退回去,不烧预算。
- // 第一次走到这里时 `claim_one` 已经把 `attempts` 加 1,写回时要 -1,
- // 否则同一次等待会让 `attempts` 慢慢爬到 `max_attempts`,最后那条
- // `failed` 是我们最不想看见的——ontology 还差一秒就绪,文档却先死了。
- //
- // **等也有期限。** 从第一次挂回去算起(记在 payload 的 `deferred_since`,不用
- // `created_at`:一批上传排队几小时是常态,那不算在等)超过 [`DEFER_WINDOW_SECS`]
- // 还在等,就不再挂回去,落到下面的普通退避、烧预算。等的那件事(比如
- // `embed_ontology`)自己一直失败时,不设期限这条任务会每 30 秒醒一次、永远排着,
- // 却没有一次被记成失败
- if let Some(retry_in) = utopia_core::is_deferred(err) {
- let secs = deferred_retry_secs(retry_in);
+ return Ok(res.rows_affected() > 0);
+ };
+
+ if let Some(secs) = deferred_secs {
+ // Deferred 退回认领消耗的次数;等待窗口耗尽后才走普通失败预算。
+ // 等待窗口和下次运行时刻固定在 handler 完成时,不随写回重试滑动。
let res = sqlx::query(
"UPDATE jobs SET status = 'queued', last_error = $2,
attempts = GREATEST(0, attempts - 1),
- run_at = now() + make_interval(secs => $3::float8),
+ run_at = $4::timestamptz + make_interval(secs => $5::float8),
payload = payload || jsonb_build_object('deferred_since',
- COALESCE(payload->>'deferred_since', now()::text)),
+ COALESCE(payload->>'deferred_since', $4::timestamptz::text)),
updated_at = now()
- WHERE id = $1
- AND COALESCE((payload->>'deferred_since')::timestamptz, now())
- > now() - make_interval(secs => $4::float8)",
+ WHERE id = $1 AND status = 'running' AND locked_at = $3
+ AND COALESCE((payload->>'deferred_since')::timestamptz, $4)
+ > $4 - make_interval(secs => $6::float8)",
)
.bind(job.id)
- .bind(&text)
- .bind(secs as f64)
+ .bind(error)
+ .bind(job.locked_at)
+ .bind(completed_at)
+ .bind(*secs as f64)
.bind(DEFER_WINDOW_SECS as f64)
.execute(pool)
.await?;
if res.rows_affected() > 0 {
- return Ok(());
+ return Ok(true);
}
+ // 0 行也可能是旧领取;普通失败分支保留相同保护,不能覆盖新执行。
}
- let Some(backoff_secs) = retry_delay(job.attempts, job.max_attempts, false) else {
- sqlx::query(
- "UPDATE jobs SET status = 'failed', last_error = $2, updated_at = now() WHERE id = $1",
- )
- .bind(job.id)
- .bind(&text)
- .execute(pool)
- .await?;
- return Ok(());
+
+ let status = if backoff_secs.is_some() {
+ "queued"
+ } else {
+ "failed"
};
- sqlx::query(
- "UPDATE jobs SET status = 'queued', last_error = $2,
- run_at = now() + make_interval(secs => $3::float8),
+ let res = sqlx::query(
+ "UPDATE jobs SET status = $3, last_error = $4,
+ run_at = CASE WHEN $5::float8 IS NULL THEN run_at
+ ELSE $6::timestamptz + make_interval(secs => $5::float8) END,
updated_at = now()
- WHERE id = $1",
+ WHERE id = $1 AND status = 'running' AND locked_at = $2",
)
.bind(job.id)
- .bind(&text)
- .bind(backoff_secs as f64)
+ .bind(job.locked_at)
+ .bind(status)
+ .bind(error)
+ .bind(backoff_secs.map(|secs| secs as f64))
+ .bind(completed_at)
.execute(pool)
.await?;
+ Ok(res.rows_affected() > 0)
+}
+
+/// 集成测试的单次写回入口;worker 保存结果后由 persist_with_retry 写回。
+#[doc(hidden)]
+pub async fn mark_failed(pool: &PgPool, job: &Job, err: &anyhow::Error) -> AppResult<()> {
+ persist_outcome(pool, job, &JobOutcome::failed(job, err), Utc::now()).await?;
Ok(())
}
+async fn persist_with_retry(
+ pool: &PgPool,
+ job: &Job,
+ outcome: &JobOutcome,
+ completed_at: DateTime<Utc>,
+) {
+ let mut delay = Duration::from_secs(1);
+ loop {
+ if pool.is_closed() {
+ tracing::warn!(
+ job_id = job.id,
+ "数据库连接池已关闭,未确认的任务留待启动回收"
+ );
+ return;
+ }
+ let result = tokio::select! {
+ _ = pool.close_event() => {
+ tracing::warn!(job_id = job.id, "数据库连接池已关闭,未确认的任务留待启动回收");
+ return;
+ }
+ result = persist_outcome(pool, job, outcome, completed_at) => result,
+ };
+ match result {
+ Ok(updated) => {
+ if !updated {
+ tracing::debug!(
+ job_id = job.id,
+ claimed_at = %job.locked_at,
+ "任务已结束或领取已失效,停止写回旧结果"
+ );
+ }
+ return;
+ }
+ Err(e) => {
+ tracing::error!(job_id = job.id, error = %e, retry_in_secs = delay.as_secs(), "任务状态写回失败,将重试同一结果");
+ }
+ }
+ tokio::select! {
+ _ = pool.close_event() => {}
+ _ = tokio::time::sleep(delay) => {}
+ }
+ delay = (delay * 2).min(Duration::from_secs(30));
+ }
+}
+
/// worker 调度循环:运行中任务数低于目标并发就继续认领(有活立即续派),
/// 空闲时等入队通知、2s 轮询兜底(#517);每个任务在独立 tokio task 中执行,长抽取不再阻塞同步。
/// `concurrency` 每轮热读——系统设置里改并发数即时生效。
@@ -522,6 +588,9 @@ where
);
let mut listener = listen_for_jobs(&pool).await;
loop {
+ if pool.is_closed() {
+ return;
+ }
let cap = concurrency.load(Ordering::Relaxed).max(1);
if running.load(Ordering::Relaxed) >= cap {
tokio::time::sleep(Duration::from_millis(200)).await;
@@ -550,16 +619,15 @@ where
Err(anyhow::anyhow!("任务处理器 panic(详情见 stderr):{join}"))
}
};
+ let completed_at = Utc::now();
let outcome = match result {
- Ok(()) => mark_done(&pool, job.id).await,
+ Ok(()) => JobOutcome::Done,
Err(e) => {
tracing::warn!(job_id = job.id, kind = %job.kind, error = %e, "任务执行失败");
- mark_failed(&pool, &job, &e).await
+ JobOutcome::failed(&job, &e)
}
};
- if let Err(e) = outcome {
- tracing::error!(job_id = job.id, error = %e, "任务状态写回失败");
- }
+ persist_with_retry(&pool, &job, &outcome, completed_at).await;
running.fetch_sub(1, Ordering::Relaxed);
});
}
@@ -574,6 +642,10 @@ where
}
}
+#[cfg(test)]
+#[path = "jobs_writeback_tests.rs"]
+mod writeback_tests;
+
#[cfg(test)]
mod tests {
use super::{deferred_retry_secs, retry_delay};
diff --git a/crates/utopia-store/src/jobs_writeback_tests.rs b/crates/utopia-store/src/jobs_writeback_tests.rs
new file mode 100644
index 00000000..824060f9
--- /dev/null
+++ b/crates/utopia-store/src/jobs_writeback_tests.rs
@@ -0,0 +1,364 @@
+//! 写回测试只操作自己的任务行,不启动会认领全库任务的 worker。
+
+use super::{persist_outcome, persist_with_retry, Job, JobOutcome, DEFER_WINDOW_SECS};
+use chrono::{DateTime, Utc};
+use serde_json::{json, Value};
+use sqlx::PgPool;
+use std::time::Duration;
+use utopia_core::Deferred;
+use uuid::Uuid;
+
+async fn running_job(pool: &PgPool, attempts: i32) -> anyhow::Result<Job> {
+ Ok(sqlx::query_as(
+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts,
+ locked_at, last_error)
+ VALUES ($1, '{}', 'running', $2, 3, now(), 'previous failure')
+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",
+ )
+ .bind(format!("test_writeback_{}", Uuid::now_v7()))
+ .bind(attempts)
+ .fetch_one(pool)
+ .await?)
+}
+
+async fn claim_job(pool: &PgPool, id: i64) -> anyhow::Result<Job> {
+ Ok(sqlx::query_as(
+ "UPDATE jobs SET status = 'running', attempts = attempts + 1,
+ locked_at = now(), updated_at = now()
+ WHERE id = $1 AND status = 'queued'
+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",
+ )
+ .bind(id)
+ .fetch_one(pool)
+ .await?)
+}
+
+#[derive(Debug, PartialEq, sqlx::FromRow)]
+struct JobState {
+ status: String,
+ attempts: i32,
+ locked_at: Option<DateTime<Utc>>,
+ last_error: Option<String>,
+ run_at: DateTime<Utc>,
+ updated_at: DateTime<Utc>,
+ payload: Value,
+}
+
+async fn state(pool: &PgPool, id: i64) -> anyhow::Result<JobState> {
+ Ok(sqlx::query_as(
+ "SELECT status, attempts, locked_at, last_error, run_at, updated_at, payload
+ FROM jobs WHERE id = $1",
+ )
+ .bind(id)
+ .fetch_one(pool)
+ .await?)
+}
+
+async fn remove(pool: &PgPool, id: i64) -> anyhow::Result<()> {
+ sqlx::query("DELETE FROM jobs WHERE id = $1")
+ .bind(id)
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn a_refused_write_retries_the_computed_result() -> anyhow::Result<()> {
+ let Some(url) = crate::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let job = running_job(&pool, 1).await?;
+ let object_name = format!("test_writeback_retry_{}", Uuid::now_v7().simple());
+
+ let result: anyhow::Result<()> = async {
+ // sequence 不随拒写事务回滚,计数能证明第一次写入确实失败。
+ sqlx::raw_sql(&format!(
+ "CREATE SEQUENCE {object_name};
+ CREATE FUNCTION {object_name}() RETURNS trigger LANGUAGE plpgsql AS $$
+ BEGIN
+ IF nextval('{object_name}') = 1 THEN
+ RAISE EXCEPTION 'injected result writeback failure';
+ END IF;
+ RETURN NEW;
+ END; $$;
+ CREATE TRIGGER {object_name} BEFORE UPDATE ON jobs
+ FOR EACH ROW WHEN (OLD.id = {id} AND OLD.kind = '{kind}'
+ AND OLD.status = 'running' AND NEW.status = 'done')
+ EXECUTE FUNCTION {object_name}();",
+ id = job.id,
+ kind = job.kind,
+ ))
+ .execute(&pool)
+ .await?;
+
+ tokio::time::timeout(
+ Duration::from_secs(10),
+ persist_with_retry(&pool, &job, &JobOutcome::Done, Utc::now()),
+ )
+ .await?;
+ let writes: i64 = sqlx::query_scalar(&format!("SELECT last_value FROM {object_name}"))
+ .fetch_one(&pool)
+ .await?;
+ anyhow::ensure!(
+ writes == 2,
+ "expected one refused write and one successful retry"
+ );
+ let written = state(&pool, job.id).await?;
+ anyhow::ensure!(written.status == "done" && written.last_error.is_none());
+ anyhow::ensure!(written.attempts == job.attempts);
+ anyhow::ensure!(written.locked_at == Some(job.locked_at));
+ Ok(())
+ }
+ .await;
+
+ let dropped = sqlx::raw_sql(&format!(
+ "DROP TRIGGER IF EXISTS {object_name} ON jobs;
+ DROP FUNCTION IF EXISTS {object_name}();
+ DROP SEQUENCE IF EXISTS {object_name};"
+ ))
+ .execute(&pool)
+ .await;
+ let removed = remove(&pool, job.id).await;
+ pool.close().await;
+ result?;
+ dropped?;
+ removed?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn an_old_claim_cannot_overwrite_any_new_running_result() -> anyhow::Result<()> {
+ let Some(url) = crate::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let completed_at = DateTime::<Utc>::from_timestamp(1_700_000_000, 0).unwrap();
+ let ordinary = anyhow::anyhow!("network failure").context("provider request");
+ let deferred = anyhow::anyhow!("index busy").context(Deferred::new(Duration::from_secs(17)));
+
+ // 窗口内和窗口外的旧 Deferred 都不能把新认领写成失败。
+ for expired in [false, true] {
+ let job = running_job(&pool, 1).await?;
+ let payload = if expired {
+ json!({"deferred_since": (completed_at - chrono::Duration::hours(2)).to_rfc3339()})
+ } else {
+ json!({})
+ };
+ // 先提交重排,再从新事务认领;不伪造认领时间,也不认领其他测试的任务。
+ sqlx::query("UPDATE jobs SET status = 'queued', attempts = 0, payload = $2 WHERE id = $1")
+ .bind(job.id)
+ .bind(payload)
+ .execute(&pool)
+ .await?;
+ let new_job = claim_job(&pool, job.id).await?;
+ assert_eq!(new_job.attempts, job.attempts);
+ assert_ne!(new_job.locked_at, job.locked_at);
+ let new_execution = state(&pool, job.id).await?;
+
+ for outcome in [
+ JobOutcome::Done,
+ JobOutcome::failed(&job, &ordinary),
+ JobOutcome::failed(&job, &deferred),
+ ] {
+ assert!(!persist_outcome(&pool, &job, &outcome, completed_at).await?);
+ assert_eq!(state(&pool, job.id).await?, new_execution);
+ }
+ remove(&pool, job.id).await?;
+ }
+ pool.close().await;
+ Ok(())
+}
+
+#[tokio::test]
+async fn a_repeated_ack_preserves_the_first_writeback() -> anyhow::Result<()> {
+ let Some(url) = crate::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let completed_at = DateTime::<Utc>::from_timestamp(1_700_000_000, 0).unwrap();
+ let deferred = anyhow::anyhow!("index busy").context(Deferred::new(Duration::from_secs(17)));
+ let ordinary =
+ anyhow::anyhow!("provider detail: request rejected").context("outer request context");
+
+ for (error, starts_deferral) in [
+ (None, false),
+ (Some(&ordinary), false),
+ (Some(&deferred), true),
+ ] {
+ let job = running_job(&pool, 2).await?;
+ let outcome = match error {
+ Some(error) => JobOutcome::failed(&job, error),
+ None => JobOutcome::Done,
+ };
+ assert!(persist_outcome(&pool, &job, &outcome, completed_at).await?);
+ let first_ack = state(&pool, job.id).await?;
+ if starts_deferral {
+ let since: DateTime<Utc> = sqlx::query_scalar("SELECT $1::text::timestamptz")
+ .bind(first_ack.payload["deferred_since"].as_str())
+ .fetch_one(&pool)
+ .await?;
+ assert_eq!(since, completed_at);
+ }
+ let duplicate_ack = persist_outcome(
+ &pool,
+ &job,
+ &outcome,
+ completed_at + chrono::Duration::hours(1),
+ )
+ .await?;
+ assert!(!duplicate_ack);
+ assert_eq!(state(&pool, job.id).await?, first_ack);
+ remove(&pool, job.id).await?;
+ }
+ pool.close().await;
+ Ok(())
+}
+
+#[tokio::test]
+async fn deferred_window_uses_completion_time_and_current_database_payload() -> anyhow::Result<()> {
+ let Some(url) = crate::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ // 故意用早于当前时间的完成时刻,写回等待不能令有效的 Deferred 过期。
+ let completed_at = DateTime::<Utc>::from_timestamp(1_700_000_000, 0).unwrap();
+ let error = anyhow::anyhow!("index busy").context(Deferred::new(Duration::from_secs(17)));
+ for seconds_inside_window in [1, 0] {
+ let job = running_job(&pool, 2).await?;
+ let since =
+ completed_at - chrono::Duration::seconds(DEFER_WINDOW_SECS - seconds_inside_window);
+ let payload = json!({"deferred_since": since.to_rfc3339(), "kept": "value"});
+ // handler 可以在认领后更新 payload,窗口起点以已写入数据库的值为准。
+ sqlx::query("UPDATE jobs SET payload = $2 WHERE id = $1")
+ .bind(job.id)
+ .bind(&payload)
+ .execute(&pool)
+ .await?;
+ let outcome = JobOutcome::failed(&job, &error);
+ assert!(persist_outcome(&pool, &job, &outcome, completed_at).await?);
+ let written = state(&pool, job.id).await?;
+ assert_eq!(written.status, "queued");
+ assert_eq!(written.payload, payload);
+ if seconds_inside_window == 1 {
+ assert_eq!(written.attempts, 1, "valid Deferred does not spend budget");
+ assert_eq!(written.run_at, completed_at + chrono::Duration::seconds(17));
+ } else {
+ assert_eq!(
+ written.attempts, 2,
+ "expired Deferred spends ordinary budget"
+ );
+ assert_eq!(
+ written.run_at,
+ completed_at + chrono::Duration::seconds(120)
+ );
+ }
+ remove(&pool, job.id).await?;
+ }
+ pool.close().await;
+ Ok(())
+}
+
+#[tokio::test]
+async fn a_closed_pool_does_not_keep_writeback_retry_alive() -> anyhow::Result<()> {
+ // 已关闭的惰性池不会建立连接,此测试不依赖数据库。
+ let pool = PgPool::connect_lazy("postgres://test:[email protected]:1/test")?;
+ pool.close().await;
+ let job = Job {
+ id: 0,
+ kind: "closed_pool_writeback".into(),
+ payload: json!({}),
+ attempts: 1,
+ max_attempts: 3,
+ locked_at: Utc::now(),
+ };
+ tokio::time::timeout(
+ Duration::from_secs(1),
+ super::persist_with_retry(&pool, &job, &JobOutcome::Done, Utc::now()),
+ )
+ .await?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn closing_the_pool_cancels_writeback_while_it_waits_for_a_row_lock() -> anyhow::Result<()> {
+ let Some(url) = crate::test_db::url() else {
+ return Ok(());
+ };
+ let control = sqlx::postgres::PgPoolOptions::new()
+ .max_connections(2)
+ .connect(&url)
+ .await?;
+ let job = running_job(&control, 1).await?;
+ let before = state(&control, job.id).await?;
+ let mut locked = control.begin().await?;
+ let locker_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
+ .fetch_one(&mut *locked)
+ .await?;
+ sqlx::query("SELECT id FROM jobs WHERE id = $1 FOR UPDATE")
+ .bind(job.id)
+ .execute(&mut *locked)
+ .await?;
+
+ let writer_pool = sqlx::postgres::PgPoolOptions::new()
+ .max_connections(1)
+ .connect(&url)
+ .await?;
+ let writer_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
+ .fetch_one(&writer_pool)
+ .await?;
+ let mut writer = tokio::spawn({
+ let pool = writer_pool.clone();
+ let job = job.clone();
+ async move {
+ super::persist_with_retry(&pool, &job, &JobOutcome::Done, Utc::now()).await;
+ }
+ });
+
+ let result: anyhow::Result<()> = async {
+ // 观测自己的后端确实在等这把锁,再关闭池,不靠固定睡眠猜测执行进度。
+ tokio::time::timeout(Duration::from_secs(2), async {
+ loop {
+ let waiting: bool = sqlx::query_scalar(
+ "SELECT EXISTS (SELECT 1 FROM pg_stat_activity
+ WHERE pid = $1 AND wait_event_type = 'Lock'
+ AND $2 = ANY(pg_blocking_pids(pid)))",
+ )
+ .bind(writer_pid)
+ .bind(locker_pid)
+ .fetch_one(&control)
+ .await?;
+ if waiting {
+ return anyhow::Ok(());
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await??;
+
+ let (finished, ()) = tokio::time::timeout(Duration::from_secs(2), async {
+ tokio::join!(&mut writer, writer_pool.close())
+ })
+ .await?;
+ finished?;
+ let unchanged = state(&control, job.id).await?;
+ anyhow::ensure!(
+ unchanged == before,
+ "pool closure changed the locked job: {unchanged:?}"
+ );
+ anyhow::Ok(())
+ }
+ .await;
+
+ // 失败路径也撤销自己的行锁并终止测试 task,不能把挂起的写回留给其他测试。
+ writer.abort();
+ let unlocked = locked.rollback().await;
+ let closed = tokio::time::timeout(Duration::from_secs(2), writer_pool.close()).await;
+ let removed = remove(&control, job.id).await;
+ control.close().await;
+ result?;
+ unlocked?;
+ closed?;
+ removed?;
+ Ok(())
+}
diff --git a/crates/utopia-store/src/materialize_delivery_tests.rs b/crates/utopia-store/src/materialize_delivery_tests.rs
index 97b154a4..8924c7cb 100644
--- a/crates/utopia-store/src/materialize_delivery_tests.rs
+++ b/crates/utopia-store/src/materialize_delivery_tests.rs
@@ -70,7 +70,7 @@ async fn accept(
}
async fn claim(pool: &PgPool, id: i64) -> anyhow::Result<jobs::Job> {
// Restrict the production claim SQL to this test's job, never steal work.
- Ok(sqlx::query_as("UPDATE jobs SET status='running', attempts=attempts+1, locked_at=now() WHERE id=$1 AND status='queued' RETURNING id,kind,payload,attempts,max_attempts")
+ Ok(sqlx::query_as("UPDATE jobs SET status='running', attempts=attempts+1, locked_at=now() WHERE id=$1 AND status='queued' RETURNING id,kind,payload,attempts,max_attempts,locked_at")
.bind(id).fetch_one(pool).await?)
}
async fn handle(pool: &PgPool, kb: Uuid, job: &jobs::Job) -> anyhow::Result<()> {
diff --git a/crates/utopia-store/tests/human_phrase_materialization_delivery.rs b/crates/utopia-store/tests/human_phrase_materialization_delivery.rs
index 6f32429d..e9f0952f 100644
--- a/crates/utopia-store/tests/human_phrase_materialization_delivery.rs
+++ b/crates/utopia-store/tests/human_phrase_materialization_delivery.rs
@@ -43,7 +43,7 @@ async fn accept(
}
async fn claim(pool: &PgPool, id: i64) -> anyhow::Result<jobs::Job> {
// Restrict the production claim SQL to this test's job, never steal work.
- Ok(sqlx::query_as("UPDATE jobs SET status='running', attempts=attempts+1, locked_at=now() WHERE id=$1 AND status='queued' RETURNING id,kind,payload,attempts,max_attempts")
+ Ok(sqlx::query_as("UPDATE jobs SET status='running', attempts=attempts+1, locked_at=now() WHERE id=$1 AND status='queued' RETURNING id,kind,payload,attempts,max_attempts,locked_at")
.bind(id).fetch_one(pool).await?)
}
async fn handle(pool: &PgPool, kb: Uuid, job: &jobs::Job) -> anyhow::Result<()> {
diff --git a/crates/utopia-store/tests/store/a_deferred_job_does_not_spend_its_budget.rs b/crates/utopia-store/tests/store/a_deferred_job_does_not_spend_its_budget.rs
index 3c375c7d..0ecc4c08 100644
--- a/crates/utopia-store/tests/store/a_deferred_job_does_not_spend_its_budget.rs
+++ b/crates/utopia-store/tests/store/a_deferred_job_does_not_spend_its_budget.rs
@@ -26,24 +26,18 @@ async fn a_deferred_job_does_not_spend_its_budget() -> anyhow::Result<()> {
// 直接插一行「正在跑」的任务——attempts 已经由 `claim_one` 加过一次,
// 这一关的语义是 `mark_failed(Deferred)` 写回时把它退回去。
- let (id,): (i64,) = sqlx::query_as(
- "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)
+ let job: jobs::Job = sqlx::query_as(
+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts, locked_at)
VALUES ('extract_document', '{\"document_id\":\"00000000-0000-0000-0000-000000000000\"}',
- 'running', 2, 3)
- RETURNING id",
+ 'running', 2, 3, now())
+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",
)
.fetch_one(&pool)
.await?;
+ let id = job.id;
// 第一次等待:本体向量补齐,30s。
let err = anyhow!("waiting on ontology index").context(Deferred::new(Duration::from_secs(30)));
- let job = jobs::Job {
- id,
- kind: "extract_document".into(),
- payload: serde_json::json!({}),
- attempts: 2,
- max_attempts: 3,
- };
jobs::mark_failed(&pool, &job, &err).await?;
let (status, attempts, last_error): (String, i32, String) =
@@ -74,10 +68,14 @@ async fn a_deferred_job_does_not_spend_its_budget() -> anyhow::Result<()> {
// 第二次等待:同一个任务、同一段等待。**预算不烧**——
// 这是与「限流重试」的关键区别。
let err2 = anyhow!("waiting on ontology index").context(Deferred::new(Duration::from_secs(30)));
- let job2 = jobs::Job {
- attempts: 1, // 第一次之后的状态
- ..job.clone()
- };
+ let job2: jobs::Job = sqlx::query_as(
+ "UPDATE jobs SET status='running', attempts=attempts+1, locked_at=now()
+ WHERE id=$1 AND status='queued'
+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",
+ )
+ .bind(id)
+ .fetch_one(&pool)
+ .await?;
jobs::mark_failed(&pool, &job2, &err2).await?;
let (status2, attempts2, _): (String, i32, String) =
sqlx::query_as("SELECT status, attempts, last_error FROM jobs WHERE id = $1")
@@ -85,7 +83,7 @@ async fn a_deferred_job_does_not_spend_its_budget() -> anyhow::Result<()> {
.fetch_one(&pool)
.await?;
assert_eq!(status2, "queued");
- assert_eq!(attempts2, 0, "两次 Deferred 之后 attempts 应该退到 0");
+ assert_eq!(attempts2, 1, "第二次认领再 Deferred 不应消耗重试预算");
// 收尾:避免污染下一次跑(其它测试共享同一个库)
sqlx::query("DELETE FROM jobs WHERE id = $1")
@@ -104,25 +102,19 @@ async fn terminal_wins_over_deferred_when_both_attached() -> anyhow::Result<()>
};
let pool = PgPool::connect(&url).await?;
- let (id,): (i64,) = sqlx::query_as(
- "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)
+ let job: jobs::Job = sqlx::query_as(
+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts, locked_at)
VALUES ('extract_document', '{\"document_id\":\"00000000-0000-0000-0000-000000000000\"}',
- 'running', 1, 3)
- RETURNING id",
+ 'running', 1, 3, now())
+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",
)
.fetch_one(&pool)
.await?;
+ let id = job.id;
// 先挂 Deferred,再挂 Terminal。Terminal 是「最后一次改主意」的那一个。
let err = anyhow!("balance gone after waiting").context(Deferred::new(Duration::from_secs(30)));
let err = err.context(utopia_core::Terminal);
- let job = jobs::Job {
- id,
- kind: "extract_document".into(),
- payload: serde_json::json!({}),
- attempts: 1,
- max_attempts: 3,
- };
jobs::mark_failed(&pool, &job, &err).await?;
let (status, attempts): (String, i32) =
@@ -151,20 +143,15 @@ async fn a_wait_past_its_window_is_a_failure() -> anyhow::Result<()> {
let pool = PgPool::connect(&url).await?;
// 第一次挂回去时记下开始等的时刻
- let (fresh,): (i64,) = sqlx::query_as(
- "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)
- VALUES ('extract_document', '{}', 'running', 1, 3) RETURNING id",
+ let job: jobs::Job = sqlx::query_as(
+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts, locked_at)
+ VALUES ('extract_document', '{}', 'running', 1, 3, now())
+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",
)
.fetch_one(&pool)
.await?;
+ let fresh = job.id;
let err = anyhow!("waiting on ontology index").context(Deferred::new(Duration::from_secs(30)));
- let job = jobs::Job {
- id: fresh,
- kind: "extract_document".into(),
- payload: serde_json::json!({}),
- attempts: 1,
- max_attempts: 3,
- };
jobs::mark_failed(&pool, &job, &err).await?;
let since: Option<String> =
sqlx::query_scalar("SELECT payload->>'deferred_since' FROM jobs WHERE id = $1")
@@ -174,17 +161,17 @@ async fn a_wait_past_its_window_is_a_failure() -> anyhow::Result<()> {
assert!(since.is_some(), "第一次等待应当记下 deferred_since");
// 已经等过了期限:不再退回 attempts,走普通退避
- let (stale,): (i64,) = sqlx::query_as(
- "INSERT INTO jobs (kind, payload, status, attempts, max_attempts)
+ let job: jobs::Job = sqlx::query_as(
+ "INSERT INTO jobs (kind, payload, status, attempts, max_attempts, locked_at)
VALUES ('extract_document',
jsonb_build_object('deferred_since', (now() - make_interval(secs => $1::float8))::text),
- 'running', 1, 3)
- RETURNING id",
+ 'running', 1, 3, now())
+ RETURNING id, kind, payload, attempts, max_attempts, locked_at",
)
.bind((jobs::DEFER_WINDOW_SECS + 60) as f64)
.fetch_one(&pool)
.await?;
- let job = jobs::Job { id: stale, ..job };
+ let stale = job.id;
jobs::mark_failed(&pool, &job, &err).await?;
let (status, attempts): (String, i32) =
sqlx::query_as("SELECT status, attempts FROM jobs WHERE id = $1")
diff --git a/docs/decisions/0051-a-human-phrase-decision-carries-its-materialization-work.md b/docs/decisions/0051-a-human-phrase-decision-carries-its-materialization-work.md
index 7623a6f5..525557c7 100644
--- a/docs/decisions/0051-a-human-phrase-decision-carries-its-materialization-work.md
+++ b/docs/decisions/0051-a-human-phrase-decision-carries-its-materialization-work.md
@@ -1,6 +1,6 @@
# 0051 · A human phrase decision carries its materialization work
-- **Status**: Implemented 2026-09-23 (#876)
+- **Status**: Implemented · 2026-09-23 (#876) · revised 2026-10-12 ([#1137](https://github.com/deeplethe/utopia/pull/1137))
- **Written**: 2026-09-21
- **Related**: [0044](0044-the-ontology-is-a-view-over-what-documents-say.md); [PR #841](https://github.com/deeplethe/utopia/pull/841).
@@ -81,6 +81,17 @@ notification-loss polling, old-aligner/new-handler overlap, and production route
permissions/status reads/UI E2E remain explicit acceptance work. Existing temporal
and evidence tests must pass after any extraction; the experiment is not a substitute.
+**Revision 2026-10-12 (job result writeback, [#1137](https://github.com/deeplethe/utopia/pull/1137)).**
+While the worker lives, this supersedes the failed-ack limitation above (#1106).
+Guard every outcome update by ID, `status = 'running'`, and the exact database
+`locked_at` returned by `claim_one`, including the expired Deferred fallback.
+Freeze the result, full error, retry decision and completion time; repeated writes
+cannot spend another business attempt, move the schedule or extend deferral.
+Retry persistence in the same task after 1, 2, 4, 8, 16, then at most 30 seconds;
+retain the worker slot, release connections before waiting, and stop on pool closure.
+Outcomes stay in memory: process exit may rerun business work, and persistent
+write failures retain slots. No schema change or multi-instance guarantee is added.
+
After contract approval, wire the route's existing authorization and binding lookup
to a same-transaction decision+job function; register the pure handler in main;
add job status authorization and completion/failure events; update Review and both