World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
//! 任务队列:Postgres `FOR UPDATE SKIP LOCKED` 消费。
//! worker 与 API 同进程(tokio task),失败按 30s * attempts² 退避重试——
//! 除非处理器把它标成了 `utopia_core::Terminal`,那种一次就到此为止(见 [`retry_delay`])。
//! 并发消费:调度循环按"运行中 < 目标数"续派,任务在独立 task 执行;
//! 目标数经 AtomicUsize 热读——系统设置里改并发即时生效,无需重启。
use chrono::{DateTime, Utc};
use sqlx::{postgres::PgListener, PgPool, Postgres, Transaction};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use utopia_core::AppResult;
use uuid::Uuid;
pub const JOB_CHANNEL: &str = "utopia_jobs";
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct Job {
pub id: i64,
pub kind: String,
pub payload: serde_json::Value,
pub attempts: i32,
pub max_attempts: i32,
pub locked_at: DateTime<Utc>,
}
/// 同种任务、同样载荷已经排着就不再排:一批文档各自抽完都想触发同一个库级任务
/// (类型消解、对账),排十次跑十次是浪费,而且后九次看到的是第一次跑完的库。
/// 返回 None = 已有一条在排队
/// 排一件事,除非同样的一件**已经排着或正在跑**。
///
/// 与 [`enqueue_unless_queued`] 的差别只在「正在跑」那一半,而那一半正是批量编辑
/// 会踩的:本体页一口气建 28 条属性,每建一条排一次,第一条开跑之后的 27 条都不再
/// 与「排着的」撞上,于是排成 28 份,串行跑 28 遍。用这个入口的任务必须在**结束时
/// 自己回头看有没有新活**(对齐的两个任务都这么做),否则跑着时来的那些变化会丢。
///
/// `after` 是**去抖**:一批编辑里第一条排下一个几秒后才跑的任务,其余每一条都撞上
/// 这个「排着的」而不再排。没有它,空库上一次运行只要几百毫秒,42 次创建照样排出
/// 十几份——「正在跑」这一半只有在任务真的在跑时才挡得住。
pub async fn enqueue_unless_pending(
pool: &PgPool,
kind: &str,
payload: serde_json::Value,
after: Duration,
) -> AppResult<Option<i64>> {
let mut tx = pool.begin().await?;
let row: Option<(i64,)> = sqlx::query_as(
"INSERT INTO jobs (kind, payload, run_at)
SELECT $1, $2, now() + make_interval(secs => $3)
WHERE NOT EXISTS (SELECT 1 FROM jobs
WHERE kind = $1 AND payload = $2
AND status IN ('queued', 'running'))
RETURNING id",
)
.bind(kind)
.bind(payload)
.bind(after.as_secs_f64())
.fetch_optional(&mut *tx)
.await?;
if row.is_some() {
notify_worker_tx(&mut tx).await?;
}
tx.commit().await?;
Ok(row.map(|(id,)| id))
}
pub async fn enqueue_unless_queued(
pool: &PgPool,
kind: &str,
payload: serde_json::Value,
) -> AppResult<Option<i64>> {
let mut tx = pool.begin().await?;
let id = enqueue_unless_queued_tx(&mut tx, kind, payload).await?;
tx.commit().await?;
Ok(id)
}
/// 同 [`enqueue_unless_queued`],但排在调用方的事务里:任务与事务里别的写一起提交、
/// 一起回滚。人定一个类别词时,判定和它的短语对齐任务要么都在、要么都不在——先提交
/// 判定再排,进程在两步之间退出就只剩判定,它改了的签名没人去重判
pub async fn enqueue_unless_queued_tx(
tx: &mut Transaction<'_, Postgres>,
kind: &str,
payload: serde_json::Value,
) -> AppResult<Option<i64>> {
let row: Option<(i64,)> = sqlx::query_as(
"INSERT INTO jobs (kind, payload)
SELECT $1, $2
WHERE NOT EXISTS (SELECT 1 FROM jobs WHERE kind = $1 AND payload = $2 AND status = 'queued')
RETURNING id",
)
.bind(kind)
.bind(payload)
.fetch_optional(&mut **tx)
.await?;
if row.is_some() {
notify_worker_tx(tx).await?;
}
Ok(row.map(|(id,)| id))
}
/// 同 [`enqueue_unless_queued`],但晚一点跑。挡的只是排着的,不挡在跑的——调用方
/// 正是那个在跑的任务、想给自己之后再排一个的时候,用这个而不是 `enqueue_unless_pending`
pub async fn enqueue_unless_queued_after(
pool: &PgPool,
kind: &str,
payload: serde_json::Value,
after: Duration,
) -> AppResult<Option<i64>> {
let mut tx = pool.begin().await?;
let row: Option<(i64,)> = sqlx::query_as(
"INSERT INTO jobs (kind, payload, run_at)
SELECT $1, $2, now() + make_interval(secs => $3)
WHERE NOT EXISTS (SELECT 1 FROM jobs WHERE kind = $1 AND payload = $2 AND status = 'queued')
RETURNING id",
)
.bind(kind)
.bind(payload)
.bind(after.as_secs_f64())
.fetch_optional(&mut *tx)
.await?;
if row.is_some() {
notify_worker_tx(&mut tx).await?;
}
tx.commit().await?;
Ok(row.map(|(id,)| id))
}
/// 这个库有没有这一种任务排着或跑着(按 payload 里的 kb_id 看,不比整份 payload:
/// 同一种任务的 payload 可能多带一个 reask)
pub async fn pending_for_kb(pool: &PgPool, kind: &str, kb_id: Uuid) -> AppResult<bool> {
Ok(sqlx::query_scalar(
"SELECT EXISTS (SELECT 1 FROM jobs
WHERE kind = $1 AND status IN ('queued', 'running')
AND payload->>'kb_id' = $2)",
)
.bind(kind)
.bind(kb_id.to_string())
.fetch_one(pool)
.await?)
}
pub async fn enqueue(pool: &PgPool, kind: &str, payload: serde_json::Value) -> AppResult<i64> {
enqueue_with_max_attempts(pool, kind, payload, 3).await
}
/// 入队并显式指定有限的重试预算。调用方不能把一个外部系统的任务变成无限重试。
pub async fn enqueue_with_max_attempts(
pool: &PgPool,
kind: &str,
payload: serde_json::Value,
max_attempts: i32,
) -> AppResult<i64> {
if !(1..=32).contains(&max_attempts) {
return Err(utopia_core::AppError::Validation(
"max_attempts must be between 1 and 32".into(),
));
}
let mut tx = pool.begin().await?;
let id = enqueue_with_max_attempts_tx(&mut tx, kind, payload, max_attempts).await?;
tx.commit().await?;
Ok(id)
}
/// 与 ledger 变更共用事务;不能先写 job、再写 entry,否则进程退出窗口会留下
/// 一个无法从 entry 找到的 hydration 任务。
pub async fn enqueue_with_max_attempts_tx(
tx: &mut Transaction<'_, Postgres>,
kind: &str,
payload: serde_json::Value,
max_attempts: i32,
) -> AppResult<i64> {
if !(1..=32).contains(&max_attempts) {
return Err(utopia_core::AppError::Validation(
"max_attempts must be between 1 and 32".into(),
));
}
let (id,): (i64,) = sqlx::query_as(
"INSERT INTO jobs (kind, payload, max_attempts) VALUES ($1, $2, $3) RETURNING id",
)
.bind(kind)
.bind(payload)
.bind(max_attempts)
.fetch_one(&mut **tx)
.await?;
notify_worker_tx(tx).await?;
Ok(id)
}
/// 同 [`enqueue_with_max_attempts_tx`],但 `after` 之后才到期。从插入这一刻
/// (`clock_timestamp()`)算起:`now()` 是事务开始的时刻,事务长了,提交时延迟已经用掉
pub async fn enqueue_with_max_attempts_after_tx(
tx: &mut Transaction<'_, Postgres>,
kind: &str,
payload: serde_json::Value,
max_attempts: i32,
after: Duration,
) -> AppResult<i64> {
if !(1..=32).contains(&max_attempts) {
return Err(utopia_core::AppError::Validation(
"max_attempts must be between 1 and 32".into(),
));
}
let (id,): (i64,) = sqlx::query_as(
"INSERT INTO jobs (kind, payload, max_attempts, run_at)
VALUES ($1, $2, $3, clock_timestamp() + make_interval(secs => $4))
RETURNING id",
)
.bind(kind)
.bind(payload)
.bind(max_attempts)
.bind(after.as_secs_f64())
.fetch_one(&mut **tx)
.await?;
notify_worker_tx(tx).await?;
Ok(id)
}
/// Wake idle workers only after the transaction containing the job is committed.
/// PostgreSQL delivers `NOTIFY` at commit, so a worker can never wake up before
/// the row it needs to claim is visible.
pub(crate) async fn notify_worker_tx(tx: &mut Transaction<'_, Postgres>) -> AppResult<()> {
sqlx::query("SELECT pg_notify($1, '')")
.bind(JOB_CHANNEL)
.execute(&mut **tx)
.await?;
Ok(())
}
/// 空闲轮询间隔。有通知时用不到它;它兜的是 `run_at` 在未来的重试、
/// 重连期间丢掉的通知,以及几个 worker 被同一条通知叫醒后没抢到的那些(#517)。
pub const IDLE_POLL: Duration = Duration::from_secs(2);
/// 一次空闲等待是怎么结束的。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Wake {
/// 有人入队,通知到了
Notified,
/// 等满一个轮询间隔,什么也没来
Polled,
}
/// 建立唤醒监听。`None` = 建不起来,worker 只靠轮询,行为与没有通知时一样。
///
/// 这条连接从池里取出后**一直握着**:LISTEN 是连接级状态,还回去就没了。
/// 池上限(`db.rs`)从此少一条给别人用;jobs 只起一个 worker,所以只少这一条。
pub async fn listen_for_jobs(pool: &PgPool) -> Option<PgListener> {
let mut listener = match PgListener::connect_with(pool).await {
Ok(listener) => listener,
Err(e) => {
tracing::error!(error = %e, "jobs worker 建立通知监听失败,退回轮询");
return None;
}
};
if let Err(e) = listener.listen(JOB_CHANNEL).await {
tracing::error!(error = %e, "jobs worker 监听通知失败,退回轮询");
return None;
}
Some(listener)
}
/// 空闲等待:通知先到就先醒,否则等满 `poll`。
///
/// 监听出错不能立刻再认领:库能认领而监听连接重连不上(池满就是一种)的话,
/// 循环会变成「认领一次、警告一行」的紧循环。出错就按轮询的节奏睡一觉,
/// 下一轮 `recv` 自己会重连。
pub async fn wait_for_work(listener: Option<&mut PgListener>, poll: Duration) -> Wake {
let Some(listener) = listener else {
tokio::time::sleep(poll).await;
return Wake::Polled;
};
match tokio::time::timeout(poll, listener.recv()).await {
Ok(Ok(_)) => Wake::Notified,
Ok(Err(e)) => {
tracing::warn!(error = %e, "jobs worker 通知监听失败,继续轮询");
tokio::time::sleep(poll).await;
Wake::Polled
}
Err(_) => Wake::Polled,
}
}
/// 重排失败任务的范围(#216)。三个条件都可空,空 = 不限。
///
/// **按库圈要解 payload**:任务表没有 kb 列,payload 只带 `document_id` /
/// `source_id` / `kb_id` 三种之一,各自解到库。没有库的系统任务只在不限库时才动
#[derive(Debug, Default, Clone, Copy)]
pub struct RequeueScope<'a> {
pub kb_id: Option<Uuid>,
pub kind: Option<&'a str>,
/// 只排这个时刻之后失败的——告警上的「再跑一遍」圈的正是那次故障窗口
pub failed_since: Option<DateTime<Utc>>,
}
/// 库范围的 SQL 谓词,`$N` 是库 id;`requeue_failed` 与 `failed_count` 共用
const KB_SCOPE: &str = "(
(j.payload ? 'kb_id' AND j.payload->>'kb_id' = $KB::text)
OR (j.payload ? 'document_id' AND EXISTS (
SELECT 1 FROM documents d
WHERE d.id::text = j.payload->>'document_id' AND d.kb_id = $KB))
OR (j.payload ? 'source_id' AND EXISTS (
SELECT 1 FROM sources s
WHERE s.id::text = j.payload->>'source_id' AND s.kb_id = $KB)))";
/// 把范围内的 failed 任务放回队列:`attempts` 归零、立即到期。
///
/// RSS hydration is excluded: its source admission policy owns retries.
/// 其他处理器都是幂等的(启动时回收孤儿就靠这一点);
/// 此前 `failed` 是终点,余额耗尽一批文档全失败,充值之后只能逐个点或整源重抽
pub async fn requeue_failed(pool: &PgPool, scope: RequeueScope<'_>) -> AppResult<u64> {
let mut tx = pool.begin().await?;
let sql = format!(
"UPDATE jobs j
SET status = 'queued', attempts = 0, run_at = now(), updated_at = now()
WHERE j.status = 'failed' AND j.kind <> 'hydrate_rss_entry'
AND ($1::text IS NULL OR j.kind = $1)
AND ($2::timestamptz IS NULL OR j.updated_at >= $2)
AND ($3::uuid IS NULL OR {})",
KB_SCOPE.replace("$KB", "$3")
);
let res = sqlx::query(&sql)
.bind(scope.kind)
.bind(scope.failed_since)
.bind(scope.kb_id)
.execute(&mut *tx)
.await?;
let count = res.rows_affected();
if count > 0 {
notify_worker_tx(&mut tx).await?;
}
tx.commit().await?;
Ok(count)
}
/// 范围内 failed 的条数——设置页那一行「N 个失败任务」
pub async fn failed_count(pool: &PgPool, kb_id: Option<Uuid>) -> AppResult<i64> {
let sql = format!(
"SELECT count(*) FROM jobs j
WHERE j.status = 'failed' AND ($1::uuid IS NULL OR {})",
KB_SCOPE.replace("$KB", "$1")
);
Ok(sqlx::query_scalar(&sql).bind(kb_id).fetch_one(pool).await?)
}
/// 一个任务此刻的样子,给「我刚排下去的那件事跑完了没」这个问题用(0051)。
#[derive(Debug, Clone, serde::Serialize, sqlx::FromRow)]
pub struct JobStatus {
pub id: i64,
pub kind: String,
pub status: String,
pub attempts: i32,
pub max_attempts: i32,
pub last_error: Option<String>,
pub run_at: chrono::DateTime<chrono::Utc>,
pub updated_at: chrono::DateTime<chrono::Utc>,
}
/// 按 id 读一个任务,**且只在它属于这个库时**。授权跟着库走:能看这个库的人能看
/// 它的任务;别的库的任务 id 猜对了也只得到 None,与看不见的文档一样答 404。
pub async fn status_in_kb(pool: &PgPool, kb_id: Uuid, id: i64) -> AppResult<Option<JobStatus>> {
let sql = format!(
"SELECT j.id, j.kind, j.status, j.attempts, j.max_attempts, j.last_error, j.run_at, j.updated_at
FROM jobs j WHERE j.id = $1 AND {}",
KB_SCOPE.replace("$KB", "$2")
);
Ok(sqlx::query_as(&sql)
.bind(id)
.bind(kb_id)
.fetch_optional(pool)
.await?)
}
/// 认领一个到期任务;没有则返回 None。
///
/// 不调对话模型的任务排在前面(`job_priority`,迁移 0116,#1300):同步、文档入库这些几秒
/// 的活,不该等在几百个更早的抽取后面
async fn claim_one(pool: &PgPool) -> AppResult<Option<Job>> {
let job = sqlx::query_as(
"UPDATE jobs SET status = 'running', locked_at = now(),
attempts = attempts + 1, updated_at = now()
WHERE id = (
SELECT id FROM jobs
WHERE status = 'queued' AND run_at <= now()
ORDER BY job_priority(kind), run_at
FOR UPDATE SKIP LOCKED
LIMIT 1
)
RETURNING id, kind, payload, attempts, max_attempts, locked_at",
)
.fetch_optional(pool)
.await?;
Ok(job)
}
/// 下一次重试等多久;`None` = 到此为止。
///
/// 两个理由到此为止:**次数用完了**,或者**处理器说了这次不会因为重试而变好**
/// (`utopia_core::Terminal`,见 issue #195)。后者以前不存在,于是余额耗尽的
/// 任务照样把三次退避走完——七分钟里余额不会自己长回来,那三次只是把同一句
/// 错误重说三遍,还把运维该看见的「失败」推迟了七分钟。
///
/// 限流相反,它正是这套退避的服务对象:配额会自己恢复。#176 把两类分开就是
/// 为了让它们各走各的路,而重试策略当时没跟上。
///
/// 抽成纯函数是为了能测——决定在这里,写库只是把决定落下去。
fn retry_delay(attempts: i32, max_attempts: i32, terminal: bool) -> Option<i64> {
if terminal || attempts >= max_attempts {
return None;
}
Some(30i64 * i64::from(attempts) * i64::from(attempts))
}
/// 处理器挂上 `Deferred { retry_in }` 时下一次重试的等待秒数。**与失败次数无关**——
/// 这是 `Deferred` 跟默认退避的关键区别:默认的 `30s × attempts²` 是「再试一次
/// 也许能好」的递增,而 `Deferred` 是「现在条件不满足,`retry_in` 之后再试」,
/// 与第几次没有关系。同一个等待条件挂回队列两次,两次都得到同一个 `run_at`
/// 偏移,没有 60s、120s 的递增(#526)。
///
/// 一条任务最多连续等多久(见 `mark_failed`)。一小时够一个大本体在远端嵌入模型上
/// 补齐;过了还没好,多半是补齐任务自己在失败,该让这条抽取按失败处理、被人看见
pub const DEFER_WINDOW_SECS: i64 = 60 * 60;
fn deferred_retry_secs(retry_in: std::time::Duration) -> i64 {
// 截断到秒:底层 `run_at` 是 timestamptz,亚秒精度存不住,而几十毫秒也不值得
// 一行浮点换算。向上取整——少等一秒比早跑一秒好,前者无害,后者会把还在跑的
// embedding job 撞回锁上。
let secs = retry_in.as_secs_f64().ceil();
if !secs.is_finite() || secs < 1.0 {
1
} else {
secs as i64
}
}
enum JobOutcome {
Done,
Failed {
error: String,
backoff_secs: Option<i64>,
deferred_secs: Option<i64>,
},
}
impl JobOutcome {
fn failed(job: &Job, err: &anyhow::Error) -> Self {
let terminal = utopia_core::is_terminal(err);
// Terminal 优先;结果写回重试沿用同一分类和完整错误原文。
let deferred_secs = if terminal {
None
} else {
utopia_core::is_deferred(err).map(deferred_retry_secs)
};
Self::Failed {
error: format!("{err:#}"),
backoff_secs: retry_delay(job.attempts, job.max_attempts, terminal),
deferred_secs,
}
}
}
/// true = 本次写入;false = 已结束或领取失效。两者都结束这次写回。
async fn persist_outcome(
pool: &PgPool,
job: &Job,
outcome: &JobOutcome,
completed_at: DateTime<Utc>,
) -> AppResult<bool> {
let JobOutcome::Failed {
error,
backoff_secs,
deferred_secs,
} = outcome
else {
// 成功要清掉前一次错误,避免任务显示 done 却仍带着失败原因。
let res = sqlx::query(
"UPDATE jobs SET status = 'done', last_error = NULL, updated_at = now()
WHERE id = $1 AND status = 'running' AND locked_at = $2",
)
.bind(job.id)
.bind(job.locked_at)
.execute(pool)
.await?;
return Ok(res.rows_affected() > 0);
};
if let Some(secs) = deferred_secs {
// Deferred 退回认领消耗的次数;等待窗口耗尽后才走普通失败预算。
// 等待窗口和下次运行时刻固定在 handler 完成时,不随写回重试滑动。
let res = sqlx::query(
"UPDATE jobs SET status = 'queued', last_error = $2,
attempts = GREATEST(0, attempts - 1),
run_at = $4::timestamptz + make_interval(secs => $5::float8),
payload = payload || jsonb_build_object('deferred_since',
COALESCE(payload->>'deferred_since', $4::timestamptz::text)),
updated_at = now()
WHERE id = $1 AND status = 'running' AND locked_at = $3
AND COALESCE((payload->>'deferred_since')::timestamptz, $4)
> $4 - make_interval(secs => $6::float8)",
)
.bind(job.id)
.bind(error)
.bind(job.locked_at)
.bind(completed_at)
.bind(*secs as f64)
.bind(DEFER_WINDOW_SECS as f64)
.execute(pool)
.await?;
if res.rows_affected() > 0 {
return Ok(true);
}
// 0 行也可能是旧领取;普通失败分支保留相同保护,不能覆盖新执行。
}
let status = if backoff_secs.is_some() {
"queued"
} else {
"failed"
};
let res = sqlx::query(
"UPDATE jobs SET status = $3, last_error = $4,
run_at = CASE WHEN $5::float8 IS NULL THEN run_at
ELSE $6::timestamptz + make_interval(secs => $5::float8) END,
updated_at = now()
WHERE id = $1 AND status = 'running' AND locked_at = $2",
)
.bind(job.id)
.bind(job.locked_at)
.bind(status)
.bind(error)
.bind(backoff_secs.map(|secs| secs as f64))
.bind(completed_at)
.execute(pool)
.await?;
Ok(res.rows_affected() > 0)
}
/// 集成测试的单次写回入口;worker 保存结果后由 persist_with_retry 写回。
#[doc(hidden)]
pub async fn mark_failed(pool: &PgPool, job: &Job, err: &anyhow::Error) -> AppResult<()> {
persist_outcome(pool, job, &JobOutcome::failed(job, err), Utc::now()).await?;
Ok(())
}
async fn persist_with_retry(
pool: &PgPool,
job: &Job,
outcome: &JobOutcome,
completed_at: DateTime<Utc>,
) {
let mut delay = Duration::from_secs(1);
loop {
if pool.is_closed() {
tracing::warn!(
job_id = job.id,
"数据库连接池已关闭,未确认的任务留待启动回收"
);
return;
}
let result = tokio::select! {
_ = pool.close_event() => {
tracing::warn!(job_id = job.id, "数据库连接池已关闭,未确认的任务留待启动回收");
return;
}
result = persist_outcome(pool, job, outcome, completed_at) => result,
};
match result {
Ok(updated) => {
if !updated {
tracing::debug!(
job_id = job.id,
claimed_at = %job.locked_at,
"任务已结束或领取已失效,停止写回旧结果"
);
}
return;
}
Err(e) => {
tracing::error!(job_id = job.id, error = %e, retry_in_secs = delay.as_secs(), "任务状态写回失败,将重试同一结果");
}
}
tokio::select! {
_ = pool.close_event() => {}
_ = tokio::time::sleep(delay) => {}
}
delay = (delay * 2).min(Duration::from_secs(30));
}
}
/// worker 调度循环:运行中任务数低于目标并发就继续认领(有活立即续派),
/// 空闲时等入队通知、2s 轮询兜底(#517);每个任务在独立 tokio task 中执行,长抽取不再阻塞同步。
/// `concurrency` 每轮热读——系统设置里改并发数即时生效。
/// 任务分发逻辑由调用方以 handler 注入(store 不依赖上层 crate)。
pub async fn run_worker<F, Fut>(pool: PgPool, concurrency: Arc<AtomicUsize>, handler: F)
where
F: Fn(Job) -> Fut + Clone + Send + Sync + 'static,
Fut: std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
{
let running = Arc::new(AtomicUsize::new(0));
// 孤儿回收:进程被杀时 running 任务无人收尸,文档会永远停在 extracting。
// 单进程部署下,启动时仍为 running 的必是孤儿——一律重排队(处理器均幂等)。
match sqlx::query(
"UPDATE jobs SET status = 'queued', locked_at = NULL, updated_at = now()
WHERE status = 'running'",
)
.execute(&pool)
.await
{
Ok(r) if r.rows_affected() > 0 => {
tracing::warn!(
count = r.rows_affected(),
"回收孤儿任务(上次进程退出时正在运行)"
);
}
Ok(_) => {}
Err(e) => tracing::error!(error = %e, "孤儿任务回收失败"),
}
tracing::info!(
concurrency = concurrency.load(Ordering::Relaxed),
"jobs worker 已启动"
);
let mut listener = listen_for_jobs(&pool).await;
loop {
if pool.is_closed() {
return;
}
let cap = concurrency.load(Ordering::Relaxed).max(1);
if running.load(Ordering::Relaxed) >= cap {
tokio::time::sleep(Duration::from_millis(200)).await;
continue;
}
match claim_one(&pool).await {
Ok(Some(job)) => {
running.fetch_add(1, Ordering::Relaxed);
let pool = pool.clone();
let handler = handler.clone();
let running = running.clone();
tokio::spawn(async move {
// **处理器 panic 也要收尸。** 直接 `handler(job).await` 的话,
// panic 会把这个 spawn 出来的 future 一起掀掉:`mark_failed`
// 不会跑(任务行永远停在 running,无错误无重试),`running`
// 也不会减(每 panic 一次就永久少一个并发名额,攒够 cap 之后
// 整个队列不再认领任何任务)。套一层 spawn,panic 变成
// JoinError 拿回来,两件事就都还在。
let inner = {
let (handler, job) = (handler.clone(), job.clone());
tokio::spawn(async move { handler(job).await })
};
let result = match inner.await {
Ok(r) => r,
Err(join) => {
Err(anyhow::anyhow!("任务处理器 panic(详情见 stderr):{join}"))
}
};
let completed_at = Utc::now();
let outcome = match result {
Ok(()) => JobOutcome::Done,
Err(e) => {
tracing::warn!(job_id = job.id, kind = %job.kind, error = %e, "任务执行失败");
JobOutcome::failed(&job, &e)
}
};
persist_with_retry(&pool, &job, &outcome, completed_at).await;
running.fetch_sub(1, Ordering::Relaxed);
});
}
Ok(None) => {
wait_for_work(listener.as_mut(), IDLE_POLL).await;
}
Err(e) => {
tracing::error!(error = %e, "任务认领失败,5s 后重试");
tokio::time::sleep(Duration::from_secs(5)).await;
}
}
}
}
#[cfg(test)]
#[path = "jobs_writeback_tests.rs"]
mod writeback_tests;
#[cfg(test)]
#[path = "jobs_claim_tests.rs"]
mod claim_tests;
#[cfg(test)]
mod tests {
use super::{deferred_retry_secs, retry_delay};
use std::time::Duration;
/// 退避照旧:30s、120s、270s,第三次之后放弃。
#[test]
fn an_ordinary_failure_backs_off_and_then_gives_up() {
assert_eq!(retry_delay(1, 3, false), Some(30));
assert_eq!(retry_delay(2, 3, false), Some(120));
assert_eq!(retry_delay(3, 3, false), None);
}
/// 标成没救的**第一次就到此为止**——那三次退避加起来是七分钟,
/// 而余额不会在七分钟里自己长回来(#195)。
#[test]
fn a_terminal_failure_does_not_spend_the_budget() {
assert_eq!(retry_delay(1, 3, true), None);
}
/// `Deferred` 不随失败次数递增——同一等待条件两次排队得到的 `run_at`
/// 偏移相同,不会出现 60s、120s 的递增(#526)。
#[test]
fn deferred_retry_is_independent_of_attempts() {
assert_eq!(
deferred_retry_secs(Duration::from_secs(30)),
deferred_retry_secs(Duration::from_secs(30))
);
// 截断到秒,向上取整:29.5s → 30s
assert_eq!(deferred_retry_secs(Duration::from_millis(29_500)), 30);
// 0/负数/NaN 都给 1s 下界——至少等一秒,比立刻重试的轮询间隔还短就是浪费
assert_eq!(deferred_retry_secs(Duration::ZERO), 1);
assert_eq!(deferred_retry_secs(Duration::from_nanos(500)), 1);
}
}