Skip to content

World's first open-source enterprise world model.

Wake idle workers when jobs are queued (#521)

RKS131aefa65b79
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index 9e55f39b..7d0500c1 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -1134,6 +1134,7 @@ pub async fn queue_extraction_one(pool: &PgPool, document_id: Uuid) -> AppResult
.bind(document_id)
.fetch_one(&mut *tx)
.await?;
+ crate::jobs::notify_worker_tx(&mut tx).await?;
tx.commit().await?;
Ok(job_id)
}
@@ -1354,6 +1355,7 @@ pub async fn queue_extraction(
.bind(&ids)
.execute(&mut *tx)
.await?;
+ crate::jobs::notify_worker_tx(&mut tx).await?;
tx.commit().await?;
Ok(ids)
}
diff --git a/crates/utopia-store/src/jobs.rs b/crates/utopia-store/src/jobs.rs
index 6954bf23..7f3e6779 100644
--- a/crates/utopia-store/src/jobs.rs
+++ b/crates/utopia-store/src/jobs.rs
@@ -5,13 +5,15 @@
//! 目标数经 AtomicUsize 热读——系统设置里改并发即时生效,无需重启。
use chrono::{DateTime, Utc};
-use sqlx::{PgPool, Postgres, Transaction};
+use sqlx::{postgres::PgListener, PgPool, Postgres, Transaction};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use utopia_core::AppResult;
use uuid::Uuid;
+pub const JOB_CHANNEL: &str = "utopia_jobs";
+
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct Job {
pub id: i64,
@@ -29,6 +31,7 @@ pub async fn enqueue_unless_queued(
kind: &str,
payload: serde_json::Value,
) -> AppResult<Option<i64>> {
+ let mut tx = pool.begin().await?;
let row: Option<(i64,)> = sqlx::query_as(
"INSERT INTO jobs (kind, payload)
SELECT $1, $2
@@ -37,8 +40,12 @@ pub async fn enqueue_unless_queued(
)
.bind(kind)
.bind(payload)
- .fetch_optional(pool)
+ .fetch_optional(&mut *tx)
.await?;
+ if row.is_some() {
+ notify_worker_tx(&mut tx).await?;
+ }
+ tx.commit().await?;
Ok(row.map(|(id,)| id))
}
@@ -58,14 +65,9 @@ pub async fn enqueue_with_max_attempts(
"max_attempts must be between 1 and 32".into(),
));
}
- let (id,): (i64,) = sqlx::query_as(
- "INSERT INTO jobs (kind, payload, max_attempts) VALUES ($1, $2, $3) RETURNING id",
- )
- .bind(kind)
- .bind(payload)
- .bind(max_attempts)
- .fetch_one(pool)
- .await?;
+ let mut tx = pool.begin().await?;
+ let id = enqueue_with_max_attempts_tx(&mut tx, kind, payload, max_attempts).await?;
+ tx.commit().await?;
Ok(id)
}
@@ -90,9 +92,74 @@ pub async fn enqueue_with_max_attempts_tx(
.bind(max_attempts)
.fetch_one(&mut **tx)
.await?;
+ notify_worker_tx(tx).await?;
Ok(id)
}
+/// Wake idle workers only after the transaction containing the job is committed.
+/// PostgreSQL delivers `NOTIFY` at commit, so a worker can never wake up before
+/// the row it needs to claim is visible.
+pub(crate) async fn notify_worker_tx(tx: &mut Transaction<'_, Postgres>) -> AppResult<()> {
+ sqlx::query("SELECT pg_notify($1, '')")
+ .bind(JOB_CHANNEL)
+ .execute(&mut **tx)
+ .await?;
+ Ok(())
+}
+
+/// 空闲轮询间隔。有通知时用不到它;它兜的是 `run_at` 在未来的重试、
+/// 重连期间丢掉的通知,以及几个 worker 被同一条通知叫醒后没抢到的那些(#517)。
+pub const IDLE_POLL: Duration = Duration::from_secs(2);
+
+/// 一次空闲等待是怎么结束的。
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Wake {
+ /// 有人入队,通知到了
+ Notified,
+ /// 等满一个轮询间隔,什么也没来
+ Polled,
+}
+
+/// 建立唤醒监听。`None` = 建不起来,worker 只靠轮询,行为与没有通知时一样。
+///
+/// 这条连接从池里取出后**一直握着**:LISTEN 是连接级状态,还回去就没了。
+/// 池上限(`db.rs`)从此少一条给别人用;jobs 只起一个 worker,所以只少这一条。
+pub async fn listen_for_jobs(pool: &PgPool) -> Option<PgListener> {
+ let mut listener = match PgListener::connect_with(pool).await {
+ Ok(listener) => listener,
+ Err(e) => {
+ tracing::error!(error = %e, "jobs worker 建立通知监听失败,退回轮询");
+ return None;
+ }
+ };
+ if let Err(e) = listener.listen(JOB_CHANNEL).await {
+ tracing::error!(error = %e, "jobs worker 监听通知失败,退回轮询");
+ return None;
+ }
+ Some(listener)
+}
+
+/// 空闲等待:通知先到就先醒,否则等满 `poll`。
+///
+/// 监听出错不能立刻再认领:库能认领而监听连接重连不上(池满就是一种)的话,
+/// 循环会变成「认领一次、警告一行」的紧循环。出错就按轮询的节奏睡一觉,
+/// 下一轮 `recv` 自己会重连。
+pub async fn wait_for_work(listener: Option<&mut PgListener>, poll: Duration) -> Wake {
+ let Some(listener) = listener else {
+ tokio::time::sleep(poll).await;
+ return Wake::Polled;
+ };
+ match tokio::time::timeout(poll, listener.recv()).await {
+ Ok(Ok(_)) => Wake::Notified,
+ Ok(Err(e)) => {
+ tracing::warn!(error = %e, "jobs worker 通知监听失败,继续轮询");
+ tokio::time::sleep(poll).await;
+ Wake::Polled
+ }
+ Err(_) => Wake::Polled,
+ }
+}
+
/// 重排失败任务的范围(#216)。三个条件都可空,空 = 不限。
///
/// **按库圈要解 payload**:任务表没有 kb 列,payload 只带 `document_id` /
@@ -121,6 +188,7 @@ const KB_SCOPE: &str = "(
/// 其他处理器都是幂等的(启动时回收孤儿就靠这一点);
/// 此前 `failed` 是终点,余额耗尽一批文档全失败,充值之后只能逐个点或整源重抽
pub async fn requeue_failed(pool: &PgPool, scope: RequeueScope<'_>) -> AppResult<u64> {
+ let mut tx = pool.begin().await?;
let sql = format!(
"UPDATE jobs j
SET status = 'queued', attempts = 0, run_at = now(), updated_at = now()
@@ -134,9 +202,14 @@ pub async fn requeue_failed(pool: &PgPool, scope: RequeueScope<'_>) -> AppResult
.bind(scope.kind)
.bind(scope.failed_since)
.bind(scope.kb_id)
- .execute(pool)
+ .execute(&mut *tx)
.await?;
- Ok(res.rows_affected())
+ let count = res.rows_affected();
+ if count > 0 {
+ notify_worker_tx(&mut tx).await?;
+ }
+ tx.commit().await?;
+ Ok(count)
}
/// 范围内 failed 的条数——设置页那一行「N 个失败任务」
@@ -231,7 +304,7 @@ async fn mark_failed(pool: &PgPool, job: &Job, err: &anyhow::Error) -> AppResult
}
/// worker 调度循环:运行中任务数低于目标并发就继续认领(有活立即续派),
-/// 空闲时 2s 轮询;每个任务在独立 tokio task 中执行,长抽取不再阻塞同步。
+/// 空闲时等入队通知、2s 轮询兜底(#517);每个任务在独立 tokio task 中执行,长抽取不再阻塞同步。
/// `concurrency` 每轮热读——系统设置里改并发数即时生效。
/// 任务分发逻辑由调用方以 handler 注入(store 不依赖上层 crate)。
pub async fn run_worker<F, Fut>(pool: PgPool, concurrency: Arc<AtomicUsize>, handler: F)
@@ -262,6 +335,7 @@ where
concurrency = concurrency.load(Ordering::Relaxed),
"jobs worker 已启动"
);
+ let mut listener = listen_for_jobs(&pool).await;
loop {
let cap = concurrency.load(Ordering::Relaxed).max(1);
if running.load(Ordering::Relaxed) >= cap {
@@ -304,7 +378,9 @@ where
running.fetch_sub(1, Ordering::Relaxed);
});
}
- Ok(None) => tokio::time::sleep(Duration::from_secs(2)).await,
+ Ok(None) => {
+ wait_for_work(listener.as_mut(), IDLE_POLL).await;
+ }
Err(e) => {
tracing::error!(error = %e, "任务认领失败,5s 后重试");
tokio::time::sleep(Duration::from_secs(5)).await;
diff --git a/crates/utopia-store/src/rss_full_content.rs b/crates/utopia-store/src/rss_full_content.rs
index b56171de..be1bd169 100644
--- a/crates/utopia-store/src/rss_full_content.rs
+++ b/crates/utopia-store/src/rss_full_content.rs
@@ -74,7 +74,7 @@ pub async fn requeue_failed(pool: &PgPool, scope: crate::jobs::RequeueScope<'_>)
ORDER BY j.id LIMIT $4 FOR UPDATE OF e, j",
).bind(source_id).bind(generation).bind(scope.failed_since).bind(capacity)
.fetch_all(&mut *tx).await?;
- count += sqlx::query(
+ let requeued = sqlx::query(
"UPDATE jobs SET status = 'queued', attempts = 0, last_error = NULL,
locked_at = NULL, run_at = now(), updated_at = now()
WHERE id = ANY($1) AND status = 'failed'",
@@ -83,6 +83,10 @@ pub async fn requeue_failed(pool: &PgPool, scope: crate::jobs::RequeueScope<'_>)
.execute(&mut *tx)
.await?
.rows_affected();
+ if requeued > 0 {
+ crate::jobs::notify_worker_tx(&mut tx).await?;
+ }
+ count += requeued;
tx.commit().await?;
}
Ok(count)
diff --git a/crates/utopia-store/tests/a_queued_job_wakes_an_idle_worker.rs b/crates/utopia-store/tests/a_queued_job_wakes_an_idle_worker.rs
new file mode 100644
index 00000000..bd718649
--- /dev/null
+++ b/crates/utopia-store/tests/a_queued_job_wakes_an_idle_worker.rs
@@ -0,0 +1,120 @@
+//! #517:入队的任务不等轮询,空闲的 worker 立刻醒。
+//!
+//! worker 空闲时不再干睡两秒,而是等 `utopia_jobs` 频道的通知,两秒轮询只做兜底
+//! (`run_at` 在未来的重试、重连期间丢掉的通知、几个 worker 被一条通知叫醒后没抢到的)。
+//! 这里守的是 worker 真正用的那个等待函数 `jobs::wait_for_work`,不起完整 worker:
+//! 完整 worker 会认领库里任何排队任务,共用一个测试库时会替别的测试把活干掉。
+//!
+//! 三件事:
+//! 1. **入队叫得醒。** 通知在轮询间隔之内到达,等待以 `Notified` 结束。
+//! 2. **通知只在提交之后。** 插入还在未提交的事务里时,等待只能靠轮询结束;提交后才醒。
+//! 这是 #517 点名的顺序隐患:先醒后见行的话,醒来一无所获,又回到轮询。
+//! 3. **没有通知也会醒。** 轮询那一臂还在,等满间隔以 `Polled` 结束。
+//!
+//! 三个测试共用一个频道,串行跑:并行的话彼此的通知会互相叫醒。
+//! 要对着一个没有别的进程往里写任务的库跑。
+
+use sqlx::PgPool;
+use std::time::{Duration, Instant};
+use tokio::sync::Mutex;
+use utopia_store::jobs::{self, Wake};
+
+/// 短轮询:测试只关心「通知比轮询快」,不必等真正的两秒。
+const POLL: Duration = Duration::from_millis(300);
+static SERIAL: Mutex<()> = Mutex::const_new(());
+
+async fn connect() -> anyhow::Result<Option<PgPool>> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(None);
+ };
+ Ok(Some(PgPool::connect(&url).await?))
+}
+
+async fn listener(pool: &PgPool) -> anyhow::Result<sqlx::postgres::PgListener> {
+ jobs::listen_for_jobs(pool)
+ .await
+ .ok_or_else(|| anyhow::anyhow!("the wake-up listener could not be built"))
+}
+
+async fn remove(pool: &PgPool, id: i64) -> anyhow::Result<()> {
+ sqlx::query("DELETE FROM jobs WHERE id = $1")
+ .bind(id)
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn a_queued_job_wakes_an_idle_worker() -> anyhow::Result<()> {
+ let _serial = SERIAL.lock().await;
+ let Some(pool) = connect().await? else {
+ return Ok(());
+ };
+ let mut listener = listener(&pool).await?;
+
+ let started = Instant::now();
+ let id = jobs::enqueue(&pool, "wake_test", serde_json::json!({})).await?;
+ let wake = jobs::wait_for_work(Some(&mut listener), POLL).await;
+ let waited = started.elapsed();
+ remove(&pool, id).await?;
+
+ assert_eq!(
+ wake,
+ Wake::Notified,
+ "an enqueue must end the idle wait with a notification"
+ );
+ assert!(
+ waited < POLL,
+ "woke after {waited:?}, which is not faster than the {POLL:?} poll"
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn a_notification_never_arrives_before_the_row() -> anyhow::Result<()> {
+ let _serial = SERIAL.lock().await;
+ let Some(pool) = connect().await? else {
+ return Ok(());
+ };
+ let mut listener = listener(&pool).await?;
+
+ let mut tx = pool.begin().await?;
+ let id =
+ jobs::enqueue_with_max_attempts_tx(&mut tx, "wake_test", serde_json::json!({}), 3).await?;
+ let before_commit = jobs::wait_for_work(Some(&mut listener), POLL).await;
+ tx.commit().await?;
+ let after_commit = jobs::wait_for_work(Some(&mut listener), POLL).await;
+ remove(&pool, id).await?;
+
+ assert_eq!(
+ before_commit,
+ Wake::Polled,
+ "nothing may wake a worker while the row is still invisible"
+ );
+ assert_eq!(
+ after_commit,
+ Wake::Notified,
+ "the commit is what wakes the worker"
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn the_poll_still_fires_without_a_notification() -> anyhow::Result<()> {
+ let _serial = SERIAL.lock().await;
+ let Some(pool) = connect().await? else {
+ return Ok(());
+ };
+ let mut listener = listener(&pool).await?;
+
+ let started = Instant::now();
+ let wake = jobs::wait_for_work(Some(&mut listener), POLL).await;
+ let waited = started.elapsed();
+
+ assert_eq!(wake, Wake::Polled);
+ assert!(
+ waited >= POLL,
+ "the poll arm returned after {waited:?}, before its {POLL:?}"
+ );
+ Ok(())
+}