World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Wake idle workers when jobs are queued (#521)
RKS131aefa65b79
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rsindex 9e55f39b..7d0500c1 100644--- a/crates/utopia-store/src/documents.rs+++ b/crates/utopia-store/src/documents.rs@@ -1134,6 +1134,7 @@ pub async fn queue_extraction_one(pool: &PgPool, document_id: Uuid) -> AppResult.bind(document_id).fetch_one(&mut *tx).await?;+ crate::jobs::notify_worker_tx(&mut tx).await?;tx.commit().await?;Ok(job_id)}@@ -1354,6 +1355,7 @@ pub async fn queue_extraction(.bind(&ids).execute(&mut *tx).await?;+ crate::jobs::notify_worker_tx(&mut tx).await?;tx.commit().await?;Ok(ids)}diff --git a/crates/utopia-store/src/jobs.rs b/crates/utopia-store/src/jobs.rsindex 6954bf23..7f3e6779 100644--- a/crates/utopia-store/src/jobs.rs+++ b/crates/utopia-store/src/jobs.rs@@ -5,13 +5,15 @@//! 目标数经 AtomicUsize 热读——系统设置里改并发即时生效,无需重启。use chrono::{DateTime, Utc};-use sqlx::{PgPool, Postgres, Transaction};+use sqlx::{postgres::PgListener, PgPool, Postgres, Transaction};use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::Arc;use std::time::Duration;use utopia_core::AppResult;use uuid::Uuid;+pub const JOB_CHANNEL: &str = "utopia_jobs";+#[derive(Debug, Clone, sqlx::FromRow)]pub struct Job {pub id: i64,@@ -29,6 +31,7 @@ pub async fn enqueue_unless_queued(kind: &str,payload: serde_json::Value,) -> AppResult<Option<i64>> {+ let mut tx = pool.begin().await?;let row: Option<(i64,)> = sqlx::query_as("INSERT INTO jobs (kind, payload)SELECT $1, $2@@ -37,8 +40,12 @@ pub async fn enqueue_unless_queued().bind(kind).bind(payload)- .fetch_optional(pool)+ .fetch_optional(&mut *tx).await?;+ if row.is_some() {+ notify_worker_tx(&mut tx).await?;+ }+ tx.commit().await?;Ok(row.map(|(id,)| id))}@@ -58,14 +65,9 @@ pub async fn enqueue_with_max_attempts("max_attempts must be between 1 and 32".into(),));}- let (id,): (i64,) = sqlx::query_as(- "INSERT INTO jobs (kind, payload, max_attempts) VALUES ($1, $2, $3) RETURNING id",- )- .bind(kind)- .bind(payload)- .bind(max_attempts)- .fetch_one(pool)- .await?;+ let mut tx = pool.begin().await?;+ let id = enqueue_with_max_attempts_tx(&mut tx, kind, payload, max_attempts).await?;+ tx.commit().await?;Ok(id)}@@ -90,9 +92,74 @@ pub async fn enqueue_with_max_attempts_tx(.bind(max_attempts).fetch_one(&mut **tx).await?;+ notify_worker_tx(tx).await?;Ok(id)}+/// Wake idle workers only after the transaction containing the job is committed.+/// PostgreSQL delivers `NOTIFY` at commit, so a worker can never wake up before+/// the row it needs to claim is visible.+pub(crate) async fn notify_worker_tx(tx: &mut Transaction<'_, Postgres>) -> AppResult<()> {+ sqlx::query("SELECT pg_notify($1, '')")+ .bind(JOB_CHANNEL)+ .execute(&mut **tx)+ .await?;+ Ok(())+}++/// 空闲轮询间隔。有通知时用不到它;它兜的是 `run_at` 在未来的重试、+/// 重连期间丢掉的通知,以及几个 worker 被同一条通知叫醒后没抢到的那些(#517)。+pub const IDLE_POLL: Duration = Duration::from_secs(2);++/// 一次空闲等待是怎么结束的。+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Wake {+ /// 有人入队,通知到了+ Notified,+ /// 等满一个轮询间隔,什么也没来+ Polled,+}++/// 建立唤醒监听。`None` = 建不起来,worker 只靠轮询,行为与没有通知时一样。+///+/// 这条连接从池里取出后**一直握着**:LISTEN 是连接级状态,还回去就没了。+/// 池上限(`db.rs`)从此少一条给别人用;jobs 只起一个 worker,所以只少这一条。+pub async fn listen_for_jobs(pool: &PgPool) -> Option<PgListener> {+ let mut listener = match PgListener::connect_with(pool).await {+ Ok(listener) => listener,+ Err(e) => {+ tracing::error!(error = %e, "jobs worker 建立通知监听失败,退回轮询");+ return None;+ }+ };+ if let Err(e) = listener.listen(JOB_CHANNEL).await {+ tracing::error!(error = %e, "jobs worker 监听通知失败,退回轮询");+ return None;+ }+ Some(listener)+}++/// 空闲等待:通知先到就先醒,否则等满 `poll`。+///+/// 监听出错不能立刻再认领:库能认领而监听连接重连不上(池满就是一种)的话,+/// 循环会变成「认领一次、警告一行」的紧循环。出错就按轮询的节奏睡一觉,+/// 下一轮 `recv` 自己会重连。+pub async fn wait_for_work(listener: Option<&mut PgListener>, poll: Duration) -> Wake {+ let Some(listener) = listener else {+ tokio::time::sleep(poll).await;+ return Wake::Polled;+ };+ match tokio::time::timeout(poll, listener.recv()).await {+ Ok(Ok(_)) => Wake::Notified,+ Ok(Err(e)) => {+ tracing::warn!(error = %e, "jobs worker 通知监听失败,继续轮询");+ tokio::time::sleep(poll).await;+ Wake::Polled+ }+ Err(_) => Wake::Polled,+ }+}+/// 重排失败任务的范围(#216)。三个条件都可空,空 = 不限。////// **按库圈要解 payload**:任务表没有 kb 列,payload 只带 `document_id` /@@ -121,6 +188,7 @@ const KB_SCOPE: &str = "(/// 其他处理器都是幂等的(启动时回收孤儿就靠这一点);/// 此前 `failed` 是终点,余额耗尽一批文档全失败,充值之后只能逐个点或整源重抽pub async fn requeue_failed(pool: &PgPool, scope: RequeueScope<'_>) -> AppResult<u64> {+ let mut tx = pool.begin().await?;let sql = format!("UPDATE jobs jSET status = 'queued', attempts = 0, run_at = now(), updated_at = now()@@ -134,9 +202,14 @@ pub async fn requeue_failed(pool: &PgPool, scope: RequeueScope<'_>) -> AppResult.bind(scope.kind).bind(scope.failed_since).bind(scope.kb_id)- .execute(pool)+ .execute(&mut *tx).await?;- Ok(res.rows_affected())+ let count = res.rows_affected();+ if count > 0 {+ notify_worker_tx(&mut tx).await?;+ }+ tx.commit().await?;+ Ok(count)}/// 范围内 failed 的条数——设置页那一行「N 个失败任务」@@ -231,7 +304,7 @@ async fn mark_failed(pool: &PgPool, job: &Job, err: &anyhow::Error) -> AppResult}/// worker 调度循环:运行中任务数低于目标并发就继续认领(有活立即续派),-/// 空闲时 2s 轮询;每个任务在独立 tokio task 中执行,长抽取不再阻塞同步。+/// 空闲时等入队通知、2s 轮询兜底(#517);每个任务在独立 tokio task 中执行,长抽取不再阻塞同步。/// `concurrency` 每轮热读——系统设置里改并发数即时生效。/// 任务分发逻辑由调用方以 handler 注入(store 不依赖上层 crate)。pub async fn run_worker<F, Fut>(pool: PgPool, concurrency: Arc<AtomicUsize>, handler: F)@@ -262,6 +335,7 @@ whereconcurrency = concurrency.load(Ordering::Relaxed),"jobs worker 已启动");+ let mut listener = listen_for_jobs(&pool).await;loop {let cap = concurrency.load(Ordering::Relaxed).max(1);if running.load(Ordering::Relaxed) >= cap {@@ -304,7 +378,9 @@ whererunning.fetch_sub(1, Ordering::Relaxed);});}- Ok(None) => tokio::time::sleep(Duration::from_secs(2)).await,+ Ok(None) => {+ wait_for_work(listener.as_mut(), IDLE_POLL).await;+ }Err(e) => {tracing::error!(error = %e, "任务认领失败,5s 后重试");tokio::time::sleep(Duration::from_secs(5)).await;diff --git a/crates/utopia-store/src/rss_full_content.rs b/crates/utopia-store/src/rss_full_content.rsindex b56171de..be1bd169 100644--- a/crates/utopia-store/src/rss_full_content.rs+++ b/crates/utopia-store/src/rss_full_content.rs@@ -74,7 +74,7 @@ pub async fn requeue_failed(pool: &PgPool, scope: crate::jobs::RequeueScope<'_>)ORDER BY j.id LIMIT $4 FOR UPDATE OF e, j",).bind(source_id).bind(generation).bind(scope.failed_since).bind(capacity).fetch_all(&mut *tx).await?;- count += sqlx::query(+ let requeued = sqlx::query("UPDATE jobs SET status = 'queued', attempts = 0, last_error = NULL,locked_at = NULL, run_at = now(), updated_at = now()WHERE id = ANY($1) AND status = 'failed'",@@ -83,6 +83,10 @@ pub async fn requeue_failed(pool: &PgPool, scope: crate::jobs::RequeueScope<'_>).execute(&mut *tx).await?.rows_affected();+ if requeued > 0 {+ crate::jobs::notify_worker_tx(&mut tx).await?;+ }+ count += requeued;tx.commit().await?;}Ok(count)diff --git a/crates/utopia-store/tests/a_queued_job_wakes_an_idle_worker.rs b/crates/utopia-store/tests/a_queued_job_wakes_an_idle_worker.rsnew file mode 100644index 00000000..bd718649--- /dev/null+++ b/crates/utopia-store/tests/a_queued_job_wakes_an_idle_worker.rs@@ -0,0 +1,120 @@+//! #517:入队的任务不等轮询,空闲的 worker 立刻醒。+//!+//! worker 空闲时不再干睡两秒,而是等 `utopia_jobs` 频道的通知,两秒轮询只做兜底+//! (`run_at` 在未来的重试、重连期间丢掉的通知、几个 worker 被一条通知叫醒后没抢到的)。+//! 这里守的是 worker 真正用的那个等待函数 `jobs::wait_for_work`,不起完整 worker:+//! 完整 worker 会认领库里任何排队任务,共用一个测试库时会替别的测试把活干掉。+//!+//! 三件事:+//! 1. **入队叫得醒。** 通知在轮询间隔之内到达,等待以 `Notified` 结束。+//! 2. **通知只在提交之后。** 插入还在未提交的事务里时,等待只能靠轮询结束;提交后才醒。+//! 这是 #517 点名的顺序隐患:先醒后见行的话,醒来一无所获,又回到轮询。+//! 3. **没有通知也会醒。** 轮询那一臂还在,等满间隔以 `Polled` 结束。+//!+//! 三个测试共用一个频道,串行跑:并行的话彼此的通知会互相叫醒。+//! 要对着一个没有别的进程往里写任务的库跑。++use sqlx::PgPool;+use std::time::{Duration, Instant};+use tokio::sync::Mutex;+use utopia_store::jobs::{self, Wake};++/// 短轮询:测试只关心「通知比轮询快」,不必等真正的两秒。+const POLL: Duration = Duration::from_millis(300);+static SERIAL: Mutex<()> = Mutex::const_new(());++async fn connect() -> anyhow::Result<Option<PgPool>> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(None);+ };+ Ok(Some(PgPool::connect(&url).await?))+}++async fn listener(pool: &PgPool) -> anyhow::Result<sqlx::postgres::PgListener> {+ jobs::listen_for_jobs(pool)+ .await+ .ok_or_else(|| anyhow::anyhow!("the wake-up listener could not be built"))+}++async fn remove(pool: &PgPool, id: i64) -> anyhow::Result<()> {+ sqlx::query("DELETE FROM jobs WHERE id = $1")+ .bind(id)+ .execute(pool)+ .await?;+ Ok(())+}++#[tokio::test]+async fn a_queued_job_wakes_an_idle_worker() -> anyhow::Result<()> {+ let _serial = SERIAL.lock().await;+ let Some(pool) = connect().await? else {+ return Ok(());+ };+ let mut listener = listener(&pool).await?;++ let started = Instant::now();+ let id = jobs::enqueue(&pool, "wake_test", serde_json::json!({})).await?;+ let wake = jobs::wait_for_work(Some(&mut listener), POLL).await;+ let waited = started.elapsed();+ remove(&pool, id).await?;++ assert_eq!(+ wake,+ Wake::Notified,+ "an enqueue must end the idle wait with a notification"+ );+ assert!(+ waited < POLL,+ "woke after {waited:?}, which is not faster than the {POLL:?} poll"+ );+ Ok(())+}++#[tokio::test]+async fn a_notification_never_arrives_before_the_row() -> anyhow::Result<()> {+ let _serial = SERIAL.lock().await;+ let Some(pool) = connect().await? else {+ return Ok(());+ };+ let mut listener = listener(&pool).await?;++ let mut tx = pool.begin().await?;+ let id =+ jobs::enqueue_with_max_attempts_tx(&mut tx, "wake_test", serde_json::json!({}), 3).await?;+ let before_commit = jobs::wait_for_work(Some(&mut listener), POLL).await;+ tx.commit().await?;+ let after_commit = jobs::wait_for_work(Some(&mut listener), POLL).await;+ remove(&pool, id).await?;++ assert_eq!(+ before_commit,+ Wake::Polled,+ "nothing may wake a worker while the row is still invisible"+ );+ assert_eq!(+ after_commit,+ Wake::Notified,+ "the commit is what wakes the worker"+ );+ Ok(())+}++#[tokio::test]+async fn the_poll_still_fires_without_a_notification() -> anyhow::Result<()> {+ let _serial = SERIAL.lock().await;+ let Some(pool) = connect().await? else {+ return Ok(());+ };+ let mut listener = listener(&pool).await?;++ let started = Instant::now();+ let wake = jobs::wait_for_work(Some(&mut listener), POLL).await;+ let waited = started.elapsed();++ assert_eq!(wake, Wake::Polled);+ assert!(+ waited >= POLL,+ "the poll arm returned after {waited:?}, before its {POLL:?}"+ );+ Ok(())+}