Skip to content

World's first open-source enterprise world model.

Bound lock waits for human phrase alignment review decisions (#798, #800) (#864)

Royceecc85e70b000
diff --git a/crates/utopia-server/src/api/review_routes.rs b/crates/utopia-server/src/api/review_routes.rs
index f324f508..9a62f3bb 100644
--- a/crates/utopia-server/src/api/review_routes.rs
+++ b/crates/utopia-server/src/api/review_routes.rs
@@ -1277,7 +1277,7 @@ pub async fn decide_alignment_phrase(
},
)
.await?;
- let typed = utopia_store::materialize::materialize(&state.pool, kb_id).await?;
+ let typed = utopia_store::materialize::materialize_human(&state.pool, kb_id).await?;
let _ = utopia_store::audit::record(
&state.pool,
Some(kb_id),
diff --git a/crates/utopia-store/src/materialize.rs b/crates/utopia-store/src/materialize.rs
index 8079ab36..f8044a25 100644
--- a/crates/utopia-store/src/materialize.rs
+++ b/crates/utopia-store/src/materialize.rs
@@ -14,7 +14,7 @@
//! 没有」的(陈述, 绑定)对补上——有同断言的行就并进去,没有才新建。没有模型调用。
use sqlx::PgPool;
-use utopia_core::AppResult;
+use utopia_core::{AppError, AppResult};
use uuid::Uuid;
use crate::graph::{insert_fact_on, FactObject, Validity};
@@ -57,6 +57,9 @@ struct Due {
}
/// 对一个库重算一遍。
+///
+/// Worker 走这一条:等多久都行,因为它没人在屏幕前面等。人的判定不走这条:
+/// 见 `materialize_human`,那是另一条带预算的入口
pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {
// A worker and a human review can recompute the same base concurrently.
// Serialize before reading due statements, and use this connection for the
@@ -71,6 +74,57 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {
Ok(outcome)
}
+/// 人的入口(#798 跟进、#800 复盘):与 `materialize` 同样的活,外加一个
+/// 事务级的 `lock_timeout` 预算,超时映射到表「语言可读的重试提示」。
+///
+/// 设计要点(沿用 #828 的同一条理由):
+/// - `lock_timeout` 是**等锁的预算**,不是请求总预算,也不是查询执行预算
+/// - worker 不动也不动:背景不会因为 5xx 醒不了,没人在屏幕前面等
+/// - advisory 锁本身在事务提交时释放(pg_advisory_xact_lock),所以这里的
+/// 等待只在等 worker 的 1 次 materialize 完成;不会等下一个做多次锁
+/// - `55P03` 是 `lock_not_available`,PostgreSQL 唯一的「这个锁等不到」错
+pub async fn materialize_human(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {
+ let mut tx = pool.begin().await?;
+ let result = async {
+ // 2 秒是 0033 的「人点一下」的预算:再长就显成 spinner 了。
+ // worker 的等待策略不在这一格里——它走 `materialize`,没预算
+ sqlx::query("SET LOCAL lock_timeout = '2s'")
+ .execute(&mut *tx)
+ .await?;
+ sqlx::query("SELECT pg_advisory_xact_lock(hashtext('typed_materialize'), hashtext($1))")
+ .bind(kb_id.to_string())
+ .execute(&mut *tx)
+ .await?;
+ materialize_in_tx(&mut tx, kb_id).await
+ }
+ .await;
+ match result {
+ Ok(outcome) => {
+ tx.commit().await?;
+ Ok(outcome)
+ }
+ Err(error) => {
+ // 关掉失败路径的回滚要先把连接释放掉,再走错误映射。
+ // 双重错误(回滚也炸了)保留两者,别只报回滚
+ if let Err(rollback) = tx.rollback().await {
+ return Err(AppError::Other(anyhow::Error::new(error).context(format!(
+ "rolling back human materialization: {rollback}"
+ ))));
+ }
+ if matches!(&error, AppError::Db(sqlx::Error::Database(e))
+ if e.code().as_deref() == Some("55P03"))
+ {
+ return Err(AppError::CodedConflict {
+ code: "alignment_busy",
+ message: "This base is being recomputed by another operation. Please try again shortly."
+ .into(),
+ });
+ }
+ Err(error)
+ }
+ }
+}
+
async fn materialize_in_tx(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
kb_id: Uuid,
diff --git a/crates/utopia-store/tests/human_materialization_has_a_lock_budget.rs b/crates/utopia-store/tests/human_materialization_has_a_lock_budget.rs
new file mode 100644
index 00000000..428618e9
--- /dev/null
+++ b/crates/utopia-store/tests/human_materialization_has_a_lock_budget.rs
@@ -0,0 +1,242 @@
+//! A person clicking Review should never wait indefinitely behind a background
+//! materialization. The human entry point (`materialize_human`) sets a 2-second
+//! `lock_timeout` and maps 55P03 to `AppError::CodedConflict { code: "alignment_busy" }`.
+//!
+//! The worker entry point (`materialize`) keeps its existing unbounded wait —
+//! workers don't have a spinner to look at. This test pins the *human* half only.
+//!
+//! `#798` survey follow-up; same pattern as `#828`'s `decide_and_apply_human`.
+
+use sqlx::{postgres::PgPoolOptions, PgPool};
+use utopia_store::{materialize, phrase_bindings};
+use uuid::Uuid;
+
+async fn wait_for_blocked_descendants(
+ pool: &PgPool,
+ blocker: i32,
+ count: i64,
+) -> anyhow::Result<()> {
+ tokio::time::timeout(std::time::Duration::from_secs(10), async {
+ loop {
+ let blocked: i64 = sqlx::query_scalar(
+ "WITH RECURSIVE blocked(pid) AS (
+ SELECT pid FROM pg_stat_activity WHERE $1=ANY(pg_blocking_pids(pid))
+ UNION SELECT p.pid FROM pg_stat_activity p JOIN blocked b ON b.pid=ANY(pg_blocking_pids(p.pid))
+ ) SELECT count(*) FROM blocked",
+ )
+ .bind(blocker)
+ .fetch_one(pool)
+ .await?;
+ if blocked >= count {
+ return anyhow::Ok(());
+ }
+ tokio::task::yield_now().await;
+ }
+ })
+ .await??;
+ Ok(())
+}
+
+/// A human materialization that hits a worker-held advisory lock returns
+/// CodedConflict("alignment_busy") within a couple of seconds, not after the
+/// worker releases naturally.
+#[tokio::test]
+async fn human_materialization_against_a_held_worker_lock_returns_alignment_busy(
+) -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let control = PgPool::connect(&url).await?;
+ utopia_store::db::migrate(&control).await?;
+ let pool = PgPoolOptions::new()
+ .max_connections(2)
+ .connect(&url)
+ .await?;
+ let (org, ws, kb, subject, object, property, statement) = (
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ );
+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'human-materialize-budget')")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'human-materialize-budget')")
+ .bind(ws)
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'human-materialize-budget')")
+ .bind(kb).bind(ws).execute(&pool).await?;
+ sqlx::query(
+ "INSERT INTO entities(id,kb_id,canonical_name) VALUES($1,$2,'Acme'),($3,$2,'London')",
+ )
+ .bind(subject)
+ .bind(kb)
+ .bind(object)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO relation_types(id,kb_id,key,label,temporal) VALUES($1,$2,'based_in','based in','state')")
+ .bind(property).bind(kb).execute(&pool).await?;
+ sqlx::query("INSERT INTO facts(id,kb_id,subject_id,object_id,layer,phrase) VALUES($1,$2,$3,$4,'open','based in')")
+ .bind(statement).bind(kb).bind(subject).bind(object).execute(&pool).await?;
+ let signature = phrase_bindings::signatures(&pool, kb).await?.remove(0);
+ phrase_bindings::decide(
+ &pool,
+ kb,
+ &signature,
+ phrase_bindings::Decision {
+ relation_type_id: Some(property),
+ direction: Some("forward"),
+ status: "bound",
+ votes: &serde_json::json!({}),
+ decided_by: "agent",
+ },
+ )
+ .await?;
+
+ // Hold the typed_materialize advisory lock on a separate connection so the
+ // human materialization has to wait for it. We DO NOT commit until the test
+ // asserts the human request has already returned 409 with alignment_busy.
+ let mut gate = pool.begin().await?;
+ let blocker: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
+ .fetch_one(&mut *gate)
+ .await?;
+ sqlx::query("SELECT pg_advisory_xact_lock(hashtext('typed_materialize'), hashtext($1))")
+ .bind(kb.to_string())
+ .execute(&mut *gate)
+ .await?;
+ let human_pool = pool.clone();
+ let human_kb = kb;
+ let human = tokio::spawn(async move {
+ // Allow up to 10s wall-clock. Without the fix this would still be
+ // running at 10s and the spawned task would not have returned.
+ tokio::time::timeout(
+ std::time::Duration::from_secs(10),
+ materialize::materialize_human(&human_pool, human_kb),
+ )
+ .await
+ });
+ // The human materialization is on a different connection but the same
+ // pool — pool size is 2 so it should queue on `gate` and reach the lock
+ // wait within a few hundred ms.
+ wait_for_blocked_descendants(&control, blocker, 1).await?;
+ let result = human.await??;
+ let err = result.expect_err("the human materialization must have hit lock_timeout");
+ let utopia_core::AppError::CodedConflict { code, message } = err else {
+ panic!("expected CodedConflict, got: {err:?}");
+ };
+ assert_eq!(code, "alignment_busy");
+ assert!(message.contains("try again"), "message: {message}");
+
+ // Now release the gate and confirm a *worker* materialize (no budget)
+ // completes normally on this base.
+ gate.commit().await?;
+ let outcome = materialize::materialize(&pool, kb).await?;
+ assert_eq!(
+ outcome.added, 1,
+ "the bound statement should produce one typed fact"
+ );
+ sqlx::query("DELETE FROM organizations WHERE id=$1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ pool.close().await;
+ control.close().await;
+ Ok(())
+}
+
+/// When the typed_materialize lock is free, materialize_human completes
+/// normally (no regression on the worker side).
+#[tokio::test]
+async fn human_materialization_with_no_contention_matches_succeeds() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPoolOptions::new()
+ .max_connections(2)
+ .connect(&url)
+ .await?;
+ utopia_store::db::migrate(&pool).await?;
+ let (org, ws, kb, subject, object, property, statement) = (
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ );
+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'human-materialize-no-contention')")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'human-materialize-no-contention')",
+ )
+ .bind(ws)
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'human-materialize-no-contention')")
+ .bind(kb).bind(ws).execute(&pool).await?;
+ sqlx::query(
+ "INSERT INTO entities(id,kb_id,canonical_name) VALUES($1,$2,'Acme'),($3,$2,'London')",
+ )
+ .bind(subject)
+ .bind(kb)
+ .bind(object)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO relation_types(id,kb_id,key,label,temporal) VALUES($1,$2,'based_in','based in','state')")
+ .bind(property).bind(kb).execute(&pool).await?;
+ sqlx::query("INSERT INTO facts(id,kb_id,subject_id,object_id,layer,phrase) VALUES($1,$2,$3,$4,'open','based in')")
+ .bind(statement).bind(kb).bind(subject).bind(object).execute(&pool).await?;
+ let signature = phrase_bindings::signatures(&pool, kb).await?.remove(0);
+ phrase_bindings::decide(
+ &pool,
+ kb,
+ &signature,
+ phrase_bindings::Decision {
+ relation_type_id: Some(property),
+ direction: Some("forward"),
+ status: "bound",
+ votes: &serde_json::json!({}),
+ decided_by: "agent",
+ },
+ )
+ .await?;
+
+ let outcome = materialize::materialize_human(&pool, kb).await?;
+ assert_eq!(outcome.added, 1, "one typed fact should be added");
+
+ // The SET LOCAL inside materialize_human must not leak to the session —
+ // otherwise the next caller on this connection inherits our 2-second
+ // budget, which would be wrong for a worker. Capture the value before and
+ // after, and assert they're equal. Postgres' session default is `0`
+ // (wait forever) on a clean connection; we don't depend on that.
+ let before: Option<String> = sqlx::query_scalar("SHOW lock_timeout")
+ .fetch_optional(&pool)
+ .await?;
+ // Run materialize_human once more — if SET LOCAL had leaked, this second
+ // call would still see `2s` (which we never set at session level).
+ let _ = materialize::materialize_human(&pool, kb).await?;
+ let after: Option<String> = sqlx::query_scalar("SHOW lock_timeout")
+ .fetch_optional(&pool)
+ .await?;
+ assert_eq!(
+ before, after,
+ "lock_timeout should not leak out of materialize_human's transaction; before={before:?} after={after:?}"
+ );
+
+ sqlx::query("DELETE FROM organizations WHERE id=$1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ pool.close().await;
+ Ok(())
+}