World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Bound lock waits for human phrase alignment review decisions (#798, #800) (#864)
Royceecc85e70b000
diff --git a/crates/utopia-server/src/api/review_routes.rs b/crates/utopia-server/src/api/review_routes.rsindex f324f508..9a62f3bb 100644--- a/crates/utopia-server/src/api/review_routes.rs+++ b/crates/utopia-server/src/api/review_routes.rs@@ -1277,7 +1277,7 @@ pub async fn decide_alignment_phrase(},).await?;- let typed = utopia_store::materialize::materialize(&state.pool, kb_id).await?;+ let typed = utopia_store::materialize::materialize_human(&state.pool, kb_id).await?;let _ = utopia_store::audit::record(&state.pool,Some(kb_id),diff --git a/crates/utopia-store/src/materialize.rs b/crates/utopia-store/src/materialize.rsindex 8079ab36..f8044a25 100644--- a/crates/utopia-store/src/materialize.rs+++ b/crates/utopia-store/src/materialize.rs@@ -14,7 +14,7 @@//! 没有」的(陈述, 绑定)对补上——有同断言的行就并进去,没有才新建。没有模型调用。use sqlx::PgPool;-use utopia_core::AppResult;+use utopia_core::{AppError, AppResult};use uuid::Uuid;use crate::graph::{insert_fact_on, FactObject, Validity};@@ -57,6 +57,9 @@ struct Due {}/// 对一个库重算一遍。+///+/// Worker 走这一条:等多久都行,因为它没人在屏幕前面等。人的判定不走这条:+/// 见 `materialize_human`,那是另一条带预算的入口pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {// A worker and a human review can recompute the same base concurrently.// Serialize before reading due statements, and use this connection for the@@ -71,6 +74,57 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {Ok(outcome)}+/// 人的入口(#798 跟进、#800 复盘):与 `materialize` 同样的活,外加一个+/// 事务级的 `lock_timeout` 预算,超时映射到表「语言可读的重试提示」。+///+/// 设计要点(沿用 #828 的同一条理由):+/// - `lock_timeout` 是**等锁的预算**,不是请求总预算,也不是查询执行预算+/// - worker 不动也不动:背景不会因为 5xx 醒不了,没人在屏幕前面等+/// - advisory 锁本身在事务提交时释放(pg_advisory_xact_lock),所以这里的+/// 等待只在等 worker 的 1 次 materialize 完成;不会等下一个做多次锁+/// - `55P03` 是 `lock_not_available`,PostgreSQL 唯一的「这个锁等不到」错+pub async fn materialize_human(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {+ let mut tx = pool.begin().await?;+ let result = async {+ // 2 秒是 0033 的「人点一下」的预算:再长就显成 spinner 了。+ // worker 的等待策略不在这一格里——它走 `materialize`,没预算+ sqlx::query("SET LOCAL lock_timeout = '2s'")+ .execute(&mut *tx)+ .await?;+ sqlx::query("SELECT pg_advisory_xact_lock(hashtext('typed_materialize'), hashtext($1))")+ .bind(kb_id.to_string())+ .execute(&mut *tx)+ .await?;+ materialize_in_tx(&mut tx, kb_id).await+ }+ .await;+ match result {+ Ok(outcome) => {+ tx.commit().await?;+ Ok(outcome)+ }+ Err(error) => {+ // 关掉失败路径的回滚要先把连接释放掉,再走错误映射。+ // 双重错误(回滚也炸了)保留两者,别只报回滚+ if let Err(rollback) = tx.rollback().await {+ return Err(AppError::Other(anyhow::Error::new(error).context(format!(+ "rolling back human materialization: {rollback}"+ ))));+ }+ if matches!(&error, AppError::Db(sqlx::Error::Database(e))+ if e.code().as_deref() == Some("55P03"))+ {+ return Err(AppError::CodedConflict {+ code: "alignment_busy",+ message: "This base is being recomputed by another operation. Please try again shortly."+ .into(),+ });+ }+ Err(error)+ }+ }+}+async fn materialize_in_tx(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,kb_id: Uuid,diff --git a/crates/utopia-store/tests/human_materialization_has_a_lock_budget.rs b/crates/utopia-store/tests/human_materialization_has_a_lock_budget.rsnew file mode 100644index 00000000..428618e9--- /dev/null+++ b/crates/utopia-store/tests/human_materialization_has_a_lock_budget.rs@@ -0,0 +1,242 @@+//! A person clicking Review should never wait indefinitely behind a background+//! materialization. The human entry point (`materialize_human`) sets a 2-second+//! `lock_timeout` and maps 55P03 to `AppError::CodedConflict { code: "alignment_busy" }`.+//!+//! The worker entry point (`materialize`) keeps its existing unbounded wait —+//! workers don't have a spinner to look at. This test pins the *human* half only.+//!+//! `#798` survey follow-up; same pattern as `#828`'s `decide_and_apply_human`.++use sqlx::{postgres::PgPoolOptions, PgPool};+use utopia_store::{materialize, phrase_bindings};+use uuid::Uuid;++async fn wait_for_blocked_descendants(+ pool: &PgPool,+ blocker: i32,+ count: i64,+) -> anyhow::Result<()> {+ tokio::time::timeout(std::time::Duration::from_secs(10), async {+ loop {+ let blocked: i64 = sqlx::query_scalar(+ "WITH RECURSIVE blocked(pid) AS (+ SELECT pid FROM pg_stat_activity WHERE $1=ANY(pg_blocking_pids(pid))+ UNION SELECT p.pid FROM pg_stat_activity p JOIN blocked b ON b.pid=ANY(pg_blocking_pids(p.pid))+ ) SELECT count(*) FROM blocked",+ )+ .bind(blocker)+ .fetch_one(pool)+ .await?;+ if blocked >= count {+ return anyhow::Ok(());+ }+ tokio::task::yield_now().await;+ }+ })+ .await??;+ Ok(())+}++/// A human materialization that hits a worker-held advisory lock returns+/// CodedConflict("alignment_busy") within a couple of seconds, not after the+/// worker releases naturally.+#[tokio::test]+async fn human_materialization_against_a_held_worker_lock_returns_alignment_busy(+) -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let control = PgPool::connect(&url).await?;+ utopia_store::db::migrate(&control).await?;+ let pool = PgPoolOptions::new()+ .max_connections(2)+ .connect(&url)+ .await?;+ let (org, ws, kb, subject, object, property, statement) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'human-materialize-budget')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'human-materialize-budget')")+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'human-materialize-budget')")+ .bind(kb).bind(ws).execute(&pool).await?;+ sqlx::query(+ "INSERT INTO entities(id,kb_id,canonical_name) VALUES($1,$2,'Acme'),($3,$2,'London')",+ )+ .bind(subject)+ .bind(kb)+ .bind(object)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO relation_types(id,kb_id,key,label,temporal) VALUES($1,$2,'based_in','based in','state')")+ .bind(property).bind(kb).execute(&pool).await?;+ sqlx::query("INSERT INTO facts(id,kb_id,subject_id,object_id,layer,phrase) VALUES($1,$2,$3,$4,'open','based in')")+ .bind(statement).bind(kb).bind(subject).bind(object).execute(&pool).await?;+ let signature = phrase_bindings::signatures(&pool, kb).await?.remove(0);+ phrase_bindings::decide(+ &pool,+ kb,+ &signature,+ phrase_bindings::Decision {+ relation_type_id: Some(property),+ direction: Some("forward"),+ status: "bound",+ votes: &serde_json::json!({}),+ decided_by: "agent",+ },+ )+ .await?;++ // Hold the typed_materialize advisory lock on a separate connection so the+ // human materialization has to wait for it. We DO NOT commit until the test+ // asserts the human request has already returned 409 with alignment_busy.+ let mut gate = pool.begin().await?;+ let blocker: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")+ .fetch_one(&mut *gate)+ .await?;+ sqlx::query("SELECT pg_advisory_xact_lock(hashtext('typed_materialize'), hashtext($1))")+ .bind(kb.to_string())+ .execute(&mut *gate)+ .await?;+ let human_pool = pool.clone();+ let human_kb = kb;+ let human = tokio::spawn(async move {+ // Allow up to 10s wall-clock. Without the fix this would still be+ // running at 10s and the spawned task would not have returned.+ tokio::time::timeout(+ std::time::Duration::from_secs(10),+ materialize::materialize_human(&human_pool, human_kb),+ )+ .await+ });+ // The human materialization is on a different connection but the same+ // pool — pool size is 2 so it should queue on `gate` and reach the lock+ // wait within a few hundred ms.+ wait_for_blocked_descendants(&control, blocker, 1).await?;+ let result = human.await??;+ let err = result.expect_err("the human materialization must have hit lock_timeout");+ let utopia_core::AppError::CodedConflict { code, message } = err else {+ panic!("expected CodedConflict, got: {err:?}");+ };+ assert_eq!(code, "alignment_busy");+ assert!(message.contains("try again"), "message: {message}");++ // Now release the gate and confirm a *worker* materialize (no budget)+ // completes normally on this base.+ gate.commit().await?;+ let outcome = materialize::materialize(&pool, kb).await?;+ assert_eq!(+ outcome.added, 1,+ "the bound statement should produce one typed fact"+ );+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&pool)+ .await?;+ pool.close().await;+ control.close().await;+ Ok(())+}++/// When the typed_materialize lock is free, materialize_human completes+/// normally (no regression on the worker side).+#[tokio::test]+async fn human_materialization_with_no_contention_matches_succeeds() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPoolOptions::new()+ .max_connections(2)+ .connect(&url)+ .await?;+ utopia_store::db::migrate(&pool).await?;+ let (org, ws, kb, subject, object, property, statement) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'human-materialize-no-contention')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'human-materialize-no-contention')",+ )+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'human-materialize-no-contention')")+ .bind(kb).bind(ws).execute(&pool).await?;+ sqlx::query(+ "INSERT INTO entities(id,kb_id,canonical_name) VALUES($1,$2,'Acme'),($3,$2,'London')",+ )+ .bind(subject)+ .bind(kb)+ .bind(object)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO relation_types(id,kb_id,key,label,temporal) VALUES($1,$2,'based_in','based in','state')")+ .bind(property).bind(kb).execute(&pool).await?;+ sqlx::query("INSERT INTO facts(id,kb_id,subject_id,object_id,layer,phrase) VALUES($1,$2,$3,$4,'open','based in')")+ .bind(statement).bind(kb).bind(subject).bind(object).execute(&pool).await?;+ let signature = phrase_bindings::signatures(&pool, kb).await?.remove(0);+ phrase_bindings::decide(+ &pool,+ kb,+ &signature,+ phrase_bindings::Decision {+ relation_type_id: Some(property),+ direction: Some("forward"),+ status: "bound",+ votes: &serde_json::json!({}),+ decided_by: "agent",+ },+ )+ .await?;++ let outcome = materialize::materialize_human(&pool, kb).await?;+ assert_eq!(outcome.added, 1, "one typed fact should be added");++ // The SET LOCAL inside materialize_human must not leak to the session —+ // otherwise the next caller on this connection inherits our 2-second+ // budget, which would be wrong for a worker. Capture the value before and+ // after, and assert they're equal. Postgres' session default is `0`+ // (wait forever) on a clean connection; we don't depend on that.+ let before: Option<String> = sqlx::query_scalar("SHOW lock_timeout")+ .fetch_optional(&pool)+ .await?;+ // Run materialize_human once more — if SET LOCAL had leaked, this second+ // call would still see `2s` (which we never set at session level).+ let _ = materialize::materialize_human(&pool, kb).await?;+ let after: Option<String> = sqlx::query_scalar("SHOW lock_timeout")+ .fetch_optional(&pool)+ .await?;+ assert_eq!(+ before, after,+ "lock_timeout should not leak out of materialize_human's transaction; before={before:?} after={after:?}"+ );++ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&pool)+ .await?;+ pool.close().await;+ Ok(())+}