World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Serialize typed materialization on one transaction connection (#797)
Maya Kida70402d26e41
diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rsindex 1b832fca..5e80bbe3 100644--- a/crates/utopia-store/src/graph.rs+++ b/crates/utopia-store/src/graph.rs@@ -433,7 +433,10 @@ impl Temporal {/// 谓词的时间语义。没有谓词(0010)按状态——三者里唯一不丢信息的那个,与导入本体时/// 的判断一致-pub async fn predicate_temporal(pool: &PgPool, predicate_id: Option<Uuid>) -> AppResult<Temporal> {+pub async fn predicate_temporal<'e>(+ pool: impl sqlx::Executor<'e, Database = sqlx::Postgres>,+ predicate_id: Option<Uuid>,+) -> AppResult<Temporal> {let Some(id) = predicate_id else {return Ok(Temporal::State);};@@ -536,10 +539,36 @@ async fn insert_fact_inner(object: FactObject<'_>,validity: Validity<'_>,confidence: f32,+) -> AppResult<(Uuid, bool)> {+ let mut conn = pool.acquire().await?;+ insert_fact_on(+ &mut conn,+ kb_id,+ subject_id,+ predicate_id,+ object,+ validity,+ confidence,+ )+ .await+}++/// The same insertion semantics on a caller-owned connection/transaction.+#[allow(clippy::too_many_arguments)]+pub(crate) async fn insert_fact_on(+ conn: &mut sqlx::PgConnection,+ kb_id: Uuid,+ subject_id: Uuid,+ // None = 本体里没有对应的关系。原意不丢——它在证据的 proposed_predicate 里,+ // 显示时由 fact_surface_predicate() 取回(见 `facts.predicate_id`)+ predicate_id: Option<Uuid>,+ object: FactObject<'_>,+ validity: Validity<'_>,+ confidence: f32,) -> AppResult<(Uuid, bool)> {// 按谓词的时间语义归一(0031):事件两端同一刻,恒常无日期。写在这里而不是各个// 写入者那儿——抽取、点头、人自己写的事实都经过这一个门- let temporal = predicate_temporal(pool, predicate_id).await?;+ let temporal = predicate_temporal(&mut *conn, predicate_id).await?;let validity = validity.under(temporal).truncated();let same_sql = match object {FactObject::Entity(_) => {@@ -561,7 +590,7 @@ async fn insert_fact_inner(FactObject::Entity(id) => q.bind(id),FactObject::Value(v) => q.bind(v),};- let same: Vec<FactSpanRow> = q.fetch_all(pool).await?;+ let same: Vec<FactSpanRow> = q.fetch_all(&mut *conn).await?;// 「结束了,不知哪天」的观察撞上同断言的**开放行**(0022 / #393):关上它。// 不并进去——并进去等于把「它结束了」这唯一带来的信息丢掉(同 valid_from 那条// 精确重复的路会这么干);也不另立一行——另立一行让两条各说各话,开放的那条@@ -578,7 +607,8 @@ async fn insert_fact_inner(.max_by_key(|(_, vf, _, _)| *vf);if let Some((open, _, _, _)) = open {if let Some(closed) =- crate::temporal::close_with_unknown_end(pool, *open, validity.attested_at).await?+ crate::temporal::close_with_unknown_end(&mut *conn, *open, validity.attested_at)+ .await?{return Ok((closed, true));}@@ -592,11 +622,12 @@ async fn insert_fact_inner(&& validity.from.is_none_or(|f| Some(f) == *vf)}) {if let Some(stated) =- crate::temporal::state_derived_end(pool, *ended, None, validity.attested_at).await?+ crate::temporal::state_derived_end(&mut *conn, *ended, None, validity.attested_at)+ .await?{return Ok((stated, true));}- attest_earlier(pool, *ended, validity.attested_at).await?;+ attest_earlier(&mut *conn, *ended, validity.attested_at).await?;return Ok((*ended, false));}}@@ -611,7 +642,7 @@ async fn insert_fact_inner(if let Some((ended, _, _, _)) = same.iter().find(|(_, _, vt, _)| *vt == Some(to)) {let precision = validity.to_precision.unwrap_or("day");if let Some(stated) = crate::temporal::state_derived_end(- pool,+ &mut *conn,*ended,Some((to, precision)),validity.attested_at,@@ -620,7 +651,7 @@ async fn insert_fact_inner({return Ok((stated, true));}- attest_earlier(pool, *ended, validity.attested_at).await?;+ attest_earlier(&mut *conn, *ended, validity.attested_at).await?;return Ok((*ended, false));}let open = same@@ -631,7 +662,7 @@ async fn insert_fact_inner(.max_by_key(|(_, vf, _, _)| *vf);if let Some((open, _, _, _)) = open {if let Some(closed) = crate::temporal::close_superseded(- pool,+ &mut *conn,*open,to,validity.to_precision.unwrap_or("day"),@@ -649,7 +680,7 @@ async fn insert_fact_inner(if temporal == Temporal::State && vt.is_none() && vtp.is_none() {if let Some(to) = validity.to {if let Some(closed) = crate::temporal::close_superseded(- pool,+ &mut *conn,*existing,to,validity.to_precision.unwrap_or("day"),@@ -665,14 +696,18 @@ async fn insert_fact_inner(let stated_to = validity.to.map(|to| (to, validity.to_precision.unwrap_or("day")));- if let Some(stated) =- crate::temporal::state_derived_end(pool, *existing, stated_to, validity.attested_at)- .await?+ if let Some(stated) = crate::temporal::state_derived_end(+ &mut *conn,+ *existing,+ stated_to,+ validity.attested_at,+ )+ .await?{return Ok((stated, true));}}- attest_earlier(pool, *existing, validity.attested_at).await?;+ attest_earlier(&mut *conn, *existing, validity.attested_at).await?;return Ok((*existing, false));}// 弱化陈述:新观察无时间,同断言已有开放行 → 并入(取起点最新的开放行)。@@ -684,7 +719,7 @@ async fn insert_fact_inner(.filter(|(_, _, vt, _)| vt.is_none() || temporal == Temporal::Event).max_by_key(|(_, vf, _, _)| *vf){- attest_earlier(pool, *existing, validity.attested_at).await?;+ attest_earlier(&mut *conn, *existing, validity.attested_at).await?;return Ok((*existing, false));}// 没有开放行,但这次观察的文档日期落在某条**已关上**的行里:说的是那一段,不是@@ -696,7 +731,7 @@ async fn insert_fact_inner(.iter().find(|(_, vf, vt, _)| vt.is_some_and(|t| at <= t) && vf.is_none_or(|f| f <= at)){- attest_earlier(pool, *existing, validity.attested_at).await?;+ attest_earlier(&mut *conn, *existing, validity.attested_at).await?;return Ok((*existing, false));}}@@ -766,19 +801,19 @@ async fn insert_fact_inner(.bind(confidence).bind(validity.attested_at).bind(validity.from_grade)- .execute(pool)+ .execute(&mut *conn).await?;// 时间精化:裸行(无时无终的同断言)被本次带时间的观察取代——作废+链上,证据随行if let Some(old_id) = refine_target {sqlx::query("UPDATE facts SET invalidated_at = now() WHERE id = $1").bind(old_id)- .execute(pool)+ .execute(&mut *conn).await?;sqlx::query("UPDATE facts SET supersedes = $2 WHERE id = $1").bind(id).bind(old_id)- .execute(pool)+ .execute(&mut *conn).await?;sqlx::query(// 表层谓词随证据一起搬:精化的是时间,不是原文说了什么。引文的偏移一起搬@@ -791,7 +826,7 @@ async fn insert_fact_inner().bind(id).bind(old_id)- .execute(pool)+ .execute(&mut *conn).await?;// 边上的属性也随行(0037):裸行上已有的金额、职务不因为精化了时间而丢sqlx::query(@@ -802,7 +837,7 @@ async fn insert_fact_inner().bind(id).bind(old_id)- .execute(pool)+ .execute(&mut *conn).await?;}Ok((id, true))@@ -811,8 +846,8 @@ async fn insert_fact_inner(/// 同一断言又被观察到一次:锚点只往早挪(0022)。更早的文档是更早的证据;/// 更晚的什么也不改——一条事实从有证据的那一刻起成立,之后再被提到不会把它/// 往后推。`None`(此刻)也不动它:此刻不会早于任何已有的证据。-async fn attest_earlier(- pool: &PgPool,+async fn attest_earlier<'e>(+ pool: impl sqlx::Executor<'e, Database = sqlx::Postgres>,fact_id: Uuid,at: Option<chrono::DateTime<chrono::Utc>>,) -> AppResult<()> {diff --git a/crates/utopia-store/src/materialize.rs b/crates/utopia-store/src/materialize.rsindex 7ec7c0cf..b331f59b 100644--- a/crates/utopia-store/src/materialize.rs+++ b/crates/utopia-store/src/materialize.rs@@ -5,7 +5,7 @@//! 时间、来源时间、置信度从陈述来;证据与限定各复制一份;来源记在 `typed_fact_sources`,//! `from_statement_id` 是第一条。带 mood 限定的陈述不算。//!-//! 写行走类型化图谱本来的门([`crate::graph::insert_fact`] / [`insert_value_fact`]):同断言+//! 写行走类型化图谱本来的门([`crate::graph::insert_fact`] / [`crate::graph::insert_value_fact`]):同断言//! 同起点复用那一行,裸行被带时间的观察取代并链上,「结束了」关上开着的行——两份文档说//! 同一件事,时间线上是一条边。//!@@ -17,7 +17,7 @@ use sqlx::PgPool;use utopia_core::AppResult;use uuid::Uuid;-use crate::graph::{insert_fact, insert_value_fact, Validity};+use crate::graph::{insert_fact_on, FactObject, Validity};/// 陈述与绑定对得上的条件:短语归一后相等,两端的类相同(空也相同),宾语是不是字面值相同。/// `s` 是开放陈述(facts),`se`/`oe` 是它两端的实体,`b` 是 phrase_bindings@@ -58,6 +58,14 @@ struct Due {/// 对一个库重算一遍。pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {+ // A worker and a human review can recompute the same base concurrently.+ // Serialize before reading due statements, and use this connection for the+ // whole recompute so waiting runs cannot exhaust the pool with lock holders.+ let mut tx = pool.begin().await?;+ sqlx::query("SELECT pg_advisory_xact_lock(hashtext('typed_materialize'), hashtext($1))")+ .bind(kb_id.to_string())+ .execute(&mut *tx)+ .await?;// 1. 删不再成立的来源:陈述死了、行死了、签名没绑着、属性或方向变了、陈述带了 moodsqlx::query(&format!("DELETE FROM typed_fact_sources src@@ -80,7 +88,7 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {WHERE q.fact_id = s.id AND q.role = 'mood'))")).bind(kb_id)- .execute(pool)+ .execute(&mut *tx).await?;// 2. 作废来源全空的类型化行:只动算出来的行(带 from_statement_id 的),人写的不碰@@ -92,7 +100,7 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {AND NOT EXISTS (SELECT 1 FROM typed_fact_sources src WHERE src.fact_id = t.id)",).bind(kb_id)- .execute(pool)+ .execute(&mut *tx).await?.rows_affected();@@ -119,7 +127,7 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {ORDER BY s.id")).bind(kb_id)- .fetch_all(pool)+ .fetch_all(&mut *tx).await?;let (mut added, mut merged) = (0u64, 0u64);@@ -135,36 +143,36 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {};let (fact, new) = match (reverse, d.object_id, &d.object_value) {(true, Some(object), _) => {- insert_fact(- pool,+ insert_fact_on(+ &mut tx,kb_id,object,Some(d.property),- d.subject_id,+ FactObject::Entity(d.subject_id),validity,d.confidence,).await?}(false, Some(object), _) => {- insert_fact(- pool,+ insert_fact_on(+ &mut tx,kb_id,d.subject_id,Some(d.property),- object,+ FactObject::Entity(object),validity,d.confidence,).await?}(false, None, Some(value)) => {- insert_value_fact(- pool,+ insert_fact_on(+ &mut tx,kb_id,d.subject_id,Some(d.property),- value,+ FactObject::Value(value),validity,d.confidence,)@@ -177,7 +185,7 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {sqlx::query("UPDATE facts SET from_statement_id = $2 WHERE id = $1 AND from_statement_id IS NULL").bind(fact).bind(d.statement)- .execute(pool)+ .execute(&mut *tx).await?;// 新行取代了一条裸行(时间精化,supersedes 链上):被取代那行的来源跟着搬过来,// 这一轮就收敛,不等下一轮把旧来源当「不成立」删掉再补@@ -189,7 +197,7 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {ON CONFLICT DO NOTHING",).bind(fact)- .execute(pool)+ .execute(&mut *tx).await?;sqlx::query("DELETE FROM typed_fact_sources src@@ -197,7 +205,7 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {WHERE n.id = $1 AND src.fact_id = n.supersedes",).bind(fact)- .execute(pool)+ .execute(&mut *tx).await?;} else {merged += 1;@@ -208,7 +216,7 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {).bind(fact).bind(d.statement)- .execute(pool)+ .execute(&mut *tx).await?;// 证据与限定各抄一份:证据是同一段原文的同一处引文;限定照角色词原样带过去sqlx::query(@@ -221,7 +229,7 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {).bind(fact).bind(d.statement)- .execute(pool)+ .execute(&mut *tx).await?;sqlx::query("INSERT INTO statement_qualifiers (fact_id, role, value, entity_id)@@ -230,9 +238,10 @@ pub async fn materialize(pool: &PgPool, kb_id: Uuid) -> AppResult<Outcome> {).bind(fact).bind(d.statement)- .execute(pool)+ .execute(&mut *tx).await?;}+ tx.commit().await?;Ok(Outcome {retired,added,diff --git a/crates/utopia-store/src/temporal.rs b/crates/utopia-store/src/temporal.rsindex 90c97acc..f6913e35 100644--- a/crates/utopia-store/src/temporal.rs+++ b/crates/utopia-store/src/temporal.rs@@ -869,8 +869,8 @@ pub async fn retract(pool: &PgPool, kb_id: Uuid, fact_id: Uuid) -> AppResult<boo/// 作废 + 改写成**写明的**终点:旧行记 invalidated_at(认知轴),插入闭合区间的修正行/// (世界轴),证据引用随行复制。原文说它在哪天结束、人裁决把它关上,都走这里——/// 这个终点此后不再由引擎重算。返回修正行 id;`None` = 这条已被作废,没动。-pub async fn close_superseded(- pool: &PgPool,+pub async fn close_superseded<'a>(+ pool: impl sqlx::Acquire<'a, Database = Postgres>,fact_id: Uuid,valid_to: DateTime<Utc>,valid_to_precision: &str,@@ -891,8 +891,8 @@ pub async fn close_superseded(/// `attested_from` 从旧行继承——没起点的裸行靠它记着第一份证据,读出来是「从那时起」。/// 这是原文写明的结束,引擎不重算。证据引用随行复制。返回修正行 id;`None` = 这条已不是/// 开放行,没动。-pub async fn close_with_unknown_end(- pool: &PgPool,+pub async fn close_with_unknown_end<'a>(+ pool: impl sqlx::Acquire<'a, Database = Postgres>,fact_id: Uuid,attested_at: Option<DateTime<Utc>>,) -> AppResult<Option<Uuid>> {@@ -906,8 +906,8 @@ pub async fn close_with_unknown_end(/// 原文说出了一条**引擎关上**的行的终点:改写成原文说的,此后不再重算(#679 第三轮评审)。/// `valid_to` 为 `None` 是「结束了,不知哪天」,锚在 `attested_at`。返回修正行 id;/// 行已作废,或它的终点本来就是写明的,返回 `None`-pub async fn state_derived_end(- pool: &PgPool,+pub async fn state_derived_end<'a>(+ pool: impl sqlx::Acquire<'a, Database = Postgres>,fact_id: Uuid,valid_to: Option<(DateTime<Utc>, &str)>,attested_at: Option<DateTime<Utc>>,diff --git a/crates/utopia-store/tests/materialization_is_serial.rs b/crates/utopia-store/tests/materialization_is_serial.rsnew file mode 100644index 00000000..89c42cec--- /dev/null+++ b/crates/utopia-store/tests/materialization_is_serial.rs@@ -0,0 +1,146 @@+//! Two materializers must not create two typed facts from the same statement.+use sqlx::{postgres::PgPoolOptions, PgPool};+use utopia_store::{materialize, phrase_bindings};+use uuid::Uuid;++async fn blocked_descendants(pool: &PgPool, blocker: i32) -> anyhow::Result<i64> {+ Ok(sqlx::query_scalar("WITH RECURSIVE blocked(pid) AS (+ SELECT pid FROM pg_stat_activity WHERE $1=ANY(pg_blocking_pids(pid))+ UNION SELECT p.pid FROM pg_stat_activity p JOIN blocked b ON b.pid=ANY(pg_blocking_pids(p.pid))+ ) SELECT count(*) FROM blocked")+ .bind(blocker).fetch_one(pool).await?)+}+async fn wait_for(pool: &PgPool, blocker: i32, count: i64) -> anyhow::Result<()> {+ tokio::time::timeout(std::time::Duration::from_secs(10), async {+ while blocked_descendants(pool, blocker).await? < count {+ tokio::task::yield_now().await;+ }+ anyhow::Ok(())+ })+ .await??;+ Ok(())+}++#[tokio::test]+async fn overlapping_materializations_create_one_typed_fact() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let control = PgPool::connect(&url).await?;+ utopia_store::db::migrate(&control).await?;+ // Exactly two connections for two competing materializers. The gate/observer+ // uses a separate pool so it does not affect the workers' connection budget.+ let pool = PgPoolOptions::new()+ .max_connections(2)+ .connect(&url)+ .await?;+ let (org, ws, kb, subject, object, property, statement) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'materialize-race')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'materialize-race')")+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'materialize-race')",+ )+ .bind(kb)+ .bind(ws)+ .execute(&pool)+ .await?;+ for (id, name) in [(subject, "Acme"), (object, "London")] {+ sqlx::query("INSERT INTO entities(id,kb_id,canonical_name) VALUES($1,$2,$3)")+ .bind(id)+ .bind(kb)+ .bind(name)+ .execute(&pool)+ .await?;+ }+ sqlx::query("INSERT INTO relation_types(id,kb_id,key,label,temporal) VALUES($1,$2,'based_in','based in','state')").bind(property).bind(kb).execute(&pool).await?;+ sqlx::query("INSERT INTO facts(id,kb_id,subject_id,object_id,layer,phrase) VALUES($1,$2,$3,$4,'open','based in')").bind(statement).bind(kb).bind(subject).bind(object).execute(&pool).await?;+ let signature = phrase_bindings::signatures(&pool, kb).await?.remove(0);+ phrase_bindings::decide(+ &pool,+ kb,+ &signature,+ phrase_bindings::Decision {+ relation_type_id: Some(property),+ direction: Some("forward"),+ status: "bound",+ votes: &serde_json::json!({}),+ decided_by: "agent",+ },+ )+ .await?;+ let name = format!("materialize_gate_{}", kb.simple());+ sqlx::query(&format!("CREATE FUNCTION {name}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN+ PERFORM pg_advisory_xact_lock(hashtext('materialize-test'),hashtext(TG_ARGV[0])); RETURN NEW;+ END $$")).execute(&control).await?;+ sqlx::query(&format!(+ "CREATE TRIGGER {name} BEFORE INSERT ON facts FOR EACH ROW+ WHEN (NEW.kb_id='{kb}'::uuid AND NEW.layer='typed') EXECUTE FUNCTION {name}('{kb}')"+ ))+ .execute(&control)+ .await?;+ let run=async {+ let mut gate=control.begin().await?;+ let blocker:i32=sqlx::query_scalar("SELECT pg_backend_pid()").fetch_one(&mut *gate).await?;+ sqlx::query("SELECT pg_advisory_xact_lock(hashtext('materialize-test'),hashtext($1))")+ .bind(kb.to_string()).execute(&mut *gate).await?;+ let a_pool=pool.clone();+ let a=tokio::spawn(async move { materialize::materialize(&a_pool,kb).await });+ wait_for(&control,blocker,1).await?;+ let b_pool=pool.clone();+ let b=tokio::spawn(async move { materialize::materialize(&b_pool,kb).await });+ // Old code: both insert triggers wait on the gate. Fixed code: the second+ // materializer waits on the first, which waits on the gate.+ wait_for(&control,blocker,2).await?;+ gate.commit().await?;+ a.await??; b.await??;+ let live=materialize::count(&pool,kb).await?;+ let sources:i64=sqlx::query_scalar("SELECT count(*) FROM typed_fact_sources WHERE statement_id=$1").bind(statement).fetch_one(&pool).await?;+ anyhow::ensure!(live == 1,"overlapping runs produced {live} live typed facts; expected one");+ anyhow::ensure!(sources == 1,"the statement has {sources} typed projections; expected one");+ // One connection is enough for a subsequent materialization, including+ // the nested graph/temporal operations.+ let one=PgPoolOptions::new().max_connections(1).connect(&url).await?;+ anyhow::ensure!(tokio::time::timeout(std::time::Duration::from_secs(10),materialize::materialize(&one,kb)).await?? == materialize::Outcome::default());+ // A later statement closes the existing fact. This exercises a nested+ // temporal transaction on that same single connection (a savepoint).+ sqlx::query("INSERT INTO facts(id,kb_id,subject_id,object_id,layer,phrase,valid_to,valid_to_precision) VALUES($1,$2,$3,$4,'open','based in','2020-01-01','day')")+ .bind(Uuid::now_v7()).bind(kb).bind(subject).bind(object).execute(&one).await?;+ let closed=tokio::time::timeout(std::time::Duration::from_secs(10),materialize::materialize(&one,kb)).await??;+ anyhow::ensure!(closed.added == 1,"the closing statement should create one corrected fact");+ anyhow::ensure!(materialize::count(&one,kb).await? == 1);+ let ended:bool=sqlx::query_scalar("SELECT valid_to='2020-01-01'::timestamptz FROM facts WHERE kb_id=$1 AND layer='typed' AND invalidated_at IS NULL")+ .bind(kb).fetch_one(&one).await?;+ anyhow::ensure!(ended,"the temporal rewrite must commit with the projection");+ anyhow::ensure!(materialize::materialize(&one,kb).await? == materialize::Outcome::default());+ one.close().await;+ anyhow::Ok(())+ }.await;+ sqlx::query(&format!("DROP TRIGGER {name} ON facts"))+ .execute(&control)+ .await?;+ sqlx::query(&format!("DROP FUNCTION {name}()"))+ .execute(&control)+ .await?;+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&control)+ .await?;+ pool.close().await;+ control.close().await;+ run+}