World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Rows rewritten before #967 get their statements back, and their duplicates are retired (#972)
Maya Kid0fbcd387f04d
diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rsindex a35a8ab3..3e341fc4 100644--- a/crates/utopia-cli/src/main.rs+++ b/crates/utopia-cli/src/main.rs@@ -82,7 +82,7 @@ struct ManifestDataDir {/// not a side effect of a code change.// 是迁移文件的**个数**,不是最大的编号(守卫 `schema_version_policy_compares_against_current`// 按个数比):编号有空缺时两者不同——0071 由一个开放 PR 占着,0072 先落,个数是 71-const CURRENT_SCHEMA_VERSION: u32 = 86;+const CURRENT_SCHEMA_VERSION: u32 = 87;fn main() -> anyhow::Result<()> {dotenvy::dotenv().ok();diff --git a/crates/utopia-store/tests/a_rewritten_row_gets_its_statements_back.rs b/crates/utopia-store/tests/a_rewritten_row_gets_its_statements_back.rsnew file mode 100644index 00000000..4ca9938d--- /dev/null+++ b/crates/utopia-store/tests/a_rewritten_row_gets_its_statements_back.rs@@ -0,0 +1,302 @@+//! 迁移 0096:#967 之前改写出来的行找回它由哪些陈述算出,之后物化重算出的重复行作废。+//!+//! 坏的状态照它当时的来路造出来。一条陈述「Zhang San —leads→ Aurora,自 2023-01-10」+//! 绑到 leads,一条已批准的短语规则从同一句蕴含 works_on。物化之后得到类型化行与隐含行。+//! 两行都经 `correct_interval` 把起点改成 2023-02-01,再把修正行的链接抹掉——那正是+//! #967 之前人改区间写出来的样子。再物化一次,旧代码的后果照样出现:陈述没有活着的行+//! 代表,照原区间又算出两行(D 与 D2)。+//!+//! 跑 0096 之后:两条修正行找回陈述与规则来源,D 与 D2 作废而仍在表里;人手写又改过的行、+//! #967 之后带着链接改写的行都不动;人点头的事实给一条裸行补上起点(时间精化,另一次观察)+//! 也不动——它链着一条物化行,却不是改写;再物化什么也不做,2023-01-20 那一刻没有人在管;+//! 再跑一遍迁移,一样不动。+//!+//! 迁移的语句不按 kb 过滤,改的是全库的行,所以这个测试在 tests/ 顶层自成一个二进制+//! (与 0057 的回填同理,见 `tests/store/main.rs`)。+//!+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。++use chrono::{DateTime, Utc};+use sqlx::PgPool;+use utopia_store::graph::{self, Validity};+use utopia_store::materialize::{materialize, Outcome};+use uuid::Uuid;++const MIGRATION: &str =+ include_str!("../../../migrations/0096_a_rewritten_row_gets_its_statements_back.sql");++fn t(s: &str) -> DateTime<Utc> {+ s.parse().expect("fixed timestamp")+}++/// 一行的链接:from_statement_id、implied、两张来源表里的陈述+async fn links(+ pool: &PgPool,+ fact: Uuid,+) -> anyhow::Result<(Option<Uuid>, bool, Vec<Uuid>, Vec<Uuid>)> {+ let (from, implied): (Option<Uuid>, bool) =+ sqlx::query_as("SELECT from_statement_id, implied FROM facts WHERE id = $1")+ .bind(fact)+ .fetch_one(pool)+ .await?;+ let typed: Vec<Uuid> = sqlx::query_scalar(+ "SELECT statement_id FROM typed_fact_sources WHERE fact_id = $1 ORDER BY statement_id",+ )+ .bind(fact)+ .fetch_all(pool)+ .await?;+ let by_rule: Vec<Uuid> = sqlx::query_scalar(+ "SELECT statement_id FROM implied_fact_sources WHERE fact_id = $1 ORDER BY statement_id",+ )+ .bind(fact)+ .fetch_all(pool)+ .await?;+ Ok((from, implied, typed, by_rule))+}++async fn live(pool: &PgPool, fact: Uuid) -> anyhow::Result<bool> {+ Ok(+ sqlx::query_scalar("SELECT invalidated_at IS NULL FROM facts WHERE id = $1")+ .bind(fact)+ .fetch_one(pool)+ .await?,+ )+}++/// 修正行的链接抹掉:#967 之前的 `correct_interval` 写出来就是这样+async fn as_before_967(pool: &PgPool, fact: Uuid) -> anyhow::Result<()> {+ sqlx::query("UPDATE facts SET from_statement_id = NULL, implied = FALSE WHERE id = $1")+ .bind(fact)+ .execute(pool)+ .await?;+ sqlx::query("DELETE FROM typed_fact_sources WHERE fact_id = $1")+ .bind(fact)+ .execute(pool)+ .await?;+ sqlx::query("DELETE FROM implied_fact_sources WHERE fact_id = $1")+ .bind(fact)+ .execute(pool)+ .await?;+ Ok(())+}++async fn correct_start(pool: &PgPool, fact: Uuid, from: &str) -> anyhow::Result<Uuid> {+ Ok(utopia_store::temporal::correct_interval(+ pool,+ fact,+ Validity {+ from: Some(t(from)),+ from_precision: Some("day"),+ ..Default::default()+ },+ )+ .await?+ .expect("the row was live"))+}++/// 活着的某一谓词的类型化行(主语、宾语给定)+async fn rows(+ pool: &PgPool,+ kb: Uuid,+ subject: Uuid,+ predicate: Uuid,+) -> anyhow::Result<Vec<Uuid>> {+ Ok(sqlx::query_scalar(+ "SELECT id FROM facts+ WHERE kb_id = $1 AND layer = 'typed' AND subject_id = $2 AND predicate_id = $3+ AND invalidated_at IS NULL+ ORDER BY id",+ )+ .bind(kb)+ .bind(subject)+ .bind(predicate)+ .fetch_all(pool)+ .await?)+}++#[tokio::test]+async fn a_rewritten_row_gets_its_statements_back() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'rewritten-row-test')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'rewritten-row-test')")+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'rewritten-row-test')",+ )+ .bind(kb)+ .bind(ws)+ .execute(&pool)+ .await?;++ let run = async {+ let id = Uuid::now_v7;+ let (person, project, leads, works_on, rule) = (id(), id(), id(), id(), id());+ let (zhang, li, wang, zhou, aurora, helios, nova) =+ (id(), id(), id(), id(), id(), id(), id());+ let (charter, kickoff, bare, by_hand) = (id(), id(), id(), id());+ // 插进去的只有本地生成的 UUID 和写死的日期+ sqlx::raw_sql(&format!(+ r#"+ INSERT INTO entity_types (id, kb_id, key, label, color, shape) VALUES+ ('{person}','{kb}','person','person','#7fd0ff','circle'),+ ('{project}','{kb}','project','project','#7fd0ff','circle');+ INSERT INTO relation_types (id, kb_id, key, label, temporal) VALUES+ ('{leads}','{kb}','leads','leads','state'),+ ('{works_on}','{kb}','works_on','works on','state');+ INSERT INTO entities (id, kb_id, type_id, canonical_name) VALUES+ ('{zhang}','{kb}','{person}','Zhang San'),+ ('{li}','{kb}','{person}','Li Si'),+ ('{wang}','{kb}','{person}','Wang Wu'),+ ('{zhou}','{kb}','{person}','Zhou Qi'),+ ('{aurora}','{kb}','{project}','Aurora'),+ ('{helios}','{kb}','{project}','Helios'),+ ('{nova}','{kb}','{project}','Nova');+ -- 三条陈述:张三那条走 #967 之前的路,李四那条走 #967 之后的路,周七那条没有起点,+ -- 之后由人点头的事实补上+ INSERT INTO facts (id, kb_id, subject_id, object_id, layer, phrase, confidence,+ valid_from, valid_from_precision, valid_from_grade) VALUES+ ('{charter}','{kb}','{zhang}','{aurora}','open','leads',0.9,'2023-01-10','day','A'),+ ('{kickoff}','{kb}','{li}','{helios}','open','leads',0.9,'2024-08-12','day','A'),+ ('{bare}','{kb}','{zhou}','{nova}','open','leads',0.9,NULL,NULL,NULL);+ INSERT INTO phrase_bindings (id, kb_id, phrase, subject_type_id, object_type_id,+ object_is_value, relation_type_id, direction, status) VALUES+ ('{binding}','{kb}','leads','{person}','{project}',false,'{leads}','forward','bound');+ INSERT INTO implication_rules (id, kb_id, trigger, phrase, subject_type_id, object_type_id,+ object_is_value, conclude_property_id, reading, status,+ decided_by) VALUES+ ('{rule}','{kb}','phrase','leads','{person}','{project}',false,'{works_on}',NULL,+ 'approved','person');+ -- 人手写的一行:链上没有物化出来的行+ INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, layer, confidence,+ valid_from, valid_from_precision) VALUES+ ('{by_hand}','{kb}','{wang}','{leads}','{helios}','typed',1.0,'2022-01-01','day');+ "#,+ binding = id(),+ ))+ .execute(&pool)+ .await?;++ // 物化:三条陈述各一行 leads,每句另有一行规则蕴含的 works_on+ materialize(&pool, kb).await?;+ let typed = rows(&pool, kb, zhang, leads).await?;+ let by_rule = rows(&pool, kb, zhang, works_on).await?;+ assert_eq!((typed.len(), by_rule.len()), (1, 1));++ // #967 之前人改区间:修正行写出来没有链接+ let corrected = correct_start(&pool, typed[0], "2023-02-01T00:00:00Z").await?;+ let corrected_rule = correct_start(&pool, by_rule[0], "2023-02-01T00:00:00Z").await?;+ as_before_967(&pool, corrected).await?;+ as_before_967(&pool, corrected_rule).await?;+ // 对照:#967 之后改的一行带着链接;人手写的一行被改过+ let after = rows(&pool, kb, li, leads).await?;+ let after = correct_start(&pool, after[0], "2024-08-01T00:00:00Z").await?;+ let by_hand = correct_start(&pool, by_hand, "2022-02-01T00:00:00Z").await?;+ let after_links = links(&pool, after).await?;+ assert_eq!(after_links.0, Some(kickoff), "#967 之后改写带着链接");+ // 对照:人点头的事实给裸行补上起点——新行链着那条物化行,却是另一次观察+ let (refined, _) = graph::insert_fact(+ &pool,+ kb,+ zhou,+ Some(leads),+ nova,+ Validity {+ from: Some(t("2023-05-01T00:00:00Z")),+ from_precision: Some("day"),+ ..Default::default()+ },+ 1.0,+ )+ .await?;+ let (supersedes,): (Option<Uuid>,) =+ sqlx::query_as("SELECT supersedes FROM facts WHERE id = $1")+ .bind(refined)+ .fetch_one(&pool)+ .await?;+ assert!(supersedes.is_some(), "the confirmed fact refines the bare row");++ // 旧代码的后果:再物化一次,照陈述又算出两行,与修正行并排+ let second = materialize(&pool, kb).await?;+ assert_eq!(+ (second.added, second.implied, second.merged),+ (1, 1, 1),+ "the two duplicates, and the bare statement merged into the refined row: {second:?}"+ );+ let refined_links = links(&pool, refined).await?;+ assert_eq!(refined_links, (None, false, vec![bare], vec![]));+ let duplicate = *rows(&pool, kb, zhang, leads)+ .await?+ .iter()+ .find(|f| **f != corrected)+ .expect("a duplicate of the corrected leads row");+ let duplicate_rule = *rows(&pool, kb, zhang, works_on)+ .await?+ .iter()+ .find(|f| **f != corrected_rule)+ .expect("a duplicate of the corrected works_on row");++ sqlx::raw_sql(MIGRATION).execute(&pool).await?;++ // 两条修正行找回陈述与规则来源+ assert_eq!(+ links(&pool, corrected).await?,+ (Some(charter), false, vec![charter], vec![])+ );+ assert_eq!(+ links(&pool, corrected_rule).await?,+ (None, true, vec![], vec![charter])+ );+ // 重复行作废,不删+ assert!(!live(&pool, duplicate).await? && !live(&pool, duplicate_rule).await?);+ // 对照不动+ assert_eq!(links(&pool, after).await?, after_links);+ assert_eq!(links(&pool, by_hand).await?, (None, false, vec![], vec![]));+ assert_eq!(links(&pool, refined).await?, refined_links, "a refinement is not a rewrite");+ assert!(+ live(&pool, by_hand).await? && live(&pool, after).await? && live(&pool, refined).await?+ );++ // 再物化什么也不做:陈述由修正行代表,规则的来源也在+ assert_eq!(materialize(&pool, kb).await?, Outcome::default());+ let (nodes, edges) =+ graph::neighborhood(&pool, kb, aurora, 1, Some(t("2023-01-20T00:00:00Z")), None)+ .await?;+ let names: Vec<&str> = edges+ .iter()+ .filter_map(|e| nodes.iter().find(|n| n.id == e.source))+ .map(|n| n.name.as_str())+ .collect();+ assert!(names.is_empty(), "nobody on 2023-01-20: {names:?}");++ // 再跑一遍,一样不动+ sqlx::raw_sql(MIGRATION).execute(&pool).await?;+ assert_eq!(+ links(&pool, corrected).await?,+ (Some(charter), false, vec![charter], vec![])+ );+ assert_eq!(+ rows(&pool, kb, zhang, leads).await?,+ vec![corrected],+ "one live leads row"+ );+ assert_eq!(rows(&pool, kb, zhang, works_on).await?, vec![corrected_rule]);+ anyhow::Ok(())+ }+ .await;++ sqlx::query("DELETE FROM organizations WHERE id = $1")+ .bind(org)+ .execute(&pool)+ .await?;+ run+}diff --git a/crates/utopia-store/tests/store/main.rs b/crates/utopia-store/tests/store/main.rsindex 91142b89..576590da 100644--- a/crates/utopia-store/tests/store/main.rs+++ b/crates/utopia-store/tests/store/main.rs@@ -18,6 +18,8 @@//! - `the_engine_redraws_only_what_it_drew`:把迁移 0057 的回填语句原样跑一遍,那条//! `UPDATE facts` 不按 kb 过滤,锁住全库的行,和正在重算时间线的测试互相等死——//! 合并二进制的实验里 30% 的运行栽在这里。+//! - `a_rewritten_row_gets_its_statements_back`:把迁移 0096 原样跑一遍,它同样不按 kb+//! 过滤——修正行、重复行都按全库找,会改到别的测试正在物化、改写的行。//!//! 新加测试默认放这个目录;只有碰上面那类全局状态时才放顶层,并在这里补一行为什么。mod a_batch_decides_like_a_person;diff --git a/migrations/0096_a_rewritten_row_gets_its_statements_back.sql b/migrations/0096_a_rewritten_row_gets_its_statements_back.sqlnew file mode 100644index 00000000..53256198--- /dev/null+++ b/migrations/0096_a_rewritten_row_gets_its_statements_back.sql@@ -0,0 +1,137 @@+-- 改写出来的行找回它由哪些陈述算出(#967 的后续)。+--+-- 物化出来的行被改写(作废旧行、插修正行)时,要把「它由哪些陈述、哪条规则算出」带到+-- 修正行上:`from_statement_id`、`implied`、`typed_fact_sources`、`implied_fact_sources`。+-- 闭合与搬移从 #911 起带着,人改区间从 #967 起带着。在那之前改写出来的行没有这些:+-- 陈述不再被活着的类型化行代表,画面上它按原话、原来的区间回来;下一轮物化还照陈述+-- 再算一行,与修正行并排。这里把链接还给修正行,把后来算出来的重复行作废。+--+-- 修正行:活着的类型化行,有 supersedes,自己没有 from_statement_id、也不是 implied,+-- 顺着 supersedes 往上能走到一条带链接的行(从陈述物化的,或规则算的)——离它最近的+-- 那条就是它的来源行。人手写的行,链上没有这样的行,不动;带着链接改写出来的行不在此列。+--+-- 往上走的每一步都得是**改写**:新行与它作废的旧行在同一个事务里写下,新行的+-- recorded_at 就是旧行的 invalidated_at(两个都是 now())。闭合、搬移、人改区间都是+-- 这样。一条后来的观察给裸行补上起点(`insert_fact` 的时间精化)也会链上 supersedes,+-- 但它是另一次观察、另一个事务写的:人点头确认的事实不是陈述算出来的,不给它挂上+-- 陈述——挂上了,陈述一撤,它会被当成算出来的行作废。+--+-- 它找回的陈述:来源行的 from_statement_id,加上链上各行还留着的来源(来源行一作废,+-- 下一轮物化会删掉它的 typed_fact_sources;implied_fact_sources 还在)。+--+-- 重复行:同库、同主语谓词宾语、活着、不是修正行,来源落在修正行找回的陈述里。一条陈述+-- 在一个谓词下只该有一行,所以它只能是链接丢了之后物化重算的。它的来源也归修正行;+-- 它自己**作废,不删**——账本只追加,记录轴倒回去仍看得见它活过。+--+-- 已知的边界:重复行活着时可能按旧起点关上过别的行,那一端要等这条时间线下次重算才回到+-- 修正后的起点,迁移跑不了引擎的重算。来源陈述后来撤回了的修正行,找回链接之后,下一轮+-- 物化会把它作废——与 #967 之后改写出来的行一样:人改的是时间,不是出处。+--+-- 普通语句(sqlx 把每个迁移包在一个事务里);两个计数用 NOTICE 报出来,再跑一遍是 0 和 0。+DO $$+DECLARE+ repaired bigint;+ retired bigint;+BEGIN+ -- 修正行与它最近的来源行(depth = 从修正行往上走几步到它);每一步都是同一个事务里的改写+ CREATE TEMP TABLE repair_rows ON COMMIT DROP AS+ WITH RECURSIVE up (fact_id, prev, depth) AS (+ SELECT f.id, p.id, 1+ FROM facts f+ JOIN facts p ON p.id = f.supersedes AND p.invalidated_at = f.recorded_at+ WHERE f.layer = 'typed' AND f.invalidated_at IS NULL+ AND f.from_statement_id IS NULL AND NOT f.implied+ UNION ALL+ SELECT up.fact_id, pp.id, up.depth + 1+ FROM up+ JOIN facts p ON p.id = up.prev+ JOIN facts pp ON pp.id = p.supersedes AND pp.invalidated_at = p.recorded_at+ WHERE p.from_statement_id IS NULL AND NOT p.implied+ )+ SELECT DISTINCT ON (up.fact_id)+ up.fact_id, up.depth, p.from_statement_id, p.implied+ FROM up+ JOIN facts p ON p.id = up.prev+ WHERE p.from_statement_id IS NOT NULL OR p.implied+ ORDER BY up.fact_id, up.depth;++ -- 链上的每一行:修正行自己,一直到来源行+ CREATE TEMP TABLE repair_chain ON COMMIT DROP AS+ WITH RECURSIVE down (fact_id, member, depth) AS (+ SELECT r.fact_id, r.fact_id, 0 FROM repair_rows r+ UNION ALL+ SELECT down.fact_id, f.supersedes, down.depth + 1+ FROM down+ JOIN facts f ON f.id = down.member+ JOIN repair_rows r ON r.fact_id = down.fact_id+ WHERE f.supersedes IS NOT NULL AND down.depth < r.depth+ )+ SELECT fact_id, member FROM down;++ -- 找回的陈述与规则来源+ CREATE TEMP TABLE repair_statements ON COMMIT DROP AS+ SELECT fact_id, from_statement_id AS statement_id+ FROM repair_rows+ WHERE from_statement_id IS NOT NULL+ UNION+ SELECT c.fact_id, s.statement_id+ FROM repair_chain c+ JOIN typed_fact_sources s ON s.fact_id = c.member;++ CREATE TEMP TABLE repair_implied ON COMMIT DROP AS+ SELECT DISTINCT c.fact_id, i.rule_id, i.statement_id, i.entity_id+ FROM repair_chain c+ JOIN implied_fact_sources i ON i.fact_id = c.member;++ -- 物化之后照同一陈述(或同一规则的同一来源)重算出来的重复行+ CREATE TEMP TABLE repair_duplicates ON COMMIT DROP AS+ SELECT DISTINCT r.fact_id, d.id AS duplicate+ FROM repair_rows r+ JOIN facts f ON f.id = r.fact_id+ JOIN facts d ON d.kb_id = f.kb_id AND d.id <> f.id+ AND d.layer = 'typed' AND d.invalidated_at IS NULL+ AND d.subject_id = f.subject_id+ AND d.predicate_id IS NOT DISTINCT FROM f.predicate_id+ AND d.object_id IS NOT DISTINCT FROM f.object_id+ AND d.object_value IS NOT DISTINCT FROM f.object_value+ WHERE NOT EXISTS (SELECT 1 FROM repair_rows x WHERE x.fact_id = d.id)+ AND (EXISTS (SELECT 1 FROM repair_statements s+ WHERE s.fact_id = r.fact_id+ AND (s.statement_id = d.from_statement_id+ OR EXISTS (SELECT 1 FROM typed_fact_sources t+ WHERE t.fact_id = d.id AND t.statement_id = s.statement_id)))+ OR EXISTS (SELECT 1 FROM repair_implied i+ JOIN implied_fact_sources x ON x.fact_id = d.id AND x.rule_id = i.rule_id+ WHERE i.fact_id = r.fact_id+ AND x.statement_id IS NOT DISTINCT FROM i.statement_id+ AND x.entity_id IS NOT DISTINCT FROM i.entity_id));++ UPDATE facts f+ SET from_statement_id = r.from_statement_id, implied = r.implied+ FROM repair_rows r+ WHERE f.id = r.fact_id;+ GET DIAGNOSTICS repaired = ROW_COUNT;++ INSERT INTO typed_fact_sources (fact_id, statement_id)+ SELECT fact_id, statement_id FROM repair_statements+ UNION+ SELECT d.fact_id, t.statement_id+ FROM repair_duplicates d+ JOIN typed_fact_sources t ON t.fact_id = d.duplicate+ ON CONFLICT DO NOTHING;++ INSERT INTO implied_fact_sources (fact_id, rule_id, statement_id, entity_id)+ SELECT fact_id, rule_id, statement_id, entity_id FROM repair_implied+ UNION+ SELECT d.fact_id, x.rule_id, x.statement_id, x.entity_id+ FROM repair_duplicates d+ JOIN implied_fact_sources x ON x.fact_id = d.duplicate+ ON CONFLICT DO NOTHING;++ UPDATE facts+ SET invalidated_at = now()+ WHERE id IN (SELECT duplicate FROM repair_duplicates) AND invalidated_at IS NULL;+ GET DIAGNOSTICS retired = ROW_COUNT;++ RAISE NOTICE '0096:% 条改写出来的行找回了陈述,% 条重复行作废', repaired, retired;+END $$;