World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
List phrase signatures and kind words in one pass instead of rescanning the base once per group (#1108)
evolv3.ai2d73d5ff0378
diff --git a/crates/utopia-store/src/phrase_bindings.rs b/crates/utopia-store/src/phrase_bindings.rsindex f658aea0..b4c1fe9c 100644--- a/crates/utopia-store/src/phrase_bindings.rs+++ b/crates/utopia-store/src/phrase_bindings.rs@@ -73,37 +73,34 @@ pub async fn signatures(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<PhraseSignaWHERE f.kb_id = $1 AND f.layer = 'open' AND f.invalidated_at IS NULLAND f.phrase IS NOT NULL AND btrim(f.phrase) <> ''),- grouped AS (- SELECT phrase, subject_type_id, object_type_id, object_is_value, count(*) AS count- FROM live GROUP BY phrase, subject_type_id, object_type_id, object_is_value- ),- picked AS (- SELECT g.phrase, g.subject_type_id, g.object_type_id, g.object_is_value,- l.subject_name || ' —' || l.spelling || '→ ' || l.object_name AS example,- coalesce(substr(c.text, l.quote_start + 1, l.quote_end - l.quote_start), '') AS quote,- row_number() OVER (PARTITION BY g.phrase, g.subject_type_id, g.object_type_id, g.object_is_value+ -- 每组的计数和前 3 条例句在一趟里聚出来。从前每组的例句、引文各是一个关联子查询,+ -- 每个都把全部陈述重扫一遍,耗时随开放陈述数平方增长:3.6 万条时一趟约 200 秒(#1096)。+ -- 计数也在这里取,不另聚一份再按四个键用 IS NOT DISTINCT FROM 接回来——那种连接+ -- Postgres 做不了哈希;分块的原文只为留下的那三条去读+ ranked AS (+ SELECT l.*,+ row_number() OVER (PARTITION BY l.phrase, l.subject_type_id, l.object_type_id, l.object_is_valueORDER BY l.recorded_at, l.id) AS rn- FROM grouped g- JOIN live l ON l.phrase = g.phrase- AND l.subject_type_id IS NOT DISTINCT FROM g.subject_type_id- AND l.object_type_id IS NOT DISTINCT FROM g.object_type_id- AND l.object_is_value = g.object_is_value- LEFT JOIN chunks c ON c.id = l.chunk_id+ FROM live l+ ),+ grouped AS (+ SELECT r.phrase, r.subject_type_id, r.object_type_id, r.object_is_value, count(*) AS count,+ array_agg(r.subject_name || ' —' || r.spelling || '→ ' || r.object_name ORDER BY r.rn)+ FILTER (WHERE r.rn <= 3) AS examples,+ array_agg(coalesce(substr(c.text, r.quote_start + 1, r.quote_end - r.quote_start), '') ORDER BY r.rn)+ FILTER (WHERE r.rn <= 3) AS quotes+ FROM ranked r+ LEFT JOIN chunks c ON c.id = r.chunk_id AND r.rn <= 3+ GROUP BY r.phrase, r.subject_type_id, r.object_type_id, r.object_is_value)SELECT g.phrase, g.subject_type_id, st.key AS subject_type_key,g.object_type_id, ot.key AS object_type_key, g.object_is_value, g.count,- ARRAY(SELECT p.example FROM picked p- WHERE p.phrase = g.phrase AND p.subject_type_id IS NOT DISTINCT FROM g.subject_type_id- AND p.object_type_id IS NOT DISTINCT FROM g.object_type_id- AND p.object_is_value = g.object_is_value AND p.rn <= 3 ORDER BY p.rn) AS examples,- ARRAY(SELECT p.quote FROM picked p- WHERE p.phrase = g.phrase AND p.subject_type_id IS NOT DISTINCT FROM g.subject_type_id- AND p.object_type_id IS NOT DISTINCT FROM g.object_type_id- AND p.object_is_value = g.object_is_value AND p.rn <= 3 ORDER BY p.rn) AS quotes+ coalesce(g.examples, '{{}}') AS examples, coalesce(g.quotes, '{{}}') AS quotesFROM grouped gLEFT JOIN entity_types st ON st.id = g.subject_type_idLEFT JOIN entity_types ot ON ot.id = g.object_type_id- ORDER BY g.count DESC, g.phrase",+ -- 计数和短语相同、两端的类不同的签名,顺序也得定下来:对齐按这个顺序分批送模型+ ORDER BY g.count DESC, g.phrase, g.subject_type_id, g.object_type_id, g.object_is_value",phrase = phrase_sql("f.phrase"));Ok(sqlx::query_as(&sql).bind(kb_id).fetch_all(pool).await?)diff --git a/crates/utopia-store/src/type_bindings.rs b/crates/utopia-store/src/type_bindings.rsindex 5c685977..e83fe701 100644--- a/crates/utopia-store/src/type_bindings.rs+++ b/crates/utopia-store/src/type_bindings.rs@@ -67,28 +67,42 @@ pub async fn signatures<'e>(WHERE e.kb_id = $1 AND e.merged_into IS NULLAND e.specific_type IS NOT NULL AND btrim(e.specific_type) <> ''),+ -- 写法、例名、关系短语的前 3 个各在一趟里聚出来。从前每个词各跑三个关联子查询,+ -- 每个都把全部实体(短语那个还连着陈述)重扫一遍,耗时随库的大小平方增长(#1096)grouped AS (- SELECT kind_word, count(*) AS count FROM live GROUP BY kind_word+ SELECT kind_word, count(*) AS count,+ (array_agg(canonical_name ORDER BY (description IS NOT NULL), created_at, id))[1:3] AS examples+ FROM live GROUP BY kind_word+ ),+ spellings AS (+ SELECT kind_word, spelling,+ row_number() OVER (PARTITION BY kind_word ORDER BY count(*) DESC, spelling) AS rn+ FROM live GROUP BY kind_word, spelling+ ),+ words AS (+ SELECT kind_word, array_agg(spelling ORDER BY rn) AS words+ FROM spellings WHERE rn <= 3 GROUP BY kind_word+ ),+ phrase_counts AS (+ SELECT l.kind_word, f.phrase,+ row_number() OVER (PARTITION BY l.kind_word ORDER BY count(*) DESC, f.phrase) AS rn+ FROM facts f JOIN live l ON l.id = f.subject_id+ WHERE f.kb_id = $1 AND f.layer = 'open' AND f.invalidated_at IS NULL+ AND f.phrase IS NOT NULL+ GROUP BY l.kind_word, f.phrase+ ),+ phrases AS (+ SELECT kind_word, array_agg(phrase ORDER BY rn) AS phrases+ FROM phrase_counts WHERE rn <= 3 GROUP BY kind_word)SELECT g.kind_word,g.count,- ARRAY(SELECT s.spelling FROM (- SELECT l.spelling, count(*) AS n FROM live l- WHERE l.kind_word = g.kind_word- GROUP BY l.spelling ORDER BY n DESC, l.spelling LIMIT 3) s- ) AS words,- ARRAY(SELECT l.canonical_name FROM live l- WHERE l.kind_word = g.kind_word- ORDER BY (l.description IS NOT NULL), l.created_at, l.id LIMIT 3- ) AS examples,- ARRAY(SELECT s.phrase FROM (- SELECT f.phrase, count(*) AS n- FROM facts f JOIN live l ON l.id = f.subject_id- WHERE f.kb_id = $1 AND f.layer = 'open' AND f.invalidated_at IS NULL- AND f.phrase IS NOT NULL AND l.kind_word = g.kind_word- GROUP BY f.phrase ORDER BY n DESC, f.phrase LIMIT 3) s- ) AS phrases+ coalesce(w.words, '{{}}') AS words,+ g.examples,+ coalesce(p.phrases, '{{}}') AS phrasesFROM grouped g+ LEFT JOIN words w ON w.kind_word = g.kind_word+ LEFT JOIN phrases p ON p.kind_word = g.kind_wordORDER BY g.count DESC, g.kind_word",kind = kind_word_sql("e.specific_type"));diff --git a/crates/utopia-store/tests/store/main.rs b/crates/utopia-store/tests/store/main.rsindex 3f386de1..e2de9ad5 100644--- a/crates/utopia-store/tests/store/main.rs+++ b/crates/utopia-store/tests/store/main.rs@@ -151,3 +151,4 @@ mod the_floor_under_retrieval;mod the_nearest_chunk_is_found_however_it_is_reached;mod the_second_clock_can_be_rewound;mod the_world_axis_reaches_the_second;+mod tied_signatures_come_in_a_fixed_order;diff --git a/crates/utopia-store/tests/store/tied_signatures_come_in_a_fixed_order.rs b/crates/utopia-store/tests/store/tied_signatures_come_in_a_fixed_order.rsnew file mode 100644index 00000000..6059ee85--- /dev/null+++ b/crates/utopia-store/tests/store/tied_signatures_come_in_a_fixed_order.rs@@ -0,0 +1,207 @@+//! 签名与类别词的扫描一趟聚完(#1096):多于三条的组只取最早的三条例句,引文跟着自己的+//! 例句走;计数与短语都相同、两端的类不同的签名按类的 id 定序,每次一样。+use sqlx::PgPool;+use utopia_store::graph::{self, FactObject};+use utopia_store::{phrase_bindings, type_bindings};+use uuid::Uuid;++/// 一条签名的键与计数:短语、主语的类、宾语的类、宾语是不是字面值、陈述数+type Shape = (String, Option<Uuid>, Option<Uuid>, bool, i64);++#[tokio::test]+async fn tied_signatures_come_in_a_fixed_order() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ utopia_store::db::migrate(&pool).await?;+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ // v7 按生成先后递增:a 类的 id 小于 b 类+ let (a, b) = (Uuid::now_v7(), Uuid::now_v7());+ sqlx::raw_sql(&format!(+ "INSERT INTO organizations(id,name) VALUES ('{org}','tied-signatures');+ INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','tied-signatures');+ INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','tied-signatures');+ INSERT INTO entity_types(id,kb_id,key,label,color,shape) VALUES+ ('{a}','{kb}','a','A','#000','circle'), ('{b}','{kb}','b','B','#000','circle');"+ ))+ .execute(&pool)+ .await?;+ let result = async {+ // 类别词 company 下四种写法,各 4、3、2、1 个实体:写法只留前三+ let mut subjects = Vec::new();+ for (spelling, n) in [+ ("Company", 4),+ ("company", 3),+ ("COMPANY", 2),+ ("CoMpany", 1),+ ] {+ for i in 0..n {+ subjects.push(entity(&pool, kb, &format!("{spelling}-{i}"), a, spelling).await?);+ }+ }+ let target = entity(&pool, kb, "target", b, "place").await?;++ // 一组五条:例句取最早的三条,引文跟着各自的陈述(第一条没有证据,引文为空)+ let mut supplies = Vec::new();+ for s in &subjects[..5] {+ supplies.push(say(&pool, kb, *s, "supplies", FactObject::Entity(target)).await?);+ }+ let doc = Uuid::now_v7();+ sqlx::query("INSERT INTO documents(id,kb_id,filename,sha256) VALUES($1,$2,$3,$3)")+ .bind(doc)+ .bind(kb)+ .bind(doc.to_string())+ .execute(&pool)+ .await?;+ for (seq, fact) in supplies.iter().enumerate().skip(1) {+ let (chunk, text) = (Uuid::now_v7(), format!("q{seq}"));+ sqlx::query("INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES($1,$2,$3,$4,$5)")+ .bind(chunk)+ .bind(kb)+ .bind(doc)+ .bind(seq as i32)+ .bind(&text)+ .execute(&pool)+ .await?;+ graph::add_evidence_located(&pool, *fact, chunk, Some(&text), None, Some((0, 2)))+ .await?;+ }++ // 三组「owns」各两条,计数与短语都相同,只差两端的类与宾语是不是字面值+ let (x, y) = (+ serde_json::json!({ "value": "x" }),+ serde_json::json!({ "value": "y" }),+ );+ for s in &subjects[..2] {+ say(&pool, kb, target, "owns", FactObject::Entity(*s)).await?; // b → a+ say(&pool, kb, *s, "owns", FactObject::Entity(target)).await?; // a → b+ }+ say(&pool, kb, subjects[0], "owns", FactObject::Value(&x)).await?; // a → 字面值+ say(&pool, kb, subjects[1], "owns", FactObject::Value(&y)).await?;++ let sigs = phrase_bindings::signatures(&pool, kb).await?;+ let shape: Vec<Shape> = sigs+ .iter()+ .map(|s| {+ (+ s.phrase.clone(),+ s.subject_type_id,+ s.object_type_id,+ s.object_is_value,+ s.count,+ )+ })+ .collect();+ anyhow::ensure!(+ shape+ == vec![+ ("supplies".into(), Some(a), Some(b), false, 5),+ // 平手按主语的类、宾语的类(空的在后)、宾语是不是字面值+ ("owns".into(), Some(a), Some(b), false, 2),+ ("owns".into(), Some(a), None, true, 2),+ ("owns".into(), Some(b), Some(a), false, 2),+ ],+ "{shape:#?}"+ );+ let supplied = &sigs[0];+ anyhow::ensure!(+ supplied.examples+ == vec![+ "Company-0 —supplies→ target",+ "Company-1 —supplies→ target",+ "Company-2 —supplies→ target",+ ],+ "the first three by when they were recorded: {:?}",+ supplied.examples+ );+ anyhow::ensure!(+ supplied.quotes == vec!["", "q1", "q2"],+ "each quote follows its own example: {:?}",+ supplied.quotes+ );+ anyhow::ensure!(sigs[2].examples == vec!["Company-0 —owns→ x", "Company-1 —owns→ y"]);+ anyhow::ensure!(sigs[2].quotes == vec!["", ""]);+ for _ in 0..3 {+ let again: Vec<_> = phrase_bindings::signatures(&pool, kb)+ .await?+ .iter()+ .map(|s| (s.phrase.clone(), s.subject_type_id, s.object_type_id))+ .collect();+ let first: Vec<_> = sigs+ .iter()+ .map(|s| (s.phrase.clone(), s.subject_type_id, s.object_type_id))+ .collect();+ anyhow::ensure!(again == first, "the order holds from call to call");+ }++ let kinds = type_bindings::signatures(&pool, kb).await?;+ let company = kinds+ .iter()+ .find(|k| k.kind_word == "company")+ .ok_or_else(|| anyhow::anyhow!("no company: {kinds:?}"))?;+ anyhow::ensure!(company.count == 10);+ anyhow::ensure!(+ company.words == vec!["Company", "company", "COMPANY"],+ "the three most frequent spellings: {:?}",+ company.words+ );+ anyhow::ensure!(company.examples == vec!["Company-0", "Company-1", "Company-2"]);+ anyhow::ensure!(+ company.phrases == vec!["supplies", "owns"],+ "phrases by how often a company says them: {:?}",+ company.phrases+ );+ anyhow::ensure!(+ kinds+ .iter()+ .map(|k| k.kind_word.as_str())+ .collect::<Vec<_>>()+ == vec!["company", "place"]+ );+ anyhow::Ok(())+ }+ .await;+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&pool)+ .await?;+ pool.close().await;+ result+}++async fn entity(+ pool: &PgPool,+ kb: Uuid,+ name: &str,+ type_id: Uuid,+ kind_word: &str,+) -> anyhow::Result<Uuid> {+ let id = Uuid::now_v7();+ sqlx::query(+ "INSERT INTO entities (id, kb_id, canonical_name, type_id, specific_type)+ VALUES ($1, $2, $3, $4, $5)",+ )+ .bind(id)+ .bind(kb)+ .bind(name)+ .bind(type_id)+ .bind(kind_word)+ .execute(pool)+ .await?;+ Ok(id)+}++async fn say(+ pool: &PgPool,+ kb: Uuid,+ subject: Uuid,+ phrase: &str,+ object: FactObject<'_>,+) -> anyhow::Result<Uuid> {+ Ok(+ graph::insert_open_statement(pool, kb, subject, phrase, object, None, 1.0)+ .await?+ .0,+ )+}