Skip to content

World's first open-source enterprise world model.

List phrase signatures and kind words in one pass instead of rescanning the base once per group (#1108)

evolv3.ai2d73d5ff0378
diff --git a/crates/utopia-store/src/phrase_bindings.rs b/crates/utopia-store/src/phrase_bindings.rs
index f658aea0..b4c1fe9c 100644
--- a/crates/utopia-store/src/phrase_bindings.rs
+++ b/crates/utopia-store/src/phrase_bindings.rs
@@ -73,37 +73,34 @@ pub async fn signatures(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<PhraseSigna
WHERE f.kb_id = $1 AND f.layer = 'open' AND f.invalidated_at IS NULL
AND f.phrase IS NOT NULL AND btrim(f.phrase) <> ''
),
- grouped AS (
- SELECT phrase, subject_type_id, object_type_id, object_is_value, count(*) AS count
- FROM live GROUP BY phrase, subject_type_id, object_type_id, object_is_value
- ),
- picked AS (
- SELECT g.phrase, g.subject_type_id, g.object_type_id, g.object_is_value,
- l.subject_name || ' —' || l.spelling || '→ ' || l.object_name AS example,
- coalesce(substr(c.text, l.quote_start + 1, l.quote_end - l.quote_start), '') AS quote,
- row_number() OVER (PARTITION BY g.phrase, g.subject_type_id, g.object_type_id, g.object_is_value
+ -- 每组的计数和前 3 条例句在一趟里聚出来。从前每组的例句、引文各是一个关联子查询,
+ -- 每个都把全部陈述重扫一遍,耗时随开放陈述数平方增长:3.6 万条时一趟约 200 秒(#1096)。
+ -- 计数也在这里取,不另聚一份再按四个键用 IS NOT DISTINCT FROM 接回来——那种连接
+ -- Postgres 做不了哈希;分块的原文只为留下的那三条去读
+ ranked AS (
+ SELECT l.*,
+ row_number() OVER (PARTITION BY l.phrase, l.subject_type_id, l.object_type_id, l.object_is_value
ORDER BY l.recorded_at, l.id) AS rn
- FROM grouped g
- JOIN live l ON l.phrase = g.phrase
- AND l.subject_type_id IS NOT DISTINCT FROM g.subject_type_id
- AND l.object_type_id IS NOT DISTINCT FROM g.object_type_id
- AND l.object_is_value = g.object_is_value
- LEFT JOIN chunks c ON c.id = l.chunk_id
+ FROM live l
+ ),
+ grouped AS (
+ SELECT r.phrase, r.subject_type_id, r.object_type_id, r.object_is_value, count(*) AS count,
+ array_agg(r.subject_name || ' —' || r.spelling || '→ ' || r.object_name ORDER BY r.rn)
+ FILTER (WHERE r.rn <= 3) AS examples,
+ array_agg(coalesce(substr(c.text, r.quote_start + 1, r.quote_end - r.quote_start), '') ORDER BY r.rn)
+ FILTER (WHERE r.rn <= 3) AS quotes
+ FROM ranked r
+ LEFT JOIN chunks c ON c.id = r.chunk_id AND r.rn <= 3
+ GROUP BY r.phrase, r.subject_type_id, r.object_type_id, r.object_is_value
)
SELECT g.phrase, g.subject_type_id, st.key AS subject_type_key,
g.object_type_id, ot.key AS object_type_key, g.object_is_value, g.count,
- ARRAY(SELECT p.example FROM picked p
- WHERE p.phrase = g.phrase AND p.subject_type_id IS NOT DISTINCT FROM g.subject_type_id
- AND p.object_type_id IS NOT DISTINCT FROM g.object_type_id
- AND p.object_is_value = g.object_is_value AND p.rn <= 3 ORDER BY p.rn) AS examples,
- ARRAY(SELECT p.quote FROM picked p
- WHERE p.phrase = g.phrase AND p.subject_type_id IS NOT DISTINCT FROM g.subject_type_id
- AND p.object_type_id IS NOT DISTINCT FROM g.object_type_id
- AND p.object_is_value = g.object_is_value AND p.rn <= 3 ORDER BY p.rn) AS quotes
+ coalesce(g.examples, '{{}}') AS examples, coalesce(g.quotes, '{{}}') AS quotes
FROM grouped g
LEFT JOIN entity_types st ON st.id = g.subject_type_id
LEFT JOIN entity_types ot ON ot.id = g.object_type_id
- ORDER BY g.count DESC, g.phrase",
+ -- 计数和短语相同、两端的类不同的签名,顺序也得定下来:对齐按这个顺序分批送模型
+ ORDER BY g.count DESC, g.phrase, g.subject_type_id, g.object_type_id, g.object_is_value",
phrase = phrase_sql("f.phrase")
);
Ok(sqlx::query_as(&sql).bind(kb_id).fetch_all(pool).await?)
diff --git a/crates/utopia-store/src/type_bindings.rs b/crates/utopia-store/src/type_bindings.rs
index 5c685977..e83fe701 100644
--- a/crates/utopia-store/src/type_bindings.rs
+++ b/crates/utopia-store/src/type_bindings.rs
@@ -67,28 +67,42 @@ pub async fn signatures<'e>(
WHERE e.kb_id = $1 AND e.merged_into IS NULL
AND e.specific_type IS NOT NULL AND btrim(e.specific_type) <> ''
),
+ -- 写法、例名、关系短语的前 3 个各在一趟里聚出来。从前每个词各跑三个关联子查询,
+ -- 每个都把全部实体(短语那个还连着陈述)重扫一遍,耗时随库的大小平方增长(#1096)
grouped AS (
- SELECT kind_word, count(*) AS count FROM live GROUP BY kind_word
+ SELECT kind_word, count(*) AS count,
+ (array_agg(canonical_name ORDER BY (description IS NOT NULL), created_at, id))[1:3] AS examples
+ FROM live GROUP BY kind_word
+ ),
+ spellings AS (
+ SELECT kind_word, spelling,
+ row_number() OVER (PARTITION BY kind_word ORDER BY count(*) DESC, spelling) AS rn
+ FROM live GROUP BY kind_word, spelling
+ ),
+ words AS (
+ SELECT kind_word, array_agg(spelling ORDER BY rn) AS words
+ FROM spellings WHERE rn <= 3 GROUP BY kind_word
+ ),
+ phrase_counts AS (
+ SELECT l.kind_word, f.phrase,
+ row_number() OVER (PARTITION BY l.kind_word ORDER BY count(*) DESC, f.phrase) AS rn
+ FROM facts f JOIN live l ON l.id = f.subject_id
+ WHERE f.kb_id = $1 AND f.layer = 'open' AND f.invalidated_at IS NULL
+ AND f.phrase IS NOT NULL
+ GROUP BY l.kind_word, f.phrase
+ ),
+ phrases AS (
+ SELECT kind_word, array_agg(phrase ORDER BY rn) AS phrases
+ FROM phrase_counts WHERE rn <= 3 GROUP BY kind_word
)
SELECT g.kind_word,
g.count,
- ARRAY(SELECT s.spelling FROM (
- SELECT l.spelling, count(*) AS n FROM live l
- WHERE l.kind_word = g.kind_word
- GROUP BY l.spelling ORDER BY n DESC, l.spelling LIMIT 3) s
- ) AS words,
- ARRAY(SELECT l.canonical_name FROM live l
- WHERE l.kind_word = g.kind_word
- ORDER BY (l.description IS NOT NULL), l.created_at, l.id LIMIT 3
- ) AS examples,
- ARRAY(SELECT s.phrase FROM (
- SELECT f.phrase, count(*) AS n
- FROM facts f JOIN live l ON l.id = f.subject_id
- WHERE f.kb_id = $1 AND f.layer = 'open' AND f.invalidated_at IS NULL
- AND f.phrase IS NOT NULL AND l.kind_word = g.kind_word
- GROUP BY f.phrase ORDER BY n DESC, f.phrase LIMIT 3) s
- ) AS phrases
+ coalesce(w.words, '{{}}') AS words,
+ g.examples,
+ coalesce(p.phrases, '{{}}') AS phrases
FROM grouped g
+ LEFT JOIN words w ON w.kind_word = g.kind_word
+ LEFT JOIN phrases p ON p.kind_word = g.kind_word
ORDER BY g.count DESC, g.kind_word",
kind = kind_word_sql("e.specific_type")
);
diff --git a/crates/utopia-store/tests/store/main.rs b/crates/utopia-store/tests/store/main.rs
index 3f386de1..e2de9ad5 100644
--- a/crates/utopia-store/tests/store/main.rs
+++ b/crates/utopia-store/tests/store/main.rs
@@ -151,3 +151,4 @@ mod the_floor_under_retrieval;
mod the_nearest_chunk_is_found_however_it_is_reached;
mod the_second_clock_can_be_rewound;
mod the_world_axis_reaches_the_second;
+mod tied_signatures_come_in_a_fixed_order;
diff --git a/crates/utopia-store/tests/store/tied_signatures_come_in_a_fixed_order.rs b/crates/utopia-store/tests/store/tied_signatures_come_in_a_fixed_order.rs
new file mode 100644
index 00000000..6059ee85
--- /dev/null
+++ b/crates/utopia-store/tests/store/tied_signatures_come_in_a_fixed_order.rs
@@ -0,0 +1,207 @@
+//! 签名与类别词的扫描一趟聚完(#1096):多于三条的组只取最早的三条例句,引文跟着自己的
+//! 例句走;计数与短语都相同、两端的类不同的签名按类的 id 定序,每次一样。
+use sqlx::PgPool;
+use utopia_store::graph::{self, FactObject};
+use utopia_store::{phrase_bindings, type_bindings};
+use uuid::Uuid;
+
+/// 一条签名的键与计数:短语、主语的类、宾语的类、宾语是不是字面值、陈述数
+type Shape = (String, Option<Uuid>, Option<Uuid>, bool, i64);
+
+#[tokio::test]
+async fn tied_signatures_come_in_a_fixed_order() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ utopia_store::db::migrate(&pool).await?;
+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ // v7 按生成先后递增:a 类的 id 小于 b 类
+ let (a, b) = (Uuid::now_v7(), Uuid::now_v7());
+ sqlx::raw_sql(&format!(
+ "INSERT INTO organizations(id,name) VALUES ('{org}','tied-signatures');
+ INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','tied-signatures');
+ INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','tied-signatures');
+ INSERT INTO entity_types(id,kb_id,key,label,color,shape) VALUES
+ ('{a}','{kb}','a','A','#000','circle'), ('{b}','{kb}','b','B','#000','circle');"
+ ))
+ .execute(&pool)
+ .await?;
+ let result = async {
+ // 类别词 company 下四种写法,各 4、3、2、1 个实体:写法只留前三
+ let mut subjects = Vec::new();
+ for (spelling, n) in [
+ ("Company", 4),
+ ("company", 3),
+ ("COMPANY", 2),
+ ("CoMpany", 1),
+ ] {
+ for i in 0..n {
+ subjects.push(entity(&pool, kb, &format!("{spelling}-{i}"), a, spelling).await?);
+ }
+ }
+ let target = entity(&pool, kb, "target", b, "place").await?;
+
+ // 一组五条:例句取最早的三条,引文跟着各自的陈述(第一条没有证据,引文为空)
+ let mut supplies = Vec::new();
+ for s in &subjects[..5] {
+ supplies.push(say(&pool, kb, *s, "supplies", FactObject::Entity(target)).await?);
+ }
+ let doc = Uuid::now_v7();
+ sqlx::query("INSERT INTO documents(id,kb_id,filename,sha256) VALUES($1,$2,$3,$3)")
+ .bind(doc)
+ .bind(kb)
+ .bind(doc.to_string())
+ .execute(&pool)
+ .await?;
+ for (seq, fact) in supplies.iter().enumerate().skip(1) {
+ let (chunk, text) = (Uuid::now_v7(), format!("q{seq}"));
+ sqlx::query("INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES($1,$2,$3,$4,$5)")
+ .bind(chunk)
+ .bind(kb)
+ .bind(doc)
+ .bind(seq as i32)
+ .bind(&text)
+ .execute(&pool)
+ .await?;
+ graph::add_evidence_located(&pool, *fact, chunk, Some(&text), None, Some((0, 2)))
+ .await?;
+ }
+
+ // 三组「owns」各两条,计数与短语都相同,只差两端的类与宾语是不是字面值
+ let (x, y) = (
+ serde_json::json!({ "value": "x" }),
+ serde_json::json!({ "value": "y" }),
+ );
+ for s in &subjects[..2] {
+ say(&pool, kb, target, "owns", FactObject::Entity(*s)).await?; // b → a
+ say(&pool, kb, *s, "owns", FactObject::Entity(target)).await?; // a → b
+ }
+ say(&pool, kb, subjects[0], "owns", FactObject::Value(&x)).await?; // a → 字面值
+ say(&pool, kb, subjects[1], "owns", FactObject::Value(&y)).await?;
+
+ let sigs = phrase_bindings::signatures(&pool, kb).await?;
+ let shape: Vec<Shape> = sigs
+ .iter()
+ .map(|s| {
+ (
+ s.phrase.clone(),
+ s.subject_type_id,
+ s.object_type_id,
+ s.object_is_value,
+ s.count,
+ )
+ })
+ .collect();
+ anyhow::ensure!(
+ shape
+ == vec![
+ ("supplies".into(), Some(a), Some(b), false, 5),
+ // 平手按主语的类、宾语的类(空的在后)、宾语是不是字面值
+ ("owns".into(), Some(a), Some(b), false, 2),
+ ("owns".into(), Some(a), None, true, 2),
+ ("owns".into(), Some(b), Some(a), false, 2),
+ ],
+ "{shape:#?}"
+ );
+ let supplied = &sigs[0];
+ anyhow::ensure!(
+ supplied.examples
+ == vec![
+ "Company-0 —supplies→ target",
+ "Company-1 —supplies→ target",
+ "Company-2 —supplies→ target",
+ ],
+ "the first three by when they were recorded: {:?}",
+ supplied.examples
+ );
+ anyhow::ensure!(
+ supplied.quotes == vec!["", "q1", "q2"],
+ "each quote follows its own example: {:?}",
+ supplied.quotes
+ );
+ anyhow::ensure!(sigs[2].examples == vec!["Company-0 —owns→ x", "Company-1 —owns→ y"]);
+ anyhow::ensure!(sigs[2].quotes == vec!["", ""]);
+ for _ in 0..3 {
+ let again: Vec<_> = phrase_bindings::signatures(&pool, kb)
+ .await?
+ .iter()
+ .map(|s| (s.phrase.clone(), s.subject_type_id, s.object_type_id))
+ .collect();
+ let first: Vec<_> = sigs
+ .iter()
+ .map(|s| (s.phrase.clone(), s.subject_type_id, s.object_type_id))
+ .collect();
+ anyhow::ensure!(again == first, "the order holds from call to call");
+ }
+
+ let kinds = type_bindings::signatures(&pool, kb).await?;
+ let company = kinds
+ .iter()
+ .find(|k| k.kind_word == "company")
+ .ok_or_else(|| anyhow::anyhow!("no company: {kinds:?}"))?;
+ anyhow::ensure!(company.count == 10);
+ anyhow::ensure!(
+ company.words == vec!["Company", "company", "COMPANY"],
+ "the three most frequent spellings: {:?}",
+ company.words
+ );
+ anyhow::ensure!(company.examples == vec!["Company-0", "Company-1", "Company-2"]);
+ anyhow::ensure!(
+ company.phrases == vec!["supplies", "owns"],
+ "phrases by how often a company says them: {:?}",
+ company.phrases
+ );
+ anyhow::ensure!(
+ kinds
+ .iter()
+ .map(|k| k.kind_word.as_str())
+ .collect::<Vec<_>>()
+ == vec!["company", "place"]
+ );
+ anyhow::Ok(())
+ }
+ .await;
+ sqlx::query("DELETE FROM organizations WHERE id=$1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ pool.close().await;
+ result
+}
+
+async fn entity(
+ pool: &PgPool,
+ kb: Uuid,
+ name: &str,
+ type_id: Uuid,
+ kind_word: &str,
+) -> anyhow::Result<Uuid> {
+ let id = Uuid::now_v7();
+ sqlx::query(
+ "INSERT INTO entities (id, kb_id, canonical_name, type_id, specific_type)
+ VALUES ($1, $2, $3, $4, $5)",
+ )
+ .bind(id)
+ .bind(kb)
+ .bind(name)
+ .bind(type_id)
+ .bind(kind_word)
+ .execute(pool)
+ .await?;
+ Ok(id)
+}
+
+async fn say(
+ pool: &PgPool,
+ kb: Uuid,
+ subject: Uuid,
+ phrase: &str,
+ object: FactObject<'_>,
+) -> anyhow::Result<Uuid> {
+ Ok(
+ graph::insert_open_statement(pool, kb, subject, phrase, object, None, 1.0)
+ .await?
+ .0,
+ )
+}