World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Keep each phrase signature's vector so an alignment run embeds only texts it has not seen (#1186)
evolv3.aib82f5e2e1180
diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rsindex 45c8d256..09ca883a 100644--- a/crates/utopia-cli/src/main.rs+++ b/crates/utopia-cli/src/main.rs@@ -82,7 +82,7 @@ struct ManifestDataDir {/// not a side effect of a code change.// 是迁移文件的**个数**,不是最大的编号(守卫 `schema_version_policy_compares_against_current`// 按个数比):编号有空缺时两者不同——0071 由一个开放 PR 占着,0072 先落,个数是 71-const CURRENT_SCHEMA_VERSION: u32 = 98;+const CURRENT_SCHEMA_VERSION: u32 = 99;fn main() -> anyhow::Result<()> {dotenvy::dotenv().ok();diff --git a/crates/utopia-server/src/phrase_alignment.rs b/crates/utopia-server/src/phrase_alignment.rsindex c5628045..b6e9269a 100644--- a/crates/utopia-server/src/phrase_alignment.rs+++ b/crates/utopia-server/src/phrase_alignment.rs@@ -17,6 +17,7 @@use crate::extraction::chat_retrying_rate_limits_at;use crate::llm_util;use crate::state::AppState;+use sha2::{Digest, Sha256};use std::collections::{HashMap, HashSet};use utopia_core::models::RelationTypeView;use utopia_extract::phrase_align::{@@ -269,7 +270,10 @@ type Shortlist = HashMap<SignatureKey, Vec<Uuid>>;/// 按相关度给候选多的签名开短名单:签名的文本(短语加一条例句)嵌入后,与属性的向量比/// 近(`embed_ontology` 建的那份),留最近的 [`SHORTLIST`] 条;标签里的词出现在短语里的/// 属性无论远近都留着("based in" 对 "based in")。没配嵌入模型、属性还没向量、或候选本来-/// 就不多的签名不进表——那时模型看的还是全部结构候选+/// 就不多的签名不进表——那时模型看的还是全部结构候选。+///+/// 签名的向量按 (库, 嵌入模型, 文本的哈希) 存在 `signature_vectors` 里,一轮只嵌没见过的+/// 文本(#1097):签名的文本两轮之间几乎不变,从前每轮全部重嵌,判的可能只是新来的几十条async fn shortlist(state: &AppState,settings: &utopia_core::models::LlmSettings,@@ -278,17 +282,28 @@ async fn shortlist(full: &Considered<'_>,) -> anyhow::Result<Shortlist> {let mut out = Shortlist::new();- let Some(client) = llm_util::embed_client(settings) else {+ let (Some(client), Some(model)) = (+ llm_util::embed_client(settings),+ settings.embed_model.as_deref(),+ ) else {return Ok(out);};let wide: Vec<&PhraseSignature> = sigs.iter().filter(|s| full.get(&s.key()).is_some_and(|(f, _)| f.len() > SHORTLIST)).collect();+ let pool = &state.pool;+ let texts: Vec<String> = wide.iter().map(|s| signature_text(s)).collect();+ let hashes: Vec<Vec<u8>> = texts+ .iter()+ .map(|t| Sha256::digest(t.as_bytes()).to_vec())+ .collect();+ // 这一轮不再出现的文本、别的模型嵌的向量删掉。按全部候选多的签名留,不只按下面嵌了的:+ // 一批嵌入失败时,上一轮存好的不跟着丢+ utopia_store::signature_vectors::prune(pool, kb_id, model, &hashes).await?;if wide.is_empty() {return Ok(out);}- let pool = &state.pool;// 先问这一类属性有没有向量,没有的签名不嵌:嵌了最近邻也是空,照样看全部。一个没跑过// `embed_ontology` 的库从前每轮把一万七千条签名送去嵌入、短名单一条没开(#1097)。// 一类问一句,至多两句@@ -300,23 +315,25 @@ async fn shortlist(has_vectors.insert(kind, has);}}- let usable: Vec<&PhraseSignature> = wide+ let usable: Vec<usize> = (0..wide.len())+ .filter(|&i| has_vectors.get(property_kind(wide[i])) == Some(&true))+ .collect();+ let wanted: Vec<Vec<u8>> = usable.iter().map(|&i| hashes[i].clone()).collect();+ let mut vectors = utopia_store::signature_vectors::get(pool, kb_id, model, &wanted).await?;+ let reused = vectors.len();+ // 没存过的才嵌;两条签名文本相同(例句、引文都一样)只嵌一次+ let mut queued = HashSet::new();+ let missing: Vec<usize> = usable.iter().copied()- .filter(|s| has_vectors.get(property_kind(s)) == Some(&true))+ .filter(|&i| !vectors.contains_key(&hashes[i]) && queued.insert(&hashes[i])).collect();- for batch in usable.chunks(SHORTLIST_EMBED_BATCH) {- let texts: Vec<String> = batch- .iter()- .map(|s| match (s.examples.first(), s.quotes.first()) {- (Some(e), Some(q)) => format!("{} · {e} · {q}", s.phrase),- (Some(e), None) => format!("{} · {e}", s.phrase),- _ => s.phrase.clone(),- })- .collect();- let vectors = {+ let mut embedded = 0;+ for batch in missing.chunks(SHORTLIST_EMBED_BATCH) {+ let batch_texts: Vec<String> = batch.iter().map(|&i| texts[i].clone()).collect();+ let got = {let _permit = llm_util::acquire_embed(state, settings).await;- match client.embed(&texts).await {+ match client.embed(&batch_texts).await {Ok(v) if v.len() == batch.len() => v,Ok(v) => {tracing::warn!(%kb_id, sent = batch.len(), got = v.len(), "签名向量数量对不上,这一批不开短名单");@@ -328,42 +345,64 @@ async fn shortlist(}}};- for (s, vector) in batch.iter().zip(vectors) {- let (fitting, _) = &full[&s.key()];- let near = utopia_store::ontology::nearest_relation_type_ids(- pool,- kb_id,- &vector,- (fitting.len() * 2) as i64,- Some(property_kind(s)),- )- .await?;- if near.is_empty() {- // 属性还没有向量:不开短名单,模型看全部- continue;- }- let fitting_ids: Vec<Uuid> = fitting.iter().map(|p| p.id).collect();- let must_keep: Vec<Uuid> = fitting- .iter()- .filter(|p| label_in_phrase(&p.label, &s.phrase))- .map(|p| p.id)- .collect();- out.insert(- s.key(),- pick_shortlist(&near, &fitting_ids, &must_keep, SHORTLIST),- );+ let items: Vec<(Vec<u8>, Vec<f32>)> =+ batch.iter().map(|&i| hashes[i].clone()).zip(got).collect();+ utopia_store::signature_vectors::put(pool, kb_id, model, &items).await?;+ embedded += items.len();+ vectors.extend(items);+ }+ for &i in &usable {+ let s = wide[i];+ // 这一批没嵌出来:不开短名单,模型看全部+ let Some(vector) = vectors.get(&hashes[i]) else {+ continue;+ };+ let (fitting, _) = &full[&s.key()];+ let near = utopia_store::ontology::nearest_relation_type_ids(+ pool,+ kb_id,+ vector,+ (fitting.len() * 2) as i64,+ Some(property_kind(s)),+ )+ .await?;+ if near.is_empty() {+ // 属性还没有向量:不开短名单,模型看全部+ continue;}+ let fitting_ids: Vec<Uuid> = fitting.iter().map(|p| p.id).collect();+ let must_keep: Vec<Uuid> = fitting+ .iter()+ .filter(|p| label_in_phrase(&p.label, &s.phrase))+ .map(|p| p.id)+ .collect();+ out.insert(+ s.key(),+ pick_shortlist(&near, &fitting_ids, &must_keep, SHORTLIST),+ );}tracing::info!(%kb_id,wide = wide.len(),skipped = wide.len() - usable.len(),+ reused,+ embedded,shortlisted = out.len(),"候选短名单开好");Ok(out)}+/// 嵌入用的签名文本:短语加第一条例句和它的引文。缓存按它的哈希认,改了这里旧向量都对不上,+/// 下一轮全部重嵌一次+fn signature_text(s: &PhraseSignature) -> String {+ match (s.examples.first(), s.quotes.first()) {+ (Some(e), Some(q)) => format!("{} · {e} · {q}", s.phrase),+ (Some(e), None) => format!("{} · {e}", s.phrase),+ _ => s.phrase.clone(),+ }+}+/// 签名能绑的是哪一类属性:宾语是字面值的绑属性,否则绑关系fn property_kind(s: &PhraseSignature) -> &'static str {if s.object_is_value {diff --git a/crates/utopia-server/src/phrase_alignment_tests.rs b/crates/utopia-server/src/phrase_alignment_tests.rsindex e01544b9..0d3bbe11 100644--- a/crates/utopia-server/src/phrase_alignment_tests.rs+++ b/crates/utopia-server/src/phrase_alignment_tests.rs@@ -558,6 +558,66 @@ async fn signatures_are_not_embedded_while_no_property_has_a_vector() -> anyhow:run}+/// 签名的向量存下来(#1097):库没变,第二轮一次嵌入也不叫,也不再问模型。换了嵌入模型,+/// 旧向量与新的属性向量不在一个空间里,重嵌一次,旧模型的行删掉+#[tokio::test]+async fn a_second_run_over_an_unchanged_base_makes_no_embedding_call() -> anyhow::Result<()> {+ let Some(f) = Fx::new().await? else {+ return Ok(());+ };+ let run = async {+ f.enable_embeddings().await?;+ // 12 条不声明域/值域的关系,加上 based_in 共 13 > SHORTLIST:这条签名要开短名单+ for i in 0..12 {+ sqlx::query("INSERT INTO relation_types(id,kb_id,key,label,kind,temporal) VALUES($1,$2,$3,$3,'relation','state')")+ .bind(Uuid::now_v7()).bind(f.kb).bind(format!("filler_{i}")).execute(&f.pool).await?;+ }+ sqlx::query("UPDATE relation_types SET embedding = '[1,0,0,0]' WHERE kb_id=$1")+ .bind(f.kb)+ .execute(&f.pool)+ .await?;+ let cached = || async {+ sqlx::query_scalar::<_, String>(+ "SELECT model FROM signature_vectors WHERE kb_id=$1 ORDER BY model",+ )+ .bind(f.kb)+ .fetch_all(&f.pool)+ .await+ };++ f.script(bound());+ f.run().await?;+ assert_eq!(f.embed_calls(), 1, "the first run embeds the signature");+ assert_eq!(f.binding().await?.status, "bound");+ let asked = f.requests().len();++ f.run().await?;+ assert_eq!(+ f.embed_calls(),+ 1,+ "a second run over an unchanged base makes no embedding call"+ );+ assert_eq!(f.requests().len(), asked, "and asks the model nothing");+ assert_eq!(cached().await?, vec!["scripted-embed".to_string()]);++ sqlx::query("UPDATE llm_settings SET embed_model='scripted-embed-2' WHERE workspace_id=$1")+ .bind(f.ws)+ .execute(&f.pool)+ .await?;+ f.run().await?;+ assert_eq!(f.embed_calls(), 2, "a new embedding model embeds again");+ assert_eq!(+ cached().await?,+ vec!["scripted-embed-2".to_string()],+ "the old model's vectors are dropped"+ );+ anyhow::Ok(())+ }+ .await;+ f.cleanup().await?;+ run+}+#[tokio::test]async fn an_endpoint_class_change_moves_the_signature_and_the_old_row_stops_looping() -> anyhow::Result<()> {diff --git a/crates/utopia-store/src/lib.rs b/crates/utopia-store/src/lib.rsindex 0f0c8907..cdb6334f 100644--- a/crates/utopia-store/src/lib.rs+++ b/crates/utopia-store/src/lib.rs@@ -44,6 +44,7 @@ pub mod review_summary;pub mod rss_full_content;pub mod sealing;pub mod settings;+pub mod signature_vectors;pub mod sources;pub mod temporal;pub mod test_db;diff --git a/crates/utopia-store/src/signature_vectors.rs b/crates/utopia-store/src/signature_vectors.rsnew file mode 100644index 00000000..ab480f12--- /dev/null+++ b/crates/utopia-store/src/signature_vectors.rs@@ -0,0 +1,86 @@+//! 短语签名的向量(#1097,0110)。+//!+//! 短语对齐给候选多的签名开短名单,要把签名的文本(短语、一条例句、一段引文)嵌入。+//! 签名的文本在两轮之间几乎不变,从前每轮全部重嵌:库里一万七千条候选多的签名,每轮+//! 一万七千次嵌入,判的可能只是新来的几十条。这里按 (库, 嵌入模型, 文本的哈希) 存下+//! 算过的向量,一轮只嵌没见过的文本。+//!+//! 只按键读,不做近邻查询,所以不建 HNSW、不登记 `vector_index`;`embedding` 不定维,+//! 随所选嵌入模型(与 `chunks.embedding` 同一条规矩)。这是缓存不是账本:删了下一轮+//! 重嵌,结果一样。++use pgvector::Vector;+use sqlx::PgPool;+use std::collections::HashMap;+use utopia_core::AppResult;+use uuid::Uuid;++/// 这些哈希里已经存了向量的那些,哈希 → 向量+pub async fn get(+ pool: &PgPool,+ kb_id: Uuid,+ model: &str,+ hashes: &[Vec<u8>],+) -> AppResult<HashMap<Vec<u8>, Vec<f32>>> {+ if hashes.is_empty() {+ return Ok(HashMap::new());+ }+ let rows: Vec<(Vec<u8>, Vector)> = sqlx::query_as(+ "SELECT text_hash, embedding FROM signature_vectors+ WHERE kb_id = $1 AND model = $2 AND text_hash = ANY($3)",+ )+ .bind(kb_id)+ .bind(model)+ .bind(hashes)+ .fetch_all(pool)+ .await?;+ Ok(rows+ .into_iter()+ .map(|(hash, v)| (hash, v.to_vec()))+ .collect())+}++/// 存一批新嵌的向量。一个事务一次提交:逐条自动提交是一行一次 fsync。撞键(同一段文本+/// 两轮之间被别处写过)不覆盖:同一模型同一段文字,向量是同一个+pub async fn put(+ pool: &PgPool,+ kb_id: Uuid,+ model: &str,+ items: &[(Vec<u8>, Vec<f32>)],+) -> AppResult<()> {+ if items.is_empty() {+ return Ok(());+ }+ let mut tx = pool.begin().await?;+ for (hash, emb) in items {+ sqlx::query(+ "INSERT INTO signature_vectors (kb_id, model, text_hash, embedding)+ VALUES ($1, $2, $3, $4)+ ON CONFLICT DO NOTHING",+ )+ .bind(kb_id)+ .bind(model)+ .bind(hash)+ .bind(Vector::from(emb.clone()))+ .execute(&mut *tx)+ .await?;+ }+ tx.commit().await?;+ Ok(())+}++/// 删掉这个库里不再用得上的向量:哈希不在 `keep` 里的,以及别的模型嵌的(换了模型,+/// 旧向量与新属性向量不在一个空间里,留着只占地方)。`keep` 要是这一轮**全部**签名的+/// 哈希,不只是这一轮嵌了的——否则一批嵌入失败,上一轮存好的那批跟着被删。返回删了几行+pub async fn prune(pool: &PgPool, kb_id: Uuid, model: &str, keep: &[Vec<u8>]) -> AppResult<u64> {+ Ok(sqlx::query(+ "DELETE FROM signature_vectors+ WHERE kb_id = $1 AND (model <> $2 OR NOT (text_hash = ANY($3)))",+ )+ .bind(kb_id)+ .bind(model)+ .bind(keep)+ .execute(pool)+ .await?+ .rows_affected())+}diff --git a/crates/utopia-store/tests/store/a_signature_vector_lasts_while_its_text_does.rs b/crates/utopia-store/tests/store/a_signature_vector_lasts_while_its_text_does.rsnew file mode 100644index 00000000..59aead69--- /dev/null+++ b/crates/utopia-store/tests/store/a_signature_vector_lasts_while_its_text_does.rs@@ -0,0 +1,79 @@+//! 签名向量的缓存(#1097,0110):按 (库, 模型, 哈希) 取回;一轮收尾删掉这一轮不再出现的+//! 文本和别的模型嵌的行,别的库的不碰。+use sqlx::PgPool;+use utopia_store::signature_vectors;+use uuid::Uuid;++#[tokio::test]+async fn a_signature_vector_lasts_while_its_text_does() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ utopia_store::db::migrate(&pool).await?;+ let (org, ws, kb, other) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ sqlx::raw_sql(&format!(+ "INSERT INTO organizations(id,name) VALUES ('{org}','signature-vectors');+ INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','signature-vectors');+ INSERT INTO knowledge_bases(id,workspace_id,name) VALUES+ ('{kb}','{ws}','signature-vectors'), ('{other}','{ws}','signature-vectors-other');"+ ))+ .execute(&pool)+ .await?;+ let result = async {+ let (kept, gone) = (vec![1u8; 32], vec![2u8; 32]);+ signature_vectors::put(+ &pool,+ kb,+ "m1",+ &[+ (kept.clone(), vec![1.0, 0.0]),+ (gone.clone(), vec![0.0, 1.0]),+ ],+ )+ .await?;+ signature_vectors::put(&pool, kb, "m0", &[(kept.clone(), vec![0.5, 0.5])]).await?;+ signature_vectors::put(&pool, other, "m1", &[(gone.clone(), vec![0.0, 1.0])]).await?;+ // 撞键不覆盖:同一模型同一段文本,向量是同一个+ signature_vectors::put(&pool, kb, "m1", &[(kept.clone(), vec![9.0, 9.0])]).await?;++ let got = signature_vectors::get(&pool, kb, "m1", &[kept.clone(), gone.clone()]).await?;+ anyhow::ensure!(got.len() == 2, "both texts are cached: {got:?}");+ anyhow::ensure!(got[&kept] == vec![1.0, 0.0], "the first vector stays");+ let other_model =+ signature_vectors::get(&pool, kb, "m0", std::slice::from_ref(&kept)).await?;+ anyhow::ensure!(other_model[&kept] == vec![0.5, 0.5], "keyed by model too");++ let deleted =+ signature_vectors::prune(&pool, kb, "m1", std::slice::from_ref(&kept)).await?;+ anyhow::ensure!(+ deleted == 2,+ "the text that no longer occurs and the other model's row go, got {deleted}"+ );+ let left: Vec<(String, Vec<u8>)> = sqlx::query_as(+ "SELECT model, text_hash FROM signature_vectors WHERE kb_id=$1 ORDER BY model",+ )+ .bind(kb)+ .fetch_all(&pool)+ .await?;+ anyhow::ensure!(left == vec![("m1".to_string(), kept.clone())], "{left:?}");+ let untouched =+ signature_vectors::get(&pool, other, "m1", std::slice::from_ref(&gone)).await?;+ anyhow::ensure!(untouched.len() == 1, "another base's vectors are its own");++ anyhow::ensure!(signature_vectors::prune(&pool, kb, "m1", &[]).await? == 1);+ anyhow::Ok(())+ }+ .await;+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&pool)+ .await?;+ pool.close().await;+ result+}diff --git a/crates/utopia-store/tests/store/main.rs b/crates/utopia-store/tests/store/main.rsindex cde15d17..138065ca 100644--- a/crates/utopia-store/tests/store/main.rs+++ b/crates/utopia-store/tests/store/main.rs@@ -86,6 +86,7 @@ mod a_search_puts_the_exact_name_first;mod a_search_reads_the_base_as_it_was;mod a_secret_is_sealed_at_rest;mod a_signature_holds_on_every_path;+mod a_signature_vector_lasts_while_its_text_does;mod a_similar_name_is_proposed_not_merged;mod a_source_kind_is_listed_once;mod a_source_reaches_only_where_it_was_granted;diff --git a/migrations/0110_a_signature_is_embedded_once.sql b/migrations/0110_a_signature_is_embedded_once.sqlnew file mode 100644index 00000000..7884f909--- /dev/null+++ b/migrations/0110_a_signature_is_embedded_once.sql@@ -0,0 +1,18 @@+-- 一条短语签名的文本只嵌一次(#1097)。+--+-- 短语对齐给候选多的签名开短名单,要把签名的文本(短语 · 例句 · 引文)嵌入,再与属性的向量+-- 比近。这些文本在两轮之间几乎不变,从前每轮全部重嵌:一个一万七千条候选多的签名的库,+-- 每轮一万七千次嵌入,要判的可能只是新来的几十条,嵌入成了一轮的固定开销,随库线性长。+--+-- 存法:按 (库, 嵌入模型, 文本的 SHA-256) 存向量,一轮只嵌没见过的文本,并删掉这一轮不再+-- 出现的哈希、以及别的模型嵌的行。模型在键里,换了模型旧行自然用不上。`embedding` 不定维,+-- 随所选嵌入模型(与 `chunks.embedding` 同一条规矩);只按键读、不做近邻查询,所以不建+-- HNSW、不登记 `vector_index`。这是缓存不是账本——删了下一轮重嵌,结果一样——所以不进+-- 0070 的同库责任面,也不随库导出。+CREATE TABLE signature_vectors (+ kb_id UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,+ model TEXT NOT NULL,+ text_hash BYTEA NOT NULL,+ embedding vector NOT NULL,+ PRIMARY KEY (kb_id, model, text_hash)+);