Skip to content

World's first open-source enterprise world model.

Keep each phrase signature's vector so an alignment run embeds only texts it has not seen (#1186)

evolv3.aib82f5e2e1180
diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs
index 45c8d256..09ca883a 100644
--- a/crates/utopia-cli/src/main.rs
+++ b/crates/utopia-cli/src/main.rs
@@ -82,7 +82,7 @@ struct ManifestDataDir {
/// not a side effect of a code change.
// 是迁移文件的**个数**,不是最大的编号(守卫 `schema_version_policy_compares_against_current`
// 按个数比):编号有空缺时两者不同——0071 由一个开放 PR 占着,0072 先落,个数是 71
-const CURRENT_SCHEMA_VERSION: u32 = 98;
+const CURRENT_SCHEMA_VERSION: u32 = 99;
fn main() -> anyhow::Result<()> {
dotenvy::dotenv().ok();
diff --git a/crates/utopia-server/src/phrase_alignment.rs b/crates/utopia-server/src/phrase_alignment.rs
index c5628045..b6e9269a 100644
--- a/crates/utopia-server/src/phrase_alignment.rs
+++ b/crates/utopia-server/src/phrase_alignment.rs
@@ -17,6 +17,7 @@
use crate::extraction::chat_retrying_rate_limits_at;
use crate::llm_util;
use crate::state::AppState;
+use sha2::{Digest, Sha256};
use std::collections::{HashMap, HashSet};
use utopia_core::models::RelationTypeView;
use utopia_extract::phrase_align::{
@@ -269,7 +270,10 @@ type Shortlist = HashMap<SignatureKey, Vec<Uuid>>;
/// 按相关度给候选多的签名开短名单:签名的文本(短语加一条例句)嵌入后,与属性的向量比
/// 近(`embed_ontology` 建的那份),留最近的 [`SHORTLIST`] 条;标签里的词出现在短语里的
/// 属性无论远近都留着("based in" 对 "based in")。没配嵌入模型、属性还没向量、或候选本来
-/// 就不多的签名不进表——那时模型看的还是全部结构候选
+/// 就不多的签名不进表——那时模型看的还是全部结构候选。
+///
+/// 签名的向量按 (库, 嵌入模型, 文本的哈希) 存在 `signature_vectors` 里,一轮只嵌没见过的
+/// 文本(#1097):签名的文本两轮之间几乎不变,从前每轮全部重嵌,判的可能只是新来的几十条
async fn shortlist(
state: &AppState,
settings: &utopia_core::models::LlmSettings,
@@ -278,17 +282,28 @@ async fn shortlist(
full: &Considered<'_>,
) -> anyhow::Result<Shortlist> {
let mut out = Shortlist::new();
- let Some(client) = llm_util::embed_client(settings) else {
+ let (Some(client), Some(model)) = (
+ llm_util::embed_client(settings),
+ settings.embed_model.as_deref(),
+ ) else {
return Ok(out);
};
let wide: Vec<&PhraseSignature> = sigs
.iter()
.filter(|s| full.get(&s.key()).is_some_and(|(f, _)| f.len() > SHORTLIST))
.collect();
+ let pool = &state.pool;
+ let texts: Vec<String> = wide.iter().map(|s| signature_text(s)).collect();
+ let hashes: Vec<Vec<u8>> = texts
+ .iter()
+ .map(|t| Sha256::digest(t.as_bytes()).to_vec())
+ .collect();
+ // 这一轮不再出现的文本、别的模型嵌的向量删掉。按全部候选多的签名留,不只按下面嵌了的:
+ // 一批嵌入失败时,上一轮存好的不跟着丢
+ utopia_store::signature_vectors::prune(pool, kb_id, model, &hashes).await?;
if wide.is_empty() {
return Ok(out);
}
- let pool = &state.pool;
// 先问这一类属性有没有向量,没有的签名不嵌:嵌了最近邻也是空,照样看全部。一个没跑过
// `embed_ontology` 的库从前每轮把一万七千条签名送去嵌入、短名单一条没开(#1097)。
// 一类问一句,至多两句
@@ -300,23 +315,25 @@ async fn shortlist(
has_vectors.insert(kind, has);
}
}
- let usable: Vec<&PhraseSignature> = wide
+ let usable: Vec<usize> = (0..wide.len())
+ .filter(|&i| has_vectors.get(property_kind(wide[i])) == Some(&true))
+ .collect();
+ let wanted: Vec<Vec<u8>> = usable.iter().map(|&i| hashes[i].clone()).collect();
+ let mut vectors = utopia_store::signature_vectors::get(pool, kb_id, model, &wanted).await?;
+ let reused = vectors.len();
+ // 没存过的才嵌;两条签名文本相同(例句、引文都一样)只嵌一次
+ let mut queued = HashSet::new();
+ let missing: Vec<usize> = usable
.iter()
.copied()
- .filter(|s| has_vectors.get(property_kind(s)) == Some(&true))
+ .filter(|&i| !vectors.contains_key(&hashes[i]) && queued.insert(&hashes[i]))
.collect();
- for batch in usable.chunks(SHORTLIST_EMBED_BATCH) {
- let texts: Vec<String> = batch
- .iter()
- .map(|s| match (s.examples.first(), s.quotes.first()) {
- (Some(e), Some(q)) => format!("{} · {e} · {q}", s.phrase),
- (Some(e), None) => format!("{} · {e}", s.phrase),
- _ => s.phrase.clone(),
- })
- .collect();
- let vectors = {
+ let mut embedded = 0;
+ for batch in missing.chunks(SHORTLIST_EMBED_BATCH) {
+ let batch_texts: Vec<String> = batch.iter().map(|&i| texts[i].clone()).collect();
+ let got = {
let _permit = llm_util::acquire_embed(state, settings).await;
- match client.embed(&texts).await {
+ match client.embed(&batch_texts).await {
Ok(v) if v.len() == batch.len() => v,
Ok(v) => {
tracing::warn!(%kb_id, sent = batch.len(), got = v.len(), "签名向量数量对不上,这一批不开短名单");
@@ -328,42 +345,64 @@ async fn shortlist(
}
}
};
- for (s, vector) in batch.iter().zip(vectors) {
- let (fitting, _) = &full[&s.key()];
- let near = utopia_store::ontology::nearest_relation_type_ids(
- pool,
- kb_id,
- &vector,
- (fitting.len() * 2) as i64,
- Some(property_kind(s)),
- )
- .await?;
- if near.is_empty() {
- // 属性还没有向量:不开短名单,模型看全部
- continue;
- }
- let fitting_ids: Vec<Uuid> = fitting.iter().map(|p| p.id).collect();
- let must_keep: Vec<Uuid> = fitting
- .iter()
- .filter(|p| label_in_phrase(&p.label, &s.phrase))
- .map(|p| p.id)
- .collect();
- out.insert(
- s.key(),
- pick_shortlist(&near, &fitting_ids, &must_keep, SHORTLIST),
- );
+ let items: Vec<(Vec<u8>, Vec<f32>)> =
+ batch.iter().map(|&i| hashes[i].clone()).zip(got).collect();
+ utopia_store::signature_vectors::put(pool, kb_id, model, &items).await?;
+ embedded += items.len();
+ vectors.extend(items);
+ }
+ for &i in &usable {
+ let s = wide[i];
+ // 这一批没嵌出来:不开短名单,模型看全部
+ let Some(vector) = vectors.get(&hashes[i]) else {
+ continue;
+ };
+ let (fitting, _) = &full[&s.key()];
+ let near = utopia_store::ontology::nearest_relation_type_ids(
+ pool,
+ kb_id,
+ vector,
+ (fitting.len() * 2) as i64,
+ Some(property_kind(s)),
+ )
+ .await?;
+ if near.is_empty() {
+ // 属性还没有向量:不开短名单,模型看全部
+ continue;
}
+ let fitting_ids: Vec<Uuid> = fitting.iter().map(|p| p.id).collect();
+ let must_keep: Vec<Uuid> = fitting
+ .iter()
+ .filter(|p| label_in_phrase(&p.label, &s.phrase))
+ .map(|p| p.id)
+ .collect();
+ out.insert(
+ s.key(),
+ pick_shortlist(&near, &fitting_ids, &must_keep, SHORTLIST),
+ );
}
tracing::info!(
%kb_id,
wide = wide.len(),
skipped = wide.len() - usable.len(),
+ reused,
+ embedded,
shortlisted = out.len(),
"候选短名单开好"
);
Ok(out)
}
+/// 嵌入用的签名文本:短语加第一条例句和它的引文。缓存按它的哈希认,改了这里旧向量都对不上,
+/// 下一轮全部重嵌一次
+fn signature_text(s: &PhraseSignature) -> String {
+ match (s.examples.first(), s.quotes.first()) {
+ (Some(e), Some(q)) => format!("{} · {e} · {q}", s.phrase),
+ (Some(e), None) => format!("{} · {e}", s.phrase),
+ _ => s.phrase.clone(),
+ }
+}
+
/// 签名能绑的是哪一类属性:宾语是字面值的绑属性,否则绑关系
fn property_kind(s: &PhraseSignature) -> &'static str {
if s.object_is_value {
diff --git a/crates/utopia-server/src/phrase_alignment_tests.rs b/crates/utopia-server/src/phrase_alignment_tests.rs
index e01544b9..0d3bbe11 100644
--- a/crates/utopia-server/src/phrase_alignment_tests.rs
+++ b/crates/utopia-server/src/phrase_alignment_tests.rs
@@ -558,6 +558,66 @@ async fn signatures_are_not_embedded_while_no_property_has_a_vector() -> anyhow:
run
}
+/// 签名的向量存下来(#1097):库没变,第二轮一次嵌入也不叫,也不再问模型。换了嵌入模型,
+/// 旧向量与新的属性向量不在一个空间里,重嵌一次,旧模型的行删掉
+#[tokio::test]
+async fn a_second_run_over_an_unchanged_base_makes_no_embedding_call() -> anyhow::Result<()> {
+ let Some(f) = Fx::new().await? else {
+ return Ok(());
+ };
+ let run = async {
+ f.enable_embeddings().await?;
+ // 12 条不声明域/值域的关系,加上 based_in 共 13 > SHORTLIST:这条签名要开短名单
+ for i in 0..12 {
+ sqlx::query("INSERT INTO relation_types(id,kb_id,key,label,kind,temporal) VALUES($1,$2,$3,$3,'relation','state')")
+ .bind(Uuid::now_v7()).bind(f.kb).bind(format!("filler_{i}")).execute(&f.pool).await?;
+ }
+ sqlx::query("UPDATE relation_types SET embedding = '[1,0,0,0]' WHERE kb_id=$1")
+ .bind(f.kb)
+ .execute(&f.pool)
+ .await?;
+ let cached = || async {
+ sqlx::query_scalar::<_, String>(
+ "SELECT model FROM signature_vectors WHERE kb_id=$1 ORDER BY model",
+ )
+ .bind(f.kb)
+ .fetch_all(&f.pool)
+ .await
+ };
+
+ f.script(bound());
+ f.run().await?;
+ assert_eq!(f.embed_calls(), 1, "the first run embeds the signature");
+ assert_eq!(f.binding().await?.status, "bound");
+ let asked = f.requests().len();
+
+ f.run().await?;
+ assert_eq!(
+ f.embed_calls(),
+ 1,
+ "a second run over an unchanged base makes no embedding call"
+ );
+ assert_eq!(f.requests().len(), asked, "and asks the model nothing");
+ assert_eq!(cached().await?, vec!["scripted-embed".to_string()]);
+
+ sqlx::query("UPDATE llm_settings SET embed_model='scripted-embed-2' WHERE workspace_id=$1")
+ .bind(f.ws)
+ .execute(&f.pool)
+ .await?;
+ f.run().await?;
+ assert_eq!(f.embed_calls(), 2, "a new embedding model embeds again");
+ assert_eq!(
+ cached().await?,
+ vec!["scripted-embed-2".to_string()],
+ "the old model's vectors are dropped"
+ );
+ anyhow::Ok(())
+ }
+ .await;
+ f.cleanup().await?;
+ run
+}
+
#[tokio::test]
async fn an_endpoint_class_change_moves_the_signature_and_the_old_row_stops_looping(
) -> anyhow::Result<()> {
diff --git a/crates/utopia-store/src/lib.rs b/crates/utopia-store/src/lib.rs
index 0f0c8907..cdb6334f 100644
--- a/crates/utopia-store/src/lib.rs
+++ b/crates/utopia-store/src/lib.rs
@@ -44,6 +44,7 @@ pub mod review_summary;
pub mod rss_full_content;
pub mod sealing;
pub mod settings;
+pub mod signature_vectors;
pub mod sources;
pub mod temporal;
pub mod test_db;
diff --git a/crates/utopia-store/src/signature_vectors.rs b/crates/utopia-store/src/signature_vectors.rs
new file mode 100644
index 00000000..ab480f12
--- /dev/null
+++ b/crates/utopia-store/src/signature_vectors.rs
@@ -0,0 +1,86 @@
+//! 短语签名的向量(#1097,0110)。
+//!
+//! 短语对齐给候选多的签名开短名单,要把签名的文本(短语、一条例句、一段引文)嵌入。
+//! 签名的文本在两轮之间几乎不变,从前每轮全部重嵌:库里一万七千条候选多的签名,每轮
+//! 一万七千次嵌入,判的可能只是新来的几十条。这里按 (库, 嵌入模型, 文本的哈希) 存下
+//! 算过的向量,一轮只嵌没见过的文本。
+//!
+//! 只按键读,不做近邻查询,所以不建 HNSW、不登记 `vector_index`;`embedding` 不定维,
+//! 随所选嵌入模型(与 `chunks.embedding` 同一条规矩)。这是缓存不是账本:删了下一轮
+//! 重嵌,结果一样。
+
+use pgvector::Vector;
+use sqlx::PgPool;
+use std::collections::HashMap;
+use utopia_core::AppResult;
+use uuid::Uuid;
+
+/// 这些哈希里已经存了向量的那些,哈希 → 向量
+pub async fn get(
+ pool: &PgPool,
+ kb_id: Uuid,
+ model: &str,
+ hashes: &[Vec<u8>],
+) -> AppResult<HashMap<Vec<u8>, Vec<f32>>> {
+ if hashes.is_empty() {
+ return Ok(HashMap::new());
+ }
+ let rows: Vec<(Vec<u8>, Vector)> = sqlx::query_as(
+ "SELECT text_hash, embedding FROM signature_vectors
+ WHERE kb_id = $1 AND model = $2 AND text_hash = ANY($3)",
+ )
+ .bind(kb_id)
+ .bind(model)
+ .bind(hashes)
+ .fetch_all(pool)
+ .await?;
+ Ok(rows
+ .into_iter()
+ .map(|(hash, v)| (hash, v.to_vec()))
+ .collect())
+}
+
+/// 存一批新嵌的向量。一个事务一次提交:逐条自动提交是一行一次 fsync。撞键(同一段文本
+/// 两轮之间被别处写过)不覆盖:同一模型同一段文字,向量是同一个
+pub async fn put(
+ pool: &PgPool,
+ kb_id: Uuid,
+ model: &str,
+ items: &[(Vec<u8>, Vec<f32>)],
+) -> AppResult<()> {
+ if items.is_empty() {
+ return Ok(());
+ }
+ let mut tx = pool.begin().await?;
+ for (hash, emb) in items {
+ sqlx::query(
+ "INSERT INTO signature_vectors (kb_id, model, text_hash, embedding)
+ VALUES ($1, $2, $3, $4)
+ ON CONFLICT DO NOTHING",
+ )
+ .bind(kb_id)
+ .bind(model)
+ .bind(hash)
+ .bind(Vector::from(emb.clone()))
+ .execute(&mut *tx)
+ .await?;
+ }
+ tx.commit().await?;
+ Ok(())
+}
+
+/// 删掉这个库里不再用得上的向量:哈希不在 `keep` 里的,以及别的模型嵌的(换了模型,
+/// 旧向量与新属性向量不在一个空间里,留着只占地方)。`keep` 要是这一轮**全部**签名的
+/// 哈希,不只是这一轮嵌了的——否则一批嵌入失败,上一轮存好的那批跟着被删。返回删了几行
+pub async fn prune(pool: &PgPool, kb_id: Uuid, model: &str, keep: &[Vec<u8>]) -> AppResult<u64> {
+ Ok(sqlx::query(
+ "DELETE FROM signature_vectors
+ WHERE kb_id = $1 AND (model <> $2 OR NOT (text_hash = ANY($3)))",
+ )
+ .bind(kb_id)
+ .bind(model)
+ .bind(keep)
+ .execute(pool)
+ .await?
+ .rows_affected())
+}
diff --git a/crates/utopia-store/tests/store/a_signature_vector_lasts_while_its_text_does.rs b/crates/utopia-store/tests/store/a_signature_vector_lasts_while_its_text_does.rs
new file mode 100644
index 00000000..59aead69
--- /dev/null
+++ b/crates/utopia-store/tests/store/a_signature_vector_lasts_while_its_text_does.rs
@@ -0,0 +1,79 @@
+//! 签名向量的缓存(#1097,0110):按 (库, 模型, 哈希) 取回;一轮收尾删掉这一轮不再出现的
+//! 文本和别的模型嵌的行,别的库的不碰。
+use sqlx::PgPool;
+use utopia_store::signature_vectors;
+use uuid::Uuid;
+
+#[tokio::test]
+async fn a_signature_vector_lasts_while_its_text_does() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ utopia_store::db::migrate(&pool).await?;
+ let (org, ws, kb, other) = (
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ );
+ sqlx::raw_sql(&format!(
+ "INSERT INTO organizations(id,name) VALUES ('{org}','signature-vectors');
+ INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','signature-vectors');
+ INSERT INTO knowledge_bases(id,workspace_id,name) VALUES
+ ('{kb}','{ws}','signature-vectors'), ('{other}','{ws}','signature-vectors-other');"
+ ))
+ .execute(&pool)
+ .await?;
+ let result = async {
+ let (kept, gone) = (vec![1u8; 32], vec![2u8; 32]);
+ signature_vectors::put(
+ &pool,
+ kb,
+ "m1",
+ &[
+ (kept.clone(), vec![1.0, 0.0]),
+ (gone.clone(), vec![0.0, 1.0]),
+ ],
+ )
+ .await?;
+ signature_vectors::put(&pool, kb, "m0", &[(kept.clone(), vec![0.5, 0.5])]).await?;
+ signature_vectors::put(&pool, other, "m1", &[(gone.clone(), vec![0.0, 1.0])]).await?;
+ // 撞键不覆盖:同一模型同一段文本,向量是同一个
+ signature_vectors::put(&pool, kb, "m1", &[(kept.clone(), vec![9.0, 9.0])]).await?;
+
+ let got = signature_vectors::get(&pool, kb, "m1", &[kept.clone(), gone.clone()]).await?;
+ anyhow::ensure!(got.len() == 2, "both texts are cached: {got:?}");
+ anyhow::ensure!(got[&kept] == vec![1.0, 0.0], "the first vector stays");
+ let other_model =
+ signature_vectors::get(&pool, kb, "m0", std::slice::from_ref(&kept)).await?;
+ anyhow::ensure!(other_model[&kept] == vec![0.5, 0.5], "keyed by model too");
+
+ let deleted =
+ signature_vectors::prune(&pool, kb, "m1", std::slice::from_ref(&kept)).await?;
+ anyhow::ensure!(
+ deleted == 2,
+ "the text that no longer occurs and the other model's row go, got {deleted}"
+ );
+ let left: Vec<(String, Vec<u8>)> = sqlx::query_as(
+ "SELECT model, text_hash FROM signature_vectors WHERE kb_id=$1 ORDER BY model",
+ )
+ .bind(kb)
+ .fetch_all(&pool)
+ .await?;
+ anyhow::ensure!(left == vec![("m1".to_string(), kept.clone())], "{left:?}");
+ let untouched =
+ signature_vectors::get(&pool, other, "m1", std::slice::from_ref(&gone)).await?;
+ anyhow::ensure!(untouched.len() == 1, "another base's vectors are its own");
+
+ anyhow::ensure!(signature_vectors::prune(&pool, kb, "m1", &[]).await? == 1);
+ anyhow::Ok(())
+ }
+ .await;
+ sqlx::query("DELETE FROM organizations WHERE id=$1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ pool.close().await;
+ result
+}
diff --git a/crates/utopia-store/tests/store/main.rs b/crates/utopia-store/tests/store/main.rs
index cde15d17..138065ca 100644
--- a/crates/utopia-store/tests/store/main.rs
+++ b/crates/utopia-store/tests/store/main.rs
@@ -86,6 +86,7 @@ mod a_search_puts_the_exact_name_first;
mod a_search_reads_the_base_as_it_was;
mod a_secret_is_sealed_at_rest;
mod a_signature_holds_on_every_path;
+mod a_signature_vector_lasts_while_its_text_does;
mod a_similar_name_is_proposed_not_merged;
mod a_source_kind_is_listed_once;
mod a_source_reaches_only_where_it_was_granted;
diff --git a/migrations/0110_a_signature_is_embedded_once.sql b/migrations/0110_a_signature_is_embedded_once.sql
new file mode 100644
index 00000000..7884f909
--- /dev/null
+++ b/migrations/0110_a_signature_is_embedded_once.sql
@@ -0,0 +1,18 @@
+-- 一条短语签名的文本只嵌一次(#1097)。
+--
+-- 短语对齐给候选多的签名开短名单,要把签名的文本(短语 · 例句 · 引文)嵌入,再与属性的向量
+-- 比近。这些文本在两轮之间几乎不变,从前每轮全部重嵌:一个一万七千条候选多的签名的库,
+-- 每轮一万七千次嵌入,要判的可能只是新来的几十条,嵌入成了一轮的固定开销,随库线性长。
+--
+-- 存法:按 (库, 嵌入模型, 文本的 SHA-256) 存向量,一轮只嵌没见过的文本,并删掉这一轮不再
+-- 出现的哈希、以及别的模型嵌的行。模型在键里,换了模型旧行自然用不上。`embedding` 不定维,
+-- 随所选嵌入模型(与 `chunks.embedding` 同一条规矩);只按键读、不做近邻查询,所以不建
+-- HNSW、不登记 `vector_index`。这是缓存不是账本——删了下一轮重嵌,结果一样——所以不进
+-- 0070 的同库责任面,也不随库导出。
+CREATE TABLE signature_vectors (
+ kb_id UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
+ model TEXT NOT NULL,
+ text_hash BYTEA NOT NULL,
+ embedding vector NOT NULL,
+ PRIMARY KEY (kb_id, model, text_hash)
+);