World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A trimmed description is not stale (#673)
WaylandYangc927c90d7073
diff --git a/crates/utopia-store/src/ontology.rs b/crates/utopia-store/src/ontology.rsindex 79a8c210..d96de7c2 100644--- a/crates/utopia-store/src/ontology.rs+++ b/crates/utopia-store/src/ontology.rs@@ -1115,49 +1115,64 @@ pub async fn types_needing_embedding(model: &str,only: Option<TypeKind>,) -> AppResult<Vec<TypeToEmbed>> {- let mut out = Vec::new();if only == Some(TypeKind::Relation) {// 类型消解只用类,等 1633 个关系嵌完是白等六分钟return relations_needing_embedding(pool, kb_id, model).await;}- let ents: Vec<(Uuid, String, String)> = sqlx::query_as(- "SELECT id, label, coalesce(description, '') FROM entity_types- WHERE kb_id = $1- AND (embedding IS NULL- OR embedded_model IS DISTINCT FROM $2- OR embedded_text IS DISTINCT FROM- CASE WHEN coalesce(btrim(description), '') = '' THEN btrim(label)- ELSE btrim(label) || E'\n' || btrim(description) END)",+ // **陈不陈,用生成原文的同一个函数判**(#672)。从前在 SQL 里用 `btrim` 把这段字+ // 重拼一遍再比,而 `embed_text` 用的是 Rust 的 `trim`:`btrim` 只去空格,`trim`+ // 连换行、制表符一起去。schema.org 的描述结尾是 "\n ",这些行存下的原文与+ // SQL 拼出来的永远不等,于是每轮都判成陈的——补齐任务把同一批行一遍遍重嵌,+ // 抽取门控(#526)等一个永远补不齐的索引,直到期限把文档判失败。+ // 与 `mappings::needing_embedding` 同一个教训:拉回来在 Rust 里比+ let rows: Vec<StoredEmbedding> = sqlx::query_as(+ "SELECT id, label, coalesce(description, '') AS description,+ embedding IS NOT NULL AS embedded, embedded_model, embedded_text,+ label_embedding IS NOT NULL AS label_embedded, label_embedded_model, label_embedded_text+ FROM entity_types+ WHERE kb_id = $1",).bind(kb_id)- .bind(model).fetch_all(pool).await?;- out.extend(ents.into_iter().map(|(id, label, desc)| TypeToEmbed {- id,- kind: TypeKind::Entity,- text: embed_text(&label, &desc),- field: EmbedField::Full,- }));- // 只嵌 label 的那一份(见 `entity_types.label_embedding`)。短查询走这个索引——查询分了两种形状,- // 文档也得分两种,否则短查询被同义反复的类接管- let labels: Vec<(Uuid, String)> = sqlx::query_as(- "SELECT id, label FROM entity_types- WHERE kb_id = $1- AND (label_embedding IS NULL- OR label_embedded_model IS DISTINCT FROM $2- OR label_embedded_text IS DISTINCT FROM btrim(label))",- )- .bind(kb_id)- .bind(model)- .fetch_all(pool)- .await?;- out.extend(labels.into_iter().map(|(id, label)| TypeToEmbed {- id,- kind: TypeKind::Entity,- text: label.trim().to_string(),- field: EmbedField::Label,- }));+ let mut full = Vec::new();+ let mut labels = Vec::new();+ for r in rows {+ let text = embed_text(&r.label, &r.description);+ if is_stale(+ r.embedded,+ r.embedded_model.as_deref(),+ r.embedded_text.as_deref(),+ model,+ &text,+ ) {+ full.push(TypeToEmbed {+ id: r.id,+ kind: TypeKind::Entity,+ text,+ field: EmbedField::Full,+ });+ }+ // 只嵌 label 的那一份(见 `entity_types.label_embedding`)。短查询走这个索引——查询分了两种形状,+ // 文档也得分两种,否则短查询被同义反复的类接管+ let short = r.label.trim().to_string();+ if is_stale(+ r.label_embedded,+ r.label_embedded_model.as_deref(),+ r.label_embedded_text.as_deref(),+ model,+ &short,+ ) {+ labels.push(TypeToEmbed {+ id: r.id,+ kind: TypeKind::Entity,+ text: short,+ field: EmbedField::Label,+ });+ }+ }+ let mut out = full;+ out.extend(labels);if only == Some(TypeKind::Entity) {return Ok(out);}@@ -1170,28 +1185,64 @@ async fn relations_needing_embedding(kb_id: Uuid,model: &str,) -> AppResult<Vec<TypeToEmbed>> {- let mut out = Vec::new();- let rels: Vec<(Uuid, String, String)> = sqlx::query_as(- "SELECT id, label, coalesce(description, '') FROM relation_types- WHERE kb_id = $1- AND (embedding IS NULL- OR embedded_model IS DISTINCT FROM $2- OR embedded_text IS DISTINCT FROM- CASE WHEN coalesce(btrim(description), '') = '' THEN btrim(label)- ELSE btrim(label) || E'\n' || btrim(description) END)",+ // 判据同上(#672):在 Rust 里用 `embed_text` 比,不在 SQL 里重拼。关系只有整段+ // 那一份向量,label 那三列补空+ let rows: Vec<StoredEmbedding> = sqlx::query_as(+ "SELECT id, label, coalesce(description, '') AS description,+ embedding IS NOT NULL AS embedded, embedded_model, embedded_text,+ false AS label_embedded, NULL::text AS label_embedded_model,+ NULL::text AS label_embedded_text+ FROM relation_types+ WHERE kb_id = $1",).bind(kb_id)- .bind(model).fetch_all(pool).await?;- out.extend(rels.into_iter().map(|(id, label, desc)| TypeToEmbed {- id,- kind: TypeKind::Relation,- text: embed_text(&label, &desc),- // 关系没有短查询那一路,只有整段这一份- field: EmbedField::Full,- }));- Ok(out)+ Ok(rows+ .into_iter()+ .filter_map(|r| {+ let text = embed_text(&r.label, &r.description);+ is_stale(+ r.embedded,+ r.embedded_model.as_deref(),+ r.embedded_text.as_deref(),+ model,+ &text,+ )+ .then_some(TypeToEmbed {+ id: r.id,+ kind: TypeKind::Relation,+ text,+ // 关系没有短查询那一路,只有整段这一份+ field: EmbedField::Full,+ })+ })+ .collect())+}++/// 类型那一行上两份向量的现状,判陈用+#[derive(sqlx::FromRow)]+struct StoredEmbedding {+ id: Uuid,+ label: String,+ description: String,+ embedded: bool,+ embedded_model: Option<String>,+ embedded_text: Option<String>,+ label_embedded: bool,+ label_embedded_model: Option<String>,+ label_embedded_text: Option<String>,+}++/// 没嵌过、换了模型、或者当时嵌的原文与现在要嵌的不一样+fn is_stale(+ embedded: bool,+ stored_model: Option<&str>,+ stored_text: Option<&str>,+ model: &str,+ text: &str,+) -> bool {+ !embedded || stored_model != Some(model) || stored_text != Some(text)}/// 回写向量,连同"嵌的是哪段字、用的哪个模型"。三者必须同一次写入——diff --git a/crates/utopia-store/tests/a_trimmed_description_is_not_stale.rs b/crates/utopia-store/tests/a_trimmed_description_is_not_stale.rsnew file mode 100644index 00000000..3fe25ae6--- /dev/null+++ b/crates/utopia-store/tests/a_trimmed_description_is_not_stale.rs@@ -0,0 +1,133 @@+//! 嵌过的本体行不因为描述首尾的换行就永远判成陈的(#672),打在真库上。+//!+//! 陈不陈看「当时嵌的原文」与「现在要嵌的原文」是否一致。从前后者在 SQL 里用 `btrim`+//! 重拼,而原文由 Rust 的 `trim` 生成:`btrim` 只去空格,`trim` 连换行、制表符一起去。+//! schema.org 的描述结尾是 "\n ",这些行嵌完仍旧判陈——补齐任务一遍遍重嵌,抽取+//! 门控等一个永远补不齐的索引,文档到期判失败。钉三样:+//!+//! 1. **嵌过一轮就不陈**:描述结尾带换行和空格、label 首尾带制表符的类(整段与 label+//! 两份)和关系,`set_type_embeddings` 写回之后,`types_needing_embedding` 为空+//! 2. **改了描述照样重嵌**:判据没有因此变松+//! 3. **换了模型全部重嵌**+//!+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。++use sqlx::PgPool;+use utopia_store::ontology::{self, EmbedField, TypeKind};+use uuid::Uuid;++#[tokio::test]+async fn an_embedded_row_with_surrounding_whitespace_stays_embedded() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ let (place, located) = (Uuid::now_v7(), Uuid::now_v7());+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'trim-stale-test')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'trim-stale-test')")+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'trim-stale-test')",+ )+ .bind(kb)+ .bind(ws)+ .execute(&pool)+ .await?;++ let result = async {+ // schema.org 导进来的样子:描述结尾一个换行加一串空格;label 首尾再夹个制表符+ sqlx::query(+ "INSERT INTO entity_types (id, kb_id, key, label, description)+ VALUES ($1, $2, 'place', E'\\tPlace\\t', E'Entities that have a somewhat fixed, physical extension.\\n ')",+ )+ .bind(place)+ .bind(kb)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO relation_types (id, kb_id, key, label, description)+ VALUES ($1, $2, 'located_in', 'located in', E'\\n The place something is in.\\n')",+ )+ .bind(located)+ .bind(kb)+ .execute(&pool)+ .await?;++ let embed_all = |model: &'static str| {+ let pool = pool.clone();+ async move {+ let stale = ontology::types_needing_embedding(&pool, kb, model, None).await?;+ let items: Vec<_> = stale.iter().cloned().map(|t| (t, vec![0.1f32, 0.2, 0.3])).collect();+ ontology::set_type_embeddings(&pool, model, &items).await?;+ anyhow::Ok(stale)+ }+ };++ // ---- 一、第一轮三份都要嵌:类的整段与 label,关系的整段+ let first = embed_all("model-a").await?;+ let mut shape: Vec<(Uuid, &str, &str)> = first+ .iter()+ .map(|t| {+ (+ t.id,+ match t.kind {+ TypeKind::Entity => "entity",+ TypeKind::Relation => "relation",+ },+ match t.field {+ EmbedField::Full => "full",+ EmbedField::Label => "label",+ },+ )+ })+ .collect();+ shape.sort();+ let mut want = vec![+ (place, "entity", "full"),+ (place, "entity", "label"),+ (located, "relation", "full"),+ ];+ want.sort();+ assert_eq!(shape, want);+ let label = first+ .iter()+ .find(|t| t.field == EmbedField::Label)+ .map(|t| t.text.clone());+ assert_eq!(label.as_deref(), Some("Place"), "送去嵌的是去掉首尾空白的 label");++ // 嵌过就不陈:从前这里会把三份原样再端回来,一轮一轮永远补不齐+ let again = ontology::types_needing_embedding(&pool, kb, "model-a", None).await?;+ assert!(again.is_empty(), "嵌过的行又判成陈的:{again:?}");++ // ---- 二、改了描述,只那一份重嵌+ sqlx::query("UPDATE relation_types SET description = 'Where something is.' WHERE id = $1")+ .bind(located)+ .execute(&pool)+ .await?;+ let changed = ontology::types_needing_embedding(&pool, kb, "model-a", None).await?;+ assert_eq!(changed.len(), 1);+ assert_eq!(+ (changed[0].id, changed[0].text.as_str()),+ (located, "located in\nWhere something is.")+ );+ embed_all("model-a").await?;++ // ---- 三、换了模型,三份全部重嵌+ let switched = ontology::types_needing_embedding(&pool, kb, "model-b", None).await?;+ assert_eq!(switched.len(), 3);+ Ok::<_, anyhow::Error>(())+ }+ .await;+ sqlx::query("DELETE FROM organizations WHERE id = $1")+ .bind(org)+ .execute(&pool)+ .await?;+ result+}