Skip to content

World's first open-source enterprise world model.

A trimmed description is not stale (#673)

WaylandYangc927c90d7073
diff --git a/crates/utopia-store/src/ontology.rs b/crates/utopia-store/src/ontology.rs
index 79a8c210..d96de7c2 100644
--- a/crates/utopia-store/src/ontology.rs
+++ b/crates/utopia-store/src/ontology.rs
@@ -1115,49 +1115,64 @@ pub async fn types_needing_embedding(
model: &str,
only: Option<TypeKind>,
) -> AppResult<Vec<TypeToEmbed>> {
- let mut out = Vec::new();
if only == Some(TypeKind::Relation) {
// 类型消解只用类,等 1633 个关系嵌完是白等六分钟
return relations_needing_embedding(pool, kb_id, model).await;
}
- let ents: Vec<(Uuid, String, String)> = sqlx::query_as(
- "SELECT id, label, coalesce(description, '') FROM entity_types
- WHERE kb_id = $1
- AND (embedding IS NULL
- OR embedded_model IS DISTINCT FROM $2
- OR embedded_text IS DISTINCT FROM
- CASE WHEN coalesce(btrim(description), '') = '' THEN btrim(label)
- ELSE btrim(label) || E'\n' || btrim(description) END)",
+ // **陈不陈,用生成原文的同一个函数判**(#672)。从前在 SQL 里用 `btrim` 把这段字
+ // 重拼一遍再比,而 `embed_text` 用的是 Rust 的 `trim`:`btrim` 只去空格,`trim`
+ // 连换行、制表符一起去。schema.org 的描述结尾是 "\n ",这些行存下的原文与
+ // SQL 拼出来的永远不等,于是每轮都判成陈的——补齐任务把同一批行一遍遍重嵌,
+ // 抽取门控(#526)等一个永远补不齐的索引,直到期限把文档判失败。
+ // 与 `mappings::needing_embedding` 同一个教训:拉回来在 Rust 里比
+ let rows: Vec<StoredEmbedding> = sqlx::query_as(
+ "SELECT id, label, coalesce(description, '') AS description,
+ embedding IS NOT NULL AS embedded, embedded_model, embedded_text,
+ label_embedding IS NOT NULL AS label_embedded, label_embedded_model, label_embedded_text
+ FROM entity_types
+ WHERE kb_id = $1",
)
.bind(kb_id)
- .bind(model)
.fetch_all(pool)
.await?;
- out.extend(ents.into_iter().map(|(id, label, desc)| TypeToEmbed {
- id,
- kind: TypeKind::Entity,
- text: embed_text(&label, &desc),
- field: EmbedField::Full,
- }));
- // 只嵌 label 的那一份(见 `entity_types.label_embedding`)。短查询走这个索引——查询分了两种形状,
- // 文档也得分两种,否则短查询被同义反复的类接管
- let labels: Vec<(Uuid, String)> = sqlx::query_as(
- "SELECT id, label FROM entity_types
- WHERE kb_id = $1
- AND (label_embedding IS NULL
- OR label_embedded_model IS DISTINCT FROM $2
- OR label_embedded_text IS DISTINCT FROM btrim(label))",
- )
- .bind(kb_id)
- .bind(model)
- .fetch_all(pool)
- .await?;
- out.extend(labels.into_iter().map(|(id, label)| TypeToEmbed {
- id,
- kind: TypeKind::Entity,
- text: label.trim().to_string(),
- field: EmbedField::Label,
- }));
+ let mut full = Vec::new();
+ let mut labels = Vec::new();
+ for r in rows {
+ let text = embed_text(&r.label, &r.description);
+ if is_stale(
+ r.embedded,
+ r.embedded_model.as_deref(),
+ r.embedded_text.as_deref(),
+ model,
+ &text,
+ ) {
+ full.push(TypeToEmbed {
+ id: r.id,
+ kind: TypeKind::Entity,
+ text,
+ field: EmbedField::Full,
+ });
+ }
+ // 只嵌 label 的那一份(见 `entity_types.label_embedding`)。短查询走这个索引——查询分了两种形状,
+ // 文档也得分两种,否则短查询被同义反复的类接管
+ let short = r.label.trim().to_string();
+ if is_stale(
+ r.label_embedded,
+ r.label_embedded_model.as_deref(),
+ r.label_embedded_text.as_deref(),
+ model,
+ &short,
+ ) {
+ labels.push(TypeToEmbed {
+ id: r.id,
+ kind: TypeKind::Entity,
+ text: short,
+ field: EmbedField::Label,
+ });
+ }
+ }
+ let mut out = full;
+ out.extend(labels);
if only == Some(TypeKind::Entity) {
return Ok(out);
}
@@ -1170,28 +1185,64 @@ async fn relations_needing_embedding(
kb_id: Uuid,
model: &str,
) -> AppResult<Vec<TypeToEmbed>> {
- let mut out = Vec::new();
- let rels: Vec<(Uuid, String, String)> = sqlx::query_as(
- "SELECT id, label, coalesce(description, '') FROM relation_types
- WHERE kb_id = $1
- AND (embedding IS NULL
- OR embedded_model IS DISTINCT FROM $2
- OR embedded_text IS DISTINCT FROM
- CASE WHEN coalesce(btrim(description), '') = '' THEN btrim(label)
- ELSE btrim(label) || E'\n' || btrim(description) END)",
+ // 判据同上(#672):在 Rust 里用 `embed_text` 比,不在 SQL 里重拼。关系只有整段
+ // 那一份向量,label 那三列补空
+ let rows: Vec<StoredEmbedding> = sqlx::query_as(
+ "SELECT id, label, coalesce(description, '') AS description,
+ embedding IS NOT NULL AS embedded, embedded_model, embedded_text,
+ false AS label_embedded, NULL::text AS label_embedded_model,
+ NULL::text AS label_embedded_text
+ FROM relation_types
+ WHERE kb_id = $1",
)
.bind(kb_id)
- .bind(model)
.fetch_all(pool)
.await?;
- out.extend(rels.into_iter().map(|(id, label, desc)| TypeToEmbed {
- id,
- kind: TypeKind::Relation,
- text: embed_text(&label, &desc),
- // 关系没有短查询那一路,只有整段这一份
- field: EmbedField::Full,
- }));
- Ok(out)
+ Ok(rows
+ .into_iter()
+ .filter_map(|r| {
+ let text = embed_text(&r.label, &r.description);
+ is_stale(
+ r.embedded,
+ r.embedded_model.as_deref(),
+ r.embedded_text.as_deref(),
+ model,
+ &text,
+ )
+ .then_some(TypeToEmbed {
+ id: r.id,
+ kind: TypeKind::Relation,
+ text,
+ // 关系没有短查询那一路,只有整段这一份
+ field: EmbedField::Full,
+ })
+ })
+ .collect())
+}
+
+/// 类型那一行上两份向量的现状,判陈用
+#[derive(sqlx::FromRow)]
+struct StoredEmbedding {
+ id: Uuid,
+ label: String,
+ description: String,
+ embedded: bool,
+ embedded_model: Option<String>,
+ embedded_text: Option<String>,
+ label_embedded: bool,
+ label_embedded_model: Option<String>,
+ label_embedded_text: Option<String>,
+}
+
+/// 没嵌过、换了模型、或者当时嵌的原文与现在要嵌的不一样
+fn is_stale(
+ embedded: bool,
+ stored_model: Option<&str>,
+ stored_text: Option<&str>,
+ model: &str,
+ text: &str,
+) -> bool {
+ !embedded || stored_model != Some(model) || stored_text != Some(text)
}
/// 回写向量,连同"嵌的是哪段字、用的哪个模型"。三者必须同一次写入——
diff --git a/crates/utopia-store/tests/a_trimmed_description_is_not_stale.rs b/crates/utopia-store/tests/a_trimmed_description_is_not_stale.rs
new file mode 100644
index 00000000..3fe25ae6
--- /dev/null
+++ b/crates/utopia-store/tests/a_trimmed_description_is_not_stale.rs
@@ -0,0 +1,133 @@
+//! 嵌过的本体行不因为描述首尾的换行就永远判成陈的(#672),打在真库上。
+//!
+//! 陈不陈看「当时嵌的原文」与「现在要嵌的原文」是否一致。从前后者在 SQL 里用 `btrim`
+//! 重拼,而原文由 Rust 的 `trim` 生成:`btrim` 只去空格,`trim` 连换行、制表符一起去。
+//! schema.org 的描述结尾是 "\n ",这些行嵌完仍旧判陈——补齐任务一遍遍重嵌,抽取
+//! 门控等一个永远补不齐的索引,文档到期判失败。钉三样:
+//!
+//! 1. **嵌过一轮就不陈**:描述结尾带换行和空格、label 首尾带制表符的类(整段与 label
+//! 两份)和关系,`set_type_embeddings` 写回之后,`types_needing_embedding` 为空
+//! 2. **改了描述照样重嵌**:判据没有因此变松
+//! 3. **换了模型全部重嵌**
+//!
+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
+
+use sqlx::PgPool;
+use utopia_store::ontology::{self, EmbedField, TypeKind};
+use uuid::Uuid;
+
+#[tokio::test]
+async fn an_embedded_row_with_surrounding_whitespace_stays_embedded() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ let (place, located) = (Uuid::now_v7(), Uuid::now_v7());
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'trim-stale-test')")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'trim-stale-test')")
+ .bind(ws)
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'trim-stale-test')",
+ )
+ .bind(kb)
+ .bind(ws)
+ .execute(&pool)
+ .await?;
+
+ let result = async {
+ // schema.org 导进来的样子:描述结尾一个换行加一串空格;label 首尾再夹个制表符
+ sqlx::query(
+ "INSERT INTO entity_types (id, kb_id, key, label, description)
+ VALUES ($1, $2, 'place', E'\\tPlace\\t', E'Entities that have a somewhat fixed, physical extension.\\n ')",
+ )
+ .bind(place)
+ .bind(kb)
+ .execute(&pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO relation_types (id, kb_id, key, label, description)
+ VALUES ($1, $2, 'located_in', 'located in', E'\\n The place something is in.\\n')",
+ )
+ .bind(located)
+ .bind(kb)
+ .execute(&pool)
+ .await?;
+
+ let embed_all = |model: &'static str| {
+ let pool = pool.clone();
+ async move {
+ let stale = ontology::types_needing_embedding(&pool, kb, model, None).await?;
+ let items: Vec<_> = stale.iter().cloned().map(|t| (t, vec![0.1f32, 0.2, 0.3])).collect();
+ ontology::set_type_embeddings(&pool, model, &items).await?;
+ anyhow::Ok(stale)
+ }
+ };
+
+ // ---- 一、第一轮三份都要嵌:类的整段与 label,关系的整段
+ let first = embed_all("model-a").await?;
+ let mut shape: Vec<(Uuid, &str, &str)> = first
+ .iter()
+ .map(|t| {
+ (
+ t.id,
+ match t.kind {
+ TypeKind::Entity => "entity",
+ TypeKind::Relation => "relation",
+ },
+ match t.field {
+ EmbedField::Full => "full",
+ EmbedField::Label => "label",
+ },
+ )
+ })
+ .collect();
+ shape.sort();
+ let mut want = vec![
+ (place, "entity", "full"),
+ (place, "entity", "label"),
+ (located, "relation", "full"),
+ ];
+ want.sort();
+ assert_eq!(shape, want);
+ let label = first
+ .iter()
+ .find(|t| t.field == EmbedField::Label)
+ .map(|t| t.text.clone());
+ assert_eq!(label.as_deref(), Some("Place"), "送去嵌的是去掉首尾空白的 label");
+
+ // 嵌过就不陈:从前这里会把三份原样再端回来,一轮一轮永远补不齐
+ let again = ontology::types_needing_embedding(&pool, kb, "model-a", None).await?;
+ assert!(again.is_empty(), "嵌过的行又判成陈的:{again:?}");
+
+ // ---- 二、改了描述,只那一份重嵌
+ sqlx::query("UPDATE relation_types SET description = 'Where something is.' WHERE id = $1")
+ .bind(located)
+ .execute(&pool)
+ .await?;
+ let changed = ontology::types_needing_embedding(&pool, kb, "model-a", None).await?;
+ assert_eq!(changed.len(), 1);
+ assert_eq!(
+ (changed[0].id, changed[0].text.as_str()),
+ (located, "located in\nWhere something is.")
+ );
+ embed_all("model-a").await?;
+
+ // ---- 三、换了模型,三份全部重嵌
+ let switched = ontology::types_needing_embedding(&pool, kb, "model-b", None).await?;
+ assert_eq!(switched.len(), 3);
+ Ok::<_, anyhow::Error>(())
+ }
+ .await;
+ sqlx::query("DELETE FROM organizations WHERE id = $1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ result
+}