World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
fix: keep Markdown name variants on the same resolution paths (#1299)
Nash Aschenbath8750760e6f2a
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rsindex 7894eb6d..6b7ae44f 100644--- a/crates/utopia-server/src/extraction.rs+++ b/crates/utopia-server/src/extraction.rs@@ -274,7 +274,7 @@ pub(crate) async fn resolve_handle(needs_adjudication: &mut bool,human_reviews_found: &mut bool,) -> anyhow::Result<Uuid> {- let normalized = utopia_store::resolution::normalize_name(name).to_lowercase();+ let normalized = utopia_store::resolution::name_key(name);let response_excluded = response_claims.get(&normalized).cloned()@@ -584,7 +584,7 @@ mod tests {&pool,kb,Some(person),- "Zhang Wei",+ "**Zhang Wei**",None,None,None,diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rsindex 0a4fa948..6dc1aee3 100644--- a/crates/utopia-server/src/extraction_open.rs+++ b/crates/utopia-server/src/extraction_open.rs@@ -34,6 +34,7 @@ use utopia_store::extraction_drops::reason;use utopia_store::graph::FactObject;use utopia_store::graph::Validity;use utopia_store::pending::{Outcome, Proposal};+pub(crate) use utopia_store::resolution::name_key;use uuid::Uuid;/// 一块调 + 解析完后的中间产物,apply 半步读它落库(#588 cut 1)。@@ -298,7 +299,7 @@ async fn apply_open_extraction().await;}- touched_names.insert(utopia_store::resolution::normalize_name(name).to_lowercase());+ touched_names.insert(name_key(name));if !doc_entities.iter().any(|(x, _, _)| *x == id) {doc_entities.push((id, kind.to_string(), name.to_string()));known_by_name.entry(key.clone()).or_insert(id);@@ -661,7 +662,7 @@ async fn apply_open_extraction(}// 一个名字不会同时是两样东西的名字:这一块或本文档前面已经把它认成// 另一个实体的,不记- let key = utopia_store::resolution::normalize_name(name).to_lowercase();+ let key = name_key(name);let claimed = handled_by_name.get(&key).is_some_and(|ids| ids.iter().any(|x| *x != id));@@ -895,13 +896,6 @@ async fn embed_pending_names(Ok(items.len())}-pub(crate) fn name_key(name: &str) -> String {- name.split_whitespace()- .collect::<Vec<_>>()- .join(" ")- .to_lowercase()-}-/// 陈述里写的名字 → 实体。先看这一块列出的,再看本文档前面认下的(提示词里的清单)。////// **被描述的东西用到才建。** 模型会把段落里每个名词短语都列进 `e`:实测一个 25 篇的库里diff --git a/crates/utopia-store/src/names.rs b/crates/utopia-store/src/names.rsindex 687e9694..63d04f0e 100644--- a/crates/utopia-store/src/names.rs+++ b/crates/utopia-store/src/names.rs@@ -13,7 +13,7 @@//! 一条「事实」,不能冒充证据,也不能把计数撑大。use crate::graph::{add_evidence, insert_value_fact, Validity};-use crate::resolution::normalize_name;+use crate::resolution::{fold_markdown_markers, normalize_name};use sqlx::PgPool;use std::collections::HashMap;use utopia_core::models::NameView;@@ -78,6 +78,7 @@ pub async fn record(attested_at: Option<chrono::DateTime<chrono::Utc>>,) -> AppResult<Option<Uuid>> {let name = normalize_name(name);+ let name = fold_markdown_markers(&name);if name.is_empty() {return Ok(None);}diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rsindex d51ea1b1..2262834a 100644--- a/crates/utopia-store/src/resolution.rs+++ b/crates/utopia-store/src/resolution.rs@@ -50,6 +50,54 @@ pub fn normalize_name(raw: &str) -> String {mapped.split_whitespace().collect::<Vec<_>>().join(" ")}+/// Fold unambiguous Markdown formatting in names; evidence keeps the original text.+/// Input should already have passed through [`normalize_name`]. Only whole-name+/// `**...**` / `__...__` emphasis is removed; empty, nested, mismatched or+/// whitespace-padded markup is kept.+/// `__identifier__` stays literal (Unicode letters/digits and underscores count).+/// Single backticks are removed only around a slash path or a dotted filename+/// containing a letter: only letters, digits, `/`, `.`, `_` and `-` are allowed,+/// with at least one alphanumeric character and no trailing dot. This deliberately+/// leaves code, bare identifiers, numeric versions and paths with spaces alone.+pub fn fold_markdown_markers(name: &str) -> &str {+ for marker in ["**", "__", "`"] {+ let Some(inner) = name+ .strip_prefix(marker)+ .and_then(|s| s.strip_suffix(marker))+ else {+ continue;+ };+ if inner.is_empty()+ || inner.trim() != inner+ || inner.contains(['*', '`'])+ || (marker != "`"+ && (inner.starts_with('_') || inner.ends_with('_') || inner.contains("__")))+ {+ return name;+ }+ let fold = match marker {+ "__" => !inner.chars().all(|c| c.is_alphanumeric() || c == '_'),+ "`" => {+ (inner.contains('/')+ || (inner.contains('.') && inner.chars().any(char::is_alphabetic)))+ && inner.chars().any(char::is_alphanumeric)+ && !inner.ends_with('.')+ && inner+ .chars()+ .all(|c| c.is_alphanumeric() || matches!(c, '/' | '.' | '_' | '-'))+ }+ _ => true,+ };+ return if fold { inner } else { name };+ }+ name+}++/// Shared identity key for recall and document-level name claims.+pub fn name_key(raw: &str) -> String {+ fold_markdown_markers(&normalize_name(raw)).to_lowercase()+}+/// 泛用后缀词表:中文直接拼在词干后;英文按独立词算、首尾两种语序都认/// ("Phoenix Project" / "Project Phoenix")。只影响召回,判定仍走画像相似度。const GENERIC_SUFFIXES_CJK: &[&str] = &["项目", "公司", "集团", "部门", "团队"];@@ -89,10 +137,12 @@ pub fn name_stem(name: &str) -> Option<String> {None}+/// 输入应已过 [`normalize_name`]。/// mention 的召回键集合(全 lower):本名 + 词干 + 词干的泛用后缀增广。/// 增广覆盖反方向(库内是"星尘项目"、mention 只说"星尘");词干含 CJK 拼中文-/// 尾缀,否则拼英文词(两种语序)。键数 ≤10,走 (kb,type,lower(name)) 索引多点查。-pub fn recall_keys(name: &str) -> Vec<String> {+/// 尾缀,否则拼英文词(两种语序)。键数 ≤11,走 (kb,type,lower(name)) 索引多点查。+pub fn recall_keys(normalized: &str) -> Vec<String> {+ let name = fold_markdown_markers(normalized);let lower = name.to_lowercase();let base = name_stem(name).unwrap_or_else(|| lower.clone());let mut keys = vec![lower];@@ -101,6 +151,8 @@ pub fn recall_keys(name: &str) -> Vec<String> {keys.push(k);}}+ // Keep exact recall of names already stored with formatting as well.+ add(&mut keys, normalized.to_lowercase());add(&mut keys, base.clone());if base.chars().any(|c| ('\u{4E00}'..='\u{9FFF}').contains(&c)) {for suf in GENERIC_SUFFIXES_CJK {@@ -320,7 +372,7 @@ async fn propose_by_name_vector(exclude: &[Uuid],r: &mut Resolution,) -> AppResult<()> {- let mention_name = normalize_name(raw_name).to_lowercase();+ let mention_name = name_key(raw_name);let mention_family = match type_id {Some(t) => type_label(pool, t).await?@@ -398,10 +450,12 @@ async fn resolve_by_name(// and recallable on later calls, but this mention cannot attach to them.exclude: &[Uuid],) -> AppResult<Resolution> {- let name = normalize_name(raw_name);+ let normalized = normalize_name(raw_name);// 召回键 = 本名 + 泛用后缀词干及其增广("星尘"↔"星尘项目"互为候选)。// 只扩召回,归并与否仍由下方画像相似度分层定夺。- let keys = recall_keys(&name);+ let keys = recall_keys(&normalized);+ // Store the same spelling whichever form is mentioned first.+ let name = fold_markdown_markers(&normalized);// 名字召回读名字事实(0041 决定 1):本名、简称、曾用名都是 `known_as` 上的一条。// 度数不数名字事实——名字不是一条「关于它的事」,数进去每个实体都凭空多一// 度数要等真有几个候选得比时才数:每个候选一条计数子查询,大库上很贵(#1296)@@ -430,7 +484,7 @@ async fn resolve_by_name(if candidates.is_empty() {// 同类型无候选 ≠ 新名字:类型标签会漂(同一团队被抽成 organization/project/// concept),先查其它类型下的同名实体,按类型对的互斥强度分流。- return resolve_type_drift(pool, kb_id, type_id, &name, &keys, context, exclude).await;+ return resolve_type_drift(pool, kb_id, type_id, name, &keys, context, exclude).await;}let Some(ctx) = context else {@@ -517,7 +571,7 @@ async fn resolve_by_name(}}let (id, created) =- create_entity(pool, kb_id, type_id, &name, context, &weighed).await?;+ create_entity(pool, kb_id, type_id, name, context, &weighed).await?;if !created {// 并行的另一份文档刚建好它:用它的,审核对也是它排的return Ok(Resolution {@@ -526,7 +580,7 @@ async fn resolve_by_name(reviews: Vec::new(),});}- refresh_disambiguators(pool, kb_id, &name).await?;+ refresh_disambiguators(pool, kb_id, name).await?;let reviews = [(best, sim), (runner, r_sim)].into_iter().map(|(c, s)| ReviewRequest {@@ -594,7 +648,7 @@ async fn resolve_by_name(});}}- let (id, created) = create_entity(pool, kb_id, type_id, &name, context, &weighed).await?;+ let (id, created) = create_entity(pool, kb_id, type_id, name, context, &weighed).await?;if !created {// 并行的另一份文档刚建好它:用它的,审核对也是它排的return Ok(Resolution {@@ -603,7 +657,7 @@ async fn resolve_by_name(reviews: Vec::new(),});}- refresh_disambiguators(pool, kb_id, &name).await?;+ refresh_disambiguators(pool, kb_id, name).await?;let mut reviews = best_scored.filter(|(_, sim)| *sim >= SIM_NEW).map(|(c, sim)| {@@ -617,7 +671,7 @@ async fn resolve_by_name(.unwrap_or_default();// 名字互相包含的既有实体:等值召回看不见它们(前缀枚举不完),// 于是简称会静默变成第二个实体。只入队,不合并- reviews.extend(containment_reviews(pool, kb_id, type_id, &name, id, context).await?);+ reviews.extend(containment_reviews(pool, kb_id, type_id, name, id, context).await?);Ok(Resolution {entity_id: id,created: true,@@ -2555,6 +2609,38 @@ mod tests {assert_eq!(normalize_name("张 三"), "张 三");}+ #[test]+ fn markdown_folding_is_conservative() {+ for (raw, expected) in [+ ("**Pump 1**", "Pump 1"),+ ("__Pump 1__", "Pump 1"),+ ("__init__", "__init__"),+ ("`docs/setup.md`", "docs/setup.md"),+ ("`setup.md`", "setup.md"),+ ("`acme`", "`acme`"),+ ("`main()`", "`main()`"),+ ("`1.2`", "`1.2`"),+ ("**Pump 1*", "**Pump 1*"),+ ("***Pump 1***", "***Pump 1***"),+ ("** Pump 1 **", "** Pump 1 **"),+ ("****", "****"),+ ] {+ assert_eq!(fold_markdown_markers(raw), expected, "{raw:?}");+ }+ }++ #[test]+ fn recall_keys_fold_markdown_before_stemming_and_keep_exact_spelling() {+ let normalized = normalize_name(" **Phoenix Project** ");+ let keys = recall_keys(&normalized);+ assert_eq!(keys[0], name_key(&normalized));+ for key in recall_keys("Phoenix Project") {+ assert!(keys.contains(&key), "must recall {key:?}");+ }+ assert!(keys.contains(&"**phoenix project**".to_string()));+ assert!(keys.len() <= 11);+ }+#[test]fn stem_generic_suffixes() {assert_eq!(name_stem("星尘项目").as_deref(), Some("星尘"));diff --git a/crates/utopia-store/tests/store/a_similar_name_is_proposed_not_merged.rs b/crates/utopia-store/tests/store/a_similar_name_is_proposed_not_merged.rsindex f013c07b..e23e1b86 100644--- a/crates/utopia-store/tests/store/a_similar_name_is_proposed_not_merged.rs+++ b/crates/utopia-store/tests/store/a_similar_name_is_proposed_not_merged.rs@@ -275,26 +275,40 @@ async fn the_same_literal_name_is_not_proposed_twice() -> anyhow::Result<()> {let f = seed(&pool, "name-vector-literal").await?;let run = async {let same: Vec<f32> = vec![1.0, 0.0, 0.0];- // 没有上下文向量:v1 路径归并到唯一的字面候选- let r = utopia_store::resolution::resolve_mention(- &pool,- f.kb,- Some(f.device),- "海洋探测器1号",- None,- Some(&same),- None,- &[],+ // A second entity with the same name stays a channel-1 decision, including+ // when the mention carries Markdown. Neither candidate is excluded here.+ sqlx::query(+ "UPDATE entities SET canonical_name = '海洋探测器1号', type_id = $2 WHERE id = $1",)+ .bind(f.captain)+ .bind(f.device)+ .execute(&pool).await?;- assert_eq!(r.entity_id, f.probe, "字面相等走通道 1");- // 夹具里 海洋探测队长 的向量与之相同、大类又拦不住,它照常被提议;要看的只是- // 已归并到的 海洋探测器1号 自己不会再被排一对- assert!(- !vector_reviews(&r).iter().any(|(id, _)| *id == f.probe),- "已经归并到它,不再对它排 name_vector:{:?}",- r.reviews- );+ sqlx::query(+ "UPDATE facts SET object_value = '{\"value\":\"海洋探测器1号\"}' WHERE subject_id = $1",+ )+ .bind(f.captain)+ .execute(&pool)+ .await?;+ for mention in ["海洋探测器1号", "**海洋探测器1号**"] {+ let r = utopia_store::resolution::resolve_mention(+ &pool,+ f.kb,+ Some(f.device),+ mention,+ None,+ Some(&same),+ None,+ &[],+ )+ .await?;+ assert!(!r.created);+ assert!(+ vector_reviews(&r).is_empty(),+ "a namesake must not be reconsidered by name vector: {:?}",+ r.reviews+ );+ }Ok::<_, anyhow::Error>(())}.await;diff --git a/crates/utopia-store/tests/store/an_undeclared_name_is_looked_up.rs b/crates/utopia-store/tests/store/an_undeclared_name_is_looked_up.rsindex d58949ca..e373a791 100644--- a/crates/utopia-store/tests/store/an_undeclared_name_is_looked_up.rs+++ b/crates/utopia-store/tests/store/an_undeclared_name_is_looked_up.rs@@ -105,3 +105,101 @@ async fn a_known_name_is_found_by_any_spelling_and_an_unknown_one_is_not() -> anteardown(&pool, &f).await?;run}++#[tokio::test]+async fn markdown_mentions_recall_names_in_either_order() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ let f = seed(&pool).await?;+ let run = async {+ // Open-graph extraction resolves names before they have an ontology type.+ sqlx::query("UPDATE entities SET type_id = NULL WHERE id = $1")+ .bind(f.acme)+ .execute(&pool)+ .await?;+ utopia_store::names::record(&pool, f.kb, f.acme, "`docs/setup.md`", None, None).await?;+ for mention in [+ "**Acme Corporation**",+ "__Acme Corporation__",+ "**ACME**",+ "docs/setup.md",+ "`docs/setup.md`",+ ] {+ assert_eq!(+ utopia_store::resolution::existing_by_name(&pool, f.kb, mention).await?,+ Some(f.acme),+ "{mention} must recall the existing name"+ );+ let resolved = utopia_store::resolution::resolve_mention(+ &pool,+ f.kb,+ None,+ mention,+ None,+ None,+ None,+ &[],+ )+ .await?;+ assert_eq!(resolved.entity_id, f.acme);+ assert!(!resolved.created);+ assert!(resolved.reviews.is_empty());+ }+ let canonical: String =+ sqlx::query_scalar("SELECT canonical_name FROM entities WHERE id = $1")+ .bind(f.acme)+ .fetch_one(&pool)+ .await?;+ assert_eq!(canonical, "Acme Corporation");++ // Marked-first names must remain recallable by later plain mentions.+ let written = "**Pump 2**";+ let created = utopia_store::resolution::resolve_mention(+ &pool,+ f.kb,+ None,+ written,+ None,+ None,+ None,+ &[],+ )+ .await?;+ assert!(created.created);+ let canonical: String =+ sqlx::query_scalar("SELECT canonical_name FROM entities WHERE id = $1")+ .bind(created.entity_id)+ .fetch_one(&pool)+ .await?;+ assert_eq!(canonical, "Pump 2");+ let recorded: String = sqlx::query_scalar(+ "SELECT object_value->>'value' FROM facts+ WHERE subject_id = $1 AND predicate_id = $2",+ )+ .bind(created.entity_id)+ .bind(utopia_store::names::ensure_known_as(&pool, f.kb).await?)+ .fetch_one(&pool)+ .await?;+ assert_eq!(recorded, "Pump 2");+ let recalled = utopia_store::resolution::resolve_mention(+ &pool,+ f.kb,+ None,+ "Pump 2",+ None,+ None,+ None,+ &[],+ )+ .await?;+ assert_eq!(recalled.entity_id, created.entity_id);+ assert!(!recalled.created);+ assert!(recalled.reviews.is_empty());+ anyhow::Ok(())+ }+ .await;+ teardown(&pool, &f).await?;+ run+}