Skip to content

World's first open-source enterprise world model.

fix: keep Markdown name variants on the same resolution paths (#1299)

Nash Aschenbath8750760e6f2a
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs
index 7894eb6d..6b7ae44f 100644
--- a/crates/utopia-server/src/extraction.rs
+++ b/crates/utopia-server/src/extraction.rs
@@ -274,7 +274,7 @@ pub(crate) async fn resolve_handle(
needs_adjudication: &mut bool,
human_reviews_found: &mut bool,
) -> anyhow::Result<Uuid> {
- let normalized = utopia_store::resolution::normalize_name(name).to_lowercase();
+ let normalized = utopia_store::resolution::name_key(name);
let response_excluded = response_claims
.get(&normalized)
.cloned()
@@ -584,7 +584,7 @@ mod tests {
&pool,
kb,
Some(person),
- "Zhang Wei",
+ "**Zhang Wei**",
None,
None,
None,
diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs
index 0a4fa948..6dc1aee3 100644
--- a/crates/utopia-server/src/extraction_open.rs
+++ b/crates/utopia-server/src/extraction_open.rs
@@ -34,6 +34,7 @@ use utopia_store::extraction_drops::reason;
use utopia_store::graph::FactObject;
use utopia_store::graph::Validity;
use utopia_store::pending::{Outcome, Proposal};
+pub(crate) use utopia_store::resolution::name_key;
use uuid::Uuid;
/// 一块调 + 解析完后的中间产物,apply 半步读它落库(#588 cut 1)。
@@ -298,7 +299,7 @@ async fn apply_open_extraction(
)
.await;
}
- touched_names.insert(utopia_store::resolution::normalize_name(name).to_lowercase());
+ touched_names.insert(name_key(name));
if !doc_entities.iter().any(|(x, _, _)| *x == id) {
doc_entities.push((id, kind.to_string(), name.to_string()));
known_by_name.entry(key.clone()).or_insert(id);
@@ -661,7 +662,7 @@ async fn apply_open_extraction(
}
// 一个名字不会同时是两样东西的名字:这一块或本文档前面已经把它认成
// 另一个实体的,不记
- let key = utopia_store::resolution::normalize_name(name).to_lowercase();
+ let key = name_key(name);
let claimed = handled_by_name
.get(&key)
.is_some_and(|ids| ids.iter().any(|x| *x != id));
@@ -895,13 +896,6 @@ async fn embed_pending_names(
Ok(items.len())
}
-pub(crate) fn name_key(name: &str) -> String {
- name.split_whitespace()
- .collect::<Vec<_>>()
- .join(" ")
- .to_lowercase()
-}
-
/// 陈述里写的名字 → 实体。先看这一块列出的,再看本文档前面认下的(提示词里的清单)。
///
/// **被描述的东西用到才建。** 模型会把段落里每个名词短语都列进 `e`:实测一个 25 篇的库里
diff --git a/crates/utopia-store/src/names.rs b/crates/utopia-store/src/names.rs
index 687e9694..63d04f0e 100644
--- a/crates/utopia-store/src/names.rs
+++ b/crates/utopia-store/src/names.rs
@@ -13,7 +13,7 @@
//! 一条「事实」,不能冒充证据,也不能把计数撑大。
use crate::graph::{add_evidence, insert_value_fact, Validity};
-use crate::resolution::normalize_name;
+use crate::resolution::{fold_markdown_markers, normalize_name};
use sqlx::PgPool;
use std::collections::HashMap;
use utopia_core::models::NameView;
@@ -78,6 +78,7 @@ pub async fn record(
attested_at: Option<chrono::DateTime<chrono::Utc>>,
) -> AppResult<Option<Uuid>> {
let name = normalize_name(name);
+ let name = fold_markdown_markers(&name);
if name.is_empty() {
return Ok(None);
}
diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs
index d51ea1b1..2262834a 100644
--- a/crates/utopia-store/src/resolution.rs
+++ b/crates/utopia-store/src/resolution.rs
@@ -50,6 +50,54 @@ pub fn normalize_name(raw: &str) -> String {
mapped.split_whitespace().collect::<Vec<_>>().join(" ")
}
+/// Fold unambiguous Markdown formatting in names; evidence keeps the original text.
+/// Input should already have passed through [`normalize_name`]. Only whole-name
+/// `**...**` / `__...__` emphasis is removed; empty, nested, mismatched or
+/// whitespace-padded markup is kept.
+/// `__identifier__` stays literal (Unicode letters/digits and underscores count).
+/// Single backticks are removed only around a slash path or a dotted filename
+/// containing a letter: only letters, digits, `/`, `.`, `_` and `-` are allowed,
+/// with at least one alphanumeric character and no trailing dot. This deliberately
+/// leaves code, bare identifiers, numeric versions and paths with spaces alone.
+pub fn fold_markdown_markers(name: &str) -> &str {
+ for marker in ["**", "__", "`"] {
+ let Some(inner) = name
+ .strip_prefix(marker)
+ .and_then(|s| s.strip_suffix(marker))
+ else {
+ continue;
+ };
+ if inner.is_empty()
+ || inner.trim() != inner
+ || inner.contains(['*', '`'])
+ || (marker != "`"
+ && (inner.starts_with('_') || inner.ends_with('_') || inner.contains("__")))
+ {
+ return name;
+ }
+ let fold = match marker {
+ "__" => !inner.chars().all(|c| c.is_alphanumeric() || c == '_'),
+ "`" => {
+ (inner.contains('/')
+ || (inner.contains('.') && inner.chars().any(char::is_alphabetic)))
+ && inner.chars().any(char::is_alphanumeric)
+ && !inner.ends_with('.')
+ && inner
+ .chars()
+ .all(|c| c.is_alphanumeric() || matches!(c, '/' | '.' | '_' | '-'))
+ }
+ _ => true,
+ };
+ return if fold { inner } else { name };
+ }
+ name
+}
+
+/// Shared identity key for recall and document-level name claims.
+pub fn name_key(raw: &str) -> String {
+ fold_markdown_markers(&normalize_name(raw)).to_lowercase()
+}
+
/// 泛用后缀词表:中文直接拼在词干后;英文按独立词算、首尾两种语序都认
/// ("Phoenix Project" / "Project Phoenix")。只影响召回,判定仍走画像相似度。
const GENERIC_SUFFIXES_CJK: &[&str] = &["项目", "公司", "集团", "部门", "团队"];
@@ -89,10 +137,12 @@ pub fn name_stem(name: &str) -> Option<String> {
None
}
+/// 输入应已过 [`normalize_name`]。
/// mention 的召回键集合(全 lower):本名 + 词干 + 词干的泛用后缀增广。
/// 增广覆盖反方向(库内是"星尘项目"、mention 只说"星尘");词干含 CJK 拼中文
-/// 尾缀,否则拼英文词(两种语序)。键数 ≤10,走 (kb,type,lower(name)) 索引多点查。
-pub fn recall_keys(name: &str) -> Vec<String> {
+/// 尾缀,否则拼英文词(两种语序)。键数 ≤11,走 (kb,type,lower(name)) 索引多点查。
+pub fn recall_keys(normalized: &str) -> Vec<String> {
+ let name = fold_markdown_markers(normalized);
let lower = name.to_lowercase();
let base = name_stem(name).unwrap_or_else(|| lower.clone());
let mut keys = vec![lower];
@@ -101,6 +151,8 @@ pub fn recall_keys(name: &str) -> Vec<String> {
keys.push(k);
}
}
+ // Keep exact recall of names already stored with formatting as well.
+ add(&mut keys, normalized.to_lowercase());
add(&mut keys, base.clone());
if base.chars().any(|c| ('\u{4E00}'..='\u{9FFF}').contains(&c)) {
for suf in GENERIC_SUFFIXES_CJK {
@@ -320,7 +372,7 @@ async fn propose_by_name_vector(
exclude: &[Uuid],
r: &mut Resolution,
) -> AppResult<()> {
- let mention_name = normalize_name(raw_name).to_lowercase();
+ let mention_name = name_key(raw_name);
let mention_family = match type_id {
Some(t) => type_label(pool, t)
.await?
@@ -398,10 +450,12 @@ async fn resolve_by_name(
// and recallable on later calls, but this mention cannot attach to them.
exclude: &[Uuid],
) -> AppResult<Resolution> {
- let name = normalize_name(raw_name);
+ let normalized = normalize_name(raw_name);
// 召回键 = 本名 + 泛用后缀词干及其增广("星尘"↔"星尘项目"互为候选)。
// 只扩召回,归并与否仍由下方画像相似度分层定夺。
- let keys = recall_keys(&name);
+ let keys = recall_keys(&normalized);
+ // Store the same spelling whichever form is mentioned first.
+ let name = fold_markdown_markers(&normalized);
// 名字召回读名字事实(0041 决定 1):本名、简称、曾用名都是 `known_as` 上的一条。
// 度数不数名字事实——名字不是一条「关于它的事」,数进去每个实体都凭空多一
// 度数要等真有几个候选得比时才数:每个候选一条计数子查询,大库上很贵(#1296)
@@ -430,7 +484,7 @@ async fn resolve_by_name(
if candidates.is_empty() {
// 同类型无候选 ≠ 新名字:类型标签会漂(同一团队被抽成 organization/project/
// concept),先查其它类型下的同名实体,按类型对的互斥强度分流。
- return resolve_type_drift(pool, kb_id, type_id, &name, &keys, context, exclude).await;
+ return resolve_type_drift(pool, kb_id, type_id, name, &keys, context, exclude).await;
}
let Some(ctx) = context else {
@@ -517,7 +571,7 @@ async fn resolve_by_name(
}
}
let (id, created) =
- create_entity(pool, kb_id, type_id, &name, context, &weighed).await?;
+ create_entity(pool, kb_id, type_id, name, context, &weighed).await?;
if !created {
// 并行的另一份文档刚建好它:用它的,审核对也是它排的
return Ok(Resolution {
@@ -526,7 +580,7 @@ async fn resolve_by_name(
reviews: Vec::new(),
});
}
- refresh_disambiguators(pool, kb_id, &name).await?;
+ refresh_disambiguators(pool, kb_id, name).await?;
let reviews = [(best, sim), (runner, r_sim)]
.into_iter()
.map(|(c, s)| ReviewRequest {
@@ -594,7 +648,7 @@ async fn resolve_by_name(
});
}
}
- let (id, created) = create_entity(pool, kb_id, type_id, &name, context, &weighed).await?;
+ let (id, created) = create_entity(pool, kb_id, type_id, name, context, &weighed).await?;
if !created {
// 并行的另一份文档刚建好它:用它的,审核对也是它排的
return Ok(Resolution {
@@ -603,7 +657,7 @@ async fn resolve_by_name(
reviews: Vec::new(),
});
}
- refresh_disambiguators(pool, kb_id, &name).await?;
+ refresh_disambiguators(pool, kb_id, name).await?;
let mut reviews = best_scored
.filter(|(_, sim)| *sim >= SIM_NEW)
.map(|(c, sim)| {
@@ -617,7 +671,7 @@ async fn resolve_by_name(
.unwrap_or_default();
// 名字互相包含的既有实体:等值召回看不见它们(前缀枚举不完),
// 于是简称会静默变成第二个实体。只入队,不合并
- reviews.extend(containment_reviews(pool, kb_id, type_id, &name, id, context).await?);
+ reviews.extend(containment_reviews(pool, kb_id, type_id, name, id, context).await?);
Ok(Resolution {
entity_id: id,
created: true,
@@ -2555,6 +2609,38 @@ mod tests {
assert_eq!(normalize_name("张 三"), "张 三");
}
+ #[test]
+ fn markdown_folding_is_conservative() {
+ for (raw, expected) in [
+ ("**Pump 1**", "Pump 1"),
+ ("__Pump 1__", "Pump 1"),
+ ("__init__", "__init__"),
+ ("`docs/setup.md`", "docs/setup.md"),
+ ("`setup.md`", "setup.md"),
+ ("`acme`", "`acme`"),
+ ("`main()`", "`main()`"),
+ ("`1.2`", "`1.2`"),
+ ("**Pump 1*", "**Pump 1*"),
+ ("***Pump 1***", "***Pump 1***"),
+ ("** Pump 1 **", "** Pump 1 **"),
+ ("****", "****"),
+ ] {
+ assert_eq!(fold_markdown_markers(raw), expected, "{raw:?}");
+ }
+ }
+
+ #[test]
+ fn recall_keys_fold_markdown_before_stemming_and_keep_exact_spelling() {
+ let normalized = normalize_name(" **Phoenix  Project** ");
+ let keys = recall_keys(&normalized);
+ assert_eq!(keys[0], name_key(&normalized));
+ for key in recall_keys("Phoenix Project") {
+ assert!(keys.contains(&key), "must recall {key:?}");
+ }
+ assert!(keys.contains(&"**phoenix project**".to_string()));
+ assert!(keys.len() <= 11);
+ }
+
#[test]
fn stem_generic_suffixes() {
assert_eq!(name_stem("星尘项目").as_deref(), Some("星尘"));
diff --git a/crates/utopia-store/tests/store/a_similar_name_is_proposed_not_merged.rs b/crates/utopia-store/tests/store/a_similar_name_is_proposed_not_merged.rs
index f013c07b..e23e1b86 100644
--- a/crates/utopia-store/tests/store/a_similar_name_is_proposed_not_merged.rs
+++ b/crates/utopia-store/tests/store/a_similar_name_is_proposed_not_merged.rs
@@ -275,26 +275,40 @@ async fn the_same_literal_name_is_not_proposed_twice() -> anyhow::Result<()> {
let f = seed(&pool, "name-vector-literal").await?;
let run = async {
let same: Vec<f32> = vec![1.0, 0.0, 0.0];
- // 没有上下文向量:v1 路径归并到唯一的字面候选
- let r = utopia_store::resolution::resolve_mention(
- &pool,
- f.kb,
- Some(f.device),
- "海洋探测器1号",
- None,
- Some(&same),
- None,
- &[],
+ // A second entity with the same name stays a channel-1 decision, including
+ // when the mention carries Markdown. Neither candidate is excluded here.
+ sqlx::query(
+ "UPDATE entities SET canonical_name = '海洋探测器1号', type_id = $2 WHERE id = $1",
)
+ .bind(f.captain)
+ .bind(f.device)
+ .execute(&pool)
.await?;
- assert_eq!(r.entity_id, f.probe, "字面相等走通道 1");
- // 夹具里 海洋探测队长 的向量与之相同、大类又拦不住,它照常被提议;要看的只是
- // 已归并到的 海洋探测器1号 自己不会再被排一对
- assert!(
- !vector_reviews(&r).iter().any(|(id, _)| *id == f.probe),
- "已经归并到它,不再对它排 name_vector:{:?}",
- r.reviews
- );
+ sqlx::query(
+ "UPDATE facts SET object_value = '{\"value\":\"海洋探测器1号\"}' WHERE subject_id = $1",
+ )
+ .bind(f.captain)
+ .execute(&pool)
+ .await?;
+ for mention in ["海洋探测器1号", "**海洋探测器1号**"] {
+ let r = utopia_store::resolution::resolve_mention(
+ &pool,
+ f.kb,
+ Some(f.device),
+ mention,
+ None,
+ Some(&same),
+ None,
+ &[],
+ )
+ .await?;
+ assert!(!r.created);
+ assert!(
+ vector_reviews(&r).is_empty(),
+ "a namesake must not be reconsidered by name vector: {:?}",
+ r.reviews
+ );
+ }
Ok::<_, anyhow::Error>(())
}
.await;
diff --git a/crates/utopia-store/tests/store/an_undeclared_name_is_looked_up.rs b/crates/utopia-store/tests/store/an_undeclared_name_is_looked_up.rs
index d58949ca..e373a791 100644
--- a/crates/utopia-store/tests/store/an_undeclared_name_is_looked_up.rs
+++ b/crates/utopia-store/tests/store/an_undeclared_name_is_looked_up.rs
@@ -105,3 +105,101 @@ async fn a_known_name_is_found_by_any_spelling_and_an_unknown_one_is_not() -> an
teardown(&pool, &f).await?;
run
}
+
+#[tokio::test]
+async fn markdown_mentions_recall_names_in_either_order() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let f = seed(&pool).await?;
+ let run = async {
+ // Open-graph extraction resolves names before they have an ontology type.
+ sqlx::query("UPDATE entities SET type_id = NULL WHERE id = $1")
+ .bind(f.acme)
+ .execute(&pool)
+ .await?;
+ utopia_store::names::record(&pool, f.kb, f.acme, "`docs/setup.md`", None, None).await?;
+ for mention in [
+ "**Acme Corporation**",
+ "__Acme Corporation__",
+ "**ACME**",
+ "docs/setup.md",
+ "`docs/setup.md`",
+ ] {
+ assert_eq!(
+ utopia_store::resolution::existing_by_name(&pool, f.kb, mention).await?,
+ Some(f.acme),
+ "{mention} must recall the existing name"
+ );
+ let resolved = utopia_store::resolution::resolve_mention(
+ &pool,
+ f.kb,
+ None,
+ mention,
+ None,
+ None,
+ None,
+ &[],
+ )
+ .await?;
+ assert_eq!(resolved.entity_id, f.acme);
+ assert!(!resolved.created);
+ assert!(resolved.reviews.is_empty());
+ }
+ let canonical: String =
+ sqlx::query_scalar("SELECT canonical_name FROM entities WHERE id = $1")
+ .bind(f.acme)
+ .fetch_one(&pool)
+ .await?;
+ assert_eq!(canonical, "Acme Corporation");
+
+ // Marked-first names must remain recallable by later plain mentions.
+ let written = "**Pump 2**";
+ let created = utopia_store::resolution::resolve_mention(
+ &pool,
+ f.kb,
+ None,
+ written,
+ None,
+ None,
+ None,
+ &[],
+ )
+ .await?;
+ assert!(created.created);
+ let canonical: String =
+ sqlx::query_scalar("SELECT canonical_name FROM entities WHERE id = $1")
+ .bind(created.entity_id)
+ .fetch_one(&pool)
+ .await?;
+ assert_eq!(canonical, "Pump 2");
+ let recorded: String = sqlx::query_scalar(
+ "SELECT object_value->>'value' FROM facts
+ WHERE subject_id = $1 AND predicate_id = $2",
+ )
+ .bind(created.entity_id)
+ .bind(utopia_store::names::ensure_known_as(&pool, f.kb).await?)
+ .fetch_one(&pool)
+ .await?;
+ assert_eq!(recorded, "Pump 2");
+ let recalled = utopia_store::resolution::resolve_mention(
+ &pool,
+ f.kb,
+ None,
+ "Pump 2",
+ None,
+ None,
+ None,
+ &[],
+ )
+ .await?;
+ assert_eq!(recalled.entity_id, created.entity_id);
+ assert!(!recalled.created);
+ assert!(recalled.reviews.is_empty());
+ anyhow::Ok(())
+ }
+ .await;
+ teardown(&pool, &f).await?;
+ run
+}