World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
The entity-name guard reads structure, and records what it only suspects (#285)
WaylandYang60150efdfed7
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rsindex 9b4b9567..eb6246fc 100644--- a/crates/utopia-server/src/extraction.rs+++ b/crates/utopia-server/src/extraction.rs@@ -100,13 +100,70 @@ const CLAUSE_MARKERS: &[&str] = &["became", "went", "came", "did", "does", "gave", "took", "made",];+/// 情态动词:英语里**封闭类**的限定形式——不像 `fell`、`hits` 那样开放无边,也不会+/// 出现在名词短语里。`may`(月份、人名)与 `can`(容器)除外,它们兼作名词(#193)+const MODALS: &[&str] = &["could", "should", "might", "must", "shall"];++/// 结构信号:不靠词表也看得出的「这是一句话」(#193)。+///+/// 动词表挡不住新语料——英语有上千个限定形式,`could`、`fell`、`hits` 都不在那 19 个+/// 词里,而且第一个例句恰好 12 个词,卡在上限上。结构信号迁移得动:+/// 1. **句号结尾**:末词是小写词并以句号收尾(`… as a whole.`)。专名缩写 `Inc.` /+/// `Co.` 是大写开头,不误伤。+/// 2. **情态动词**:封闭类,见 [`MODALS`]。+///+/// 这两条**当场拒绝**——它们和词数上限一样是结构判据,不是又一份词表。+/// `words` 是小写化、去标点的词,`raw_last` 是保留原样的末词+fn reads_like_a_sentence(words: &[String], raw_last: Option<&str>) -> bool {+ if words.len() > 2 && words.iter().any(|w| MODALS.contains(&w.as_str())) {+ return true;+ }+ if let Some(stem) = raw_last.and_then(|w| w.strip_suffix('.')) {+ if stem.chars().count() >= 3 && stem.chars().all(|c| c.is_lowercase()) {+ return true;+ }+ }+ false+}++/// 弱信号:像从句,但不敢当场拒绝(#193)。+///+/// 守卫的样本全部来自一份语料,换一份就漏——换规则之前先要一份**跨语料的标注集**。+/// 命中只记 `clause_suspect`(例句进 `extraction_drops`),实体照常落库;攒够两份语料的+/// 样本再决定哪条升成硬规则。返回的是信号名,作为记录的 detail+fn clause_suspect(name: &str) -> Option<&'static str> {+ let words: Vec<String> = name+ .split_whitespace()+ .map(|w| {+ w.trim_matches(|c: char| !c.is_alphanumeric())+ .to_lowercase()+ })+ .collect();+ // 限定词起头的长串:「The removal … industry as a whole」;机构名也会长,但+ // 通常不以 the 起头(US District Court …),起头的(The New York Times)不长+ if words.len() >= 8 && matches!(words[0].as_str(), "a" | "an" | "the") {+ return Some("determiner_opens_a_long_string");+ }+ // 句中的关系词 / 从属连词:「the committee that reviewed …」+ if words.len() >= 5+ && words+ .iter()+ .skip(1)+ .any(|w| matches!(w.as_str(), "that" | "which" | "who" | "because" | "while"))+ {+ return Some("relative_or_subordinate_clause");+ }+ None+}+fn is_entity_name(name: &str) -> bool {let name = name.trim();if name.is_empty() || name.chars().count() > 100 {return false;}- let words: Vec<String> = name- .split_whitespace()+ let raw: Vec<&str> = name.split_whitespace().collect();+ let words: Vec<String> = raw+ .iter().map(|w| {w.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase()@@ -119,6 +176,9 @@ fn is_entity_name(name: &str) -> bool {if words.len() > 2 && words.iter().any(|w| CLAUSE_MARKERS.contains(&w.as_str())) {return false;}+ if reads_like_a_sentence(&words, raw.last().copied()) {+ return false;+ }// 部分格:`745 of OpenAI's 770 employees` 是一个数量描述,不是一个东西。// 它没有限定动词,词数也不多,上面两条都接不住它。//@@ -536,6 +596,18 @@ async fn run(state: &AppState, document_id: Uuid, proposed_by: Option<Uuid>) ->.await;continue;}+ // 守卫放行、结构却像从句:只记不挡(#193 先攒标注集)+ if let Some(signal) = clause_suspect(name) {+ drop_signal(+ state,+ doc.kb_id,+ document_id,+ utopia_store::extraction_drops::reason::CLAUSE_SUSPECT,+ signal,+ Some(name),+ )+ .await;+ }// 降级时记住模型提议的那个词:本体装不下不等于它说错了。// 只留计数的话,日后想加 model 类就找不出那 43 个实体——它们混在// concept 里面,唯一的出路是整库重抽@@ -946,6 +1018,19 @@ async fn run(state: &AppState, document_id: Uuid, proposed_by: Option<Uuid>) ->.await;continue;}+ for side in [f.subject.trim(), object_name] {+ if let Some(signal) = clause_suspect(side) {+ drop_signal(+ state,+ doc.kb_id,+ document_id,+ utopia_store::extraction_drops::reason::CLAUSE_SUSPECT,+ signal,+ Some(side),+ )+ .await;+ }+ }// 主宾未在 entities 中声明时先建出来(模型偶尔漏报)。没有 entities 那条// 记录就没有类型可依,留空即可——0009 之前这里只能塞 concept@@ -1535,7 +1620,7 @@ async fn chunk_lists(#[cfg(test)]mod name_tests {- use super::is_entity_name;+ use super::{clause_suspect, is_entity_name};/// 样本全部取自实跑出来的库(ai-timeline-ends × schema.org),不是编的。#[test]@@ -1567,6 +1652,46 @@ mod name_tests {}}+ /// #193:词表之外的句子,结构信号接住——句号结尾、情态动词。样本来自第二份语料+ #[test]+ fn a_sentence_is_caught_without_its_verb_on_the_list() {+ for s in [+ "The removal could slow down the artificial intelligence industry as a whole.",+ "Shares in Microsoft fell nearly three percent following the announcement.",+ "the board should reconsider its position",+ ] {+ assert!(!is_entity_name(s), "这是一句话,不该当成实体名:{s}");+ }+ // 大写缩写的句号、兼作名词的情态词,都不误伤+ for s in [+ "Safe Superintelligence Inc.",+ "Theresa May",+ "Trash Can Museum",+ ] {+ assert!(is_entity_name(s), "这是真实体,不该被挡:{s}");+ }+ }++ /// 弱信号只记不挡:像从句的照常落库,但留下样本+ #[test]+ fn a_suspect_is_recorded_not_rejected() {+ let s = "The committee that reviewed the merger of the two companies";+ assert!(is_entity_name(s));+ assert_eq!(clause_suspect(s), Some("determiner_opens_a_long_string"));+ assert_eq!(+ clause_suspect("committee members who reviewed the merger"),+ Some("relative_or_subordinate_clause")+ );+ for s in [+ "US District Court for the Northern District of California",+ "OpenAI's board of directors",+ "The New York Times",+ "The Men Who Stare",+ ] {+ assert_eq!(clause_suspect(s), None, "{s} 不该被怀疑");+ }+ }+/// 分词在名词短语里完全正常,列进标志词会误伤。#[test]fn a_participle_is_not_a_predicate() {diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rsindex cfcd132a..ce6c2a33 100644--- a/crates/utopia-store/src/extraction_drops.rs+++ b/crates/utopia-store/src/extraction_drops.rs@@ -39,6 +39,9 @@ pub mod reason {pub const DIRECTION_CORRECTED: &str = "direction_corrected";/// 模型输出被截断(撞上 max_tokens)→ 已完整的那些留下,尾巴丢掉pub const TRUNCATED_REPLY: &str = "truncated_reply";+ /// 守卫放行了、结构却像从句(限定词起头的长串、句中的关系词)。**只记不挡**:+ /// 实体照常落库,例句留下来——#193 要的是一份跨语料的标注集,再决定哪条升成硬规则+ pub const CLAUSE_SUSPECT: &str = "clause_suspect";}pub async fn record(diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex 71d562ad..0d7e1009 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -441,6 +441,7 @@ export const en = {domain_mismatch:"The subject does not fit the relation, and swapping would not help",not_an_entity_name: "That name is a sentence, not a thing",+ clause_suspect: "Kept, but the name reads like a clause: a sample for the guard",direction_corrected:"Subject and object were swapped to match the signature",} as Record<string, string>,diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 59fb884b..b745297b 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -401,6 +401,7 @@ export const zh: Strings = {truncated_reply: "模型的输出被截断",domain_mismatch: "主语对不上这个关系,对调也不行",not_an_entity_name: "那个名字是一句话,不是一个东西",+ clause_suspect: "已保留,但这个名字像从句:给守卫攒的样本",direction_corrected: "已按签名把主宾掰正",},reExtractSource: "重新抽取",