Skip to content

World's first open-source enterprise world model.

The entity-name guard reads structure, and records what it only suspects (#285)

WaylandYang60150efdfed7
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs
index 9b4b9567..eb6246fc 100644
--- a/crates/utopia-server/src/extraction.rs
+++ b/crates/utopia-server/src/extraction.rs
@@ -100,13 +100,70 @@ const CLAUSE_MARKERS: &[&str] = &[
"became", "went", "came", "did", "does", "gave", "took", "made",
];
+/// 情态动词:英语里**封闭类**的限定形式——不像 `fell`、`hits` 那样开放无边,也不会
+/// 出现在名词短语里。`may`(月份、人名)与 `can`(容器)除外,它们兼作名词(#193)
+const MODALS: &[&str] = &["could", "should", "might", "must", "shall"];
+
+/// 结构信号:不靠词表也看得出的「这是一句话」(#193)。
+///
+/// 动词表挡不住新语料——英语有上千个限定形式,`could`、`fell`、`hits` 都不在那 19 个
+/// 词里,而且第一个例句恰好 12 个词,卡在上限上。结构信号迁移得动:
+/// 1. **句号结尾**:末词是小写词并以句号收尾(`… as a whole.`)。专名缩写 `Inc.` /
+/// `Co.` 是大写开头,不误伤。
+/// 2. **情态动词**:封闭类,见 [`MODALS`]。
+///
+/// 这两条**当场拒绝**——它们和词数上限一样是结构判据,不是又一份词表。
+/// `words` 是小写化、去标点的词,`raw_last` 是保留原样的末词
+fn reads_like_a_sentence(words: &[String], raw_last: Option<&str>) -> bool {
+ if words.len() > 2 && words.iter().any(|w| MODALS.contains(&w.as_str())) {
+ return true;
+ }
+ if let Some(stem) = raw_last.and_then(|w| w.strip_suffix('.')) {
+ if stem.chars().count() >= 3 && stem.chars().all(|c| c.is_lowercase()) {
+ return true;
+ }
+ }
+ false
+}
+
+/// 弱信号:像从句,但不敢当场拒绝(#193)。
+///
+/// 守卫的样本全部来自一份语料,换一份就漏——换规则之前先要一份**跨语料的标注集**。
+/// 命中只记 `clause_suspect`(例句进 `extraction_drops`),实体照常落库;攒够两份语料的
+/// 样本再决定哪条升成硬规则。返回的是信号名,作为记录的 detail
+fn clause_suspect(name: &str) -> Option<&'static str> {
+ let words: Vec<String> = name
+ .split_whitespace()
+ .map(|w| {
+ w.trim_matches(|c: char| !c.is_alphanumeric())
+ .to_lowercase()
+ })
+ .collect();
+ // 限定词起头的长串:「The removal … industry as a whole」;机构名也会长,但
+ // 通常不以 the 起头(US District Court …),起头的(The New York Times)不长
+ if words.len() >= 8 && matches!(words[0].as_str(), "a" | "an" | "the") {
+ return Some("determiner_opens_a_long_string");
+ }
+ // 句中的关系词 / 从属连词:「the committee that reviewed …」
+ if words.len() >= 5
+ && words
+ .iter()
+ .skip(1)
+ .any(|w| matches!(w.as_str(), "that" | "which" | "who" | "because" | "while"))
+ {
+ return Some("relative_or_subordinate_clause");
+ }
+ None
+}
+
fn is_entity_name(name: &str) -> bool {
let name = name.trim();
if name.is_empty() || name.chars().count() > 100 {
return false;
}
- let words: Vec<String> = name
- .split_whitespace()
+ let raw: Vec<&str> = name.split_whitespace().collect();
+ let words: Vec<String> = raw
+ .iter()
.map(|w| {
w.trim_matches(|c: char| !c.is_alphanumeric())
.to_lowercase()
@@ -119,6 +176,9 @@ fn is_entity_name(name: &str) -> bool {
if words.len() > 2 && words.iter().any(|w| CLAUSE_MARKERS.contains(&w.as_str())) {
return false;
}
+ if reads_like_a_sentence(&words, raw.last().copied()) {
+ return false;
+ }
// 部分格:`745 of OpenAI's 770 employees` 是一个数量描述,不是一个东西。
// 它没有限定动词,词数也不多,上面两条都接不住它。
//
@@ -536,6 +596,18 @@ async fn run(state: &AppState, document_id: Uuid, proposed_by: Option<Uuid>) ->
.await;
continue;
}
+ // 守卫放行、结构却像从句:只记不挡(#193 先攒标注集)
+ if let Some(signal) = clause_suspect(name) {
+ drop_signal(
+ state,
+ doc.kb_id,
+ document_id,
+ utopia_store::extraction_drops::reason::CLAUSE_SUSPECT,
+ signal,
+ Some(name),
+ )
+ .await;
+ }
// 降级时记住模型提议的那个词:本体装不下不等于它说错了。
// 只留计数的话,日后想加 model 类就找不出那 43 个实体——它们混在
// concept 里面,唯一的出路是整库重抽
@@ -946,6 +1018,19 @@ async fn run(state: &AppState, document_id: Uuid, proposed_by: Option<Uuid>) ->
.await;
continue;
}
+ for side in [f.subject.trim(), object_name] {
+ if let Some(signal) = clause_suspect(side) {
+ drop_signal(
+ state,
+ doc.kb_id,
+ document_id,
+ utopia_store::extraction_drops::reason::CLAUSE_SUSPECT,
+ signal,
+ Some(side),
+ )
+ .await;
+ }
+ }
// 主宾未在 entities 中声明时先建出来(模型偶尔漏报)。没有 entities 那条
// 记录就没有类型可依,留空即可——0009 之前这里只能塞 concept
@@ -1535,7 +1620,7 @@ async fn chunk_lists(
#[cfg(test)]
mod name_tests {
- use super::is_entity_name;
+ use super::{clause_suspect, is_entity_name};
/// 样本全部取自实跑出来的库(ai-timeline-ends × schema.org),不是编的。
#[test]
@@ -1567,6 +1652,46 @@ mod name_tests {
}
}
+ /// #193:词表之外的句子,结构信号接住——句号结尾、情态动词。样本来自第二份语料
+ #[test]
+ fn a_sentence_is_caught_without_its_verb_on_the_list() {
+ for s in [
+ "The removal could slow down the artificial intelligence industry as a whole.",
+ "Shares in Microsoft fell nearly three percent following the announcement.",
+ "the board should reconsider its position",
+ ] {
+ assert!(!is_entity_name(s), "这是一句话,不该当成实体名:{s}");
+ }
+ // 大写缩写的句号、兼作名词的情态词,都不误伤
+ for s in [
+ "Safe Superintelligence Inc.",
+ "Theresa May",
+ "Trash Can Museum",
+ ] {
+ assert!(is_entity_name(s), "这是真实体,不该被挡:{s}");
+ }
+ }
+
+ /// 弱信号只记不挡:像从句的照常落库,但留下样本
+ #[test]
+ fn a_suspect_is_recorded_not_rejected() {
+ let s = "The committee that reviewed the merger of the two companies";
+ assert!(is_entity_name(s));
+ assert_eq!(clause_suspect(s), Some("determiner_opens_a_long_string"));
+ assert_eq!(
+ clause_suspect("committee members who reviewed the merger"),
+ Some("relative_or_subordinate_clause")
+ );
+ for s in [
+ "US District Court for the Northern District of California",
+ "OpenAI's board of directors",
+ "The New York Times",
+ "The Men Who Stare",
+ ] {
+ assert_eq!(clause_suspect(s), None, "{s} 不该被怀疑");
+ }
+ }
+
/// 分词在名词短语里完全正常,列进标志词会误伤。
#[test]
fn a_participle_is_not_a_predicate() {
diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rs
index cfcd132a..ce6c2a33 100644
--- a/crates/utopia-store/src/extraction_drops.rs
+++ b/crates/utopia-store/src/extraction_drops.rs
@@ -39,6 +39,9 @@ pub mod reason {
pub const DIRECTION_CORRECTED: &str = "direction_corrected";
/// 模型输出被截断(撞上 max_tokens)→ 已完整的那些留下,尾巴丢掉
pub const TRUNCATED_REPLY: &str = "truncated_reply";
+ /// 守卫放行了、结构却像从句(限定词起头的长串、句中的关系词)。**只记不挡**:
+ /// 实体照常落库,例句留下来——#193 要的是一份跨语料的标注集,再决定哪条升成硬规则
+ pub const CLAUSE_SUSPECT: &str = "clause_suspect";
}
pub async fn record(
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index 71d562ad..0d7e1009 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -441,6 +441,7 @@ export const en = {
domain_mismatch:
"The subject does not fit the relation, and swapping would not help",
not_an_entity_name: "That name is a sentence, not a thing",
+ clause_suspect: "Kept, but the name reads like a clause: a sample for the guard",
direction_corrected:
"Subject and object were swapped to match the signature",
} as Record<string, string>,
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index 59fb884b..b745297b 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -401,6 +401,7 @@ export const zh: Strings = {
truncated_reply: "模型的输出被截断",
domain_mismatch: "主语对不上这个关系,对调也不行",
not_an_entity_name: "那个名字是一句话,不是一个东西",
+ clause_suspect: "已保留,但这个名字像从句:给守卫攒的样本",
direction_corrected: "已按签名把主宾掰正",
},
reExtractSource: "重新抽取",