Skip to content

World's first open-source enterprise world model.

An undeclared name does not become a node (#568)

WaylandYang0f6ffb28e666
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs
index 26e624df..a739a6ea 100644
--- a/crates/utopia-server/src/extraction.rs
+++ b/crates/utopia-server/src/extraction.rs
@@ -1446,8 +1446,12 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:
}
}
- // 主宾未在 entities 中声明时先建出来(模型偶尔漏报)。没有 entities 那条
- // 记录就没有类型可依,留空即可——0009 之前这里只能塞 concept
+ // 主宾没在 entities 里声明时(模型偶尔漏报):库里已经有这个名字的就用它,
+ // 库里也没有的**不建**(#559)。从前这里一律先建出来、类型留空,结果
+ // 一个库里 20% 的实体是 "lawsuit against OpenAI"、"$5 billion"、"March 2024"
+ // 这样的描述——几乎全部只当过宾语,从没当过主语。漏报的实体多半在
+ // 前面的分块或别的文档里已经声明过,按名字找得到;找不到的就是描述。
+ // 描述做主语的事实丢掉并记账,做宾语的事实照落,宾语落成字面值
let subject_id = match f.subject_ref.as_deref().map(str::trim) {
Some(handle) => match referenced_entity(&ref_entities, handle) {
Some(bound) => bound.id,
@@ -1465,7 +1469,29 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:
}
},
None => {
- match no_ref_name_binding(&entity_ids, &handled_by_name, f.subject.trim()) {
+ let binding =
+ no_ref_name_binding(&entity_ids, &handled_by_name, f.subject.trim());
+ if matches!(binding, NoRefNameBinding::Missing)
+ && utopia_store::resolution::existing_by_name(
+ &state.pool,
+ doc.kb_id,
+ f.subject.trim(),
+ )
+ .await?
+ .is_none()
+ {
+ drop_signal(
+ state,
+ doc.kb_id,
+ document_id,
+ utopia_store::extraction_drops::reason::SUBJECT_NOT_DECLARED,
+ &f.predicate,
+ Some(f.subject.trim()),
+ )
+ .await;
+ continue;
+ }
+ match binding {
NoRefNameBinding::Legacy(id) => id,
NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {
touched_names.insert(
@@ -1506,28 +1532,117 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:
continue;
}
},
- None => match no_ref_name_binding(&entity_ids, &handled_by_name, object_name) {
- NoRefNameBinding::Legacy(id) => id,
- NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {
- touched_names.insert(
- utopia_store::resolution::normalize_name(object_name).to_lowercase(),
- );
- resolve_bare(
+ None => {
+ let binding = no_ref_name_binding(&entity_ids, &handled_by_name, object_name);
+ if matches!(binding, NoRefNameBinding::Missing)
+ && utopia_store::resolution::existing_by_name(
&state.pool,
doc.kb_id,
- None,
object_name,
- ctx,
- Some(&chunk.text),
- &mut doc_cache,
- &handled_by_name,
- &mut ambiguous_bare_cache,
- &mut needs_adjudication,
- &mut human_reviews_found,
)
.await?
+ .is_none()
+ {
+ // 没声明、库里也没有:宾语落成字面值。谓词照旧对本体;
+ // 被动形(`_by`)本该主宾对调,而字面值当不了主语,那条就不给谓词,
+ // 原词留在证据上
+ let predicate_id = match pred_index.lookup(f.predicate.as_str()) {
+ Some((id, false)) => Some(id),
+ Some((_, true)) => None,
+ None => {
+ let _ = utopia_store::ontology::record_miss(
+ &state.pool,
+ doc.kb_id,
+ "relation_type",
+ &f.predicate,
+ Some(&format!("{} → {}", f.subject, object_name)),
+ )
+ .await;
+ None
+ }
+ };
+ drop_signal(
+ state,
+ doc.kb_id,
+ document_id,
+ utopia_store::extraction_drops::reason::OBJECT_UNDECLARED,
+ &f.predicate,
+ Some(object_name),
+ )
+ .await;
+ let literal = serde_json::json!({ "value": object_name });
+ if await_nod {
+ if let utopia_store::pending::Outcome::Proposed(_) =
+ utopia_store::pending::propose(
+ &state.pool,
+ utopia_store::pending::Proposal {
+ kb_id: doc.kb_id,
+ subject_id,
+ predicate_id,
+ object_id: None,
+ object_value: Some(&literal),
+ proposed_predicate: Some(f.predicate.as_str()),
+ validity,
+ confidence,
+ chunk_id: chunk.id,
+ proposed_by: proposer.user_id,
+ proposed_token: proposer.token_id,
+ },
+ )
+ .await?
+ {
+ pending_count += 1;
+ }
+ continue;
+ }
+ let (fact_id, created) = utopia_store::graph::insert_value_fact(
+ &state.pool,
+ doc.kb_id,
+ subject_id,
+ predicate_id,
+ &literal,
+ validity,
+ confidence,
+ )
+ .await?;
+ touched_facts.push(fact_id);
+ utopia_store::graph::add_evidence(
+ &state.pool,
+ fact_id,
+ chunk.id,
+ f.quote.as_deref(),
+ Some(f.predicate.as_str()),
+ )
+ .await?;
+ if created {
+ fact_count += 1;
+ }
+ continue;
}
- },
+ match binding {
+ NoRefNameBinding::Legacy(id) => id,
+ NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {
+ touched_names.insert(
+ utopia_store::resolution::normalize_name(object_name)
+ .to_lowercase(),
+ );
+ resolve_bare(
+ &state.pool,
+ doc.kb_id,
+ None,
+ object_name,
+ ctx,
+ Some(&chunk.text),
+ &mut doc_cache,
+ &handled_by_name,
+ &mut ambiguous_bare_cache,
+ &mut needs_adjudication,
+ &mut human_reviews_found,
+ )
+ .await?
+ }
+ }
+ }
};
if subject_id == object_id {
continue;
diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rs
index ce6c2a33..770d20c1 100644
--- a/crates/utopia-store/src/extraction_drops.rs
+++ b/crates/utopia-store/src/extraction_drops.rs
@@ -42,6 +42,10 @@ pub mod reason {
/// 守卫放行了、结构却像从句(限定词起头的长串、句中的关系词)。**只记不挡**:
/// 实体照常落库,例句留下来——#193 要的是一份跨语料的标注集,再决定哪条升成硬规则
pub const CLAUSE_SUSPECT: &str = "clause_suspect";
+ /// 宾语既没在 entities 里声明、库里也没有叫这个名字的东西:事实照落,
+ /// 宾语落成字面值而不是节点(#559)。记下来是为了量:这一类里有多少
+ /// 本该是实体(模型漏报),有多少本来就是描述
+ pub const OBJECT_UNDECLARED: &str = "object_undeclared";
}
pub async fn record(
diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs
index f29ad63a..4d86bd41 100644
--- a/crates/utopia-store/src/resolution.rs
+++ b/crates/utopia-store/src/resolution.rs
@@ -1001,6 +1001,35 @@ async fn update_profile(pool: &PgPool, id: Uuid, n: i32, ctx: &[f32]) -> AppResu
///
/// 没有谓词的事实(`predicate_id IS NULL`,0010)不参与:原话留在证据里,
/// 不是本体承认的说法,不该被当成一个人的身份写进后缀。
+/// 库里有没有叫这个名字的实体,**不问类型**、并掉的不算。
+///
+/// 抽取用它判断一个没声明的主宾是不是已知的东西(#559):模型偶尔漏报一个实体
+/// 却在事实里用了它,那时库里多半已经有它;库里也没有的,就不是漏报,是一个
+/// 描述("lawsuit against OpenAI"),不该成节点。同名多个时取事实最多的那个——
+/// 这里只回答「有没有」,谁是谁交给消解
+pub async fn existing_by_name(
+ pool: &PgPool,
+ kb_id: Uuid,
+ raw_name: &str,
+) -> AppResult<Option<Uuid>> {
+ let name = normalize_name(raw_name);
+ let keys = recall_keys(&name);
+ let id: Option<Uuid> = sqlx::query_scalar(
+ "SELECT e.id FROM entities e
+ WHERE e.kb_id = $1 AND e.merged_into IS NULL
+ AND (lower(e.canonical_name) = ANY($2)
+ OR EXISTS (SELECT 1 FROM unnest(e.aliases) a WHERE lower(a) = ANY($2)))
+ ORDER BY (SELECT count(*) FROM facts f
+ WHERE f.subject_id = e.id OR f.object_id = e.id) DESC, e.created_at
+ LIMIT 1",
+ )
+ .bind(kb_id)
+ .bind(&keys)
+ .fetch_optional(pool)
+ .await?;
+ Ok(id)
+}
+
pub async fn refresh_disambiguators(pool: &PgPool, kb_id: Uuid, name: &str) -> AppResult<()> {
let group: Vec<(Uuid,)> = sqlx::query_as(
"SELECT id FROM entities
diff --git a/crates/utopia-store/tests/an_undeclared_name_is_looked_up.rs b/crates/utopia-store/tests/an_undeclared_name_is_looked_up.rs
new file mode 100644
index 00000000..21d0f864
--- /dev/null
+++ b/crates/utopia-store/tests/an_undeclared_name_is_looked_up.rs
@@ -0,0 +1,104 @@
+//! `existing_by_name`(#559):一个没声明的主宾是不是库里已经有的东西。
+//! 不问类型,认别名,并掉的不算;没有就是没有。
+
+use sqlx::PgPool;
+use uuid::Uuid;
+
+struct Fixture {
+ org: Uuid,
+ kb: Uuid,
+ acme: Uuid,
+ merged: Uuid,
+}
+
+async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {
+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ let (organization, acme, merged) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'lookup-test')")
+ .bind(org)
+ .execute(pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'lookup-test')")
+ .bind(ws)
+ .bind(org)
+ .execute(pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'lookup-test')",
+ )
+ .bind(kb)
+ .bind(ws)
+ .execute(pool)
+ .await?;
+ sqlx::query("INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, 'organization', 'Organization')")
+ .bind(organization)
+ .bind(kb)
+ .execute(pool)
+ .await?;
+ // 有类型的、带别名的一个;并掉的一个(同名,不该被找到)
+ sqlx::query(
+ "INSERT INTO entities (id, kb_id, type_id, canonical_name, aliases)
+ VALUES ($1, $2, $3, 'Acme Corporation', ARRAY['ACME'])",
+ )
+ .bind(acme)
+ .bind(kb)
+ .bind(organization)
+ .execute(pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO entities (id, kb_id, canonical_name, merged_into) VALUES ($1, $2, 'Nova Labs', $3)",
+ )
+ .bind(merged)
+ .bind(kb)
+ .bind(acme)
+ .execute(pool)
+ .await?;
+ Ok(Fixture {
+ org,
+ kb,
+ acme,
+ merged,
+ })
+}
+
+async fn teardown(pool: &PgPool, f: &Fixture) -> anyhow::Result<()> {
+ sqlx::query("DELETE FROM knowledge_bases WHERE id = $1")
+ .bind(f.kb)
+ .execute(pool)
+ .await?;
+ sqlx::query("DELETE FROM organizations WHERE id = $1")
+ .bind(f.org)
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn a_known_name_is_found_by_any_spelling_and_an_unknown_one_is_not() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let f = seed(&pool).await?;
+ let run = async {
+ let find = |name: &'static str| {
+ let pool = pool.clone();
+ async move { utopia_store::resolution::existing_by_name(&pool, f.kb, name).await }
+ };
+ assert_eq!(find("Acme Corporation").await?, Some(f.acme), "原名");
+ assert_eq!(find("acme corporation").await?, Some(f.acme), "大小写不论");
+ assert_eq!(find("ACME").await?, Some(f.acme), "别名也算");
+ assert_eq!(
+ find("Acme").await?,
+ Some(f.acme),
+ "泛用后缀词干互推:Acme ↔ Acme Corporation"
+ );
+ assert_eq!(find("Nova Labs").await?, None, "并掉的不算");
+ assert_eq!(find("lawsuit against Acme").await?, None, "描述不是实体");
+ let _ = f.merged;
+ anyhow::Ok(())
+ }
+ .await;
+ teardown(&pool, &f).await?;
+ run
+}