World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
An undeclared name does not become a node (#568)
WaylandYang0f6ffb28e666
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rsindex 26e624df..a739a6ea 100644--- a/crates/utopia-server/src/extraction.rs+++ b/crates/utopia-server/src/extraction.rs@@ -1446,8 +1446,12 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:}}- // 主宾未在 entities 中声明时先建出来(模型偶尔漏报)。没有 entities 那条- // 记录就没有类型可依,留空即可——0009 之前这里只能塞 concept+ // 主宾没在 entities 里声明时(模型偶尔漏报):库里已经有这个名字的就用它,+ // 库里也没有的**不建**(#559)。从前这里一律先建出来、类型留空,结果+ // 一个库里 20% 的实体是 "lawsuit against OpenAI"、"$5 billion"、"March 2024"+ // 这样的描述——几乎全部只当过宾语,从没当过主语。漏报的实体多半在+ // 前面的分块或别的文档里已经声明过,按名字找得到;找不到的就是描述。+ // 描述做主语的事实丢掉并记账,做宾语的事实照落,宾语落成字面值let subject_id = match f.subject_ref.as_deref().map(str::trim) {Some(handle) => match referenced_entity(&ref_entities, handle) {Some(bound) => bound.id,@@ -1465,7 +1469,29 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:}},None => {- match no_ref_name_binding(&entity_ids, &handled_by_name, f.subject.trim()) {+ let binding =+ no_ref_name_binding(&entity_ids, &handled_by_name, f.subject.trim());+ if matches!(binding, NoRefNameBinding::Missing)+ && utopia_store::resolution::existing_by_name(+ &state.pool,+ doc.kb_id,+ f.subject.trim(),+ )+ .await?+ .is_none()+ {+ drop_signal(+ state,+ doc.kb_id,+ document_id,+ utopia_store::extraction_drops::reason::SUBJECT_NOT_DECLARED,+ &f.predicate,+ Some(f.subject.trim()),+ )+ .await;+ continue;+ }+ match binding {NoRefNameBinding::Legacy(id) => id,NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {touched_names.insert(@@ -1506,28 +1532,117 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:continue;}},- None => match no_ref_name_binding(&entity_ids, &handled_by_name, object_name) {- NoRefNameBinding::Legacy(id) => id,- NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {- touched_names.insert(- utopia_store::resolution::normalize_name(object_name).to_lowercase(),- );- resolve_bare(+ None => {+ let binding = no_ref_name_binding(&entity_ids, &handled_by_name, object_name);+ if matches!(binding, NoRefNameBinding::Missing)+ && utopia_store::resolution::existing_by_name(&state.pool,doc.kb_id,- None,object_name,- ctx,- Some(&chunk.text),- &mut doc_cache,- &handled_by_name,- &mut ambiguous_bare_cache,- &mut needs_adjudication,- &mut human_reviews_found,).await?+ .is_none()+ {+ // 没声明、库里也没有:宾语落成字面值。谓词照旧对本体;+ // 被动形(`_by`)本该主宾对调,而字面值当不了主语,那条就不给谓词,+ // 原词留在证据上+ let predicate_id = match pred_index.lookup(f.predicate.as_str()) {+ Some((id, false)) => Some(id),+ Some((_, true)) => None,+ None => {+ let _ = utopia_store::ontology::record_miss(+ &state.pool,+ doc.kb_id,+ "relation_type",+ &f.predicate,+ Some(&format!("{} → {}", f.subject, object_name)),+ )+ .await;+ None+ }+ };+ drop_signal(+ state,+ doc.kb_id,+ document_id,+ utopia_store::extraction_drops::reason::OBJECT_UNDECLARED,+ &f.predicate,+ Some(object_name),+ )+ .await;+ let literal = serde_json::json!({ "value": object_name });+ if await_nod {+ if let utopia_store::pending::Outcome::Proposed(_) =+ utopia_store::pending::propose(+ &state.pool,+ utopia_store::pending::Proposal {+ kb_id: doc.kb_id,+ subject_id,+ predicate_id,+ object_id: None,+ object_value: Some(&literal),+ proposed_predicate: Some(f.predicate.as_str()),+ validity,+ confidence,+ chunk_id: chunk.id,+ proposed_by: proposer.user_id,+ proposed_token: proposer.token_id,+ },+ )+ .await?+ {+ pending_count += 1;+ }+ continue;+ }+ let (fact_id, created) = utopia_store::graph::insert_value_fact(+ &state.pool,+ doc.kb_id,+ subject_id,+ predicate_id,+ &literal,+ validity,+ confidence,+ )+ .await?;+ touched_facts.push(fact_id);+ utopia_store::graph::add_evidence(+ &state.pool,+ fact_id,+ chunk.id,+ f.quote.as_deref(),+ Some(f.predicate.as_str()),+ )+ .await?;+ if created {+ fact_count += 1;+ }+ continue;}- },+ match binding {+ NoRefNameBinding::Legacy(id) => id,+ NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {+ touched_names.insert(+ utopia_store::resolution::normalize_name(object_name)+ .to_lowercase(),+ );+ resolve_bare(+ &state.pool,+ doc.kb_id,+ None,+ object_name,+ ctx,+ Some(&chunk.text),+ &mut doc_cache,+ &handled_by_name,+ &mut ambiguous_bare_cache,+ &mut needs_adjudication,+ &mut human_reviews_found,+ )+ .await?+ }+ }+ }};if subject_id == object_id {continue;diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rsindex ce6c2a33..770d20c1 100644--- a/crates/utopia-store/src/extraction_drops.rs+++ b/crates/utopia-store/src/extraction_drops.rs@@ -42,6 +42,10 @@ pub mod reason {/// 守卫放行了、结构却像从句(限定词起头的长串、句中的关系词)。**只记不挡**:/// 实体照常落库,例句留下来——#193 要的是一份跨语料的标注集,再决定哪条升成硬规则pub const CLAUSE_SUSPECT: &str = "clause_suspect";+ /// 宾语既没在 entities 里声明、库里也没有叫这个名字的东西:事实照落,+ /// 宾语落成字面值而不是节点(#559)。记下来是为了量:这一类里有多少+ /// 本该是实体(模型漏报),有多少本来就是描述+ pub const OBJECT_UNDECLARED: &str = "object_undeclared";}pub async fn record(diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rsindex f29ad63a..4d86bd41 100644--- a/crates/utopia-store/src/resolution.rs+++ b/crates/utopia-store/src/resolution.rs@@ -1001,6 +1001,35 @@ async fn update_profile(pool: &PgPool, id: Uuid, n: i32, ctx: &[f32]) -> AppResu////// 没有谓词的事实(`predicate_id IS NULL`,0010)不参与:原话留在证据里,/// 不是本体承认的说法,不该被当成一个人的身份写进后缀。+/// 库里有没有叫这个名字的实体,**不问类型**、并掉的不算。+///+/// 抽取用它判断一个没声明的主宾是不是已知的东西(#559):模型偶尔漏报一个实体+/// 却在事实里用了它,那时库里多半已经有它;库里也没有的,就不是漏报,是一个+/// 描述("lawsuit against OpenAI"),不该成节点。同名多个时取事实最多的那个——+/// 这里只回答「有没有」,谁是谁交给消解+pub async fn existing_by_name(+ pool: &PgPool,+ kb_id: Uuid,+ raw_name: &str,+) -> AppResult<Option<Uuid>> {+ let name = normalize_name(raw_name);+ let keys = recall_keys(&name);+ let id: Option<Uuid> = sqlx::query_scalar(+ "SELECT e.id FROM entities e+ WHERE e.kb_id = $1 AND e.merged_into IS NULL+ AND (lower(e.canonical_name) = ANY($2)+ OR EXISTS (SELECT 1 FROM unnest(e.aliases) a WHERE lower(a) = ANY($2)))+ ORDER BY (SELECT count(*) FROM facts f+ WHERE f.subject_id = e.id OR f.object_id = e.id) DESC, e.created_at+ LIMIT 1",+ )+ .bind(kb_id)+ .bind(&keys)+ .fetch_optional(pool)+ .await?;+ Ok(id)+}+pub async fn refresh_disambiguators(pool: &PgPool, kb_id: Uuid, name: &str) -> AppResult<()> {let group: Vec<(Uuid,)> = sqlx::query_as("SELECT id FROM entitiesdiff --git a/crates/utopia-store/tests/an_undeclared_name_is_looked_up.rs b/crates/utopia-store/tests/an_undeclared_name_is_looked_up.rsnew file mode 100644index 00000000..21d0f864--- /dev/null+++ b/crates/utopia-store/tests/an_undeclared_name_is_looked_up.rs@@ -0,0 +1,104 @@+//! `existing_by_name`(#559):一个没声明的主宾是不是库里已经有的东西。+//! 不问类型,认别名,并掉的不算;没有就是没有。++use sqlx::PgPool;+use uuid::Uuid;++struct Fixture {+ org: Uuid,+ kb: Uuid,+ acme: Uuid,+ merged: Uuid,+}++async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ let (organization, acme, merged) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'lookup-test')")+ .bind(org)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'lookup-test')")+ .bind(ws)+ .bind(org)+ .execute(pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'lookup-test')",+ )+ .bind(kb)+ .bind(ws)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, 'organization', 'Organization')")+ .bind(organization)+ .bind(kb)+ .execute(pool)+ .await?;+ // 有类型的、带别名的一个;并掉的一个(同名,不该被找到)+ sqlx::query(+ "INSERT INTO entities (id, kb_id, type_id, canonical_name, aliases)+ VALUES ($1, $2, $3, 'Acme Corporation', ARRAY['ACME'])",+ )+ .bind(acme)+ .bind(kb)+ .bind(organization)+ .execute(pool)+ .await?;+ sqlx::query(+ "INSERT INTO entities (id, kb_id, canonical_name, merged_into) VALUES ($1, $2, 'Nova Labs', $3)",+ )+ .bind(merged)+ .bind(kb)+ .bind(acme)+ .execute(pool)+ .await?;+ Ok(Fixture {+ org,+ kb,+ acme,+ merged,+ })+}++async fn teardown(pool: &PgPool, f: &Fixture) -> anyhow::Result<()> {+ sqlx::query("DELETE FROM knowledge_bases WHERE id = $1")+ .bind(f.kb)+ .execute(pool)+ .await?;+ sqlx::query("DELETE FROM organizations WHERE id = $1")+ .bind(f.org)+ .execute(pool)+ .await?;+ Ok(())+}++#[tokio::test]+async fn a_known_name_is_found_by_any_spelling_and_an_unknown_one_is_not() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ let f = seed(&pool).await?;+ let run = async {+ let find = |name: &'static str| {+ let pool = pool.clone();+ async move { utopia_store::resolution::existing_by_name(&pool, f.kb, name).await }+ };+ assert_eq!(find("Acme Corporation").await?, Some(f.acme), "原名");+ assert_eq!(find("acme corporation").await?, Some(f.acme), "大小写不论");+ assert_eq!(find("ACME").await?, Some(f.acme), "别名也算");+ assert_eq!(+ find("Acme").await?,+ Some(f.acme),+ "泛用后缀词干互推:Acme ↔ Acme Corporation"+ );+ assert_eq!(find("Nova Labs").await?, None, "并掉的不算");+ assert_eq!(find("lawsuit against Acme").await?, None, "描述不是实体");+ let _ = f.merged;+ anyhow::Ok(())+ }+ .await;+ teardown(&pool, &f).await?;+ run+}