Skip to content

World's first open-source enterprise world model.

Extraction reads only the open graph (#736)

WaylandYang42930b1a9dec
diff --git a/Cargo.lock b/Cargo.lock
index bf685952..66de830c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5414,7 +5414,6 @@ dependencies = [
"serde",
"serde_json",
"tracing",
- "unicode-segmentation",
"utopia-llm",
]
diff --git a/Cargo.toml b/Cargo.toml
index b0e96793..bb197b5d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -64,7 +64,6 @@ text-splitter = { version = "0.32", features = ["tiktoken-rs"] }
# 分块预算按 token 数:cl100k 的排名表随 crate 内嵌,离线可用
tiktoken-rs = "0.12"
# 句子边界按 Unicode 标准(UAX #29)切,不自己写标点规则
-unicode-segmentation = "1.13"
pdf-extract = "0.12"
calamine = "0.36"
pgvector = { version = "0.4", features = ["sqlx"] }
diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs
index aa8fa8f9..d90fa4a8 100644
--- a/crates/utopia-core/src/models.rs
+++ b/crates/utopia-core/src/models.rs
@@ -1077,10 +1077,6 @@ pub struct KnowledgeBase {
/// 这次打开治理的时刻;保险丝只数它之后的撤回(0025 决定 9)。
/// 新库生下来就开着治理,这一格于是等于建库的时刻(0050)
pub governance_since: Option<DateTime<Utc>>,
- /// 开放抽取(0044 第一刀,#729):开着,抽取只写开放图谱——陈述照文档的字落库
- /// (`facts.layer = 'open'`),不读本体、不问日期。**缺省开**(0044 决定 2);关掉走
- /// 带本体的老路,它按决定 3 只是已批准本体下的可选第二路,等对齐追平就退场
- pub open_extraction: bool,
/// 多久重推一次(分钟)。见 `knowledge_bases.inference_interval_minutes`
pub inference_interval_minutes: i32,
/// 上次推完的时间。**答的是「上次看过没有」,不是「上次改过没有」**
diff --git a/crates/utopia-extract/Cargo.toml b/crates/utopia-extract/Cargo.toml
index 5a0840ce..ee407ae0 100644
--- a/crates/utopia-extract/Cargo.toml
+++ b/crates/utopia-extract/Cargo.toml
@@ -11,4 +11,3 @@ anyhow.workspace = true
chrono.workspace = true
utopia-llm.workspace = true
tracing.workspace = true
-unicode-segmentation.workspace = true
diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs
index ad441573..58fe1585 100644
--- a/crates/utopia-extract/src/lib.rs
+++ b/crates/utopia-extract/src/lib.rs
@@ -1,5 +1,9 @@
-//! utopia-extract: LLM 抽取(实体/关系/时间归一化)。
-//! 提示词注入本体类型与文档元时间;输出严格 JSON;证据引句强制(无引句降置信度)。
+//! utopia-extract:抽取用的提示词与解析。
+//!
+//! 开放抽取([`open`])是唯一的抽取契约(0044 决定 2):模型用文档自己的话写陈述,
+//! 不读本体、不算日期。这里留的是它借用的零件(已知实体的句柄、文件开头的预算、
+//! JSON 取块与补括号)、实体消解的攒批裁决,以及量与时间的读法——本体属性的采纳、
+//! 工具参数、连接器都用它们。
use chrono::{DateTime, NaiveDate, TimeZone, Utc};
use serde::Deserialize;
@@ -8,122 +12,6 @@ use utopia_llm::ChatMessage;
pub mod governor;
pub mod open;
-pub mod normalize;
-pub use normalize::{drop_quotes_from_opening, normalize_facts, Normalization};
-
-#[derive(Debug, Deserialize)]
-pub struct Extraction {
- #[serde(default)]
- pub entities: Vec<ExtractedEntity>,
- #[serde(default)]
- pub facts: Vec<ExtractedFact>,
- /// 实体在这段文字里的**别的名字**(0041 决定 2):简称、曾用名、另一种文字的写法。
- /// 模型报,服务端只核对名字与引文确实在原文里——认不认「简称」「又名」这些词是模型的事
- #[serde(default)]
- pub names: Vec<ExtractedName>,
- /// 逐项解析时被跳过的条目数。**必须报给调用方**——不报就是一次静默丢弃,
- /// 与 #108「部分抽取报告成完成」同一类错
- #[serde(skip)]
- pub skipped_entities: usize,
- #[serde(skip)]
- pub skipped_facts: usize,
- /// 模型的输出被截断,这里是修补后解析的
- #[serde(skip)]
- pub truncated: bool,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct ExtractedEntity {
- /// Identifier scoped to this extraction response. It preserves mention identity while
- /// facts are bound; it is not a persistent entity id or a resolution verdict.
- #[serde(default)]
- pub local_id: Option<String>,
- pub name: String,
- #[serde(rename = "type")]
- pub type_key: String,
- /// 模型自己的说法:它认为这最具体是个什么。**不校验、不入本体**。
- ///
- /// 存在的理由是清单里总有个"差不多"的:本体有 product,模型觉得够用就选了,
- /// 心里那个"向量数据库软件"就此丢失。实测 17 个实体的 proposed_type
- /// 全是空的,正是这个原因——而事后消解最需要的恰是这个名字:
- /// 短名字对短标签,比拿一段中文散文去匹配 "A software application." 近得多。
- #[serde(default)]
- pub specific_type: Option<String>,
-}
-
-/// 一个实体的一个别的名字。`ref` 是这次回复里的 local_id,或者提示词给的 k 句柄
-#[derive(Debug, Deserialize)]
-pub struct ExtractedName {
- #[serde(rename = "ref")]
- pub entity_ref: String,
- pub name: String,
- /// 名字出现在里面的那段原文,逐字抄
- #[serde(default)]
- pub quote: Option<String>,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct ExtractedFact {
- pub subject: String,
- /// Response-local entity handle. When present, callers must bind through it rather than
- /// guessing from the surface name.
- #[serde(default)]
- pub subject_ref: Option<String>,
- pub predicate: String,
- /// 关系事实的宾语实体名;属性事实为空
- #[serde(default)]
- pub object: Option<String>,
- #[serde(default)]
- pub object_ref: Option<String>,
- /// 属性事实的字面值(谓词是 attribute 时)
- #[serde(default)]
- pub value: Option<serde_json::Value>,
- /// **边上的属性**(0037):`{"amount": "$5 billion", "stake": "20%"}`。
- /// 只对关系事实有意义,key 必须是清单里这条关系声明过的;值照原文写,换算在服务端
- #[serde(default)]
- pub qualifiers: Option<serde_json::Map<String, serde_json::Value>>,
- #[serde(default)]
- pub valid_from: Option<String>,
- #[serde(default)]
- pub valid_to: Option<String>,
- #[serde(default)]
- pub confidence: Option<f32>,
- #[serde(default)]
- pub quote: Option<String>,
- /// 引文里逐字点名主语的那几个字(#582)。模型抄,不判断;落库时机器核对它
- /// 是不是 `subject` 那个名字——"Former OpenAI personnel" 不是 OpenAI
- #[serde(default)]
- pub subject_span: Option<String>,
- /// 同上,宾语那一侧
- #[serde(default)]
- pub object_span: Option<String>,
- /// 日期属性的值只相对一件事给出(「触发日后 45 天」),没有日历上的日期(#681 §4)。
- /// 模型判断、模型标;服务端不认这类说法的词,只看这个标记决定收不收
- #[serde(default)]
- pub relative: bool,
-}
-
-/// 提示词里的一条关系。
-///
-/// 比类多一样东西:**类型签名**。它是给模型的签名,不是闸门——在模型落笔那一刻
-/// 减少"Alice works_at 西雅图",而不是等错了再拦。事后校验面对的是既成事实
-///(丢掉可惜、留着是脏数据),签名是在写出来之前掰正。本体写错时模型看到原文
-/// 说了别的仍可覆盖;硬闸门会系统性丢数据,`part_of` 烧我们的正是那种方式。
-pub struct PromptRelation {
- pub key: String,
- pub label: String,
- pub description: String,
- /// 形如 `person|organization → vendor`,`*` 表示那一侧不限。空串 = 两侧都不限。
- /// **一律用 key**:模型要输出的就是 key,中文库里 person 的 label 是"人物",
- /// 写进签名等于教它输出一个不存在的类型(docs/decisions/0004)
- pub signature: String,
- /// 时间语义(`relation_types.temporal`):`state` / `event` / `eternal`(0031)。
- /// 只有 event 与 eternal 会在清单里带标记——状态是默认,写出来只多花 token
- pub temporal: String,
- /// 这条关系的边能带的属性,已排好版:`amount: number $`(0037)。空 = 不带
- pub qualifiers: Vec<String>,
-}
-
/// Response-scoped reference to a persistent entity; database UUIDs must never enter prompts.
pub struct KnownEntity {
pub handle: String,
@@ -131,322 +19,9 @@ pub struct KnownEntity {
pub name: String,
}
-/// 构造抽取提示词。`types` 为 (key, label, description) 三元组;
-/// description 非空时按行列出——本体里的语义指引直接决定抽取质量。
-/// `attributes` 为调用方预排版的属性行("person.salary (number, CNY): 月薪");
-/// 为空时提示词一字不变——没定义属性的库零成本。
-pub fn build_messages(
- types: &[(String, String, String)],
- relations: &[PromptRelation],
- attributes: &[String],
- doc_time: Option<&str>,
- filename: &str,
- // 本文档前面几块已经认下的实体,按首次出现排序。handle 只对这次回复有效。
- // 第一块为空——那时还没有"前面"
- known: &[KnownEntity],
- chunk_text: &str,
-) -> Vec<ChatMessage> {
- build_messages_with_opening(
- types, relations, attributes, doc_time, filename, known, None, chunk_text,
- )
-}
-
/// 文件开头进提示词的字符预算。一份补充协议的标题、生效日、当事方和「修订的是哪份
/// 协议」通常在头一千字符里;新闻稿的电头与导语也是
-pub const OPENING_BUDGET_CHARS: usize = 1500;
-
-/// 同 [`build_messages`],另带**这份文件的开头**(第一块的原文),给第二块往后用。
-///
-/// **一块是孤立抽取的,它看不见自己属于什么。** 补充协议把截止日写在第三块的表格里,
-/// 那一块只说「Article 13 的日期延至……」:改的是哪份租约、从哪天起改,都写在第一块。
-/// 模型拿不到,就只能把「Phase 2 Exercise Deadline」本身当主语(服务端按主语未声明丢掉),
-/// 或者抽出一个没有起点的日期(时态引擎没法据此关闭旧值)——Blackbaud 总部租约链
-/// 上五次改期丢了两次,抽到的三次一次都没关上旧值。开头只作背景,不从里面抽事实:
-/// 它自己那一块会抽,重复抽只会多出重复的事实
-#[allow(clippy::too_many_arguments)]
-pub fn build_messages_with_opening(
- types: &[(String, String, String)],
- relations: &[PromptRelation],
- attributes: &[String],
- doc_time: Option<&str>,
- filename: &str,
- known: &[KnownEntity],
- opening: Option<&str>,
- chunk_text: &str,
-) -> Vec<ChatMessage> {
- // **有描述时不送 label**。label 是给人看的显示名,而且它跟界面无关、
- // 跟这个库的语料语言走——中文库里 person 的 label 是"人物"。
- // `- person (人物): 有名有姓的具体的人…` 里那个"人物"相对 key 近乎零信息量,
- // 却让提示词在语料语言与标识符之间来回跳。描述为空时才拿它兜底:
- // 光一个 key 太单薄。见 docs/decisions/0004
- let fmt_list = |items: &[(String, String, String)]| {
- items
- .iter()
- .map(|(k, l, d)| {
- let d = d.trim();
- if d.is_empty() {
- format!("- {k} ({l})")
- } else {
- format!("- {k}: {d}")
- }
- })
- .collect::<Vec<_>>()
- .join("\n")
- };
- let type_list = fmt_list(types);
- // 关系行:有签名时括号里放签名,没有才退回 label。
- // `- works_at (person → organization): 一个人受雇于某个组织。`
- let rel_list = relations
- .iter()
- .map(|r| {
- let d = r.description.trim();
- let paren = if !r.signature.is_empty() {
- r.signature.clone()
- } else if d.is_empty() {
- r.label.clone()
- } else {
- String::new()
- };
- // 事件与恒常带方括号标记;状态是默认,不标(0031)
- let mark = temporal_mark(&r.temporal)
- .map(|m| format!(" [{m}]"))
- .unwrap_or_default();
- // 边上能带的属性跟在标记后面:`{amount: number $, stake: number %}`
- let mark = if r.qualifiers.is_empty() {
- mark
- } else {
- format!("{mark} {{{}}}", r.qualifiers.join(", "))
- };
- match (paren.is_empty(), d.is_empty()) {
- (false, false) => format!("- {} ({paren}){mark}: {d}", r.key),
- (false, true) => format!("- {} ({paren}){mark}", r.key),
- (true, false) => format!("- {}{mark}: {d}", r.key),
- (true, true) => format!("- {}{mark}", r.key),
- }
- })
- .collect::<Vec<_>>()
- .join("\n");
- // 标记也只在真有事件或恒常关系时解释一次;全是状态的库,提示词一字不变。
- // 说的是**写什么**而不是「它是什么」:事件的那一刻进 valid_from、valid_to 留空
- // ——不然模型照状态的样子填一个起点,账本就把一次收购读成从那天起一直持续
- let temporal_note = if relations
- .iter()
- .any(|r| temporal_mark(&r.temporal).is_some())
- {
- "\n 3b. A relation marked [event] happens at one moment: put the date it happened in \
- valid_from and leave valid_to null — it has no span and does not end. A relation \
- marked [eternal] holds regardless of time: leave both dates null."
- } else {
- ""
- };
- // 记号只在真有签名时解释一次;没有签名的库,提示词一字不变。
- // 说明用英文——提示词的**指令语言**是英文,只有 description 跟语料走
- //
- // **签名管两件事,而它们的可覆盖性不同。** 第一版把两件事混成了一句
- // "It is a hint, not a rule — when the text says otherwise, write what the
- // text says",于是模型连参数顺序也一并按原文的说法写:
- //
- // Elon Musk (person) --employee--> Microsoft
- //
- // 而 schema.org 声明的是 employee (organization → person)。实测一次跑里
- // 130 条可校验的事实有 102 条这样反着落库——**恰恰是本体包最主要的卖点失效**,
- // 选 schema.org 的理由就是「方向是声明的不是描述的」。
- //
- // 两件事分开说:
- //
- // - **哪些类型能参与**:提示不是闸门。本体可能写错,原文说西雅图就写西雅图。
- // 0001 的判断在这里不变——硬闸门会系统性丢数据,part_of 烧我们的正是那样。
- // - **参数顺序**:由签名定。顺序不是关于世界的断言,是这个 key 的编码约定;
- // 原文从来没有「说了别的方向」,它只说两个实体之间存在某种关系。
- // 反着说时该交换主宾,而不是反过来用这个关系。
- let sig_note = if relations.iter().any(|r| !r.signature.is_empty()) {
- ". A parenthesis after the key is the type signature, subject then object; \
- \"|\" means or, \"*\" means unconstrained. Which kinds of things may take part \
- is a hint, not a rule — when the text says otherwise, write what the text says. \
- The order is not a hint: the signature fixes which side is the subject. If the \
- text puts them the other way round, swap subject and object so that the subject \
- matches the left side — do not reverse the relation. For example, given \
- \"employee (organization → person)\" and a text saying \"X is an employee of Y\", \
- write Y as the subject and X as the object"
- } else {
- ""
- };
- let time_ctx = doc_time
- .map(|t| {
- format!(
- "Document date: {t}. Resolve relative time expressions (e.g. \"last year\", \
- \"this March\") to absolute dates using it as the reference."
- )
- })
- .unwrap_or_else(|| {
- "Document date unknown — only output dates explicitly written in the text.".into()
- });
-
- // 属性段按需注入:清单 + 输出说明 + 取值规则。没定义属性时完全不出现
- let attr_section = if attributes.is_empty() {
- String::new()
- } else {
- format!(
- "\nAttributes (literal-valued fields, listed as class.attribute_key; as \"predicate\" \
- use the attribute_key alone — e.g. \"salary\", not \"person.salary\" — with a \
- \"value\" instead of \"object\"):\n{}\n",
- attributes.join("\n")
- )
- };
- let attr_rules = if attributes.is_empty() {
- String::new()
- } else {
- "\n11. Attribute facts carry \"value\" (no \"object\"): number = the figure **as the text writes it, magnitude and currency included** \n (\"86亿元\", \"$5 billion\", \"4,300 人\") — never reduce it to a bare number, the server converts; date = \"YYYY[-MM[-DD]]\" (a zoned clock time only when the text gives one) — a date the text gives only relative to an event \
- (\"45 days after the Trigger Date\", \"within 30 days of closing\") has no calendar date to convert: write it as the text writes it and add \"relative\": true; bool = true/false; \
- text = a short string. Only attach an attribute to a subject of its listed class. \
- valid_from = when this value took effect, if the text or the opening of the document says so. \
- A document that changes a value set earlier — amends, extends or replaces it — makes the new value \
- hold from the date the change takes effect, which is the document's own effective date unless the text gives another."
- .to_string()
- };
- let system = format!(
- "You are a knowledge-graph extraction engine. Extract entities and factual relations \
- from the given text. Output exactly one JSON object and nothing else.\n\
- \n\
- Entity types (prefer these keys):\n{type_list}\n\
- \n\
- Relation types (prefer these keys){sig_note}:\n{rel_list}\n\
- {attr_section}\
- \n\
- Output format:\n\
- {{\"entities\":[{{\"local_id\":\"e1\",\"name\":\"entity name\",\"type\":\"type key\",\"specific_type\":\"what you would call it\"}}],\n\
- \"facts\":[{{\"subject\":\"subject entity name\",\"subject_ref\":\"e1\",\"subject_span\":\"the words in quote that name the subject\",\"predicate\":\"relation key\",\"object\":\"object entity name\",\"object_ref\":\"e2\",\"object_span\":\"the words in quote that name the object\",\n\
- \"valid_from\":\"2023-01\",\"valid_to\":null,\"confidence\":0.9,\"quote\":\"verbatim supporting quote\"}}],\n\
- \"names\":[{{\"ref\":\"e1\",\"name\":\"another name the text uses for it\",\"quote\":\"verbatim text containing that name\"}}]}}\n\
- \n\
- Rules:\n\
- 1. Give every newly listed entity a local_id unique within this response (e1, e2, ...). \
- A local_id may define at most one entity. Reuse the same local_id when this response \
- mentions the same entity again, including abbreviations; never allocate a new handle \
- merely because a mention repeats. Different handles mean the mentions should be \
- tracked separately for attribution, not that their permanent identity is proven. \
- When the text clearly describes two different entities with the same surface name, \
- list both under different local_ids. A shared name alone neither proves sameness nor \
- requires a split.\n\
- 1a. Use the canonical full name as written in the text, in the text's original language; \
- list each entity once. Text introduces a full name and then shortens it — \
- \"星云科技上海研究院\" becomes \"上海研究院\", \"Nebula Technologies Inc.\" becomes \
- \"Nebula\" — and both forms mean one entity, listed once under the fuller form. \
- Two names are two entities only when the text is talking about two things. \
- A name identifies the thing; it is not a description of its history. When the text \
- names something and then describes what happened to it, the name ends where the \
- description begins.\n\
- 1b. Every other name the text gives an entity goes into \"names\", once per name: the \
- shortened form it introduces or uses (\"上海研究院\" for \"星云科技上海研究院\"), a \
- former name, the name in another language. \"ref\" is the entity's local_id or its \
- known handle, and \"quote\" is a verbatim excerpt that contains the name. Only \
- names belong there — never a pronoun or a description (\"该公司\", \"the company\", \
- \"former employees\") — and never the name already written in entities. A name \
- must name the entity itself, not something that belongs to it: \"星云科技研发团队\" \
- names a team, not 星云科技.\n\
- 2. Every fact keeps its name fields and uses subject_ref; relation facts also use \
- object_ref. Each ref must be either a local_id defined exactly once in \
- entities or a known handle supplied with this text. An entity referenced by a known \
- handle must not be copied into entities.\n\
- 3. Dates must be \"YYYY\", \"YYYY-MM\", \"YYYY-MM-DD\", or null — never invent dates. \
- A clock time is allowed only together with its zone, as \"YYYY-MM-DDTHH:MM[:SS]Z\" \
- or with a \"+HH:MM\" offset, and only when the text or the document states that \
- zone; a time of day without a zone stays a plain date — never guess a zone.\n\
- 3a. valid_to takes a third value: \"unknown\". Use it when the text says the relation \
- has ended but does not say when — \"former CEO of X\", \"stepped down\", \"left the \
- company\", \"no longer available\", \"until recently\". Use null only for something \
- still going on. These are not interchangeable: null asserts it still holds, and \
- writing null for a relation the text says is over makes us claim the opposite of \
- the source.\n\
- 3c. A period is when a fact holds, never what it is about. A quarter, a half, a \
- fiscal or calendar year, a month, \"the three months ended July 26, 2026\" — \
- none of these is an entity and none is an object. Put the period's dates in \
- valid_from and valid_to (a fiscal period resolves to the dates the document \
- states for it) and write the figure as the fact's \"value\" — the figure alone, as it stands in the \
- quote, with nothing appended. A column of a table headed by a period is a column \
- of values that hold in that period.\n\
- {temporal_note}\n\
- 4. {time_ctx}\n\
- 5. quote must be a contiguous excerpt from the Text block; never quote the opening of the document. Every fact needs one.\n\
- 6. confidence in 0~1: 0.9 explicitly stated, 0.7 inferred, 0.5 uncertain. A value \
- the text writes out is stated whatever the layout — a sentence, a list, a table \
- cell, a schedule, the new column of an amendment that replaces an earlier term. \
- Inferred means the text does not write the value and you worked it out.\n\
- 7. If nothing can be extracted, output {{\"entities\":[],\"facts\":[]}}.\n\
- 8. If no listed relation fits, do not force the nearest one — write the predicate the \
- text itself uses, in snake_case (e.g. \"available_on\", \"runs_on\"). A relation \
- named after the text is worth more than a listed one that says something false.\n\
- 8a. The same holds for a literal the text states outright — an amount, a share count, \
- a percentage, a capacity, a date, a job title, a ticker. Write it as a fact with \
- \"value\" and no \"object\": {{\"subject\":\"NVIDIA\",\"subject_ref\":\"e1\",\
- \"predicate\":\"purchase_price\",\"value\":\"$11.9 billion\",\"confidence\":0.9,\
- \"quote\":\"...\"}}. Name the predicate after the text when no listed attribute \
- fits — \"purchase_price\", \"job_title\", \"generation_capacity\", \"record_date\". \
- Attach it to the entity the text attaches it to, and keep the literal as written, \
- units and all — except a date, which is always written in the format of rule 3 \
- (\"June 23, 2020\" is \"2020-06-23\"). A deadline or a period stated \
- relative to an event, with no calendar date, is not a date: keep it as written \
- and mark it \"relative\" as rule 11 says. \
- **A stated figure left out is the loss that costs most**: the reader \
- came for those numbers, and no later step can recover one that was never written \
- down.\n\
- 8b. A listed relation followed by {{…}} can carry those **qualifiers on the edge**: when the same sentence gives both the other entity and a figure for it — an amount, a stake, a price, a share count — write the relation with its \"object\" and put the figure in \"qualifiers\" keyed exactly as listed, **as written in the text, currency and all** (\"€30 million\", \"15亿元人民币\", never a bare number) — except a date, which takes the format of rule 3: {{\"subject\":\"Vega Capital\",\"predicate\":\"invested_in\",\"object\":\"Northwind\", \"qualifiers\":{{\"amount\":\"$5 billion\"}},…}}. Never invent a key that is not listed for that relation, and never drop the figure to keep the edge — a relation without its amount is half the sentence. A relation you name after the text (rule 8) carries its figure the same way — keyed by the listed attribute that fits it, or by the plainest word for it (\"amount\", \"stake\", \"price\") when none does.
- 8c. A **listed** relation also takes \"value\" when what the text gives is a \
- string rather than another entity — a job title, a designation, a ticker, a \
- model number. Never invent an entity for a string. And when the text introduces \
- someone by their role — \"X, founder and CEO of Y\", \"Z, co-CEO of W\", \
- \"Y's vice president of research\", \"the president of OpenAI\", \
- \"chief executive of Quora\", \"OpenAI's chief technology officer of \
- applications\" — write both facts: the tie to the organization, and \
- the role itself as a value on the person. The tie alone says they \
- are connected; the role is what the sentence was actually telling \
- you. The possessive (\"Y's <role>\", \"<role> of Y\", \"<role> at Y\"), the past \
- tense (\"was Y's <role>\", \"former <role> of Y\"), and the implied form \
- (\"appointed … as OpenAI's CTO of applications\") all carry the same \
- shape — the role is the value, the organization is the other \
- entity. Past tense and \"former\" give the tie valid_to: \"unknown\".\n\
- 8d. A list of named parties is a list of facts — one per name. \"partners \
- including A, B, C and D\" is four facts, not one; \"advisors A and B\" is two. \
- Do not collapse an enumeration into a summary or into its first member. \
- The same applies to the entities: each named party is its own entity.\n\
- 8e. subject_span and object_span are the exact words in quote that name each side. \
- Copy them; never paraphrase. When the words that do the thing are a description \
- rather than a name — \"former X employees\", \"companies using X\" — the span \
- is that description, whatever you wrote in subject.\n\
- 8f. An obligation, a deadline or a right belongs to the agreement, law or decision \
- that imposes it, even when it concerns another agreement or thing. A lease that \
- sets the last day to sign a second lease gives that deadline to the first lease; \
- the second lease is only what the deadline is about.\n\
- 9. The same holds for entity types: if none of the listed types fits, write the type \
- the text implies, in snake_case (e.g. \"model\", \"technology\"). Do not fall back \
- to a broad listed type such as \"thing\" or \"creative_work\" merely because \
- nothing specific matched — that hides the gap instead of reporting it.\n\
- 10. specific_type is required on every entity and is never checked against the list. \
- Name the most specific kind the thing is, in the words you would use for it. Write \
- it even when \"type\" already fits, and make it narrower than \"type\" wherever the \
- text supports it — type \"product\", specific_type \"vector database software\". \
- Repeat the listed type only when the text genuinely says nothing more precise.\
- {attr_rules}"
- );
-
- // 已知实体紧挨着正文:服从性靠位置,理由见 known_block 的注释
- let user = format!(
- "Source file: \"{filename}\"\n{}{}\nText:\n{chunk_text}",
- opening_block(opening),
- known_block(known)
- );
-
- vec![
- ChatMessage {
- role: "system".into(),
- content: system,
- },
- ChatMessage {
- role: "user".into(),
- content: user,
- },
- ]
-}
+pub(crate) const OPENING_BUDGET_CHARS: usize = 1500;
/// 文件开头排版成提示词里的一段。开头为空(或只有空白)时返回空串;
/// 「这一块就是开头本身」由调用方判断,那时它传 `None`。
@@ -471,75 +46,12 @@ pub(crate) fn opening_block(opening: Option<&str>) -> String {
)
}
-/// 一段描述的第一句(句子边界按 UAX #29)。按块检索出的清单只带这一句:schema.org 的
-/// 描述后半截多是用法说明与示例,一块铺上百行时它们占了清单的八成
-pub fn first_sentence(text: &str) -> &str {
- use unicode_segmentation::UnicodeSegmentation;
- text.trim().unicode_sentences().next().map_or("", str::trim)
-}
-
-/// 清单里给关系带的标记:事件 `[event]`、恒常 `[eternal]`;状态不标。
-/// 认不出的值当状态——数据库的 CHECK 只放这三个进来,这里不再报错
-fn temporal_mark(temporal: &str) -> Option<&'static str> {
- match temporal {
- "event" => Some("event"),
- "eternal" => Some("eternal"),
- _ => None,
- }
-}
-
/// 已在本文档中出现过的实体,放进提示词的字符预算。
///
/// 超出就截断(保留先出现的)。中文商业文本先出全称、主角先出场,所以
/// **首次出现顺序天然偏向那些后面会被简称的名字**。
pub(crate) const KNOWN_BUDGET_CHARS: usize = 1200;
-/// 把「本文档已经认下的实体」排版成提示词里的一段。空则返回空串。
-///
-/// **为什么在正文之前、指令贴着清单**:抽象规则打不过挨着它的具体块——本体建议
-/// 那次,语言要求就输给了紧随其后的英文 JSON 骨架,挪到骨架之后并点名它才生效。
-/// 服从性靠位置,所以指令挨着它管的数据放,两者一起挨着正文。
-///
-/// **顺带一条与放哪条消息无关的规矩:逐块变化的内容一律放最后。** 前缀缓存匹配的是
-/// token 前缀,而消息按 system→user 拼接,所以「system 末尾」与「user 开头」几乎等价;
-/// 真正会打碎缓存的是把它塞在**中间**(本体之后、规则之前),那会把规则挤出前缀。
-/// 缓存本身不归我们管——供应商开不开、报不报都由它,本部署实测 `cached=0`——
-/// 我们只负责别把它弄碎。自部署 vLLM 默认开着自动前缀缓存,那省的是算力不是钱。
-fn known_block(known: &[KnownEntity]) -> String {
- if known.is_empty() {
- return String::new();
- }
- let mut lines = Vec::new();
- let mut used = 0usize;
- for entity in known {
- used += entity.handle.chars().count()
- + entity.type_key.chars().count()
- + entity.name.chars().count()
- + 6;
- if used > KNOWN_BUDGET_CHARS {
- break;
- }
- lines.push(format!(
- " {} [{}]: {}",
- entity.handle, entity.type_key, entity.name
- ));
- }
- if lines.is_empty() {
- return String::new();
- }
- let lines = lines.join("\n");
- format!(
- "\nAlready recorded from earlier parts of this same document:\n{lines}\n\
- \n\
- If something in the text below refers to one of these, use its k-handle in the fact's \
- subject_ref/object_ref, write that exact string as the name, and give it that same \
- type — documents abbreviate after first mention \
- (\"星云科技上海研究院\" later becomes \"上海研究院\"), and the shortened form must \
- not become a second entity. If it is a different thing, name it as the text does; \
- do not force it onto this list.\n"
- )
-}
-
/// 回复里可能是 JSON 的那段文字:切掉思考过程与代码围栏。前后的废话留给调用方按
/// 括号定位——[`json_block`] 取第一个 `{` 到最后一个 `}`;开放抽取的截断修补则从
/// 第一个 `{` 取到结尾,那边的记录是数组,最后一个 `}` 不是可靠的结尾
@@ -611,102 +123,6 @@ pub(crate) fn close_brackets(head: &str) -> Option<String> {
Some(out)
}
-/// 输出被截断时,退到**最后一个完整对象**的结尾再把括号补齐。
-///
-/// 模型写到一半没了(撞上 max_tokens)时,前面那些对象是完整且正确的。
-/// 整块作废等于把已经抽对的十几条事实一起扔掉——实测 246 次调用里 4 次是这种。
-fn repair_truncated(json: &str) -> Option<String> {
- let mut cut = json.len();
- for _ in 0..64 {
- let idx = json[..cut].rfind('}')?;
- if let Some(closed) = close_brackets(&json[..=idx]) {
- if serde_json::from_str::<serde_json::Value>(&closed).is_ok() {
- return Some(closed);
- }
- }
- cut = idx;
- }
- None
-}
-
-/// **一条坏记录不该毁掉一整块。**
-///
-/// 从前这里是 `serde_json::from_str::<Extraction>`——全有或全无。一个缺 `predicate`
-/// 的对象、或者一次输出截断,整块的实体和事实一起作废,而一块里常有二十条好事实。
-/// 实测 246 次调用里 5 次这样丢掉(2%),并且会让整个 `extract_document` 任务失败、
-/// 走重试,三次之后文档标记失败。
-///
-/// 现在:先解成 `Value`(截断就先补齐括号),再逐项 `from_value`,好的收下、
-/// 坏的计数。**计数必须往外传**——静默跳过就是另一种"报告成完成"。
-pub fn parse_response(raw: &str) -> anyhow::Result<Extraction> {
- let json_str = json_block(raw)?;
- let (value, truncated) = match serde_json::from_str::<serde_json::Value>(&json_str) {
- Ok(v) => (v, false),
- Err(e) => match repair_truncated(&json_str) {
- Some(fixed) => (
- serde_json::from_str::<serde_json::Value>(&fixed)
- .map_err(|e| anyhow::anyhow!("Failed to parse extraction JSON: {e}"))?,
- true,
- ),
- // 补不回来才是真解析失败:连一个完整对象都没有
- None => anyhow::bail!("Failed to parse extraction JSON: {e}"),
- },
- };
-
- fn take<T: serde::de::DeserializeOwned>(
- value: &serde_json::Value,
- key: &str,
- ) -> (Vec<T>, usize) {
- let Some(arr) = value.get(key).and_then(|v| v.as_array()) else {
- return (Vec::new(), 0);
- };
- let mut out = Vec::with_capacity(arr.len());
- let mut skipped = 0;
- for item in arr {
- match serde_json::from_value::<T>(item.clone()) {
- Ok(v) => out.push(v),
- Err(_) => skipped += 1,
- }
- }
- (out, skipped)
- }
-
- let (mut entities, mut skipped_entities) = take::<ExtractedEntity>(&value, "entities");
- let (facts, skipped_facts) = take::<ExtractedFact>(&value, "facts");
- // 名字条目坏了不算实体或事实被跳过:丢一个名字只是少一座桥,不丢断言
- let (names, _) = take::<ExtractedName>(&value, "names");
-
- // A handle identifies exactly one entity definition within one response. Reject every
- // definition participating in a duplicate (including identical duplicates): keeping the
- // first or last would make fact attribution depend on array order. Empty handles are
- // malformed too; legacy output is represented by an absent field, not an empty id.
- let mut handle_counts = std::collections::HashMap::<String, usize>::new();
- for entity in &entities {
- if let Some(handle) = entity.local_id.as_deref() {
- *handle_counts.entry(handle.trim().to_string()).or_default() += 1;
- }
- }
- entities.retain(|entity| match entity.local_id.as_deref() {
- None => true,
- Some(handle) => {
- let handle = handle.trim();
- let valid = !handle.is_empty() && handle_counts.get(handle) == Some(&1);
- if !valid {
- skipped_entities += 1;
- }
- valid
- }
- });
- Ok(Extraction {
- entities,
- facts,
- names,
- skipped_entities,
- skipped_facts,
- truncated,
- })
-}
-
// ---------------------------------------------------------------------------
// 实体消解裁决(攒批:一次调用裁多对,LLM 只处理 embedding 分不出的灰区)
// ---------------------------------------------------------------------------
@@ -934,7 +350,7 @@ pub fn parse_leading_quantity(s: &str) -> Option<(f64, Option<String>)> {
}
/// 货币:符号、ISO 码、中英文单词,统一成符号。**只认这张表**,认不出的不猜。
-pub fn currency_unit(tok: &str) -> Option<&'static str> {
+pub(crate) fn currency_unit(tok: &str) -> Option<&'static str> {
Some(
match tok.trim_matches(|c: char| c == ',' || c == '.' || c == ';') {
"$" | "USD" | "usd" | "US$" | "dollar" | "dollars" | "美元" => "$",
@@ -1082,25 +498,6 @@ fn next_token(s: &str) -> (&str, &str) {
(&s[..end], &s[end..])
}
-/// 一个属性值落库时的样子:按 datatype 归一成 `{"value": …}`;失败返回 None,调用方记
-/// `attr_datatype`。
-///
-/// 日期属性上一个**相对**的值(#681 §4):解不成日期、模型又标了 `relative`、写的是一段非空
-/// 文字时,照原文收下,值里带 `"relative": true`。它不是日期,从不当日期比较或排序。解得成
-/// 日期的照日期存,标错了也不当相对;没标的非日期值仍然不收
-pub fn attr_object_value(
- datatype: &str,
- raw: &serde_json::Value,
- relative: bool,
-) -> Option<serde_json::Value> {
- if let Some(value) = normalize_attr_value(datatype, raw) {
- return Some(serde_json::json!({ "value": value }));
- }
- let written = raw.as_str().map(str::trim).filter(|s| !s.is_empty())?;
- (datatype == "date" && relative)
- .then(|| serde_json::json!({ "value": written, "relative": true }))
-}
-
/// 属性值按 datatype 归一。失败返回 None——宁缺勿脏,调用方跳过并记日志。
/// number 容忍千分位/空格;date 收规则 3 的格式(YYYY[-MM[-DD]]、带时区的时刻,原样保留),
/// 也收写法说得清是哪天的日期([`written_date`]),换成规则 3 的样子;bool 宽容 yes/no。
@@ -1321,271 +718,32 @@ fn split_zone(clock: &str) -> Option<(&str, chrono::Duration)> {
}
#[cfg(test)]
-mod prompt_shape_tests {
+mod tests {
use super::*;
- /// 第十份补充协议把截止日改成「触发日后 45 天」:模型标 relative,服务端照原文收;
- /// 没标的、不是日期属性的、空的都不走这条
+ /// #690:思考过程里的大括号不能把 JSON 的起止带偏。不切掉标记,"第一个 `{`" 的起点
+ /// 会提前到思考过程里;代码围栏与前后的废话同样不算数
#[test]
- fn a_relative_date_is_kept_as_written_only_when_marked() {
- let raw = serde_json::json!("45 days after the Trigger Date");
- assert_eq!(
- attr_object_value("date", &raw, true),
- Some(
- serde_json::json!({ "value": "45 days after the Trigger Date", "relative": true })
- )
- );
- assert_eq!(attr_object_value("date", &raw, false), None, "没标就不收");
- assert_eq!(
- attr_object_value("date", &serde_json::json!(" "), true),
- None
- );
- // 解得成日期的照日期存,标了 relative 也不当相对
+ fn json_block_ignores_a_think_block_and_a_fence_before_the_json() {
+ let raw = "先想想 {\"a\": 1,再回答。\n</think>{\"entities\":[{\"name\":\"张三\"}]}";
assert_eq!(
- attr_object_value("date", &serde_json::json!("2020-06-23"), true),
- Some(serde_json::json!({ "value": "2020-06-23" }))
+ json_block(raw).unwrap(),
+ "{\"entities\":[{\"name\":\"张三\"}]}"
);
- // 不是日期属性:只按它自己的 datatype 归一,relative 不起作用
- assert_eq!(
- attr_object_value("bool", &serde_json::json!("45 days after"), true),
- None
- );
- assert_eq!(
- attr_object_value("number", &serde_json::json!("1,250"), true),
- Some(serde_json::json!({ "value": 1250.0 }))
- );
- let fact: ExtractedFact = serde_json::from_value(serde_json::json!({
- "subject": "Lease", "predicate": "expansion_option_deadline",
- "value": "45 days after the Trigger Date", "relative": true
- }))
- .unwrap();
- assert!(fact.relative);
- let plain: ExtractedFact = serde_json::from_value(serde_json::json!({
- "subject": "Lease", "predicate": "expansion_option_deadline", "value": "2020-06-23"
- }))
- .unwrap();
- assert!(!plain.relative, "没写就不是");
- let msgs = build_messages(
- &[],
- &[],
- &["lease.deadline (date)".into()],
- None,
- "a.txt",
- &[],
- "text",
- );
- assert!(msgs[0].content.contains("add \"relative\": true"));
- }
-
- fn rel(key: &str, description: &str, signature: &str) -> PromptRelation {
- PromptRelation {
- key: key.into(),
- label: key.replace('_', " "),
- description: description.into(),
- signature: signature.into(),
- temporal: "state".into(),
- qualifiers: vec![],
- }
- }
-
- fn timed(key: &str, description: &str, temporal: &str) -> PromptRelation {
- PromptRelation {
- temporal: temporal.into(),
- ..rel(key, description, "")
- }
- }
-
- /// 事件与恒常在清单里带标记,说明只出现一次(0031)
- #[test]
- fn an_event_and_an_eternal_relation_are_marked() {
- let rels = vec![
- rel("works_at", "受雇于某个组织。", "person → organization"),
- timed("acquired", "One company buys another.", "event"),
- timed("capital_of", "", "eternal"),
- ];
- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");
- let s = &msgs[0].content;
- assert!(s.contains("- works_at (person → organization): 受雇于某个组织。"));
- assert!(s.contains("- acquired [event]: One company buys another."));
- // 没有描述时括号里是 label,标记跟在括号后面
- assert!(s.contains("- capital_of (capital of) [eternal]"));
- assert!(s.contains("A relation marked [event] happens at one moment"));
- assert!(s.contains("leave valid_to null"));
+ let fenced = "好的,结果如下:\n```json\n{\"entities\":[]}\n```";
+ assert_eq!(json_block(fenced).unwrap(), "{\"entities\":[]}");
+ assert!(json_block("no json here").is_err());
}
- /// **全是状态的库,提示词一字不变**:不标、不解释
+ /// 括号出现在字符串里不算结构——`"a[b"` 不是一个开括号;断在字符串中间的那一截不可用
#[test]
- fn a_base_of_states_pays_nothing_for_the_marks() {
- let rels = vec![rel("works_at", "d", "")];
- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");
- let s = &msgs[0].content;
- assert!(!s.contains("[event]"));
- assert!(!s.contains("[eternal]"));
- assert!(!s.contains("happens at one moment"));
- }
-
- /// 签名进括号,而且**一律是 key**:中文库的 label 是"人物",
- /// 写进提示词等于教模型输出一个不存在的类型。
- #[test]
- fn a_signature_takes_the_parenthesis_and_uses_keys() {
- let rels = vec![rel("works_at", "受雇于某个组织。", "person → organization")];
- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");
- assert!(msgs[0]
- .content
- .contains("- works_at (person → organization): 受雇于某个组织。"));
- }
-
- /// 多值用 `|`,空的一侧用 `*` —— 都是 key 层面的记号,不是类型名
- #[test]
- fn several_classes_join_with_a_pipe_and_an_empty_side_is_a_star() {
- let rels = vec![rel("buys_from", "", "employee|team → *")];
- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");
- assert!(msgs[0].content.contains("- buys_from (employee|team → *)"));
- }
-
- /// **没有签名的库,提示词一字不变**:记号说明也不出现。
- /// 大多数库不会声明 domain/range,不该为此付每块的 token
- #[test]
- fn a_base_without_signatures_pays_nothing() {
- let rels = vec![rel("works_at", "受雇于某个组织。", "")];
- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");
- assert!(msgs[0].content.contains("- works_at: 受雇于某个组织。"));
- assert!(!msgs[0].content.contains("type signature"));
- assert!(!msgs[0].content.contains('→'));
- }
-
- /// 签名是提示不是闸门。这句话必须在提示词里 —— 少了它,
- /// 模型会把签名当硬规则,本体写错时就系统性丢数据(part_of 那种方式)
- #[test]
- fn the_prompt_says_the_signature_is_a_hint() {
- let rels = vec![rel("works_at", "d", "person → organization")];
- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");
- assert!(msgs[0].content.contains("hint, not a rule"));
- }
-
- /// **但顺序不是提示。**
- ///
- /// 两句话必须同时在场,少哪一句都退回一种老毛病:少了「提示不是闸门」,
- /// 本体写错时系统性丢数据(part_of 那种方式);少了「顺序由签名定」,
- /// 模型按英语直觉写 `Musk --employee--> Microsoft`,而 schema.org 声明的是
- /// `employee (organization → person)`——实测一次跑里 130 条可校验的事实
- /// 有 102 条这样反着落库。
- #[test]
- fn the_prompt_says_the_order_is_not_a_hint() {
- let rels = vec![rel("employee", "d", "organization → person")];
- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");
- let c = &msgs[0].content;
- assert!(c.contains("hint, not a rule"), "类型那句丢了");
- assert!(c.contains("The order is not a hint"), "顺序那句丢了");
- assert!(
- c.contains("swap subject and object"),
- "只说了顺序重要,没说反着写时该怎么办"
- );
- assert!(
- c.contains("do not reverse the relation"),
- "少了这句,模型可能去找一个反向关系而不是交换主宾"
- );
- }
-
- #[test]
- fn an_obligation_belongs_to_the_agreement_that_imposes_it() {
- // 主租约里写着「签二期租约的截止日」,模型时而把截止日挂到二期租约上:
- // 主租约的时间线上就少了这次改期(#681 §3)
- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");
- assert!(msgs[0]
- .content
- .contains("belongs to the agreement, law or decision that imposes it"));
- }
-
- #[test]
- fn a_literal_keeps_its_units_but_a_date_takes_the_contract_format() {
- // 8a 从前说「字面值按原文写」并把日期列在字面值里,而规则 3 与属性规则要求
- // YYYY-MM-DD:两条互相打架,模型写出「June 23, 2020」,服务端按格式不合整条丢掉。
- // Blackbaud 总部租约链上各轮累计丢了二十多次
- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");
- let system = &msgs[0].content;
- assert!(system.contains("except a date, which is always written in the format of rule 3"));
- }
-
- /// 补充协议把旧条款与新日期排成一张对照表,模型把表格里读到的新日期标 0.7(当成
- /// 推断),低于 0.75 的值不许接替前一个——截止日就一直停在旧值上。规则 6 说清楚:
- /// 原文写着的值不论排成什么样都是明写
- #[test]
- fn a_value_written_in_a_table_is_stated() {
- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");
- let system = &msgs[0].content;
- assert!(system.contains("stated whatever the layout"));
- assert!(system.contains("a table cell"));
- }
-
- /// 规则编号各不相同,「按规则 N」指得到唯一的一条。从前有两条 8c、两条 10,
- /// 「as rule 10 says」说的是哪条要靠猜(#689 评审)
- #[test]
- fn every_rule_has_its_own_number_and_every_reference_lands() {
- let rels = vec![PromptRelation {
- key: "acquired".into(),
- label: "acquired".into(),
- description: String::new(),
- signature: String::new(),
- temporal: "event".into(),
- qualifiers: vec![],
- }];
- let attrs = vec!["lease.option_deadline (date)".to_string()];
- let msgs = build_messages(&[], &rels, &attrs, None, "a.txt", &[], "text");
- let system = &msgs[0].content;
- let mut labels = Vec::new();
- for line in system.lines() {
- let Some((label, _)) = line.trim_start().split_once(". ") else {
- continue;
- };
- let digits = label.trim_end_matches(|c: char| c.is_ascii_lowercase());
- if !digits.is_empty()
- && digits.chars().all(|c| c.is_ascii_digit())
- && label.len() - digits.len() <= 1
- {
- labels.push(label.to_string());
- }
- }
- let unique: std::collections::BTreeSet<_> = labels.iter().collect();
- assert_eq!(unique.len(), labels.len(), "规则编号重复:{labels:?}");
- for (i, _) in system.match_indices("rule ") {
- let n: String = system[i + 5..]
- .chars()
- .take_while(|c| c.is_ascii_alphanumeric())
- .collect();
- assert!(
- labels.contains(&n),
- "「rule {n}」指不到任何一条:{labels:?}"
- );
- }
- }
-
- #[test]
- fn a_later_chunk_reads_the_opening_of_its_document() {
- let opening = "FIFTH AMENDMENT TO LEASE AGREEMENT entered into as of February 18, 2020";
- let msgs = build_messages_with_opening(
- &[],
- &[],
- &[],
- None,
- "a.html",
- &[],
- Some(opening),
- "The Existing Dates are extended to March 17, 2020.",
- );
- let user = &msgs[1].content;
- let at_opening = user.find(opening).expect("opening is in the user message");
- let at_text = user
- .find("The Existing Dates")
- .expect("text is in the user message");
- assert!(
- at_opening < at_text,
- "the opening comes before the text it frames"
+ fn brackets_inside_strings_are_not_structure() {
+ assert_eq!(
+ close_brackets(r#"{"s": [{"a": "a[b{c"}"#).as_deref(),
+ Some(r#"{"s": [{"a": "a[b{c"}]}"#)
);
- // 没有开头时,提示词与从前一字不差
- let plain = build_messages(&[], &[], &[], None, "a.html", &[], "t");
- let framed = build_messages_with_opening(&[], &[], &[], None, "a.html", &[], None, "t");
- assert_eq!(plain[1].content, framed[1].content);
+ assert!(close_brackets(r#"{"s": "cut in the mid"#).is_none());
+ assert!(close_brackets("]").is_none());
}
#[test]
@@ -1597,136 +755,6 @@ mod prompt_shape_tests {
assert_eq!(opening_block(Some(" ")), "");
}
- /// 已知实体必须落在 **user** 消息里、紧挨着正文。
- ///
- /// 理由是服从性不是缓存:抽象规则打不过挨着它的具体块。清单放进 system 的
- /// 规则区,就会隔着输出格式、十条规则、文件名,离它要管的正文最远。
- #[test]
- fn known_entities_stay_out_of_the_system_message() {
- // 用一个规则 1 的例子里没有的名字:规则 1 也提"星云科技上海研究院",
- // 拿它断言等于测不出清单到底在哪条消息里
- let known = vec![KnownEntity {
- handle: "k1".into(),
- type_key: "organization".into(),
- name: "华瑞集团智能制造研究院".into(),
- }];
- let msgs = build_messages(&[], &[], &[], None, "a.txt", &known, "text");
- assert_eq!(msgs[0].role, "system");
- assert!(!msgs[0].content.contains("Already recorded"));
- assert!(!msgs[0].content.contains("华瑞集团智能制造研究院"));
- assert!(msgs[1]
- .content
- .contains("k1 [organization]: 华瑞集团智能制造研究院"));
- }
-
- /// 第一块没有"前面",那一段应当完全不出现——成本为零,而不是一段空标题
- #[test]
- fn the_first_chunk_carries_no_block() {
- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");
- assert!(!msgs[1].content.contains("Already recorded"));
- }
-
- /// 反向护栏必须在:给了参照物就会有人硬套(`concept` 那次的教训)
- #[test]
- fn the_block_tells_the_model_not_to_force_a_match() {
- let known = vec![KnownEntity {
- handle: "k1".into(),
- type_key: "person".into(),
- name: "陈立".into(),
- }];
- let msgs = build_messages(&[], &[], &[], None, "a.txt", &known, "text");
- assert!(msgs[1].content.contains("do not force it onto this list"));
- }
-
- /// 有描述就不送 label——中文库的 label 是中文,混进提示词只会让
- /// 标识符与语料语言来回跳,而它相对 key 近乎零信息量。
- #[test]
- fn described_types_drop_the_label() {
- let types = vec![
- (
- "person".into(),
- "人物".into(),
- "有名有姓的具体的人。".into(),
- ),
- ("event".into(), "事件".into(), String::new()),
- ];
- let msgs = build_messages(&types, &[], &[], None, "a.txt", &[], "text");
- let prompt = format!("{:?}", msgs);
- assert!(prompt.contains("- person: 有名有姓的具体的人。"));
- assert!(!prompt.contains("person (人物)"));
- // 描述为空时 label 仍是唯一的额外线索,留着
- assert!(prompt.contains("- event (事件)"));
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- /// 边上的属性(0037):清单里跟在关系后面,回复里挂在事实上。
- #[test]
- fn a_relation_lists_its_qualifiers_and_a_fact_carries_them() {
- use serde_json::json;
- let mut r = PromptRelation {
- key: "invested_in".into(),
- label: "invested in".into(),
- description: "money into a company".into(),
- signature: "organization → organization".into(),
- temporal: "event".into(),
- qualifiers: vec!["amount: number $".into(), "stake: number %".into()],
- };
- let msgs = build_messages(
- &[],
- std::slice::from_ref(&r),
- &[],
- None,
- "a.txt",
- &[],
- "text",
- );
- let prompt = format!("{:?}", msgs);
- // 签名、标记、属性清单三段顺序固定:`(签名) [event] {属性}`
- assert!(prompt.contains(
- "- invested_in (organization → organization) [event] {amount: number $, stake: number %}: money into a company"
- ), "{prompt}");
- // 不带属性的关系不多一个花括号
- r.qualifiers.clear();
- let prompt = format!(
- "{:?}",
- build_messages(
- &[],
- std::slice::from_ref(&r),
- &[],
- None,
- "a.txt",
- &[],
- "text"
- )
- );
- assert!(
- prompt.contains("- invested_in (organization → organization) [event]: money"),
- "{prompt}"
- );
- assert!(!prompt.contains("[event] {"));
-
- // 回复:qualifiers 挂在关系事实上;没写的是 None,旧回复不受影响
- let reply = r#"{"entities":[],"facts":[
- {"subject":"Vega","predicate":"invested_in","object":"Northwind",
- "qualifiers":{"amount":"$5 billion"},"confidence":0.9},
- {"subject":"Vega","predicate":"invested_in","object":"Kestrel","confidence":0.9}
- ]}"#;
- let parsed = parse_response(reply).unwrap();
- assert_eq!(parsed.facts.len(), 2);
- assert_eq!(
- parsed.facts[0]
- .qualifiers
- .as_ref()
- .and_then(|q| q.get("amount")),
- Some(&json!("$5 billion"))
- );
- assert!(parsed.facts[1].qualifiers.is_none());
- }
-
#[test]
fn a_quantity_is_the_whole_string_or_nothing() {
// 整体就是一个量:符号、量级词、千分位都读得动
@@ -1851,22 +879,6 @@ mod tests {
assert_eq!(normalize_attr_value("number", &json!("about ten")), None);
}
- #[test]
- fn a_description_is_cut_at_its_first_sentence() {
- assert_eq!(
- first_sentence(
- " The date on which the CreativeWork was created. See also dateModified.\n\nExample: 2020-01-01."
- ),
- "The date on which the CreativeWork was created."
- );
- assert_eq!(
- first_sentence("一个有名有姓的人。可以是虚构的。"),
- "一个有名有姓的人。"
- );
- assert_eq!(first_sentence("A person"), "A person");
- assert_eq!(first_sentence(" "), "");
- }
-
#[test]
fn parse_time_precisions() {
assert_eq!(parse_time("2024").unwrap().1, "year");
@@ -1997,211 +1009,6 @@ mod tests {
);
assert_eq!(normalize_attr_value("text", &json!([1])), None);
}
-
- /// **一条坏记录不该毁掉一整块。**
- ///
- /// 形态取自真实日志:`missing field \`predicate\``。模型偶尔会漏写这个字段
- /// (`related_to` 退场后它没有万能选项可挑),从前 serde 会让整块作废,
- /// 而这一块里另外两条事实是好的。
- #[test]
- fn one_malformed_fact_does_not_take_the_whole_chunk() {
- let raw = r#"{
- "entities": [{"name": "OpenAI", "type": "organization"}],
- "facts": [
- {"subject": "OpenAI", "predicate": "produces", "object": "GPT-4"},
- {"subject": "OpenAI", "object": "ChatGPT"},
- {"subject": "Sam Altman", "predicate": "leads", "object": "OpenAI"}
- ]
- }"#;
- let x = parse_response(raw).unwrap();
- assert_eq!(x.facts.len(), 2, "好的两条该留下");
- assert_eq!(x.skipped_facts, 1, "跳过的那条要报出来,不能静默");
- assert_eq!(x.entities.len(), 1);
- assert!(!x.truncated);
- }
-
- /// **输出被截断时,已经完整的那些要救回来。**
- ///
- /// 撞上 max_tokens 时模型写到一半就没了(真实日志:`EOF while parsing a list`)。
- /// 前面的对象是完整且正确的,整块作废等于把抽对的十几条一起扔掉。
- #[test]
- fn a_cut_off_reply_keeps_what_was_complete() {
- let raw = r#"{
- "entities": [{"name": "Anthropic", "type": "organization"}],
- "facts": [
- {"subject": "Anthropic", "predicate": "produces", "object": "Claude"},
- {"subject": "Dario Amodei", "predicate": "leads", "object": "Anthropic"},
- {"subject": "Anthropic", "predicate": "loca"#;
- let x = parse_response(raw).unwrap();
- assert!(x.truncated, "截断要标出来");
- assert_eq!(x.facts.len(), 2, "断点之前的两条是完整的");
- assert_eq!(x.entities.len(), 1);
- }
-
- /// 括号出现在字符串里不算结构——`"a[b"` 不是一个开括号。
- #[test]
- fn brackets_inside_strings_are_not_structure() {
- let raw =
- r#"{"entities": [], "facts": [{"subject": "a[b{c", "predicate": "p", "object": "o"}]}"#;
- let x = parse_response(raw).unwrap();
- assert_eq!(x.facts.len(), 1);
- assert!(!x.truncated, "结构完整,不该判成截断");
- }
-
- /// 连一个完整对象都没有时,仍然要报失败——**容错不是把空结果说成成功**。
- #[test]
- fn a_reply_with_nothing_complete_still_fails() {
- assert!(parse_response(r#"{"facts": [{"subject": "a"#).is_err());
- }
-
- /// 片段字段可有可无:老模型输出没有它们,照常解析
- #[test]
- fn spans_parse_and_default_to_none() {
- let with = parse_response(
- r#"{"entities":[],"facts":[{"subject":"OpenAI","predicate":"founded","object":"Anthropic","subject_span":"Former OpenAI personnel","object_span":"Anthropic"}]}"#,
- )
- .unwrap();
- assert_eq!(
- with.facts[0].subject_span.as_deref(),
- Some("Former OpenAI personnel")
- );
- assert_eq!(with.facts[0].object_span.as_deref(), Some("Anthropic"));
- let without = parse_response(
- r#"{"entities":[],"facts":[{"subject":"OpenAI","predicate":"founded","object":"Anthropic"}]}"#,
- )
- .unwrap();
- assert!(without.facts[0].subject_span.is_none());
- assert!(without.facts[0].object_span.is_none());
- }
-
- #[test]
- fn names_are_parsed_and_a_malformed_one_is_skipped() {
- let raw = r#"{"entities":[{"local_id":"e1","name":"海洋探测器1号","type":"equipment"}],
- "facts":[],
- "names":[{"ref":"e1","name":"海探1","quote":"海洋探测器1号(简称“海探1”)"},
- {"name":"no ref"}]}"#;
- let x = parse_response(raw).unwrap();
- assert_eq!(x.names.len(), 1);
- assert_eq!(x.names[0].entity_ref, "e1");
- assert_eq!(x.names[0].name, "海探1");
- assert_eq!(x.skipped_entities, 0, "a bad name is not a skipped entity");
- }
-
- #[test]
- fn the_contract_asks_for_other_names_and_forbids_descriptions() {
- let msgs = build_messages(&[], &[], &[], None, "f.txt", &[], "text");
- let system = &msgs[0].content;
- assert!(system.contains("\"names\":[{\"ref\""));
- assert!(system.contains("1b. Every other name the text gives an entity"));
- }
-
- #[test]
- fn parse_response_with_fence() {
- let raw = "好的,结果如下:\n```json\n{\"entities\":[{\"name\":\"张三\",\"type\":\"person\"}],\"facts\":[]}\n```";
- let e = parse_response(raw).unwrap();
- assert_eq!(e.entities.len(), 1);
- assert_eq!(e.entities[0].type_key, "person");
- }
-
- /// #690:思考过程里的大括号不能把 JSON 的起止带偏。
- ///
- /// 思考过程里这个没闭合的 `{` 会把"第一个 `{`"的起点提前,而修补截断的逻辑
- /// 认不出夹在中间的废话——不切掉标记,整块直接报解析失败作废。
- #[test]
- fn parse_response_ignores_a_think_block_before_the_json() {
- let raw = "先想想 {\"a\": 1,再回答。\n</think>{\"entities\":[{\"name\":\"张三\",\"type\":\"person\"}],\"facts\":[]}";
- let e = parse_response(raw).unwrap();
- assert_eq!(e.entities.len(), 1);
- assert_eq!(e.entities[0].type_key, "person");
- }
-
- #[test]
- fn handles_and_fact_refs_are_optional_and_legacy_compatible() {
- let handled = parse_response(
- r#"{"entities":[{"local_id":"e1","name":"Zhang Wei","type":"person"}],
- "facts":[{"subject":"Zhang Wei","subject_ref":"e1","predicate":"leads",
- "object":"Finance","object_ref":"e2"}]}"#,
- )
- .unwrap();
- assert_eq!(handled.entities[0].local_id.as_deref(), Some("e1"));
- assert_eq!(handled.facts[0].subject_ref.as_deref(), Some("e1"));
- assert_eq!(handled.facts[0].object_ref.as_deref(), Some("e2"));
-
- let legacy = parse_response(
- r#"{"entities":[{"name":"Zhang Wei","type":"person"}],
- "facts":[{"subject":"Zhang Wei","predicate":"leads","object":"Finance"}]}"#,
- )
- .unwrap();
- assert_eq!(legacy.entities[0].local_id, None);
- assert_eq!(legacy.facts[0].subject_ref, None);
- assert_eq!(legacy.facts[0].object_ref, None);
- }
-
- #[test]
- fn duplicate_handle_definitions_are_all_malformed() {
- let x = parse_response(
- r#"{"entities":[
- {"local_id":"e1","name":"Zhang Wei","type":"person"},
- {"local_id":"e1","name":"John Smith","type":"person"},
- {"local_id":"e2","name":"Finance","type":"organization"}],
- "facts":[]}"#,
- )
- .unwrap();
- assert_eq!(x.skipped_entities, 2);
- assert_eq!(x.entities.len(), 1);
- assert_eq!(x.entities[0].local_id.as_deref(), Some("e2"));
- }
-
- /// specific_type 在骨架里、也在规则里,且两处都说"永远要填"。
- ///
- /// 只写进骨架是不够的:**规则与骨架冲突时骨架赢**(语言那条就栽过一次)。
- /// 这里两边一致,所以要一起钉住。
- #[test]
- fn every_entity_is_asked_for_its_own_words() {
- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");
- let sys = &msgs[0].content;
- assert!(sys.contains("\"specific_type\":\"what you would call it\""));
- assert!(sys.contains("required on every entity"));
- // 关键的一句:不校验。校验它就等于又造了一个词表
- assert!(sys.contains("never checked against the list"));
- // 与 type 的关系必须说清楚,否则模型会把粗类抄一遍
- assert!(sys.contains("narrower than"));
- }
-
- #[test]
- fn extraction_contract_explains_handle_identity_without_forcing_splits() {
- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");
- let system = &msgs[0].content;
- assert!(system.contains("\"local_id\":\"e1\""));
- assert!(system.contains("\"subject_ref\":\"e1\""));
- // #578:跟 X 有关的一群人不是 X
- assert!(system.contains("subject_span and object_span are the exact words"));
- assert!(system.contains("unique within this response"));
- assert!(system.contains("Reuse the same local_id"));
- assert!(system.contains("permanent identity is proven"));
- assert!(system.contains("A shared name alone neither proves sameness nor requires a split"));
- }
-
- #[test]
- fn same_name_known_entities_keep_distinct_response_handles() {
- let known = vec![
- KnownEntity {
- handle: "k1".into(),
- type_key: "person".into(),
- name: "Zhang Wei".into(),
- },
- KnownEntity {
- handle: "k2".into(),
- type_key: "person".into(),
- name: "Zhang Wei".into(),
- },
- ];
- let msgs = build_messages(&[], &[], &[], None, "a.txt", &known, "text");
- let user = &msgs[1].content;
- assert!(user.contains("k1 [person]: Zhang Wei"));
- assert!(user.contains("k2 [person]: Zhang Wei"));
- assert!(user.contains("subject_ref/object_ref"));
- }
}
#[cfg(test)]
diff --git a/crates/utopia-extract/src/normalize.rs b/crates/utopia-extract/src/normalize.rs
deleted file mode 100644
index e2abbae4..00000000
--- a/crates/utopia-extract/src/normalize.rs
+++ /dev/null
@@ -1,1067 +0,0 @@
-//! 模型回复落库前的**形状检查**:只看结构,不看词。
-//!
-//! **分工。**读懂原文里的时间、判断一段话是不是一个东西——这是语言问题,归模型,
-//! 契约(提示词 3c)说清楚它该怎么写。这里只核对输出有没有照契约的形状写,判据一律
-//! 是结构性的:引文里有没有这段字、值是不是只有标点、一侧是不是契约的日期格式、
-//! 同一句里有没有另一条边。**不认任何一种语言的词**——第一版按英文词表认「季度」
-//!「N months ended」,中文财报一条都认不出,还把四份报告的标题当成期间删了。
-//!
-//! 每条规则做了什么都返回给服务端记进丢弃表:违约多常见、出在哪个模型,量得出来,
-//! 契约该怎么改看数说话。
-
-use crate::{read_time, ExtractedEntity, ExtractedFact, Extraction};
-use std::collections::HashSet;
-
-/// 形状检查做了什么;服务端按条记信号
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum Normalization {
- /// 值只有破折号(`—`)或是空的:表里的「无」,不是一个值,不落。**只认破折号**:
- /// `☒`、`✓` 这类符号没有字母数字,却是一格写着的内容(勾选了),照值落
- NoValue { predicate: String, written: String },
- /// 值后面有一截引文里没有的字:只留引文里有的那段。模型读对了表头、却把期间
- /// 写进了值(`(6,176) for three months ended July 27, 2025`),引文只有 `(6,176)`
- ValueTrimmed {
- predicate: String,
- kept: String,
- dropped: String,
- },
- /// 没有宾语、没有值、只带边属性:每个属性落成主语上的一条值事实。
- /// 从前整条以 object_missing 丢掉,写对了的数跟着没了
- QualifiersWithoutObject { predicate: String, values: usize },
- /// 宾语是契约格式的日期(`2026-06-30`):时间不是实体。边上的数落成值、日期进有效期;
- /// 没带数的,把写出来的那段落成值——`2028`(「2028 年起上线」)、`4000`(人数)都
- /// 解析得成年份,丢掉就把一条信息整个丢了
- TimeAsObject {
- predicate: String,
- written: String,
- values: usize,
- },
- /// 主语是契约格式的日期:数是某个东西在那一刻的数,那个东西是谁回复里没说。不落
- TimeAsSubject { predicate: String, written: String },
- /// 宾语的名字包住了另一个声明实体,而同一句、同主语、同谓词已有一条指向那个实体的边:
- /// 它**可能**是那个实体的描述。**只记,不删**:「non-GAAP net income, or earnings, per
- /// diluted share」包住了「non-GAAP net income」,却是另一个指标(每股收益)。结构上
- /// 分不出描述与另一个东西,删错了就是实体连事实一起没了
- ObjectDescribesDeclared {
- predicate: String,
- name: String,
- head: String,
- },
- /// 上面几条去掉事实之后,没有任何事实再引用的声明:不建,否则就是一个孤点
- OrphanDeclaration { name: String },
- /// 引文抄自提示词里附的文件开头,而不是这一块:开头只作背景,它自己那一块会抽到。
- /// 照落的话,证据挂在这一块上,引的却是第一块的话——引错了出处
- QuoteFromOpening { predicate: String, quote: String },
-}
-
-/// 比对用的形态:空白折叠、小写
-fn norm(s: &str) -> String {
- s.split_whitespace()
- .collect::<Vec<_>>()
- .join(" ")
- .to_lowercase()
-}
-
-/// 比对引文用的词元:一段连续的数字(中间的 `,` `.` 算在数里,`13,237`、`89.0`),或者
-/// 一段连续的非数字字母。其余字符都是分隔。
-///
-/// **按词元比,不按子串比**:从前 `10 to 15 GW` 对着「10–15 GW」,`10` 作为子串在引文
-/// 里,尾巴 `to 15 GW` 作为整段不在,于是被剪成 `10`。数字与汉字之间也切开——中文里数
-/// 贴着字写(「营收为12,345元」),不切的话一个数永远对不上
-fn tokens(t: &str) -> Vec<String> {
- let chars: Vec<char> = t.chars().collect();
- let mut out = Vec::new();
- let mut cur = String::new();
- let mut cur_digit = false;
- for (i, &c) in chars.iter().enumerate() {
- let digit = c.is_numeric();
- let joins_number =
- (c == ',' || c == '.') && cur_digit && chars.get(i + 1).is_some_and(|n| n.is_numeric());
- if joins_number {
- cur.push(c);
- continue;
- }
- if !c.is_alphanumeric() {
- if !cur.is_empty() {
- out.push(std::mem::take(&mut cur));
- }
- continue;
- }
- if !cur.is_empty() && digit != cur_digit {
- out.push(std::mem::take(&mut cur));
- }
- cur_digit = digit;
- cur.extend(c.to_lowercase());
- }
- if !cur.is_empty() {
- out.push(cur);
- }
- out
-}
-
-/// `needle` 的词元是否作为连续的一段出现在 `hay` 的词元里
-fn contains_tokens(hay: &[String], needle: &[String]) -> bool {
- !needle.is_empty() && hay.windows(needle.len()).any(|w| w == needle)
-}
-
-fn has_digit(t: &str) -> bool {
- t.chars().any(char::is_numeric)
-}
-
-/// 值后面是否挂着一截不属于它的字。返回 (保留的前缀, 去掉的尾巴)。
-///
-/// 两个条件同时成立才剪,都是结构,不认词,比的都是整词元(见 [`tokens`]):
-/// - **保留的那段在引文里,而且是一格的写法**——含数字、词元连续地出现在引文里,是原文
-/// 写的那个数;或者只有破折号(`—`),是原文那一格写的「没有」;
-/// - **尾巴自己含数字,而且那些数一个都不在引文里**——它是另一条信息(一个期间、一个
-/// 日期、另一个百分比),不是这个数的单位。尾巴里有一个数在引文里,就说明它是原文的
-/// 一部分换了写法(`10 to 15 GW` 对着「10–15 GW」),不剪。
-///
-/// 第二条要数字,是因为挂在数后面、引文里又没有的,还有一类是对的:表头上的量级与
-/// 单位(`53,954 million USD`,这一行引文只有 `53,954`,`million` 在表头「in millions」)、
-/// 模型换了写法的单位(`10 gigawatts`)、缩写的头衔(`founder and CEO`)。它们不带数字,
-/// 不剪。整个值在引文里的,一个字不动。
-fn ungrounded_tail<'a>(value: &'a str, quote: &str) -> Option<(&'a str, &'a str)> {
- let q = norm(quote);
- if q.is_empty() || q.contains(&norm(value)) {
- return None;
- }
- let quote_tokens = tokens(quote);
- let foreign_figures = |tail: &str| {
- let numbers: Vec<String> = tokens(tail).into_iter().filter(|w| has_digit(w)).collect();
- !numbers.is_empty() && numbers.iter().all(|w| !quote_tokens.contains(w))
- };
- let bounds: Vec<usize> = value
- .char_indices()
- .filter(|(i, c)| c.is_whitespace() && *i > 0)
- .map(|(i, _)| i)
- .collect();
- bounds
- .iter()
- .rev()
- .map(|&i| {
- (
- value[..i]
- .trim()
- .trim_end_matches([',', ';', ':', '\u{3001}', '\u{FF0C}']),
- value[i..].trim(),
- )
- })
- .find(|(p, r)| {
- if p.is_empty() || r.is_empty() || !foreign_figures(r) {
- return false;
- }
- if is_no_value(p) {
- return q.contains(&norm(p));
- }
- has_digit(p) && contains_tokens(&quote_tokens, &tokens(p))
- })
-}
-
-/// 表格里表示「没有」的那一格:空的,或者只有破折号。
-///
-/// **只认破折号(Unicode 的 Pd 类)**,不是「没有字母数字就算」:`☒`、`✓`、`☐` 也没有
-/// 字母数字,却是那一格真写着的内容,从前被当成空格子丢了
-fn is_no_value(written: &str) -> bool {
- written.chars().filter(|c| !c.is_whitespace()).all(|c| {
- matches!(
- c,
- '-' | '\u{058A}' | '\u{05BE}' | '\u{1400}' | '\u{1806}' | '\u{2010}'
- ..='\u{2015}'
- | '\u{2E17}'
- | '\u{2E1A}'
- | '\u{2E3A}'
- | '\u{2E3B}'
- | '\u{2E40}'
- | '\u{301C}'
- | '\u{3030}'
- | '\u{30A0}'
- | '\u{FE31}'
- | '\u{FE32}'
- | '\u{FE58}'
- | '\u{FE63}'
- | '\u{FF0D}'
- )
- })
-}
-
-/// 边属性里不是值、是单位的那几个键(与服务端同一张表)
-fn is_unit_key(k: &str) -> bool {
- matches!(
- k.trim().to_lowercase().as_str(),
- "currency" | "币种" | "货币" | "unit" | "单位"
- )
-}
-
-fn qualifier_values(f: &ExtractedFact) -> Vec<(String, serde_json::Value)> {
- f.qualifiers
- .as_ref()
- .map(|q| {
- q.iter()
- .filter(|(k, v)| !v.is_null() && !is_unit_key(k))
- .map(|(k, v)| (k.clone(), v.clone()))
- .collect()
- })
- .unwrap_or_default()
-}
-
-fn value_fact(
- f: &ExtractedFact,
- predicate: String,
- value: serde_json::Value,
- valid_from: Option<String>,
-) -> ExtractedFact {
- ExtractedFact {
- subject: f.subject.clone(),
- subject_ref: f.subject_ref.clone(),
- predicate,
- object: None,
- object_ref: None,
- value: Some(value),
- qualifiers: None,
- valid_from: valid_from.or_else(|| f.valid_from.clone()),
- valid_to: f.valid_to.clone(),
- confidence: f.confidence,
- quote: f.quote.clone(),
- subject_span: f.subject_span.clone(),
- object_span: None,
- relative: false,
- }
-}
-
-/// 一侧写的是一个时间:规则 3 的格式(`YYYY` / `YYYY-MM` / `YYYY-MM-DD`,带时区的时刻),
-/// 或写法说得清是哪天的日期(`written_date`,#688)
-fn names_a_time(s: &str) -> bool {
- read_time(s.trim()).is_some()
-}
-
-/// 引文不在这一块、却在附上的文件开头里的事实:丢掉,连同因此没人引用的声明。
-///
-/// 只比「在不在」(空白、大小写不论),不比像不像:两边都找不到的引文不归这里管——
-/// 那是模型改写了原文,与开头无关
-pub fn drop_quotes_from_opening(
- x: &mut Extraction,
- chunk_text: &str,
- opening: &str,
-) -> Vec<Normalization> {
- let (chunk, opening) = (norm(chunk_text), norm(opening));
- if opening.is_empty() {
- return Vec::new();
- }
- let before = referenced_names(&x.entities, &x.facts);
- let mut out = Vec::new();
- x.facts.retain(|f| {
- let q = norm(f.quote.as_deref().unwrap_or(""));
- let from_opening = !q.is_empty() && !chunk.contains(&q) && opening.contains(&q);
- if from_opening {
- out.push(Normalization::QuoteFromOpening {
- predicate: f.predicate.clone(),
- quote: f.quote.clone().unwrap_or_default(),
- });
- }
- !from_opening
- });
- if out.is_empty() {
- return out;
- }
- let mut after = referenced_names(&x.entities, &x.facts);
- // 与 normalize_facts 同一条:模型给它报了别的名字的声明不算孤点(0041)
- after.extend(x.names.iter().filter_map(|n| {
- x.entities
- .iter()
- .find(|e| e.local_id.as_deref().map(str::trim) == Some(n.entity_ref.trim()))
- .map(|e| e.name.trim().to_lowercase())
- }));
- let mut orphans = Vec::new();
- x.entities.retain(|e| {
- let n = e.name.trim().to_lowercase();
- let orphan = before.contains(&n) && !after.contains(&n);
- if orphan {
- orphans.push(e.name.trim().to_string());
- }
- !orphan
- });
- out.extend(
- orphans
- .into_iter()
- .map(|name| Normalization::OrphanDeclaration { name }),
- );
- out
-}
-
-/// 事实两侧绑到的声明名(小写):有句柄按句柄,没有按写出来的名字
-fn referenced_names(entities: &[ExtractedEntity], facts: &[ExtractedFact]) -> HashSet<String> {
- let handle_name = |h: Option<&String>| {
- h.and_then(|h| {
- entities
- .iter()
- .find(|e| e.local_id.as_deref().map(str::trim) == Some(h.trim()))
- .map(|e| e.name.trim().to_string())
- })
- };
- facts
- .iter()
- .flat_map(|f| {
- [
- handle_name(f.subject_ref.as_ref()).or_else(|| Some(f.subject.trim().to_string())),
- handle_name(f.object_ref.as_ref())
- .or_else(|| f.object.as_deref().map(|o| o.trim().to_string())),
- ]
- })
- .flatten()
- .map(|n| n.to_lowercase())
- .collect()
-}
-
-pub fn normalize_facts(x: &mut Extraction) -> Vec<Normalization> {
- let mut out = Vec::new();
- let entities = std::mem::take(&mut x.entities);
-
- // 一侧绑到的声明名:有句柄按句柄,没有按写出来的名字
- let handle_name = |h: Option<&String>| {
- h.and_then(|h| {
- entities
- .iter()
- .find(|e| e.local_id.as_deref().map(str::trim) == Some(h.trim()))
- .map(|e| e.name.trim().to_string())
- })
- };
- let before = referenced_names(&entities, &x.facts);
-
- let mut facts: Vec<ExtractedFact> = Vec::with_capacity(x.facts.len());
- for mut f in std::mem::take(&mut x.facts) {
- let quote = f.quote.clone().unwrap_or_default();
-
- // ---- 值 ----
- if let Some(written) = f.value.as_ref().and_then(|v| v.as_str()).map(str::to_owned) {
- // 先剪再看空:`— for three months ended July 27, 2025` 剪掉尾巴才露出那一格是空的
- let figure = ungrounded_tail(&written, &quote).map_or(written.as_str(), |(k, _)| k);
- if is_no_value(figure) {
- out.push(Normalization::NoValue {
- predicate: f.predicate.clone(),
- written,
- });
- continue;
- }
- if let Some((kept, dropped)) = ungrounded_tail(&written, &quote) {
- out.push(Normalization::ValueTrimmed {
- predicate: f.predicate.clone(),
- kept: kept.to_string(),
- dropped: dropped.to_string(),
- });
- f.value = Some(serde_json::Value::String(kept.to_string()));
- }
- }
-
- // ---- 主语是时间 ----
- let subject =
- handle_name(f.subject_ref.as_ref()).unwrap_or_else(|| f.subject.trim().to_string());
- if names_a_time(&subject) {
- out.push(Normalization::TimeAsSubject {
- predicate: f.predicate.clone(),
- written: subject,
- });
- continue;
- }
-
- let values = qualifier_values(&f);
- let object = handle_name(f.object_ref.as_ref())
- .or_else(|| f.object.as_deref().map(|o| o.trim().to_string()))
- .filter(|o| !o.is_empty());
- let has_value = f.value.as_ref().is_some_and(|v| !v.is_null());
-
- // ---- 只有边属性 ----
- if object.is_none() && !has_value && !values.is_empty() {
- for (key, value) in &values {
- let predicate = format!("{}.{}", f.predicate.trim(), key.trim());
- facts.push(value_fact(&f, predicate, value.clone(), None));
- }
- out.push(Normalization::QualifiersWithoutObject {
- predicate: f.predicate.clone(),
- values: values.len(),
- });
- continue;
- }
-
- // ---- 宾语是时间 ----
- if let Some(o) = object.as_deref().filter(|o| names_a_time(o)) {
- if values.is_empty() {
- // 没带数:写出来的那段就是值。`2028`(「2028 年起上线」)、`4000`(人数)都解析
- // 得成年份,从前整条丢掉,一条信息就没了。宾语那个声明没人引用,下面按孤点去掉
- facts.push(value_fact(
- &f,
- f.predicate.trim().to_string(),
- serde_json::Value::String(o.to_string()),
- None,
- ));
- out.push(Normalization::TimeAsObject {
- predicate: f.predicate.clone(),
- written: o.to_string(),
- values: 0,
- });
- continue;
- }
- let several = values.len() > 1;
- for (key, value) in &values {
- let predicate = if several {
- format!("{}.{}", f.predicate.trim(), key.trim())
- } else {
- f.predicate.trim().to_string()
- };
- facts.push(value_fact(
- &f,
- predicate,
- value.clone(),
- Some(o.to_string()),
- ));
- }
- out.push(Normalization::TimeAsObject {
- predicate: f.predicate.clone(),
- written: o.to_string(),
- values: values.len(),
- });
- continue;
- }
-
- facts.push(f);
- }
-
- // ---- 描述:名字包住另一个声明实体,本尊那条边同句已在。只记,不删 ----
- let declared: Vec<String> = entities.iter().map(|e| e.name.trim().to_string()).collect();
- let side = |r: Option<&String>, w: Option<&str>| {
- handle_name(r)
- .or_else(|| w.map(|s| s.trim().to_string()))
- .filter(|s| !s.is_empty())
- };
- let objects: Vec<Option<String>> = facts
- .iter()
- .map(|f| side(f.object_ref.as_ref(), f.object.as_deref()))
- .collect();
- let subjects: Vec<Option<String>> = facts
- .iter()
- .map(|f| side(f.subject_ref.as_ref(), Some(f.subject.as_str())))
- .collect();
- // 名字边界:拉丁字母数字前后不能粘着字母数字;汉字之间本来就没有空格,不设边界
- let contains_name = |outer: &str, inner: &str| {
- let (o, i) = (norm(outer), norm(inner));
- !i.is_empty()
- && o.len() > i.len()
- && o.match_indices(&i).any(|(at, _)| {
- let glued = |c: Option<char>| c.is_some_and(|c| c.is_ascii_alphanumeric());
- let edge_in = i.chars().next().is_some_and(|c| c.is_ascii_alphanumeric());
- let edge_out = i
- .chars()
- .next_back()
- .is_some_and(|c| c.is_ascii_alphanumeric());
- !(edge_in && glued(o[..at].chars().next_back()))
- && !(edge_out && glued(o[at + i.len()..].chars().next()))
- })
- };
- for i in 0..facts.len() {
- let Some(name) = objects[i].as_deref() else {
- continue;
- };
- let quote_i = norm(facts[i].quote.as_deref().unwrap_or(""));
- let head = declared.iter().find(|h| {
- contains_name(name, h)
- && (0..facts.len()).any(|j| {
- j != i
- && facts[j].predicate.eq_ignore_ascii_case(&facts[i].predicate)
- && subjects[j] == subjects[i]
- && objects[j]
- .as_deref()
- .is_some_and(|o| o.eq_ignore_ascii_case(h))
- && {
- let quote_j = norm(facts[j].quote.as_deref().unwrap_or(""));
- quote_i.contains(&quote_j) || quote_j.contains(&quote_i)
- }
- })
- });
- if let Some(head) = head {
- // 不删(见枚举上的说明):结构分不出「SB Energy 的发展」与「每股收益」这类
- // 包住了另一个名字的真指标。记下来,量得出这种形状多常见、有多少是描述
- out.push(Normalization::ObjectDescribesDeclared {
- predicate: facts[i].predicate.clone(),
- name: name.to_string(),
- head: head.clone(),
- });
- }
- }
- x.facts = facts;
-
- // ---- 被上面几条弄成孤点的声明 ----
- // 只去掉「原来有事实引用、现在没有了」的:模型一开始就只声明不连边的,不归这里管
- let mut after = referenced_names(&entities, &x.facts);
- // 模型给它报了别的名字的声明也不算孤点:那些名字要绑在它身上(0041)
- after.extend(
- x.names
- .iter()
- .filter_map(|n| handle_name(Some(&n.entity_ref)))
- .map(|n| n.to_lowercase()),
- );
- let mut orphans = Vec::new();
- let mut kept_entities = entities;
- kept_entities.retain(|e| {
- let n = e.name.trim().to_lowercase();
- let orphan = before.contains(&n) && !after.contains(&n);
- if orphan {
- orphans.push(e.name.trim().to_string());
- }
- !orphan
- });
- x.entities = kept_entities;
- out.extend(
- orphans
- .into_iter()
- .map(|name| Normalization::OrphanDeclaration { name }),
- );
- out
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::ExtractedEntity;
-
- fn fact(subject: &str, predicate: &str, object: Option<&str>, quote: &str) -> ExtractedFact {
- ExtractedFact {
- subject: subject.into(),
- subject_ref: None,
- predicate: predicate.into(),
- object: object.map(str::to_string),
- object_ref: None,
- value: None,
- qualifiers: None,
- valid_from: None,
- valid_to: None,
- confidence: Some(0.9),
- quote: Some(quote.into()),
- subject_span: Some(subject.into()),
- object_span: object.map(str::to_string),
- relative: false,
- }
- }
- fn valued(subject: &str, predicate: &str, value: &str, quote: &str) -> ExtractedFact {
- let mut f = fact(subject, predicate, None, quote);
- f.value = Some(serde_json::Value::String(value.into()));
- f
- }
- fn entity(id: &str, name: &str) -> ExtractedEntity {
- ExtractedEntity {
- local_id: Some(id.into()),
- name: name.into(),
- type_key: "organization".into(),
- specific_type: None,
- }
- }
- fn quals(pairs: &[(&str, &str)]) -> serde_json::Map<String, serde_json::Value> {
- pairs
- .iter()
- .map(|(k, v)| (k.to_string(), serde_json::Value::String(v.to_string())))
- .collect()
- }
- /// 第五份补充协议的第三块:模型从附上的开头里抄了「as of February 18, 2020」那句当引文。
- /// 那条事实丢掉,只为它而声明的实体跟着不建;引文在这一块里的照落
- #[test]
- fn a_fact_quoting_the_opening_is_dropped_with_its_orphan() {
- let opening = "THIS FIFTH AMENDMENT TO LEASE AGREEMENT is entered into as of February 18, 2020 by and between HPBB1, LLC and BLACKBAUD, INC.";
- let chunk = "The Phase 2 Exercise Deadline is hereby extended to March 17, 2020.";
- let mut x = Extraction {
- entities: vec![entity("e1", "Lease"), entity("e2", "HPBB1, LLC")],
- facts: vec![
- fact(
- "Lease",
- "landlord",
- Some("HPBB1, LLC"),
- "entered into as of February 18, 2020 by and between HPBB1, LLC",
- ),
- fact(
- "Lease",
- "expansion_option_deadline",
- None,
- "the phase 2 exercise deadline is hereby extended to March 17, 2020",
- ),
- ],
- names: Vec::new(),
- skipped_entities: 0,
- skipped_facts: 0,
- truncated: false,
- };
- let n = drop_quotes_from_opening(&mut x, chunk, opening);
- let kept: Vec<&str> = x.facts.iter().map(|f| f.predicate.as_str()).collect();
- assert_eq!(
- kept,
- ["expansion_option_deadline"],
- "大小写与空白不论,引文在这一块里的留下"
- );
- let names: Vec<&str> = x.entities.iter().map(|e| e.name.as_str()).collect();
- assert_eq!(names, ["Lease"], "只为被丢那条声明的 HPBB1 不建");
- assert!(n
- .iter()
- .any(|v| matches!(v, Normalization::QuoteFromOpening { .. })));
- assert!(n
- .iter()
- .any(|v| matches!(v, Normalization::OrphanDeclaration { .. })));
-
- // 两边都找不到的引文不归这里管;没有开头时什么都不做
- let mut y = Extraction {
- entities: vec![entity("e1", "Lease")],
- facts: vec![fact("Lease", "note", None, "a paraphrase of neither")],
- names: Vec::new(),
- skipped_entities: 0,
- skipped_facts: 0,
- truncated: false,
- };
- assert!(drop_quotes_from_opening(&mut y, chunk, opening).is_empty());
- assert!(drop_quotes_from_opening(&mut y, chunk, " ").is_empty());
- assert_eq!(y.facts.len(), 1);
- }
-
- fn run(
- entities: Vec<ExtractedEntity>,
- facts: Vec<ExtractedFact>,
- ) -> (Extraction, Vec<Normalization>) {
- let mut x = Extraction {
- entities,
- facts,
- skipped_entities: 0,
- skipped_facts: 0,
- truncated: false,
- names: Vec::new(),
- };
- let n = normalize_facts(&mut x);
- (x, n)
- }
- fn value_of(f: &ExtractedFact) -> &str {
- f.value.as_ref().unwrap().as_str().unwrap()
- }
-
- /// 截图里那 21 条:模型读对了表头,却把期间写进了值。引文里只有数
- #[test]
- fn a_tail_the_quote_does_not_contain_is_not_the_value() {
- let (x, n) = run(
- vec![entity("e1", "NVIDIA")],
- vec![
- valued(
- "NVIDIA",
- "net_cash_used",
- "(6,176) for three months ended July 27, 2025",
- "Net cash used in financing activities (6,176)",
- ),
- // 同样的形状,中文:不认词,照样剪
- valued(
- "NVIDIA",
- "经营现金流",
- "12,345 截至2025年7月27日止三个月",
- "经营活动产生的现金流量净额 | 12,345",
- ),
- valued(
- "NVIDIA",
- "revenue",
- "$89.0 billion, up 18% from the previous quarter",
- "Second-quarter revenue was $89.0 billion",
- ),
- ],
- );
- assert_eq!(value_of(&x.facts[0]), "(6,176)");
- assert_eq!(value_of(&x.facts[1]), "12,345");
- // 数后面接着另一个数:「up 18%」是另一条事实,不是这个数的一部分;逗号跟着前缀走
- assert_eq!(value_of(&x.facts[2]), "$89.0 billion");
- assert!(
- matches!(&n[0], Normalization::ValueTrimmed { dropped, .. } if dropped == "for three months ended July 27, 2025")
- );
- }
-
- #[test]
- fn a_value_grounded_in_the_quote_is_left_alone() {
- let (x, n) = run(
- vec![entity("e1", "Tench Coxe")],
- vec![
- // 整个值在引文里
- valued(
- "NVIDIA",
- "operating_expenses",
- "$9.2 billion",
- "expected to be approximately $9.2 billion and $9.0 billion",
- ),
- // 尾巴 shares 在引文别处出现过:原文的单位,不剪
- valued(
- "Tench Coxe",
- "votes_for",
- "15,411,252,412 shares",
- "Number of shares For | 15,411,252,412",
- ),
- // 规范化过的值,前缀一个词都对不上:不是这条规则管的
- valued("Vega", "amount", "$5 billion", "invested 5 billion dollars"),
- // 表头上的量级与币种:引文那一行只有数,尾巴不带数字,是它的单位,不剪
- valued(
- "NVIDIA",
- "net_income",
- "53,954 million USD",
- "Net income | $ | 53,954",
- ),
- // 模型换了写法的单位、缩写的头衔:不带数字,不剪
- valued(
- "SB Energy",
- "capacity",
- "10 gigawatts",
- "at least 10 GW of new generation",
- ),
- valued(
- "Jensen Huang",
- "job_title",
- "founder and CEO",
- "Jensen Huang, founder and chief executive officer",
- ),
- ],
- );
- let got: Vec<&str> = x.facts.iter().map(value_of).collect();
- assert_eq!(
- got,
- [
- "$9.2 billion",
- "15,411,252,412 shares",
- "$5 billion",
- "53,954 million USD",
- "10 gigawatts",
- "founder and CEO"
- ]
- );
- assert!(n.is_empty(), "{n:?}");
- }
-
- #[test]
- fn a_dash_is_no_value() {
- let (x, n) = run(
- vec![entity("e1", "NVIDIA")],
- vec![
- valued("NVIDIA", "dividend", "—", "Dividends | —"),
- // 勾选框是那一格写着的内容,不是空格子:没有字母数字,照值落
- valued(
- "NVIDIA",
- "large_accelerated_filer",
- "☒",
- "Large accelerated filer | ☒",
- ),
- valued(
- "NVIDIA",
- "emerging_growth_company",
- "☐",
- "Emerging growth company | ☐",
- ),
- valued("NVIDIA", "dividend", " – ", "Dividends | –"),
- valued("NVIDIA", "dividend", "$0.01", "Dividends | $0.01"),
- // 空的那一格后面挂着列头上的期间:剪掉尾巴,剩下的仍是空
- valued(
- "NVIDIA",
- "amount",
- "— for three months ended July 27, 2025",
- "Purchases of marketable securities | — | (6,176)",
- ),
- // 反例:前缀只有标点、在引文里,可尾巴上的数也在引文里——那个数才是值
- valued(
- "NVIDIA",
- "net_income",
- "$ 53,954 million",
- "Net income | $ | 53,954",
- ),
- ],
- );
- let kept: Vec<&str> = x.facts.iter().map(value_of).collect();
- assert_eq!(kept, ["☒", "☐", "$0.01", "$ 53,954 million"]);
- assert_eq!(
- n.iter()
- .filter(|v| matches!(v, Normalization::NoValue { .. }))
- .count(),
- 3
- );
- }
-
- /// Coxe 的形状:`vote_result` 带着数,没有宾语也没有值
- #[test]
- fn figures_on_an_edge_with_no_other_end_land_on_the_subject() {
- let mut f = fact(
- "Tench Coxe",
- "vote_result",
- None,
- "Number of shares For | 15,411,252,412",
- );
- f.qualifiers = Some(quals(&[
- ("for", "15,411,252,412"),
- ("against", "1,399,727,580"),
- ("unit", "shares"),
- ]));
- let (x, n) = run(vec![entity("e1", "Tench Coxe")], vec![f]);
- let mut got: Vec<(String, String)> = x
- .facts
- .iter()
- .map(|f| (f.predicate.clone(), value_of(f).to_string()))
- .collect();
- got.sort();
- assert_eq!(
- got,
- [
- (
- "vote_result.against".to_string(),
- "1,399,727,580".to_string()
- ),
- ("vote_result.for".to_string(), "15,411,252,412".to_string()),
- ]
- );
- assert_eq!(
- n,
- vec![Normalization::QualifiersWithoutObject {
- predicate: "vote_result".into(),
- values: 2
- }]
- );
- }
-
- /// 时间做了宾语:边上的数落成值,时间进有效期,那个时间节点不建
- #[test]
- fn a_time_object_becomes_the_validity_and_leaves_no_node() {
- let mut margin = fact(
- "NVIDIA",
- "gross_margin",
- Some("2026-06"),
- "Gross margin | 75.0%",
- );
- margin.object_ref = Some("e7".into());
- margin.qualifiers = Some(quals(&[("percentage", "75.0%")]));
- let mut bare = fact("NVIDIA", "reported_in", Some("2026"), "reported in 2026");
- bare.object_ref = Some("e8".into());
- let (x, n) = run(
- vec![
- entity("e1", "NVIDIA"),
- entity("e7", "2026-06"),
- entity("e8", "2026"),
- ],
- vec![margin, bare],
- );
- // 带数的:数落成值、日期进有效期;没带数的:写出来的那段落成值,不丢
- assert_eq!(x.facts.len(), 2);
- assert_eq!(x.facts[0].predicate, "gross_margin");
- assert_eq!(value_of(&x.facts[0]), "75.0%");
- assert_eq!(x.facts[0].valid_from.as_deref(), Some("2026-06"));
- assert_eq!(
- (x.facts[1].predicate.as_str(), value_of(&x.facts[1])),
- ("reported_in", "2026")
- );
- assert!(x.facts[1].object.is_none() && x.facts[1].object_ref.is_none());
- let names: Vec<&str> = x.entities.iter().map(|e| e.name.as_str()).collect();
- assert_eq!(names, ["NVIDIA"]);
- assert_eq!(
- n.iter()
- .filter(|v| matches!(v, Normalization::OrphanDeclaration { .. }))
- .count(),
- 2
- );
- }
-
- #[test]
- fn a_time_subject_is_not_placed() {
- let f = valued(
- "2026-07-26",
- "revenue",
- "$89.0 billion",
- "revenue was $89.0 billion",
- );
- let (x, n) = run(vec![entity("e1", "NVIDIA")], vec![f]);
- assert!(x.facts.is_empty());
- assert!(matches!(&n[0], Normalization::TimeAsSubject { .. }));
- }
-
- /// 不是契约格式的期间名(`Q2 FY27`、`第二季度`)不归这里判:那是模型的事,契约 3c 管
- #[test]
- fn a_period_name_that_is_not_a_date_is_not_second_guessed() {
- let mut f = fact(
- "NVIDIA",
- "gross_margin_for_period",
- Some("Q2 FY27"),
- "Gross margin | 75.0 | %",
- );
- f.qualifiers = Some(quals(&[("percentage", "75.0%")]));
- let (x, n) = run(
- vec![entity("e1", "NVIDIA"), entity("e7", "Q2 FY27")],
- vec![f],
- );
- assert_eq!(x.facts.len(), 1);
- assert_eq!(x.entities.len(), 2);
- assert!(n.is_empty());
- }
-
- /// SB Energy 那句:名字包住了另一个声明实体,同句同主语同谓词已有一条指向它的边。
- /// 记一条信号,事实与声明都留着——结构分不出它与下面「每股收益」那种真指标
- #[test]
- fn a_description_beside_its_head_is_flagged_not_removed() {
- let quote = "NVIDIA to invest $1.5B in SB Energy now to support SB Energy\u{2019}s growth and commitments to the Ohio community";
- let mut head = fact("NVIDIA", "invested_in", Some("SB Energy"), quote);
- head.object_ref = Some("e2".into());
- let mut desc = fact(
- "NVIDIA",
- "invested_in",
- Some("SB Energy's growth and commitments to the Ohio community"),
- quote,
- );
- desc.object_ref = Some("e6".into());
- let (x, n) = run(
- vec![
- entity("e1", "NVIDIA"),
- entity("e2", "SB Energy"),
- entity(
- "e6",
- "SB Energy's growth and commitments to the Ohio community",
- ),
- ],
- vec![head, desc],
- );
- // 只记不删:两条都在,那个声明也在(它仍被引用)
- assert_eq!(x.facts.len(), 2);
- assert_eq!(x.entities.len(), 3);
- assert!(n.iter().any(
- |v| matches!(v, Normalization::ObjectDescribesDeclared { head, .. } if head == "SB Energy")
- ));
- }
-
- /// 同样的结构,中文:不靠 's,照样认得出这个形状(只记)
- #[test]
- fn a_description_is_recognised_without_a_possessive() {
- let quote = "英伟达向星辰能源投资15亿美元,支持星辰能源在俄亥俄的发展";
- let (x, n) = run(
- vec![
- entity("e1", "英伟达"),
- entity("e2", "星辰能源"),
- entity("e3", "星辰能源在俄亥俄的发展"),
- ],
- vec![
- fact("英伟达", "投资", Some("星辰能源"), quote),
- fact("英伟达", "投资", Some("星辰能源在俄亥俄的发展"), quote),
- ],
- );
- assert_eq!(x.facts.len(), 2);
- assert_eq!(x.entities.len(), 3);
- assert!(n
- .iter()
- .any(|v| matches!(v, Normalization::ObjectDescribesDeclared { .. })));
- }
-
- /// **四处误伤,逐个钉住**(#637 实测):每一处从前都把一条真信息丢了或剪坏了
- #[test]
- fn what_the_shape_checks_used_to_break_is_kept() {
- // 一、短数字被当子串匹配:`10 to 15 GW` 对着「10–15 GW」,从前剪成 `10`
- let (x, n) = run(
- vec![entity("e1", "SB Energy")],
- vec![
- valued(
- "SB Energy",
- "planned_capacity",
- "10 to 15 GW",
- "SB Energy plans 10–15 GW of new generation",
- ),
- // 词元不是子串:`5` 不在「25 GW」里
- valued(
- "SB Energy",
- "planned_capacity",
- "5 GW by 2030",
- "SB Energy plans 25 GW by 2030",
- ),
- // 中文里数贴着字写,照样按词元对得上,尾巴那个日期不在引文里,剪
- valued(
- "SB Energy",
- "营收",
- "12,345 截至2025年7月27日止三个月",
- "营收为12,345元",
- ),
- ],
- );
- let got: Vec<&str> = x.facts.iter().map(value_of).collect();
- assert_eq!(got, ["10 to 15 GW", "5 GW by 2030", "12,345"]);
- assert_eq!(n.len(), 1, "{n:?}");
-
- // 二、勾选框:见 a_dash_is_no_value
-
- // 三、四位数的宾语:`2028`、`4000` 解析得成年份,落成值,不丢
- let mut launch = fact(
- "Aurora",
- "launch_year",
- Some("2028"),
- "Aurora goes live from 2028",
- );
- launch.object_ref = Some("e2".into());
- let mut staff = fact(
- "Acme",
- "employees",
- Some("4000"),
- "Acme employs 4000 people",
- );
- staff.object_ref = Some("e3".into());
- let (x, _) = run(
- vec![
- entity("e1", "Aurora"),
- entity("e2", "2028"),
- entity("e3", "4000"),
- entity("e4", "Acme"),
- ],
- vec![launch, staff],
- );
- let got: Vec<(&str, &str)> = x
- .facts
- .iter()
- .map(|f| (f.predicate.as_str(), value_of(f)))
- .collect();
- assert_eq!(got, [("launch_year", "2028"), ("employees", "4000")]);
- let names: Vec<&str> = x.entities.iter().map(|e| e.name.as_str()).collect();
- assert_eq!(names, ["Aurora", "Acme"], "年份那两个声明不建成节点");
-
- // 四、包住了另一个指标名的真指标:每股收益不是净利润的描述,留着
- let quote = "non-GAAP net income was $26.4 billion and non-GAAP net income, or earnings, per diluted share was $1.05";
- let (x, n) = run(
- vec![
- entity("e1", "NVIDIA"),
- entity("e2", "non-GAAP net income"),
- entity("e3", "non-GAAP net income, or earnings, per diluted share"),
- ],
- vec![
- fact(
- "NVIDIA",
- "reported_metric",
- Some("non-GAAP net income"),
- quote,
- ),
- fact(
- "NVIDIA",
- "reported_metric",
- Some("non-GAAP net income, or earnings, per diluted share"),
- quote,
- ),
- ],
- );
- assert_eq!(x.facts.len(), 2);
- assert_eq!(x.entities.len(), 3);
- assert!(n
- .iter()
- .any(|v| matches!(v, Normalization::ObjectDescribesDeclared { .. })));
- }
-
- /// 包住了别的名字、但旁边没有指向本尊的同一条边:可能真是一个东西,不动
- #[test]
- fn a_name_that_contains_another_without_a_sibling_edge_is_left_alone() {
- let quote = "Sam Altman was removed by OpenAI's board of directors";
- let (x, n) = run(
- vec![
- entity("e1", "Sam Altman"),
- entity("e2", "OpenAI"),
- entity("e3", "OpenAI's board of directors"),
- ],
- vec![fact(
- "Sam Altman",
- "removed_by",
- Some("OpenAI's board of directors"),
- quote,
- )],
- );
- assert_eq!(x.facts.len(), 1);
- assert_eq!(x.entities.len(), 3);
- assert!(n.is_empty());
- }
-}
diff --git a/crates/utopia-server/src/api/kbs.rs b/crates/utopia-server/src/api/kbs.rs
index 57f7c69c..89a4a71d 100644
--- a/crates/utopia-server/src/api/kbs.rs
+++ b/crates/utopia-server/src/api/kbs.rs
@@ -55,10 +55,6 @@ pub struct UpdateKbReq {
/// 提示词都读它。探索生成的描述在另一个字段,PATCH 不了。见 #570
#[serde(default)]
pub data_conventions: Option<String>,
- /// 开放抽取(0044 第一刀,#729):开着,抽取只写开放图谱——陈述照文档的字落库,
- /// 不读本体;关着走今天的类型化那条路。缺省关
- #[serde(default)]
- pub open_extraction: Option<bool>,
}
/// 用户可见的 KB 列表(restricted 库仅矩阵成员与系统管理员可见)。
@@ -119,7 +115,6 @@ pub async fn create(
None,
None,
None,
- None,
)
.await?;
}
@@ -202,7 +197,6 @@ pub async fn update(
req.auto_type_resolution,
req.governance,
req.data_conventions.as_deref().map(str::trim),
- req.open_extraction,
)
.await?;
// 打开开关就自动开始处理:排一轮,同库已排着的不重复
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs
index 647251c5..887aa3e9 100644
--- a/crates/utopia-server/src/extraction.rs
+++ b/crates/utopia-server/src/extraction.rs
@@ -1,13 +1,16 @@
-//! 图谱抽取任务:逐分块调用 LLM → 实体消解 v2 → 事实 + 证据写入账本。
+//! 图谱抽取任务的入口,以及开放写入路径(`extraction_open`)共用的那几块。
+//!
+//! 抽取只写开放图谱(0044 决定 2):每篇文档——记忆日志也在内——都从这里进
+//! `run_open`,提示词里没有本体,也没有带本体的第二条路。这里留下的是那条路借用的东西:
+//! 限流退避的 chat 调用、来源给置信度设的上限、片段核对、丢弃信号、未抽完的判据、
+//! 名字到实体的消解(句柄、同名审核对)。
//! 与摄入管道分离(两段式可用):索引完成即可搜可问,抽取慢慢跑。
//! 消解灰区只入审核队列并触发独立的攒批裁决任务——LLM 裁决永不阻塞本任务。
use crate::llm_util;
-use crate::ontology_index;
-use crate::predicate_match::PredicateIndex;
use crate::state::AppState;
use sqlx::PgPool;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
use std::time::Duration;
use utopia_core::models::Proposer;
use uuid::Uuid;
@@ -31,7 +34,7 @@ fn jitter(base: Duration) -> Duration {
base / 2 + Duration::from_millis(if half == 0 { 0 } else { nanos % half })
}
-/// 抽取的 chat 调用,**限流会退避重试**。
+/// 抽取的 chat 调用,**限流会退避重试**。温度由调用方定(开放抽取要 0,见 `extraction_open`)。
///
/// 限流与其他失败的区别是它会自己好,所以从前那句「跳过该分块」用在它身上
/// 就是把一分钟的等待换成永久的数据缺口——实测一次 1884 块的灌入里
@@ -43,16 +46,6 @@ fn jitter(base: Duration) -> Duration {
/// 否则一个在等的分块会挡住本来可以通过的另一个。
/// - **`Retry-After` 多数厂商不发**,所以它只是「有则更准」,判据是错误类型
/// 本身;没有它就走指数退避。
-pub(crate) async fn chat_retrying_rate_limits(
- state: &AppState,
- settings: &utopia_core::models::LlmSettings,
- client: &utopia_llm::LlmClient,
- messages: &[utopia_llm::ChatMessage],
-) -> anyhow::Result<String> {
- chat_retrying_rate_limits_at(state, settings, client, messages, None).await
-}
-
-/// 同上,指定采样温度(开放抽取要 0,见 `extraction_open`)。
pub(crate) async fn chat_retrying_rate_limits_at(
state: &AppState,
settings: &utopia_core::models::LlmSettings,
@@ -90,8 +83,6 @@ pub(crate) async fn chat_retrying_rate_limits_at(
unreachable!("循环内必定 return")
}
-const MIN_CONFIDENCE: f32 = 0.6;
-
/// 模型看图描述出来的事实,置信度压到这里(0040 决定 4):低于时态引擎自动关闭的门槛,
/// 它能进图、能被搜到和引用,却不能单独把一段正确的旧值关掉——看柱状图读错一个数字是常事,
/// 按普通事实入库的话,错的数会关掉它反驳的那个对的数,而库里没有一行说这次关闭靠的是一张图。
@@ -107,90 +98,6 @@ pub(crate) fn origin_ceiling(origin: &str, confidence: f32) -> f32 {
}
}
-/// 这串字**是不是一个东西的名字**。
-///
-/// 判据是**词数**不是字符数。字符数分不开真假:
-/// `US District Court for the Northern District of California`(57 字符)是真实体,
-/// 而 `removal was driven by growing discontent and distrust with Altman`(65 字符)
-/// 是一整个从句——两者字符数相近,词数也相近(9 vs 10),但后者带着**限定动词**。
-///
-/// 所以两条一起看:词数封顶挡住长句,而**句中的限定动词**挡住那些不长的从句。
-/// 机构名会长("US District Court for the Northern District of California"),
-/// 但不会出现 "was driven by"、"showed"、"giving off" 这种谓语。
-///
-/// 上限取 12 个词:实测真实体里最长的机构名是 9 个词,留三个词的余量。
-/// 而被挡下的那些平均 14 个词。
-const MAX_NAME_WORDS: usize = 12;
-
-/// 句子里的谓语标志。**只列限定形式**——`used`、`flying` 这类分词在名词短语里
-/// 完全正常("equipment used by X"),列进去会误伤真实体。
-const CLAUSE_MARKERS: &[&str] = &[
- "was", "were", "is", "are", "has", "have", "had", "will", "would", "showed", "said", "says",
- "became", "went", "came", "did", "does", "gave", "took", "made",
-];
-
-/// 情态动词:英语里**封闭类**的限定形式——不像 `fell`、`hits` 那样开放无边,也不会
-/// 出现在名词短语里。`may`(月份、人名)与 `can`(容器)除外,它们兼作名词(#193)
-const MODALS: &[&str] = &["could", "should", "might", "must", "shall"];
-
-/// 结构信号:不靠词表也看得出的「这是一句话」(#193)。
-///
-/// 动词表挡不住新语料——英语有上千个限定形式,`could`、`fell`、`hits` 都不在那 19 个
-/// 词里,而且第一个例句恰好 12 个词,卡在上限上。结构信号迁移得动:
-/// 1. **句号结尾**:末词是小写词并以句号收尾(`… as a whole.`)。专名缩写 `Inc.` /
-/// `Co.` 是大写开头,不误伤。
-/// 2. **情态动词**:封闭类,见 [`MODALS`]。
-///
-/// 这两条**当场拒绝**——它们和词数上限一样是结构判据,不是又一份词表。
-/// `words` 是小写化、去标点的词,`raw_last` 是保留原样的末词
-fn reads_like_a_sentence(words: &[String], raw_last: Option<&str>) -> bool {
- if words.len() > 2 && words.iter().any(|w| MODALS.contains(&w.as_str())) {
- return true;
- }
- if let Some(stem) = raw_last.and_then(|w| w.strip_suffix('.')) {
- if stem.chars().count() >= 3 && stem.chars().all(|c| c.is_lowercase()) {
- return true;
- }
- }
- false
-}
-
-/// 弱信号:像从句,但不敢当场拒绝(#193)。
-///
-/// 守卫的样本全部来自一份语料,换一份就漏——换规则之前先要一份**跨语料的标注集**。
-/// 命中只记 `clause_suspect`(例句进 `extraction_drops`),实体照常落库;攒够两份语料的
-/// 样本再决定哪条升成硬规则。返回的是信号名,作为记录的 detail
-/// 槽位片段核对的结论(#582)
-#[derive(Debug, PartialEq)]
-enum SpanVerdict {
- /// 没给片段,或片段就是所绑的那个名字(同名、同词干、名字的一部分、名字后面接着
- /// 大写的续词——"Anthropic PBC")
- Ok,
- /// 片段不在引文里:模型没照抄。当没给处理,记一笔
- NotInQuote,
- /// 片段点的是另一个声明过的实体:改绑到它
- Rebind(String),
- /// 片段是围着某个名字的短语,那个名字在里面只是修饰语(后面还有词、或带所有格):
- /// 描述,不是实体
- Described(String),
- /// 片段是所绑名字前面带了别的词:头衔("entrepreneur Tasha McCauley")还是另一件
- /// 东西("companies using OpenAI"),机器分不开。绑定照旧,只记
- Prefixed(String),
- /// 片段里没有任何声明过的名字:模型消解了指代("him"、"the company")。绑定照旧,只记
- Coreference(String),
- /// 片段抄的是事实另一侧的名字(宾语片段写成了主语):抄错了位置。绑定照旧,只记
- Misplaced(String),
-}
-
-/// 模型报给 `bound` 的别名,是不是已经声明成了另一个实体的名字(0041)。
-///
-/// 结构判据,不认词:同一个名字不会同时是两样东西的名字。模型把「海探1项目」列成
-/// 一个机构,又把它报成探测器的别名——两个答案打架,别名那个不要
-fn name_claimed_elsewhere(name: &str, bound: Uuid, declared: &HashMap<String, Uuid>) -> bool {
- let key = utopia_store::resolution::normalize_name(name).to_lowercase();
- declared.get(&key).is_some_and(|id| *id != bound)
-}
-
/// 片段在不在引文里:大小写、空白都不论
pub(crate) fn span_in_quote(span: &str, quote: &str) -> bool {
let norm = |s: &str| {
@@ -203,291 +110,6 @@ pub(crate) fn span_in_quote(span: &str, quote: &str) -> bool {
!s.is_empty() && q.contains(&s)
}
-/// 一个词:去掉两头标点和所有格后的小写形态,连同原样(看大小写用)
-#[derive(Debug, Clone)]
-struct Word<'a> {
- clean: String,
- raw: &'a str,
-}
-
-/// 片段拆成词:去两头标点、去所有格("OpenAI's" → openai)、去开头的冠词
-fn span_words(s: &str) -> Vec<Word<'_>> {
- let mut words: Vec<Word<'_>> = s
- .split_whitespace()
- .filter_map(|raw| {
- let w = raw.trim_matches(|c: char| !c.is_alphanumeric());
- let w = w
- .strip_suffix("'s")
- .or_else(|| w.strip_suffix("\u{2019}s"))
- .unwrap_or(w);
- (!w.is_empty()).then(|| Word {
- clean: w.to_lowercase(),
- raw,
- })
- })
- .collect();
- while words
- .first()
- .is_some_and(|w| matches!(w.clean.as_str(), "the" | "a" | "an"))
- {
- words.remove(0);
- }
- words
-}
-
-/// 一个词是不是专名的样子:首字符大写或数字("PBC"、"LLC"、"LX"、"Global")
-fn looks_proper(raw: &str) -> bool {
- raw.chars()
- .find(|c| c.is_alphanumeric())
- .is_some_and(|c| c.is_uppercase() || c.is_numeric())
-}
-
-/// 词序列 needle 在 hay 里连续出现的位置
-fn find_words(hay: &[Word<'_>], needle: &[Word<'_>]) -> Option<usize> {
- if needle.is_empty() || needle.len() > hay.len() {
- return None;
- }
- (0..=hay.len() - needle.len()).find(|&i| {
- needle
- .iter()
- .zip(&hay[i..])
- .all(|(n, h)| n.clean == h.clean)
- })
-}
-
-/// 片段说的是不是这个名字:同名、同词干(Acme / Acme Corp.)、泛用后缀互推——
-/// 与消解召回同一套判据(`recall_keys`);或者片段的词全在名字里("Altman" 之于
-/// "Sam Altman","Anthropic" 之于 "Anthropic, PBC")
-fn slot_matches(span: &str, name: &str) -> bool {
- let keys = |s: &str| {
- let clean = span_words(s)
- .into_iter()
- .map(|w| w.clean)
- .collect::<Vec<_>>()
- .join(" ");
- utopia_store::resolution::recall_keys(&utopia_store::resolution::normalize_name(&clean))
- };
- let (a, b) = (keys(span), keys(name));
- if !a.is_empty() && a.iter().any(|k| b.contains(k)) {
- return true;
- }
- let (s, n) = (span_words(span), span_words(name));
- !s.is_empty() && s.iter().all(|w| n.iter().any(|x| x.clean == w.clean))
-}
-
-/// **模型抄,机器判**(#582)。
-///
-/// #559、#578、#581 是同一件事的三张脸:模型把事实挂到了错的参与者身上——
-/// "former OpenAI personnel" 写成 OpenAI,"lawsuit against OpenAI" 造成节点。给每一种
-/// 形状写一条规则、配一张词表,量出来规则的服从率一半上下,词表只认见过的形状。
-/// 这里换一个问法:不问模型「这算不算实体」,让它把引文里点名每一侧的那几个字
-/// 抄出来(`subject_span` / `object_span`)。抄是模型稳定会做的事;判断交给机器:
-///
-/// 1. 片段是所绑的名字(同名、同词干、名字的一部分)→ 放行;
-/// 2. 片段是另一个声明过的实体的名字(或那名字的一部分)→ 改绑;
-/// 3. 片段里含所绑的名字、名字**后面**还有词("former OpenAI personnel"、"The Verge
-/// reporter")或名字带所有格("Anthropic's safeguards")→ 名字在里面只是修饰语,
-/// 这是描述。后面的词全是专名的样子("Anthropic PBC"、"OpenAI Global LLC")不算;
-/// 4. 名字只在片段**末尾**、前面带了词 → 头衔还是另一件东西分不开,绑定照旧、只记;
-/// 5. 片段里是别的声明过的名字带着修饰 → 描述;一个名字都没有 → 指代,绑定照旧、只记。
-///
-/// 按语法位置来的补充:名字后面紧跟逗号是同位语("Helen Toner, strategy director
-/// for …"),还是它;片段以这条事实**另一侧**的名字结尾(宾语片段抄成了主语),是抄错
-/// 位置,只记。
-///
-/// 同一套判据还用在模型**写的名字**和它 **ref 指的实体**之间(`written_verdict`):写
-/// "OpenAI employees" 却 ref 到 OpenAI,是它自己的两个答案打架,v5 那轮的最后一条假边
-/// ("Eleven employees left OpenAI … to establish Anthropic" → Anthropic founded_by OpenAI)
-/// 就是这么来的——片段 "eleven employees" 里没有名字,只看片段拦不住。
-///
-/// 只有 3 和 5 前半改动事实(主语丢、宾语落字面值);其余都是信号,让每一层的比率
-/// 按库、按模型读得出来
-fn verify_span(
- span: Option<&str>,
- name: &str,
- other: &str,
- quote: &str,
- declared: &HashMap<String, Uuid>,
-) -> SpanVerdict {
- let Some(span) = span.map(str::trim).filter(|s| !s.is_empty()) else {
- return SpanVerdict::Ok;
- };
- if !span_in_quote(span, quote) {
- return SpanVerdict::NotInQuote;
- }
- if slot_matches(span, name) {
- return SpanVerdict::Ok;
- }
- let s = span_words(span);
- let n = span_words(name);
- if s.is_empty() {
- return SpanVerdict::Coreference(span.to_string());
- }
-
- // 2. 另一个声明过的实体:精确/词干命中优先,其次名字包住片段的(取最短的那个)
- let mut others: Vec<&String> = declared.keys().filter(|k| k.as_str() != name).collect();
- others.sort();
- if let Some(k) = others.iter().find(|k| {
- let keys = |x: &str| {
- let clean = span_words(x)
- .into_iter()
- .map(|w| w.clean)
- .collect::<Vec<_>>()
- .join(" ");
- utopia_store::resolution::recall_keys(&utopia_store::resolution::normalize_name(&clean))
- };
- let (a, b) = (keys(span), keys(k));
- a.iter().any(|x| b.contains(x))
- }) {
- return SpanVerdict::Rebind((*k).clone());
- }
- // 单个词包在别的名字里不算改绑:"company" 是指代,句首的 "Stockholders" 大写也
- // 不是专名的证据(召回测量台第二轮:主语从 NVIDIA 改绑到了「年度股东大会」)。
- // 单个词只认精确/词干命中(上面 `slot_matches` 那一关)
- let names_something = s.len() >= 2;
- let mut supersets: Vec<(&String, usize)> = others
- .iter()
- .filter_map(|k| {
- let kw = span_words(k);
- (names_something && s.iter().all(|w| kw.iter().any(|x| x.clean == w.clean)))
- .then_some((*k, kw.len()))
- })
- .collect();
- supersets.sort_by_key(|(_, len)| *len);
- if let Some((k, _)) = supersets.first() {
- return SpanVerdict::Rebind((*k).clone());
- }
- // 所绑的名字在片段里的位置;名字后面紧跟逗号是同位语("TBPN, a media company in
- // California"),还是它——先于下面所有判断
- let found = find_words(&s, &n);
- if let Some(i) = found {
- let end = i + n.len();
- if end < s.len() && s[end - 1].raw.trim_end().ends_with(',') {
- return SpanVerdict::Ok;
- }
- }
-
- // 片段以事实另一侧的名字结尾:抄错了位置(宾语片段写成了主语),不是绑错了实体。
- // 「片段以别的声明名结尾就改绑到它」试过两轮(v4/v5):介词的补语、名单的最后一项、
- // 并列的后一个("Swisher and … Alex Heath")都会被当成头,错的多过对的,不做
- let ends_with = |k: &str| {
- let kw = span_words(k);
- !kw.is_empty() && kw.len() < s.len() && find_words(&s[s.len() - kw.len()..], &kw) == Some(0)
- };
- if !other.is_empty() && !slot_matches(name, other) && ends_with(other) {
- return SpanVerdict::Misplaced(span.to_string());
- }
-
- // 3 / 4. 所绑的名字在片段里
- if let Some(i) = found {
- let end = i + n.len();
- let last = s[end - 1].raw;
- let possessive = last.ends_with("'s") || last.ends_with("\u{2019}s");
- let after = &s[end..];
- // 名字后面接着 and / & 再接专名,是并列("SB Energy and SoftBank"):名字是名单里
- // 的一项,不是修饰语。连词跟冠词一样是语法词,不是词表
- let after = match after.first() {
- Some(w) if matches!(w.clean.as_str(), "and" | "&") => &after[1..],
- _ => after,
- };
- if possessive || after.iter().any(|w| !looks_proper(w.raw)) {
- return SpanVerdict::Described(span.to_string());
- }
- if after.is_empty() && i > 0 {
- return SpanVerdict::Prefixed(span.to_string());
- }
- return SpanVerdict::Ok;
- }
-
- // 5. 别的名字带着修饰,或一个名字都没有
- if others.iter().any(|k| {
- let kw = span_words(k);
- find_words(&s, &kw).is_some()
- }) {
- return SpanVerdict::Described(span.to_string());
- }
- SpanVerdict::Coreference(span.to_string())
-}
-
-fn clause_suspect(name: &str) -> Option<&'static str> {
- let words: Vec<String> = name
- .split_whitespace()
- .map(|w| {
- w.trim_matches(|c: char| !c.is_alphanumeric())
- .to_lowercase()
- })
- .collect();
- // 限定词起头的长串:「The removal … industry as a whole」;机构名也会长,但
- // 通常不以 the 起头(US District Court …),起头的(The New York Times)不长
- if words.len() >= 8 && matches!(words[0].as_str(), "a" | "an" | "the") {
- return Some("determiner_opens_a_long_string");
- }
- // 句中的关系词 / 从属连词:「the committee that reviewed …」
- if words.len() >= 5
- && words
- .iter()
- .skip(1)
- .any(|w| matches!(w.as_str(), "that" | "which" | "who" | "because" | "while"))
- {
- return Some("relative_or_subordinate_clause");
- }
- None
-}
-
-fn is_entity_name(name: &str) -> bool {
- let name = name.trim();
- if name.is_empty() || name.chars().count() > 100 {
- return false;
- }
- let raw: Vec<&str> = name.split_whitespace().collect();
- let words: Vec<String> = raw
- .iter()
- .map(|w| {
- w.trim_matches(|c: char| !c.is_alphanumeric())
- .to_lowercase()
- })
- .collect();
- if words.len() > MAX_NAME_WORDS {
- return false;
- }
- // 一个词的名字不可能是从句,别让 "Is" 这种专名被误伤
- if words.len() > 2 && words.iter().any(|w| CLAUSE_MARKERS.contains(&w.as_str())) {
- return false;
- }
- if reads_like_a_sentence(&words, raw.last().copied()) {
- return false;
- }
- // 部分格:`745 of OpenAI's 770 employees` 是一个数量描述,不是一个东西。
- // 它没有限定动词,词数也不多,上面两条都接不住它。
- //
- // 判据收得很窄——**首词是纯数字且第二词是 of**。真实体里以数字开头的
- //(`3M`、`7-Eleven`、`23andMe`)首词不是纯数字;`2023 Nobel Prize` 首词是纯数字,
- // 但第二个词不是 `of`。宽一格就会误伤它们。
- if words.len() >= 3
- && words[1] == "of"
- && !words[0].is_empty()
- && words[0].chars().all(|c| c.is_ascii_digit())
- {
- return false;
- }
- /* **整体就是一个量的,不是一个东西**:`$5 billion`、`52%`、`3.5 million`。
- 上面那条部分格只接住 `745 of …`,接不住这些。
-
- 这道闸装在**这里**才管用。抽取那边也有一道(`looks_literal`),但它前面
- 挂着 `!known_predicate`:谓词一旦是本体里列出来的关系,整段判断直接跳过。
- 实测就是这么漏的——`hasAmount` 来自 FIBO 包、抽取前就在本体里,于是
- `Microsoft hasAmount $1 billion` 里那个数额照样被造成了节点,
- 而同一个库里 `invested`(语料自己长的、当时还未知)走到了那道闸、被拦下。
- `is_entity_name` 不问谓词,主语宾语一视同仁,两条路都过它。
-
- 判据仍旧从严(见 `parse_quantity`):尾巴上有实词就不算,
- `3M`、`7-Eleven`、`23andMe`、`2025 Atlantic hurricane season` 一个都不误伤。 */
- if utopia_extract::parse_quantity(name).is_some() {
- return false;
- }
- true
-}
-
/// 记一条丢弃信号。抽取器有七处 `continue`,每一处都是"事实抽出来了、被挡掉、
/// 什么都不说"。信号写失败绝不能带垮整篇文档的抽取,所以这里吞掉错误。
pub(crate) async fn drop_signal(
@@ -541,30 +163,6 @@ pub(crate) fn incomplete_reason(unextracted: &[(i32, String)], attempted: usize)
))
}
-/// 自动扩本体的唯一入队点。**成功与失败两条路都要走到它。**
-///
-/// 开关在这里重读,而不是沿用调用方手上那份:失败路径压根没加载过 kb,
-/// 而成功路径那份是文档**开抽时**读的——一篇 73 块的文档要跑一个多小时,
-/// 期间有人在设置里关掉了开关,沿用旧值就是拿一小时前的意图办事。
-async fn enqueue_bootstrap(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> {
- if !utopia_store::kbs::get(&state.pool, kb_id)
- .await?
- .auto_extend_ontology
- {
- return Ok(());
- }
- if !utopia_store::documents::extraction_idle(&state.pool, kb_id).await? {
- return Ok(());
- }
- utopia_store::jobs::enqueue(
- &state.pool,
- "bootstrap_ontology",
- serde_json::json!({ "kb_id": kb_id }),
- )
- .await?;
- Ok(())
-}
-
/// `proposer`:这篇文档若是记忆日志,抽出的事实等人点头,据此记下「谁说的」
/// ——人,以及经 MCP 时那个 agent(0014 的令牌)。批量摄入的文档传默认值:
/// 那条路不经待确认队列,这两位都用不上
@@ -585,60 +183,12 @@ pub async fn extract_document(
.await;
if let Ok(doc) = utopia_store::documents::get(&state.pool, document_id).await {
state.emit_document(doc.kb_id, document_id);
- // **失败也要触发自动扩本体。**
- //
- // 入队从前只写在成功路径上,于是这一串会把知识库永久卡住:
- // 前 14 篇成功(每篇都看到还有别的在飞,不触发),第 15 篇重试
- // 耗尽变 failed —— 这时 extraction_idle 恰好为真(failed 不算
- // queued/extracting),可**再没有任何一篇文档会完成来做这次检查**。
- // 结果是提案堆在池子里、本体永远停在种子那几个关系、半张图永远是
- // 兜底谓词,而界面上没有任何东西说这件事发生过。
- //
- // 任务本身幂等且会重查开关与门槛,所以这里多入队一次是安全的。
- let _ = enqueue_bootstrap(state, doc.kb_id).await;
}
Err(e)
}
}
}
-/// mention → 实体 id。同一文档内同名同类型直接复用(单文档语境里罕有同名不同人,
-/// 也把消解调用摊薄到每个名字一次);跨文档歧义由 resolve_mention 的画像比对处理。
-#[allow(clippy::too_many_arguments)]
-async fn resolve(
- pool: &PgPool,
- kb_id: Uuid,
- // None = 模型给的类型不在本体里,或这个库根本还没有类(0009)
- type_id: Option<Uuid>,
- name: &str,
- ctx: Option<&[f32]>,
- // 本次提及所在分块的原文:画像分不开同名候选时的事实旁证(#331)
- text: Option<&str>,
- doc_cache: &mut HashMap<(Option<Uuid>, String), Uuid>,
- needs_adjudication: &mut bool,
-) -> anyhow::Result<Uuid> {
- let key = (
- type_id,
- utopia_store::resolution::normalize_name(name).to_lowercase(),
- );
- if let Some(id) = doc_cache.get(&key) {
- return Ok(*id);
- }
- let id = resolve_uncached(
- pool,
- kb_id,
- type_id,
- name,
- ctx,
- text,
- &[],
- needs_adjudication,
- )
- .await?;
- doc_cache.insert(key, id);
- Ok(id)
-}
-
/// Resolve without the document's name cache. Handles use this path so two identities claimed
/// separately in one response cannot collapse before their fact refs are bound.
#[allow(clippy::too_many_arguments)]
@@ -699,78 +249,6 @@ async fn create_namesake_reviews(
Ok(created)
}
-#[derive(Clone, Copy)]
-struct BoundEntity {
- id: Uuid,
- type_id: Option<Uuid>,
-}
-
-/// 模型写的名字和它 ref 指的实体对不对得上(#582)。没有 ref、或写的就是那个名字时
-/// 无事;否则把写的名字当片段核对,写的名字自己当引文(免掉在不在引文那一关)。
-/// 只有描述和改绑两种结论会用到,其余当无事
-fn written_verdict(
- written: &str,
- bound: &str,
- other: &str,
- referenced: bool,
- declared: &HashMap<String, Uuid>,
-) -> SpanVerdict {
- if !referenced || slot_matches(written, bound) {
- return SpanVerdict::Ok;
- }
- match verify_span(Some(written), bound, other, written, declared) {
- v @ (SpanVerdict::Described(_) | SpanVerdict::Rebind(_)) => v,
- _ => SpanVerdict::Ok,
- }
-}
-
-/// 一侧绑上的名字:有 ref 就是 ref 指的那个实体声明的名字,没有就是模型写的(#582)
-fn bound_name<'a>(
- ref_names: &'a HashMap<String, String>,
- reference: Option<&str>,
- written: &'a str,
-) -> &'a str {
- reference
- .map(str::trim)
- .and_then(|h| ref_names.get(h))
- .map(String::as_str)
- .unwrap_or(written)
-}
-
-fn referenced_entity(
- ref_entities: &HashMap<String, BoundEntity>,
- reference: &str,
-) -> Option<BoundEntity> {
- ref_entities.get(reference.trim()).copied()
-}
-
-#[derive(Debug, PartialEq, Eq)]
-enum NoRefNameBinding {
- Legacy(Uuid),
- AmbiguousHandled,
- Missing,
-}
-
-fn no_ref_name_binding(
- entity_ids: &HashMap<String, Uuid>,
- handled_by_name: &HashMap<String, Vec<Uuid>>,
- name: &str,
-) -> NoRefNameBinding {
- let normalized = utopia_store::resolution::normalize_name(name).to_lowercase();
- if handled_by_name
- .get(&normalized)
- .is_some_and(|ids| ids.len() > 1)
- {
- NoRefNameBinding::AmbiguousHandled
- } else {
- entity_ids
- .get(name)
- .copied()
- .map(NoRefNameBinding::Legacy)
- .unwrap_or(NoRefNameBinding::Missing)
- }
-}
-
#[allow(clippy::too_many_arguments)]
pub(crate) async fn resolve_handle(
pool: &PgPool,
@@ -836,2614 +314,37 @@ pub(crate) async fn resolve_handle(
.or_default()
.push(id);
let document_claims = handled_by_name.entry(normalized).or_default();
- if !document_claims.contains(&id) {
- document_claims.push(id);
- }
- Ok(id)
-}
-
-#[allow(clippy::too_many_arguments)]
-async fn resolve_bare(
- pool: &PgPool,
- kb_id: Uuid,
- type_id: Option<Uuid>,
- name: &str,
- ctx: Option<&[f32]>,
- text: Option<&str>,
- doc_cache: &mut HashMap<(Option<Uuid>, String), Uuid>,
- handled_by_name: &HashMap<String, Vec<Uuid>>,
- ambiguous_bare_cache: &mut HashMap<String, Uuid>,
- needs_adjudication: &mut bool,
- human_reviews_found: &mut bool,
-) -> anyhow::Result<Uuid> {
- let normalized = utopia_store::resolution::normalize_name(name).to_lowercase();
- let claims = handled_by_name
- .get(&normalized)
- .filter(|ids| ids.len() > 1)
- .cloned();
- let Some(claims) = claims else {
- return resolve(
- pool,
- kb_id,
- type_id,
- name,
- ctx,
- text,
- doc_cache,
- needs_adjudication,
- )
- .await;
- };
- if let Some(id) = ambiguous_bare_cache.get(&normalized) {
- return Ok(*id);
- }
-
- // The text supplies no evidence for choosing among the handled namesakes. Preserve the
- // fact on one document-scoped provisional entity and expose every possible identity link
- // to a person. Subsequent bare mentions in this document reuse this provisional entity.
- let id = resolve_uncached(
- pool,
- kb_id,
- type_id,
- name,
- ctx,
- text,
- &claims,
- needs_adjudication,
- )
- .await?;
- if create_namesake_reviews(pool, kb_id, id, &claims).await? {
- *human_reviews_found = true;
- }
- ambiguous_bare_cache.insert(normalized, id);
- Ok(id)
-}
-
-async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow::Result<()> {
- let doc = utopia_store::documents::get(&state.pool, document_id).await?;
- // 排队之后被删了(#268):墓碑不抽——抽出来的事实会活在一个已删除的出处上
- if doc.deleted_at.is_some() {
- tracing::info!(document = %document_id, "skipping a deleted document");
- return Ok(());
- }
- // **本体向量门控(#526)——把等待从 worker 里搬回队列。**
- //
- // 在这之前 `extraction::run` 直接调 `ontology_index::refresh` 等到补齐才动。
- // 那次等待有三个坏处:它把 worker 槽占住,让同一批次的其余文档和别的库的
- // 任务全卡在锁上;它让文档在这段时间里挂着 `extracting`,用户看见 32 篇
- // 全在「抽取中」却没一个事实落库;它用一份可能没补齐的索引作依据,抽出来
- // 的图是基于半个本体写的,再也不会被重抽。
- //
- // 换成队列内门控:本体超出提示词预算且需要嵌入时,先把 `embed_ontology`
- // 排上、把这次抽取挂回 `queued` 等 30s,让 worker 槽立刻空出来——同一批
- // 其余文档和其它库的任务都能继续认领。下次轮到这个文档时本体可能已就绪,
- // 也可能还没,那就再等一轮。**不是同一个文档在等,是同一个抽取器在等**,
- // 而等候归队列管,attempts 不烧。
- //
- // 没配嵌入模型的库不等,照旧送完整本体——那种部署本来就没有检索。
- // 等也有期限(`jobs::DEFER_WINDOW_SECS`),补齐任务一直失败时这篇按失败处理。
- let kb = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;
- if !kb.open_extraction && ontology_index::gate_required(state, doc.kb_id).await? {
- // gate_required 已经把 `embed_ontology` 入队过了(如果应该入队的话)。
- // 这里只挂等待时长,不重复 enqueue。attempts 会被 mark_failed 退回去——
- // 同一个等待条件两次排队不应该消耗两次预算。
- let err = anyhow::Error::msg("waiting for ontology index to be embedded")
- .context(utopia_core::Deferred::new(Duration::from_secs(30)));
- return Err(err);
- }
- let settings = utopia_store::settings::get(&state.pool, kb.workspace_id)
- .await?
- .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?;
- let client = llm_util::chat_client(&settings)
- .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?;
-
- // 所有权凭证:重抽会自增 epoch,本任务据此察觉自己已被接管(见分块循环)
- let my_epoch = utopia_store::documents::extract_epoch(&state.pool, document_id).await?;
- utopia_store::documents::set_graph_status(&state.pool, document_id, "extracting").await?;
- state.emit_document(doc.kb_id, document_id);
- // **开放图谱**(0044 第 1 刀,#729):开关开着的库只写文档自己的话,本体不进提示词。
- // 记忆日志也走这条路,只是它的陈述先进待确认表等人点头(0015),点头时才落成开放陈述
- if kb.open_extraction {
- let await_nod = utopia_store::memory::is_memory_document(&state.pool, document_id).await?;
- return crate::extraction_open::run_open(
- state, &doc, &kb, &settings, &client, my_epoch, proposer, await_nod,
- )
- .await;
- }
- let etypes = utopia_store::graph::entity_types(&state.pool, doc.kb_id).await?;
- // 这一轮落过的事实(新建或重复观察):结尾对它们跑一遍签名检查
- let mut touched_facts: Vec<Uuid> = Vec::new();
- let mut rtypes = utopia_store::graph::relation_types(&state.pool, doc.kb_id).await?;
- // 名字属性不进给模型的清单(0041):名字走回复里的 `names`,服务端核对它在原文里
- rtypes.retain(|r| !utopia_store::names::is_name_attribute(r));
- // 关系与属性分道:属性走字面值通道,不进关系清单。
- //
- // **本体里没有对应关系时就没有谓词**(见 `facts.predicate_id`)。原词落进
- // fact_evidence.proposed_predicate,显示时由 fact_surface_predicate() 取回。
- //
- // 从前这里是一个叫 related_to 的兜底关系,且刻意不列给模型——它摆进提示词就成了
- // 逃生舱,模型读到说不清的关系时不去写原文说法,直接挑这个万能选项。
- // 现在它连行都没有了,逃生舱和"记得别列它"这两件事一起消失。
- //
- // **这两件事必须一起做,缺一件比都不做更糟。** 只删库里的行不够,还要删
- // `DEFAULT_RELATION_TYPES` 里的种子——而这里的排除过滤已经跟着删了。于是
- // `ensure_default_ontology` 七分钟后把行种回来,`related_to` 第一次被**列进
- // 提示词给模型看**。0001 量过:359 次使用里 321 次是模型从清单上挑的。
- // 谁要往种子表里加回一个兜底关系,先看 0010——不过那张表现在已经没有了
- // (`#128`),连播种函数一起退场。
- let type_key_by_id: HashMap<Uuid, &str> =
- etypes.iter().map(|t| (t.id, t.key.as_str())).collect();
- let attr_meta: HashMap<&str, &utopia_core::models::RelationType> = rtypes
- .iter()
- .filter(|r| r.kind == "attribute")
- .map(|r| (r.key.as_str(), r))
- .collect();
- /* **边上的属性**(0037):一条关系声明过的属性定义,按关系 id 取。
- 属性定义就是 kind='attribute' 的行,datatype / unit / 换算全复用;
- 写入时按这里的 key 对模型给的 qualifiers,对不上的进丢弃表让人看见 */
- let rtype_by_id: HashMap<Uuid, &utopia_core::models::RelationType> =
- rtypes.iter().map(|r| (r.id, r)).collect();
- let attr_by_key: HashMap<String, &utopia_core::models::RelationType> = rtypes
- .iter()
- .filter(|r| r.kind == "attribute")
- .map(|r| (r.key.to_lowercase(), r))
- .collect();
- let mut qualifier_defs: HashMap<Uuid, Vec<&utopia_core::models::RelationType>> = rtypes
- .iter()
- .filter(|r| r.kind != "attribute" && !r.qualifiers.is_empty())
- .map(|r| {
- let defs = r
- .qualifiers
- .iter()
- .filter_map(|q| rtype_by_id.get(q).copied())
- .filter(|q| q.kind == "attribute")
- .collect();
- (r.id, defs)
- })
- .collect();
- let type_ids: HashMap<&str, Uuid> = etypes.iter().map(|t| (t.key.as_str(), t.id)).collect();
- let rel_ids: HashMap<&str, Uuid> = rtypes.iter().map(|r| (r.key.as_str(), r.id)).collect();
- // 模型说出的谓词往本体已有关系上落:写法、时态、被动都对齐(见 predicate_match)。
- // 没有它的时候,`produces` 明明在词表里,模型写 `produced_by` 就被降级扔了
- let pred_index = PredicateIndex::build(&rtypes);
- // 「本体认不认识这个说法」——字面值那一档与关系那一档必须用同一个判据。
- // 分开写的话,模糊匹配得上的谓词会先被字面值那一档当成属性分流走,
- // 同一个词在两条路上得到相反的回答
- let known_predicate = |p: &str| rel_ids.contains_key(p) || pred_index.lookup(p).is_some();
- // 时态对账只对带唯一性约束的状态关系生效(本体元数据):(functional, inverse_functional, temporal)
- let rel_meta: HashMap<Uuid, (bool, bool, String)> = rtypes
- .iter()
- .map(|r| {
- (
- r.id,
- (r.functional, r.inverse_functional, r.temporal.clone()),
- )
- })
- .collect();
- let type_parents: HashMap<Uuid, &[Uuid]> = etypes
- .iter()
- .map(|t| (t.id, t.parents.as_slice()))
- .collect();
-
- // **本体全铺还是按分块检索。**
- //
- // 全铺是今天的行为,小本体下它对且便宜:40 个类约 2k 字符,检索反而是多余的
- // 往返。大本体下它是灾难——schema.org 实测每个分块 108k tokens,而同一份语料
- // 只给种子类时抽到 25 个实体、给全量时只剩 18 个。**多给的那 959 个类
- // 吃掉了 7 个实体。**
- //
- // 所以按预算切换:装得下就全铺,装不下就每块检索。判据量的是**实际要排的
- // 那段字**(build_lists 自己数),不是另写一个估算公式——公式会跟排版分叉。
- let full = build_lists(&etypes, &rtypes, None, None);
- let budget = utopia_store::access::ontology_prompt_budget(&state.pool).await?;
- let retrieve_per_chunk = full.chars() > budget;
- if retrieve_per_chunk {
- tracing::info!(
- %document_id, chars = full.chars(), budget,
- classes = etypes.len(),
- "本体超出提示词预算,改为按分块检索候选"
- );
- // 走到这里说明本体超出预算且按预算逻辑需要按块检索——但本任务的等待
- // 早就在 `gate_required` 里完成(要么已经嵌好,要么通过 `Deferred` 挂回
- // 队列)。剩下的就是按块检索本身,不再有「顺便 refresh」这一步:
- // 那一步是在 worker 里等 PER_KB 锁,正是 #526 想消除的副作用。
- //
- // 留一个注释方便日后回看:若 budget 在 `gate_required` 与此处之间被改小,
- // 本文档会按全量本体抽——比 #526 之前的「等几分钟」更接近「对的那一边」。
- }
- // 内置类恒在:检索漏掉的分块仍要有地方落脚,否则模型无类可选
- let seed_classes: HashSet<Uuid> = etypes.iter().filter(|t| t.builtin).map(|t| t.id).collect();
-
- // 属性 domain 允许子类:主语类型沿 parent 链上溯命中 domain 即可
- // 沿 subClassOf 上溯。**广度优先 + 访问集**,不是单链循环:
- // 一个类可以有多个父(FOAF 的 Person 同时是 Agent 与 SpatialThing),
- // 而菱形继承会从两条路到达同一个祖先,没有访问集就会重复展开。
- //
- // 深度上限换成了访问集:写入侧 set_parents 已经查环,这里再靠"最多走十层"
- // 兜底既挡不住宽的图,也会悄悄放过深的层级。
- let type_matches_domain = |ty: Uuid, domain: Uuid| -> bool {
- let mut seen: HashSet<Uuid> = HashSet::new();
- let mut queue = vec![ty];
- while let Some(cur) = queue.pop() {
- if cur == domain {
- return true;
- }
- if !seen.insert(cur) {
- continue;
- }
- if let Some(ps) = type_parents.get(&cur) {
- queue.extend(ps.iter().copied());
- }
- }
- false
- };
- // 本轮要从头讲一遍这篇文档的故事,旧信号先清掉(重抽自动作数)
- let _ = utopia_store::extraction_drops::clear_for_document(&state.pool, document_id).await;
-
- // **记忆抽出的事实先等人点头**(0015)。一句 remember 一次一句、人就在对话里,
- // 确认成本最低的时刻就是说完那句话的时候;而批量摄入一次上万条,逐条确认
- // 不可能,那条路仍旧乐观写入 + 事后审阅。判据只有一个:这篇是不是记忆日志。
- // 实体照常消解并创建——`pending_facts.subject_id` 是外键,这是 0018 定下的取舍
- let await_nod = utopia_store::memory::is_memory_document(&state.pool, document_id).await?;
- let mut pending_count = 0usize;
-
- let doc_time = doc.doc_time.map(|t| t.format("%Y-%m-%d").to_string());
- let chunks = utopia_store::documents::chunks_for_extraction(&state.pool, document_id).await?;
- // 文件开头:序号最小的现存分块(不是还没抽的第一块)。备忘文件一个片段一块,
- // 前一段不是后一段的开头,不附
- let opening_chunk = if await_nod {
- None
- } else {
- utopia_store::documents::opening_chunk(&state.pool, document_id).await?
- };
-
- let mut doc_cache: HashMap<(Option<Uuid>, String), Uuid> = HashMap::new();
- // Identities introduced through handles, grouped only for detecting document-local
- // namesake ambiguity. A later bare mention gets its own provisional entity instead of
- // guessing among this group.
- let mut handled_by_name: HashMap<String, Vec<Uuid>> = HashMap::new();
- let mut ambiguous_bare_cache: HashMap<String, Uuid> = HashMap::new();
- let mut touched_names: HashSet<String> = HashSet::new();
- // 本文档已经认下的实体,按首次出现排序,送进后续分块的提示词。
- //
- // **按 entity_id 去重,不按名字**:第 3 块写"上海研究院"若消解到了第 1 块的
- // "星云科技上海研究院",那它不该以第二个名字进清单——清单里每个实体只有
- // 一个展示形态,就是这篇文档第一次用的那个。中文里全称先出现,所以这也是较全的那个。
- let mut doc_entities: Vec<(Uuid, String, String)> = Vec::new();
- // 整块没抽成的:(seq, 原因)。收尾时据此拒绝把这篇文档标成 done
- let mut unextracted: Vec<(i32, String)> = Vec::new();
- let mut needs_adjudication = false;
- let mut human_reviews_found = false;
- let mut conflicts_found = false;
- let mut fact_count = 0usize;
- // 不设分块上限:静默截断等于丢知识,长文档的成本由部署者自己权衡
- // (成本优化走 prompt 前缀缓存与更新时 chunk 级跳过,而非丢数据)
- for chunk in chunks.iter() {
- // 被接管则安静退场:不写 failed、不碰状态,舞台留给新任务。
- // 检查放在调用 LLM 之前——取消粒度即一个分块,不必等整篇跑完
- if utopia_store::documents::extract_epoch(&state.pool, document_id).await? != my_epoch {
- tracing::info!(%document_id, "抽取任务已被新一轮接管,退出");
- return Ok(());
- }
- let ctx: Option<&[f32]> = chunk.embedding.as_ref().map(|v| v.as_slice());
- // 本体装得下就用全量那份;装不下就拿**这一块自己的向量**检索候选。
- // 向量是现成的——实体消解本来就在用它(上面那个 ctx),检索一次
- // 嵌入都不用加。检索不出来(没配嵌入模型、或这块没向量)就退回全量:
- // 提示词大是慢,没有类可选是抽不出东西
- let lists = if retrieve_per_chunk {
- match ctx {
- Some(v) => {
- chunk_lists(state, doc.kb_id, v, &etypes, &rtypes, &seed_classes, budget)
- .await
- .unwrap_or(None)
- }
- None => None,
- }
- } else {
- None
- };
- let lists = lists.as_ref().unwrap_or(&full);
- let known: Vec<utopia_extract::KnownEntity> = doc_entities
- .iter()
- .enumerate()
- .map(|(index, (_, type_key, name))| utopia_extract::KnownEntity {
- handle: format!("k{}", index + 1),
- type_key: type_key.clone(),
- name: name.clone(),
- })
- .collect();
- // 这一块就是开头本身时不再重复一遍
- let opening = opening_chunk
- .as_ref()
- .filter(|(id, _)| *id != chunk.id)
- .map(|(_, text)| text.as_str());
- let messages = utopia_extract::build_messages_with_opening(
- &lists.types,
- &lists.relations,
- &lists.attributes,
- doc_time.as_deref(),
- &doc.filename,
- &known,
- opening,
- &chunk.text,
- );
- // 这两处 continue 跳过的是**整个分块**——它一条事实都没产出。
- // 记下来,收尾时据此决定这篇文档算不算抽完(见循环之后)
- let reply = match chat_retrying_rate_limits(state, &settings, &client, &messages).await {
- Ok(r) => r,
- Err(e) => {
- tracing::warn!(%document_id, seq = chunk.seq, error = %e, "抽取调用失败,跳过该分块");
- unextracted.push((chunk.seq, format!("调用失败:{e}")));
- continue;
- }
- };
- // 模型的原话只在 debug 级别看得到:查它对哪几个字段怎么填(#582 的片段)时开
- tracing::debug!(%document_id, seq = chunk.seq, reply = %reply, "抽取原始回复");
- let mut extraction = match utopia_extract::parse_response(&reply) {
- Ok(x) => x,
- Err(e) => {
- tracing::warn!(%document_id, seq = chunk.seq, error = %e, "抽取结果解析失败,跳过该分块");
- unextracted.push((chunk.seq, format!("结果解析失败:{e}")));
- continue;
- }
- };
- // **跳过了什么必须说出来。** 逐项解析救回了整块,但被跳过的那几条
- // 如果不落信号,就成了另一种「部分抽取报告成完成」(#108 修过一次)
- if extraction.truncated {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::TRUNCATED_REPLY,
- &format!("分块 #{} 的输出被截断", chunk.seq),
- None,
- )
- .await;
- }
- let skipped = extraction.skipped_entities + extraction.skipped_facts;
- if skipped > 0 {
- tracing::warn!(
- %document_id,
- seq = chunk.seq,
- entities = extraction.skipped_entities,
- facts = extraction.skipped_facts,
- "跳过了结构不合的条目"
- );
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::MALFORMED_ITEM,
- &format!(
- "分块 #{} 跳过 {} 个实体 / {} 条事实",
- chunk.seq, extraction.skipped_entities, extraction.skipped_facts
- ),
- None,
- )
- .await;
- }
-
- // 实体消解:名称 → 实体 id(本分块的事实按原文名字连线)
- // **落库前先查形状**(utopia_extract::normalize):只看结构、不看词——引文里有没有
- // 这段字、值是不是只有标点、一侧是不是契约的日期、同句有没有另一条边。读懂时间
- // 归模型(提示词 3c),这里只核对它照没照契约写,做了什么都记进丢弃表
- let from_opening = match opening {
- Some(text) => {
- utopia_extract::drop_quotes_from_opening(&mut extraction, &chunk.text, text)
- }
- None => Vec::new(),
- };
- for n in from_opening
- .into_iter()
- .chain(utopia_extract::normalize_facts(&mut extraction))
- {
- use utopia_extract::Normalization as N;
- use utopia_store::extraction_drops::reason;
- let (r, detail, example) = match n {
- N::NoValue { predicate, written } => (reason::NO_VALUE, predicate, written),
- N::ValueTrimmed {
- predicate,
- kept,
- dropped,
- } => (
- reason::VALUE_TRIMMED,
- predicate,
- format!("{kept} ✂ {dropped}"),
- ),
- N::QualifiersWithoutObject { predicate, values } => (
- reason::QUALIFIERS_WITHOUT_OBJECT,
- predicate,
- format!("{values} value(s) moved onto the subject"),
- ),
- N::TimeAsObject {
- predicate,
- written,
- values,
- } => (
- reason::TIME_AS_OBJECT,
- predicate,
- if values == 0 {
- format!("{written} kept as a value")
- } else {
- format!("{written} → {values} value(s)")
- },
- ),
- N::TimeAsSubject { predicate, written } => {
- (reason::TIME_AS_SUBJECT, predicate, written)
- }
- N::ObjectDescribesDeclared {
- predicate,
- name,
- head,
- } => (
- reason::OBJECT_DESCRIBES_DECLARED,
- predicate,
- format!("{name} ← {head}"),
- ),
- N::OrphanDeclaration { name } => {
- (reason::ORPHAN_DECLARATION, "entity".to_string(), name)
- }
- N::QuoteFromOpening { predicate, quote } => {
- (reason::QUOTE_FROM_OPENING, predicate, quote)
- }
- };
- drop_signal(state, doc.kb_id, document_id, r, &detail, Some(&example)).await;
- }
- let mut entity_ids: HashMap<String, Uuid> = HashMap::new();
- // 名称 → 声明类型(属性 domain 校验用:salary 不能挂在 Organization 上)
- let mut entity_type_of: HashMap<String, Option<Uuid>> = HashMap::new();
- let mut ref_entities: HashMap<String, BoundEntity> = doc_entities
- .iter()
- .enumerate()
- .map(|(index, (id, type_key, _))| {
- (
- format!("k{}", index + 1),
- BoundEntity {
- id: *id,
- type_id: type_ids.get(type_key.as_str()).copied(),
- },
- )
- })
- .collect();
- // 句柄 → 声明的名字:片段核对要对着**绑上的**那个名字看(#582)。模型写
- // "OpenAI personnel" 却 ref 到 e1=OpenAI 时,错在 ref 上,片段对着 "OpenAI"
- // 才看得出来
- let mut ref_names: HashMap<String, String> = doc_entities
- .iter()
- .enumerate()
- .map(|(index, (_, _, name))| (format!("k{}", index + 1), name.clone()))
- .collect();
- let mut response_claims: HashMap<String, Vec<Uuid>> = HashMap::new();
-
- // Resolve handled definitions first. This matters for a mixed response: a later
- // handle-less same-name item must see the ambiguity rather than winning by array order.
- for e in extraction
- .entities
- .iter()
- .filter(|e| e.local_id.is_some())
- .chain(extraction.entities.iter().filter(|e| e.local_id.is_none()))
- {
- let name = e.name.trim();
- if !is_entity_name(name) {
- // 从前这里是静默 `continue`——正是 `drop_signal` 当初为之而建的
- // 那种"抽出来了、被挡掉、什么都不说"
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::NOT_AN_ENTITY_NAME,
- &e.type_key,
- Some(name),
- )
- .await;
- continue;
- }
- // 守卫放行、结构却像从句:只记不挡(#193 先攒标注集)
- if let Some(signal) = clause_suspect(name) {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::CLAUSE_SUSPECT,
- signal,
- Some(name),
- )
- .await;
- }
- if let Some(handle) = e.local_id.as_deref().map(str::trim) {
- if ref_entities.contains_key(handle) {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::MALFORMED_ITEM,
- "local_id collides with a known handle",
- Some(handle),
- )
- .await;
- continue;
- }
- }
- // 降级时记住模型提议的那个词:本体装不下不等于它说错了。
- // 只留计数的话,日后想加 model 类就找不出那 43 个实体——它们混在
- // concept 里面,唯一的出路是整库重抽
- let mut proposed: Option<&str> = None;
- let type_id = match type_ids.get(e.type_key.as_str()) {
- Some(id) => Some(*id),
- None => {
- // 白名单外类型:**留空**,并记入未匹配统计(本体扩展的信号)。
- //
- // 从前这里降级到 concept 那行哨兵。现在「还没判出来」就是
- // `type_id IS NULL`(0009)——实体照常建、事实照常落、证据照常有,
- // 只是暂时没有类型。之后装一个包再跑类型消解,它会被重新分配
- let _ = utopia_store::ontology::record_miss(
- &state.pool,
- doc.kb_id,
- "entity_type",
- &e.type_key,
- Some(name),
- )
- .await;
- proposed = Some(e.type_key.as_str());
- None
- }
- };
- let normalized = utopia_store::resolution::normalize_name(name).to_lowercase();
- touched_names.insert(normalized.clone());
- let id = if let Some(handle) = e.local_id.as_deref() {
- let handle = handle.trim();
- let id = resolve_handle(
- &state.pool,
- doc.kb_id,
- type_id,
- name,
- ctx,
- Some(&chunk.text),
- &mut response_claims,
- &mut handled_by_name,
- &mut ambiguous_bare_cache,
- &mut needs_adjudication,
- &mut human_reviews_found,
- )
- .await?;
- ref_entities.insert(handle.to_string(), BoundEntity { id, type_id });
- ref_names.insert(handle.to_string(), name.to_string());
- id
- } else {
- resolve_bare(
- &state.pool,
- doc.kb_id,
- type_id,
- name,
- ctx,
- Some(&chunk.text),
- &mut doc_cache,
- &handled_by_name,
- &mut ambiguous_bare_cache,
- &mut needs_adjudication,
- &mut human_reviews_found,
- )
- .await?
- };
- if let Some(p) = proposed {
- let _ = utopia_store::resolution::set_proposed_type(&state.pool, id, p).await;
- }
- // 模型写下的这个名字就在这一块原文里时,给这条名字事实补出处(0041)。
- // 给已知句柄时它照提示词写的是清单上的全称,这一块里未必有——那就不补,
- // 这一块用的别的写法走下面的 `names`。
- // 记忆日志里的不补:那一句算不算出处,要等人点头(0018)
- if !await_nod && span_in_quote(name, &chunk.text) {
- let _ = utopia_store::names::record(
- &state.pool,
- doc.kb_id,
- id,
- name,
- Some(utopia_store::names::NameSource {
- chunk_id: chunk.id,
- quote: name,
- }),
- doc.doc_time,
- )
- .await;
- }
- // 模型自己的说法。**跟 proposed_type 分开存**:那一列的含义是
- // "本体里没有",增长回路靠它的稀有性设门槛;这一列每个实体都有。
- // 与粗类同名的不记——那不是更具体的说法,只是把清单抄了一遍
- if let Some(st) = e
- .specific_type
- .as_deref()
- .map(str::trim)
- .filter(|s| !s.is_empty())
- .filter(|s| !s.eq_ignore_ascii_case(&e.type_key))
- {
- let _ = utopia_store::resolution::set_specific_type(&state.pool, id, st).await;
- }
- // 只记模型自己声明过类型的:主宾兜底那条路没有类型可依,
- // 把一个猜出来的类型放进清单等于让后续分块照着猜的抄。
- //
- // 本体装不下那个类时用模型自己的说法(proposed):这份清单是给后文
- // 认人用的,"同一个名字别写成两个实体"才是它的活。从前这里只能写死
- // concept,反倒把几个不同的词抹平成同一个标签
- if !doc_entities.iter().any(|(eid, _, _)| *eid == id) {
- let tk = type_id
- .and_then(|t| type_key_by_id.get(&t).copied())
- .or(proposed)
- .unwrap_or("?");
- doc_entities.push((id, tk.to_string(), name.to_string()));
- }
- // Legacy name maps retain their old last-write-wins behavior. Handled facts bind
- // through ref_entities; inserting handled definitions here only supports mixed
- // model output when the surface name is unambiguous.
- entity_ids.insert(name.to_string(), id);
- entity_type_of.insert(name.to_string(), type_id);
- }
-
- // 实体的别的名字(0041 决定 2)。**名字与引文都要在这一块原文里**:名字是召回的桥,
- // 一座凭空造的桥会把两个不相干的实体接到一起。认不认「简称」「又名」是模型的事,
- // 服务端不认词,只核对它抄的字是不是真在原文里
- // 这次回复声明的名字,加上本文档前面几块认下的:别名撞上它们之一就不收
- let declared_names: HashMap<String, Uuid> = entity_ids
- .iter()
- .map(|(name, id)| (name.as_str(), *id))
- .chain(
- doc_entities
- .iter()
- .map(|(id, _, name)| (name.as_str(), *id)),
- )
- .map(|(name, id)| {
- (
- utopia_store::resolution::normalize_name(name).to_lowercase(),
- id,
- )
- })
- .collect();
- for n in &extraction.names {
- let name = n.name.trim();
- let Some(bound) = ref_entities.get(n.entity_ref.trim()) else {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::MALFORMED_ITEM,
- "name ref is not a declared handle",
- Some(name),
- )
- .await;
- continue;
- };
- let quote = n
- .quote
- .as_deref()
- .map(str::trim)
- .filter(|q| !q.is_empty())
- .unwrap_or(name);
- if name_claimed_elsewhere(name, bound.id, &declared_names) {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::NAME_CLAIMED_BY_ANOTHER,
- &n.entity_ref,
- Some(name),
- )
- .await;
- continue;
- }
- if !is_entity_name(name)
- || !span_in_quote(name, quote)
- || !span_in_quote(quote, &chunk.text)
- {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::NAME_NOT_IN_TEXT,
- &n.entity_ref,
- Some(name),
- )
- .await;
- continue;
- }
- // 记忆日志读到的名字与它的其它事实一样等人点头(0018):名字是召回的桥,
- // 一句没确认过的话不该先把桥搭上。点头之后是一条普通的名字事实;
- // 这一步不配对,同名的配对等下一次在文档里读到它
- if await_nod {
- let known_as = utopia_store::names::ensure_known_as(&state.pool, doc.kb_id).await?;
- let value = serde_json::json!({
- "value": utopia_store::resolution::normalize_name(name)
- });
- if let utopia_store::pending::Outcome::Proposed(_) = utopia_store::pending::propose(
- &state.pool,
- utopia_store::pending::Proposal {
- kb_id: doc.kb_id,
- subject_id: bound.id,
- predicate_id: Some(known_as),
- object_id: None,
- object_value: Some(&value),
- proposed_predicate: Some(utopia_store::names::KNOWN_AS),
- validity: utopia_store::graph::Validity {
- attested_at: doc.doc_time,
- ..Default::default()
- },
- confidence: 1.0,
- chunk_id: chunk.id,
- proposed_by: proposer.user_id,
- proposed_token: proposer.token_id,
- phrase: None,
- qualifiers: None,
- time_words: None,
- quote_span: None,
- },
- )
- .await?
- {
- pending_count += 1;
- }
- continue;
- }
- utopia_store::names::record(
- &state.pool,
- doc.kb_id,
- bound.id,
- name,
- Some(utopia_store::names::NameSource {
- chunk_id: chunk.id,
- quote,
- }),
- doc.doc_time,
- )
- .await?;
- // 别的实体已经叫这个名字:送去裁决,不合并
- if utopia_store::names::pair_shared_name(&state.pool, doc.kb_id, bound.id, name).await?
- > 0
- {
- needs_adjudication = true;
- }
- }
-
- // 改绑的候选:这次回复声明的实体,加上提示词里给过的库内实体(#582)
- let span_declared: HashMap<String, Uuid> = {
- let mut m = entity_ids.clone();
- for (id, _, name) in &doc_entities {
- m.entry(name.clone()).or_insert(*id);
- }
- m
- };
- for f in &extraction.facts {
- let confidence = f.confidence.unwrap_or(0.7).clamp(0.0, 1.0);
- if confidence < MIN_CONFIDENCE {
- // 设计上的阈值,但用户同样无从知道"抽到了,只是不够自信"
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::LOW_CONFIDENCE,
- &f.predicate,
- Some(&format!("{} ({:.0}%)", f.subject, confidence * 100.0)),
- )
- .await;
- continue;
- }
- let confidence = origin_ceiling(&chunk.origin, confidence);
- let validity =
- validity_of(f.valid_from.as_deref(), f.valid_to.as_deref(), doc.doc_time);
-
- // 属性事实:谓词命中属性 → 字面值通道。datatype 校验失败宁缺勿脏;
- // domain 校验(含子类上溯)挡住"把 salary 挂到 Organization"这类张冠李戴。
- // 模型偶尔照抄清单里的 "person.salary" 全限定名——剥掉类前缀再查一次
- let attr_hit = attr_meta.get(f.predicate.as_str()).or_else(|| {
- f.predicate
- .rsplit_once('.')
- .and_then(|(_, k)| attr_meta.get(k))
- });
- if let Some(attr) = attr_hit {
- let subject_name = f.subject.trim();
- // 主语没在 entities 里声明:类型不明,domain 无从校验,属性不落。
- // 关系路径遇到同样的缺失会兜底按 concept 消解——这里学不来,
- // 按 concept 解出来 domain 照样不匹配,只是从这里掉进下面那一档
- let bound = match f.subject_ref.as_deref().map(str::trim) {
- Some(handle) => referenced_entity(&ref_entities, handle),
- None => {
- match no_ref_name_binding(&entity_ids, &handled_by_name, subject_name) {
- NoRefNameBinding::Legacy(id) => entity_type_of
- .get(subject_name)
- .copied()
- .map(|type_id| BoundEntity { id, type_id }),
- NoRefNameBinding::Missing => None,
- NoRefNameBinding::AmbiguousHandled => {
- let type_id = entity_type_of.get(subject_name).copied().flatten();
- touched_names.insert(
- utopia_store::resolution::normalize_name(subject_name)
- .to_lowercase(),
- );
- let id = resolve_bare(
- &state.pool,
- doc.kb_id,
- type_id,
- subject_name,
- ctx,
- Some(&chunk.text),
- &mut doc_cache,
- &handled_by_name,
- &mut ambiguous_bare_cache,
- &mut needs_adjudication,
- &mut human_reviews_found,
- )
- .await?;
- Some(BoundEntity { id, type_id })
- }
- }
- }
- };
- let Some(BoundEntity {
- id: subject_id,
- type_id: subject_type,
- }) = bound
- else {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- if f.subject_ref.is_some() {
- utopia_store::extraction_drops::reason::MALFORMED_ITEM
- } else {
- utopia_store::extraction_drops::reason::SUBJECT_NOT_DECLARED
- },
- &attr.key,
- Some(f.subject_ref.as_deref().unwrap_or(subject_name)),
- )
- .await;
- continue;
- };
- // 不可达:store 层强制 attribute 必有 domain,且 domain 不可改,
- // 无 domain 的属性连提示词都进不去。留着是防御,不需要信号
- if attr.domains.is_empty() {
- continue;
- }
- // **任一 domain 命中即可**:属性挂在多个类下时,主语属于其中之一就算数
- if !attr
- .domains
- .iter()
- .any(|d| subject_type.is_some_and(|t| type_matches_domain(t, *d)))
- {
- let subj_key = subject_type
- .and_then(|t| type_key_by_id.get(&t).copied())
- .unwrap_or("?");
- let dom_key = attr
- .domains
- .iter()
- .filter_map(|d| type_key_by_id.get(d).copied())
- .collect::<Vec<_>>()
- .join("|");
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::ATTR_DOMAIN_MISMATCH,
- &format!("{}@{subj_key} (wants {dom_key})", attr.key),
- Some(subject_name),
- )
- .await;
- continue;
- }
- let raw = match (&f.value, &f.object) {
- (Some(v), _) => v.clone(),
- // 模型偶尔把值放进 object:宽容接住
- (None, Some(o)) if !o.trim().is_empty() => {
- serde_json::Value::String(o.trim().to_string())
- }
- _ => {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::ATTR_NO_VALUE,
- &attr.key,
- Some(subject_name),
- )
- .await;
- continue;
- }
- };
- let datatype = attr.datatype.as_deref().unwrap_or("text");
- // 只相对一件事给出的日期(「触发日后 45 天」,#681 §4)照原文收下、带着标记:
- // 它是新的状态值,时态引擎照常用它接替前一个截止日
- let Some(mut object_value) =
- utopia_extract::attr_object_value(datatype, &raw, f.relative)
- else {
- tracing::debug!(%document_id, attr = attr.key, ?raw, "属性值不合 datatype,跳过");
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::ATTR_DATATYPE,
- &format!("{} ({datatype})", attr.key),
- Some(&format!("{subject_name} → {raw}")),
- )
- .await;
- continue;
- };
- // 单位随事实落笔:类型上的单位以后改了,旧值仍按记录时的单位读。
- // 记哪个单位照 `unit_for`——从前这里无条件盖上声明的单位,实测
- //「提供 500 兆瓦的风电」被模型记成金额,再盖上 ¥ 就成了 500 块钱
- if let Some(u) = unit_for(&raw, datatype, None, attr.unit.as_deref()) {
- object_value["unit"] = serde_json::json!(u);
- }
- if await_nod {
- if let utopia_store::pending::Outcome::Proposed(_) =
- utopia_store::pending::propose(
- &state.pool,
- utopia_store::pending::Proposal {
- kb_id: doc.kb_id,
- subject_id,
- predicate_id: Some(attr.id),
- object_id: None,
- object_value: Some(&object_value),
- proposed_predicate: Some(f.predicate.as_str()),
- validity,
- confidence,
- chunk_id: chunk.id,
- proposed_by: proposer.user_id,
- proposed_token: proposer.token_id,
- phrase: None,
- qualifiers: None,
- time_words: None,
- quote_span: None,
- },
- )
- .await?
- {
- pending_count += 1;
- }
- continue;
- }
- let (fact_id, created) = utopia_store::graph::insert_value_fact(
- &state.pool,
- doc.kb_id,
- subject_id,
- Some(attr.id),
- &object_value,
- validity,
- confidence,
- )
- .await?;
- touched_facts.push(fact_id);
- // 属性谓词也留原词:模型偶尔照抄 "person.salary" 全限定名,
- // 命中的是剥掉前缀后的 key,原样是什么值得留着
- utopia_store::graph::add_evidence(
- &state.pool,
- fact_id,
- chunk.id,
- f.quote.as_deref(),
- Some(f.predicate.as_str()),
- )
- .await?;
- if created {
- fact_count += 1;
- }
- // 单值属性 = functional:新值闭合旧值(属性历史由此而来)。并进已有断言的也对:
- // 这份证据的日期可能更早,时间线的形状跟着变(#679)
- if attr.functional && attr.temporal == "state" {
- let report = utopia_store::temporal::reconcile_new_fact(
- &state.pool,
- doc.kb_id,
- fact_id,
- subject_id,
- attr.id,
- None,
- Some(&object_value),
- utopia_store::temporal::Uniqueness::SubjectSide,
- validity,
- confidence,
- )
- .await?;
- if report.conflicts > 0 {
- conflicts_found = true;
- }
- }
- continue;
- }
-
- // **词表外的字面值:既不丢,也不给它编一个实体。**
- //
- // 走到这里说明谓词既不是已知属性也还没查关系表。它带着字面值时
- // 有两种走法,从前两种都不好:
- // value 有而 object 空 → 掉进下面的"宾语必填",整条静默消失;
- // object 里塞着字面值 → 按 concept 消解,凭空造出一个叫「2015」
- // 的实体,图里多一个假节点,事后再修还得改事实的形状。
- // 现在都存成 object_value 且没有谓词:值在图里、有证据、有时态,
- // 原词进 proposed_predicate,消解那一遍只需换谓词,形状已经是对的。
- //
- // object 里的东西算不算字面值,判据从严:**模型没把它声明成实体**,
- // 且**它整体就是一个量或一个日期**。"杭州"两条都不满足,"2015"、
- // "$5 billion"、"52%" 都满足;"900 million weekly active users"
- // 不满足——尾巴上还有实词,它说的就不再只是那个数了。
- // 文本值的属性(schema.org 里 323 个)在这一档仍会变成实体——
- // 那里没有可靠判据,猜错会吃掉真实体,不猜
- // 第二格是表层谓词:通常就是模型写的谓词;值旁边挂着一个没声明的宾语短语时,
- // 短语并进来(见下面那一档)
- let literal: Option<(serde_json::Value, String)> =
- match (&f.value, f.object.as_deref().map(str::trim)) {
- // **给了值、没给宾语——不管这个谓词本体认不认识。**
- //
- // 从前这里卡着 `!known_predicate`:`job_title` 在 schema.org 里是关系
- //(它的 range 是 `Text|DefinedTerm`,含一个类就走关系通道),于是模型
- // 写 `job_title` + "founder and CEO" 时既进不了属性档、又在关系档因为
- // 缺宾语被丢掉——`object_missing` 实测 69 次,四篇文档里每个人的职务
- // 就是这么没的。谓词认不认识与「这条事实带的是值还是实体」无关:
- // 值在手上就收下,原词进 proposed_predicate,等本体采纳时再换谓词,
- // 形状已经是对的(0010)
- (Some(v), None | Some("")) => Some((v.clone(), f.predicate.clone())),
- /* **宾语整体是一个量:一律当值收下**,不问谓词认不认识、
- 也不问模型有没有把它声明成实体。
-
- 下面那一档卡着 `!known_predicate`,理由是本体说得上话的时候
- 别去二猜模型。可量值这里没有可猜的余地:一个数额不会因为
- 谓词恰好在本体里就变成一个东西。实测漏的正是这一格——
- `hasAmount` 来自 FIBO 包、抽取前就在本体里,
- `Microsoft hasAmount $1 billion` 于是绕过下面那一档,
- 把数额造成了节点;同一个库里 `invested` 当时还未知,
- 走到下面那一档、被拦住了。同一个数额,两种下场。
-
- 收下而不是丢掉:`is_entity_name` 那道闸现在也拦纯量值,
- 不在这里接住的话,这条事实会连同那个数一起进丢弃表。
- 原词照旧进 `proposed_predicate`,采纳时再换谓词 */
- (_, Some(o)) if utopia_extract::parse_quantity(o).is_some() => Some((
- serde_json::Value::String(o.to_string()),
- f.predicate.clone(),
- )),
- /* **给了值、宾语却是一个没声明的短语:值收下,短语并进表层谓词**(#685)。
-
- `build` + 宾语「new energy generation」+ 值「at least 10 GW」:宾语不是
- 回复里声明的实体、没有句柄、也不是本文档认下的名字,于是上面几档都接
- 不住,关系那条路又因为它不是实体把整条丢掉——那个数跟着没了。模型把
- 同一句话写成纯值(「at least 10 GW of new energy generation」)时就落得
- 下,落不落全看它挑了两种同样说得通的写法里的哪一种。
-
- 只看结构:值在、宾语没有句柄、宾语不在已声明的名字里。宾语是个认得的
- 实体时照旧走边。短语不丢,进表层谓词(`build new energy generation`),
- 采纳时再换谓词;原句在证据里 */
- (Some(v), Some(o))
- if undeclared_beside_value(f.object_ref.as_deref(), o, &span_declared) =>
- {
- Some((v.clone(), format!("{} {o}", f.predicate.trim())))
- }
- (_, Some(o))
- if !o.is_empty()
- && !known_predicate(f.predicate.as_str())
- && !entity_ids.contains_key(o)
- && looks_literal(o) =>
- {
- Some((
- serde_json::Value::String(o.to_string()),
- f.predicate.clone(),
- ))
- }
- _ => None,
- };
- if let Some((value, surface)) = literal {
- let subject_name = f.subject.trim();
- // **主语按关系那条路解,不要求它在本次回复里重新声明过。**
- //
- // 从前这里只查 `entity_ids`,模型用「已知实体」句柄带进来的、
- // 或者只写了名字没重列的主语一律落空——实测一轮 51 块里
- // `subject_not_declared` 丢掉 113 条,丢的是 NVIDIA 的营收、
- // 净利、每股收益,是十位董事的赞成票与反对票,全是有名有姓的
- // 主语。属性那一档要求主语有类型(domain 要校验),这一档没有
- // domain 可校验,也就没有理由比关系那条路更严
- let subject_id = match f.subject_ref.as_deref().map(str::trim) {
- Some(handle) => match referenced_entity(&ref_entities, handle) {
- Some(bound) => bound.id,
- None => {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::MALFORMED_ITEM,
- &f.predicate,
- Some(handle),
- )
- .await;
- continue;
- }
- },
- None => {
- match no_ref_name_binding(&entity_ids, &handled_by_name, subject_name) {
- NoRefNameBinding::Legacy(id) => id,
- NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {
- touched_names.insert(
- utopia_store::resolution::normalize_name(subject_name)
- .to_lowercase(),
- );
- resolve_bare(
- &state.pool,
- doc.kb_id,
- entity_type_of.get(subject_name).copied().flatten(),
- subject_name,
- ctx,
- Some(&chunk.text),
- &mut doc_cache,
- &handled_by_name,
- &mut ambiguous_bare_cache,
- &mut needs_adjudication,
- &mut human_reviews_found,
- )
- .await?
- }
- }
- }
- };
- let _ = utopia_store::ontology::record_miss(
- &state.pool,
- doc.kb_id,
- "attribute_type",
- &surface,
- Some(&format!("{subject_name} → {value}")),
- )
- .await;
- /* 值照原文落笔(提示词 8a 要的就是「units and all」),**单位另记一格**。
- 采纳成属性时按 datatype 把 `$5 billion` 换算成 5e9,那一步只看得懂
- 数;符号丢在原文里就再也取不出来了,而「5000000000」少了那个 `$`
- 就不知道是钱还是别的什么 */
- let unit = value
- .as_str()
- .and_then(utopia_extract::parse_quantity)
- .and_then(|(_, u)| u);
- let mut literal = serde_json::json!({ "value": value });
- if let Some(unit) = unit {
- literal["unit"] = serde_json::Value::String(unit);
- }
- let literal = literal;
- if await_nod {
- if let utopia_store::pending::Outcome::Proposed(_) =
- utopia_store::pending::propose(
- &state.pool,
- utopia_store::pending::Proposal {
- kb_id: doc.kb_id,
- subject_id,
- predicate_id: None,
- object_id: None,
- object_value: Some(&literal),
- proposed_predicate: Some(surface.as_str()),
- validity,
- confidence,
- chunk_id: chunk.id,
- proposed_by: proposer.user_id,
- proposed_token: proposer.token_id,
- phrase: None,
- qualifiers: None,
- time_words: None,
- quote_span: None,
- },
- )
- .await?
- {
- pending_count += 1;
- }
- continue;
- }
- let (fact_id, created) = utopia_store::graph::insert_value_fact(
- &state.pool,
- doc.kb_id,
- subject_id,
- None,
- &literal,
- validity,
- confidence,
- )
- .await?;
- touched_facts.push(fact_id);
- utopia_store::graph::add_evidence(
- &state.pool,
- fact_id,
- chunk.id,
- f.quote.as_deref(),
- Some(surface.as_str()),
- )
- .await?;
- if created {
- fact_count += 1;
- }
- continue;
- }
-
- // 关系事实:宾语必填
- let Some(object_name) = f.object.as_deref().map(str::trim).filter(|s| !s.is_empty())
- else {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::OBJECT_MISSING,
- &f.predicate,
- Some(f.subject.trim()),
- )
- .await;
- continue;
- };
- // **未声明的主宾也要过同一道判据。**
- //
- // 从前守卫只装在上面那条声明实体的路上,而这里绕过了它:模型把一整句话
- // 写进 `object`、那句话没出现在 entities 里,这里就转头把它造成了实体。
- // 实测(ai-timeline-ends × schema.org)421 个实体里 76 个无类型,
- // 最长的那个 111 字符——"thermal-imaging equipment used by volunteers
- // flying over the site showed at least 33 generators giving off heat",
- // 那是一整个从句,不是一个东西。**守卫拦住了前门,后门是开的。**
- //
- // 这类实体的害处不止于此:它们永远匹配不到别处的任何提及,
- // 在图上是孤点(实测 59 个),还会拖累消解——每一个都要跟已有实体比一遍。
- if !is_entity_name(f.subject.trim()) || !is_entity_name(object_name) {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::NOT_AN_ENTITY_NAME,
- &f.predicate,
- Some(if is_entity_name(f.subject.trim()) {
- object_name
- } else {
- f.subject.trim()
- }),
- )
- .await;
- continue;
- }
- for side in [f.subject.trim(), object_name] {
- if let Some(signal) = clause_suspect(side) {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::CLAUSE_SUSPECT,
- signal,
- Some(side),
- )
- .await;
- }
- }
-
- // **槽位片段核对**(#582,取代 #578 的词表):模型交出引文里点名每一侧的那几个
- // 字,机器核对(`verify_span`)。描述做主语不落、做宾语落成字面值;点了别的
- // 实体就改绑;其余都只记不拦
- let quote_text = f.quote.as_deref().unwrap_or("");
- // 对着绑上的名字看:有 ref 就是 ref 指的那个实体的名字,没有就是模型写的名字
- let bound_subject = bound_name(&ref_names, f.subject_ref.as_deref(), f.subject.trim());
- let bound_object = bound_name(&ref_names, f.object_ref.as_deref(), object_name);
- // 片段说了算;片段没说清(放行、不在引文、前缀、指代)时,再看模型写的名字
- // 跟它 ref 指的实体对不对得上
- let decisive = |v: &SpanVerdict| {
- matches!(
- v,
- SpanVerdict::Described(_) | SpanVerdict::Rebind(_) | SpanVerdict::Misplaced(_)
- )
- };
- let mut subject_verdict = verify_span(
- f.subject_span.as_deref(),
- bound_subject,
- bound_object,
- quote_text,
- &span_declared,
- );
- if !decisive(&subject_verdict) {
- let w = written_verdict(
- f.subject.trim(),
- bound_subject,
- bound_object,
- f.subject_ref.is_some(),
- &span_declared,
- );
- if decisive(&w) {
- subject_verdict = w;
- }
- }
- let mut object_verdict = verify_span(
- f.object_span.as_deref(),
- bound_object,
- bound_subject,
- quote_text,
- &span_declared,
- );
- if !decisive(&object_verdict) {
- let w = written_verdict(
- object_name,
- bound_object,
- bound_subject,
- f.object_ref.is_some(),
- &span_declared,
- );
- if decisive(&w) {
- object_verdict = w;
- }
- }
- let mut rebound_subject: Option<String> = None;
- let mut rebound_object: Option<String> = None;
- let mut object_described: Option<String> = None;
- let mut subject_described = false;
- for (side, verdict, span, bound) in [
- (
- "subject",
- &subject_verdict,
- f.subject_span.as_deref(),
- bound_subject,
- ),
- (
- "object",
- &object_verdict,
- f.object_span.as_deref(),
- bound_object,
- ),
- ] {
- let (reason, example) = match verdict {
- SpanVerdict::Ok => continue,
- SpanVerdict::NotInQuote => (
- utopia_store::extraction_drops::reason::SPAN_NOT_IN_QUOTE,
- format!("{} ({bound})", span.unwrap_or("")),
- ),
- SpanVerdict::Rebind(name) => {
- if side == "subject" {
- rebound_subject = Some(name.clone());
- } else {
- rebound_object = Some(name.clone());
- }
- (
- utopia_store::extraction_drops::reason::SPAN_REBOUND,
- format!("{} → {name} (was {bound})", span.unwrap_or("")),
- )
- }
- SpanVerdict::Described(text) => {
- if side == "subject" {
- subject_described = true;
- (
- utopia_store::extraction_drops::reason::SUBJECT_DESCRIBED,
- format!("{text} ({bound})"),
- )
- } else {
- object_described = Some(text.clone());
- (
- utopia_store::extraction_drops::reason::OBJECT_DESCRIBED,
- format!("{text} ({bound})"),
- )
- }
- }
- SpanVerdict::Prefixed(text) => (
- utopia_store::extraction_drops::reason::SPAN_PREFIXED,
- format!("{text} ({bound})"),
- ),
- SpanVerdict::Coreference(text) => (
- utopia_store::extraction_drops::reason::SPAN_COREFERENCE,
- format!("{text} ({bound})"),
- ),
- SpanVerdict::Misplaced(text) => (
- utopia_store::extraction_drops::reason::SPAN_MISPLACED,
- format!("{text} ({bound})"),
- ),
- };
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- reason,
- &f.predicate,
- Some(&example),
- )
- .await;
- }
- if subject_described {
- continue;
- }
- let subject_name: &str = rebound_subject.as_deref().unwrap_or(f.subject.trim());
- let subject_ref_eff: Option<&str> = if rebound_subject.is_some() {
- None
- } else {
- f.subject_ref.as_deref().map(str::trim)
- };
- let object_name: &str = rebound_object.as_deref().unwrap_or(object_name);
- let object_ref_eff: Option<&str> =
- if rebound_object.is_some() || object_described.is_some() {
- None
- } else {
- f.object_ref.as_deref().map(str::trim)
- };
- // 主宾没在 entities 里声明时(模型偶尔漏报):库里已经有这个名字的就用它,
- // 库里也没有的**不建**(#559)。从前这里一律先建出来、类型留空,结果
- // 一个库里 20% 的实体是 "lawsuit against OpenAI"、"$5 billion"、"March 2024"
- // 这样的描述——几乎全部只当过宾语,从没当过主语。漏报的实体多半在
- // 前面的分块或别的文档里已经声明过,按名字找得到;找不到的就是描述。
- // 描述做主语的事实丢掉并记账,做宾语的事实照落,宾语落成字面值
- let subject_id = match subject_ref_eff {
- Some(handle) => match referenced_entity(&ref_entities, handle) {
- Some(bound) => bound.id,
- None => {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::MALFORMED_ITEM,
- &f.predicate,
- Some(handle),
- )
- .await;
- continue;
- }
- },
- None => {
- let binding = no_ref_name_binding(&entity_ids, &handled_by_name, subject_name);
- if matches!(binding, NoRefNameBinding::Missing)
- && utopia_store::resolution::existing_by_name(
- &state.pool,
- doc.kb_id,
- subject_name,
- )
- .await?
- .is_none()
- {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::SUBJECT_NOT_DECLARED,
- &f.predicate,
- Some(subject_name),
- )
- .await;
- continue;
- }
- match binding {
- NoRefNameBinding::Legacy(id) => id,
- NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {
- touched_names.insert(
- utopia_store::resolution::normalize_name(subject_name)
- .to_lowercase(),
- );
- resolve_bare(
- &state.pool,
- doc.kb_id,
- None,
- subject_name,
- ctx,
- Some(&chunk.text),
- &mut doc_cache,
- &handled_by_name,
- &mut ambiguous_bare_cache,
- &mut needs_adjudication,
- &mut human_reviews_found,
- )
- .await?
- }
- }
- }
- };
- let object_id = match object_ref_eff {
- Some(handle) => match referenced_entity(&ref_entities, handle) {
- Some(bound) => bound.id,
- None => {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::MALFORMED_ITEM,
- &f.predicate,
- Some(handle),
- )
- .await;
- continue;
- }
- },
- None => {
- let binding = no_ref_name_binding(&entity_ids, &handled_by_name, object_name);
- if object_described.is_some()
- || (matches!(binding, NoRefNameBinding::Missing)
- && utopia_store::resolution::existing_by_name(
- &state.pool,
- doc.kb_id,
- object_name,
- )
- .await?
- .is_none())
- {
- // 没声明、库里也没有,或者片段说它是个描述:宾语落成字面值。谓词照旧对本体;
- // 被动形(`_by`)本该主宾对调,而字面值当不了主语,那条就不给谓词,
- // 原词留在证据上
- let predicate_id = match pred_index.lookup(f.predicate.as_str()) {
- Some((id, false)) => Some(id),
- Some((_, true)) => None,
- None => {
- let _ = utopia_store::ontology::record_miss(
- &state.pool,
- doc.kb_id,
- "relation_type",
- &f.predicate,
- Some(&format!("{} → {}", f.subject, object_name)),
- )
- .await;
- None
- }
- };
- let literal_text: &str = object_described.as_deref().unwrap_or(object_name);
- // 描述那一路在上面核对时已经记过账;这里只记「没声明」的
- if object_described.is_none() {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::OBJECT_UNDECLARED,
- &f.predicate,
- Some(object_name),
- )
- .await;
- }
- let literal = serde_json::json!({ "value": literal_text });
- if await_nod {
- if let utopia_store::pending::Outcome::Proposed(_) =
- utopia_store::pending::propose(
- &state.pool,
- utopia_store::pending::Proposal {
- kb_id: doc.kb_id,
- subject_id,
- predicate_id,
- object_id: None,
- object_value: Some(&literal),
- proposed_predicate: Some(f.predicate.as_str()),
- validity,
- confidence,
- chunk_id: chunk.id,
- proposed_by: proposer.user_id,
- proposed_token: proposer.token_id,
- phrase: None,
- qualifiers: None,
- time_words: None,
- quote_span: None,
- },
- )
- .await?
- {
- pending_count += 1;
- }
- continue;
- }
- let (fact_id, created) = utopia_store::graph::insert_value_fact(
- &state.pool,
- doc.kb_id,
- subject_id,
- predicate_id,
- &literal,
- validity,
- confidence,
- )
- .await?;
- touched_facts.push(fact_id);
- utopia_store::graph::add_evidence(
- &state.pool,
- fact_id,
- chunk.id,
- f.quote.as_deref(),
- Some(f.predicate.as_str()),
- )
- .await?;
- if created {
- fact_count += 1;
- }
- continue;
- }
- match binding {
- NoRefNameBinding::Legacy(id) => id,
- NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {
- touched_names.insert(
- utopia_store::resolution::normalize_name(object_name)
- .to_lowercase(),
- );
- resolve_bare(
- &state.pool,
- doc.kb_id,
- None,
- object_name,
- ctx,
- Some(&chunk.text),
- &mut doc_cache,
- &handled_by_name,
- &mut ambiguous_bare_cache,
- &mut needs_adjudication,
- &mut human_reviews_found,
- )
- .await?
- }
- }
- }
- };
- if subject_id == object_id {
- continue;
- }
- // 先尽量落到本体已有的关系上(写法/时态/被动),**落不上才降级**为 related_to
- // 并记入未匹配统计。降级会把原意抹平成"有关联"——原词写进证据行的
- // proposed_predicate,是这条事实身上唯一还留着原意的地方(谓词消解据此映射回本体)
- let (predicate_id, swap) = match pred_index.lookup(f.predicate.as_str()) {
- Some((id, swap)) => (Some(id), swap),
- None => {
- let _ = utopia_store::ontology::record_miss(
- &state.pool,
- doc.kb_id,
- "relation_type",
- &f.predicate,
- Some(&format!("{} → {}", f.subject, object_name)),
- )
- .await;
- // 本体里没有对应的关系 → **就是没有谓词**(见 `facts.predicate_id`)。
- // 原意留在证据的 proposed_predicate 里,显示时取回。
- // 从前这里落到 related_to 上,还要额外担心"兜底关系被删了"——
- // 那条失败模式连同它的 continue 一起消失了
- (None, false)
- }
- };
- // 被动说法命中的是同一条边的反向:`ChatGPT produced_by OpenAI` 与
- // `OpenAI produces ChatGPT` 是同一条边,存的时候要按本体的方向来,
- // 否则它跟已有的那 130 条 produces 各存各的,图上是两条相反的箭头
- let (subject_id, object_id) = if swap {
- (object_id, subject_id)
- } else {
- (subject_id, object_id)
- };
-
- // **主语违反 domain、而宾语符合时,按本体声明的方向把它掰正。**
- //
- // 先试过提示词,三轮都没赢:违反率从 57% 压到 35%,但压下去的全是
- // 类型判错那一半;**真·反向纹丝不动**(22.7% → 17.1% → 17.6%,
- // 后两个在噪声里)。模型看得见 `employee (organization → person)`,
- // 就是不照做——英语的 "X is an employee of Y" 太强。
- //
- // 这不是新原则:`produced_by` 命中 `produces` 时(见上面那个 `swap`)
- // 早就在自动翻转主宾了,区别只在触发条件是**措辞**还是**签名**。
- //
- // 当初反对自动对调的理由是实体类型不可靠——实测 Elon Musk 被判成
- // `researcher`。那个前提已经不成立:祖先地板与签名类恒在修好之后,
- // 同一批人判成了 `person`。而判据本身很窄——**主语违反且宾语符合**,
- // 两侧都要对上才动。
- //
- // **但绝不静默。** 掰正要留信号:0001 反对的是「用可能错的声明驱动
- // 自动动作」,而看得见、可复查、可反悔的动作不属于那一类。
- let (predicate_id, subject_id, object_id) = if let Some(pid) = predicate_id {
- // 类型**从库里的实体读**,不用抽取器手上那份 `entity_type_of`:
- // 那份只覆盖模型在这一块里声明过的实体,而宾语常是别处已存在的实体,
- // 这一块没重新声明它,于是查不到、判不了、掰不动。实测差别不小——
- // 用声明那份时反向只降到 7.0%,剩下的正是宾语类型查不到的那些
- //
- // 判断本身在 store(`ontology::judge_direction`),**与采纳共用**(#190):
- // 写谓词的路不止这一条,守卫只装在一条上就等于没装。查不出来(库错)
- // 按没有判据处理,照原样落——宁可少掰一条,不能因为一次查询失败丢事实
- let fit = utopia_store::ontology::judge_direction(
- &state.pool,
- pid,
- subject_id,
- object_id,
- )
- .await
- .unwrap_or(utopia_store::ontology::Fit::Unchecked);
- match fit {
- utopia_store::ontology::Fit::Swap => {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::DIRECTION_CORRECTED,
- &f.predicate,
- Some(&format!(
- "{} → {} 按签名掰正为 {} → {}",
- f.subject,
- f.object.as_deref().unwrap_or("?"),
- f.object.as_deref().unwrap_or("?"),
- f.subject
- )),
- )
- .await;
- (Some(pid), object_id, subject_id)
- }
- utopia_store::ontology::Fit::Neither => {
- // **对调也不合法 → 退回没有谓词。**
- //
- // 这不是方向问题,是这个关系压根不适用:schema.org 的
- // `affectedBy` 是医学检验用的,模型要表达「受……影响」时按名字
- // 撞了上来;`amount` 属于融资工具而不是公司,模型没造那个中间
- // 节点就把边挂到了公司上。
- //
- // 从前照原样落库,等于**用本体的名义说一件本体不同意的事**——
- // 图上写着 "OpenAI affectedBy …",读者会以为那是一条医学断言。
- // 这是自信的错误,比空谓词严重得多。
- //
- // 退回空谓词不丢信息:原词落进 `fact_evidence.proposed_predicate`,
- // 显示时由 `fact_surface_predicate()` 取回(0010)。主宾、时间、
- // 证据全都留着,只是不再冒认一个本体关系。**诚实的沉默。**
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::DOMAIN_MISMATCH,
- &f.predicate,
- Some(&format!(
- "{} — {} → 主宾都对不上,退回原文说法",
- f.subject, f.predicate
- )),
- )
- .await;
- (None, subject_id, object_id)
- }
- utopia_store::ontology::Fit::Keep | utopia_store::ontology::Fit::Unchecked => {
- (Some(pid), subject_id, object_id)
- }
- }
- } else {
- (predicate_id, subject_id, object_id)
- };
-
- if await_nod {
- if let utopia_store::pending::Outcome::Proposed(_) = utopia_store::pending::propose(
- &state.pool,
- utopia_store::pending::Proposal {
- kb_id: doc.kb_id,
- subject_id,
- predicate_id,
- object_id: Some(object_id),
- object_value: None,
- proposed_predicate: Some(f.predicate.as_str()),
- validity,
- confidence,
- chunk_id: chunk.id,
- proposed_by: proposer.user_id,
- proposed_token: proposer.token_id,
- phrase: None,
- qualifiers: None,
- time_words: None,
- quote_span: None,
- },
- )
- .await?
- {
- pending_count += 1;
- }
- continue;
- }
- {
- let (fact_id, created) = utopia_store::graph::insert_fact(
- &state.pool,
- doc.kb_id,
- subject_id,
- predicate_id,
- object_id,
- validity,
- confidence,
- )
- .await?;
- touched_facts.push(fact_id);
- /* **边上的属性落笔**(0037)。属性不进去重键:`insert_fact` 复用了旧行也照写——
- 同一条边再听到一次带了金额的,是同一条边补上金额。
- 值照 datatype 换算,单位另记一格(与 object_value 同形);
- key 不在声明里、换不动、与已记的不一致——三种都进丢弃表,不静默 */
- /* 谓词未知(0010,说法在证据上)时属性照写:值落在事实上,声明等
- 关系被采纳时在 `adopt` 里补。不写的话,`invested_in` 在 schema.org
- 库里是未知说法,一句话里的 $1.5 billion 就没有地方放——召回台上
- `oh-invest` 那一条正是这么丢的 */
- if let Some(quals) = f.qualifiers.as_ref() {
- let pid = predicate_id;
- // 克隆出这一组引用:下面撞上已有属性时要往 qualifier_defs 里追加声明
- let defs: Vec<&utopia_core::models::RelationType> = pid
- .and_then(|p| qualifier_defs.get(&p).cloned())
- .unwrap_or_default();
- /* 模型常把币种单独写成一个键(`"amount": "1500000000", "currency": "CNY"`),
- 而不是写进数额里。那不是一个属性,是数额的单位——先把它拿出来,
- 数值属性解不出单位时用它,别让它作为未知 key 进丢弃表 */
- let sibling_currency: Option<&'static str> = quals
- .iter()
- .find(|(k, _)| {
- matches!(
- k.trim().to_lowercase().as_str(),
- "currency" | "币种" | "货币" | "unit" | "单位"
- )
- })
- .and_then(|(_, v)| v.as_str())
- .and_then(utopia_extract::currency_unit);
- for (key, raw) in quals {
- // 模型对没提到的属性会写 null:那是「原文没说」,不是坏值,不记
- if raw.is_null() {
- continue;
- }
- if matches!(
- key.trim().to_lowercase().as_str(),
- "currency" | "币种" | "货币" | "unit" | "单位"
- ) {
- continue;
- }
- let declared = defs
- .iter()
- .find(|q| q.key.eq_ignore_ascii_case(key.trim()))
- .copied();
- /* **未知 key 撞上本库已有的属性定义 → 补一条声明,不丢值。**
- 实测不声明时模型照样写 `amount`、`stake`、`round`,八条全进
- 丢弃表——而这三个属性定义明明都在库里,缺的只是关系上的一条
- 声明。补声明不新建任何东西、可撤(本体页取消勾选即可),
- 所以跟自动扩本体走同一个开关 */
- let adopted = match declared {
- Some(d) => Some(d),
- /* 谓词还未知(0010,说法在证据上):没有关系可声明,值绑到库里
- 已有的属性定义、先落在事实上,采纳时 `adopt` 再补声明。这一步
- 不动本体,所以不看自动扩本体的开关 */
- None if pid.is_none() => {
- attr_by_key.get(&key.trim().to_lowercase()).copied()
- }
- None if kb.auto_extend_ontology => {
- match attr_by_key.get(&key.trim().to_lowercase()).copied() {
- Some(attr) => {
- let pid = pid.expect("checked above");
- match utopia_store::ontology::add_relation_qualifier(
- &state.pool,
- doc.kb_id,
- pid,
- attr.id,
- )
- .await
- {
- Ok(()) => {
- tracing::info!(kb_id = %doc.kb_id, relation = %f.predicate, qualifier = %attr.key, "边上的属性按语料补了声明");
- qualifier_defs.entry(pid).or_default().push(attr);
- Some(attr)
- }
- Err(e) => {
- tracing::warn!(kb_id = %doc.kb_id, error = %e, "补声明失败");
- None
- }
- }
- }
- None => {
- tracing::debug!(kb_id = %doc.kb_id, relation = %f.predicate, key = %key, attrs = attr_by_key.len(), "边上的属性:key 撞不上本库任何属性定义");
- None
- }
- }
- }
- None => {
- tracing::debug!(kb_id = %doc.kb_id, relation = %f.predicate, key = %key, auto_extend = kb.auto_extend_ontology, "边上的属性:未声明且不自动扩本体");
- None
- }
- };
- let Some(def) = adopted else {
- /* **绑不上属性定义的数也不丢。** 库里没有这个属性(schema.org 里
- `amount` 是关系不是属性)、或本体冻着不让扩——从前这里进丢弃表,
- 数就只剩丢弃表里的一个样例。现在照 8a 的样子落成主语上的一条
- 字面值事实:值按原文、单位另记一格、原词 `关系.键` 进证据的
- proposed_predicate,缺的定义记进 ontology_misses(0010 的样子),
- 采纳时人来决定它归哪儿。图里有它、有证据、能查到 */
- let wording = format!("{}.{}", f.predicate.trim(), key.trim());
- let unit = raw
- .as_str()
- .and_then(utopia_extract::parse_leading_quantity)
- .and_then(|(_, u)| u)
- .or_else(|| sibling_currency.map(str::to_string));
- let mut literal = serde_json::json!({ "value": raw });
- if let Some(u) = unit {
- literal["unit"] = serde_json::Value::String(u);
- }
- let _ = utopia_store::ontology::record_miss(
- &state.pool,
- doc.kb_id,
- "attribute_type",
- &wording,
- Some(&format!("{} → {raw}", f.subject.trim())),
- )
- .await;
- let (literal_id, _) = utopia_store::graph::insert_value_fact(
- &state.pool,
- doc.kb_id,
- subject_id,
- None,
- &literal,
- validity,
- confidence,
- )
- .await?;
- touched_facts.push(literal_id);
- utopia_store::graph::add_evidence(
- &state.pool,
- literal_id,
- chunk.id,
- f.quote.as_deref(),
- Some(wording.as_str()),
- )
- .await?;
- tracing::debug!(kb_id = %doc.kb_id, wording = %wording, "边上的属性绑不上定义,落成主语上的字面值");
- continue;
- };
- let dt = def.datatype.as_deref().unwrap_or("text");
- let Some(normalized) = utopia_extract::normalize_attr_value(dt, raw) else {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::QUALIFIER_DATATYPE,
- &format!("{}.{} ({dt})", f.predicate, def.key),
- Some(&raw.to_string()),
- )
- .await;
- continue;
- };
- let mut value = serde_json::json!({ "value": normalized });
- if let Some(u) = unit_for(raw, dt, sibling_currency, def.unit.as_deref()) {
- value["unit"] = serde_json::Value::String(u);
- }
- let write = utopia_store::graph::upsert_fact_qualifier(
- &state.pool,
- fact_id,
- def.id,
- &value,
- )
- .await?;
- if write == utopia_store::graph::QualifierWrite::Conflict {
- drop_signal(
- state,
- doc.kb_id,
- document_id,
- utopia_store::extraction_drops::reason::QUALIFIER_CONFLICT,
- &format!("{}.{}", f.predicate, def.key),
- Some(&raw.to_string()),
- )
- .await;
- }
- }
- }
- // 重复观察也要挂证据:多来源相互印证,任一来源删除后事实不孤儿化。
- // 表层谓词随每次观察落笔——甲块说 "runs on"、乙块说 "optimized for"
- // 会并进同一条事实,放事实上就是先写者胜,放证据上两个都留着
- utopia_store::graph::add_evidence(
- &state.pool,
- fact_id,
- chunk.id,
- f.quote.as_deref(),
- Some(f.predicate.as_str()),
- )
- .await?;
- if created {
- fact_count += 1;
- }
- // 时态对账:带唯一性约束的状态关系落新事实即检测矛盾(纯规则点查,
- // 自动闭合走"作废+改写",拿不准进 fact_conflicts 人裁)。并进已有断言的
- // 也对:多了一份证据,时间线的形状可能跟着变(#679)
- // 没有谓词就没有关系元数据,也就不参与时态对账——
- // 一条说不出是什么关系的边,本来就不可能带唯一性约束
- if let Some((pid, (func, inv_func, temporal))) =
- predicate_id.and_then(|id| rel_meta.get(&id).map(|m| (id, m)))
- {
- if temporal == "state" {
- let mut directions = Vec::new();
- if *func {
- directions.push(utopia_store::temporal::Uniqueness::SubjectSide);
- }
- if *inv_func {
- directions.push(utopia_store::temporal::Uniqueness::ObjectSide);
- }
- for dir in directions {
- let report = utopia_store::temporal::reconcile_new_fact(
- &state.pool,
- doc.kb_id,
- fact_id,
- subject_id,
- pid,
- Some(object_id),
- None,
- dir,
- validity,
- confidence,
- )
- .await?;
- if report.conflicts > 0 {
- conflicts_found = true;
- }
- }
- }
- }
- }
- }
-
- // 本块抽取完成即打标:更新时被认领的块携带标记跳过;中断的抽取可续跑
- // (LLM 调用/解析失败的块在上方 continue 掉,不打标,下次重试)
- utopia_store::documents::mark_chunk_extracted(&state.pool, chunk.id).await?;
- }
- // 队列里多了东西才叫醒人:Review 的计数与对话里那张确认卡都靠这一声
- if pending_count > 0 {
- tracing::info!(%document_id, pending_count, "记忆抽出的事实进了待确认队列");
- state.emit_pending(doc.kb_id);
- state.emit_review(doc.kb_id);
- }
-
- // 消歧后缀在实体创建时算会早于其事实写入——收尾时对本文档涉及的名字统一刷新
- touched_names.extend(doc_cache.keys().map(|(_, name)| name.clone()));
- for name in &touched_names {
- utopia_store::resolution::refresh_disambiguators(&state.pool, doc.kb_id, name).await?;
- }
-
- // 出口再验一次:接管可能发生在最后一个分块之后,那时循环里的检查已经跑完。
- // 漏掉这里,被顶替的任务会把 done 写在一篇 extracted_at 刚被清空的文档上——
- // 界面显示"已完成",实则一条都没抽,要等新任务开跑才纠正回来。
- if utopia_store::documents::extract_epoch(&state.pool, document_id).await? != my_epoch {
- tracing::info!(%document_id, "抽取任务已被新一轮接管,收尾时退出");
- return Ok(());
- }
-
- // **有分块没抽成就不许标 done。**
- //
- // 从前这里无条件写 done:一次网络抖动让六篇文档 60 块里只抽成 12 块,
- // 六篇全部显示"抽取完成",八成的内容没进图,而界面上没有任何东西说出来。
- // 失败只进了日志,而只进日志的错误等于没有错误。
- //
- // 返回 Err 之后这条链是完整的:`extract_document` 落 graph_failed + 原因,
- // 界面上那篇文档变成可点开看错误的 failed;任务按 30s×attempts² 退避重试,
- // 而已抽成的分块带着 extracted_at 会被跳过——所以重试很便宜,网络恢复就自愈。
- // 重试耗尽才留在 failed,那时它说的是实话。
- //
- // 判据是"全抽完"而不是某个比例:任何比例都是拍的,而这里本来就有一个
- // 不需要拍的判据——**每一块都成了才叫抽完**。
- if let Some(msg) = incomplete_reason(&unextracted, chunks.len()) {
- return Err(anyhow::anyhow!(msg));
- }
- // 刚落的事实立刻过一遍签名。写入时只掰方向(judge_direction);掰不动的
- // ——两个方向都对不上、或宾语没类型判不了——从前要等人按 Review 里的
- // Run check 才露面,Axioms 一直是 0,图里却躺着反向事实(#222)。
- // 检查失败不影响抽取本身:事实已经在库里,下一次 Run check 仍然查得到
- if !touched_facts.is_empty() {
- match utopia_store::reasoning::signature_breaks(
- &state.pool,
- doc.kb_id,
- Some(&touched_facts),
- )
- .await
- {
- Ok(broken) if !broken.is_empty() => {
- match utopia_store::reasoning::record_signature_breaks(
- &state.pool,
- doc.kb_id,
- &broken,
- )
- .await
- {
- Ok(_) => state.emit_review(doc.kb_id),
- Err(e) => {
- tracing::warn!(%document_id, error = %e, "抽取后的签名违规没记进队列")
- }
- }
- }
- Ok(_) => {}
- Err(e) => tracing::warn!(%document_id, error = %e, "抽取后的签名检查失败"),
- }
- }
- utopia_store::documents::set_graph_status(&state.pool, document_id, "done").await?;
- state.emit_document(doc.kb_id, document_id);
-
- // 灰区对进了审核队列 → 触发攒批裁决任务(独立后台跑,抽取本身到此已完成)。
- // 治理开着(0025)排的是治理任务:灰区对与直接转人工的对都从那条先进先出的
- // 队列走,先读台账再裁;同库已排着的不重复
- if kb.governance {
- if needs_adjudication || human_reviews_found {
- utopia_store::jobs::enqueue_unless_queued(
- &state.pool,
- "govern",
- serde_json::json!({ "kb_id": doc.kb_id }),
- )
- .await?;
- }
- } else if needs_adjudication {
- // 同库已排着的不重复——与下面的 resolve_types 一样。一批文档同时抽完
- // 会各排一个,而它们读到的是同一批待裁项
- utopia_store::jobs::enqueue_unless_queued(
- &state.pool,
- "adjudicate_entities",
- serde_json::json!({ "kb_id": doc.kb_id }),
- )
- .await?;
- }
- if needs_adjudication || human_reviews_found || conflicts_found {
- state.emit_review(doc.kb_id);
- }
- // 类型消解排队自动跑(0016 C2):开关开着就排一个库级任务,同库已排着的不重复。
- // 任务自己只看引擎没看过的实体、只自动落地子树内精化的那一档
- if kb.auto_type_resolution {
- utopia_store::jobs::enqueue_unless_queued(
- &state.pool,
- "resolve_types",
- serde_json::json!({ "kb_id": doc.kb_id }),
- )
- .await?;
- }
- // 自动扩本体:开关开着、且这一批都抽完了,由最后一篇触发。
- // 判据是显式开关而不是"本体有没有被碰过"——后者是从行为推断意图,
- // 推错的后果很荒唐(在提案上点一次 Add 就永久关掉建议),而且一旦为假
- // 就永不再真,本体会冻结在第一批文档碰巧包含的词汇上。
- // 并发下可能入队两次,任务自己会重查开关与状态
- enqueue_bootstrap(state, doc.kb_id).await?;
-
- tracing::info!(%document_id, facts = fact_count, "图谱抽取完成");
- Ok(())
-}
-
-/// 一条事实同时给了值和宾语时,宾语是不是一个**没声明的短语**(#685):没有句柄,
-/// 也不是本次回复或本文档前面认下的名字(大小写不计)。是的话这条事实按值落,
-/// 宾语短语并进表层谓词;不是的话宾语是个实体,照旧走边
-fn undeclared_beside_value(
- object_ref: Option<&str>,
- object: &str,
- declared: &HashMap<String, Uuid>,
-) -> bool {
- let object = object.trim();
- !object.is_empty()
- && object_ref.map(str::trim).is_none_or(str::is_empty)
- && !declared
- .keys()
- .any(|name| name.trim().eq_ignore_ascii_case(object))
-}
-
-/// 宾语位上的这串东西,是不是一个字面值而不是实体的名字。
-///
-/// **只认数字与日期。** 这是个会吃掉真实体的判断,所以宁可漏认:
-/// 漏了不过是维持今天的行为(造一个 concept 实体),认错了却是把一个
-/// 真实体降成一段文本,图里少一个节点。
-///
-/// "2015"、"2023-03"、"6" 认;"杭州"、"首席技术官"、"3M"、"V3" 不认。
-/// 调用方还额外要求模型**没有**把它声明成实体——两道门一起过才算数。
-/// 模型给的区间两端 → 落库的有效区间。
-///
-/// **两端各记各的粒度**(见 `facts.valid_to_precision`)。从前一个精度列描述两个端点,
-/// 于是「2020 年开始、2023-05-06 结束」这种只能共用一个值。两端都用 `read_time` 读:
-/// 规则 3 的格式,或写法说得清是哪天的日期(#688),精度随写了几位。
-///
-/// 模型给的 valid_to = "unknown" 表示**原文说它结束了、但没说哪天**。
-/// read_time 解不出它(本来就不是日期),落在这里显式认掉——
-/// 不认的话它退化成 None,那条事实就又变回"仍在持续"了
-fn validity_of(
- valid_from: Option<&str>,
- valid_to: Option<&str>,
- doc_time: Option<chrono::DateTime<chrono::Utc>>,
-) -> utopia_store::graph::Validity<'static> {
- let from = valid_from.and_then(utopia_extract::read_time);
- let to = valid_to.and_then(utopia_extract::read_time);
- let ended_unknown = valid_to
- .map(str::trim)
- .is_some_and(|v| v.eq_ignore_ascii_case(utopia_store::graph::ENDED_UNKNOWN));
- utopia_store::graph::Validity {
- from: from.map(|(t, _)| t),
- from_precision: from.map(|(_, p)| p),
- to: to.map(|(t, _)| t),
- to_precision: to
- .map(|(_, p)| p)
- .or(ended_unknown.then_some(utopia_store::graph::ENDED_UNKNOWN)),
- // 这次观察出自哪一天的文档(0022):没起点的事实从它起成立,结束了
- // 不知哪天的到它为止。没有文档日期就是记下的此刻——账本能给的最好的
- attested_at: doc_time,
- }
-}
-
-fn looks_literal(s: &str) -> bool {
- let s = s.trim();
- if s.is_empty() {
- return false;
- }
- /* **整体是一个量**:可选货币符号 + 数字 + 可选量级词 + 可选百分号,
- 此外一个词都不许有(判据与例子见 `parse_quantity`)。
-
- 从前这里只认裸数字(`s.parse::<f64>()`),于是 `$5 billion` 两头不着:
- 它不是裸数字、也解不成日期,掉进关系那条路,凭空长出一个叫「$5 billion」
- 的节点。同名的又会并成一个点,于是 SSI Inc. 与 Nvidia 因为都出现过这个
- 数额而在图上相连——一条没有含义的路径。实测一个 1415 实体的库里,8 个
- 这样的点、15 条事实指着它们,而**没有任何一条拿它们当主语**:
- 一个从不当主语、只当宾语、名字整体是个量的东西,是值,不是实体。
-
- `parse_quantity` 已经把裸数字那一档包含在内(`"42"` → 42),
- 所以这里不必再单留一条。全角「2015」仍旧解不动,仍旧是想要的 */
- if utopia_extract::parse_quantity(s).is_some() {
- return true;
- }
- // 日期:复用抽取侧那个解析器,它认 2015 / 2015-03 / 2015-03-01,也认写出来的日期(#688)
- utopia_extract::read_time(s).is_some()
-}
-
-/// 提示词里那三段清单:类、关系、属性。
-///
-/// **抽出来是为了让"全给"和"按分块检索"共用同一段排版逻辑。**两条路各排一份
-/// 的话迟早分叉,而分叉在这里的后果是提示词说的与代码认的不是一回事。
-struct PromptLists {
- types: Vec<(String, String, String)>,
- relations: Vec<utopia_extract::PromptRelation>,
- attributes: Vec<String>,
-}
-
-impl PromptLists {
- /// 这三段铺进提示词有多长。budget 判据用它——**量的是实际要排的那段字**,
- /// 不是另写一个估算公式(公式会跟排版分叉)。
- fn chars(&self) -> usize {
- self.types
- .iter()
- .map(|(k, l, d)| k.len() + l.len() + d.len() + 6)
- .sum::<usize>()
- + self
- .relations
- .iter()
- .map(|r| r.key.len() + r.label.len() + r.description.len() + r.signature.len() + 8)
- .sum::<usize>()
- + self.attributes.iter().map(|a| a.len() + 1).sum::<usize>()
- }
-}
-
-/// 把当前本体在提示词里的字符数算出来。**空铺**(不筛类/关系)——这就是
-/// `extract_document` 用的「全铺」档,也是判断「要不要按块检索」的标准。
-///
-/// 抽成独立函数是因为 `ontology_index::gate_required`(#526)要在加载抽取器
-/// 之前问一次预算——那时 `build_lists` 还没被调用。两个路径必须用同一个判据,
-/// 否则 gate 的判定会和实际的「全铺」走分。
-pub(crate) fn full_ontology_chars(
- etypes: &[utopia_core::models::EntityType],
- rtypes: &[utopia_core::models::RelationType],
-) -> usize {
- build_lists(etypes, rtypes, None, None).chars()
-}
-
-/// 从一个**选择集**排出三段清单。`None` = 全给(本体小于预算时的老路)。
-///
-/// 三处细节都是选择带来的,全给时它们不会触发:
-///
-/// 1. **签名只能提到选中的类**。`works_at (person → organization)` 里那两个 key
-/// 必须是模型看得见的——写一个没铺出去的类名,等于教它输出一个不存在的类型。
-/// 整侧都没选中就退回 `*`。
-/// 2. **属性跟着 domain 走**。属性行是 `class.attr`,它的类没铺出去这行就没意义。
-/// 这也顺带解决了属性段(占提示词 28%)的裁剪,不用单独处理。
-/// 3. **内置类恒在**。检索漏掉的分块仍然要有地方落脚,否则模型无类可选。
-fn build_lists(
- etypes: &[utopia_core::models::EntityType],
- rtypes: &[utopia_core::models::RelationType],
- classes: Option<&HashSet<Uuid>>,
- rels: Option<&HashSet<Uuid>>,
-) -> PromptLists {
- let picked_class = |id: &Uuid| classes.is_none_or(|s| s.contains(id));
- // 按块检索(有选择集)时描述只带第一句:检索那条路只有大的导入本体才走,一块铺上百行,
- // 描述占清单的八成(#701)。全铺的是装得下预算的小本体,描述原样
- let describe = |d: &str| -> String {
- if classes.is_some() {
- utopia_extract::first_sentence(d).to_string()
- } else {
- d.to_string()
- }
- };
- // 边上能带的属性:关系.qualifiers → 属性行(0037)。这里只排版,写入侧另有一份同样的查法
- let rtype_by_id: HashMap<Uuid, &utopia_core::models::RelationType> =
- rtypes.iter().map(|r| (r.id, r)).collect();
- let qualifier_line = |r: &utopia_core::models::RelationType| -> Vec<String> {
- r.qualifiers
- .iter()
- .filter_map(|q| rtype_by_id.get(q).copied())
- .filter(|q| q.kind == "attribute")
- .map(|q| {
- let dt = q.datatype.as_deref().unwrap_or("text");
- match q.unit.as_deref().filter(|u| !u.is_empty()) {
- Some(u) => format!("{}: {dt} {u}", q.key),
- None => format!("{}: {dt}", q.key),
- }
- })
- .collect()
- };
- let picked_rel = |id: &Uuid| rels.is_none_or(|s| s.contains(id));
- let key_of: HashMap<Uuid, &str> = etypes
- .iter()
- .filter(|t| picked_class(&t.id))
- .map(|t| (t.id, t.key.as_str()))
- .collect();
-
- let types = etypes
- .iter()
- .filter(|t| picked_class(&t.id))
- .map(|t| (t.key.clone(), t.label.clone(), describe(&t.description)))
- .collect();
-
- // 一侧的类一个都没铺出去就写 `*`:签名是导向,指向看不见的类只会误导
- let sig_of = |ids: &[Uuid]| -> String {
- let mut keys: Vec<&str> = ids
- .iter()
- .filter_map(|id| key_of.get(id).copied())
- .collect();
- if keys.is_empty() {
- return "*".into();
- }
- keys.sort_unstable();
- keys.join("|")
- };
- let relations = rtypes
- .iter()
- .filter(|r| r.kind != "attribute")
- .filter(|r| picked_rel(&r.id))
- .map(|r| {
- let signature = if r.domains.is_empty() && r.ranges.is_empty() {
- String::new()
- } else {
- format!("{} → {}", sig_of(&r.domains), sig_of(&r.ranges))
- };
- utopia_extract::PromptRelation {
- key: r.key.clone(),
- label: r.label.clone(),
- description: describe(&r.description),
- signature,
- temporal: r.temporal.clone(),
- // `amount: number $`——模型要按这个 key 写,单位提醒它别换算
- qualifiers: qualifier_line(r),
- }
- })
- .collect();
-
- let attributes = rtypes
- .iter()
- .filter(|r| r.kind == "attribute" && picked_rel(&r.id))
- .flat_map(|r| r.domains.iter().map(move |d| (r, d)))
- .filter_map(|(r, domain_id)| {
- let class_key = key_of.get(domain_id)?;
- let dt = r.datatype.as_deref().unwrap_or("text");
- let spec = match &r.unit {
- Some(u) if !u.is_empty() => format!("{dt}, {u}"),
- _ => dt.to_string(),
- };
- let d = describe(&r.description);
- let d = d.trim();
- Some(if d.is_empty() {
- format!("- {class_key}.{} ({spec})", r.key)
- } else {
- format!("- {class_key}.{} ({spec}): {d}", r.key)
- })
- })
- .collect();
-
- PromptLists {
- types,
- relations,
- attributes,
- }
-}
-
-/// 每块检索多少个类 / 关系 / 属性。**待测**——跟预算一样,定它们要那条曲线。
-const PER_CHUNK_CLASSES: i64 = 40;
-const PER_CHUNK_RELATIONS: i64 = 30;
-const PER_CHUNK_ATTRIBUTES: i64 = 30;
-/// 「这批类身上声明的关系/属性」这道地板给多少名额。
-///
-/// 比按相似度那 30 个宽得多,因为它的池子已经被 domain 收窄过一轮——
-/// schema.org 里 person + organization + corporation 三个类身上一共只有 86 个关系。
-/// 上限只是防病态情况(一块认出上百个类),不是筛选手段。
-const PER_CHUNK_DOMAIN_RELATIONS: i64 = 120;
-const PER_CHUNK_DOMAIN_ATTRIBUTES: i64 = 40;
-
-/// 按这一块的向量检索候选,排出这一块专用的三段清单。
-///
-/// 检索失败返回 `Ok(None)` 而不是错误:调用方会退回全量。提示词大是慢,
-/// 没有类可选是抽不出东西——两者之间选前者。
-async fn chunk_lists(
- state: &AppState,
- kb_id: Uuid,
- embedding: &[f32],
- etypes: &[utopia_core::models::EntityType],
- rtypes: &[utopia_core::models::RelationType],
- seed_classes: &HashSet<Uuid>,
- budget: usize,
-) -> anyhow::Result<Option<PromptLists>> {
- let mut classes: HashSet<Uuid> = seed_classes.clone();
- classes.extend(
- utopia_store::ontology::nearest_entity_type_ids(
- &state.pool,
- kb_id,
- embedding,
- PER_CHUNK_CLASSES,
- )
- .await?,
- );
- // **命中什么,就把它的祖先一起铺出去。**
- //
- // 向量检索天然偏爱字面出现在正文里的叶子类。实测一个讲 Sutskever 的分块,
- // 976 个类按距离排:`researcher` 第 4、`corporation` 第 27,
- // 而 `organization` 第 177、`person` 第 359——**前 40 名里一个泛化基类都没有**。
- // 正文写的是 "a researcher at"、"the corporation",从不写 "person"。
- //
- // 两个症状,同一个根因:
- //
- // - 实体判成 `researcher`(schema.org 里它是 `Audience` 的子类,不是人),
- // 于是 `works_for (domain=person)` 全成了违规
- // - `employee (organization → person)` 的签名**退化成 `(* → *)`**——`sig_of`
- // 只认铺出去的类,一侧没铺就写 `*`。模型根本没见过那个方向约束
- //
- // 从前这道地板由 `seed_classes`(`builtin` 的类)兜着,`build_lists` 的注释写着
- // 「内置类恒在:检索漏掉的分块仍然要有地方落脚」。种子退场后(#128)判据就悬空了——
- // 它当初碰巧等价,只因为种子类正好是那几个通用类。
- //
- // 用祖先补这道地板,比维护一张"通用类"清单好:**继承链本来就是本体自己声明的
- // 泛化关系**,谁是谁的上位不需要我们再判断一次。代价是每块多铺几层祖先。
- if !classes.is_empty() {
- let picked: Vec<Uuid> = classes.iter().copied().collect();
- classes.extend(utopia_store::ontology::ancestors_of(&state.pool, &picked).await?);
- }
- // 关系与属性分开检索:两段在提示词里是分开的,混在一起取会让其中一段
- // 被另一段挤空。检索回来是按距离排好的,排队时交替取,两段一起往下让
- let nearest = interleave(
- utopia_store::ontology::nearest_relation_type_ids(
- &state.pool,
- kb_id,
- embedding,
- PER_CHUNK_RELATIONS,
- Some("relation"),
- )
- .await?,
- utopia_store::ontology::nearest_relation_type_ids(
- &state.pool,
- kb_id,
- embedding,
- PER_CHUNK_ATTRIBUTES,
- Some("attribute"),
- )
- .await?,
- );
- let rels: HashSet<Uuid> = nearest.iter().copied().collect();
- // **一个关系被铺出去,它签名点名的类就得跟着铺。**
- //
- // 类与关系是各自独立检索的,而签名依赖两者的交集——`sig_of` 只认铺出去的类,
- // 一侧没铺就写 `*`。于是常出现这种局面:`employee` 跟正文语义相近被捞了进来,
- // 而它的 `organization`(第 795 名)与 `person`(第 630 名)离正文字面很远,
- // 一个都没捞到,签名退化成 `(* → *)`——**方向约束整个消失**,模型按英语直觉
- // 写 `Musk --employee--> Microsoft`,而 schema.org 声明的是 organization → person。
- //
- // 上面那道祖先地板治不了这种块:它从"命中的叶子"往上长,而这里一个相关的
- // 叶子都没命中,没有叶子也就没有祖先。
- //
- // `sig_of` 的注释说「签名指向看不见的类只会误导」——顾虑是对的,但抹掉签名
- // 是拿丢失方向来换。**把类拉进来**两头都保住:模型看得见那个类,签名也排得出。
- // 顺带还对:这些类正是模型马上要用来判类型的那些,`employee` 在场就说明
- // 这一块讲的是雇佣,`organization`/`person` 本来就该在候选里——
- // 按字面相似度捞不到它们,但**本体的结构知道**。
- let signature_classes = |kept: &HashSet<Uuid>| -> HashSet<Uuid> {
- rtypes
- .iter()
- .filter(|r| kept.contains(&r.id))
- .flat_map(|r| r.domains.iter().chain(r.ranges.iter()).copied())
- .collect()
- };
- let mut domain_pool = classes.clone();
- domain_pool.extend(signature_classes(&rels));
-
- // **类进来了,就把本体声明在它们身上的关系也铺出去。**
- //
- // 上面那道祖先地板治的是「类捞不到」,这道治的是「关系捞不到」——同一个
- // 病的两侧。实测一块讲「Jensen Huang, founder and CEO of NVIDIA」的正文:
- // `employee` 排第 10 进了窗口,模型就用了它;而 `founder` 排 267、
- // `job_title` 排 618、`has_occupation` 排 811,一个都没进——**四篇文档里
- // 每个人的职务因此全部没落进图,而且不留任何丢弃信号:模型没被问到,
- // 也就什么都没说,drops 与 misses 两张表都看不见它**(2026-09-08 实测)。
- //
- // 收窄的判据是本体自己声明的 domain,不是又一次相似度猜测:这一块认出了
- // person 与 organization,那么「本体说人和组织能有什么」就该摆在模型面前。
- // 同一块里 `founder` 升到 29、`job_title` 升到 55(池子 86)。
- //
- // **放在 `sig_classes` 之后,是为了不让它反过来撑大类清单。** 放在前面时
- // 这批关系的 range 会顺着签名规则把一大票类拉进来,同一块的提示词从 11.7k
- // 涨到 21.4k——为了一个谓词付两倍的钱。它们的 domain 侧本来就在清单里
- // (地板正是这么选出来的),range 侧退化成 `*` 可以接受:这道地板要办的事
- // 是「让模型看见这个说法存在」,不是把签名补全。
- let domain_ids: Vec<Uuid> = domain_pool.iter().copied().collect();
- let mut on_domains = Vec::new();
- for (limit, kind) in [
- (PER_CHUNK_DOMAIN_RELATIONS, "relation"),
- (PER_CHUNK_DOMAIN_ATTRIBUTES, "attribute"),
- ] {
- on_domains.push(
- utopia_store::ontology::nearest_relation_type_ids_in_domains(
- &state.pool,
- kb_id,
- embedding,
- limit,
- Some(kind),
- &domain_ids,
- )
- .await?,
- );
- }
- let domain_attributes = on_domains.pop().unwrap_or_default();
- let domain_relations = on_domains.pop().unwrap_or_default();
-
- // 排队:按相似度检索到的在前,地板补进来的在后
- let mut seen = rels.clone();
- let mut ranked = nearest;
- ranked.extend(
- interleave(domain_relations, domain_attributes)
- .into_iter()
- .filter(|id| seen.insert(*id)),
- );
-
- // 一个候选都没检索到 = 索引还没建好,退回全量而不是给一份空清单
- if classes.len() <= seed_classes.len() && ranked.is_empty() {
- return Ok(None);
- }
-
- // **这一块的清单也守那个预算**(#701)。预算原本只判「全铺装不装得下」,检索出来的
- // 清单没有上限:三道地板叠上去,schema.org 一块铺到 5.6 万字符,是预算的 2.3 倍,
- // 提示词两万 token 里正文不到 2%。
- //
- // 按排队顺序取前 k 个,每个带着它的结构(签名点名的类;祖先在类清单里已经有了),量
- // 实际排出来的那段字,取装得下的最大 k。多一个候选只会多几行,长度随 k 单调,二分就行。
- // 地板补进来的关系不拉签名类,理由见上
- let lists_for = |k: usize| {
- let kept: HashSet<Uuid> = ranked[..k].iter().copied().collect();
- let near_kept: HashSet<Uuid> = kept.intersection(&rels).copied().collect();
- let mut picked = classes.clone();
- picked.extend(signature_classes(&near_kept));
- build_lists(etypes, rtypes, Some(&picked), Some(&kept))
- };
- let (mut lo, mut hi) = (0usize, ranked.len());
- while lo < hi {
- let mid = (lo + hi).div_ceil(2);
- if lists_for(mid).chars() <= budget {
- lo = mid;
- } else {
- hi = mid - 1;
- }
- }
- if lo < ranked.len() {
- tracing::debug!(
- kept = lo,
- retrieved = ranked.len(),
- budget,
- "按块清单按预算截断"
- );
+ if !document_claims.contains(&id) {
+ document_claims.push(id);
}
- Ok(Some(lists_for(lo)))
+ Ok(id)
}
-/// 两串按距离排好的 id 交替并成一串:a0 b0 a1 b1 …,短的那串用完就接着排长的
-fn interleave(a: Vec<Uuid>, b: Vec<Uuid>) -> Vec<Uuid> {
- let mut out = Vec::with_capacity(a.len() + b.len());
- let (mut a, mut b) = (a.into_iter(), b.into_iter());
- loop {
- match (a.next(), b.next()) {
- (None, None) => return out,
- (x, y) => out.extend(x.into_iter().chain(y)),
- }
+/// 每篇文档都走开放图谱(0044 决定 2):只写文档自己的话,本体不进提示词。
+/// 记忆日志同样从这里进,只是它的陈述先进待确认表等人点头(0015),点头时才落成开放陈述
+async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow::Result<()> {
+ let doc = utopia_store::documents::get(&state.pool, document_id).await?;
+ // 排队之后被删了(#268):墓碑不抽——抽出来的事实会活在一个已删除的出处上
+ if doc.deleted_at.is_some() {
+ tracing::info!(document = %document_id, "skipping a deleted document");
+ return Ok(());
}
-}
+ let kb = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;
+ let settings = utopia_store::settings::get(&state.pool, kb.workspace_id)
+ .await?
+ .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?;
+ let client = llm_util::chat_client(&settings)
+ .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?;
-/// 一条值该记什么单位(#600「单位是读出来的,不是猜的」,实体上的属性与边上的属性同一条规矩)。
-///
-/// 原文里认得出的用原文的(€、¥、%、"EUR 30 million" 的 €);模型把币种单独写成
-/// 一个键时用那个;原文里写着声明的那个单位("4300 人" 对 "人")也算读出来的。
-/// 原文带着一个认不出的单位记号("500 兆瓦"、"francs")时**不能**拿声明的缺省顶上——
-/// 实测 `EUR 30 million` 被存成了 `$`,`500 兆瓦` 被存成了 500 块钱;单位写错比不写更糟。
-/// 只有原文完全没有单位记号(一个光秃秃的数)才落回声明的缺省。
-fn unit_for(
- raw: &serde_json::Value,
- datatype: &str,
- sibling_currency: Option<&str>,
- declared: Option<&str>,
-) -> Option<String> {
- // 文本、日期、布尔值没有单位可言:"3年"、"30日" 是文本,尾巴上的字不是单位
- // ——实测「期限=3年」被记成了 `3年 年`
- if datatype != "number" {
- return None;
- }
- let declared = declared.map(str::trim).filter(|u| !u.is_empty());
- let text = raw.as_str();
- if let Some(u) = text
- .and_then(utopia_extract::parse_leading_quantity)
- .and_then(|(_, u)| u)
- {
- return Some(u);
- }
- if let (Some(c), "number") = (sibling_currency, datatype) {
- return Some(c.to_string());
- }
- if let (Some(t), Some(d)) = (text, declared) {
- if t.contains(d) {
- return Some(d.to_string());
- }
- }
- let has_unit_token = text.is_some_and(|t| {
- t.chars()
- .any(|c| c.is_alphabetic() || matches!(c, '$' | '€' | '£' | '¥' | '₩' | '₹' | '%'))
- });
- if has_unit_token {
- None
- } else {
- declared.map(str::to_string)
- }
+ // 所有权凭证:重抽会自增 epoch,任务据此察觉自己已被接管(见 `run_open` 的分块循环)
+ let my_epoch = utopia_store::documents::extract_epoch(&state.pool, document_id).await?;
+ utopia_store::documents::set_graph_status(&state.pool, document_id, "extracting").await?;
+ state.emit_document(doc.kb_id, document_id);
+ let await_nod = utopia_store::memory::is_memory_document(&state.pool, document_id).await?;
+ crate::extraction_open::run_open(
+ state, &doc, &kb, &settings, &client, my_epoch, proposer, await_nod,
+ )
+ .await
}
#[cfg(test)]
@@ -3454,7 +355,7 @@ mod origin_ceiling_tests {
fn a_described_fact_cannot_close_a_value_by_itself() {
let ceiling = origin_ceiling("described", 0.95);
assert!(ceiling < utopia_store::temporal::AUTO_CLOSE_MIN_CONFIDENCE);
- assert!(ceiling >= MIN_CONFIDENCE, "it still enters the graph");
+ assert_eq!(ceiling, DESCRIBED_CEILING, "it still enters the graph");
assert_eq!(origin_ceiling("described", 0.62), 0.62);
for origin in ["stated", "ocr", "transcribed"] {
assert_eq!(origin_ceiling(origin, 0.95), 0.95);
@@ -3462,625 +363,11 @@ mod origin_ceiling_tests {
}
}
-#[cfg(test)]
-mod unit_for_tests {
- use super::unit_for;
- use serde_json::{json, Value};
-
- fn s(t: &str) -> Value {
- Value::String(t.to_string())
- }
-
- #[test]
- fn a_unit_is_read_from_the_text_before_anything_else() {
- assert_eq!(
- unit_for(&s("EUR 30 million"), "number", None, Some("$")).as_deref(),
- Some("€")
- );
- assert_eq!(
- unit_for(&s("8.6亿元"), "number", None, Some("$")).as_deref(),
- Some("¥")
- );
- assert_eq!(
- unit_for(&s("12%"), "number", None, Some("¥")).as_deref(),
- Some("%")
- );
- assert_eq!(
- unit_for(&s("$5 billion"), "number", None, None).as_deref(),
- Some("$")
- );
- }
-
- #[test]
- fn a_sibling_currency_is_the_unit_of_a_bare_number() {
- assert_eq!(
- unit_for(&s("1500000000"), "number", Some("CNY"), Some("$")).as_deref(),
- Some("CNY")
- );
- // 文本型属性没有币种可言
- assert_eq!(unit_for(&s("B 轮"), "text", Some("CNY"), None), None);
- // 文本值尾巴上的字也不是单位:"3年" 是期限的写法,不是 3 个「年」
- assert_eq!(unit_for(&s("3年"), "text", None, Some("")), None);
- assert_eq!(unit_for(&s("30日"), "text", None, None), None);
- }
-
- #[test]
- fn the_declared_unit_written_in_the_text_counts_as_read() {
- assert_eq!(
- unit_for(&s("4300 人"), "number", None, Some("人")).as_deref(),
- Some("人")
- );
- }
-
- #[test]
- fn an_unknown_unit_token_is_never_overwritten_by_the_default() {
- // 宽松扫描把尾巴上的记号当单位读出来:记的是原文的单位,不是声明的 ¥
- assert_eq!(
- unit_for(&s("500 兆瓦"), "number", None, Some("¥")).as_deref(),
- Some("兆瓦")
- );
- assert_eq!(
- unit_for(&s("30 million francs"), "number", None, Some("$")).as_deref(),
- Some("francs")
- );
- // 扫描读不出、原文却明明带着字:也不拿缺省顶上
- assert_eq!(
- unit_for(&s("about five hundred"), "number", None, Some("$")),
- None
- );
- }
-
- #[test]
- fn a_bare_figure_takes_the_declared_default() {
- assert_eq!(
- unit_for(&s("4300"), "number", None, Some("人")).as_deref(),
- Some("人")
- );
- assert_eq!(
- unit_for(&json!(4300), "number", None, Some("人")).as_deref(),
- Some("人")
- );
- assert_eq!(unit_for(&s("4300"), "number", None, Some("")), None);
- }
-}
-
-#[cfg(test)]
-mod name_tests {
- use super::{clause_suspect, is_entity_name};
-
- /// 样本全部取自实跑出来的库(ai-timeline-ends × schema.org),不是编的。
- #[test]
- fn a_clause_is_not_a_thing() {
- for s in [
- "thermal-imaging equipment used by volunteers flying over the site showed at least 33 generators giving off heat",
- "about the same amount of power as the Tennessee Valley Authority's large gas-fired power plant nearby",
- "removal was driven by growing discontent and distrust with Altman",
- "a risk of developing cancer at four times the national average in 2013",
- "745 of OpenAI's 770 employees",
- ] {
- assert!(!is_entity_name(s), "这是一句话,不该当成实体名:{s}");
- }
- }
-
- /// 一个数额不是一个东西。
- ///
- /// 样本取自实跑出来的库:`$5 billion`、`$1 billion`、`$30 billion` 各自成过节点,
- /// 而且同名的会并成一个点——SSI Inc. 与 Nvidia 因为都出现过「$5 billion」
- /// 在图上相连,那条路径没有任何含义。判据的窄处在**尾巴**:
- /// 后面还有实词的一律放行,因为那时它说的就不再只是那个数。
- #[test]
- fn a_quantity_is_not_a_thing() {
- for s in ["$5 billion", "€1.5 million", "52%", "3.5 million", "35,000"] {
- assert!(!is_entity_name(s), "这是一个量,不该当成实体名:{s}");
- }
- // **以数字开头的真实体一个都不能误伤。** 量级词只认全写,所以 `3M` 解不动;
- // 后面挂着实词的,尾巴那一条接住
- for s in [
- "3M",
- "7-Eleven",
- "23andMe",
- "2025 Atlantic hurricane season",
- "900 million weekly active users",
- "1000 Islands",
- "$10 billion investment",
- ] {
- assert!(is_entity_name(s), "这是真实体,不该被挡:{s}");
- }
- }
-
- /// **真实体会长,但不带谓语。** 判据是词数 + 限定动词,不是字符数——
- /// 下面第一个 57 字符,比上面那条 65 字符的从句还短不了多少
- #[test]
- fn a_long_name_is_still_a_name() {
- for s in [
- "US District Court for the Northern District of California",
- "United States District Court for the District of Delaware",
- "OpenAI's board of directors",
- "Safe Superintelligence Inc.",
- "École Polytechnique",
- "GPT-4",
- ] {
- assert!(is_entity_name(s), "这是真实体,不该被挡:{s}");
- }
- }
-
- /// #193:词表之外的句子,结构信号接住——句号结尾、情态动词。样本来自第二份语料
- #[test]
- fn a_sentence_is_caught_without_its_verb_on_the_list() {
- for s in [
- "The removal could slow down the artificial intelligence industry as a whole.",
- "Shares in Microsoft fell nearly three percent following the announcement.",
- "the board should reconsider its position",
- ] {
- assert!(!is_entity_name(s), "这是一句话,不该当成实体名:{s}");
- }
- // 大写缩写的句号、兼作名词的情态词,都不误伤
- for s in [
- "Safe Superintelligence Inc.",
- "Theresa May",
- "Trash Can Museum",
- ] {
- assert!(is_entity_name(s), "这是真实体,不该被挡:{s}");
- }
- }
-
- /// 弱信号只记不挡:像从句的照常落库,但留下样本
- #[test]
- fn a_suspect_is_recorded_not_rejected() {
- let s = "The committee that reviewed the merger of the two companies";
- assert!(is_entity_name(s));
- assert_eq!(clause_suspect(s), Some("determiner_opens_a_long_string"));
- assert_eq!(
- clause_suspect("committee members who reviewed the merger"),
- Some("relative_or_subordinate_clause")
- );
- for s in [
- "US District Court for the Northern District of California",
- "OpenAI's board of directors",
- "The New York Times",
- "The Men Who Stare",
- ] {
- assert_eq!(clause_suspect(s), None, "{s} 不该被怀疑");
- }
- }
-
- /// 分词在名词短语里完全正常,列进标志词会误伤。
- #[test]
- fn a_participle_is_not_a_predicate() {
- assert!(is_entity_name("equipment used by volunteers"));
- assert!(is_entity_name("Gas-Burning Turbines"));
- }
-
- /// 短名字不做从句判断:`Is` 之类可能是专名的一部分。
- #[test]
- fn a_short_name_is_never_a_clause() {
- assert!(is_entity_name("Was"));
- assert!(is_entity_name("Is Elon"));
- }
-
- #[test]
- fn an_empty_name_is_not_a_name() {
- assert!(!is_entity_name(""));
- assert!(!is_entity_name(" "));
- }
-}
-
#[cfg(test)]
mod tests {
-
- #[test]
- fn two_ranked_lists_take_turns_and_the_longer_one_finishes() {
- let ids: Vec<Uuid> = (0..5).map(|_| Uuid::now_v7()).collect();
- let (a, b) = (vec![ids[0], ids[1], ids[2]], vec![ids[3]]);
- assert_eq!(
- super::interleave(a, b),
- vec![ids[0], ids[3], ids[1], ids[2]]
- );
- assert!(super::interleave(Vec::new(), Vec::new()).is_empty());
- }
-
- #[test]
- fn a_name_declared_for_another_entity_is_not_an_alias() {
- let (probe, project) = (Uuid::now_v7(), Uuid::now_v7());
- let declared = HashMap::from([
- ("海洋探测器1号".to_string(), probe),
- ("海探1项目".to_string(), project),
- ]);
- assert!(name_claimed_elsewhere("海探1项目", probe, &declared));
- assert!(
- name_claimed_elsewhere(" 海探1项目 ", probe, &declared),
- "空白不论"
- );
- assert!(
- !name_claimed_elsewhere("海探1", probe, &declared),
- "没人声明过的名字照收"
- );
- assert!(
- !name_claimed_elsewhere("海洋探测器1号", probe, &declared),
- "自己的名字不算撞"
- );
- }
- use super::{name_claimed_elsewhere, slot_matches, span_in_quote, verify_span, SpanVerdict};
- fn declared(names: &[&str]) -> HashMap<String, Uuid> {
- names
- .iter()
- .map(|n| (n.to_string(), Uuid::now_v7()))
- .collect()
- }
-
- /// 片段核对:#578 的两句真实错例;改绑;头衔与指代只记;正常的放行
- #[test]
- fn a_span_that_names_a_description_is_not_the_entity() {
- let d = declared(&[
- "OpenAI",
- "Anthropic",
- "Sam Altman",
- "OpenAI's board of directors",
- "The Verge",
- "Tasha McCauley",
- ]);
- let quote = "Former OpenAI personnel have founded competing AI companies Anthropic, SpaceXAI, Safe Superintelligence Inc., and Thinking Machines Lab";
- // 名字后面还有词:名字只是修饰语
- assert_eq!(
- verify_span(Some("Former OpenAI personnel"), "OpenAI", "", quote, &d),
- SpanVerdict::Described("Former OpenAI personnel".into())
- );
- assert_eq!(
- verify_span(Some("Anthropic"), "Anthropic", "", quote, &d),
- SpanVerdict::Ok
- );
- // 所有格:名字只是修饰语
- let q3 = "Claude bypassed Anthropic's safeguards";
- assert_eq!(
- verify_span(Some("Anthropic's safeguards"), "Anthropic", "", q3, &d),
- SpanVerdict::Described("Anthropic's safeguards".into())
- );
- // 另一个名字带着修饰,绑到了第三方
- assert_eq!(
- verify_span(
- Some("The Verge reporter"),
- "Sam Altman",
- "",
- "a The Verge reporter wrote",
- &d
- ),
- SpanVerdict::Described("The Verge reporter".into())
- );
- // 名字前面带了词:头衔还是另一件东西分不开,只记
- let q2 = "Over one hundred companies using OpenAI contacted Anthropic";
- assert_eq!(
- verify_span(Some("companies using OpenAI"), "OpenAI", "", q2, &d),
- SpanVerdict::Prefixed("companies using OpenAI".into())
- );
- assert_eq!(
- verify_span(
- Some("entrepreneur Tasha McCauley"),
- "Tasha McCauley",
- "",
- "with entrepreneur Tasha McCauley on the board",
- &d
- ),
- SpanVerdict::Prefixed("entrepreneur Tasha McCauley".into())
- );
- // 片段点了另一个声明过的实体:改绑,不丢
- assert_eq!(
- verify_span(
- Some("Sam Altman"),
- "OpenAI",
- "",
- "Sam Altman announced the deal",
- &d
- ),
- SpanVerdict::Rebind("Sam Altman".into())
- );
- // 单个词包在别的名字里不改绑(#595):句首大写不是专名的证据。"Altman" 绑错到
- // OpenAI 时只记指代——改绑要么精确/词干命中,要么两个词以上
- assert_eq!(
- verify_span(
- Some("Altman"),
- "OpenAI",
- "",
- "Altman announced the deal",
- &d
- ),
- SpanVerdict::Coreference("Altman".into())
- );
- assert_eq!(
- verify_span(
- Some("Stockholders"),
- "NVIDIA",
- "",
- "Stockholders approved the election of each of our ten (10) director nominees",
- &declared(&[
- "NVIDIA",
- "2026 Annual Meeting of Stockholders of NVIDIA Corporation"
- ])
- ),
- SpanVerdict::Coreference("Stockholders".into())
- );
- assert_eq!(
- verify_span(
- Some("OpenAI's board"),
- "OpenAI",
- "",
- "OpenAI's board removed Sam Altman as CEO",
- &d
- ),
- SpanVerdict::Rebind("OpenAI's board of directors".into())
- );
- // 指代:没有任何名字,绑定照旧、只记
- assert_eq!(
- verify_span(
- Some("him"),
- "Sam Altman",
- "",
- "the board reinstated him",
- &d
- ),
- SpanVerdict::Coreference("him".into())
- );
- // 单个普通词包在别的名字里也是指代,不改绑("the company" ≠ "for-profit company")
- let d2 = declared(&[
- "OpenAI",
- "for-profit company",
- "Satya Nadella",
- "Jony Ive",
- "Apple",
- "Microsoft",
- "Helen Toner",
- "Fidji Simo",
- ]);
- assert_eq!(
- verify_span(
- Some("the company"),
- "OpenAI",
- "",
- "the company took over",
- &d2
- ),
- SpanVerdict::Coreference("the company".into())
- );
- // 片段以另一个名字结尾:不猜它是头(v4/v5 里猜错的多过猜对的),照描述处理
- assert_eq!(
- verify_span(
- Some("Microsoft chief executive Satya Nadella"),
- "Microsoft",
- "OpenAI",
- "convinced Microsoft chief executive Satya Nadella",
- &d2
- ),
- SpanVerdict::Described("Microsoft chief executive Satya Nadella".into())
- );
- assert_eq!(
- verify_span(
- Some("Swisher and The Verge reporter Alex Heath"),
- "Kara Swisher",
- "",
- "Swisher and The Verge reporter Alex Heath stated",
- &declared(&["Kara Swisher", "Alex Heath", "The Verge"])
- ),
- SpanVerdict::Described("Swisher and The Verge reporter Alex Heath".into())
- );
- // 以另一侧的名字结尾:抄错了位置,绑定照旧、只记
- assert_eq!(
- verify_span(
- Some("CEO of Applications: Fidji Simo"),
- "OpenAI",
- "Fidji Simo",
- "CEO of Applications: Fidji Simo",
- &d2
- ),
- SpanVerdict::Misplaced("CEO of Applications: Fidji Simo".into())
- );
- // 末尾的名字是介词的补语或名单的最后一项,不是头:不改绑(v4 的四个错例)
- let d3 = declared(&[
- "OpenAI",
- "companies using OpenAI",
- "TBPN",
- "California",
- "Pioneer Building",
- "San Francisco",
- "Anthropic",
- ]);
- assert_eq!(
- verify_span(
- Some("Over one hundred companies using OpenAI"),
- "companies using OpenAI",
- "Anthropic",
- "Over one hundred companies using OpenAI contacted Anthropic",
- &d3
- ),
- SpanVerdict::Prefixed("Over one hundred companies using OpenAI".into())
- );
- assert_eq!(
- verify_span(
- Some("his vested equity in OpenAI"),
- "vested equity",
- "Sam Altman",
- "forfeited his vested equity in OpenAI",
- &d3
- ),
- SpanVerdict::Described("his vested equity in OpenAI".into())
- );
- assert_eq!(
- verify_span(
- Some("TBPN, a media company in California"),
- "TBPN",
- "OpenAI",
- "acquired TBPN, a media company in California",
- &d3
- ),
- SpanVerdict::Ok
- );
- assert_eq!(
- verify_span(
- Some("the Pioneer Building in the Mission District, San Francisco"),
- "Pioneer Building",
- "OpenAI",
- "located in the Pioneer Building in the Mission District, San Francisco",
- &d3
- ),
- SpanVerdict::Described(
- "the Pioneer Building in the Mission District, San Francisco".into()
- )
- );
- // 名字后面紧跟逗号:同位语,还是它
- assert_eq!(
- verify_span(
- Some("Helen Toner, strategy director for the Center for Security and Emerging Technology"),
- "Helen Toner",
- "OpenAI",
- "and Helen Toner, strategy director for the Center for Security and Emerging Technology",
- &d2
- ),
- SpanVerdict::Ok
- );
- // 没给片段:老行为
- assert_eq!(verify_span(None, "OpenAI", "", quote, &d), SpanVerdict::Ok);
- // 片段不在引文里:只记不拦
- assert_eq!(
- verify_span(Some("OpenAI staff"), "OpenAI", "", q2, &d),
- SpanVerdict::NotInQuote
- );
- }
-
- /// 模型写的名字和它 ref 指的实体打架:写 "OpenAI employees" 却指向 OpenAI
- #[test]
- fn a_written_name_that_describes_its_reference_is_a_description() {
- use super::written_verdict;
- let d = declared(&[
- "OpenAI",
- "Anthropic",
- "Sam Altman",
- "OpenAI's board of directors",
- ]);
- // v5 的最后一条假边:片段 "eleven employees" 没有名字拦不住,写的名字拦得住
- assert_eq!(
- written_verdict("OpenAI employees", "OpenAI", "Anthropic", true, &d),
- SpanVerdict::Described("OpenAI employees".into())
- );
- assert_eq!(
- written_verdict("Former OpenAI personnel", "OpenAI", "Anthropic", true, &d),
- SpanVerdict::Described("Former OpenAI personnel".into())
- );
- // 写的是另一个声明过的实体:改绑
- assert_eq!(
- written_verdict(
- "OpenAI's board of directors",
- "OpenAI",
- "Sam Altman",
- true,
- &d
- ),
- SpanVerdict::Rebind("OpenAI's board of directors".into())
- );
- // 写的就是那个名字(含后缀、部分)、没有 ref、或是指代:无事
- assert_eq!(
- written_verdict("OpenAI, Inc.", "OpenAI", "", true, &d),
- SpanVerdict::Ok
- );
- assert_eq!(
- written_verdict("Altman", "Sam Altman", "", true, &d),
- SpanVerdict::Ok
- );
- assert_eq!(
- written_verdict("OpenAI employees", "OpenAI", "", false, &d),
- SpanVerdict::Ok
- );
- assert_eq!(
- written_verdict("the company", "OpenAI", "", true, &d),
- SpanVerdict::Ok
- );
- }
-
- /// 名字后面接着 and 再接专名,是并列的一项,不是描述(#595);接着 and 再接小写
- /// 的描述还是描述
- #[test]
- fn a_name_in_a_coordination_is_one_of_the_list() {
- let d = declared(&["SB Energy", "SoftBank", "OpenAI"]);
- let q = "SB Energy and SoftBank will build at least 10 GW of new energy generation";
- assert_eq!(
- verify_span(Some("SB Energy and SoftBank"), "SB Energy", "", q, &d),
- SpanVerdict::Ok
- );
- assert_eq!(
- verify_span(
- Some("SB Energy & SoftBank"),
- "SB Energy",
- "",
- "SB Energy & SoftBank will build",
- &d
- ),
- SpanVerdict::Ok
- );
- // 绑在后一项上:名字前面带词,只记
- assert_eq!(
- verify_span(Some("SB Energy and SoftBank"), "SoftBank", "", q, &d),
- SpanVerdict::Prefixed("SB Energy and SoftBank".into())
- );
- assert_eq!(
- verify_span(
- Some("OpenAI and its investors"),
- "OpenAI",
- "",
- "OpenAI and its investors agreed",
- &d
- ),
- SpanVerdict::Described("OpenAI and its investors".into())
- );
- }
-
- /// 名字后面接着专名样子的续词是同一个东西;接着小写的词就不是
- #[test]
- fn a_name_continued_in_capitals_is_the_same_thing() {
- let d = declared(&["Anthropic", "OpenAI"]);
- assert_eq!(
- verify_span(
- Some("Anthropic PBC"),
- "Anthropic",
- "",
- "Anthropic PBC filed",
- &d
- ),
- SpanVerdict::Ok
- );
- assert_eq!(
- verify_span(
- Some("OpenAI Global, LLC"),
- "OpenAI",
- "",
- "OpenAI Global, LLC is the for-profit arm",
- &d
- ),
- SpanVerdict::Ok
- );
- assert_eq!(
- verify_span(
- Some("OpenAI employees"),
- "OpenAI",
- "",
- "OpenAI employees left",
- &d
- ),
- SpanVerdict::Described("OpenAI employees".into())
- );
- }
-
- #[test]
- fn a_slot_matches_its_name_by_stem_and_suffix() {
- assert!(slot_matches("OpenAI", "OpenAI, Inc."));
- assert!(slot_matches("Acme", "Acme Corp."));
- assert!(slot_matches("openai", "OpenAI"));
- assert!(slot_matches("OpenAI's", "OpenAI"));
- assert!(slot_matches("Altman", "Sam Altman"));
- assert!(slot_matches("Anthropic", "Anthropic, PBC"));
- assert!(slot_matches(
- "the Center for Security and Emerging Technology",
- "Center for Security and Emerging Technology"
- ));
- assert!(!slot_matches("Former OpenAI personnel", "OpenAI"));
- assert!(!slot_matches("Anthropic", "OpenAI"));
- }
+ use super::{incomplete_reason, resolve_handle, span_in_quote};
+ use std::collections::HashMap;
+ use uuid::Uuid;
#[test]
fn a_span_is_found_in_its_quote_regardless_of_case_and_spacing() {
@@ -4095,109 +382,6 @@ mod tests {
assert!(!span_in_quote("", "anything"));
}
- use super::{
- incomplete_reason, looks_literal, no_ref_name_binding, referenced_entity, resolve_bare,
- resolve_handle, validity_of, BoundEntity, NoRefNameBinding,
- };
- use std::collections::HashMap;
- use uuid::Uuid;
-
- #[test]
- fn quantities_and_dates_count_as_literals() {
- // 认:这些出现在宾语位上时是值,不是实体
- for yes in [
- "2015",
- "2023-03",
- "2024-01-15",
- "1200",
- "62.5",
- "-3",
- // 带符号与量级词的量。从前这一档不认,于是图上长出一个叫
- // 「$5 billion」的节点,同名的还并成一个,把毫不相干的两家公司连起来
- "$5 billion",
- "€1.5 million",
- "52%",
- "35,000",
- // 合同照原文写的日期(#688)
- "June 23, 2020",
- "17 Mar. 2020",
- "2020年3月17日",
- ] {
- assert!(looks_literal(yes), "{yes} 该认成字面值");
- }
- // 不认:判错的代价是把一个真实体降成一段文本,所以宁可漏
- for no in [
- "杭州",
- "首席技术官",
- "3M",
- "V3",
- "深蓝存储",
- "",
- " ",
- "2015", // 全角数字:不是我们要处理的形态,交给实体路径
- // 尾巴上还有实词:它说的不再只是那个数
- "900 million weekly active users",
- "2025 Atlantic hurricane season",
- "$10 billion investment",
- "8GW data center",
- ] {
- assert!(!looks_literal(no), "{no} 不该认成字面值");
- }
- }
-
- /// 区间两端照合同原文写(#688):读成日期,精度随写了几位;「unknown」仍是结束了不知哪天
- #[test]
- fn a_written_start_keeps_the_precision_it_was_written_with() {
- let day = validity_of(Some("June 8, 2020"), None, None);
- assert_eq!(
- day.from.map(|t| t.date_naive().to_string()).as_deref(),
- Some("2020-06-08")
- );
- assert_eq!(day.from_precision, Some("day"));
- assert!(!day.has_ended());
-
- let month = validity_of(Some("March 2020"), Some("17 Mar. 2021"), None);
- assert_eq!(month.from_precision, Some("month"));
- assert_eq!(
- month.to.map(|t| t.date_naive().to_string()).as_deref(),
- Some("2021-03-17")
- );
- assert_eq!(month.to_precision, Some("day"));
-
- // 说不清几月几号的不当起点
- let ambiguous = validity_of(Some("03/04/2020"), Some("unknown"), None);
- assert_eq!((ambiguous.from, ambiguous.from_precision), (None, None));
- assert_eq!(ambiguous.to_precision, Some("unknown"));
- }
-
- #[test]
- fn no_ref_names_bypass_legacy_map_only_after_two_handle_claims() {
- let (a, b) = (Uuid::now_v7(), Uuid::now_v7());
- let entity_ids = HashMap::from([("Zhang Wei".to_string(), b)]);
- let normalized = utopia_store::resolution::normalize_name("Zhang Wei").to_lowercase();
-
- assert_eq!(
- no_ref_name_binding(&entity_ids, &HashMap::new(), "Zhang Wei"),
- NoRefNameBinding::Legacy(b)
- );
- assert_eq!(
- no_ref_name_binding(
- &entity_ids,
- &HashMap::from([(normalized.clone(), vec![a])]),
- "Zhang Wei",
- ),
- NoRefNameBinding::Legacy(b)
- );
- assert_eq!(
- no_ref_name_binding(
- &entity_ids,
- &HashMap::from([(normalized, vec![a, b])]),
- "Zhang Wei",
- ),
- NoRefNameBinding::AmbiguousHandled
- );
- }
-
/// 这条判据存在的理由:一次网络抖动让六篇文档 60 块里只抽成 12 块,
/// 六篇**全部显示"抽取完成"**,八成的内容没进图,界面上没有任何东西说出来。
#[test]
@@ -4229,7 +413,7 @@ mod tests {
}
#[tokio::test]
- async fn namesake_handles_keep_fact_attribution_and_bare_mentions_get_c() -> anyhow::Result<()>
+ async fn namesake_handles_keep_fact_attribution_and_a_fresh_handle_gets_c() -> anyhow::Result<()>
{
let Some(url) = utopia_store::test_db::url() else {
return Ok(());
@@ -4264,32 +448,6 @@ mod tests {
.await?;
let run = async {
- let mut legacy_cache = HashMap::new();
- let mut legacy_needs_adjudication = false;
- let legacy_first = super::resolve(
- &pool,
- kb,
- Some(person),
- "Alice",
- None,
- None,
- &mut legacy_cache,
- &mut legacy_needs_adjudication,
- )
- .await?;
- let legacy_second = super::resolve(
- &pool,
- kb,
- Some(person),
- "Alice",
- None,
- None,
- &mut legacy_cache,
- &mut legacy_needs_adjudication,
- )
- .await?;
- assert_eq!(legacy_first, legacy_second, "legacy name cache is unchanged");
-
let mut response_claims = HashMap::new();
let mut document_claims = HashMap::new();
let mut bare_cache = HashMap::new();
@@ -4329,25 +487,8 @@ mod tests {
"namesakes must not wake the LLM worker"
);
- let refs = HashMap::from([
- (
- "e1".to_string(),
- BoundEntity {
- id: a,
- type_id: Some(person),
- },
- ),
- (
- "e2".to_string(),
- BoundEntity {
- id: b,
- type_id: Some(person),
- },
- ),
- ]);
- assert_eq!(referenced_entity(&refs, "e1").map(|x| x.id), Some(a));
- assert_eq!(referenced_entity(&refs, "e2").map(|x| x.id), Some(b));
- assert!(referenced_entity(&refs, "missing").is_none());
+ // 同一回复里两个句柄各绑各的实体:事实跟着句柄走,不跟着名字走
+ let refs = HashMap::from([("e1", a), ("e2", b)]);
let (finance, platform) = (Uuid::now_v7(), Uuid::now_v7());
let works_at = Uuid::now_v7();
sqlx::query(
@@ -4366,7 +507,7 @@ mod tests {
.await?;
}
for (handle, object) in [("e1", finance), ("e2", platform)] {
- let subject = referenced_entity(&refs, handle).expect("valid ref").id;
+ let subject = refs[handle];
utopia_store::graph::insert_fact(
&pool,
kb,
@@ -4400,29 +541,25 @@ mod tests {
assert!(labels.contains(&(a, Some("Finance".to_string()))));
assert!(labels.contains(&(b, Some("Platform Engineering".to_string()))));
- let mut doc_cache = HashMap::new();
- // The legacy surface map still ends on B, but a no-ref fact must not use that
- // last-write-wins value once handles have claimed two distinct Zhang Weis.
- let entity_ids = HashMap::from([("Zhang Wei".to_string(), b)]);
- assert_eq!(entity_ids.get("Zhang Wei"), Some(&b));
- assert_eq!(
- super::no_ref_name_binding(&entity_ids, &document_claims, "Zhang Wei"),
- super::NoRefNameBinding::AmbiguousHandled
- );
- let c = resolve_bare(
+ // 后一次回复给同一个名字开了一个新句柄,没挑 A 或 B:文本里没有挑的依据,
+ // 不许猜。事实落到一个文档级的 C 上,对 A、对 B 各一条人工审核对
+ let mut later_response_claims = HashMap::new();
+ let c = resolve_handle(
&pool,
kb,
Some(person),
"Zhang Wei",
None,
None,
- &mut doc_cache,
- &document_claims,
+ &mut later_response_claims,
+ &mut document_claims,
&mut bare_cache,
&mut needs_adjudication,
&mut human_reviews,
)
.await?;
+ assert_ne!(c, a);
+ assert_ne!(c, b);
utopia_store::graph::insert_fact(
&pool,
kb,
@@ -4433,20 +570,26 @@ mod tests {
0.9,
)
.await?;
- let c_again = resolve_bare(
+ // 再往后的回复再开一个新句柄,仍是那个 C:一个新拼写不是猜 A 或 B 的许可
+ let mut another_response_claims = HashMap::new();
+ let c_again = resolve_handle(
&pool,
kb,
None,
"Zhang Wei",
None,
None,
- &mut doc_cache,
- &document_claims,
+ &mut another_response_claims,
+ &mut document_claims,
&mut bare_cache,
&mut needs_adjudication,
&mut human_reviews,
)
.await?;
+ assert_eq!(
+ c_again, c,
+ "a later response cannot evade A/B ambiguity by inventing a new e-handle"
+ );
utopia_store::graph::insert_fact(
&pool,
kb,
@@ -4457,12 +600,6 @@ mod tests {
0.9,
)
.await?;
- assert_ne!(c, a);
- assert_ne!(c, b);
- assert_eq!(
- c_again, c,
- "later bare mentions must reuse document-local C"
- );
let c_objects: Vec<Uuid> = sqlx::query_scalar(
"SELECT object_id FROM facts
WHERE kb_id = $1 AND subject_id = $2 AND object_id IS NOT NULL ORDER BY object_id",
@@ -4474,25 +611,6 @@ mod tests {
assert_eq!(c_objects.len(), 2);
assert!(c_objects.contains(&finance));
assert!(c_objects.contains(&platform));
- let mut later_response_claims = HashMap::new();
- let c_via_fresh_handle = resolve_handle(
- &pool,
- kb,
- Some(person),
- "Zhang Wei",
- None,
- None,
- &mut later_response_claims,
- &mut document_claims,
- &mut bare_cache,
- &mut needs_adjudication,
- &mut human_reviews,
- )
- .await?;
- assert_eq!(
- c_via_fresh_handle, c,
- "a later response cannot evade A/B ambiguity by inventing a new e-handle"
- );
let reviews = utopia_store::resolution::list_reviews(
&pool,
@@ -4520,11 +638,10 @@ mod tests {
run
}
- /// #270:跨文档的裸 mention 同名并列——不靠 handle,靠画像相似度。库里已有两个
- /// 「张伟」,画像质心一模一样(同一 chunk 播的种)。新 mention 对两人打出同一个分。
- /// 旧路径在最高分 ≥ SIM_ATTACH 时静默 attach 到先遇到的那个;修好之后 `resolve`
- /// 走的这条路要:新建第三个实体、两条**人工**审核对、且绝不唤醒 LLM 裁决器
- /// (否则两条几乎相同的画像会被自动并掉,正是要防的)。
+ /// #270:跨文档的同名并列——不靠句柄,靠画像相似度。库里已有两个「张伟」,画像质心
+ /// 一模一样(同一 chunk 播的种)。新提及对两人打出同一个分。旧路径在最高分 ≥ SIM_ATTACH
+ /// 时静默 attach 到先遇到的那个;修好之后走句柄这条路要:新建第三个实体、两条**人工**
+ /// 审核对、且绝不唤醒 LLM 裁决器(否则两条几乎相同的画像会被自动并掉,正是要防的)。
#[tokio::test]
async fn a_profile_tie_across_documents_files_human_reviews_not_an_attach() -> anyhow::Result<()>
{
@@ -4578,17 +695,22 @@ mod tests {
// 与两人画像都一致的上下文:打出的余弦相同 → 分不开
let ctx: Vec<f32> = vec![1.0, 0.0, 0.0];
- let mut doc_cache = HashMap::new();
- let mut needs_adjudication = false;
- let c = super::resolve(
+ let mut response_claims = HashMap::new();
+ let mut document_claims = HashMap::new();
+ let mut bare_cache = HashMap::new();
+ let (mut needs_adjudication, mut human_reviews) = (false, false);
+ let c = resolve_handle(
&pool,
kb,
Some(person),
"Zhang Wei",
Some(&ctx),
None,
- &mut doc_cache,
+ &mut response_claims,
+ &mut document_claims,
+ &mut bare_cache,
&mut needs_adjudication,
+ &mut human_reviews,
)
.await?;
@@ -4629,36 +751,3 @@ mod tests {
run
}
}
-
-#[cfg(test)]
-mod undeclared_beside_value_tests {
- use super::undeclared_beside_value;
- use std::collections::HashMap;
- use uuid::Uuid;
-
- #[test]
- fn a_phrase_beside_a_value_is_not_an_entity() {
- let mut declared = HashMap::new();
- declared.insert("SB Energy".to_string(), Uuid::nil());
- declared.insert("Microsoft".to_string(), Uuid::nil());
- // 回复里的真形状:没句柄、没声明 → 值落下,短语进谓词
- assert!(undeclared_beside_value(
- None,
- "new energy generation",
- &declared
- ));
- assert!(undeclared_beside_value(
- Some(" "),
- "new regional grid infrastructure",
- &declared
- ));
- // 宾语有句柄,或者是认下的名字(大小写不计)→ 是实体,走边
- assert!(!undeclared_beside_value(
- Some("e2"),
- "new energy generation",
- &declared
- ));
- assert!(!undeclared_beside_value(None, "microsoft", &declared));
- assert!(!undeclared_beside_value(None, " ", &declared));
- }
-}
diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs
index 92436221..c31f1517 100644
--- a/crates/utopia-server/src/extraction_open.rs
+++ b/crates/utopia-server/src/extraction_open.rs
@@ -24,6 +24,7 @@ use crate::extraction::{
span_in_quote,
};
use crate::state::AppState;
+use sqlx::PgPool;
use std::collections::{HashMap, HashSet};
use utopia_core::models::{Document, KnowledgeBase, LlmSettings, Proposer};
use utopia_store::extraction_drops::reason;
@@ -76,6 +77,46 @@ fn name_key(name: &str) -> String {
.to_lowercase()
}
+/// 陈述里写的名字 → 实体。先看这一块列出的,再看本文档前面认下的(提示词里的清单)。
+///
+/// **被描述的东西用到才建。** 模型会把段落里每个名词短语都列进 `e`:实测一个 25 篇的库里
+/// 848 个实体有 616 个是被描述的,其中 181 个没挂任何陈述,还有 155 字的从句。没人指着的
+/// 描述不是实体,只是一句话的一部分——所以描述先记在 `deferred` 里,第一条指到它的陈述
+/// 才把它建出来;同一篇里同一段描述只建一次(`described`)
+async fn place(
+ pool: &PgPool,
+ kb_id: Uuid,
+ local: &mut HashMap<String, Uuid>,
+ known: &HashMap<String, Uuid>,
+ deferred: &HashMap<String, (String, String)>,
+ described: &mut HashMap<String, Uuid>,
+ name: &str,
+) -> anyhow::Result<Option<Uuid>> {
+ let key = name_key(name);
+ if let Some(id) = local.get(&key).or_else(|| known.get(&key)) {
+ return Ok(Some(*id));
+ }
+ let Some((text, kind)) = deferred.get(&key) else {
+ return Ok(None);
+ };
+ let id = match described.get(&key) {
+ Some(id) => *id,
+ None => {
+ let id = utopia_store::resolution::create_described(
+ pool,
+ kb_id,
+ text,
+ (!kind.is_empty()).then_some(kind.as_str()),
+ )
+ .await?;
+ described.insert(key.clone(), id);
+ id
+ }
+ };
+ local.insert(key, id);
+ Ok(Some(id))
+}
+
/// `await_nod`:这是记忆日志(0015)——陈述不直接落库,原样进待确认表,人点头时才成为开放陈述。
/// `proposer`:那句话是谁、经哪枚令牌说的(0026),随待确认项一起记
#[allow(clippy::too_many_arguments)]
@@ -195,6 +236,8 @@ pub(crate) async fn run_open(
// ---- 东西:有名字的走身份消解,被描述的建成没有名字事实的实体 ----
let mut response_claims: HashMap<String, Vec<Uuid>> = HashMap::new();
let mut local: HashMap<String, Uuid> = HashMap::new();
+ // 被描述的东西:先记下名字和类别词,等陈述指到它再建(见 `place`)
+ let mut deferred: HashMap<String, (String, String)> = HashMap::new();
for e in &extraction.entities {
let name = e.name.trim();
if name.is_empty() {
@@ -248,28 +291,11 @@ pub(crate) async fn run_open(
}
id
} else {
- match described.get(&key) {
- Some(id) => *id,
- None => {
- let id = utopia_store::resolution::create_described(
- pool,
- kb_id,
- name,
- (!kind.is_empty()).then_some(kind),
- )
- .await?;
- described.insert(key.clone(), id);
- id
- }
- }
+ deferred.insert(key, (name.to_string(), kind.to_string()));
+ continue;
};
local.insert(key, id);
}
- // 陈述里写的名字 → 实体:先看这一块列出的,再看本文档前面认下的(提示词里的清单)
- let resolve_name = |name: &str| -> Option<Uuid> {
- let key = name_key(name);
- local.get(&key).or_else(|| known_by_name.get(&key)).copied()
- };
// ---- 陈述 ----
for s in &extraction.statements {
@@ -277,7 +303,17 @@ pub(crate) async fn run_open(
if phrase.is_empty() {
continue;
}
- let Some(subject) = resolve_name(&s.subject) else {
+ let Some(subject) = place(
+ pool,
+ kb_id,
+ &mut local,
+ &known_by_name,
+ &deferred,
+ &mut described,
+ &s.subject,
+ )
+ .await?
+ else {
drop_signal(
state,
kb_id,
@@ -299,7 +335,17 @@ pub(crate) async fn run_open(
.map(str::trim)
.filter(|v| v.chars().any(char::is_alphanumeric));
let object = match s.object.as_deref().map(str::trim).filter(|o| !o.is_empty()) {
- Some(name) => match resolve_name(name) {
+ Some(name) => match place(
+ pool,
+ kb_id,
+ &mut local,
+ &known_by_name,
+ &deferred,
+ &mut described,
+ name,
+ )
+ .await?
+ {
Some(id) => Some(id),
None => {
drop_signal(
@@ -376,7 +422,17 @@ pub(crate) async fn run_open(
if role.is_empty() || text.is_empty() {
continue;
}
- match resolve_name(text) {
+ match place(
+ pool,
+ kb_id,
+ &mut local,
+ &known_by_name,
+ &deferred,
+ &mut described,
+ text,
+ )
+ .await?
+ {
Some(id) => qualifiers.push((role, None, Some(id))),
None => qualifiers.push((role, Some(serde_json::json!(text)), None)),
}
@@ -500,7 +556,17 @@ pub(crate) async fn run_open(
if name.is_empty() {
continue;
}
- let Some(id) = resolve_name(&n.entity) else {
+ let Some(id) = place(
+ pool,
+ kb_id,
+ &mut local,
+ &known_by_name,
+ &deferred,
+ &mut described,
+ &n.entity,
+ )
+ .await?
+ else {
drop_signal(
state,
kb_id,
diff --git a/crates/utopia-server/src/ontology_index.rs b/crates/utopia-server/src/ontology_index.rs
index b3e05bce..7b1ef762 100644
--- a/crates/utopia-server/src/ontology_index.rs
+++ b/crates/utopia-server/src/ontology_index.rs
@@ -64,64 +64,6 @@ pub async fn refresh(state: &AppState, kb_id: Uuid) -> anyhow::Result<usize> {
refresh_scoped(state, kb_id, None).await
}
-/// 抽取前的门控(#526)。**走这条路径的抽取器必须先问这个再继续**——
-/// 不问而直接抽,要么抽出来的图是基于半个本体的、要么抽取 worker 在
-/// `refresh` 锁上空转把 32 个并发槽占死。
-///
-/// 返回值语义:
-/// - `Ok(false)`:本体小,全铺就行;本体大但已经在向量里了;或者没配嵌入模型
-/// (照旧全铺)。继续抽。
-/// - `Err(anyhow::Error::context(Deferred{30s}))`:本体超出预算且向量还没补齐,
-/// 调用方应该把这个错误原样往上抛——`jobs::mark_failed` 认 `Deferred`,把任务
-/// 挂回 `queued` 等 30s。worker 槽立刻空出来,下一轮再试。
-pub async fn gate_required(state: &AppState, kb_id: Uuid) -> anyhow::Result<bool> {
- let etypes = utopia_store::graph::entity_types(&state.pool, kb_id).await?;
- let mut rtypes = utopia_store::graph::relation_types(&state.pool, kb_id).await?;
- // 名字属性不进本体向量索引:检索出来就会被当成一条可抽的属性递给模型
- rtypes.retain(|r| !utopia_store::names::is_name_attribute(r));
- let budget = utopia_store::access::ontology_prompt_budget(&state.pool).await?;
- let chars = crate::extraction::full_ontology_chars(&etypes, &rtypes);
- if chars <= budget {
- return Ok(false);
- }
- // 超出预算:需要检索候选。检索候选要先有向量——向量是否就绪取决于
- // `types_needing_embedding`,而它要求一个具体的嵌入模型名。
- //
- // KB 拿不到就不再静默当成「没事」——`run()` 后面还会再读一次同样的字段,
- // 让那次错误冒上来但归因不清。`Err` 直接归到这里,运维日志一眼能找到。
- let kb = utopia_store::kbs::get(&state.pool, kb_id).await?;
- let Some(settings) = utopia_store::settings::get(&state.pool, kb.workspace_id).await? else {
- // 没设模型就走全铺那条路——和原行为一致。配置阶段不该在这里就报错
- return Ok(false);
- };
- // 与 `refresh_scoped` 同一个判据:客户端与模型名缺一个,补齐就什么都不做,
- // 等下去只会空等到期限
- let (Some(_), Some(embed_model)) = (
- llm_util::embed_client(&settings),
- settings.embed_model.clone(),
- ) else {
- // 超预算 + 没配嵌入模型:照旧按全量本体抽。那种部署本来就没有检索,
- // 没有什么可等;判成失败会让只配了对话模型的库一篇都抽不出来
- return Ok(false);
- };
- let stale =
- utopia_store::ontology::types_needing_embedding(&state.pool, kb_id, &embed_model, None)
- .await?;
- if stale.is_empty() {
- // 超预算但所有类型都已嵌好——这个组合很罕见,意味着库被改大后又改小了,
- // 但代码路径是合法的。继续抽
- return Ok(false);
- }
- // 入队补齐任务——同库已排着的不重复(`enqueue_unless_queued` 的契约)
- utopia_store::jobs::enqueue_unless_queued(
- &state.pool,
- "embed_ontology",
- serde_json::json!({ "kb_id": kb_id }),
- )
- .await?;
- Ok(true)
-}
-
/// 只补一半。调用方清楚自己要哪一半时用它——类型消解只用类,
/// 等关系嵌完是白等。补漏的那一半有后台任务兜着。
pub async fn refresh_scoped(
diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rs
index 0ba9cad0..15cf8021 100644
--- a/crates/utopia-store/src/extraction_drops.rs
+++ b/crates/utopia-store/src/extraction_drops.rs
@@ -13,91 +13,24 @@ use utopia_core::AppResult;
use uuid::Uuid;
/// 原因码。前端按这个查文案,所以是稳定契约,不要改字面量。
+/// 都由开放抽取那条路(0044)发出;类型化那条路的原因码随它一起退场了
pub mod reason {
- /// 主语没在 entities 里声明 → 类型不明,属性无法校验 domain
- pub const SUBJECT_NOT_DECLARED: &str = "subject_not_declared";
- /// 属性挂在了不该挂的类上(salary 挂到 Organization)
- pub const ATTR_DOMAIN_MISMATCH: &str = "attr_domain_mismatch";
- /// 属性事实既没给 value 也没给 object
- pub const ATTR_NO_VALUE: &str = "attr_no_value";
- /// 值不合 datatype,归一化失败
- pub const ATTR_DATATYPE: &str = "attr_datatype";
- /// 模型在边上写了一个这条关系没声明过的属性 key(0037)
- pub const QUALIFIER_UNKNOWN: &str = "qualifier_unknown";
- /// 边上属性的值换不成它声明的 datatype
- pub const QUALIFIER_DATATYPE: &str = "qualifier_datatype";
- /// 同一条边再听到一次,属性值与已记的不一致——先记下,不覆盖
- pub const QUALIFIER_CONFLICT: &str = "qualifier_conflict";
- /// 模型自报置信度低于阈值
- pub const LOW_CONFIDENCE: &str = "low_confidence";
- /// 关系事实缺宾语
+ /// 陈述缺宾语也缺值
pub const OBJECT_MISSING: &str = "object_missing";
- /// 模型给的这一条不合结构(缺 predicate 之类)→ 只跳这一条,不牵连整块
+ /// 模型给的这一条不合结构(缺短语之类)→ 只跳这一条,不牵连整块
pub const MALFORMED_ITEM: &str = "malformed_item";
- /// 主语的类型对不上关系声明的 domain,**且对调也不合法**——那是选错了关系
- /// 或类型判错,不是方向问题。照原样落库 + 记信号,交给人看,不猜
- pub const DOMAIN_MISMATCH: &str = "domain_mismatch";
- /// 模型给的"实体名"其实是一整句话或从句——不是一个东西的名字。
- /// 这类东西永远匹配不到别处的提及,在图上是孤点,还会拖累消解
- pub const NOT_AN_ENTITY_NAME: &str = "not_an_entity_name";
- /// 主语违反 domain 而宾语符合,已按本体声明的方向把主宾掰正。
- /// **动作必须留痕**:自动的、看不见的改写才是 0001 反对的那种
- pub const DIRECTION_CORRECTED: &str = "direction_corrected";
/// 模型输出被截断(撞上 max_tokens)→ 已完整的那些留下,尾巴丢掉
pub const TRUNCATED_REPLY: &str = "truncated_reply";
- /// 守卫放行了、结构却像从句(限定词起头的长串、句中的关系词)。**只记不挡**:
- /// 实体照常落库,例句留下来——#193 要的是一份跨语料的标注集,再决定哪条升成硬规则
- pub const CLAUSE_SUSPECT: &str = "clause_suspect";
- /// 主语是「跟 X 有关的一群人」而写成了 X(#578):"former OpenAI personnel" 不是
- /// OpenAI。事实不落,例句是那个短语;这一类该由提示词的规则改写到有名字的一侧,
- /// 这里只量它还错多少
- pub const SUBJECT_SHORTENED: &str = "subject_shortened";
- /// 主宾片段不在引文里(#582):模型没照抄。事实照旧处理,只记下来
- pub const SPAN_NOT_IN_QUOTE: &str = "span_not_in_quote";
/// 模型报的别名(或它的引文)不在这一块原文里(0041 决定 2):不记这个名字。
/// 名字是召回的桥,一座凭空的桥会把两个不相干的实体接到一起
pub const NAME_NOT_IN_TEXT: &str = "name_not_in_text";
/// 模型报的别名,这次回复(或本文档前面几块)里已经是另一个实体的名字(0041 决定 2):
/// 一个名字不会同时是两样东西的名字。「海探1项目」声明成了一个机构,就不是探测器的别名
pub const NAME_CLAIMED_BY_ANOTHER: &str = "name_claimed_by_another";
- /// 主语片段是个描述,不是任何声明过的实体的名字:事实不落(#582,取代 #578 的词表)
- pub const SUBJECT_DESCRIBED: &str = "subject_described";
- /// 宾语片段是个描述:事实照落,宾语落成字面值(#582)
- pub const OBJECT_DESCRIBED: &str = "object_described";
- /// 片段点的是另一个声明过的实体:改绑到它(#582)
- pub const SPAN_REBOUND: &str = "span_rebound";
- /// 片段是所绑名字前面带了别的词("entrepreneur Tasha McCauley" / "companies using
- /// OpenAI"):头衔还是另一件东西,机器分不开,绑定照旧,只记(#582)
- pub const SPAN_PREFIXED: &str = "span_prefixed";
- /// 片段里没有任何声明过的名字("him" / "the company"):模型消解了指代,无从核对,
- /// 绑定照旧,只记(#582)
- pub const SPAN_COREFERENCE: &str = "span_coreference";
- /// 片段抄的是事实**另一侧**的名字(宾语片段写成了主语):抄错了位置,不是绑错了
- /// 实体。绑定照旧,只记(#582)
- pub const SPAN_MISPLACED: &str = "span_misplaced";
- /// 宾语既没在 entities 里声明、库里也没有叫这个名字的东西:事实照落,
+ /// 宾语既没在回复里列出、库里也没有叫这个名字的东西:陈述照落,
/// 宾语落成字面值而不是节点(#559)。记下来是为了量:这一类里有多少
/// 本该是实体(模型漏报),有多少本来就是描述
pub const OBJECT_UNDECLARED: &str = "object_undeclared";
- /// 以下都来自 `utopia_extract::normalize`:只看结构、不看词的形状检查
- /// 值只有破折号(`—`):表里的「无」,不落
- pub const NO_VALUE: &str = "no_value";
- /// 值后面有一截引文里没有的字:只留引文里有的那段。只记
- pub const VALUE_TRIMMED: &str = "value_trimmed";
- /// 没有宾语也没有值、只带边属性:属性落成主语上的值事实。只记
- pub const QUALIFIERS_WITHOUT_OBJECT: &str = "qualifiers_without_object";
- /// 宾语是契约格式的日期:数落成值、日期进有效期;没带数的把写出来的那段落成值。只记
- pub const TIME_AS_OBJECT: &str = "time_as_object";
- /// 主语是契约格式的日期:数是谁的回复里没说,不落
- pub const TIME_AS_SUBJECT: &str = "time_as_subject";
- /// 宾语名字包住另一个声明实体、同句已有指向本尊的边:可能是描述,也可能是另一个
- /// 东西(每股收益包住了净利润)。只记,不删
- pub const OBJECT_DESCRIBES_DECLARED: &str = "object_describes_declared";
- /// 上面几条去掉事实后没人引用的声明:不建
- pub const ORPHAN_DECLARATION: &str = "orphan_declaration";
- /// 引文抄自提示词里附的文件开头、不在这一块:证据会挂错出处,不落
- pub const QUOTE_FROM_OPENING: &str = "quote_from_opening";
- /// 以下三条来自开放抽取那条路(0044 第一刀,#729)
/// 模型的引文在这一块里找不到原样的一句:陈述照落、引文照记,只是没有偏移
/// (`quote_start` / `quote_end` 留空)。这个信号数的是有多少条没定位到
pub const QUOTE_NOT_IN_CHUNK: &str = "quote_not_in_chunk";
diff --git a/crates/utopia-store/src/kbs.rs b/crates/utopia-store/src/kbs.rs
index 14077b09..af2baec8 100644
--- a/crates/utopia-store/src/kbs.rs
+++ b/crates/utopia-store/src/kbs.rs
@@ -110,7 +110,6 @@ pub async fn update(
auto_type_resolution: Option<bool>,
governance: Option<bool>,
data_conventions: Option<&str>,
- open_extraction: Option<bool>,
) -> AppResult<KnowledgeBase> {
// 改语言不回头重写已有的类——它们已经是这个库的数据,可能有人手工调过。
// 这一列往后管的是**新**描述(自动扩本体、AI 建议)写成什么语言
@@ -152,8 +151,6 @@ pub async fn update(
ELSE governance_since END,
-- 人写的约定;清空要送空串,送 null 等于不改(与 description 同一约定)
data_conventions = COALESCE($11, data_conventions),
- -- 开放抽取(0044 第一刀):抽取只写开放图谱;关着走类型化那条路
- open_extraction = COALESCE($12, open_extraction),
updated_at = now()
WHERE id = $1 RETURNING *",
)
@@ -168,7 +165,6 @@ pub async fn update(
.bind(auto_type_resolution)
.bind(governance)
.bind(data_conventions)
- .bind(open_extraction)
.fetch_optional(pool)
.await?
.ok_or(AppError::NotFound)
diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs
index 514a29ce..10f7dd29 100644
--- a/crates/utopia-store/src/resolution.rs
+++ b/crates/utopia-store/src/resolution.rs
@@ -261,7 +261,10 @@ pub async fn resolve_mention(
WHERE (f.subject_id = e.id OR f.object_id = e.id)
AND f.invalidated_at IS NULL AND {not_name}) AS degree
FROM entities e
- WHERE e.kb_id = $1 AND e.type_id = $2 AND e.merged_into IS NULL
+ -- IS NOT DISTINCT FROM 而不是 =(0009 的那个陷阱):开放图谱里的实体都没有类
+ -- (类由对齐来定),`type_id = NULL` 永远不成立,同名的它就永远撞不上——
+ -- 实测一个库里 Securities and Exchange Commission 与它的全大写写法成了两个实体
+ WHERE e.kb_id = $1 AND e.type_id IS NOT DISTINCT FROM $2 AND e.merged_into IS NULL
-- 被描述的东西没有名字(0044):它的 canonical_name 只是显示用的描述,
-- 不是召回的桥——两篇文档里描述得一样的两个东西不能因此接到一起
AND e.description IS NULL
diff --git a/crates/utopia-store/tests/an_untyped_name_meets_its_namesake.rs b/crates/utopia-store/tests/an_untyped_name_meets_its_namesake.rs
new file mode 100644
index 00000000..afce5bed
--- /dev/null
+++ b/crates/utopia-store/tests/an_untyped_name_meets_its_namesake.rs
@@ -0,0 +1,78 @@
+//! 没有类的实体也要撞得上同名者(0044 第一刀之后的实测)。
+//!
+//! 开放图谱里的实体都没有类:类由对齐来定,抽取只记文档的字。`resolve_mention` 的候选
+//! 查询从前写的是 `e.type_id = $2`,传 NULL 时这个条件永远不成立——于是同一个库里
+//! 「Securities and Exchange Commission」和它的全大写写法成了两个实体,只有建实体时
+//! 的精确同名匹配还兜得住。这里守的是:两次没有类的提及,写法只差大小写,落到同一个实体。
+//!
+//! 没有 `UTOPIA_DATABASE_URL` 时跳过。自建自拆,绝不碰已有的库。
+
+use sqlx::PgPool;
+use uuid::Uuid;
+
+#[tokio::test]
+async fn an_untyped_mention_attaches_to_its_untyped_namesake() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'untyped-namesake-test')")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'untyped-namesake-test')",
+ )
+ .bind(ws)
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'untyped-namesake-test')",
+ )
+ .bind(kb)
+ .bind(ws)
+ .execute(&pool)
+ .await?;
+
+ let first = utopia_store::resolution::resolve_mention(
+ &pool,
+ kb,
+ None,
+ "Securities and Exchange Commission",
+ None,
+ None,
+ &[],
+ )
+ .await?;
+ assert!(first.created, "第一次提及要建实体");
+ let again = utopia_store::resolution::resolve_mention(
+ &pool,
+ kb,
+ None,
+ "SECURITIES AND EXCHANGE COMMISSION",
+ None,
+ None,
+ &[],
+ )
+ .await?;
+ assert_eq!(
+ again.entity_id, first.entity_id,
+ "只差大小写的同名提及要落到同一个没有类的实体上"
+ );
+ assert!(!again.created);
+ let n: i64 = sqlx::query_scalar(
+ "SELECT count(*) FROM entities WHERE kb_id = $1 AND merged_into IS NULL",
+ )
+ .bind(kb)
+ .fetch_one(&pool)
+ .await?;
+ assert_eq!(n, 1);
+
+ sqlx::query("DELETE FROM organizations WHERE id = $1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ Ok(())
+}
diff --git a/crates/utopia-store/tests/exploration_describes_the_data.rs b/crates/utopia-store/tests/exploration_describes_the_data.rs
index fdf1dcc3..20d564ea 100644
--- a/crates/utopia-store/tests/exploration_describes_the_data.rs
+++ b/crates/utopia-store/tests/exploration_describes_the_data.rs
@@ -51,7 +51,6 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> an
None,
None,
Some("Amounts are in cents.\nis_test = 1 is excluded."),
- None,
)
.await?;
// 探索写描述与问题
@@ -102,7 +101,6 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> an
None,
None,
Some("cents only"),
- None,
)
.await?;
let got = utopia_store::kbs::get(&pool, kb).await?;
@@ -121,7 +119,6 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> an
None,
None,
no,
- None,
)
.await?;
assert_eq!(
diff --git a/migrations/0063_extraction_reads_only_the_open_graph.sql b/migrations/0063_extraction_reads_only_the_open_graph.sql
new file mode 100644
index 00000000..a648c5ee
--- /dev/null
+++ b/migrations/0063_extraction_reads_only_the_open_graph.sql
@@ -0,0 +1,10 @@
+-- 抽取只写开放图谱(0044 决定 2):开关退场。
+--
+-- 0061 加这一列时给带本体的老路留了一个关掉的口子——按当时的决定 3,它是已批准本体下
+-- 的可选第二路,等对齐追平就退场。现在定了:没有遗留层,也没有可选的类型化第二路。
+-- 每篇文档(记忆日志也在内)都走 `extraction_open::run_open`,本体不进提示词;类型化的
+-- 事实由对齐从开放陈述算出来,不由抽取写。一个再也没有第二个取值的开关不该留在表里:
+-- 留着,读的人会以为还有一条路可以切回去。
+--
+-- 没有已发布的版本,直接删列;已有的 `layer = 'typed'` 行不动,它们是对齐的输入。
+ALTER TABLE knowledge_bases DROP COLUMN open_extraction;
diff --git a/scripts/bench/README.md b/scripts/bench/README.md
index 7449ae1d..39a1cba2 100644
--- a/scripts/bench/README.md
+++ b/scripts/bench/README.md
@@ -65,13 +65,12 @@ node scripts/bench/fetch-sec-filings.mjs # 一次就够
node scripts/bench/recall.mjs --kb <id> # 一轮:清空 → 重抽 → 打分
node scripts/bench/recall.mjs --kb <id> --score # 只打分
node scripts/bench/recall.mjs --kb <id> --reprocess # 改了解析器:连分块一起重来
-node scripts/bench/recall.mjs --kb <id> --typed # 带本体的老路(可选第二路);缺省走开放图谱(0044 第 1 刀)
node scripts/bench/judge_open.mjs --kb <id> --sample 200 # 开放陈述有多少不是原文说的(门槛 2%)
```
-缺省(库的 `open_extraction` 开着)抽的是开放图谱:陈述按原文短语落成 `layer='open'` 的事实行,
-限定挂在 `statement_qualifiers`,时间词进 `time_mentions`;打分口径不变(谓词取
-`proposed_predicate`,限定也拼进那一行),所以 `--typed` 那条老路和它的分数直接可比。`judge_open.mjs` 是
+抽取写的是开放图谱(0044 决定 2,没有带本体的第二条路):陈述按原文短语落成 `layer='open'` 的
+事实行,限定挂在 `statement_qualifiers`,时间词进 `time_mentions`;打分口径照旧(谓词取
+`proposed_predicate`,限定也拼进那一行),所以下表里带本体那些轮次的分数仍然可比。`judge_open.mjs` 是
反面的尺子:抽样让一个裁判模型读原文判 stated / misworded / not_stated;裁判端点用
`BENCH_JUDGE_BASE / BENCH_JUDGE_KEY / BENCH_JUDGE_MODEL`,不给就用工作区的对话模型
(和抽取同一个模型,数字要打折看)。
diff --git a/scripts/bench/recall.mjs b/scripts/bench/recall.mjs
index 18d93fc8..cc9f85b9 100644
--- a/scripts/bench/recall.mjs
+++ b/scripts/bench/recall.mjs
@@ -16,7 +16,6 @@
// node scripts/bench/recall.mjs --kb <id> # 已有库(本体向量已就绪)
// node scripts/bench/recall.mjs --kb <id> --score # 只打分,不重抽
// node scripts/bench/recall.mjs --kb <id> --reprocess # 改了解析器:连分块一起重来
-// node scripts/bench/recall.mjs --kb <id> --typed # 带本体的老路(可选第二路);缺省是开放图谱(0044 第 1 刀)
//
// 环境变量:BENCH_BASE / BENCH_EMAIL / BENCH_PASSWORD / BENCH_PSQL(同 run.mjs)。
//
@@ -212,8 +211,7 @@ psql(`DELETE FROM ontology_misses WHERE kb_id='${KB}'`);
if (packTs) psql(`DELETE FROM relation_types WHERE kb_id='${KB}' AND created_at > '${packTs}'::timestamptz + interval '1 second'`);
// 会在抽取之后改本体、增派生事实的开关关掉,两轮的本体与打分口径才一样。治理开着:
// 库生下来就开着它(0050),量的是产品本来的样子
-psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE, materialize_inferences=FALSE, governance=TRUE, open_extraction=${args.typed ? "FALSE" : "TRUE"} WHERE id='${KB}'`);
-console.log(args.typed ? "带本体的老路:本体进提示词" : "开放图谱:本体不进提示词,陈述按原文短语落库");
+psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE, materialize_inferences=FALSE, governance=TRUE WHERE id='${KB}'`);
psql(`UPDATE chunks SET extracted_at=NULL WHERE document_id IN (SELECT id FROM documents WHERE kb_id='${KB}' AND filename IN (${names}))`);
console.log(`本体 ${num(`SELECT count(*) FROM relation_types WHERE kb_id='${KB}'`)} 个关系 / ${num(`SELECT count(*) FROM entity_types WHERE kb_id='${KB}'`)} 个类`);
diff --git a/scripts/bench/temporal.mjs b/scripts/bench/temporal.mjs
index 9debb330..4a7dc0a1 100644
--- a/scripts/bench/temporal.mjs
+++ b/scripts/bench/temporal.mjs
@@ -437,6 +437,7 @@ async function acceptUniqueness(kb, axioms) {
const byId = new Map(relation_types.map((r) => [r.id, r]));
const rows = [];
for (const c of candidates) {
+ // 候选来自类型化的事实(predicate_id);开放陈述带的是 `phrase`、没有 predicate_id,对不上的跳过
const r = byId.get(c.predicate_id);
if (!r) continue;
const declared = wanted.get(c.key);
diff --git a/web/src/api.ts b/web/src/api.ts
index a193ae77..9f4a110a 100644
--- a/web/src/api.ts
+++ b/web/src/api.ts
@@ -102,9 +102,6 @@ export interface Kb {
/** 抽取遇到本体外的说法时,是否允许系统自动补进本体并改写等它的事实。
关掉不影响"留意":未匹配统计照常累积可见,只是变成你点一下的提案 */
auto_extend_ontology: boolean;
- /** 开放抽取(0044 第一刀):开着,抽取只写开放图谱——陈述照文档的字落库,
- * 不读本体;关着走今天的类型化那条路。缺省关;这一刀没有开关界面 */
- open_extraction: boolean;
/** 把推出来的事实写进账本(R1)。**缺省关**——推理往图里加东西,
* 而声明可能是错的,不该在用户没表态时就按它改图 */
materialize_inferences: boolean;
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index ba0421cc..a2a48450 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -503,20 +503,9 @@ export const en = {
"Each line says why, and how many.",
dropsExample: "e.g.",
dropReason: {
- attr_domain_mismatch: "Attribute on the wrong class",
- subject_not_declared: "Subject type unknown",
- attr_no_value: "Attribute had no value",
- attr_datatype: "Value did not match the datatype",
- low_confidence: "Below the confidence threshold",
object_missing: "Relation had no object",
malformed_item: "The model's item did not fit the schema",
truncated_reply: "The model's reply was cut off",
- domain_mismatch:
- "The subject does not fit the relation, and swapping would not help",
- not_an_entity_name: "That name is a sentence, not a thing",
- clause_suspect: "Kept, but the name reads like a clause: a sample for the guard",
- direction_corrected:
- "Subject and object were swapped to match the signature",
quote_not_in_chunk: "Kept, but the quoted sentence is not in the text verbatim",
time_not_in_quote: "A time mention's words are not in the text",
unknown_ref: "The item points at nothing in the reply",
@@ -2023,7 +2012,6 @@ export const en = {
conflictReason: {
no_time: "new fact has no date",
simultaneous: "same start date",
- low_confidence: "low confidence",
} as Record<string, string>,
conflictVs: "vs",
conflictSince: (d: string) => `since ${d}`,
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index 6b5c03ce..ec8fe671 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -454,18 +454,9 @@ export const zh: Strings = {
"这些是从文档里抽出来的,但在写入途中被挡下了。每行说明原因与条数。",
dropsExample: "例如",
dropReason: {
- attr_domain_mismatch: "属性挂在了错误的类上",
- subject_not_declared: "主语类型未知",
- attr_no_value: "属性没有取值",
- attr_datatype: "取值与数据类型不符",
- low_confidence: "低于置信度阈值",
object_missing: "关系缺少宾语",
malformed_item: "模型给的这一条结构不合",
truncated_reply: "模型的输出被截断",
- domain_mismatch: "主语对不上这个关系,对调也不行",
- not_an_entity_name: "那个名字是一句话,不是一个东西",
- clause_suspect: "已保留,但这个名字像从句:给守卫攒的样本",
- direction_corrected: "已按签名把主宾掰正",
quote_not_in_chunk: "已保留,但引文不是原文原样",
time_not_in_quote: "时间词不在原文里",
unknown_ref: "这一条指向的东西不在回复里",
@@ -1764,7 +1755,6 @@ export const zh: Strings = {
conflictReason: {
no_time: "新事实没有日期",
simultaneous: "起始日期相同",
- low_confidence: "置信度低",
},
conflictVs: "对",
conflictSince: (d: string) => `自 ${d} 起`,