World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Extraction reads only the open graph (#736)
WaylandYang42930b1a9dec
diff --git a/Cargo.lock b/Cargo.lockindex bf685952..66de830c 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5414,7 +5414,6 @@ dependencies = ["serde","serde_json","tracing",- "unicode-segmentation","utopia-llm",]diff --git a/Cargo.toml b/Cargo.tomlindex b0e96793..bb197b5d 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -64,7 +64,6 @@ text-splitter = { version = "0.32", features = ["tiktoken-rs"] }# 分块预算按 token 数:cl100k 的排名表随 crate 内嵌,离线可用tiktoken-rs = "0.12"# 句子边界按 Unicode 标准(UAX #29)切,不自己写标点规则-unicode-segmentation = "1.13"pdf-extract = "0.12"calamine = "0.36"pgvector = { version = "0.4", features = ["sqlx"] }diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rsindex aa8fa8f9..d90fa4a8 100644--- a/crates/utopia-core/src/models.rs+++ b/crates/utopia-core/src/models.rs@@ -1077,10 +1077,6 @@ pub struct KnowledgeBase {/// 这次打开治理的时刻;保险丝只数它之后的撤回(0025 决定 9)。/// 新库生下来就开着治理,这一格于是等于建库的时刻(0050)pub governance_since: Option<DateTime<Utc>>,- /// 开放抽取(0044 第一刀,#729):开着,抽取只写开放图谱——陈述照文档的字落库- /// (`facts.layer = 'open'`),不读本体、不问日期。**缺省开**(0044 决定 2);关掉走- /// 带本体的老路,它按决定 3 只是已批准本体下的可选第二路,等对齐追平就退场- pub open_extraction: bool,/// 多久重推一次(分钟)。见 `knowledge_bases.inference_interval_minutes`pub inference_interval_minutes: i32,/// 上次推完的时间。**答的是「上次看过没有」,不是「上次改过没有」**diff --git a/crates/utopia-extract/Cargo.toml b/crates/utopia-extract/Cargo.tomlindex 5a0840ce..ee407ae0 100644--- a/crates/utopia-extract/Cargo.toml+++ b/crates/utopia-extract/Cargo.toml@@ -11,4 +11,3 @@ anyhow.workspace = truechrono.workspace = trueutopia-llm.workspace = truetracing.workspace = true-unicode-segmentation.workspace = truediff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rsindex ad441573..58fe1585 100644--- a/crates/utopia-extract/src/lib.rs+++ b/crates/utopia-extract/src/lib.rs@@ -1,5 +1,9 @@-//! utopia-extract: LLM 抽取(实体/关系/时间归一化)。-//! 提示词注入本体类型与文档元时间;输出严格 JSON;证据引句强制(无引句降置信度)。+//! utopia-extract:抽取用的提示词与解析。+//!+//! 开放抽取([`open`])是唯一的抽取契约(0044 决定 2):模型用文档自己的话写陈述,+//! 不读本体、不算日期。这里留的是它借用的零件(已知实体的句柄、文件开头的预算、+//! JSON 取块与补括号)、实体消解的攒批裁决,以及量与时间的读法——本体属性的采纳、+//! 工具参数、连接器都用它们。use chrono::{DateTime, NaiveDate, TimeZone, Utc};use serde::Deserialize;@@ -8,122 +12,6 @@ use utopia_llm::ChatMessage;pub mod governor;pub mod open;-pub mod normalize;-pub use normalize::{drop_quotes_from_opening, normalize_facts, Normalization};--#[derive(Debug, Deserialize)]-pub struct Extraction {- #[serde(default)]- pub entities: Vec<ExtractedEntity>,- #[serde(default)]- pub facts: Vec<ExtractedFact>,- /// 实体在这段文字里的**别的名字**(0041 决定 2):简称、曾用名、另一种文字的写法。- /// 模型报,服务端只核对名字与引文确实在原文里——认不认「简称」「又名」这些词是模型的事- #[serde(default)]- pub names: Vec<ExtractedName>,- /// 逐项解析时被跳过的条目数。**必须报给调用方**——不报就是一次静默丢弃,- /// 与 #108「部分抽取报告成完成」同一类错- #[serde(skip)]- pub skipped_entities: usize,- #[serde(skip)]- pub skipped_facts: usize,- /// 模型的输出被截断,这里是修补后解析的- #[serde(skip)]- pub truncated: bool,-}--#[derive(Debug, Deserialize)]-pub struct ExtractedEntity {- /// Identifier scoped to this extraction response. It preserves mention identity while- /// facts are bound; it is not a persistent entity id or a resolution verdict.- #[serde(default)]- pub local_id: Option<String>,- pub name: String,- #[serde(rename = "type")]- pub type_key: String,- /// 模型自己的说法:它认为这最具体是个什么。**不校验、不入本体**。- ///- /// 存在的理由是清单里总有个"差不多"的:本体有 product,模型觉得够用就选了,- /// 心里那个"向量数据库软件"就此丢失。实测 17 个实体的 proposed_type- /// 全是空的,正是这个原因——而事后消解最需要的恰是这个名字:- /// 短名字对短标签,比拿一段中文散文去匹配 "A software application." 近得多。- #[serde(default)]- pub specific_type: Option<String>,-}--/// 一个实体的一个别的名字。`ref` 是这次回复里的 local_id,或者提示词给的 k 句柄-#[derive(Debug, Deserialize)]-pub struct ExtractedName {- #[serde(rename = "ref")]- pub entity_ref: String,- pub name: String,- /// 名字出现在里面的那段原文,逐字抄- #[serde(default)]- pub quote: Option<String>,-}--#[derive(Debug, Deserialize)]-pub struct ExtractedFact {- pub subject: String,- /// Response-local entity handle. When present, callers must bind through it rather than- /// guessing from the surface name.- #[serde(default)]- pub subject_ref: Option<String>,- pub predicate: String,- /// 关系事实的宾语实体名;属性事实为空- #[serde(default)]- pub object: Option<String>,- #[serde(default)]- pub object_ref: Option<String>,- /// 属性事实的字面值(谓词是 attribute 时)- #[serde(default)]- pub value: Option<serde_json::Value>,- /// **边上的属性**(0037):`{"amount": "$5 billion", "stake": "20%"}`。- /// 只对关系事实有意义,key 必须是清单里这条关系声明过的;值照原文写,换算在服务端- #[serde(default)]- pub qualifiers: Option<serde_json::Map<String, serde_json::Value>>,- #[serde(default)]- pub valid_from: Option<String>,- #[serde(default)]- pub valid_to: Option<String>,- #[serde(default)]- pub confidence: Option<f32>,- #[serde(default)]- pub quote: Option<String>,- /// 引文里逐字点名主语的那几个字(#582)。模型抄,不判断;落库时机器核对它- /// 是不是 `subject` 那个名字——"Former OpenAI personnel" 不是 OpenAI- #[serde(default)]- pub subject_span: Option<String>,- /// 同上,宾语那一侧- #[serde(default)]- pub object_span: Option<String>,- /// 日期属性的值只相对一件事给出(「触发日后 45 天」),没有日历上的日期(#681 §4)。- /// 模型判断、模型标;服务端不认这类说法的词,只看这个标记决定收不收- #[serde(default)]- pub relative: bool,-}--/// 提示词里的一条关系。-///-/// 比类多一样东西:**类型签名**。它是给模型的签名,不是闸门——在模型落笔那一刻-/// 减少"Alice works_at 西雅图",而不是等错了再拦。事后校验面对的是既成事实-///(丢掉可惜、留着是脏数据),签名是在写出来之前掰正。本体写错时模型看到原文-/// 说了别的仍可覆盖;硬闸门会系统性丢数据,`part_of` 烧我们的正是那种方式。-pub struct PromptRelation {- pub key: String,- pub label: String,- pub description: String,- /// 形如 `person|organization → vendor`,`*` 表示那一侧不限。空串 = 两侧都不限。- /// **一律用 key**:模型要输出的就是 key,中文库里 person 的 label 是"人物",- /// 写进签名等于教它输出一个不存在的类型(docs/decisions/0004)- pub signature: String,- /// 时间语义(`relation_types.temporal`):`state` / `event` / `eternal`(0031)。- /// 只有 event 与 eternal 会在清单里带标记——状态是默认,写出来只多花 token- pub temporal: String,- /// 这条关系的边能带的属性,已排好版:`amount: number $`(0037)。空 = 不带- pub qualifiers: Vec<String>,-}-/// Response-scoped reference to a persistent entity; database UUIDs must never enter prompts.pub struct KnownEntity {pub handle: String,@@ -131,322 +19,9 @@ pub struct KnownEntity {pub name: String,}-/// 构造抽取提示词。`types` 为 (key, label, description) 三元组;-/// description 非空时按行列出——本体里的语义指引直接决定抽取质量。-/// `attributes` 为调用方预排版的属性行("person.salary (number, CNY): 月薪");-/// 为空时提示词一字不变——没定义属性的库零成本。-pub fn build_messages(- types: &[(String, String, String)],- relations: &[PromptRelation],- attributes: &[String],- doc_time: Option<&str>,- filename: &str,- // 本文档前面几块已经认下的实体,按首次出现排序。handle 只对这次回复有效。- // 第一块为空——那时还没有"前面"- known: &[KnownEntity],- chunk_text: &str,-) -> Vec<ChatMessage> {- build_messages_with_opening(- types, relations, attributes, doc_time, filename, known, None, chunk_text,- )-}-/// 文件开头进提示词的字符预算。一份补充协议的标题、生效日、当事方和「修订的是哪份/// 协议」通常在头一千字符里;新闻稿的电头与导语也是-pub const OPENING_BUDGET_CHARS: usize = 1500;--/// 同 [`build_messages`],另带**这份文件的开头**(第一块的原文),给第二块往后用。-///-/// **一块是孤立抽取的,它看不见自己属于什么。** 补充协议把截止日写在第三块的表格里,-/// 那一块只说「Article 13 的日期延至……」:改的是哪份租约、从哪天起改,都写在第一块。-/// 模型拿不到,就只能把「Phase 2 Exercise Deadline」本身当主语(服务端按主语未声明丢掉),-/// 或者抽出一个没有起点的日期(时态引擎没法据此关闭旧值)——Blackbaud 总部租约链-/// 上五次改期丢了两次,抽到的三次一次都没关上旧值。开头只作背景,不从里面抽事实:-/// 它自己那一块会抽,重复抽只会多出重复的事实-#[allow(clippy::too_many_arguments)]-pub fn build_messages_with_opening(- types: &[(String, String, String)],- relations: &[PromptRelation],- attributes: &[String],- doc_time: Option<&str>,- filename: &str,- known: &[KnownEntity],- opening: Option<&str>,- chunk_text: &str,-) -> Vec<ChatMessage> {- // **有描述时不送 label**。label 是给人看的显示名,而且它跟界面无关、- // 跟这个库的语料语言走——中文库里 person 的 label 是"人物"。- // `- person (人物): 有名有姓的具体的人…` 里那个"人物"相对 key 近乎零信息量,- // 却让提示词在语料语言与标识符之间来回跳。描述为空时才拿它兜底:- // 光一个 key 太单薄。见 docs/decisions/0004- let fmt_list = |items: &[(String, String, String)]| {- items- .iter()- .map(|(k, l, d)| {- let d = d.trim();- if d.is_empty() {- format!("- {k} ({l})")- } else {- format!("- {k}: {d}")- }- })- .collect::<Vec<_>>()- .join("\n")- };- let type_list = fmt_list(types);- // 关系行:有签名时括号里放签名,没有才退回 label。- // `- works_at (person → organization): 一个人受雇于某个组织。`- let rel_list = relations- .iter()- .map(|r| {- let d = r.description.trim();- let paren = if !r.signature.is_empty() {- r.signature.clone()- } else if d.is_empty() {- r.label.clone()- } else {- String::new()- };- // 事件与恒常带方括号标记;状态是默认,不标(0031)- let mark = temporal_mark(&r.temporal)- .map(|m| format!(" [{m}]"))- .unwrap_or_default();- // 边上能带的属性跟在标记后面:`{amount: number $, stake: number %}`- let mark = if r.qualifiers.is_empty() {- mark- } else {- format!("{mark} {{{}}}", r.qualifiers.join(", "))- };- match (paren.is_empty(), d.is_empty()) {- (false, false) => format!("- {} ({paren}){mark}: {d}", r.key),- (false, true) => format!("- {} ({paren}){mark}", r.key),- (true, false) => format!("- {}{mark}: {d}", r.key),- (true, true) => format!("- {}{mark}", r.key),- }- })- .collect::<Vec<_>>()- .join("\n");- // 标记也只在真有事件或恒常关系时解释一次;全是状态的库,提示词一字不变。- // 说的是**写什么**而不是「它是什么」:事件的那一刻进 valid_from、valid_to 留空- // ——不然模型照状态的样子填一个起点,账本就把一次收购读成从那天起一直持续- let temporal_note = if relations- .iter()- .any(|r| temporal_mark(&r.temporal).is_some())- {- "\n 3b. A relation marked [event] happens at one moment: put the date it happened in \- valid_from and leave valid_to null — it has no span and does not end. A relation \- marked [eternal] holds regardless of time: leave both dates null."- } else {- ""- };- // 记号只在真有签名时解释一次;没有签名的库,提示词一字不变。- // 说明用英文——提示词的**指令语言**是英文,只有 description 跟语料走- //- // **签名管两件事,而它们的可覆盖性不同。** 第一版把两件事混成了一句- // "It is a hint, not a rule — when the text says otherwise, write what the- // text says",于是模型连参数顺序也一并按原文的说法写:- //- // Elon Musk (person) --employee--> Microsoft- //- // 而 schema.org 声明的是 employee (organization → person)。实测一次跑里- // 130 条可校验的事实有 102 条这样反着落库——**恰恰是本体包最主要的卖点失效**,- // 选 schema.org 的理由就是「方向是声明的不是描述的」。- //- // 两件事分开说:- //- // - **哪些类型能参与**:提示不是闸门。本体可能写错,原文说西雅图就写西雅图。- // 0001 的判断在这里不变——硬闸门会系统性丢数据,part_of 烧我们的正是那样。- // - **参数顺序**:由签名定。顺序不是关于世界的断言,是这个 key 的编码约定;- // 原文从来没有「说了别的方向」,它只说两个实体之间存在某种关系。- // 反着说时该交换主宾,而不是反过来用这个关系。- let sig_note = if relations.iter().any(|r| !r.signature.is_empty()) {- ". A parenthesis after the key is the type signature, subject then object; \- \"|\" means or, \"*\" means unconstrained. Which kinds of things may take part \- is a hint, not a rule — when the text says otherwise, write what the text says. \- The order is not a hint: the signature fixes which side is the subject. If the \- text puts them the other way round, swap subject and object so that the subject \- matches the left side — do not reverse the relation. For example, given \- \"employee (organization → person)\" and a text saying \"X is an employee of Y\", \- write Y as the subject and X as the object"- } else {- ""- };- let time_ctx = doc_time- .map(|t| {- format!(- "Document date: {t}. Resolve relative time expressions (e.g. \"last year\", \- \"this March\") to absolute dates using it as the reference."- )- })- .unwrap_or_else(|| {- "Document date unknown — only output dates explicitly written in the text.".into()- });-- // 属性段按需注入:清单 + 输出说明 + 取值规则。没定义属性时完全不出现- let attr_section = if attributes.is_empty() {- String::new()- } else {- format!(- "\nAttributes (literal-valued fields, listed as class.attribute_key; as \"predicate\" \- use the attribute_key alone — e.g. \"salary\", not \"person.salary\" — with a \- \"value\" instead of \"object\"):\n{}\n",- attributes.join("\n")- )- };- let attr_rules = if attributes.is_empty() {- String::new()- } else {- "\n11. Attribute facts carry \"value\" (no \"object\"): number = the figure **as the text writes it, magnitude and currency included** \n (\"86亿元\", \"$5 billion\", \"4,300 人\") — never reduce it to a bare number, the server converts; date = \"YYYY[-MM[-DD]]\" (a zoned clock time only when the text gives one) — a date the text gives only relative to an event \- (\"45 days after the Trigger Date\", \"within 30 days of closing\") has no calendar date to convert: write it as the text writes it and add \"relative\": true; bool = true/false; \- text = a short string. Only attach an attribute to a subject of its listed class. \- valid_from = when this value took effect, if the text or the opening of the document says so. \- A document that changes a value set earlier — amends, extends or replaces it — makes the new value \- hold from the date the change takes effect, which is the document's own effective date unless the text gives another."- .to_string()- };- let system = format!(- "You are a knowledge-graph extraction engine. Extract entities and factual relations \- from the given text. Output exactly one JSON object and nothing else.\n\- \n\- Entity types (prefer these keys):\n{type_list}\n\- \n\- Relation types (prefer these keys){sig_note}:\n{rel_list}\n\- {attr_section}\- \n\- Output format:\n\- {{\"entities\":[{{\"local_id\":\"e1\",\"name\":\"entity name\",\"type\":\"type key\",\"specific_type\":\"what you would call it\"}}],\n\- \"facts\":[{{\"subject\":\"subject entity name\",\"subject_ref\":\"e1\",\"subject_span\":\"the words in quote that name the subject\",\"predicate\":\"relation key\",\"object\":\"object entity name\",\"object_ref\":\"e2\",\"object_span\":\"the words in quote that name the object\",\n\- \"valid_from\":\"2023-01\",\"valid_to\":null,\"confidence\":0.9,\"quote\":\"verbatim supporting quote\"}}],\n\- \"names\":[{{\"ref\":\"e1\",\"name\":\"another name the text uses for it\",\"quote\":\"verbatim text containing that name\"}}]}}\n\- \n\- Rules:\n\- 1. Give every newly listed entity a local_id unique within this response (e1, e2, ...). \- A local_id may define at most one entity. Reuse the same local_id when this response \- mentions the same entity again, including abbreviations; never allocate a new handle \- merely because a mention repeats. Different handles mean the mentions should be \- tracked separately for attribution, not that their permanent identity is proven. \- When the text clearly describes two different entities with the same surface name, \- list both under different local_ids. A shared name alone neither proves sameness nor \- requires a split.\n\- 1a. Use the canonical full name as written in the text, in the text's original language; \- list each entity once. Text introduces a full name and then shortens it — \- \"星云科技上海研究院\" becomes \"上海研究院\", \"Nebula Technologies Inc.\" becomes \- \"Nebula\" — and both forms mean one entity, listed once under the fuller form. \- Two names are two entities only when the text is talking about two things. \- A name identifies the thing; it is not a description of its history. When the text \- names something and then describes what happened to it, the name ends where the \- description begins.\n\- 1b. Every other name the text gives an entity goes into \"names\", once per name: the \- shortened form it introduces or uses (\"上海研究院\" for \"星云科技上海研究院\"), a \- former name, the name in another language. \"ref\" is the entity's local_id or its \- known handle, and \"quote\" is a verbatim excerpt that contains the name. Only \- names belong there — never a pronoun or a description (\"该公司\", \"the company\", \- \"former employees\") — and never the name already written in entities. A name \- must name the entity itself, not something that belongs to it: \"星云科技研发团队\" \- names a team, not 星云科技.\n\- 2. Every fact keeps its name fields and uses subject_ref; relation facts also use \- object_ref. Each ref must be either a local_id defined exactly once in \- entities or a known handle supplied with this text. An entity referenced by a known \- handle must not be copied into entities.\n\- 3. Dates must be \"YYYY\", \"YYYY-MM\", \"YYYY-MM-DD\", or null — never invent dates. \- A clock time is allowed only together with its zone, as \"YYYY-MM-DDTHH:MM[:SS]Z\" \- or with a \"+HH:MM\" offset, and only when the text or the document states that \- zone; a time of day without a zone stays a plain date — never guess a zone.\n\- 3a. valid_to takes a third value: \"unknown\". Use it when the text says the relation \- has ended but does not say when — \"former CEO of X\", \"stepped down\", \"left the \- company\", \"no longer available\", \"until recently\". Use null only for something \- still going on. These are not interchangeable: null asserts it still holds, and \- writing null for a relation the text says is over makes us claim the opposite of \- the source.\n\- 3c. A period is when a fact holds, never what it is about. A quarter, a half, a \- fiscal or calendar year, a month, \"the three months ended July 26, 2026\" — \- none of these is an entity and none is an object. Put the period's dates in \- valid_from and valid_to (a fiscal period resolves to the dates the document \- states for it) and write the figure as the fact's \"value\" — the figure alone, as it stands in the \- quote, with nothing appended. A column of a table headed by a period is a column \- of values that hold in that period.\n\- {temporal_note}\n\- 4. {time_ctx}\n\- 5. quote must be a contiguous excerpt from the Text block; never quote the opening of the document. Every fact needs one.\n\- 6. confidence in 0~1: 0.9 explicitly stated, 0.7 inferred, 0.5 uncertain. A value \- the text writes out is stated whatever the layout — a sentence, a list, a table \- cell, a schedule, the new column of an amendment that replaces an earlier term. \- Inferred means the text does not write the value and you worked it out.\n\- 7. If nothing can be extracted, output {{\"entities\":[],\"facts\":[]}}.\n\- 8. If no listed relation fits, do not force the nearest one — write the predicate the \- text itself uses, in snake_case (e.g. \"available_on\", \"runs_on\"). A relation \- named after the text is worth more than a listed one that says something false.\n\- 8a. The same holds for a literal the text states outright — an amount, a share count, \- a percentage, a capacity, a date, a job title, a ticker. Write it as a fact with \- \"value\" and no \"object\": {{\"subject\":\"NVIDIA\",\"subject_ref\":\"e1\",\- \"predicate\":\"purchase_price\",\"value\":\"$11.9 billion\",\"confidence\":0.9,\- \"quote\":\"...\"}}. Name the predicate after the text when no listed attribute \- fits — \"purchase_price\", \"job_title\", \"generation_capacity\", \"record_date\". \- Attach it to the entity the text attaches it to, and keep the literal as written, \- units and all — except a date, which is always written in the format of rule 3 \- (\"June 23, 2020\" is \"2020-06-23\"). A deadline or a period stated \- relative to an event, with no calendar date, is not a date: keep it as written \- and mark it \"relative\" as rule 11 says. \- **A stated figure left out is the loss that costs most**: the reader \- came for those numbers, and no later step can recover one that was never written \- down.\n\- 8b. A listed relation followed by {{…}} can carry those **qualifiers on the edge**: when the same sentence gives both the other entity and a figure for it — an amount, a stake, a price, a share count — write the relation with its \"object\" and put the figure in \"qualifiers\" keyed exactly as listed, **as written in the text, currency and all** (\"€30 million\", \"15亿元人民币\", never a bare number) — except a date, which takes the format of rule 3: {{\"subject\":\"Vega Capital\",\"predicate\":\"invested_in\",\"object\":\"Northwind\", \"qualifiers\":{{\"amount\":\"$5 billion\"}},…}}. Never invent a key that is not listed for that relation, and never drop the figure to keep the edge — a relation without its amount is half the sentence. A relation you name after the text (rule 8) carries its figure the same way — keyed by the listed attribute that fits it, or by the plainest word for it (\"amount\", \"stake\", \"price\") when none does.- 8c. A **listed** relation also takes \"value\" when what the text gives is a \- string rather than another entity — a job title, a designation, a ticker, a \- model number. Never invent an entity for a string. And when the text introduces \- someone by their role — \"X, founder and CEO of Y\", \"Z, co-CEO of W\", \- \"Y's vice president of research\", \"the president of OpenAI\", \- \"chief executive of Quora\", \"OpenAI's chief technology officer of \- applications\" — write both facts: the tie to the organization, and \- the role itself as a value on the person. The tie alone says they \- are connected; the role is what the sentence was actually telling \- you. The possessive (\"Y's <role>\", \"<role> of Y\", \"<role> at Y\"), the past \- tense (\"was Y's <role>\", \"former <role> of Y\"), and the implied form \- (\"appointed … as OpenAI's CTO of applications\") all carry the same \- shape — the role is the value, the organization is the other \- entity. Past tense and \"former\" give the tie valid_to: \"unknown\".\n\- 8d. A list of named parties is a list of facts — one per name. \"partners \- including A, B, C and D\" is four facts, not one; \"advisors A and B\" is two. \- Do not collapse an enumeration into a summary or into its first member. \- The same applies to the entities: each named party is its own entity.\n\- 8e. subject_span and object_span are the exact words in quote that name each side. \- Copy them; never paraphrase. When the words that do the thing are a description \- rather than a name — \"former X employees\", \"companies using X\" — the span \- is that description, whatever you wrote in subject.\n\- 8f. An obligation, a deadline or a right belongs to the agreement, law or decision \- that imposes it, even when it concerns another agreement or thing. A lease that \- sets the last day to sign a second lease gives that deadline to the first lease; \- the second lease is only what the deadline is about.\n\- 9. The same holds for entity types: if none of the listed types fits, write the type \- the text implies, in snake_case (e.g. \"model\", \"technology\"). Do not fall back \- to a broad listed type such as \"thing\" or \"creative_work\" merely because \- nothing specific matched — that hides the gap instead of reporting it.\n\- 10. specific_type is required on every entity and is never checked against the list. \- Name the most specific kind the thing is, in the words you would use for it. Write \- it even when \"type\" already fits, and make it narrower than \"type\" wherever the \- text supports it — type \"product\", specific_type \"vector database software\". \- Repeat the listed type only when the text genuinely says nothing more precise.\- {attr_rules}"- );-- // 已知实体紧挨着正文:服从性靠位置,理由见 known_block 的注释- let user = format!(- "Source file: \"{filename}\"\n{}{}\nText:\n{chunk_text}",- opening_block(opening),- known_block(known)- );-- vec![- ChatMessage {- role: "system".into(),- content: system,- },- ChatMessage {- role: "user".into(),- content: user,- },- ]-}+pub(crate) const OPENING_BUDGET_CHARS: usize = 1500;/// 文件开头排版成提示词里的一段。开头为空(或只有空白)时返回空串;/// 「这一块就是开头本身」由调用方判断,那时它传 `None`。@@ -471,75 +46,12 @@ pub(crate) fn opening_block(opening: Option<&str>) -> String {)}-/// 一段描述的第一句(句子边界按 UAX #29)。按块检索出的清单只带这一句:schema.org 的-/// 描述后半截多是用法说明与示例,一块铺上百行时它们占了清单的八成-pub fn first_sentence(text: &str) -> &str {- use unicode_segmentation::UnicodeSegmentation;- text.trim().unicode_sentences().next().map_or("", str::trim)-}--/// 清单里给关系带的标记:事件 `[event]`、恒常 `[eternal]`;状态不标。-/// 认不出的值当状态——数据库的 CHECK 只放这三个进来,这里不再报错-fn temporal_mark(temporal: &str) -> Option<&'static str> {- match temporal {- "event" => Some("event"),- "eternal" => Some("eternal"),- _ => None,- }-}-/// 已在本文档中出现过的实体,放进提示词的字符预算。////// 超出就截断(保留先出现的)。中文商业文本先出全称、主角先出场,所以/// **首次出现顺序天然偏向那些后面会被简称的名字**。pub(crate) const KNOWN_BUDGET_CHARS: usize = 1200;-/// 把「本文档已经认下的实体」排版成提示词里的一段。空则返回空串。-///-/// **为什么在正文之前、指令贴着清单**:抽象规则打不过挨着它的具体块——本体建议-/// 那次,语言要求就输给了紧随其后的英文 JSON 骨架,挪到骨架之后并点名它才生效。-/// 服从性靠位置,所以指令挨着它管的数据放,两者一起挨着正文。-///-/// **顺带一条与放哪条消息无关的规矩:逐块变化的内容一律放最后。** 前缀缓存匹配的是-/// token 前缀,而消息按 system→user 拼接,所以「system 末尾」与「user 开头」几乎等价;-/// 真正会打碎缓存的是把它塞在**中间**(本体之后、规则之前),那会把规则挤出前缀。-/// 缓存本身不归我们管——供应商开不开、报不报都由它,本部署实测 `cached=0`——-/// 我们只负责别把它弄碎。自部署 vLLM 默认开着自动前缀缓存,那省的是算力不是钱。-fn known_block(known: &[KnownEntity]) -> String {- if known.is_empty() {- return String::new();- }- let mut lines = Vec::new();- let mut used = 0usize;- for entity in known {- used += entity.handle.chars().count()- + entity.type_key.chars().count()- + entity.name.chars().count()- + 6;- if used > KNOWN_BUDGET_CHARS {- break;- }- lines.push(format!(- " {} [{}]: {}",- entity.handle, entity.type_key, entity.name- ));- }- if lines.is_empty() {- return String::new();- }- let lines = lines.join("\n");- format!(- "\nAlready recorded from earlier parts of this same document:\n{lines}\n\- \n\- If something in the text below refers to one of these, use its k-handle in the fact's \- subject_ref/object_ref, write that exact string as the name, and give it that same \- type — documents abbreviate after first mention \- (\"星云科技上海研究院\" later becomes \"上海研究院\"), and the shortened form must \- not become a second entity. If it is a different thing, name it as the text does; \- do not force it onto this list.\n"- )-}-/// 回复里可能是 JSON 的那段文字:切掉思考过程与代码围栏。前后的废话留给调用方按/// 括号定位——[`json_block`] 取第一个 `{` 到最后一个 `}`;开放抽取的截断修补则从/// 第一个 `{` 取到结尾,那边的记录是数组,最后一个 `}` 不是可靠的结尾@@ -611,102 +123,6 @@ pub(crate) fn close_brackets(head: &str) -> Option<String> {Some(out)}-/// 输出被截断时,退到**最后一个完整对象**的结尾再把括号补齐。-///-/// 模型写到一半没了(撞上 max_tokens)时,前面那些对象是完整且正确的。-/// 整块作废等于把已经抽对的十几条事实一起扔掉——实测 246 次调用里 4 次是这种。-fn repair_truncated(json: &str) -> Option<String> {- let mut cut = json.len();- for _ in 0..64 {- let idx = json[..cut].rfind('}')?;- if let Some(closed) = close_brackets(&json[..=idx]) {- if serde_json::from_str::<serde_json::Value>(&closed).is_ok() {- return Some(closed);- }- }- cut = idx;- }- None-}--/// **一条坏记录不该毁掉一整块。**-///-/// 从前这里是 `serde_json::from_str::<Extraction>`——全有或全无。一个缺 `predicate`-/// 的对象、或者一次输出截断,整块的实体和事实一起作废,而一块里常有二十条好事实。-/// 实测 246 次调用里 5 次这样丢掉(2%),并且会让整个 `extract_document` 任务失败、-/// 走重试,三次之后文档标记失败。-///-/// 现在:先解成 `Value`(截断就先补齐括号),再逐项 `from_value`,好的收下、-/// 坏的计数。**计数必须往外传**——静默跳过就是另一种"报告成完成"。-pub fn parse_response(raw: &str) -> anyhow::Result<Extraction> {- let json_str = json_block(raw)?;- let (value, truncated) = match serde_json::from_str::<serde_json::Value>(&json_str) {- Ok(v) => (v, false),- Err(e) => match repair_truncated(&json_str) {- Some(fixed) => (- serde_json::from_str::<serde_json::Value>(&fixed)- .map_err(|e| anyhow::anyhow!("Failed to parse extraction JSON: {e}"))?,- true,- ),- // 补不回来才是真解析失败:连一个完整对象都没有- None => anyhow::bail!("Failed to parse extraction JSON: {e}"),- },- };-- fn take<T: serde::de::DeserializeOwned>(- value: &serde_json::Value,- key: &str,- ) -> (Vec<T>, usize) {- let Some(arr) = value.get(key).and_then(|v| v.as_array()) else {- return (Vec::new(), 0);- };- let mut out = Vec::with_capacity(arr.len());- let mut skipped = 0;- for item in arr {- match serde_json::from_value::<T>(item.clone()) {- Ok(v) => out.push(v),- Err(_) => skipped += 1,- }- }- (out, skipped)- }-- let (mut entities, mut skipped_entities) = take::<ExtractedEntity>(&value, "entities");- let (facts, skipped_facts) = take::<ExtractedFact>(&value, "facts");- // 名字条目坏了不算实体或事实被跳过:丢一个名字只是少一座桥,不丢断言- let (names, _) = take::<ExtractedName>(&value, "names");-- // A handle identifies exactly one entity definition within one response. Reject every- // definition participating in a duplicate (including identical duplicates): keeping the- // first or last would make fact attribution depend on array order. Empty handles are- // malformed too; legacy output is represented by an absent field, not an empty id.- let mut handle_counts = std::collections::HashMap::<String, usize>::new();- for entity in &entities {- if let Some(handle) = entity.local_id.as_deref() {- *handle_counts.entry(handle.trim().to_string()).or_default() += 1;- }- }- entities.retain(|entity| match entity.local_id.as_deref() {- None => true,- Some(handle) => {- let handle = handle.trim();- let valid = !handle.is_empty() && handle_counts.get(handle) == Some(&1);- if !valid {- skipped_entities += 1;- }- valid- }- });- Ok(Extraction {- entities,- facts,- names,- skipped_entities,- skipped_facts,- truncated,- })-}-// ---------------------------------------------------------------------------// 实体消解裁决(攒批:一次调用裁多对,LLM 只处理 embedding 分不出的灰区)// ---------------------------------------------------------------------------@@ -934,7 +350,7 @@ pub fn parse_leading_quantity(s: &str) -> Option<(f64, Option<String>)> {}/// 货币:符号、ISO 码、中英文单词,统一成符号。**只认这张表**,认不出的不猜。-pub fn currency_unit(tok: &str) -> Option<&'static str> {+pub(crate) fn currency_unit(tok: &str) -> Option<&'static str> {Some(match tok.trim_matches(|c: char| c == ',' || c == '.' || c == ';') {"$" | "USD" | "usd" | "US$" | "dollar" | "dollars" | "美元" => "$",@@ -1082,25 +498,6 @@ fn next_token(s: &str) -> (&str, &str) {(&s[..end], &s[end..])}-/// 一个属性值落库时的样子:按 datatype 归一成 `{"value": …}`;失败返回 None,调用方记-/// `attr_datatype`。-///-/// 日期属性上一个**相对**的值(#681 §4):解不成日期、模型又标了 `relative`、写的是一段非空-/// 文字时,照原文收下,值里带 `"relative": true`。它不是日期,从不当日期比较或排序。解得成-/// 日期的照日期存,标错了也不当相对;没标的非日期值仍然不收-pub fn attr_object_value(- datatype: &str,- raw: &serde_json::Value,- relative: bool,-) -> Option<serde_json::Value> {- if let Some(value) = normalize_attr_value(datatype, raw) {- return Some(serde_json::json!({ "value": value }));- }- let written = raw.as_str().map(str::trim).filter(|s| !s.is_empty())?;- (datatype == "date" && relative)- .then(|| serde_json::json!({ "value": written, "relative": true }))-}-/// 属性值按 datatype 归一。失败返回 None——宁缺勿脏,调用方跳过并记日志。/// number 容忍千分位/空格;date 收规则 3 的格式(YYYY[-MM[-DD]]、带时区的时刻,原样保留),/// 也收写法说得清是哪天的日期([`written_date`]),换成规则 3 的样子;bool 宽容 yes/no。@@ -1321,271 +718,32 @@ fn split_zone(clock: &str) -> Option<(&str, chrono::Duration)> {}#[cfg(test)]-mod prompt_shape_tests {+mod tests {use super::*;- /// 第十份补充协议把截止日改成「触发日后 45 天」:模型标 relative,服务端照原文收;- /// 没标的、不是日期属性的、空的都不走这条+ /// #690:思考过程里的大括号不能把 JSON 的起止带偏。不切掉标记,"第一个 `{`" 的起点+ /// 会提前到思考过程里;代码围栏与前后的废话同样不算数#[test]- fn a_relative_date_is_kept_as_written_only_when_marked() {- let raw = serde_json::json!("45 days after the Trigger Date");- assert_eq!(- attr_object_value("date", &raw, true),- Some(- serde_json::json!({ "value": "45 days after the Trigger Date", "relative": true })- )- );- assert_eq!(attr_object_value("date", &raw, false), None, "没标就不收");- assert_eq!(- attr_object_value("date", &serde_json::json!(" "), true),- None- );- // 解得成日期的照日期存,标了 relative 也不当相对+ fn json_block_ignores_a_think_block_and_a_fence_before_the_json() {+ let raw = "先想想 {\"a\": 1,再回答。\n</think>{\"entities\":[{\"name\":\"张三\"}]}";assert_eq!(- attr_object_value("date", &serde_json::json!("2020-06-23"), true),- Some(serde_json::json!({ "value": "2020-06-23" }))+ json_block(raw).unwrap(),+ "{\"entities\":[{\"name\":\"张三\"}]}");- // 不是日期属性:只按它自己的 datatype 归一,relative 不起作用- assert_eq!(- attr_object_value("bool", &serde_json::json!("45 days after"), true),- None- );- assert_eq!(- attr_object_value("number", &serde_json::json!("1,250"), true),- Some(serde_json::json!({ "value": 1250.0 }))- );- let fact: ExtractedFact = serde_json::from_value(serde_json::json!({- "subject": "Lease", "predicate": "expansion_option_deadline",- "value": "45 days after the Trigger Date", "relative": true- }))- .unwrap();- assert!(fact.relative);- let plain: ExtractedFact = serde_json::from_value(serde_json::json!({- "subject": "Lease", "predicate": "expansion_option_deadline", "value": "2020-06-23"- }))- .unwrap();- assert!(!plain.relative, "没写就不是");- let msgs = build_messages(- &[],- &[],- &["lease.deadline (date)".into()],- None,- "a.txt",- &[],- "text",- );- assert!(msgs[0].content.contains("add \"relative\": true"));- }-- fn rel(key: &str, description: &str, signature: &str) -> PromptRelation {- PromptRelation {- key: key.into(),- label: key.replace('_', " "),- description: description.into(),- signature: signature.into(),- temporal: "state".into(),- qualifiers: vec![],- }- }-- fn timed(key: &str, description: &str, temporal: &str) -> PromptRelation {- PromptRelation {- temporal: temporal.into(),- ..rel(key, description, "")- }- }-- /// 事件与恒常在清单里带标记,说明只出现一次(0031)- #[test]- fn an_event_and_an_eternal_relation_are_marked() {- let rels = vec![- rel("works_at", "受雇于某个组织。", "person → organization"),- timed("acquired", "One company buys another.", "event"),- timed("capital_of", "", "eternal"),- ];- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");- let s = &msgs[0].content;- assert!(s.contains("- works_at (person → organization): 受雇于某个组织。"));- assert!(s.contains("- acquired [event]: One company buys another."));- // 没有描述时括号里是 label,标记跟在括号后面- assert!(s.contains("- capital_of (capital of) [eternal]"));- assert!(s.contains("A relation marked [event] happens at one moment"));- assert!(s.contains("leave valid_to null"));+ let fenced = "好的,结果如下:\n```json\n{\"entities\":[]}\n```";+ assert_eq!(json_block(fenced).unwrap(), "{\"entities\":[]}");+ assert!(json_block("no json here").is_err());}- /// **全是状态的库,提示词一字不变**:不标、不解释+ /// 括号出现在字符串里不算结构——`"a[b"` 不是一个开括号;断在字符串中间的那一截不可用#[test]- fn a_base_of_states_pays_nothing_for_the_marks() {- let rels = vec![rel("works_at", "d", "")];- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");- let s = &msgs[0].content;- assert!(!s.contains("[event]"));- assert!(!s.contains("[eternal]"));- assert!(!s.contains("happens at one moment"));- }-- /// 签名进括号,而且**一律是 key**:中文库的 label 是"人物",- /// 写进提示词等于教模型输出一个不存在的类型。- #[test]- fn a_signature_takes_the_parenthesis_and_uses_keys() {- let rels = vec![rel("works_at", "受雇于某个组织。", "person → organization")];- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");- assert!(msgs[0]- .content- .contains("- works_at (person → organization): 受雇于某个组织。"));- }-- /// 多值用 `|`,空的一侧用 `*` —— 都是 key 层面的记号,不是类型名- #[test]- fn several_classes_join_with_a_pipe_and_an_empty_side_is_a_star() {- let rels = vec![rel("buys_from", "", "employee|team → *")];- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");- assert!(msgs[0].content.contains("- buys_from (employee|team → *)"));- }-- /// **没有签名的库,提示词一字不变**:记号说明也不出现。- /// 大多数库不会声明 domain/range,不该为此付每块的 token- #[test]- fn a_base_without_signatures_pays_nothing() {- let rels = vec![rel("works_at", "受雇于某个组织。", "")];- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");- assert!(msgs[0].content.contains("- works_at: 受雇于某个组织。"));- assert!(!msgs[0].content.contains("type signature"));- assert!(!msgs[0].content.contains('→'));- }-- /// 签名是提示不是闸门。这句话必须在提示词里 —— 少了它,- /// 模型会把签名当硬规则,本体写错时就系统性丢数据(part_of 那种方式)- #[test]- fn the_prompt_says_the_signature_is_a_hint() {- let rels = vec![rel("works_at", "d", "person → organization")];- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");- assert!(msgs[0].content.contains("hint, not a rule"));- }-- /// **但顺序不是提示。**- ///- /// 两句话必须同时在场,少哪一句都退回一种老毛病:少了「提示不是闸门」,- /// 本体写错时系统性丢数据(part_of 那种方式);少了「顺序由签名定」,- /// 模型按英语直觉写 `Musk --employee--> Microsoft`,而 schema.org 声明的是- /// `employee (organization → person)`——实测一次跑里 130 条可校验的事实- /// 有 102 条这样反着落库。- #[test]- fn the_prompt_says_the_order_is_not_a_hint() {- let rels = vec![rel("employee", "d", "organization → person")];- let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text");- let c = &msgs[0].content;- assert!(c.contains("hint, not a rule"), "类型那句丢了");- assert!(c.contains("The order is not a hint"), "顺序那句丢了");- assert!(- c.contains("swap subject and object"),- "只说了顺序重要,没说反着写时该怎么办"- );- assert!(- c.contains("do not reverse the relation"),- "少了这句,模型可能去找一个反向关系而不是交换主宾"- );- }-- #[test]- fn an_obligation_belongs_to_the_agreement_that_imposes_it() {- // 主租约里写着「签二期租约的截止日」,模型时而把截止日挂到二期租约上:- // 主租约的时间线上就少了这次改期(#681 §3)- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");- assert!(msgs[0]- .content- .contains("belongs to the agreement, law or decision that imposes it"));- }-- #[test]- fn a_literal_keeps_its_units_but_a_date_takes_the_contract_format() {- // 8a 从前说「字面值按原文写」并把日期列在字面值里,而规则 3 与属性规则要求- // YYYY-MM-DD:两条互相打架,模型写出「June 23, 2020」,服务端按格式不合整条丢掉。- // Blackbaud 总部租约链上各轮累计丢了二十多次- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");- let system = &msgs[0].content;- assert!(system.contains("except a date, which is always written in the format of rule 3"));- }-- /// 补充协议把旧条款与新日期排成一张对照表,模型把表格里读到的新日期标 0.7(当成- /// 推断),低于 0.75 的值不许接替前一个——截止日就一直停在旧值上。规则 6 说清楚:- /// 原文写着的值不论排成什么样都是明写- #[test]- fn a_value_written_in_a_table_is_stated() {- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");- let system = &msgs[0].content;- assert!(system.contains("stated whatever the layout"));- assert!(system.contains("a table cell"));- }-- /// 规则编号各不相同,「按规则 N」指得到唯一的一条。从前有两条 8c、两条 10,- /// 「as rule 10 says」说的是哪条要靠猜(#689 评审)- #[test]- fn every_rule_has_its_own_number_and_every_reference_lands() {- let rels = vec![PromptRelation {- key: "acquired".into(),- label: "acquired".into(),- description: String::new(),- signature: String::new(),- temporal: "event".into(),- qualifiers: vec![],- }];- let attrs = vec!["lease.option_deadline (date)".to_string()];- let msgs = build_messages(&[], &rels, &attrs, None, "a.txt", &[], "text");- let system = &msgs[0].content;- let mut labels = Vec::new();- for line in system.lines() {- let Some((label, _)) = line.trim_start().split_once(". ") else {- continue;- };- let digits = label.trim_end_matches(|c: char| c.is_ascii_lowercase());- if !digits.is_empty()- && digits.chars().all(|c| c.is_ascii_digit())- && label.len() - digits.len() <= 1- {- labels.push(label.to_string());- }- }- let unique: std::collections::BTreeSet<_> = labels.iter().collect();- assert_eq!(unique.len(), labels.len(), "规则编号重复:{labels:?}");- for (i, _) in system.match_indices("rule ") {- let n: String = system[i + 5..]- .chars()- .take_while(|c| c.is_ascii_alphanumeric())- .collect();- assert!(- labels.contains(&n),- "「rule {n}」指不到任何一条:{labels:?}"- );- }- }-- #[test]- fn a_later_chunk_reads_the_opening_of_its_document() {- let opening = "FIFTH AMENDMENT TO LEASE AGREEMENT entered into as of February 18, 2020";- let msgs = build_messages_with_opening(- &[],- &[],- &[],- None,- "a.html",- &[],- Some(opening),- "The Existing Dates are extended to March 17, 2020.",- );- let user = &msgs[1].content;- let at_opening = user.find(opening).expect("opening is in the user message");- let at_text = user- .find("The Existing Dates")- .expect("text is in the user message");- assert!(- at_opening < at_text,- "the opening comes before the text it frames"+ fn brackets_inside_strings_are_not_structure() {+ assert_eq!(+ close_brackets(r#"{"s": [{"a": "a[b{c"}"#).as_deref(),+ Some(r#"{"s": [{"a": "a[b{c"}]}"#));- // 没有开头时,提示词与从前一字不差- let plain = build_messages(&[], &[], &[], None, "a.html", &[], "t");- let framed = build_messages_with_opening(&[], &[], &[], None, "a.html", &[], None, "t");- assert_eq!(plain[1].content, framed[1].content);+ assert!(close_brackets(r#"{"s": "cut in the mid"#).is_none());+ assert!(close_brackets("]").is_none());}#[test]@@ -1597,136 +755,6 @@ mod prompt_shape_tests {assert_eq!(opening_block(Some(" ")), "");}- /// 已知实体必须落在 **user** 消息里、紧挨着正文。- ///- /// 理由是服从性不是缓存:抽象规则打不过挨着它的具体块。清单放进 system 的- /// 规则区,就会隔着输出格式、十条规则、文件名,离它要管的正文最远。- #[test]- fn known_entities_stay_out_of_the_system_message() {- // 用一个规则 1 的例子里没有的名字:规则 1 也提"星云科技上海研究院",- // 拿它断言等于测不出清单到底在哪条消息里- let known = vec![KnownEntity {- handle: "k1".into(),- type_key: "organization".into(),- name: "华瑞集团智能制造研究院".into(),- }];- let msgs = build_messages(&[], &[], &[], None, "a.txt", &known, "text");- assert_eq!(msgs[0].role, "system");- assert!(!msgs[0].content.contains("Already recorded"));- assert!(!msgs[0].content.contains("华瑞集团智能制造研究院"));- assert!(msgs[1]- .content- .contains("k1 [organization]: 华瑞集团智能制造研究院"));- }-- /// 第一块没有"前面",那一段应当完全不出现——成本为零,而不是一段空标题- #[test]- fn the_first_chunk_carries_no_block() {- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");- assert!(!msgs[1].content.contains("Already recorded"));- }-- /// 反向护栏必须在:给了参照物就会有人硬套(`concept` 那次的教训)- #[test]- fn the_block_tells_the_model_not_to_force_a_match() {- let known = vec![KnownEntity {- handle: "k1".into(),- type_key: "person".into(),- name: "陈立".into(),- }];- let msgs = build_messages(&[], &[], &[], None, "a.txt", &known, "text");- assert!(msgs[1].content.contains("do not force it onto this list"));- }-- /// 有描述就不送 label——中文库的 label 是中文,混进提示词只会让- /// 标识符与语料语言来回跳,而它相对 key 近乎零信息量。- #[test]- fn described_types_drop_the_label() {- let types = vec![- (- "person".into(),- "人物".into(),- "有名有姓的具体的人。".into(),- ),- ("event".into(), "事件".into(), String::new()),- ];- let msgs = build_messages(&types, &[], &[], None, "a.txt", &[], "text");- let prompt = format!("{:?}", msgs);- assert!(prompt.contains("- person: 有名有姓的具体的人。"));- assert!(!prompt.contains("person (人物)"));- // 描述为空时 label 仍是唯一的额外线索,留着- assert!(prompt.contains("- event (事件)"));- }-}--#[cfg(test)]-mod tests {- use super::*;-- /// 边上的属性(0037):清单里跟在关系后面,回复里挂在事实上。- #[test]- fn a_relation_lists_its_qualifiers_and_a_fact_carries_them() {- use serde_json::json;- let mut r = PromptRelation {- key: "invested_in".into(),- label: "invested in".into(),- description: "money into a company".into(),- signature: "organization → organization".into(),- temporal: "event".into(),- qualifiers: vec!["amount: number $".into(), "stake: number %".into()],- };- let msgs = build_messages(- &[],- std::slice::from_ref(&r),- &[],- None,- "a.txt",- &[],- "text",- );- let prompt = format!("{:?}", msgs);- // 签名、标记、属性清单三段顺序固定:`(签名) [event] {属性}`- assert!(prompt.contains(- "- invested_in (organization → organization) [event] {amount: number $, stake: number %}: money into a company"- ), "{prompt}");- // 不带属性的关系不多一个花括号- r.qualifiers.clear();- let prompt = format!(- "{:?}",- build_messages(- &[],- std::slice::from_ref(&r),- &[],- None,- "a.txt",- &[],- "text"- )- );- assert!(- prompt.contains("- invested_in (organization → organization) [event]: money"),- "{prompt}"- );- assert!(!prompt.contains("[event] {"));-- // 回复:qualifiers 挂在关系事实上;没写的是 None,旧回复不受影响- let reply = r#"{"entities":[],"facts":[- {"subject":"Vega","predicate":"invested_in","object":"Northwind",- "qualifiers":{"amount":"$5 billion"},"confidence":0.9},- {"subject":"Vega","predicate":"invested_in","object":"Kestrel","confidence":0.9}- ]}"#;- let parsed = parse_response(reply).unwrap();- assert_eq!(parsed.facts.len(), 2);- assert_eq!(- parsed.facts[0]- .qualifiers- .as_ref()- .and_then(|q| q.get("amount")),- Some(&json!("$5 billion"))- );- assert!(parsed.facts[1].qualifiers.is_none());- }-#[test]fn a_quantity_is_the_whole_string_or_nothing() {// 整体就是一个量:符号、量级词、千分位都读得动@@ -1851,22 +879,6 @@ mod tests {assert_eq!(normalize_attr_value("number", &json!("about ten")), None);}- #[test]- fn a_description_is_cut_at_its_first_sentence() {- assert_eq!(- first_sentence(- " The date on which the CreativeWork was created. See also dateModified.\n\nExample: 2020-01-01."- ),- "The date on which the CreativeWork was created."- );- assert_eq!(- first_sentence("一个有名有姓的人。可以是虚构的。"),- "一个有名有姓的人。"- );- assert_eq!(first_sentence("A person"), "A person");- assert_eq!(first_sentence(" "), "");- }-#[test]fn parse_time_precisions() {assert_eq!(parse_time("2024").unwrap().1, "year");@@ -1997,211 +1009,6 @@ mod tests {);assert_eq!(normalize_attr_value("text", &json!([1])), None);}-- /// **一条坏记录不该毁掉一整块。**- ///- /// 形态取自真实日志:`missing field \`predicate\``。模型偶尔会漏写这个字段- /// (`related_to` 退场后它没有万能选项可挑),从前 serde 会让整块作废,- /// 而这一块里另外两条事实是好的。- #[test]- fn one_malformed_fact_does_not_take_the_whole_chunk() {- let raw = r#"{- "entities": [{"name": "OpenAI", "type": "organization"}],- "facts": [- {"subject": "OpenAI", "predicate": "produces", "object": "GPT-4"},- {"subject": "OpenAI", "object": "ChatGPT"},- {"subject": "Sam Altman", "predicate": "leads", "object": "OpenAI"}- ]- }"#;- let x = parse_response(raw).unwrap();- assert_eq!(x.facts.len(), 2, "好的两条该留下");- assert_eq!(x.skipped_facts, 1, "跳过的那条要报出来,不能静默");- assert_eq!(x.entities.len(), 1);- assert!(!x.truncated);- }-- /// **输出被截断时,已经完整的那些要救回来。**- ///- /// 撞上 max_tokens 时模型写到一半就没了(真实日志:`EOF while parsing a list`)。- /// 前面的对象是完整且正确的,整块作废等于把抽对的十几条一起扔掉。- #[test]- fn a_cut_off_reply_keeps_what_was_complete() {- let raw = r#"{- "entities": [{"name": "Anthropic", "type": "organization"}],- "facts": [- {"subject": "Anthropic", "predicate": "produces", "object": "Claude"},- {"subject": "Dario Amodei", "predicate": "leads", "object": "Anthropic"},- {"subject": "Anthropic", "predicate": "loca"#;- let x = parse_response(raw).unwrap();- assert!(x.truncated, "截断要标出来");- assert_eq!(x.facts.len(), 2, "断点之前的两条是完整的");- assert_eq!(x.entities.len(), 1);- }-- /// 括号出现在字符串里不算结构——`"a[b"` 不是一个开括号。- #[test]- fn brackets_inside_strings_are_not_structure() {- let raw =- r#"{"entities": [], "facts": [{"subject": "a[b{c", "predicate": "p", "object": "o"}]}"#;- let x = parse_response(raw).unwrap();- assert_eq!(x.facts.len(), 1);- assert!(!x.truncated, "结构完整,不该判成截断");- }-- /// 连一个完整对象都没有时,仍然要报失败——**容错不是把空结果说成成功**。- #[test]- fn a_reply_with_nothing_complete_still_fails() {- assert!(parse_response(r#"{"facts": [{"subject": "a"#).is_err());- }-- /// 片段字段可有可无:老模型输出没有它们,照常解析- #[test]- fn spans_parse_and_default_to_none() {- let with = parse_response(- r#"{"entities":[],"facts":[{"subject":"OpenAI","predicate":"founded","object":"Anthropic","subject_span":"Former OpenAI personnel","object_span":"Anthropic"}]}"#,- )- .unwrap();- assert_eq!(- with.facts[0].subject_span.as_deref(),- Some("Former OpenAI personnel")- );- assert_eq!(with.facts[0].object_span.as_deref(), Some("Anthropic"));- let without = parse_response(- r#"{"entities":[],"facts":[{"subject":"OpenAI","predicate":"founded","object":"Anthropic"}]}"#,- )- .unwrap();- assert!(without.facts[0].subject_span.is_none());- assert!(without.facts[0].object_span.is_none());- }-- #[test]- fn names_are_parsed_and_a_malformed_one_is_skipped() {- let raw = r#"{"entities":[{"local_id":"e1","name":"海洋探测器1号","type":"equipment"}],- "facts":[],- "names":[{"ref":"e1","name":"海探1","quote":"海洋探测器1号(简称“海探1”)"},- {"name":"no ref"}]}"#;- let x = parse_response(raw).unwrap();- assert_eq!(x.names.len(), 1);- assert_eq!(x.names[0].entity_ref, "e1");- assert_eq!(x.names[0].name, "海探1");- assert_eq!(x.skipped_entities, 0, "a bad name is not a skipped entity");- }-- #[test]- fn the_contract_asks_for_other_names_and_forbids_descriptions() {- let msgs = build_messages(&[], &[], &[], None, "f.txt", &[], "text");- let system = &msgs[0].content;- assert!(system.contains("\"names\":[{\"ref\""));- assert!(system.contains("1b. Every other name the text gives an entity"));- }-- #[test]- fn parse_response_with_fence() {- let raw = "好的,结果如下:\n```json\n{\"entities\":[{\"name\":\"张三\",\"type\":\"person\"}],\"facts\":[]}\n```";- let e = parse_response(raw).unwrap();- assert_eq!(e.entities.len(), 1);- assert_eq!(e.entities[0].type_key, "person");- }-- /// #690:思考过程里的大括号不能把 JSON 的起止带偏。- ///- /// 思考过程里这个没闭合的 `{` 会把"第一个 `{`"的起点提前,而修补截断的逻辑- /// 认不出夹在中间的废话——不切掉标记,整块直接报解析失败作废。- #[test]- fn parse_response_ignores_a_think_block_before_the_json() {- let raw = "先想想 {\"a\": 1,再回答。\n</think>{\"entities\":[{\"name\":\"张三\",\"type\":\"person\"}],\"facts\":[]}";- let e = parse_response(raw).unwrap();- assert_eq!(e.entities.len(), 1);- assert_eq!(e.entities[0].type_key, "person");- }-- #[test]- fn handles_and_fact_refs_are_optional_and_legacy_compatible() {- let handled = parse_response(- r#"{"entities":[{"local_id":"e1","name":"Zhang Wei","type":"person"}],- "facts":[{"subject":"Zhang Wei","subject_ref":"e1","predicate":"leads",- "object":"Finance","object_ref":"e2"}]}"#,- )- .unwrap();- assert_eq!(handled.entities[0].local_id.as_deref(), Some("e1"));- assert_eq!(handled.facts[0].subject_ref.as_deref(), Some("e1"));- assert_eq!(handled.facts[0].object_ref.as_deref(), Some("e2"));-- let legacy = parse_response(- r#"{"entities":[{"name":"Zhang Wei","type":"person"}],- "facts":[{"subject":"Zhang Wei","predicate":"leads","object":"Finance"}]}"#,- )- .unwrap();- assert_eq!(legacy.entities[0].local_id, None);- assert_eq!(legacy.facts[0].subject_ref, None);- assert_eq!(legacy.facts[0].object_ref, None);- }-- #[test]- fn duplicate_handle_definitions_are_all_malformed() {- let x = parse_response(- r#"{"entities":[- {"local_id":"e1","name":"Zhang Wei","type":"person"},- {"local_id":"e1","name":"John Smith","type":"person"},- {"local_id":"e2","name":"Finance","type":"organization"}],- "facts":[]}"#,- )- .unwrap();- assert_eq!(x.skipped_entities, 2);- assert_eq!(x.entities.len(), 1);- assert_eq!(x.entities[0].local_id.as_deref(), Some("e2"));- }-- /// specific_type 在骨架里、也在规则里,且两处都说"永远要填"。- ///- /// 只写进骨架是不够的:**规则与骨架冲突时骨架赢**(语言那条就栽过一次)。- /// 这里两边一致,所以要一起钉住。- #[test]- fn every_entity_is_asked_for_its_own_words() {- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");- let sys = &msgs[0].content;- assert!(sys.contains("\"specific_type\":\"what you would call it\""));- assert!(sys.contains("required on every entity"));- // 关键的一句:不校验。校验它就等于又造了一个词表- assert!(sys.contains("never checked against the list"));- // 与 type 的关系必须说清楚,否则模型会把粗类抄一遍- assert!(sys.contains("narrower than"));- }-- #[test]- fn extraction_contract_explains_handle_identity_without_forcing_splits() {- let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");- let system = &msgs[0].content;- assert!(system.contains("\"local_id\":\"e1\""));- assert!(system.contains("\"subject_ref\":\"e1\""));- // #578:跟 X 有关的一群人不是 X- assert!(system.contains("subject_span and object_span are the exact words"));- assert!(system.contains("unique within this response"));- assert!(system.contains("Reuse the same local_id"));- assert!(system.contains("permanent identity is proven"));- assert!(system.contains("A shared name alone neither proves sameness nor requires a split"));- }-- #[test]- fn same_name_known_entities_keep_distinct_response_handles() {- let known = vec![- KnownEntity {- handle: "k1".into(),- type_key: "person".into(),- name: "Zhang Wei".into(),- },- KnownEntity {- handle: "k2".into(),- type_key: "person".into(),- name: "Zhang Wei".into(),- },- ];- let msgs = build_messages(&[], &[], &[], None, "a.txt", &known, "text");- let user = &msgs[1].content;- assert!(user.contains("k1 [person]: Zhang Wei"));- assert!(user.contains("k2 [person]: Zhang Wei"));- assert!(user.contains("subject_ref/object_ref"));- }}#[cfg(test)]diff --git a/crates/utopia-extract/src/normalize.rs b/crates/utopia-extract/src/normalize.rsdeleted file mode 100644index e2abbae4..00000000--- a/crates/utopia-extract/src/normalize.rs+++ /dev/null@@ -1,1067 +0,0 @@-//! 模型回复落库前的**形状检查**:只看结构,不看词。-//!-//! **分工。**读懂原文里的时间、判断一段话是不是一个东西——这是语言问题,归模型,-//! 契约(提示词 3c)说清楚它该怎么写。这里只核对输出有没有照契约的形状写,判据一律-//! 是结构性的:引文里有没有这段字、值是不是只有标点、一侧是不是契约的日期格式、-//! 同一句里有没有另一条边。**不认任何一种语言的词**——第一版按英文词表认「季度」-//!「N months ended」,中文财报一条都认不出,还把四份报告的标题当成期间删了。-//!-//! 每条规则做了什么都返回给服务端记进丢弃表:违约多常见、出在哪个模型,量得出来,-//! 契约该怎么改看数说话。--use crate::{read_time, ExtractedEntity, ExtractedFact, Extraction};-use std::collections::HashSet;--/// 形状检查做了什么;服务端按条记信号-#[derive(Debug, Clone, PartialEq, Eq)]-pub enum Normalization {- /// 值只有破折号(`—`)或是空的:表里的「无」,不是一个值,不落。**只认破折号**:- /// `☒`、`✓` 这类符号没有字母数字,却是一格写着的内容(勾选了),照值落- NoValue { predicate: String, written: String },- /// 值后面有一截引文里没有的字:只留引文里有的那段。模型读对了表头、却把期间- /// 写进了值(`(6,176) for three months ended July 27, 2025`),引文只有 `(6,176)`- ValueTrimmed {- predicate: String,- kept: String,- dropped: String,- },- /// 没有宾语、没有值、只带边属性:每个属性落成主语上的一条值事实。- /// 从前整条以 object_missing 丢掉,写对了的数跟着没了- QualifiersWithoutObject { predicate: String, values: usize },- /// 宾语是契约格式的日期(`2026-06-30`):时间不是实体。边上的数落成值、日期进有效期;- /// 没带数的,把写出来的那段落成值——`2028`(「2028 年起上线」)、`4000`(人数)都- /// 解析得成年份,丢掉就把一条信息整个丢了- TimeAsObject {- predicate: String,- written: String,- values: usize,- },- /// 主语是契约格式的日期:数是某个东西在那一刻的数,那个东西是谁回复里没说。不落- TimeAsSubject { predicate: String, written: String },- /// 宾语的名字包住了另一个声明实体,而同一句、同主语、同谓词已有一条指向那个实体的边:- /// 它**可能**是那个实体的描述。**只记,不删**:「non-GAAP net income, or earnings, per- /// diluted share」包住了「non-GAAP net income」,却是另一个指标(每股收益)。结构上- /// 分不出描述与另一个东西,删错了就是实体连事实一起没了- ObjectDescribesDeclared {- predicate: String,- name: String,- head: String,- },- /// 上面几条去掉事实之后,没有任何事实再引用的声明:不建,否则就是一个孤点- OrphanDeclaration { name: String },- /// 引文抄自提示词里附的文件开头,而不是这一块:开头只作背景,它自己那一块会抽到。- /// 照落的话,证据挂在这一块上,引的却是第一块的话——引错了出处- QuoteFromOpening { predicate: String, quote: String },-}--/// 比对用的形态:空白折叠、小写-fn norm(s: &str) -> String {- s.split_whitespace()- .collect::<Vec<_>>()- .join(" ")- .to_lowercase()-}--/// 比对引文用的词元:一段连续的数字(中间的 `,` `.` 算在数里,`13,237`、`89.0`),或者-/// 一段连续的非数字字母。其余字符都是分隔。-///-/// **按词元比,不按子串比**:从前 `10 to 15 GW` 对着「10–15 GW」,`10` 作为子串在引文-/// 里,尾巴 `to 15 GW` 作为整段不在,于是被剪成 `10`。数字与汉字之间也切开——中文里数-/// 贴着字写(「营收为12,345元」),不切的话一个数永远对不上-fn tokens(t: &str) -> Vec<String> {- let chars: Vec<char> = t.chars().collect();- let mut out = Vec::new();- let mut cur = String::new();- let mut cur_digit = false;- for (i, &c) in chars.iter().enumerate() {- let digit = c.is_numeric();- let joins_number =- (c == ',' || c == '.') && cur_digit && chars.get(i + 1).is_some_and(|n| n.is_numeric());- if joins_number {- cur.push(c);- continue;- }- if !c.is_alphanumeric() {- if !cur.is_empty() {- out.push(std::mem::take(&mut cur));- }- continue;- }- if !cur.is_empty() && digit != cur_digit {- out.push(std::mem::take(&mut cur));- }- cur_digit = digit;- cur.extend(c.to_lowercase());- }- if !cur.is_empty() {- out.push(cur);- }- out-}--/// `needle` 的词元是否作为连续的一段出现在 `hay` 的词元里-fn contains_tokens(hay: &[String], needle: &[String]) -> bool {- !needle.is_empty() && hay.windows(needle.len()).any(|w| w == needle)-}--fn has_digit(t: &str) -> bool {- t.chars().any(char::is_numeric)-}--/// 值后面是否挂着一截不属于它的字。返回 (保留的前缀, 去掉的尾巴)。-///-/// 两个条件同时成立才剪,都是结构,不认词,比的都是整词元(见 [`tokens`]):-/// - **保留的那段在引文里,而且是一格的写法**——含数字、词元连续地出现在引文里,是原文-/// 写的那个数;或者只有破折号(`—`),是原文那一格写的「没有」;-/// - **尾巴自己含数字,而且那些数一个都不在引文里**——它是另一条信息(一个期间、一个-/// 日期、另一个百分比),不是这个数的单位。尾巴里有一个数在引文里,就说明它是原文的-/// 一部分换了写法(`10 to 15 GW` 对着「10–15 GW」),不剪。-///-/// 第二条要数字,是因为挂在数后面、引文里又没有的,还有一类是对的:表头上的量级与-/// 单位(`53,954 million USD`,这一行引文只有 `53,954`,`million` 在表头「in millions」)、-/// 模型换了写法的单位(`10 gigawatts`)、缩写的头衔(`founder and CEO`)。它们不带数字,-/// 不剪。整个值在引文里的,一个字不动。-fn ungrounded_tail<'a>(value: &'a str, quote: &str) -> Option<(&'a str, &'a str)> {- let q = norm(quote);- if q.is_empty() || q.contains(&norm(value)) {- return None;- }- let quote_tokens = tokens(quote);- let foreign_figures = |tail: &str| {- let numbers: Vec<String> = tokens(tail).into_iter().filter(|w| has_digit(w)).collect();- !numbers.is_empty() && numbers.iter().all(|w| !quote_tokens.contains(w))- };- let bounds: Vec<usize> = value- .char_indices()- .filter(|(i, c)| c.is_whitespace() && *i > 0)- .map(|(i, _)| i)- .collect();- bounds- .iter()- .rev()- .map(|&i| {- (- value[..i]- .trim()- .trim_end_matches([',', ';', ':', '\u{3001}', '\u{FF0C}']),- value[i..].trim(),- )- })- .find(|(p, r)| {- if p.is_empty() || r.is_empty() || !foreign_figures(r) {- return false;- }- if is_no_value(p) {- return q.contains(&norm(p));- }- has_digit(p) && contains_tokens("e_tokens, &tokens(p))- })-}--/// 表格里表示「没有」的那一格:空的,或者只有破折号。-///-/// **只认破折号(Unicode 的 Pd 类)**,不是「没有字母数字就算」:`☒`、`✓`、`☐` 也没有-/// 字母数字,却是那一格真写着的内容,从前被当成空格子丢了-fn is_no_value(written: &str) -> bool {- written.chars().filter(|c| !c.is_whitespace()).all(|c| {- matches!(- c,- '-' | '\u{058A}' | '\u{05BE}' | '\u{1400}' | '\u{1806}' | '\u{2010}'- ..='\u{2015}'- | '\u{2E17}'- | '\u{2E1A}'- | '\u{2E3A}'- | '\u{2E3B}'- | '\u{2E40}'- | '\u{301C}'- | '\u{3030}'- | '\u{30A0}'- | '\u{FE31}'- | '\u{FE32}'- | '\u{FE58}'- | '\u{FE63}'- | '\u{FF0D}'- )- })-}--/// 边属性里不是值、是单位的那几个键(与服务端同一张表)-fn is_unit_key(k: &str) -> bool {- matches!(- k.trim().to_lowercase().as_str(),- "currency" | "币种" | "货币" | "unit" | "单位"- )-}--fn qualifier_values(f: &ExtractedFact) -> Vec<(String, serde_json::Value)> {- f.qualifiers- .as_ref()- .map(|q| {- q.iter()- .filter(|(k, v)| !v.is_null() && !is_unit_key(k))- .map(|(k, v)| (k.clone(), v.clone()))- .collect()- })- .unwrap_or_default()-}--fn value_fact(- f: &ExtractedFact,- predicate: String,- value: serde_json::Value,- valid_from: Option<String>,-) -> ExtractedFact {- ExtractedFact {- subject: f.subject.clone(),- subject_ref: f.subject_ref.clone(),- predicate,- object: None,- object_ref: None,- value: Some(value),- qualifiers: None,- valid_from: valid_from.or_else(|| f.valid_from.clone()),- valid_to: f.valid_to.clone(),- confidence: f.confidence,- quote: f.quote.clone(),- subject_span: f.subject_span.clone(),- object_span: None,- relative: false,- }-}--/// 一侧写的是一个时间:规则 3 的格式(`YYYY` / `YYYY-MM` / `YYYY-MM-DD`,带时区的时刻),-/// 或写法说得清是哪天的日期(`written_date`,#688)-fn names_a_time(s: &str) -> bool {- read_time(s.trim()).is_some()-}--/// 引文不在这一块、却在附上的文件开头里的事实:丢掉,连同因此没人引用的声明。-///-/// 只比「在不在」(空白、大小写不论),不比像不像:两边都找不到的引文不归这里管——-/// 那是模型改写了原文,与开头无关-pub fn drop_quotes_from_opening(- x: &mut Extraction,- chunk_text: &str,- opening: &str,-) -> Vec<Normalization> {- let (chunk, opening) = (norm(chunk_text), norm(opening));- if opening.is_empty() {- return Vec::new();- }- let before = referenced_names(&x.entities, &x.facts);- let mut out = Vec::new();- x.facts.retain(|f| {- let q = norm(f.quote.as_deref().unwrap_or(""));- let from_opening = !q.is_empty() && !chunk.contains(&q) && opening.contains(&q);- if from_opening {- out.push(Normalization::QuoteFromOpening {- predicate: f.predicate.clone(),- quote: f.quote.clone().unwrap_or_default(),- });- }- !from_opening- });- if out.is_empty() {- return out;- }- let mut after = referenced_names(&x.entities, &x.facts);- // 与 normalize_facts 同一条:模型给它报了别的名字的声明不算孤点(0041)- after.extend(x.names.iter().filter_map(|n| {- x.entities- .iter()- .find(|e| e.local_id.as_deref().map(str::trim) == Some(n.entity_ref.trim()))- .map(|e| e.name.trim().to_lowercase())- }));- let mut orphans = Vec::new();- x.entities.retain(|e| {- let n = e.name.trim().to_lowercase();- let orphan = before.contains(&n) && !after.contains(&n);- if orphan {- orphans.push(e.name.trim().to_string());- }- !orphan- });- out.extend(- orphans- .into_iter()- .map(|name| Normalization::OrphanDeclaration { name }),- );- out-}--/// 事实两侧绑到的声明名(小写):有句柄按句柄,没有按写出来的名字-fn referenced_names(entities: &[ExtractedEntity], facts: &[ExtractedFact]) -> HashSet<String> {- let handle_name = |h: Option<&String>| {- h.and_then(|h| {- entities- .iter()- .find(|e| e.local_id.as_deref().map(str::trim) == Some(h.trim()))- .map(|e| e.name.trim().to_string())- })- };- facts- .iter()- .flat_map(|f| {- [- handle_name(f.subject_ref.as_ref()).or_else(|| Some(f.subject.trim().to_string())),- handle_name(f.object_ref.as_ref())- .or_else(|| f.object.as_deref().map(|o| o.trim().to_string())),- ]- })- .flatten()- .map(|n| n.to_lowercase())- .collect()-}--pub fn normalize_facts(x: &mut Extraction) -> Vec<Normalization> {- let mut out = Vec::new();- let entities = std::mem::take(&mut x.entities);-- // 一侧绑到的声明名:有句柄按句柄,没有按写出来的名字- let handle_name = |h: Option<&String>| {- h.and_then(|h| {- entities- .iter()- .find(|e| e.local_id.as_deref().map(str::trim) == Some(h.trim()))- .map(|e| e.name.trim().to_string())- })- };- let before = referenced_names(&entities, &x.facts);-- let mut facts: Vec<ExtractedFact> = Vec::with_capacity(x.facts.len());- for mut f in std::mem::take(&mut x.facts) {- let quote = f.quote.clone().unwrap_or_default();-- // ---- 值 ----- if let Some(written) = f.value.as_ref().and_then(|v| v.as_str()).map(str::to_owned) {- // 先剪再看空:`— for three months ended July 27, 2025` 剪掉尾巴才露出那一格是空的- let figure = ungrounded_tail(&written, "e).map_or(written.as_str(), |(k, _)| k);- if is_no_value(figure) {- out.push(Normalization::NoValue {- predicate: f.predicate.clone(),- written,- });- continue;- }- if let Some((kept, dropped)) = ungrounded_tail(&written, "e) {- out.push(Normalization::ValueTrimmed {- predicate: f.predicate.clone(),- kept: kept.to_string(),- dropped: dropped.to_string(),- });- f.value = Some(serde_json::Value::String(kept.to_string()));- }- }-- // ---- 主语是时间 ----- let subject =- handle_name(f.subject_ref.as_ref()).unwrap_or_else(|| f.subject.trim().to_string());- if names_a_time(&subject) {- out.push(Normalization::TimeAsSubject {- predicate: f.predicate.clone(),- written: subject,- });- continue;- }-- let values = qualifier_values(&f);- let object = handle_name(f.object_ref.as_ref())- .or_else(|| f.object.as_deref().map(|o| o.trim().to_string()))- .filter(|o| !o.is_empty());- let has_value = f.value.as_ref().is_some_and(|v| !v.is_null());-- // ---- 只有边属性 ----- if object.is_none() && !has_value && !values.is_empty() {- for (key, value) in &values {- let predicate = format!("{}.{}", f.predicate.trim(), key.trim());- facts.push(value_fact(&f, predicate, value.clone(), None));- }- out.push(Normalization::QualifiersWithoutObject {- predicate: f.predicate.clone(),- values: values.len(),- });- continue;- }-- // ---- 宾语是时间 ----- if let Some(o) = object.as_deref().filter(|o| names_a_time(o)) {- if values.is_empty() {- // 没带数:写出来的那段就是值。`2028`(「2028 年起上线」)、`4000`(人数)都解析- // 得成年份,从前整条丢掉,一条信息就没了。宾语那个声明没人引用,下面按孤点去掉- facts.push(value_fact(- &f,- f.predicate.trim().to_string(),- serde_json::Value::String(o.to_string()),- None,- ));- out.push(Normalization::TimeAsObject {- predicate: f.predicate.clone(),- written: o.to_string(),- values: 0,- });- continue;- }- let several = values.len() > 1;- for (key, value) in &values {- let predicate = if several {- format!("{}.{}", f.predicate.trim(), key.trim())- } else {- f.predicate.trim().to_string()- };- facts.push(value_fact(- &f,- predicate,- value.clone(),- Some(o.to_string()),- ));- }- out.push(Normalization::TimeAsObject {- predicate: f.predicate.clone(),- written: o.to_string(),- values: values.len(),- });- continue;- }-- facts.push(f);- }-- // ---- 描述:名字包住另一个声明实体,本尊那条边同句已在。只记,不删 ----- let declared: Vec<String> = entities.iter().map(|e| e.name.trim().to_string()).collect();- let side = |r: Option<&String>, w: Option<&str>| {- handle_name(r)- .or_else(|| w.map(|s| s.trim().to_string()))- .filter(|s| !s.is_empty())- };- let objects: Vec<Option<String>> = facts- .iter()- .map(|f| side(f.object_ref.as_ref(), f.object.as_deref()))- .collect();- let subjects: Vec<Option<String>> = facts- .iter()- .map(|f| side(f.subject_ref.as_ref(), Some(f.subject.as_str())))- .collect();- // 名字边界:拉丁字母数字前后不能粘着字母数字;汉字之间本来就没有空格,不设边界- let contains_name = |outer: &str, inner: &str| {- let (o, i) = (norm(outer), norm(inner));- !i.is_empty()- && o.len() > i.len()- && o.match_indices(&i).any(|(at, _)| {- let glued = |c: Option<char>| c.is_some_and(|c| c.is_ascii_alphanumeric());- let edge_in = i.chars().next().is_some_and(|c| c.is_ascii_alphanumeric());- let edge_out = i- .chars()- .next_back()- .is_some_and(|c| c.is_ascii_alphanumeric());- !(edge_in && glued(o[..at].chars().next_back()))- && !(edge_out && glued(o[at + i.len()..].chars().next()))- })- };- for i in 0..facts.len() {- let Some(name) = objects[i].as_deref() else {- continue;- };- let quote_i = norm(facts[i].quote.as_deref().unwrap_or(""));- let head = declared.iter().find(|h| {- contains_name(name, h)- && (0..facts.len()).any(|j| {- j != i- && facts[j].predicate.eq_ignore_ascii_case(&facts[i].predicate)- && subjects[j] == subjects[i]- && objects[j]- .as_deref()- .is_some_and(|o| o.eq_ignore_ascii_case(h))- && {- let quote_j = norm(facts[j].quote.as_deref().unwrap_or(""));- quote_i.contains("e_j) || quote_j.contains("e_i)- }- })- });- if let Some(head) = head {- // 不删(见枚举上的说明):结构分不出「SB Energy 的发展」与「每股收益」这类- // 包住了另一个名字的真指标。记下来,量得出这种形状多常见、有多少是描述- out.push(Normalization::ObjectDescribesDeclared {- predicate: facts[i].predicate.clone(),- name: name.to_string(),- head: head.clone(),- });- }- }- x.facts = facts;-- // ---- 被上面几条弄成孤点的声明 ----- // 只去掉「原来有事实引用、现在没有了」的:模型一开始就只声明不连边的,不归这里管- let mut after = referenced_names(&entities, &x.facts);- // 模型给它报了别的名字的声明也不算孤点:那些名字要绑在它身上(0041)- after.extend(- x.names- .iter()- .filter_map(|n| handle_name(Some(&n.entity_ref)))- .map(|n| n.to_lowercase()),- );- let mut orphans = Vec::new();- let mut kept_entities = entities;- kept_entities.retain(|e| {- let n = e.name.trim().to_lowercase();- let orphan = before.contains(&n) && !after.contains(&n);- if orphan {- orphans.push(e.name.trim().to_string());- }- !orphan- });- x.entities = kept_entities;- out.extend(- orphans- .into_iter()- .map(|name| Normalization::OrphanDeclaration { name }),- );- out-}--#[cfg(test)]-mod tests {- use super::*;- use crate::ExtractedEntity;-- fn fact(subject: &str, predicate: &str, object: Option<&str>, quote: &str) -> ExtractedFact {- ExtractedFact {- subject: subject.into(),- subject_ref: None,- predicate: predicate.into(),- object: object.map(str::to_string),- object_ref: None,- value: None,- qualifiers: None,- valid_from: None,- valid_to: None,- confidence: Some(0.9),- quote: Some(quote.into()),- subject_span: Some(subject.into()),- object_span: object.map(str::to_string),- relative: false,- }- }- fn valued(subject: &str, predicate: &str, value: &str, quote: &str) -> ExtractedFact {- let mut f = fact(subject, predicate, None, quote);- f.value = Some(serde_json::Value::String(value.into()));- f- }- fn entity(id: &str, name: &str) -> ExtractedEntity {- ExtractedEntity {- local_id: Some(id.into()),- name: name.into(),- type_key: "organization".into(),- specific_type: None,- }- }- fn quals(pairs: &[(&str, &str)]) -> serde_json::Map<String, serde_json::Value> {- pairs- .iter()- .map(|(k, v)| (k.to_string(), serde_json::Value::String(v.to_string())))- .collect()- }- /// 第五份补充协议的第三块:模型从附上的开头里抄了「as of February 18, 2020」那句当引文。- /// 那条事实丢掉,只为它而声明的实体跟着不建;引文在这一块里的照落- #[test]- fn a_fact_quoting_the_opening_is_dropped_with_its_orphan() {- let opening = "THIS FIFTH AMENDMENT TO LEASE AGREEMENT is entered into as of February 18, 2020 by and between HPBB1, LLC and BLACKBAUD, INC.";- let chunk = "The Phase 2 Exercise Deadline is hereby extended to March 17, 2020.";- let mut x = Extraction {- entities: vec![entity("e1", "Lease"), entity("e2", "HPBB1, LLC")],- facts: vec![- fact(- "Lease",- "landlord",- Some("HPBB1, LLC"),- "entered into as of February 18, 2020 by and between HPBB1, LLC",- ),- fact(- "Lease",- "expansion_option_deadline",- None,- "the phase 2 exercise deadline is hereby extended to March 17, 2020",- ),- ],- names: Vec::new(),- skipped_entities: 0,- skipped_facts: 0,- truncated: false,- };- let n = drop_quotes_from_opening(&mut x, chunk, opening);- let kept: Vec<&str> = x.facts.iter().map(|f| f.predicate.as_str()).collect();- assert_eq!(- kept,- ["expansion_option_deadline"],- "大小写与空白不论,引文在这一块里的留下"- );- let names: Vec<&str> = x.entities.iter().map(|e| e.name.as_str()).collect();- assert_eq!(names, ["Lease"], "只为被丢那条声明的 HPBB1 不建");- assert!(n- .iter()- .any(|v| matches!(v, Normalization::QuoteFromOpening { .. })));- assert!(n- .iter()- .any(|v| matches!(v, Normalization::OrphanDeclaration { .. })));-- // 两边都找不到的引文不归这里管;没有开头时什么都不做- let mut y = Extraction {- entities: vec![entity("e1", "Lease")],- facts: vec![fact("Lease", "note", None, "a paraphrase of neither")],- names: Vec::new(),- skipped_entities: 0,- skipped_facts: 0,- truncated: false,- };- assert!(drop_quotes_from_opening(&mut y, chunk, opening).is_empty());- assert!(drop_quotes_from_opening(&mut y, chunk, " ").is_empty());- assert_eq!(y.facts.len(), 1);- }-- fn run(- entities: Vec<ExtractedEntity>,- facts: Vec<ExtractedFact>,- ) -> (Extraction, Vec<Normalization>) {- let mut x = Extraction {- entities,- facts,- skipped_entities: 0,- skipped_facts: 0,- truncated: false,- names: Vec::new(),- };- let n = normalize_facts(&mut x);- (x, n)- }- fn value_of(f: &ExtractedFact) -> &str {- f.value.as_ref().unwrap().as_str().unwrap()- }-- /// 截图里那 21 条:模型读对了表头,却把期间写进了值。引文里只有数- #[test]- fn a_tail_the_quote_does_not_contain_is_not_the_value() {- let (x, n) = run(- vec![entity("e1", "NVIDIA")],- vec![- valued(- "NVIDIA",- "net_cash_used",- "(6,176) for three months ended July 27, 2025",- "Net cash used in financing activities (6,176)",- ),- // 同样的形状,中文:不认词,照样剪- valued(- "NVIDIA",- "经营现金流",- "12,345 截至2025年7月27日止三个月",- "经营活动产生的现金流量净额 | 12,345",- ),- valued(- "NVIDIA",- "revenue",- "$89.0 billion, up 18% from the previous quarter",- "Second-quarter revenue was $89.0 billion",- ),- ],- );- assert_eq!(value_of(&x.facts[0]), "(6,176)");- assert_eq!(value_of(&x.facts[1]), "12,345");- // 数后面接着另一个数:「up 18%」是另一条事实,不是这个数的一部分;逗号跟着前缀走- assert_eq!(value_of(&x.facts[2]), "$89.0 billion");- assert!(- matches!(&n[0], Normalization::ValueTrimmed { dropped, .. } if dropped == "for three months ended July 27, 2025")- );- }-- #[test]- fn a_value_grounded_in_the_quote_is_left_alone() {- let (x, n) = run(- vec![entity("e1", "Tench Coxe")],- vec![- // 整个值在引文里- valued(- "NVIDIA",- "operating_expenses",- "$9.2 billion",- "expected to be approximately $9.2 billion and $9.0 billion",- ),- // 尾巴 shares 在引文别处出现过:原文的单位,不剪- valued(- "Tench Coxe",- "votes_for",- "15,411,252,412 shares",- "Number of shares For | 15,411,252,412",- ),- // 规范化过的值,前缀一个词都对不上:不是这条规则管的- valued("Vega", "amount", "$5 billion", "invested 5 billion dollars"),- // 表头上的量级与币种:引文那一行只有数,尾巴不带数字,是它的单位,不剪- valued(- "NVIDIA",- "net_income",- "53,954 million USD",- "Net income | $ | 53,954",- ),- // 模型换了写法的单位、缩写的头衔:不带数字,不剪- valued(- "SB Energy",- "capacity",- "10 gigawatts",- "at least 10 GW of new generation",- ),- valued(- "Jensen Huang",- "job_title",- "founder and CEO",- "Jensen Huang, founder and chief executive officer",- ),- ],- );- let got: Vec<&str> = x.facts.iter().map(value_of).collect();- assert_eq!(- got,- [- "$9.2 billion",- "15,411,252,412 shares",- "$5 billion",- "53,954 million USD",- "10 gigawatts",- "founder and CEO"- ]- );- assert!(n.is_empty(), "{n:?}");- }-- #[test]- fn a_dash_is_no_value() {- let (x, n) = run(- vec![entity("e1", "NVIDIA")],- vec![- valued("NVIDIA", "dividend", "—", "Dividends | —"),- // 勾选框是那一格写着的内容,不是空格子:没有字母数字,照值落- valued(- "NVIDIA",- "large_accelerated_filer",- "☒",- "Large accelerated filer | ☒",- ),- valued(- "NVIDIA",- "emerging_growth_company",- "☐",- "Emerging growth company | ☐",- ),- valued("NVIDIA", "dividend", " – ", "Dividends | –"),- valued("NVIDIA", "dividend", "$0.01", "Dividends | $0.01"),- // 空的那一格后面挂着列头上的期间:剪掉尾巴,剩下的仍是空- valued(- "NVIDIA",- "amount",- "— for three months ended July 27, 2025",- "Purchases of marketable securities | — | (6,176)",- ),- // 反例:前缀只有标点、在引文里,可尾巴上的数也在引文里——那个数才是值- valued(- "NVIDIA",- "net_income",- "$ 53,954 million",- "Net income | $ | 53,954",- ),- ],- );- let kept: Vec<&str> = x.facts.iter().map(value_of).collect();- assert_eq!(kept, ["☒", "☐", "$0.01", "$ 53,954 million"]);- assert_eq!(- n.iter()- .filter(|v| matches!(v, Normalization::NoValue { .. }))- .count(),- 3- );- }-- /// Coxe 的形状:`vote_result` 带着数,没有宾语也没有值- #[test]- fn figures_on_an_edge_with_no_other_end_land_on_the_subject() {- let mut f = fact(- "Tench Coxe",- "vote_result",- None,- "Number of shares For | 15,411,252,412",- );- f.qualifiers = Some(quals(&[- ("for", "15,411,252,412"),- ("against", "1,399,727,580"),- ("unit", "shares"),- ]));- let (x, n) = run(vec![entity("e1", "Tench Coxe")], vec![f]);- let mut got: Vec<(String, String)> = x- .facts- .iter()- .map(|f| (f.predicate.clone(), value_of(f).to_string()))- .collect();- got.sort();- assert_eq!(- got,- [- (- "vote_result.against".to_string(),- "1,399,727,580".to_string()- ),- ("vote_result.for".to_string(), "15,411,252,412".to_string()),- ]- );- assert_eq!(- n,- vec![Normalization::QualifiersWithoutObject {- predicate: "vote_result".into(),- values: 2- }]- );- }-- /// 时间做了宾语:边上的数落成值,时间进有效期,那个时间节点不建- #[test]- fn a_time_object_becomes_the_validity_and_leaves_no_node() {- let mut margin = fact(- "NVIDIA",- "gross_margin",- Some("2026-06"),- "Gross margin | 75.0%",- );- margin.object_ref = Some("e7".into());- margin.qualifiers = Some(quals(&[("percentage", "75.0%")]));- let mut bare = fact("NVIDIA", "reported_in", Some("2026"), "reported in 2026");- bare.object_ref = Some("e8".into());- let (x, n) = run(- vec![- entity("e1", "NVIDIA"),- entity("e7", "2026-06"),- entity("e8", "2026"),- ],- vec![margin, bare],- );- // 带数的:数落成值、日期进有效期;没带数的:写出来的那段落成值,不丢- assert_eq!(x.facts.len(), 2);- assert_eq!(x.facts[0].predicate, "gross_margin");- assert_eq!(value_of(&x.facts[0]), "75.0%");- assert_eq!(x.facts[0].valid_from.as_deref(), Some("2026-06"));- assert_eq!(- (x.facts[1].predicate.as_str(), value_of(&x.facts[1])),- ("reported_in", "2026")- );- assert!(x.facts[1].object.is_none() && x.facts[1].object_ref.is_none());- let names: Vec<&str> = x.entities.iter().map(|e| e.name.as_str()).collect();- assert_eq!(names, ["NVIDIA"]);- assert_eq!(- n.iter()- .filter(|v| matches!(v, Normalization::OrphanDeclaration { .. }))- .count(),- 2- );- }-- #[test]- fn a_time_subject_is_not_placed() {- let f = valued(- "2026-07-26",- "revenue",- "$89.0 billion",- "revenue was $89.0 billion",- );- let (x, n) = run(vec![entity("e1", "NVIDIA")], vec![f]);- assert!(x.facts.is_empty());- assert!(matches!(&n[0], Normalization::TimeAsSubject { .. }));- }-- /// 不是契约格式的期间名(`Q2 FY27`、`第二季度`)不归这里判:那是模型的事,契约 3c 管- #[test]- fn a_period_name_that_is_not_a_date_is_not_second_guessed() {- let mut f = fact(- "NVIDIA",- "gross_margin_for_period",- Some("Q2 FY27"),- "Gross margin | 75.0 | %",- );- f.qualifiers = Some(quals(&[("percentage", "75.0%")]));- let (x, n) = run(- vec![entity("e1", "NVIDIA"), entity("e7", "Q2 FY27")],- vec![f],- );- assert_eq!(x.facts.len(), 1);- assert_eq!(x.entities.len(), 2);- assert!(n.is_empty());- }-- /// SB Energy 那句:名字包住了另一个声明实体,同句同主语同谓词已有一条指向它的边。- /// 记一条信号,事实与声明都留着——结构分不出它与下面「每股收益」那种真指标- #[test]- fn a_description_beside_its_head_is_flagged_not_removed() {- let quote = "NVIDIA to invest $1.5B in SB Energy now to support SB Energy\u{2019}s growth and commitments to the Ohio community";- let mut head = fact("NVIDIA", "invested_in", Some("SB Energy"), quote);- head.object_ref = Some("e2".into());- let mut desc = fact(- "NVIDIA",- "invested_in",- Some("SB Energy's growth and commitments to the Ohio community"),- quote,- );- desc.object_ref = Some("e6".into());- let (x, n) = run(- vec![- entity("e1", "NVIDIA"),- entity("e2", "SB Energy"),- entity(- "e6",- "SB Energy's growth and commitments to the Ohio community",- ),- ],- vec![head, desc],- );- // 只记不删:两条都在,那个声明也在(它仍被引用)- assert_eq!(x.facts.len(), 2);- assert_eq!(x.entities.len(), 3);- assert!(n.iter().any(- |v| matches!(v, Normalization::ObjectDescribesDeclared { head, .. } if head == "SB Energy")- ));- }-- /// 同样的结构,中文:不靠 's,照样认得出这个形状(只记)- #[test]- fn a_description_is_recognised_without_a_possessive() {- let quote = "英伟达向星辰能源投资15亿美元,支持星辰能源在俄亥俄的发展";- let (x, n) = run(- vec![- entity("e1", "英伟达"),- entity("e2", "星辰能源"),- entity("e3", "星辰能源在俄亥俄的发展"),- ],- vec![- fact("英伟达", "投资", Some("星辰能源"), quote),- fact("英伟达", "投资", Some("星辰能源在俄亥俄的发展"), quote),- ],- );- assert_eq!(x.facts.len(), 2);- assert_eq!(x.entities.len(), 3);- assert!(n- .iter()- .any(|v| matches!(v, Normalization::ObjectDescribesDeclared { .. })));- }-- /// **四处误伤,逐个钉住**(#637 实测):每一处从前都把一条真信息丢了或剪坏了- #[test]- fn what_the_shape_checks_used_to_break_is_kept() {- // 一、短数字被当子串匹配:`10 to 15 GW` 对着「10–15 GW」,从前剪成 `10`- let (x, n) = run(- vec![entity("e1", "SB Energy")],- vec![- valued(- "SB Energy",- "planned_capacity",- "10 to 15 GW",- "SB Energy plans 10–15 GW of new generation",- ),- // 词元不是子串:`5` 不在「25 GW」里- valued(- "SB Energy",- "planned_capacity",- "5 GW by 2030",- "SB Energy plans 25 GW by 2030",- ),- // 中文里数贴着字写,照样按词元对得上,尾巴那个日期不在引文里,剪- valued(- "SB Energy",- "营收",- "12,345 截至2025年7月27日止三个月",- "营收为12,345元",- ),- ],- );- let got: Vec<&str> = x.facts.iter().map(value_of).collect();- assert_eq!(got, ["10 to 15 GW", "5 GW by 2030", "12,345"]);- assert_eq!(n.len(), 1, "{n:?}");-- // 二、勾选框:见 a_dash_is_no_value-- // 三、四位数的宾语:`2028`、`4000` 解析得成年份,落成值,不丢- let mut launch = fact(- "Aurora",- "launch_year",- Some("2028"),- "Aurora goes live from 2028",- );- launch.object_ref = Some("e2".into());- let mut staff = fact(- "Acme",- "employees",- Some("4000"),- "Acme employs 4000 people",- );- staff.object_ref = Some("e3".into());- let (x, _) = run(- vec![- entity("e1", "Aurora"),- entity("e2", "2028"),- entity("e3", "4000"),- entity("e4", "Acme"),- ],- vec![launch, staff],- );- let got: Vec<(&str, &str)> = x- .facts- .iter()- .map(|f| (f.predicate.as_str(), value_of(f)))- .collect();- assert_eq!(got, [("launch_year", "2028"), ("employees", "4000")]);- let names: Vec<&str> = x.entities.iter().map(|e| e.name.as_str()).collect();- assert_eq!(names, ["Aurora", "Acme"], "年份那两个声明不建成节点");-- // 四、包住了另一个指标名的真指标:每股收益不是净利润的描述,留着- let quote = "non-GAAP net income was $26.4 billion and non-GAAP net income, or earnings, per diluted share was $1.05";- let (x, n) = run(- vec![- entity("e1", "NVIDIA"),- entity("e2", "non-GAAP net income"),- entity("e3", "non-GAAP net income, or earnings, per diluted share"),- ],- vec![- fact(- "NVIDIA",- "reported_metric",- Some("non-GAAP net income"),- quote,- ),- fact(- "NVIDIA",- "reported_metric",- Some("non-GAAP net income, or earnings, per diluted share"),- quote,- ),- ],- );- assert_eq!(x.facts.len(), 2);- assert_eq!(x.entities.len(), 3);- assert!(n- .iter()- .any(|v| matches!(v, Normalization::ObjectDescribesDeclared { .. })));- }-- /// 包住了别的名字、但旁边没有指向本尊的同一条边:可能真是一个东西,不动- #[test]- fn a_name_that_contains_another_without_a_sibling_edge_is_left_alone() {- let quote = "Sam Altman was removed by OpenAI's board of directors";- let (x, n) = run(- vec![- entity("e1", "Sam Altman"),- entity("e2", "OpenAI"),- entity("e3", "OpenAI's board of directors"),- ],- vec![fact(- "Sam Altman",- "removed_by",- Some("OpenAI's board of directors"),- quote,- )],- );- assert_eq!(x.facts.len(), 1);- assert_eq!(x.entities.len(), 3);- assert!(n.is_empty());- }-}diff --git a/crates/utopia-server/src/api/kbs.rs b/crates/utopia-server/src/api/kbs.rsindex 57f7c69c..89a4a71d 100644--- a/crates/utopia-server/src/api/kbs.rs+++ b/crates/utopia-server/src/api/kbs.rs@@ -55,10 +55,6 @@ pub struct UpdateKbReq {/// 提示词都读它。探索生成的描述在另一个字段,PATCH 不了。见 #570#[serde(default)]pub data_conventions: Option<String>,- /// 开放抽取(0044 第一刀,#729):开着,抽取只写开放图谱——陈述照文档的字落库,- /// 不读本体;关着走今天的类型化那条路。缺省关- #[serde(default)]- pub open_extraction: Option<bool>,}/// 用户可见的 KB 列表(restricted 库仅矩阵成员与系统管理员可见)。@@ -119,7 +115,6 @@ pub async fn create(None,None,None,- None,).await?;}@@ -202,7 +197,6 @@ pub async fn update(req.auto_type_resolution,req.governance,req.data_conventions.as_deref().map(str::trim),- req.open_extraction,).await?;// 打开开关就自动开始处理:排一轮,同库已排着的不重复diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rsindex 647251c5..887aa3e9 100644--- a/crates/utopia-server/src/extraction.rs+++ b/crates/utopia-server/src/extraction.rs@@ -1,13 +1,16 @@-//! 图谱抽取任务:逐分块调用 LLM → 实体消解 v2 → 事实 + 证据写入账本。+//! 图谱抽取任务的入口,以及开放写入路径(`extraction_open`)共用的那几块。+//!+//! 抽取只写开放图谱(0044 决定 2):每篇文档——记忆日志也在内——都从这里进+//! `run_open`,提示词里没有本体,也没有带本体的第二条路。这里留下的是那条路借用的东西:+//! 限流退避的 chat 调用、来源给置信度设的上限、片段核对、丢弃信号、未抽完的判据、+//! 名字到实体的消解(句柄、同名审核对)。//! 与摄入管道分离(两段式可用):索引完成即可搜可问,抽取慢慢跑。//! 消解灰区只入审核队列并触发独立的攒批裁决任务——LLM 裁决永不阻塞本任务。use crate::llm_util;-use crate::ontology_index;-use crate::predicate_match::PredicateIndex;use crate::state::AppState;use sqlx::PgPool;-use std::collections::{HashMap, HashSet};+use std::collections::HashMap;use std::time::Duration;use utopia_core::models::Proposer;use uuid::Uuid;@@ -31,7 +34,7 @@ fn jitter(base: Duration) -> Duration {base / 2 + Duration::from_millis(if half == 0 { 0 } else { nanos % half })}-/// 抽取的 chat 调用,**限流会退避重试**。+/// 抽取的 chat 调用,**限流会退避重试**。温度由调用方定(开放抽取要 0,见 `extraction_open`)。////// 限流与其他失败的区别是它会自己好,所以从前那句「跳过该分块」用在它身上/// 就是把一分钟的等待换成永久的数据缺口——实测一次 1884 块的灌入里@@ -43,16 +46,6 @@ fn jitter(base: Duration) -> Duration {/// 否则一个在等的分块会挡住本来可以通过的另一个。/// - **`Retry-After` 多数厂商不发**,所以它只是「有则更准」,判据是错误类型/// 本身;没有它就走指数退避。-pub(crate) async fn chat_retrying_rate_limits(- state: &AppState,- settings: &utopia_core::models::LlmSettings,- client: &utopia_llm::LlmClient,- messages: &[utopia_llm::ChatMessage],-) -> anyhow::Result<String> {- chat_retrying_rate_limits_at(state, settings, client, messages, None).await-}--/// 同上,指定采样温度(开放抽取要 0,见 `extraction_open`)。pub(crate) async fn chat_retrying_rate_limits_at(state: &AppState,settings: &utopia_core::models::LlmSettings,@@ -90,8 +83,6 @@ pub(crate) async fn chat_retrying_rate_limits_at(unreachable!("循环内必定 return")}-const MIN_CONFIDENCE: f32 = 0.6;-/// 模型看图描述出来的事实,置信度压到这里(0040 决定 4):低于时态引擎自动关闭的门槛,/// 它能进图、能被搜到和引用,却不能单独把一段正确的旧值关掉——看柱状图读错一个数字是常事,/// 按普通事实入库的话,错的数会关掉它反驳的那个对的数,而库里没有一行说这次关闭靠的是一张图。@@ -107,90 +98,6 @@ pub(crate) fn origin_ceiling(origin: &str, confidence: f32) -> f32 {}}-/// 这串字**是不是一个东西的名字**。-///-/// 判据是**词数**不是字符数。字符数分不开真假:-/// `US District Court for the Northern District of California`(57 字符)是真实体,-/// 而 `removal was driven by growing discontent and distrust with Altman`(65 字符)-/// 是一整个从句——两者字符数相近,词数也相近(9 vs 10),但后者带着**限定动词**。-///-/// 所以两条一起看:词数封顶挡住长句,而**句中的限定动词**挡住那些不长的从句。-/// 机构名会长("US District Court for the Northern District of California"),-/// 但不会出现 "was driven by"、"showed"、"giving off" 这种谓语。-///-/// 上限取 12 个词:实测真实体里最长的机构名是 9 个词,留三个词的余量。-/// 而被挡下的那些平均 14 个词。-const MAX_NAME_WORDS: usize = 12;--/// 句子里的谓语标志。**只列限定形式**——`used`、`flying` 这类分词在名词短语里-/// 完全正常("equipment used by X"),列进去会误伤真实体。-const CLAUSE_MARKERS: &[&str] = &[- "was", "were", "is", "are", "has", "have", "had", "will", "would", "showed", "said", "says",- "became", "went", "came", "did", "does", "gave", "took", "made",-];--/// 情态动词:英语里**封闭类**的限定形式——不像 `fell`、`hits` 那样开放无边,也不会-/// 出现在名词短语里。`may`(月份、人名)与 `can`(容器)除外,它们兼作名词(#193)-const MODALS: &[&str] = &["could", "should", "might", "must", "shall"];--/// 结构信号:不靠词表也看得出的「这是一句话」(#193)。-///-/// 动词表挡不住新语料——英语有上千个限定形式,`could`、`fell`、`hits` 都不在那 19 个-/// 词里,而且第一个例句恰好 12 个词,卡在上限上。结构信号迁移得动:-/// 1. **句号结尾**:末词是小写词并以句号收尾(`… as a whole.`)。专名缩写 `Inc.` /-/// `Co.` 是大写开头,不误伤。-/// 2. **情态动词**:封闭类,见 [`MODALS`]。-///-/// 这两条**当场拒绝**——它们和词数上限一样是结构判据,不是又一份词表。-/// `words` 是小写化、去标点的词,`raw_last` 是保留原样的末词-fn reads_like_a_sentence(words: &[String], raw_last: Option<&str>) -> bool {- if words.len() > 2 && words.iter().any(|w| MODALS.contains(&w.as_str())) {- return true;- }- if let Some(stem) = raw_last.and_then(|w| w.strip_suffix('.')) {- if stem.chars().count() >= 3 && stem.chars().all(|c| c.is_lowercase()) {- return true;- }- }- false-}--/// 弱信号:像从句,但不敢当场拒绝(#193)。-///-/// 守卫的样本全部来自一份语料,换一份就漏——换规则之前先要一份**跨语料的标注集**。-/// 命中只记 `clause_suspect`(例句进 `extraction_drops`),实体照常落库;攒够两份语料的-/// 样本再决定哪条升成硬规则。返回的是信号名,作为记录的 detail-/// 槽位片段核对的结论(#582)-#[derive(Debug, PartialEq)]-enum SpanVerdict {- /// 没给片段,或片段就是所绑的那个名字(同名、同词干、名字的一部分、名字后面接着- /// 大写的续词——"Anthropic PBC")- Ok,- /// 片段不在引文里:模型没照抄。当没给处理,记一笔- NotInQuote,- /// 片段点的是另一个声明过的实体:改绑到它- Rebind(String),- /// 片段是围着某个名字的短语,那个名字在里面只是修饰语(后面还有词、或带所有格):- /// 描述,不是实体- Described(String),- /// 片段是所绑名字前面带了别的词:头衔("entrepreneur Tasha McCauley")还是另一件- /// 东西("companies using OpenAI"),机器分不开。绑定照旧,只记- Prefixed(String),- /// 片段里没有任何声明过的名字:模型消解了指代("him"、"the company")。绑定照旧,只记- Coreference(String),- /// 片段抄的是事实另一侧的名字(宾语片段写成了主语):抄错了位置。绑定照旧,只记- Misplaced(String),-}--/// 模型报给 `bound` 的别名,是不是已经声明成了另一个实体的名字(0041)。-///-/// 结构判据,不认词:同一个名字不会同时是两样东西的名字。模型把「海探1项目」列成-/// 一个机构,又把它报成探测器的别名——两个答案打架,别名那个不要-fn name_claimed_elsewhere(name: &str, bound: Uuid, declared: &HashMap<String, Uuid>) -> bool {- let key = utopia_store::resolution::normalize_name(name).to_lowercase();- declared.get(&key).is_some_and(|id| *id != bound)-}-/// 片段在不在引文里:大小写、空白都不论pub(crate) fn span_in_quote(span: &str, quote: &str) -> bool {let norm = |s: &str| {@@ -203,291 +110,6 @@ pub(crate) fn span_in_quote(span: &str, quote: &str) -> bool {!s.is_empty() && q.contains(&s)}-/// 一个词:去掉两头标点和所有格后的小写形态,连同原样(看大小写用)-#[derive(Debug, Clone)]-struct Word<'a> {- clean: String,- raw: &'a str,-}--/// 片段拆成词:去两头标点、去所有格("OpenAI's" → openai)、去开头的冠词-fn span_words(s: &str) -> Vec<Word<'_>> {- let mut words: Vec<Word<'_>> = s- .split_whitespace()- .filter_map(|raw| {- let w = raw.trim_matches(|c: char| !c.is_alphanumeric());- let w = w- .strip_suffix("'s")- .or_else(|| w.strip_suffix("\u{2019}s"))- .unwrap_or(w);- (!w.is_empty()).then(|| Word {- clean: w.to_lowercase(),- raw,- })- })- .collect();- while words- .first()- .is_some_and(|w| matches!(w.clean.as_str(), "the" | "a" | "an"))- {- words.remove(0);- }- words-}--/// 一个词是不是专名的样子:首字符大写或数字("PBC"、"LLC"、"LX"、"Global")-fn looks_proper(raw: &str) -> bool {- raw.chars()- .find(|c| c.is_alphanumeric())- .is_some_and(|c| c.is_uppercase() || c.is_numeric())-}--/// 词序列 needle 在 hay 里连续出现的位置-fn find_words(hay: &[Word<'_>], needle: &[Word<'_>]) -> Option<usize> {- if needle.is_empty() || needle.len() > hay.len() {- return None;- }- (0..=hay.len() - needle.len()).find(|&i| {- needle- .iter()- .zip(&hay[i..])- .all(|(n, h)| n.clean == h.clean)- })-}--/// 片段说的是不是这个名字:同名、同词干(Acme / Acme Corp.)、泛用后缀互推——-/// 与消解召回同一套判据(`recall_keys`);或者片段的词全在名字里("Altman" 之于-/// "Sam Altman","Anthropic" 之于 "Anthropic, PBC")-fn slot_matches(span: &str, name: &str) -> bool {- let keys = |s: &str| {- let clean = span_words(s)- .into_iter()- .map(|w| w.clean)- .collect::<Vec<_>>()- .join(" ");- utopia_store::resolution::recall_keys(&utopia_store::resolution::normalize_name(&clean))- };- let (a, b) = (keys(span), keys(name));- if !a.is_empty() && a.iter().any(|k| b.contains(k)) {- return true;- }- let (s, n) = (span_words(span), span_words(name));- !s.is_empty() && s.iter().all(|w| n.iter().any(|x| x.clean == w.clean))-}--/// **模型抄,机器判**(#582)。-///-/// #559、#578、#581 是同一件事的三张脸:模型把事实挂到了错的参与者身上——-/// "former OpenAI personnel" 写成 OpenAI,"lawsuit against OpenAI" 造成节点。给每一种-/// 形状写一条规则、配一张词表,量出来规则的服从率一半上下,词表只认见过的形状。-/// 这里换一个问法:不问模型「这算不算实体」,让它把引文里点名每一侧的那几个字-/// 抄出来(`subject_span` / `object_span`)。抄是模型稳定会做的事;判断交给机器:-///-/// 1. 片段是所绑的名字(同名、同词干、名字的一部分)→ 放行;-/// 2. 片段是另一个声明过的实体的名字(或那名字的一部分)→ 改绑;-/// 3. 片段里含所绑的名字、名字**后面**还有词("former OpenAI personnel"、"The Verge-/// reporter")或名字带所有格("Anthropic's safeguards")→ 名字在里面只是修饰语,-/// 这是描述。后面的词全是专名的样子("Anthropic PBC"、"OpenAI Global LLC")不算;-/// 4. 名字只在片段**末尾**、前面带了词 → 头衔还是另一件东西分不开,绑定照旧、只记;-/// 5. 片段里是别的声明过的名字带着修饰 → 描述;一个名字都没有 → 指代,绑定照旧、只记。-///-/// 按语法位置来的补充:名字后面紧跟逗号是同位语("Helen Toner, strategy director-/// for …"),还是它;片段以这条事实**另一侧**的名字结尾(宾语片段抄成了主语),是抄错-/// 位置,只记。-///-/// 同一套判据还用在模型**写的名字**和它 **ref 指的实体**之间(`written_verdict`):写-/// "OpenAI employees" 却 ref 到 OpenAI,是它自己的两个答案打架,v5 那轮的最后一条假边-/// ("Eleven employees left OpenAI … to establish Anthropic" → Anthropic founded_by OpenAI)-/// 就是这么来的——片段 "eleven employees" 里没有名字,只看片段拦不住。-///-/// 只有 3 和 5 前半改动事实(主语丢、宾语落字面值);其余都是信号,让每一层的比率-/// 按库、按模型读得出来-fn verify_span(- span: Option<&str>,- name: &str,- other: &str,- quote: &str,- declared: &HashMap<String, Uuid>,-) -> SpanVerdict {- let Some(span) = span.map(str::trim).filter(|s| !s.is_empty()) else {- return SpanVerdict::Ok;- };- if !span_in_quote(span, quote) {- return SpanVerdict::NotInQuote;- }- if slot_matches(span, name) {- return SpanVerdict::Ok;- }- let s = span_words(span);- let n = span_words(name);- if s.is_empty() {- return SpanVerdict::Coreference(span.to_string());- }-- // 2. 另一个声明过的实体:精确/词干命中优先,其次名字包住片段的(取最短的那个)- let mut others: Vec<&String> = declared.keys().filter(|k| k.as_str() != name).collect();- others.sort();- if let Some(k) = others.iter().find(|k| {- let keys = |x: &str| {- let clean = span_words(x)- .into_iter()- .map(|w| w.clean)- .collect::<Vec<_>>()- .join(" ");- utopia_store::resolution::recall_keys(&utopia_store::resolution::normalize_name(&clean))- };- let (a, b) = (keys(span), keys(k));- a.iter().any(|x| b.contains(x))- }) {- return SpanVerdict::Rebind((*k).clone());- }- // 单个词包在别的名字里不算改绑:"company" 是指代,句首的 "Stockholders" 大写也- // 不是专名的证据(召回测量台第二轮:主语从 NVIDIA 改绑到了「年度股东大会」)。- // 单个词只认精确/词干命中(上面 `slot_matches` 那一关)- let names_something = s.len() >= 2;- let mut supersets: Vec<(&String, usize)> = others- .iter()- .filter_map(|k| {- let kw = span_words(k);- (names_something && s.iter().all(|w| kw.iter().any(|x| x.clean == w.clean)))- .then_some((*k, kw.len()))- })- .collect();- supersets.sort_by_key(|(_, len)| *len);- if let Some((k, _)) = supersets.first() {- return SpanVerdict::Rebind((*k).clone());- }- // 所绑的名字在片段里的位置;名字后面紧跟逗号是同位语("TBPN, a media company in- // California"),还是它——先于下面所有判断- let found = find_words(&s, &n);- if let Some(i) = found {- let end = i + n.len();- if end < s.len() && s[end - 1].raw.trim_end().ends_with(',') {- return SpanVerdict::Ok;- }- }-- // 片段以事实另一侧的名字结尾:抄错了位置(宾语片段写成了主语),不是绑错了实体。- // 「片段以别的声明名结尾就改绑到它」试过两轮(v4/v5):介词的补语、名单的最后一项、- // 并列的后一个("Swisher and … Alex Heath")都会被当成头,错的多过对的,不做- let ends_with = |k: &str| {- let kw = span_words(k);- !kw.is_empty() && kw.len() < s.len() && find_words(&s[s.len() - kw.len()..], &kw) == Some(0)- };- if !other.is_empty() && !slot_matches(name, other) && ends_with(other) {- return SpanVerdict::Misplaced(span.to_string());- }-- // 3 / 4. 所绑的名字在片段里- if let Some(i) = found {- let end = i + n.len();- let last = s[end - 1].raw;- let possessive = last.ends_with("'s") || last.ends_with("\u{2019}s");- let after = &s[end..];- // 名字后面接着 and / & 再接专名,是并列("SB Energy and SoftBank"):名字是名单里- // 的一项,不是修饰语。连词跟冠词一样是语法词,不是词表- let after = match after.first() {- Some(w) if matches!(w.clean.as_str(), "and" | "&") => &after[1..],- _ => after,- };- if possessive || after.iter().any(|w| !looks_proper(w.raw)) {- return SpanVerdict::Described(span.to_string());- }- if after.is_empty() && i > 0 {- return SpanVerdict::Prefixed(span.to_string());- }- return SpanVerdict::Ok;- }-- // 5. 别的名字带着修饰,或一个名字都没有- if others.iter().any(|k| {- let kw = span_words(k);- find_words(&s, &kw).is_some()- }) {- return SpanVerdict::Described(span.to_string());- }- SpanVerdict::Coreference(span.to_string())-}--fn clause_suspect(name: &str) -> Option<&'static str> {- let words: Vec<String> = name- .split_whitespace()- .map(|w| {- w.trim_matches(|c: char| !c.is_alphanumeric())- .to_lowercase()- })- .collect();- // 限定词起头的长串:「The removal … industry as a whole」;机构名也会长,但- // 通常不以 the 起头(US District Court …),起头的(The New York Times)不长- if words.len() >= 8 && matches!(words[0].as_str(), "a" | "an" | "the") {- return Some("determiner_opens_a_long_string");- }- // 句中的关系词 / 从属连词:「the committee that reviewed …」- if words.len() >= 5- && words- .iter()- .skip(1)- .any(|w| matches!(w.as_str(), "that" | "which" | "who" | "because" | "while"))- {- return Some("relative_or_subordinate_clause");- }- None-}--fn is_entity_name(name: &str) -> bool {- let name = name.trim();- if name.is_empty() || name.chars().count() > 100 {- return false;- }- let raw: Vec<&str> = name.split_whitespace().collect();- let words: Vec<String> = raw- .iter()- .map(|w| {- w.trim_matches(|c: char| !c.is_alphanumeric())- .to_lowercase()- })- .collect();- if words.len() > MAX_NAME_WORDS {- return false;- }- // 一个词的名字不可能是从句,别让 "Is" 这种专名被误伤- if words.len() > 2 && words.iter().any(|w| CLAUSE_MARKERS.contains(&w.as_str())) {- return false;- }- if reads_like_a_sentence(&words, raw.last().copied()) {- return false;- }- // 部分格:`745 of OpenAI's 770 employees` 是一个数量描述,不是一个东西。- // 它没有限定动词,词数也不多,上面两条都接不住它。- //- // 判据收得很窄——**首词是纯数字且第二词是 of**。真实体里以数字开头的- //(`3M`、`7-Eleven`、`23andMe`)首词不是纯数字;`2023 Nobel Prize` 首词是纯数字,- // 但第二个词不是 `of`。宽一格就会误伤它们。- if words.len() >= 3- && words[1] == "of"- && !words[0].is_empty()- && words[0].chars().all(|c| c.is_ascii_digit())- {- return false;- }- /* **整体就是一个量的,不是一个东西**:`$5 billion`、`52%`、`3.5 million`。- 上面那条部分格只接住 `745 of …`,接不住这些。-- 这道闸装在**这里**才管用。抽取那边也有一道(`looks_literal`),但它前面- 挂着 `!known_predicate`:谓词一旦是本体里列出来的关系,整段判断直接跳过。- 实测就是这么漏的——`hasAmount` 来自 FIBO 包、抽取前就在本体里,于是- `Microsoft hasAmount $1 billion` 里那个数额照样被造成了节点,- 而同一个库里 `invested`(语料自己长的、当时还未知)走到了那道闸、被拦下。- `is_entity_name` 不问谓词,主语宾语一视同仁,两条路都过它。-- 判据仍旧从严(见 `parse_quantity`):尾巴上有实词就不算,- `3M`、`7-Eleven`、`23andMe`、`2025 Atlantic hurricane season` 一个都不误伤。 */- if utopia_extract::parse_quantity(name).is_some() {- return false;- }- true-}-/// 记一条丢弃信号。抽取器有七处 `continue`,每一处都是"事实抽出来了、被挡掉、/// 什么都不说"。信号写失败绝不能带垮整篇文档的抽取,所以这里吞掉错误。pub(crate) async fn drop_signal(@@ -541,30 +163,6 @@ pub(crate) fn incomplete_reason(unextracted: &[(i32, String)], attempted: usize)))}-/// 自动扩本体的唯一入队点。**成功与失败两条路都要走到它。**-///-/// 开关在这里重读,而不是沿用调用方手上那份:失败路径压根没加载过 kb,-/// 而成功路径那份是文档**开抽时**读的——一篇 73 块的文档要跑一个多小时,-/// 期间有人在设置里关掉了开关,沿用旧值就是拿一小时前的意图办事。-async fn enqueue_bootstrap(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> {- if !utopia_store::kbs::get(&state.pool, kb_id)- .await?- .auto_extend_ontology- {- return Ok(());- }- if !utopia_store::documents::extraction_idle(&state.pool, kb_id).await? {- return Ok(());- }- utopia_store::jobs::enqueue(- &state.pool,- "bootstrap_ontology",- serde_json::json!({ "kb_id": kb_id }),- )- .await?;- Ok(())-}-/// `proposer`:这篇文档若是记忆日志,抽出的事实等人点头,据此记下「谁说的」/// ——人,以及经 MCP 时那个 agent(0014 的令牌)。批量摄入的文档传默认值:/// 那条路不经待确认队列,这两位都用不上@@ -585,60 +183,12 @@ pub async fn extract_document(.await;if let Ok(doc) = utopia_store::documents::get(&state.pool, document_id).await {state.emit_document(doc.kb_id, document_id);- // **失败也要触发自动扩本体。**- //- // 入队从前只写在成功路径上,于是这一串会把知识库永久卡住:- // 前 14 篇成功(每篇都看到还有别的在飞,不触发),第 15 篇重试- // 耗尽变 failed —— 这时 extraction_idle 恰好为真(failed 不算- // queued/extracting),可**再没有任何一篇文档会完成来做这次检查**。- // 结果是提案堆在池子里、本体永远停在种子那几个关系、半张图永远是- // 兜底谓词,而界面上没有任何东西说这件事发生过。- //- // 任务本身幂等且会重查开关与门槛,所以这里多入队一次是安全的。- let _ = enqueue_bootstrap(state, doc.kb_id).await;}Err(e)}}}-/// mention → 实体 id。同一文档内同名同类型直接复用(单文档语境里罕有同名不同人,-/// 也把消解调用摊薄到每个名字一次);跨文档歧义由 resolve_mention 的画像比对处理。-#[allow(clippy::too_many_arguments)]-async fn resolve(- pool: &PgPool,- kb_id: Uuid,- // None = 模型给的类型不在本体里,或这个库根本还没有类(0009)- type_id: Option<Uuid>,- name: &str,- ctx: Option<&[f32]>,- // 本次提及所在分块的原文:画像分不开同名候选时的事实旁证(#331)- text: Option<&str>,- doc_cache: &mut HashMap<(Option<Uuid>, String), Uuid>,- needs_adjudication: &mut bool,-) -> anyhow::Result<Uuid> {- let key = (- type_id,- utopia_store::resolution::normalize_name(name).to_lowercase(),- );- if let Some(id) = doc_cache.get(&key) {- return Ok(*id);- }- let id = resolve_uncached(- pool,- kb_id,- type_id,- name,- ctx,- text,- &[],- needs_adjudication,- )- .await?;- doc_cache.insert(key, id);- Ok(id)-}-/// Resolve without the document's name cache. Handles use this path so two identities claimed/// separately in one response cannot collapse before their fact refs are bound.#[allow(clippy::too_many_arguments)]@@ -699,78 +249,6 @@ async fn create_namesake_reviews(Ok(created)}-#[derive(Clone, Copy)]-struct BoundEntity {- id: Uuid,- type_id: Option<Uuid>,-}--/// 模型写的名字和它 ref 指的实体对不对得上(#582)。没有 ref、或写的就是那个名字时-/// 无事;否则把写的名字当片段核对,写的名字自己当引文(免掉在不在引文那一关)。-/// 只有描述和改绑两种结论会用到,其余当无事-fn written_verdict(- written: &str,- bound: &str,- other: &str,- referenced: bool,- declared: &HashMap<String, Uuid>,-) -> SpanVerdict {- if !referenced || slot_matches(written, bound) {- return SpanVerdict::Ok;- }- match verify_span(Some(written), bound, other, written, declared) {- v @ (SpanVerdict::Described(_) | SpanVerdict::Rebind(_)) => v,- _ => SpanVerdict::Ok,- }-}--/// 一侧绑上的名字:有 ref 就是 ref 指的那个实体声明的名字,没有就是模型写的(#582)-fn bound_name<'a>(- ref_names: &'a HashMap<String, String>,- reference: Option<&str>,- written: &'a str,-) -> &'a str {- reference- .map(str::trim)- .and_then(|h| ref_names.get(h))- .map(String::as_str)- .unwrap_or(written)-}--fn referenced_entity(- ref_entities: &HashMap<String, BoundEntity>,- reference: &str,-) -> Option<BoundEntity> {- ref_entities.get(reference.trim()).copied()-}--#[derive(Debug, PartialEq, Eq)]-enum NoRefNameBinding {- Legacy(Uuid),- AmbiguousHandled,- Missing,-}--fn no_ref_name_binding(- entity_ids: &HashMap<String, Uuid>,- handled_by_name: &HashMap<String, Vec<Uuid>>,- name: &str,-) -> NoRefNameBinding {- let normalized = utopia_store::resolution::normalize_name(name).to_lowercase();- if handled_by_name- .get(&normalized)- .is_some_and(|ids| ids.len() > 1)- {- NoRefNameBinding::AmbiguousHandled- } else {- entity_ids- .get(name)- .copied()- .map(NoRefNameBinding::Legacy)- .unwrap_or(NoRefNameBinding::Missing)- }-}-#[allow(clippy::too_many_arguments)]pub(crate) async fn resolve_handle(pool: &PgPool,@@ -836,2614 +314,37 @@ pub(crate) async fn resolve_handle(.or_default().push(id);let document_claims = handled_by_name.entry(normalized).or_default();- if !document_claims.contains(&id) {- document_claims.push(id);- }- Ok(id)-}--#[allow(clippy::too_many_arguments)]-async fn resolve_bare(- pool: &PgPool,- kb_id: Uuid,- type_id: Option<Uuid>,- name: &str,- ctx: Option<&[f32]>,- text: Option<&str>,- doc_cache: &mut HashMap<(Option<Uuid>, String), Uuid>,- handled_by_name: &HashMap<String, Vec<Uuid>>,- ambiguous_bare_cache: &mut HashMap<String, Uuid>,- needs_adjudication: &mut bool,- human_reviews_found: &mut bool,-) -> anyhow::Result<Uuid> {- let normalized = utopia_store::resolution::normalize_name(name).to_lowercase();- let claims = handled_by_name- .get(&normalized)- .filter(|ids| ids.len() > 1)- .cloned();- let Some(claims) = claims else {- return resolve(- pool,- kb_id,- type_id,- name,- ctx,- text,- doc_cache,- needs_adjudication,- )- .await;- };- if let Some(id) = ambiguous_bare_cache.get(&normalized) {- return Ok(*id);- }-- // The text supplies no evidence for choosing among the handled namesakes. Preserve the- // fact on one document-scoped provisional entity and expose every possible identity link- // to a person. Subsequent bare mentions in this document reuse this provisional entity.- let id = resolve_uncached(- pool,- kb_id,- type_id,- name,- ctx,- text,- &claims,- needs_adjudication,- )- .await?;- if create_namesake_reviews(pool, kb_id, id, &claims).await? {- *human_reviews_found = true;- }- ambiguous_bare_cache.insert(normalized, id);- Ok(id)-}--async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow::Result<()> {- let doc = utopia_store::documents::get(&state.pool, document_id).await?;- // 排队之后被删了(#268):墓碑不抽——抽出来的事实会活在一个已删除的出处上- if doc.deleted_at.is_some() {- tracing::info!(document = %document_id, "skipping a deleted document");- return Ok(());- }- // **本体向量门控(#526)——把等待从 worker 里搬回队列。**- //- // 在这之前 `extraction::run` 直接调 `ontology_index::refresh` 等到补齐才动。- // 那次等待有三个坏处:它把 worker 槽占住,让同一批次的其余文档和别的库的- // 任务全卡在锁上;它让文档在这段时间里挂着 `extracting`,用户看见 32 篇- // 全在「抽取中」却没一个事实落库;它用一份可能没补齐的索引作依据,抽出来- // 的图是基于半个本体写的,再也不会被重抽。- //- // 换成队列内门控:本体超出提示词预算且需要嵌入时,先把 `embed_ontology`- // 排上、把这次抽取挂回 `queued` 等 30s,让 worker 槽立刻空出来——同一批- // 其余文档和其它库的任务都能继续认领。下次轮到这个文档时本体可能已就绪,- // 也可能还没,那就再等一轮。**不是同一个文档在等,是同一个抽取器在等**,- // 而等候归队列管,attempts 不烧。- //- // 没配嵌入模型的库不等,照旧送完整本体——那种部署本来就没有检索。- // 等也有期限(`jobs::DEFER_WINDOW_SECS`),补齐任务一直失败时这篇按失败处理。- let kb = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;- if !kb.open_extraction && ontology_index::gate_required(state, doc.kb_id).await? {- // gate_required 已经把 `embed_ontology` 入队过了(如果应该入队的话)。- // 这里只挂等待时长,不重复 enqueue。attempts 会被 mark_failed 退回去——- // 同一个等待条件两次排队不应该消耗两次预算。- let err = anyhow::Error::msg("waiting for ontology index to be embedded")- .context(utopia_core::Deferred::new(Duration::from_secs(30)));- return Err(err);- }- let settings = utopia_store::settings::get(&state.pool, kb.workspace_id)- .await?- .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?;- let client = llm_util::chat_client(&settings)- .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?;-- // 所有权凭证:重抽会自增 epoch,本任务据此察觉自己已被接管(见分块循环)- let my_epoch = utopia_store::documents::extract_epoch(&state.pool, document_id).await?;- utopia_store::documents::set_graph_status(&state.pool, document_id, "extracting").await?;- state.emit_document(doc.kb_id, document_id);- // **开放图谱**(0044 第 1 刀,#729):开关开着的库只写文档自己的话,本体不进提示词。- // 记忆日志也走这条路,只是它的陈述先进待确认表等人点头(0015),点头时才落成开放陈述- if kb.open_extraction {- let await_nod = utopia_store::memory::is_memory_document(&state.pool, document_id).await?;- return crate::extraction_open::run_open(- state, &doc, &kb, &settings, &client, my_epoch, proposer, await_nod,- )- .await;- }- let etypes = utopia_store::graph::entity_types(&state.pool, doc.kb_id).await?;- // 这一轮落过的事实(新建或重复观察):结尾对它们跑一遍签名检查- let mut touched_facts: Vec<Uuid> = Vec::new();- let mut rtypes = utopia_store::graph::relation_types(&state.pool, doc.kb_id).await?;- // 名字属性不进给模型的清单(0041):名字走回复里的 `names`,服务端核对它在原文里- rtypes.retain(|r| !utopia_store::names::is_name_attribute(r));- // 关系与属性分道:属性走字面值通道,不进关系清单。- //- // **本体里没有对应关系时就没有谓词**(见 `facts.predicate_id`)。原词落进- // fact_evidence.proposed_predicate,显示时由 fact_surface_predicate() 取回。- //- // 从前这里是一个叫 related_to 的兜底关系,且刻意不列给模型——它摆进提示词就成了- // 逃生舱,模型读到说不清的关系时不去写原文说法,直接挑这个万能选项。- // 现在它连行都没有了,逃生舱和"记得别列它"这两件事一起消失。- //- // **这两件事必须一起做,缺一件比都不做更糟。** 只删库里的行不够,还要删- // `DEFAULT_RELATION_TYPES` 里的种子——而这里的排除过滤已经跟着删了。于是- // `ensure_default_ontology` 七分钟后把行种回来,`related_to` 第一次被**列进- // 提示词给模型看**。0001 量过:359 次使用里 321 次是模型从清单上挑的。- // 谁要往种子表里加回一个兜底关系,先看 0010——不过那张表现在已经没有了- // (`#128`),连播种函数一起退场。- let type_key_by_id: HashMap<Uuid, &str> =- etypes.iter().map(|t| (t.id, t.key.as_str())).collect();- let attr_meta: HashMap<&str, &utopia_core::models::RelationType> = rtypes- .iter()- .filter(|r| r.kind == "attribute")- .map(|r| (r.key.as_str(), r))- .collect();- /* **边上的属性**(0037):一条关系声明过的属性定义,按关系 id 取。- 属性定义就是 kind='attribute' 的行,datatype / unit / 换算全复用;- 写入时按这里的 key 对模型给的 qualifiers,对不上的进丢弃表让人看见 */- let rtype_by_id: HashMap<Uuid, &utopia_core::models::RelationType> =- rtypes.iter().map(|r| (r.id, r)).collect();- let attr_by_key: HashMap<String, &utopia_core::models::RelationType> = rtypes- .iter()- .filter(|r| r.kind == "attribute")- .map(|r| (r.key.to_lowercase(), r))- .collect();- let mut qualifier_defs: HashMap<Uuid, Vec<&utopia_core::models::RelationType>> = rtypes- .iter()- .filter(|r| r.kind != "attribute" && !r.qualifiers.is_empty())- .map(|r| {- let defs = r- .qualifiers- .iter()- .filter_map(|q| rtype_by_id.get(q).copied())- .filter(|q| q.kind == "attribute")- .collect();- (r.id, defs)- })- .collect();- let type_ids: HashMap<&str, Uuid> = etypes.iter().map(|t| (t.key.as_str(), t.id)).collect();- let rel_ids: HashMap<&str, Uuid> = rtypes.iter().map(|r| (r.key.as_str(), r.id)).collect();- // 模型说出的谓词往本体已有关系上落:写法、时态、被动都对齐(见 predicate_match)。- // 没有它的时候,`produces` 明明在词表里,模型写 `produced_by` 就被降级扔了- let pred_index = PredicateIndex::build(&rtypes);- // 「本体认不认识这个说法」——字面值那一档与关系那一档必须用同一个判据。- // 分开写的话,模糊匹配得上的谓词会先被字面值那一档当成属性分流走,- // 同一个词在两条路上得到相反的回答- let known_predicate = |p: &str| rel_ids.contains_key(p) || pred_index.lookup(p).is_some();- // 时态对账只对带唯一性约束的状态关系生效(本体元数据):(functional, inverse_functional, temporal)- let rel_meta: HashMap<Uuid, (bool, bool, String)> = rtypes- .iter()- .map(|r| {- (- r.id,- (r.functional, r.inverse_functional, r.temporal.clone()),- )- })- .collect();- let type_parents: HashMap<Uuid, &[Uuid]> = etypes- .iter()- .map(|t| (t.id, t.parents.as_slice()))- .collect();-- // **本体全铺还是按分块检索。**- //- // 全铺是今天的行为,小本体下它对且便宜:40 个类约 2k 字符,检索反而是多余的- // 往返。大本体下它是灾难——schema.org 实测每个分块 108k tokens,而同一份语料- // 只给种子类时抽到 25 个实体、给全量时只剩 18 个。**多给的那 959 个类- // 吃掉了 7 个实体。**- //- // 所以按预算切换:装得下就全铺,装不下就每块检索。判据量的是**实际要排的- // 那段字**(build_lists 自己数),不是另写一个估算公式——公式会跟排版分叉。- let full = build_lists(&etypes, &rtypes, None, None);- let budget = utopia_store::access::ontology_prompt_budget(&state.pool).await?;- let retrieve_per_chunk = full.chars() > budget;- if retrieve_per_chunk {- tracing::info!(- %document_id, chars = full.chars(), budget,- classes = etypes.len(),- "本体超出提示词预算,改为按分块检索候选"- );- // 走到这里说明本体超出预算且按预算逻辑需要按块检索——但本任务的等待- // 早就在 `gate_required` 里完成(要么已经嵌好,要么通过 `Deferred` 挂回- // 队列)。剩下的就是按块检索本身,不再有「顺便 refresh」这一步:- // 那一步是在 worker 里等 PER_KB 锁,正是 #526 想消除的副作用。- //- // 留一个注释方便日后回看:若 budget 在 `gate_required` 与此处之间被改小,- // 本文档会按全量本体抽——比 #526 之前的「等几分钟」更接近「对的那一边」。- }- // 内置类恒在:检索漏掉的分块仍要有地方落脚,否则模型无类可选- let seed_classes: HashSet<Uuid> = etypes.iter().filter(|t| t.builtin).map(|t| t.id).collect();-- // 属性 domain 允许子类:主语类型沿 parent 链上溯命中 domain 即可- // 沿 subClassOf 上溯。**广度优先 + 访问集**,不是单链循环:- // 一个类可以有多个父(FOAF 的 Person 同时是 Agent 与 SpatialThing),- // 而菱形继承会从两条路到达同一个祖先,没有访问集就会重复展开。- //- // 深度上限换成了访问集:写入侧 set_parents 已经查环,这里再靠"最多走十层"- // 兜底既挡不住宽的图,也会悄悄放过深的层级。- let type_matches_domain = |ty: Uuid, domain: Uuid| -> bool {- let mut seen: HashSet<Uuid> = HashSet::new();- let mut queue = vec![ty];- while let Some(cur) = queue.pop() {- if cur == domain {- return true;- }- if !seen.insert(cur) {- continue;- }- if let Some(ps) = type_parents.get(&cur) {- queue.extend(ps.iter().copied());- }- }- false- };- // 本轮要从头讲一遍这篇文档的故事,旧信号先清掉(重抽自动作数)- let _ = utopia_store::extraction_drops::clear_for_document(&state.pool, document_id).await;-- // **记忆抽出的事实先等人点头**(0015)。一句 remember 一次一句、人就在对话里,- // 确认成本最低的时刻就是说完那句话的时候;而批量摄入一次上万条,逐条确认- // 不可能,那条路仍旧乐观写入 + 事后审阅。判据只有一个:这篇是不是记忆日志。- // 实体照常消解并创建——`pending_facts.subject_id` 是外键,这是 0018 定下的取舍- let await_nod = utopia_store::memory::is_memory_document(&state.pool, document_id).await?;- let mut pending_count = 0usize;-- let doc_time = doc.doc_time.map(|t| t.format("%Y-%m-%d").to_string());- let chunks = utopia_store::documents::chunks_for_extraction(&state.pool, document_id).await?;- // 文件开头:序号最小的现存分块(不是还没抽的第一块)。备忘文件一个片段一块,- // 前一段不是后一段的开头,不附- let opening_chunk = if await_nod {- None- } else {- utopia_store::documents::opening_chunk(&state.pool, document_id).await?- };-- let mut doc_cache: HashMap<(Option<Uuid>, String), Uuid> = HashMap::new();- // Identities introduced through handles, grouped only for detecting document-local- // namesake ambiguity. A later bare mention gets its own provisional entity instead of- // guessing among this group.- let mut handled_by_name: HashMap<String, Vec<Uuid>> = HashMap::new();- let mut ambiguous_bare_cache: HashMap<String, Uuid> = HashMap::new();- let mut touched_names: HashSet<String> = HashSet::new();- // 本文档已经认下的实体,按首次出现排序,送进后续分块的提示词。- //- // **按 entity_id 去重,不按名字**:第 3 块写"上海研究院"若消解到了第 1 块的- // "星云科技上海研究院",那它不该以第二个名字进清单——清单里每个实体只有- // 一个展示形态,就是这篇文档第一次用的那个。中文里全称先出现,所以这也是较全的那个。- let mut doc_entities: Vec<(Uuid, String, String)> = Vec::new();- // 整块没抽成的:(seq, 原因)。收尾时据此拒绝把这篇文档标成 done- let mut unextracted: Vec<(i32, String)> = Vec::new();- let mut needs_adjudication = false;- let mut human_reviews_found = false;- let mut conflicts_found = false;- let mut fact_count = 0usize;- // 不设分块上限:静默截断等于丢知识,长文档的成本由部署者自己权衡- // (成本优化走 prompt 前缀缓存与更新时 chunk 级跳过,而非丢数据)- for chunk in chunks.iter() {- // 被接管则安静退场:不写 failed、不碰状态,舞台留给新任务。- // 检查放在调用 LLM 之前——取消粒度即一个分块,不必等整篇跑完- if utopia_store::documents::extract_epoch(&state.pool, document_id).await? != my_epoch {- tracing::info!(%document_id, "抽取任务已被新一轮接管,退出");- return Ok(());- }- let ctx: Option<&[f32]> = chunk.embedding.as_ref().map(|v| v.as_slice());- // 本体装得下就用全量那份;装不下就拿**这一块自己的向量**检索候选。- // 向量是现成的——实体消解本来就在用它(上面那个 ctx),检索一次- // 嵌入都不用加。检索不出来(没配嵌入模型、或这块没向量)就退回全量:- // 提示词大是慢,没有类可选是抽不出东西- let lists = if retrieve_per_chunk {- match ctx {- Some(v) => {- chunk_lists(state, doc.kb_id, v, &etypes, &rtypes, &seed_classes, budget)- .await- .unwrap_or(None)- }- None => None,- }- } else {- None- };- let lists = lists.as_ref().unwrap_or(&full);- let known: Vec<utopia_extract::KnownEntity> = doc_entities- .iter()- .enumerate()- .map(|(index, (_, type_key, name))| utopia_extract::KnownEntity {- handle: format!("k{}", index + 1),- type_key: type_key.clone(),- name: name.clone(),- })- .collect();- // 这一块就是开头本身时不再重复一遍- let opening = opening_chunk- .as_ref()- .filter(|(id, _)| *id != chunk.id)- .map(|(_, text)| text.as_str());- let messages = utopia_extract::build_messages_with_opening(- &lists.types,- &lists.relations,- &lists.attributes,- doc_time.as_deref(),- &doc.filename,- &known,- opening,- &chunk.text,- );- // 这两处 continue 跳过的是**整个分块**——它一条事实都没产出。- // 记下来,收尾时据此决定这篇文档算不算抽完(见循环之后)- let reply = match chat_retrying_rate_limits(state, &settings, &client, &messages).await {- Ok(r) => r,- Err(e) => {- tracing::warn!(%document_id, seq = chunk.seq, error = %e, "抽取调用失败,跳过该分块");- unextracted.push((chunk.seq, format!("调用失败:{e}")));- continue;- }- };- // 模型的原话只在 debug 级别看得到:查它对哪几个字段怎么填(#582 的片段)时开- tracing::debug!(%document_id, seq = chunk.seq, reply = %reply, "抽取原始回复");- let mut extraction = match utopia_extract::parse_response(&reply) {- Ok(x) => x,- Err(e) => {- tracing::warn!(%document_id, seq = chunk.seq, error = %e, "抽取结果解析失败,跳过该分块");- unextracted.push((chunk.seq, format!("结果解析失败:{e}")));- continue;- }- };- // **跳过了什么必须说出来。** 逐项解析救回了整块,但被跳过的那几条- // 如果不落信号,就成了另一种「部分抽取报告成完成」(#108 修过一次)- if extraction.truncated {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::TRUNCATED_REPLY,- &format!("分块 #{} 的输出被截断", chunk.seq),- None,- )- .await;- }- let skipped = extraction.skipped_entities + extraction.skipped_facts;- if skipped > 0 {- tracing::warn!(- %document_id,- seq = chunk.seq,- entities = extraction.skipped_entities,- facts = extraction.skipped_facts,- "跳过了结构不合的条目"- );- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::MALFORMED_ITEM,- &format!(- "分块 #{} 跳过 {} 个实体 / {} 条事实",- chunk.seq, extraction.skipped_entities, extraction.skipped_facts- ),- None,- )- .await;- }-- // 实体消解:名称 → 实体 id(本分块的事实按原文名字连线)- // **落库前先查形状**(utopia_extract::normalize):只看结构、不看词——引文里有没有- // 这段字、值是不是只有标点、一侧是不是契约的日期、同句有没有另一条边。读懂时间- // 归模型(提示词 3c),这里只核对它照没照契约写,做了什么都记进丢弃表- let from_opening = match opening {- Some(text) => {- utopia_extract::drop_quotes_from_opening(&mut extraction, &chunk.text, text)- }- None => Vec::new(),- };- for n in from_opening- .into_iter()- .chain(utopia_extract::normalize_facts(&mut extraction))- {- use utopia_extract::Normalization as N;- use utopia_store::extraction_drops::reason;- let (r, detail, example) = match n {- N::NoValue { predicate, written } => (reason::NO_VALUE, predicate, written),- N::ValueTrimmed {- predicate,- kept,- dropped,- } => (- reason::VALUE_TRIMMED,- predicate,- format!("{kept} ✂ {dropped}"),- ),- N::QualifiersWithoutObject { predicate, values } => (- reason::QUALIFIERS_WITHOUT_OBJECT,- predicate,- format!("{values} value(s) moved onto the subject"),- ),- N::TimeAsObject {- predicate,- written,- values,- } => (- reason::TIME_AS_OBJECT,- predicate,- if values == 0 {- format!("{written} kept as a value")- } else {- format!("{written} → {values} value(s)")- },- ),- N::TimeAsSubject { predicate, written } => {- (reason::TIME_AS_SUBJECT, predicate, written)- }- N::ObjectDescribesDeclared {- predicate,- name,- head,- } => (- reason::OBJECT_DESCRIBES_DECLARED,- predicate,- format!("{name} ← {head}"),- ),- N::OrphanDeclaration { name } => {- (reason::ORPHAN_DECLARATION, "entity".to_string(), name)- }- N::QuoteFromOpening { predicate, quote } => {- (reason::QUOTE_FROM_OPENING, predicate, quote)- }- };- drop_signal(state, doc.kb_id, document_id, r, &detail, Some(&example)).await;- }- let mut entity_ids: HashMap<String, Uuid> = HashMap::new();- // 名称 → 声明类型(属性 domain 校验用:salary 不能挂在 Organization 上)- let mut entity_type_of: HashMap<String, Option<Uuid>> = HashMap::new();- let mut ref_entities: HashMap<String, BoundEntity> = doc_entities- .iter()- .enumerate()- .map(|(index, (id, type_key, _))| {- (- format!("k{}", index + 1),- BoundEntity {- id: *id,- type_id: type_ids.get(type_key.as_str()).copied(),- },- )- })- .collect();- // 句柄 → 声明的名字:片段核对要对着**绑上的**那个名字看(#582)。模型写- // "OpenAI personnel" 却 ref 到 e1=OpenAI 时,错在 ref 上,片段对着 "OpenAI"- // 才看得出来- let mut ref_names: HashMap<String, String> = doc_entities- .iter()- .enumerate()- .map(|(index, (_, _, name))| (format!("k{}", index + 1), name.clone()))- .collect();- let mut response_claims: HashMap<String, Vec<Uuid>> = HashMap::new();-- // Resolve handled definitions first. This matters for a mixed response: a later- // handle-less same-name item must see the ambiguity rather than winning by array order.- for e in extraction- .entities- .iter()- .filter(|e| e.local_id.is_some())- .chain(extraction.entities.iter().filter(|e| e.local_id.is_none()))- {- let name = e.name.trim();- if !is_entity_name(name) {- // 从前这里是静默 `continue`——正是 `drop_signal` 当初为之而建的- // 那种"抽出来了、被挡掉、什么都不说"- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::NOT_AN_ENTITY_NAME,- &e.type_key,- Some(name),- )- .await;- continue;- }- // 守卫放行、结构却像从句:只记不挡(#193 先攒标注集)- if let Some(signal) = clause_suspect(name) {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::CLAUSE_SUSPECT,- signal,- Some(name),- )- .await;- }- if let Some(handle) = e.local_id.as_deref().map(str::trim) {- if ref_entities.contains_key(handle) {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::MALFORMED_ITEM,- "local_id collides with a known handle",- Some(handle),- )- .await;- continue;- }- }- // 降级时记住模型提议的那个词:本体装不下不等于它说错了。- // 只留计数的话,日后想加 model 类就找不出那 43 个实体——它们混在- // concept 里面,唯一的出路是整库重抽- let mut proposed: Option<&str> = None;- let type_id = match type_ids.get(e.type_key.as_str()) {- Some(id) => Some(*id),- None => {- // 白名单外类型:**留空**,并记入未匹配统计(本体扩展的信号)。- //- // 从前这里降级到 concept 那行哨兵。现在「还没判出来」就是- // `type_id IS NULL`(0009)——实体照常建、事实照常落、证据照常有,- // 只是暂时没有类型。之后装一个包再跑类型消解,它会被重新分配- let _ = utopia_store::ontology::record_miss(- &state.pool,- doc.kb_id,- "entity_type",- &e.type_key,- Some(name),- )- .await;- proposed = Some(e.type_key.as_str());- None- }- };- let normalized = utopia_store::resolution::normalize_name(name).to_lowercase();- touched_names.insert(normalized.clone());- let id = if let Some(handle) = e.local_id.as_deref() {- let handle = handle.trim();- let id = resolve_handle(- &state.pool,- doc.kb_id,- type_id,- name,- ctx,- Some(&chunk.text),- &mut response_claims,- &mut handled_by_name,- &mut ambiguous_bare_cache,- &mut needs_adjudication,- &mut human_reviews_found,- )- .await?;- ref_entities.insert(handle.to_string(), BoundEntity { id, type_id });- ref_names.insert(handle.to_string(), name.to_string());- id- } else {- resolve_bare(- &state.pool,- doc.kb_id,- type_id,- name,- ctx,- Some(&chunk.text),- &mut doc_cache,- &handled_by_name,- &mut ambiguous_bare_cache,- &mut needs_adjudication,- &mut human_reviews_found,- )- .await?- };- if let Some(p) = proposed {- let _ = utopia_store::resolution::set_proposed_type(&state.pool, id, p).await;- }- // 模型写下的这个名字就在这一块原文里时,给这条名字事实补出处(0041)。- // 给已知句柄时它照提示词写的是清单上的全称,这一块里未必有——那就不补,- // 这一块用的别的写法走下面的 `names`。- // 记忆日志里的不补:那一句算不算出处,要等人点头(0018)- if !await_nod && span_in_quote(name, &chunk.text) {- let _ = utopia_store::names::record(- &state.pool,- doc.kb_id,- id,- name,- Some(utopia_store::names::NameSource {- chunk_id: chunk.id,- quote: name,- }),- doc.doc_time,- )- .await;- }- // 模型自己的说法。**跟 proposed_type 分开存**:那一列的含义是- // "本体里没有",增长回路靠它的稀有性设门槛;这一列每个实体都有。- // 与粗类同名的不记——那不是更具体的说法,只是把清单抄了一遍- if let Some(st) = e- .specific_type- .as_deref()- .map(str::trim)- .filter(|s| !s.is_empty())- .filter(|s| !s.eq_ignore_ascii_case(&e.type_key))- {- let _ = utopia_store::resolution::set_specific_type(&state.pool, id, st).await;- }- // 只记模型自己声明过类型的:主宾兜底那条路没有类型可依,- // 把一个猜出来的类型放进清单等于让后续分块照着猜的抄。- //- // 本体装不下那个类时用模型自己的说法(proposed):这份清单是给后文- // 认人用的,"同一个名字别写成两个实体"才是它的活。从前这里只能写死- // concept,反倒把几个不同的词抹平成同一个标签- if !doc_entities.iter().any(|(eid, _, _)| *eid == id) {- let tk = type_id- .and_then(|t| type_key_by_id.get(&t).copied())- .or(proposed)- .unwrap_or("?");- doc_entities.push((id, tk.to_string(), name.to_string()));- }- // Legacy name maps retain their old last-write-wins behavior. Handled facts bind- // through ref_entities; inserting handled definitions here only supports mixed- // model output when the surface name is unambiguous.- entity_ids.insert(name.to_string(), id);- entity_type_of.insert(name.to_string(), type_id);- }-- // 实体的别的名字(0041 决定 2)。**名字与引文都要在这一块原文里**:名字是召回的桥,- // 一座凭空造的桥会把两个不相干的实体接到一起。认不认「简称」「又名」是模型的事,- // 服务端不认词,只核对它抄的字是不是真在原文里- // 这次回复声明的名字,加上本文档前面几块认下的:别名撞上它们之一就不收- let declared_names: HashMap<String, Uuid> = entity_ids- .iter()- .map(|(name, id)| (name.as_str(), *id))- .chain(- doc_entities- .iter()- .map(|(id, _, name)| (name.as_str(), *id)),- )- .map(|(name, id)| {- (- utopia_store::resolution::normalize_name(name).to_lowercase(),- id,- )- })- .collect();- for n in &extraction.names {- let name = n.name.trim();- let Some(bound) = ref_entities.get(n.entity_ref.trim()) else {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::MALFORMED_ITEM,- "name ref is not a declared handle",- Some(name),- )- .await;- continue;- };- let quote = n- .quote- .as_deref()- .map(str::trim)- .filter(|q| !q.is_empty())- .unwrap_or(name);- if name_claimed_elsewhere(name, bound.id, &declared_names) {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::NAME_CLAIMED_BY_ANOTHER,- &n.entity_ref,- Some(name),- )- .await;- continue;- }- if !is_entity_name(name)- || !span_in_quote(name, quote)- || !span_in_quote(quote, &chunk.text)- {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::NAME_NOT_IN_TEXT,- &n.entity_ref,- Some(name),- )- .await;- continue;- }- // 记忆日志读到的名字与它的其它事实一样等人点头(0018):名字是召回的桥,- // 一句没确认过的话不该先把桥搭上。点头之后是一条普通的名字事实;- // 这一步不配对,同名的配对等下一次在文档里读到它- if await_nod {- let known_as = utopia_store::names::ensure_known_as(&state.pool, doc.kb_id).await?;- let value = serde_json::json!({- "value": utopia_store::resolution::normalize_name(name)- });- if let utopia_store::pending::Outcome::Proposed(_) = utopia_store::pending::propose(- &state.pool,- utopia_store::pending::Proposal {- kb_id: doc.kb_id,- subject_id: bound.id,- predicate_id: Some(known_as),- object_id: None,- object_value: Some(&value),- proposed_predicate: Some(utopia_store::names::KNOWN_AS),- validity: utopia_store::graph::Validity {- attested_at: doc.doc_time,- ..Default::default()- },- confidence: 1.0,- chunk_id: chunk.id,- proposed_by: proposer.user_id,- proposed_token: proposer.token_id,- phrase: None,- qualifiers: None,- time_words: None,- quote_span: None,- },- )- .await?- {- pending_count += 1;- }- continue;- }- utopia_store::names::record(- &state.pool,- doc.kb_id,- bound.id,- name,- Some(utopia_store::names::NameSource {- chunk_id: chunk.id,- quote,- }),- doc.doc_time,- )- .await?;- // 别的实体已经叫这个名字:送去裁决,不合并- if utopia_store::names::pair_shared_name(&state.pool, doc.kb_id, bound.id, name).await?- > 0- {- needs_adjudication = true;- }- }-- // 改绑的候选:这次回复声明的实体,加上提示词里给过的库内实体(#582)- let span_declared: HashMap<String, Uuid> = {- let mut m = entity_ids.clone();- for (id, _, name) in &doc_entities {- m.entry(name.clone()).or_insert(*id);- }- m- };- for f in &extraction.facts {- let confidence = f.confidence.unwrap_or(0.7).clamp(0.0, 1.0);- if confidence < MIN_CONFIDENCE {- // 设计上的阈值,但用户同样无从知道"抽到了,只是不够自信"- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::LOW_CONFIDENCE,- &f.predicate,- Some(&format!("{} ({:.0}%)", f.subject, confidence * 100.0)),- )- .await;- continue;- }- let confidence = origin_ceiling(&chunk.origin, confidence);- let validity =- validity_of(f.valid_from.as_deref(), f.valid_to.as_deref(), doc.doc_time);-- // 属性事实:谓词命中属性 → 字面值通道。datatype 校验失败宁缺勿脏;- // domain 校验(含子类上溯)挡住"把 salary 挂到 Organization"这类张冠李戴。- // 模型偶尔照抄清单里的 "person.salary" 全限定名——剥掉类前缀再查一次- let attr_hit = attr_meta.get(f.predicate.as_str()).or_else(|| {- f.predicate- .rsplit_once('.')- .and_then(|(_, k)| attr_meta.get(k))- });- if let Some(attr) = attr_hit {- let subject_name = f.subject.trim();- // 主语没在 entities 里声明:类型不明,domain 无从校验,属性不落。- // 关系路径遇到同样的缺失会兜底按 concept 消解——这里学不来,- // 按 concept 解出来 domain 照样不匹配,只是从这里掉进下面那一档- let bound = match f.subject_ref.as_deref().map(str::trim) {- Some(handle) => referenced_entity(&ref_entities, handle),- None => {- match no_ref_name_binding(&entity_ids, &handled_by_name, subject_name) {- NoRefNameBinding::Legacy(id) => entity_type_of- .get(subject_name)- .copied()- .map(|type_id| BoundEntity { id, type_id }),- NoRefNameBinding::Missing => None,- NoRefNameBinding::AmbiguousHandled => {- let type_id = entity_type_of.get(subject_name).copied().flatten();- touched_names.insert(- utopia_store::resolution::normalize_name(subject_name)- .to_lowercase(),- );- let id = resolve_bare(- &state.pool,- doc.kb_id,- type_id,- subject_name,- ctx,- Some(&chunk.text),- &mut doc_cache,- &handled_by_name,- &mut ambiguous_bare_cache,- &mut needs_adjudication,- &mut human_reviews_found,- )- .await?;- Some(BoundEntity { id, type_id })- }- }- }- };- let Some(BoundEntity {- id: subject_id,- type_id: subject_type,- }) = bound- else {- drop_signal(- state,- doc.kb_id,- document_id,- if f.subject_ref.is_some() {- utopia_store::extraction_drops::reason::MALFORMED_ITEM- } else {- utopia_store::extraction_drops::reason::SUBJECT_NOT_DECLARED- },- &attr.key,- Some(f.subject_ref.as_deref().unwrap_or(subject_name)),- )- .await;- continue;- };- // 不可达:store 层强制 attribute 必有 domain,且 domain 不可改,- // 无 domain 的属性连提示词都进不去。留着是防御,不需要信号- if attr.domains.is_empty() {- continue;- }- // **任一 domain 命中即可**:属性挂在多个类下时,主语属于其中之一就算数- if !attr- .domains- .iter()- .any(|d| subject_type.is_some_and(|t| type_matches_domain(t, *d)))- {- let subj_key = subject_type- .and_then(|t| type_key_by_id.get(&t).copied())- .unwrap_or("?");- let dom_key = attr- .domains- .iter()- .filter_map(|d| type_key_by_id.get(d).copied())- .collect::<Vec<_>>()- .join("|");- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::ATTR_DOMAIN_MISMATCH,- &format!("{}@{subj_key} (wants {dom_key})", attr.key),- Some(subject_name),- )- .await;- continue;- }- let raw = match (&f.value, &f.object) {- (Some(v), _) => v.clone(),- // 模型偶尔把值放进 object:宽容接住- (None, Some(o)) if !o.trim().is_empty() => {- serde_json::Value::String(o.trim().to_string())- }- _ => {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::ATTR_NO_VALUE,- &attr.key,- Some(subject_name),- )- .await;- continue;- }- };- let datatype = attr.datatype.as_deref().unwrap_or("text");- // 只相对一件事给出的日期(「触发日后 45 天」,#681 §4)照原文收下、带着标记:- // 它是新的状态值,时态引擎照常用它接替前一个截止日- let Some(mut object_value) =- utopia_extract::attr_object_value(datatype, &raw, f.relative)- else {- tracing::debug!(%document_id, attr = attr.key, ?raw, "属性值不合 datatype,跳过");- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::ATTR_DATATYPE,- &format!("{} ({datatype})", attr.key),- Some(&format!("{subject_name} → {raw}")),- )- .await;- continue;- };- // 单位随事实落笔:类型上的单位以后改了,旧值仍按记录时的单位读。- // 记哪个单位照 `unit_for`——从前这里无条件盖上声明的单位,实测- //「提供 500 兆瓦的风电」被模型记成金额,再盖上 ¥ 就成了 500 块钱- if let Some(u) = unit_for(&raw, datatype, None, attr.unit.as_deref()) {- object_value["unit"] = serde_json::json!(u);- }- if await_nod {- if let utopia_store::pending::Outcome::Proposed(_) =- utopia_store::pending::propose(- &state.pool,- utopia_store::pending::Proposal {- kb_id: doc.kb_id,- subject_id,- predicate_id: Some(attr.id),- object_id: None,- object_value: Some(&object_value),- proposed_predicate: Some(f.predicate.as_str()),- validity,- confidence,- chunk_id: chunk.id,- proposed_by: proposer.user_id,- proposed_token: proposer.token_id,- phrase: None,- qualifiers: None,- time_words: None,- quote_span: None,- },- )- .await?- {- pending_count += 1;- }- continue;- }- let (fact_id, created) = utopia_store::graph::insert_value_fact(- &state.pool,- doc.kb_id,- subject_id,- Some(attr.id),- &object_value,- validity,- confidence,- )- .await?;- touched_facts.push(fact_id);- // 属性谓词也留原词:模型偶尔照抄 "person.salary" 全限定名,- // 命中的是剥掉前缀后的 key,原样是什么值得留着- utopia_store::graph::add_evidence(- &state.pool,- fact_id,- chunk.id,- f.quote.as_deref(),- Some(f.predicate.as_str()),- )- .await?;- if created {- fact_count += 1;- }- // 单值属性 = functional:新值闭合旧值(属性历史由此而来)。并进已有断言的也对:- // 这份证据的日期可能更早,时间线的形状跟着变(#679)- if attr.functional && attr.temporal == "state" {- let report = utopia_store::temporal::reconcile_new_fact(- &state.pool,- doc.kb_id,- fact_id,- subject_id,- attr.id,- None,- Some(&object_value),- utopia_store::temporal::Uniqueness::SubjectSide,- validity,- confidence,- )- .await?;- if report.conflicts > 0 {- conflicts_found = true;- }- }- continue;- }-- // **词表外的字面值:既不丢,也不给它编一个实体。**- //- // 走到这里说明谓词既不是已知属性也还没查关系表。它带着字面值时- // 有两种走法,从前两种都不好:- // value 有而 object 空 → 掉进下面的"宾语必填",整条静默消失;- // object 里塞着字面值 → 按 concept 消解,凭空造出一个叫「2015」- // 的实体,图里多一个假节点,事后再修还得改事实的形状。- // 现在都存成 object_value 且没有谓词:值在图里、有证据、有时态,- // 原词进 proposed_predicate,消解那一遍只需换谓词,形状已经是对的。- //- // object 里的东西算不算字面值,判据从严:**模型没把它声明成实体**,- // 且**它整体就是一个量或一个日期**。"杭州"两条都不满足,"2015"、- // "$5 billion"、"52%" 都满足;"900 million weekly active users"- // 不满足——尾巴上还有实词,它说的就不再只是那个数了。- // 文本值的属性(schema.org 里 323 个)在这一档仍会变成实体——- // 那里没有可靠判据,猜错会吃掉真实体,不猜- // 第二格是表层谓词:通常就是模型写的谓词;值旁边挂着一个没声明的宾语短语时,- // 短语并进来(见下面那一档)- let literal: Option<(serde_json::Value, String)> =- match (&f.value, f.object.as_deref().map(str::trim)) {- // **给了值、没给宾语——不管这个谓词本体认不认识。**- //- // 从前这里卡着 `!known_predicate`:`job_title` 在 schema.org 里是关系- //(它的 range 是 `Text|DefinedTerm`,含一个类就走关系通道),于是模型- // 写 `job_title` + "founder and CEO" 时既进不了属性档、又在关系档因为- // 缺宾语被丢掉——`object_missing` 实测 69 次,四篇文档里每个人的职务- // 就是这么没的。谓词认不认识与「这条事实带的是值还是实体」无关:- // 值在手上就收下,原词进 proposed_predicate,等本体采纳时再换谓词,- // 形状已经是对的(0010)- (Some(v), None | Some("")) => Some((v.clone(), f.predicate.clone())),- /* **宾语整体是一个量:一律当值收下**,不问谓词认不认识、- 也不问模型有没有把它声明成实体。-- 下面那一档卡着 `!known_predicate`,理由是本体说得上话的时候- 别去二猜模型。可量值这里没有可猜的余地:一个数额不会因为- 谓词恰好在本体里就变成一个东西。实测漏的正是这一格——- `hasAmount` 来自 FIBO 包、抽取前就在本体里,- `Microsoft hasAmount $1 billion` 于是绕过下面那一档,- 把数额造成了节点;同一个库里 `invested` 当时还未知,- 走到下面那一档、被拦住了。同一个数额,两种下场。-- 收下而不是丢掉:`is_entity_name` 那道闸现在也拦纯量值,- 不在这里接住的话,这条事实会连同那个数一起进丢弃表。- 原词照旧进 `proposed_predicate`,采纳时再换谓词 */- (_, Some(o)) if utopia_extract::parse_quantity(o).is_some() => Some((- serde_json::Value::String(o.to_string()),- f.predicate.clone(),- )),- /* **给了值、宾语却是一个没声明的短语:值收下,短语并进表层谓词**(#685)。-- `build` + 宾语「new energy generation」+ 值「at least 10 GW」:宾语不是- 回复里声明的实体、没有句柄、也不是本文档认下的名字,于是上面几档都接- 不住,关系那条路又因为它不是实体把整条丢掉——那个数跟着没了。模型把- 同一句话写成纯值(「at least 10 GW of new energy generation」)时就落得- 下,落不落全看它挑了两种同样说得通的写法里的哪一种。-- 只看结构:值在、宾语没有句柄、宾语不在已声明的名字里。宾语是个认得的- 实体时照旧走边。短语不丢,进表层谓词(`build new energy generation`),- 采纳时再换谓词;原句在证据里 */- (Some(v), Some(o))- if undeclared_beside_value(f.object_ref.as_deref(), o, &span_declared) =>- {- Some((v.clone(), format!("{} {o}", f.predicate.trim())))- }- (_, Some(o))- if !o.is_empty()- && !known_predicate(f.predicate.as_str())- && !entity_ids.contains_key(o)- && looks_literal(o) =>- {- Some((- serde_json::Value::String(o.to_string()),- f.predicate.clone(),- ))- }- _ => None,- };- if let Some((value, surface)) = literal {- let subject_name = f.subject.trim();- // **主语按关系那条路解,不要求它在本次回复里重新声明过。**- //- // 从前这里只查 `entity_ids`,模型用「已知实体」句柄带进来的、- // 或者只写了名字没重列的主语一律落空——实测一轮 51 块里- // `subject_not_declared` 丢掉 113 条,丢的是 NVIDIA 的营收、- // 净利、每股收益,是十位董事的赞成票与反对票,全是有名有姓的- // 主语。属性那一档要求主语有类型(domain 要校验),这一档没有- // domain 可校验,也就没有理由比关系那条路更严- let subject_id = match f.subject_ref.as_deref().map(str::trim) {- Some(handle) => match referenced_entity(&ref_entities, handle) {- Some(bound) => bound.id,- None => {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::MALFORMED_ITEM,- &f.predicate,- Some(handle),- )- .await;- continue;- }- },- None => {- match no_ref_name_binding(&entity_ids, &handled_by_name, subject_name) {- NoRefNameBinding::Legacy(id) => id,- NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {- touched_names.insert(- utopia_store::resolution::normalize_name(subject_name)- .to_lowercase(),- );- resolve_bare(- &state.pool,- doc.kb_id,- entity_type_of.get(subject_name).copied().flatten(),- subject_name,- ctx,- Some(&chunk.text),- &mut doc_cache,- &handled_by_name,- &mut ambiguous_bare_cache,- &mut needs_adjudication,- &mut human_reviews_found,- )- .await?- }- }- }- };- let _ = utopia_store::ontology::record_miss(- &state.pool,- doc.kb_id,- "attribute_type",- &surface,- Some(&format!("{subject_name} → {value}")),- )- .await;- /* 值照原文落笔(提示词 8a 要的就是「units and all」),**单位另记一格**。- 采纳成属性时按 datatype 把 `$5 billion` 换算成 5e9,那一步只看得懂- 数;符号丢在原文里就再也取不出来了,而「5000000000」少了那个 `$`- 就不知道是钱还是别的什么 */- let unit = value- .as_str()- .and_then(utopia_extract::parse_quantity)- .and_then(|(_, u)| u);- let mut literal = serde_json::json!({ "value": value });- if let Some(unit) = unit {- literal["unit"] = serde_json::Value::String(unit);- }- let literal = literal;- if await_nod {- if let utopia_store::pending::Outcome::Proposed(_) =- utopia_store::pending::propose(- &state.pool,- utopia_store::pending::Proposal {- kb_id: doc.kb_id,- subject_id,- predicate_id: None,- object_id: None,- object_value: Some(&literal),- proposed_predicate: Some(surface.as_str()),- validity,- confidence,- chunk_id: chunk.id,- proposed_by: proposer.user_id,- proposed_token: proposer.token_id,- phrase: None,- qualifiers: None,- time_words: None,- quote_span: None,- },- )- .await?- {- pending_count += 1;- }- continue;- }- let (fact_id, created) = utopia_store::graph::insert_value_fact(- &state.pool,- doc.kb_id,- subject_id,- None,- &literal,- validity,- confidence,- )- .await?;- touched_facts.push(fact_id);- utopia_store::graph::add_evidence(- &state.pool,- fact_id,- chunk.id,- f.quote.as_deref(),- Some(surface.as_str()),- )- .await?;- if created {- fact_count += 1;- }- continue;- }-- // 关系事实:宾语必填- let Some(object_name) = f.object.as_deref().map(str::trim).filter(|s| !s.is_empty())- else {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::OBJECT_MISSING,- &f.predicate,- Some(f.subject.trim()),- )- .await;- continue;- };- // **未声明的主宾也要过同一道判据。**- //- // 从前守卫只装在上面那条声明实体的路上,而这里绕过了它:模型把一整句话- // 写进 `object`、那句话没出现在 entities 里,这里就转头把它造成了实体。- // 实测(ai-timeline-ends × schema.org)421 个实体里 76 个无类型,- // 最长的那个 111 字符——"thermal-imaging equipment used by volunteers- // flying over the site showed at least 33 generators giving off heat",- // 那是一整个从句,不是一个东西。**守卫拦住了前门,后门是开的。**- //- // 这类实体的害处不止于此:它们永远匹配不到别处的任何提及,- // 在图上是孤点(实测 59 个),还会拖累消解——每一个都要跟已有实体比一遍。- if !is_entity_name(f.subject.trim()) || !is_entity_name(object_name) {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::NOT_AN_ENTITY_NAME,- &f.predicate,- Some(if is_entity_name(f.subject.trim()) {- object_name- } else {- f.subject.trim()- }),- )- .await;- continue;- }- for side in [f.subject.trim(), object_name] {- if let Some(signal) = clause_suspect(side) {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::CLAUSE_SUSPECT,- signal,- Some(side),- )- .await;- }- }-- // **槽位片段核对**(#582,取代 #578 的词表):模型交出引文里点名每一侧的那几个- // 字,机器核对(`verify_span`)。描述做主语不落、做宾语落成字面值;点了别的- // 实体就改绑;其余都只记不拦- let quote_text = f.quote.as_deref().unwrap_or("");- // 对着绑上的名字看:有 ref 就是 ref 指的那个实体的名字,没有就是模型写的名字- let bound_subject = bound_name(&ref_names, f.subject_ref.as_deref(), f.subject.trim());- let bound_object = bound_name(&ref_names, f.object_ref.as_deref(), object_name);- // 片段说了算;片段没说清(放行、不在引文、前缀、指代)时,再看模型写的名字- // 跟它 ref 指的实体对不对得上- let decisive = |v: &SpanVerdict| {- matches!(- v,- SpanVerdict::Described(_) | SpanVerdict::Rebind(_) | SpanVerdict::Misplaced(_)- )- };- let mut subject_verdict = verify_span(- f.subject_span.as_deref(),- bound_subject,- bound_object,- quote_text,- &span_declared,- );- if !decisive(&subject_verdict) {- let w = written_verdict(- f.subject.trim(),- bound_subject,- bound_object,- f.subject_ref.is_some(),- &span_declared,- );- if decisive(&w) {- subject_verdict = w;- }- }- let mut object_verdict = verify_span(- f.object_span.as_deref(),- bound_object,- bound_subject,- quote_text,- &span_declared,- );- if !decisive(&object_verdict) {- let w = written_verdict(- object_name,- bound_object,- bound_subject,- f.object_ref.is_some(),- &span_declared,- );- if decisive(&w) {- object_verdict = w;- }- }- let mut rebound_subject: Option<String> = None;- let mut rebound_object: Option<String> = None;- let mut object_described: Option<String> = None;- let mut subject_described = false;- for (side, verdict, span, bound) in [- (- "subject",- &subject_verdict,- f.subject_span.as_deref(),- bound_subject,- ),- (- "object",- &object_verdict,- f.object_span.as_deref(),- bound_object,- ),- ] {- let (reason, example) = match verdict {- SpanVerdict::Ok => continue,- SpanVerdict::NotInQuote => (- utopia_store::extraction_drops::reason::SPAN_NOT_IN_QUOTE,- format!("{} ({bound})", span.unwrap_or("")),- ),- SpanVerdict::Rebind(name) => {- if side == "subject" {- rebound_subject = Some(name.clone());- } else {- rebound_object = Some(name.clone());- }- (- utopia_store::extraction_drops::reason::SPAN_REBOUND,- format!("{} → {name} (was {bound})", span.unwrap_or("")),- )- }- SpanVerdict::Described(text) => {- if side == "subject" {- subject_described = true;- (- utopia_store::extraction_drops::reason::SUBJECT_DESCRIBED,- format!("{text} ({bound})"),- )- } else {- object_described = Some(text.clone());- (- utopia_store::extraction_drops::reason::OBJECT_DESCRIBED,- format!("{text} ({bound})"),- )- }- }- SpanVerdict::Prefixed(text) => (- utopia_store::extraction_drops::reason::SPAN_PREFIXED,- format!("{text} ({bound})"),- ),- SpanVerdict::Coreference(text) => (- utopia_store::extraction_drops::reason::SPAN_COREFERENCE,- format!("{text} ({bound})"),- ),- SpanVerdict::Misplaced(text) => (- utopia_store::extraction_drops::reason::SPAN_MISPLACED,- format!("{text} ({bound})"),- ),- };- drop_signal(- state,- doc.kb_id,- document_id,- reason,- &f.predicate,- Some(&example),- )- .await;- }- if subject_described {- continue;- }- let subject_name: &str = rebound_subject.as_deref().unwrap_or(f.subject.trim());- let subject_ref_eff: Option<&str> = if rebound_subject.is_some() {- None- } else {- f.subject_ref.as_deref().map(str::trim)- };- let object_name: &str = rebound_object.as_deref().unwrap_or(object_name);- let object_ref_eff: Option<&str> =- if rebound_object.is_some() || object_described.is_some() {- None- } else {- f.object_ref.as_deref().map(str::trim)- };- // 主宾没在 entities 里声明时(模型偶尔漏报):库里已经有这个名字的就用它,- // 库里也没有的**不建**(#559)。从前这里一律先建出来、类型留空,结果- // 一个库里 20% 的实体是 "lawsuit against OpenAI"、"$5 billion"、"March 2024"- // 这样的描述——几乎全部只当过宾语,从没当过主语。漏报的实体多半在- // 前面的分块或别的文档里已经声明过,按名字找得到;找不到的就是描述。- // 描述做主语的事实丢掉并记账,做宾语的事实照落,宾语落成字面值- let subject_id = match subject_ref_eff {- Some(handle) => match referenced_entity(&ref_entities, handle) {- Some(bound) => bound.id,- None => {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::MALFORMED_ITEM,- &f.predicate,- Some(handle),- )- .await;- continue;- }- },- None => {- let binding = no_ref_name_binding(&entity_ids, &handled_by_name, subject_name);- if matches!(binding, NoRefNameBinding::Missing)- && utopia_store::resolution::existing_by_name(- &state.pool,- doc.kb_id,- subject_name,- )- .await?- .is_none()- {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::SUBJECT_NOT_DECLARED,- &f.predicate,- Some(subject_name),- )- .await;- continue;- }- match binding {- NoRefNameBinding::Legacy(id) => id,- NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {- touched_names.insert(- utopia_store::resolution::normalize_name(subject_name)- .to_lowercase(),- );- resolve_bare(- &state.pool,- doc.kb_id,- None,- subject_name,- ctx,- Some(&chunk.text),- &mut doc_cache,- &handled_by_name,- &mut ambiguous_bare_cache,- &mut needs_adjudication,- &mut human_reviews_found,- )- .await?- }- }- }- };- let object_id = match object_ref_eff {- Some(handle) => match referenced_entity(&ref_entities, handle) {- Some(bound) => bound.id,- None => {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::MALFORMED_ITEM,- &f.predicate,- Some(handle),- )- .await;- continue;- }- },- None => {- let binding = no_ref_name_binding(&entity_ids, &handled_by_name, object_name);- if object_described.is_some()- || (matches!(binding, NoRefNameBinding::Missing)- && utopia_store::resolution::existing_by_name(- &state.pool,- doc.kb_id,- object_name,- )- .await?- .is_none())- {- // 没声明、库里也没有,或者片段说它是个描述:宾语落成字面值。谓词照旧对本体;- // 被动形(`_by`)本该主宾对调,而字面值当不了主语,那条就不给谓词,- // 原词留在证据上- let predicate_id = match pred_index.lookup(f.predicate.as_str()) {- Some((id, false)) => Some(id),- Some((_, true)) => None,- None => {- let _ = utopia_store::ontology::record_miss(- &state.pool,- doc.kb_id,- "relation_type",- &f.predicate,- Some(&format!("{} → {}", f.subject, object_name)),- )- .await;- None- }- };- let literal_text: &str = object_described.as_deref().unwrap_or(object_name);- // 描述那一路在上面核对时已经记过账;这里只记「没声明」的- if object_described.is_none() {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::OBJECT_UNDECLARED,- &f.predicate,- Some(object_name),- )- .await;- }- let literal = serde_json::json!({ "value": literal_text });- if await_nod {- if let utopia_store::pending::Outcome::Proposed(_) =- utopia_store::pending::propose(- &state.pool,- utopia_store::pending::Proposal {- kb_id: doc.kb_id,- subject_id,- predicate_id,- object_id: None,- object_value: Some(&literal),- proposed_predicate: Some(f.predicate.as_str()),- validity,- confidence,- chunk_id: chunk.id,- proposed_by: proposer.user_id,- proposed_token: proposer.token_id,- phrase: None,- qualifiers: None,- time_words: None,- quote_span: None,- },- )- .await?- {- pending_count += 1;- }- continue;- }- let (fact_id, created) = utopia_store::graph::insert_value_fact(- &state.pool,- doc.kb_id,- subject_id,- predicate_id,- &literal,- validity,- confidence,- )- .await?;- touched_facts.push(fact_id);- utopia_store::graph::add_evidence(- &state.pool,- fact_id,- chunk.id,- f.quote.as_deref(),- Some(f.predicate.as_str()),- )- .await?;- if created {- fact_count += 1;- }- continue;- }- match binding {- NoRefNameBinding::Legacy(id) => id,- NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => {- touched_names.insert(- utopia_store::resolution::normalize_name(object_name)- .to_lowercase(),- );- resolve_bare(- &state.pool,- doc.kb_id,- None,- object_name,- ctx,- Some(&chunk.text),- &mut doc_cache,- &handled_by_name,- &mut ambiguous_bare_cache,- &mut needs_adjudication,- &mut human_reviews_found,- )- .await?- }- }- }- };- if subject_id == object_id {- continue;- }- // 先尽量落到本体已有的关系上(写法/时态/被动),**落不上才降级**为 related_to- // 并记入未匹配统计。降级会把原意抹平成"有关联"——原词写进证据行的- // proposed_predicate,是这条事实身上唯一还留着原意的地方(谓词消解据此映射回本体)- let (predicate_id, swap) = match pred_index.lookup(f.predicate.as_str()) {- Some((id, swap)) => (Some(id), swap),- None => {- let _ = utopia_store::ontology::record_miss(- &state.pool,- doc.kb_id,- "relation_type",- &f.predicate,- Some(&format!("{} → {}", f.subject, object_name)),- )- .await;- // 本体里没有对应的关系 → **就是没有谓词**(见 `facts.predicate_id`)。- // 原意留在证据的 proposed_predicate 里,显示时取回。- // 从前这里落到 related_to 上,还要额外担心"兜底关系被删了"——- // 那条失败模式连同它的 continue 一起消失了- (None, false)- }- };- // 被动说法命中的是同一条边的反向:`ChatGPT produced_by OpenAI` 与- // `OpenAI produces ChatGPT` 是同一条边,存的时候要按本体的方向来,- // 否则它跟已有的那 130 条 produces 各存各的,图上是两条相反的箭头- let (subject_id, object_id) = if swap {- (object_id, subject_id)- } else {- (subject_id, object_id)- };-- // **主语违反 domain、而宾语符合时,按本体声明的方向把它掰正。**- //- // 先试过提示词,三轮都没赢:违反率从 57% 压到 35%,但压下去的全是- // 类型判错那一半;**真·反向纹丝不动**(22.7% → 17.1% → 17.6%,- // 后两个在噪声里)。模型看得见 `employee (organization → person)`,- // 就是不照做——英语的 "X is an employee of Y" 太强。- //- // 这不是新原则:`produced_by` 命中 `produces` 时(见上面那个 `swap`)- // 早就在自动翻转主宾了,区别只在触发条件是**措辞**还是**签名**。- //- // 当初反对自动对调的理由是实体类型不可靠——实测 Elon Musk 被判成- // `researcher`。那个前提已经不成立:祖先地板与签名类恒在修好之后,- // 同一批人判成了 `person`。而判据本身很窄——**主语违反且宾语符合**,- // 两侧都要对上才动。- //- // **但绝不静默。** 掰正要留信号:0001 反对的是「用可能错的声明驱动- // 自动动作」,而看得见、可复查、可反悔的动作不属于那一类。- let (predicate_id, subject_id, object_id) = if let Some(pid) = predicate_id {- // 类型**从库里的实体读**,不用抽取器手上那份 `entity_type_of`:- // 那份只覆盖模型在这一块里声明过的实体,而宾语常是别处已存在的实体,- // 这一块没重新声明它,于是查不到、判不了、掰不动。实测差别不小——- // 用声明那份时反向只降到 7.0%,剩下的正是宾语类型查不到的那些- //- // 判断本身在 store(`ontology::judge_direction`),**与采纳共用**(#190):- // 写谓词的路不止这一条,守卫只装在一条上就等于没装。查不出来(库错)- // 按没有判据处理,照原样落——宁可少掰一条,不能因为一次查询失败丢事实- let fit = utopia_store::ontology::judge_direction(- &state.pool,- pid,- subject_id,- object_id,- )- .await- .unwrap_or(utopia_store::ontology::Fit::Unchecked);- match fit {- utopia_store::ontology::Fit::Swap => {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::DIRECTION_CORRECTED,- &f.predicate,- Some(&format!(- "{} → {} 按签名掰正为 {} → {}",- f.subject,- f.object.as_deref().unwrap_or("?"),- f.object.as_deref().unwrap_or("?"),- f.subject- )),- )- .await;- (Some(pid), object_id, subject_id)- }- utopia_store::ontology::Fit::Neither => {- // **对调也不合法 → 退回没有谓词。**- //- // 这不是方向问题,是这个关系压根不适用:schema.org 的- // `affectedBy` 是医学检验用的,模型要表达「受……影响」时按名字- // 撞了上来;`amount` 属于融资工具而不是公司,模型没造那个中间- // 节点就把边挂到了公司上。- //- // 从前照原样落库,等于**用本体的名义说一件本体不同意的事**——- // 图上写着 "OpenAI affectedBy …",读者会以为那是一条医学断言。- // 这是自信的错误,比空谓词严重得多。- //- // 退回空谓词不丢信息:原词落进 `fact_evidence.proposed_predicate`,- // 显示时由 `fact_surface_predicate()` 取回(0010)。主宾、时间、- // 证据全都留着,只是不再冒认一个本体关系。**诚实的沉默。**- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::DOMAIN_MISMATCH,- &f.predicate,- Some(&format!(- "{} — {} → 主宾都对不上,退回原文说法",- f.subject, f.predicate- )),- )- .await;- (None, subject_id, object_id)- }- utopia_store::ontology::Fit::Keep | utopia_store::ontology::Fit::Unchecked => {- (Some(pid), subject_id, object_id)- }- }- } else {- (predicate_id, subject_id, object_id)- };-- if await_nod {- if let utopia_store::pending::Outcome::Proposed(_) = utopia_store::pending::propose(- &state.pool,- utopia_store::pending::Proposal {- kb_id: doc.kb_id,- subject_id,- predicate_id,- object_id: Some(object_id),- object_value: None,- proposed_predicate: Some(f.predicate.as_str()),- validity,- confidence,- chunk_id: chunk.id,- proposed_by: proposer.user_id,- proposed_token: proposer.token_id,- phrase: None,- qualifiers: None,- time_words: None,- quote_span: None,- },- )- .await?- {- pending_count += 1;- }- continue;- }- {- let (fact_id, created) = utopia_store::graph::insert_fact(- &state.pool,- doc.kb_id,- subject_id,- predicate_id,- object_id,- validity,- confidence,- )- .await?;- touched_facts.push(fact_id);- /* **边上的属性落笔**(0037)。属性不进去重键:`insert_fact` 复用了旧行也照写——- 同一条边再听到一次带了金额的,是同一条边补上金额。- 值照 datatype 换算,单位另记一格(与 object_value 同形);- key 不在声明里、换不动、与已记的不一致——三种都进丢弃表,不静默 */- /* 谓词未知(0010,说法在证据上)时属性照写:值落在事实上,声明等- 关系被采纳时在 `adopt` 里补。不写的话,`invested_in` 在 schema.org- 库里是未知说法,一句话里的 $1.5 billion 就没有地方放——召回台上- `oh-invest` 那一条正是这么丢的 */- if let Some(quals) = f.qualifiers.as_ref() {- let pid = predicate_id;- // 克隆出这一组引用:下面撞上已有属性时要往 qualifier_defs 里追加声明- let defs: Vec<&utopia_core::models::RelationType> = pid- .and_then(|p| qualifier_defs.get(&p).cloned())- .unwrap_or_default();- /* 模型常把币种单独写成一个键(`"amount": "1500000000", "currency": "CNY"`),- 而不是写进数额里。那不是一个属性,是数额的单位——先把它拿出来,- 数值属性解不出单位时用它,别让它作为未知 key 进丢弃表 */- let sibling_currency: Option<&'static str> = quals- .iter()- .find(|(k, _)| {- matches!(- k.trim().to_lowercase().as_str(),- "currency" | "币种" | "货币" | "unit" | "单位"- )- })- .and_then(|(_, v)| v.as_str())- .and_then(utopia_extract::currency_unit);- for (key, raw) in quals {- // 模型对没提到的属性会写 null:那是「原文没说」,不是坏值,不记- if raw.is_null() {- continue;- }- if matches!(- key.trim().to_lowercase().as_str(),- "currency" | "币种" | "货币" | "unit" | "单位"- ) {- continue;- }- let declared = defs- .iter()- .find(|q| q.key.eq_ignore_ascii_case(key.trim()))- .copied();- /* **未知 key 撞上本库已有的属性定义 → 补一条声明,不丢值。**- 实测不声明时模型照样写 `amount`、`stake`、`round`,八条全进- 丢弃表——而这三个属性定义明明都在库里,缺的只是关系上的一条- 声明。补声明不新建任何东西、可撤(本体页取消勾选即可),- 所以跟自动扩本体走同一个开关 */- let adopted = match declared {- Some(d) => Some(d),- /* 谓词还未知(0010,说法在证据上):没有关系可声明,值绑到库里- 已有的属性定义、先落在事实上,采纳时 `adopt` 再补声明。这一步- 不动本体,所以不看自动扩本体的开关 */- None if pid.is_none() => {- attr_by_key.get(&key.trim().to_lowercase()).copied()- }- None if kb.auto_extend_ontology => {- match attr_by_key.get(&key.trim().to_lowercase()).copied() {- Some(attr) => {- let pid = pid.expect("checked above");- match utopia_store::ontology::add_relation_qualifier(- &state.pool,- doc.kb_id,- pid,- attr.id,- )- .await- {- Ok(()) => {- tracing::info!(kb_id = %doc.kb_id, relation = %f.predicate, qualifier = %attr.key, "边上的属性按语料补了声明");- qualifier_defs.entry(pid).or_default().push(attr);- Some(attr)- }- Err(e) => {- tracing::warn!(kb_id = %doc.kb_id, error = %e, "补声明失败");- None- }- }- }- None => {- tracing::debug!(kb_id = %doc.kb_id, relation = %f.predicate, key = %key, attrs = attr_by_key.len(), "边上的属性:key 撞不上本库任何属性定义");- None- }- }- }- None => {- tracing::debug!(kb_id = %doc.kb_id, relation = %f.predicate, key = %key, auto_extend = kb.auto_extend_ontology, "边上的属性:未声明且不自动扩本体");- None- }- };- let Some(def) = adopted else {- /* **绑不上属性定义的数也不丢。** 库里没有这个属性(schema.org 里- `amount` 是关系不是属性)、或本体冻着不让扩——从前这里进丢弃表,- 数就只剩丢弃表里的一个样例。现在照 8a 的样子落成主语上的一条- 字面值事实:值按原文、单位另记一格、原词 `关系.键` 进证据的- proposed_predicate,缺的定义记进 ontology_misses(0010 的样子),- 采纳时人来决定它归哪儿。图里有它、有证据、能查到 */- let wording = format!("{}.{}", f.predicate.trim(), key.trim());- let unit = raw- .as_str()- .and_then(utopia_extract::parse_leading_quantity)- .and_then(|(_, u)| u)- .or_else(|| sibling_currency.map(str::to_string));- let mut literal = serde_json::json!({ "value": raw });- if let Some(u) = unit {- literal["unit"] = serde_json::Value::String(u);- }- let _ = utopia_store::ontology::record_miss(- &state.pool,- doc.kb_id,- "attribute_type",- &wording,- Some(&format!("{} → {raw}", f.subject.trim())),- )- .await;- let (literal_id, _) = utopia_store::graph::insert_value_fact(- &state.pool,- doc.kb_id,- subject_id,- None,- &literal,- validity,- confidence,- )- .await?;- touched_facts.push(literal_id);- utopia_store::graph::add_evidence(- &state.pool,- literal_id,- chunk.id,- f.quote.as_deref(),- Some(wording.as_str()),- )- .await?;- tracing::debug!(kb_id = %doc.kb_id, wording = %wording, "边上的属性绑不上定义,落成主语上的字面值");- continue;- };- let dt = def.datatype.as_deref().unwrap_or("text");- let Some(normalized) = utopia_extract::normalize_attr_value(dt, raw) else {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::QUALIFIER_DATATYPE,- &format!("{}.{} ({dt})", f.predicate, def.key),- Some(&raw.to_string()),- )- .await;- continue;- };- let mut value = serde_json::json!({ "value": normalized });- if let Some(u) = unit_for(raw, dt, sibling_currency, def.unit.as_deref()) {- value["unit"] = serde_json::Value::String(u);- }- let write = utopia_store::graph::upsert_fact_qualifier(- &state.pool,- fact_id,- def.id,- &value,- )- .await?;- if write == utopia_store::graph::QualifierWrite::Conflict {- drop_signal(- state,- doc.kb_id,- document_id,- utopia_store::extraction_drops::reason::QUALIFIER_CONFLICT,- &format!("{}.{}", f.predicate, def.key),- Some(&raw.to_string()),- )- .await;- }- }- }- // 重复观察也要挂证据:多来源相互印证,任一来源删除后事实不孤儿化。- // 表层谓词随每次观察落笔——甲块说 "runs on"、乙块说 "optimized for"- // 会并进同一条事实,放事实上就是先写者胜,放证据上两个都留着- utopia_store::graph::add_evidence(- &state.pool,- fact_id,- chunk.id,- f.quote.as_deref(),- Some(f.predicate.as_str()),- )- .await?;- if created {- fact_count += 1;- }- // 时态对账:带唯一性约束的状态关系落新事实即检测矛盾(纯规则点查,- // 自动闭合走"作废+改写",拿不准进 fact_conflicts 人裁)。并进已有断言的- // 也对:多了一份证据,时间线的形状可能跟着变(#679)- // 没有谓词就没有关系元数据,也就不参与时态对账——- // 一条说不出是什么关系的边,本来就不可能带唯一性约束- if let Some((pid, (func, inv_func, temporal))) =- predicate_id.and_then(|id| rel_meta.get(&id).map(|m| (id, m)))- {- if temporal == "state" {- let mut directions = Vec::new();- if *func {- directions.push(utopia_store::temporal::Uniqueness::SubjectSide);- }- if *inv_func {- directions.push(utopia_store::temporal::Uniqueness::ObjectSide);- }- for dir in directions {- let report = utopia_store::temporal::reconcile_new_fact(- &state.pool,- doc.kb_id,- fact_id,- subject_id,- pid,- Some(object_id),- None,- dir,- validity,- confidence,- )- .await?;- if report.conflicts > 0 {- conflicts_found = true;- }- }- }- }- }- }-- // 本块抽取完成即打标:更新时被认领的块携带标记跳过;中断的抽取可续跑- // (LLM 调用/解析失败的块在上方 continue 掉,不打标,下次重试)- utopia_store::documents::mark_chunk_extracted(&state.pool, chunk.id).await?;- }- // 队列里多了东西才叫醒人:Review 的计数与对话里那张确认卡都靠这一声- if pending_count > 0 {- tracing::info!(%document_id, pending_count, "记忆抽出的事实进了待确认队列");- state.emit_pending(doc.kb_id);- state.emit_review(doc.kb_id);- }-- // 消歧后缀在实体创建时算会早于其事实写入——收尾时对本文档涉及的名字统一刷新- touched_names.extend(doc_cache.keys().map(|(_, name)| name.clone()));- for name in &touched_names {- utopia_store::resolution::refresh_disambiguators(&state.pool, doc.kb_id, name).await?;- }-- // 出口再验一次:接管可能发生在最后一个分块之后,那时循环里的检查已经跑完。- // 漏掉这里,被顶替的任务会把 done 写在一篇 extracted_at 刚被清空的文档上——- // 界面显示"已完成",实则一条都没抽,要等新任务开跑才纠正回来。- if utopia_store::documents::extract_epoch(&state.pool, document_id).await? != my_epoch {- tracing::info!(%document_id, "抽取任务已被新一轮接管,收尾时退出");- return Ok(());- }-- // **有分块没抽成就不许标 done。**- //- // 从前这里无条件写 done:一次网络抖动让六篇文档 60 块里只抽成 12 块,- // 六篇全部显示"抽取完成",八成的内容没进图,而界面上没有任何东西说出来。- // 失败只进了日志,而只进日志的错误等于没有错误。- //- // 返回 Err 之后这条链是完整的:`extract_document` 落 graph_failed + 原因,- // 界面上那篇文档变成可点开看错误的 failed;任务按 30s×attempts² 退避重试,- // 而已抽成的分块带着 extracted_at 会被跳过——所以重试很便宜,网络恢复就自愈。- // 重试耗尽才留在 failed,那时它说的是实话。- //- // 判据是"全抽完"而不是某个比例:任何比例都是拍的,而这里本来就有一个- // 不需要拍的判据——**每一块都成了才叫抽完**。- if let Some(msg) = incomplete_reason(&unextracted, chunks.len()) {- return Err(anyhow::anyhow!(msg));- }- // 刚落的事实立刻过一遍签名。写入时只掰方向(judge_direction);掰不动的- // ——两个方向都对不上、或宾语没类型判不了——从前要等人按 Review 里的- // Run check 才露面,Axioms 一直是 0,图里却躺着反向事实(#222)。- // 检查失败不影响抽取本身:事实已经在库里,下一次 Run check 仍然查得到- if !touched_facts.is_empty() {- match utopia_store::reasoning::signature_breaks(- &state.pool,- doc.kb_id,- Some(&touched_facts),- )- .await- {- Ok(broken) if !broken.is_empty() => {- match utopia_store::reasoning::record_signature_breaks(- &state.pool,- doc.kb_id,- &broken,- )- .await- {- Ok(_) => state.emit_review(doc.kb_id),- Err(e) => {- tracing::warn!(%document_id, error = %e, "抽取后的签名违规没记进队列")- }- }- }- Ok(_) => {}- Err(e) => tracing::warn!(%document_id, error = %e, "抽取后的签名检查失败"),- }- }- utopia_store::documents::set_graph_status(&state.pool, document_id, "done").await?;- state.emit_document(doc.kb_id, document_id);-- // 灰区对进了审核队列 → 触发攒批裁决任务(独立后台跑,抽取本身到此已完成)。- // 治理开着(0025)排的是治理任务:灰区对与直接转人工的对都从那条先进先出的- // 队列走,先读台账再裁;同库已排着的不重复- if kb.governance {- if needs_adjudication || human_reviews_found {- utopia_store::jobs::enqueue_unless_queued(- &state.pool,- "govern",- serde_json::json!({ "kb_id": doc.kb_id }),- )- .await?;- }- } else if needs_adjudication {- // 同库已排着的不重复——与下面的 resolve_types 一样。一批文档同时抽完- // 会各排一个,而它们读到的是同一批待裁项- utopia_store::jobs::enqueue_unless_queued(- &state.pool,- "adjudicate_entities",- serde_json::json!({ "kb_id": doc.kb_id }),- )- .await?;- }- if needs_adjudication || human_reviews_found || conflicts_found {- state.emit_review(doc.kb_id);- }- // 类型消解排队自动跑(0016 C2):开关开着就排一个库级任务,同库已排着的不重复。- // 任务自己只看引擎没看过的实体、只自动落地子树内精化的那一档- if kb.auto_type_resolution {- utopia_store::jobs::enqueue_unless_queued(- &state.pool,- "resolve_types",- serde_json::json!({ "kb_id": doc.kb_id }),- )- .await?;- }- // 自动扩本体:开关开着、且这一批都抽完了,由最后一篇触发。- // 判据是显式开关而不是"本体有没有被碰过"——后者是从行为推断意图,- // 推错的后果很荒唐(在提案上点一次 Add 就永久关掉建议),而且一旦为假- // 就永不再真,本体会冻结在第一批文档碰巧包含的词汇上。- // 并发下可能入队两次,任务自己会重查开关与状态- enqueue_bootstrap(state, doc.kb_id).await?;-- tracing::info!(%document_id, facts = fact_count, "图谱抽取完成");- Ok(())-}--/// 一条事实同时给了值和宾语时,宾语是不是一个**没声明的短语**(#685):没有句柄,-/// 也不是本次回复或本文档前面认下的名字(大小写不计)。是的话这条事实按值落,-/// 宾语短语并进表层谓词;不是的话宾语是个实体,照旧走边-fn undeclared_beside_value(- object_ref: Option<&str>,- object: &str,- declared: &HashMap<String, Uuid>,-) -> bool {- let object = object.trim();- !object.is_empty()- && object_ref.map(str::trim).is_none_or(str::is_empty)- && !declared- .keys()- .any(|name| name.trim().eq_ignore_ascii_case(object))-}--/// 宾语位上的这串东西,是不是一个字面值而不是实体的名字。-///-/// **只认数字与日期。** 这是个会吃掉真实体的判断,所以宁可漏认:-/// 漏了不过是维持今天的行为(造一个 concept 实体),认错了却是把一个-/// 真实体降成一段文本,图里少一个节点。-///-/// "2015"、"2023-03"、"6" 认;"杭州"、"首席技术官"、"3M"、"V3" 不认。-/// 调用方还额外要求模型**没有**把它声明成实体——两道门一起过才算数。-/// 模型给的区间两端 → 落库的有效区间。-///-/// **两端各记各的粒度**(见 `facts.valid_to_precision`)。从前一个精度列描述两个端点,-/// 于是「2020 年开始、2023-05-06 结束」这种只能共用一个值。两端都用 `read_time` 读:-/// 规则 3 的格式,或写法说得清是哪天的日期(#688),精度随写了几位。-///-/// 模型给的 valid_to = "unknown" 表示**原文说它结束了、但没说哪天**。-/// read_time 解不出它(本来就不是日期),落在这里显式认掉——-/// 不认的话它退化成 None,那条事实就又变回"仍在持续"了-fn validity_of(- valid_from: Option<&str>,- valid_to: Option<&str>,- doc_time: Option<chrono::DateTime<chrono::Utc>>,-) -> utopia_store::graph::Validity<'static> {- let from = valid_from.and_then(utopia_extract::read_time);- let to = valid_to.and_then(utopia_extract::read_time);- let ended_unknown = valid_to- .map(str::trim)- .is_some_and(|v| v.eq_ignore_ascii_case(utopia_store::graph::ENDED_UNKNOWN));- utopia_store::graph::Validity {- from: from.map(|(t, _)| t),- from_precision: from.map(|(_, p)| p),- to: to.map(|(t, _)| t),- to_precision: to- .map(|(_, p)| p)- .or(ended_unknown.then_some(utopia_store::graph::ENDED_UNKNOWN)),- // 这次观察出自哪一天的文档(0022):没起点的事实从它起成立,结束了- // 不知哪天的到它为止。没有文档日期就是记下的此刻——账本能给的最好的- attested_at: doc_time,- }-}--fn looks_literal(s: &str) -> bool {- let s = s.trim();- if s.is_empty() {- return false;- }- /* **整体是一个量**:可选货币符号 + 数字 + 可选量级词 + 可选百分号,- 此外一个词都不许有(判据与例子见 `parse_quantity`)。-- 从前这里只认裸数字(`s.parse::<f64>()`),于是 `$5 billion` 两头不着:- 它不是裸数字、也解不成日期,掉进关系那条路,凭空长出一个叫「$5 billion」- 的节点。同名的又会并成一个点,于是 SSI Inc. 与 Nvidia 因为都出现过这个- 数额而在图上相连——一条没有含义的路径。实测一个 1415 实体的库里,8 个- 这样的点、15 条事实指着它们,而**没有任何一条拿它们当主语**:- 一个从不当主语、只当宾语、名字整体是个量的东西,是值,不是实体。-- `parse_quantity` 已经把裸数字那一档包含在内(`"42"` → 42),- 所以这里不必再单留一条。全角「2015」仍旧解不动,仍旧是想要的 */- if utopia_extract::parse_quantity(s).is_some() {- return true;- }- // 日期:复用抽取侧那个解析器,它认 2015 / 2015-03 / 2015-03-01,也认写出来的日期(#688)- utopia_extract::read_time(s).is_some()-}--/// 提示词里那三段清单:类、关系、属性。-///-/// **抽出来是为了让"全给"和"按分块检索"共用同一段排版逻辑。**两条路各排一份-/// 的话迟早分叉,而分叉在这里的后果是提示词说的与代码认的不是一回事。-struct PromptLists {- types: Vec<(String, String, String)>,- relations: Vec<utopia_extract::PromptRelation>,- attributes: Vec<String>,-}--impl PromptLists {- /// 这三段铺进提示词有多长。budget 判据用它——**量的是实际要排的那段字**,- /// 不是另写一个估算公式(公式会跟排版分叉)。- fn chars(&self) -> usize {- self.types- .iter()- .map(|(k, l, d)| k.len() + l.len() + d.len() + 6)- .sum::<usize>()- + self- .relations- .iter()- .map(|r| r.key.len() + r.label.len() + r.description.len() + r.signature.len() + 8)- .sum::<usize>()- + self.attributes.iter().map(|a| a.len() + 1).sum::<usize>()- }-}--/// 把当前本体在提示词里的字符数算出来。**空铺**(不筛类/关系)——这就是-/// `extract_document` 用的「全铺」档,也是判断「要不要按块检索」的标准。-///-/// 抽成独立函数是因为 `ontology_index::gate_required`(#526)要在加载抽取器-/// 之前问一次预算——那时 `build_lists` 还没被调用。两个路径必须用同一个判据,-/// 否则 gate 的判定会和实际的「全铺」走分。-pub(crate) fn full_ontology_chars(- etypes: &[utopia_core::models::EntityType],- rtypes: &[utopia_core::models::RelationType],-) -> usize {- build_lists(etypes, rtypes, None, None).chars()-}--/// 从一个**选择集**排出三段清单。`None` = 全给(本体小于预算时的老路)。-///-/// 三处细节都是选择带来的,全给时它们不会触发:-///-/// 1. **签名只能提到选中的类**。`works_at (person → organization)` 里那两个 key-/// 必须是模型看得见的——写一个没铺出去的类名,等于教它输出一个不存在的类型。-/// 整侧都没选中就退回 `*`。-/// 2. **属性跟着 domain 走**。属性行是 `class.attr`,它的类没铺出去这行就没意义。-/// 这也顺带解决了属性段(占提示词 28%)的裁剪,不用单独处理。-/// 3. **内置类恒在**。检索漏掉的分块仍然要有地方落脚,否则模型无类可选。-fn build_lists(- etypes: &[utopia_core::models::EntityType],- rtypes: &[utopia_core::models::RelationType],- classes: Option<&HashSet<Uuid>>,- rels: Option<&HashSet<Uuid>>,-) -> PromptLists {- let picked_class = |id: &Uuid| classes.is_none_or(|s| s.contains(id));- // 按块检索(有选择集)时描述只带第一句:检索那条路只有大的导入本体才走,一块铺上百行,- // 描述占清单的八成(#701)。全铺的是装得下预算的小本体,描述原样- let describe = |d: &str| -> String {- if classes.is_some() {- utopia_extract::first_sentence(d).to_string()- } else {- d.to_string()- }- };- // 边上能带的属性:关系.qualifiers → 属性行(0037)。这里只排版,写入侧另有一份同样的查法- let rtype_by_id: HashMap<Uuid, &utopia_core::models::RelationType> =- rtypes.iter().map(|r| (r.id, r)).collect();- let qualifier_line = |r: &utopia_core::models::RelationType| -> Vec<String> {- r.qualifiers- .iter()- .filter_map(|q| rtype_by_id.get(q).copied())- .filter(|q| q.kind == "attribute")- .map(|q| {- let dt = q.datatype.as_deref().unwrap_or("text");- match q.unit.as_deref().filter(|u| !u.is_empty()) {- Some(u) => format!("{}: {dt} {u}", q.key),- None => format!("{}: {dt}", q.key),- }- })- .collect()- };- let picked_rel = |id: &Uuid| rels.is_none_or(|s| s.contains(id));- let key_of: HashMap<Uuid, &str> = etypes- .iter()- .filter(|t| picked_class(&t.id))- .map(|t| (t.id, t.key.as_str()))- .collect();-- let types = etypes- .iter()- .filter(|t| picked_class(&t.id))- .map(|t| (t.key.clone(), t.label.clone(), describe(&t.description)))- .collect();-- // 一侧的类一个都没铺出去就写 `*`:签名是导向,指向看不见的类只会误导- let sig_of = |ids: &[Uuid]| -> String {- let mut keys: Vec<&str> = ids- .iter()- .filter_map(|id| key_of.get(id).copied())- .collect();- if keys.is_empty() {- return "*".into();- }- keys.sort_unstable();- keys.join("|")- };- let relations = rtypes- .iter()- .filter(|r| r.kind != "attribute")- .filter(|r| picked_rel(&r.id))- .map(|r| {- let signature = if r.domains.is_empty() && r.ranges.is_empty() {- String::new()- } else {- format!("{} → {}", sig_of(&r.domains), sig_of(&r.ranges))- };- utopia_extract::PromptRelation {- key: r.key.clone(),- label: r.label.clone(),- description: describe(&r.description),- signature,- temporal: r.temporal.clone(),- // `amount: number $`——模型要按这个 key 写,单位提醒它别换算- qualifiers: qualifier_line(r),- }- })- .collect();-- let attributes = rtypes- .iter()- .filter(|r| r.kind == "attribute" && picked_rel(&r.id))- .flat_map(|r| r.domains.iter().map(move |d| (r, d)))- .filter_map(|(r, domain_id)| {- let class_key = key_of.get(domain_id)?;- let dt = r.datatype.as_deref().unwrap_or("text");- let spec = match &r.unit {- Some(u) if !u.is_empty() => format!("{dt}, {u}"),- _ => dt.to_string(),- };- let d = describe(&r.description);- let d = d.trim();- Some(if d.is_empty() {- format!("- {class_key}.{} ({spec})", r.key)- } else {- format!("- {class_key}.{} ({spec}): {d}", r.key)- })- })- .collect();-- PromptLists {- types,- relations,- attributes,- }-}--/// 每块检索多少个类 / 关系 / 属性。**待测**——跟预算一样,定它们要那条曲线。-const PER_CHUNK_CLASSES: i64 = 40;-const PER_CHUNK_RELATIONS: i64 = 30;-const PER_CHUNK_ATTRIBUTES: i64 = 30;-/// 「这批类身上声明的关系/属性」这道地板给多少名额。-///-/// 比按相似度那 30 个宽得多,因为它的池子已经被 domain 收窄过一轮——-/// schema.org 里 person + organization + corporation 三个类身上一共只有 86 个关系。-/// 上限只是防病态情况(一块认出上百个类),不是筛选手段。-const PER_CHUNK_DOMAIN_RELATIONS: i64 = 120;-const PER_CHUNK_DOMAIN_ATTRIBUTES: i64 = 40;--/// 按这一块的向量检索候选,排出这一块专用的三段清单。-///-/// 检索失败返回 `Ok(None)` 而不是错误:调用方会退回全量。提示词大是慢,-/// 没有类可选是抽不出东西——两者之间选前者。-async fn chunk_lists(- state: &AppState,- kb_id: Uuid,- embedding: &[f32],- etypes: &[utopia_core::models::EntityType],- rtypes: &[utopia_core::models::RelationType],- seed_classes: &HashSet<Uuid>,- budget: usize,-) -> anyhow::Result<Option<PromptLists>> {- let mut classes: HashSet<Uuid> = seed_classes.clone();- classes.extend(- utopia_store::ontology::nearest_entity_type_ids(- &state.pool,- kb_id,- embedding,- PER_CHUNK_CLASSES,- )- .await?,- );- // **命中什么,就把它的祖先一起铺出去。**- //- // 向量检索天然偏爱字面出现在正文里的叶子类。实测一个讲 Sutskever 的分块,- // 976 个类按距离排:`researcher` 第 4、`corporation` 第 27,- // 而 `organization` 第 177、`person` 第 359——**前 40 名里一个泛化基类都没有**。- // 正文写的是 "a researcher at"、"the corporation",从不写 "person"。- //- // 两个症状,同一个根因:- //- // - 实体判成 `researcher`(schema.org 里它是 `Audience` 的子类,不是人),- // 于是 `works_for (domain=person)` 全成了违规- // - `employee (organization → person)` 的签名**退化成 `(* → *)`**——`sig_of`- // 只认铺出去的类,一侧没铺就写 `*`。模型根本没见过那个方向约束- //- // 从前这道地板由 `seed_classes`(`builtin` 的类)兜着,`build_lists` 的注释写着- // 「内置类恒在:检索漏掉的分块仍然要有地方落脚」。种子退场后(#128)判据就悬空了——- // 它当初碰巧等价,只因为种子类正好是那几个通用类。- //- // 用祖先补这道地板,比维护一张"通用类"清单好:**继承链本来就是本体自己声明的- // 泛化关系**,谁是谁的上位不需要我们再判断一次。代价是每块多铺几层祖先。- if !classes.is_empty() {- let picked: Vec<Uuid> = classes.iter().copied().collect();- classes.extend(utopia_store::ontology::ancestors_of(&state.pool, &picked).await?);- }- // 关系与属性分开检索:两段在提示词里是分开的,混在一起取会让其中一段- // 被另一段挤空。检索回来是按距离排好的,排队时交替取,两段一起往下让- let nearest = interleave(- utopia_store::ontology::nearest_relation_type_ids(- &state.pool,- kb_id,- embedding,- PER_CHUNK_RELATIONS,- Some("relation"),- )- .await?,- utopia_store::ontology::nearest_relation_type_ids(- &state.pool,- kb_id,- embedding,- PER_CHUNK_ATTRIBUTES,- Some("attribute"),- )- .await?,- );- let rels: HashSet<Uuid> = nearest.iter().copied().collect();- // **一个关系被铺出去,它签名点名的类就得跟着铺。**- //- // 类与关系是各自独立检索的,而签名依赖两者的交集——`sig_of` 只认铺出去的类,- // 一侧没铺就写 `*`。于是常出现这种局面:`employee` 跟正文语义相近被捞了进来,- // 而它的 `organization`(第 795 名)与 `person`(第 630 名)离正文字面很远,- // 一个都没捞到,签名退化成 `(* → *)`——**方向约束整个消失**,模型按英语直觉- // 写 `Musk --employee--> Microsoft`,而 schema.org 声明的是 organization → person。- //- // 上面那道祖先地板治不了这种块:它从"命中的叶子"往上长,而这里一个相关的- // 叶子都没命中,没有叶子也就没有祖先。- //- // `sig_of` 的注释说「签名指向看不见的类只会误导」——顾虑是对的,但抹掉签名- // 是拿丢失方向来换。**把类拉进来**两头都保住:模型看得见那个类,签名也排得出。- // 顺带还对:这些类正是模型马上要用来判类型的那些,`employee` 在场就说明- // 这一块讲的是雇佣,`organization`/`person` 本来就该在候选里——- // 按字面相似度捞不到它们,但**本体的结构知道**。- let signature_classes = |kept: &HashSet<Uuid>| -> HashSet<Uuid> {- rtypes- .iter()- .filter(|r| kept.contains(&r.id))- .flat_map(|r| r.domains.iter().chain(r.ranges.iter()).copied())- .collect()- };- let mut domain_pool = classes.clone();- domain_pool.extend(signature_classes(&rels));-- // **类进来了,就把本体声明在它们身上的关系也铺出去。**- //- // 上面那道祖先地板治的是「类捞不到」,这道治的是「关系捞不到」——同一个- // 病的两侧。实测一块讲「Jensen Huang, founder and CEO of NVIDIA」的正文:- // `employee` 排第 10 进了窗口,模型就用了它;而 `founder` 排 267、- // `job_title` 排 618、`has_occupation` 排 811,一个都没进——**四篇文档里- // 每个人的职务因此全部没落进图,而且不留任何丢弃信号:模型没被问到,- // 也就什么都没说,drops 与 misses 两张表都看不见它**(2026-09-08 实测)。- //- // 收窄的判据是本体自己声明的 domain,不是又一次相似度猜测:这一块认出了- // person 与 organization,那么「本体说人和组织能有什么」就该摆在模型面前。- // 同一块里 `founder` 升到 29、`job_title` 升到 55(池子 86)。- //- // **放在 `sig_classes` 之后,是为了不让它反过来撑大类清单。** 放在前面时- // 这批关系的 range 会顺着签名规则把一大票类拉进来,同一块的提示词从 11.7k- // 涨到 21.4k——为了一个谓词付两倍的钱。它们的 domain 侧本来就在清单里- // (地板正是这么选出来的),range 侧退化成 `*` 可以接受:这道地板要办的事- // 是「让模型看见这个说法存在」,不是把签名补全。- let domain_ids: Vec<Uuid> = domain_pool.iter().copied().collect();- let mut on_domains = Vec::new();- for (limit, kind) in [- (PER_CHUNK_DOMAIN_RELATIONS, "relation"),- (PER_CHUNK_DOMAIN_ATTRIBUTES, "attribute"),- ] {- on_domains.push(- utopia_store::ontology::nearest_relation_type_ids_in_domains(- &state.pool,- kb_id,- embedding,- limit,- Some(kind),- &domain_ids,- )- .await?,- );- }- let domain_attributes = on_domains.pop().unwrap_or_default();- let domain_relations = on_domains.pop().unwrap_or_default();-- // 排队:按相似度检索到的在前,地板补进来的在后- let mut seen = rels.clone();- let mut ranked = nearest;- ranked.extend(- interleave(domain_relations, domain_attributes)- .into_iter()- .filter(|id| seen.insert(*id)),- );-- // 一个候选都没检索到 = 索引还没建好,退回全量而不是给一份空清单- if classes.len() <= seed_classes.len() && ranked.is_empty() {- return Ok(None);- }-- // **这一块的清单也守那个预算**(#701)。预算原本只判「全铺装不装得下」,检索出来的- // 清单没有上限:三道地板叠上去,schema.org 一块铺到 5.6 万字符,是预算的 2.3 倍,- // 提示词两万 token 里正文不到 2%。- //- // 按排队顺序取前 k 个,每个带着它的结构(签名点名的类;祖先在类清单里已经有了),量- // 实际排出来的那段字,取装得下的最大 k。多一个候选只会多几行,长度随 k 单调,二分就行。- // 地板补进来的关系不拉签名类,理由见上- let lists_for = |k: usize| {- let kept: HashSet<Uuid> = ranked[..k].iter().copied().collect();- let near_kept: HashSet<Uuid> = kept.intersection(&rels).copied().collect();- let mut picked = classes.clone();- picked.extend(signature_classes(&near_kept));- build_lists(etypes, rtypes, Some(&picked), Some(&kept))- };- let (mut lo, mut hi) = (0usize, ranked.len());- while lo < hi {- let mid = (lo + hi).div_ceil(2);- if lists_for(mid).chars() <= budget {- lo = mid;- } else {- hi = mid - 1;- }- }- if lo < ranked.len() {- tracing::debug!(- kept = lo,- retrieved = ranked.len(),- budget,- "按块清单按预算截断"- );+ if !document_claims.contains(&id) {+ document_claims.push(id);}- Ok(Some(lists_for(lo)))+ Ok(id)}-/// 两串按距离排好的 id 交替并成一串:a0 b0 a1 b1 …,短的那串用完就接着排长的-fn interleave(a: Vec<Uuid>, b: Vec<Uuid>) -> Vec<Uuid> {- let mut out = Vec::with_capacity(a.len() + b.len());- let (mut a, mut b) = (a.into_iter(), b.into_iter());- loop {- match (a.next(), b.next()) {- (None, None) => return out,- (x, y) => out.extend(x.into_iter().chain(y)),- }+/// 每篇文档都走开放图谱(0044 决定 2):只写文档自己的话,本体不进提示词。+/// 记忆日志同样从这里进,只是它的陈述先进待确认表等人点头(0015),点头时才落成开放陈述+async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow::Result<()> {+ let doc = utopia_store::documents::get(&state.pool, document_id).await?;+ // 排队之后被删了(#268):墓碑不抽——抽出来的事实会活在一个已删除的出处上+ if doc.deleted_at.is_some() {+ tracing::info!(document = %document_id, "skipping a deleted document");+ return Ok(());}-}+ let kb = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;+ let settings = utopia_store::settings::get(&state.pool, kb.workspace_id)+ .await?+ .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?;+ let client = llm_util::chat_client(&settings)+ .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?;-/// 一条值该记什么单位(#600「单位是读出来的,不是猜的」,实体上的属性与边上的属性同一条规矩)。-///-/// 原文里认得出的用原文的(€、¥、%、"EUR 30 million" 的 €);模型把币种单独写成-/// 一个键时用那个;原文里写着声明的那个单位("4300 人" 对 "人")也算读出来的。-/// 原文带着一个认不出的单位记号("500 兆瓦"、"francs")时**不能**拿声明的缺省顶上——-/// 实测 `EUR 30 million` 被存成了 `$`,`500 兆瓦` 被存成了 500 块钱;单位写错比不写更糟。-/// 只有原文完全没有单位记号(一个光秃秃的数)才落回声明的缺省。-fn unit_for(- raw: &serde_json::Value,- datatype: &str,- sibling_currency: Option<&str>,- declared: Option<&str>,-) -> Option<String> {- // 文本、日期、布尔值没有单位可言:"3年"、"30日" 是文本,尾巴上的字不是单位- // ——实测「期限=3年」被记成了 `3年 年`- if datatype != "number" {- return None;- }- let declared = declared.map(str::trim).filter(|u| !u.is_empty());- let text = raw.as_str();- if let Some(u) = text- .and_then(utopia_extract::parse_leading_quantity)- .and_then(|(_, u)| u)- {- return Some(u);- }- if let (Some(c), "number") = (sibling_currency, datatype) {- return Some(c.to_string());- }- if let (Some(t), Some(d)) = (text, declared) {- if t.contains(d) {- return Some(d.to_string());- }- }- let has_unit_token = text.is_some_and(|t| {- t.chars()- .any(|c| c.is_alphabetic() || matches!(c, '$' | '€' | '£' | '¥' | '₩' | '₹' | '%'))- });- if has_unit_token {- None- } else {- declared.map(str::to_string)- }+ // 所有权凭证:重抽会自增 epoch,任务据此察觉自己已被接管(见 `run_open` 的分块循环)+ let my_epoch = utopia_store::documents::extract_epoch(&state.pool, document_id).await?;+ utopia_store::documents::set_graph_status(&state.pool, document_id, "extracting").await?;+ state.emit_document(doc.kb_id, document_id);+ let await_nod = utopia_store::memory::is_memory_document(&state.pool, document_id).await?;+ crate::extraction_open::run_open(+ state, &doc, &kb, &settings, &client, my_epoch, proposer, await_nod,+ )+ .await}#[cfg(test)]@@ -3454,7 +355,7 @@ mod origin_ceiling_tests {fn a_described_fact_cannot_close_a_value_by_itself() {let ceiling = origin_ceiling("described", 0.95);assert!(ceiling < utopia_store::temporal::AUTO_CLOSE_MIN_CONFIDENCE);- assert!(ceiling >= MIN_CONFIDENCE, "it still enters the graph");+ assert_eq!(ceiling, DESCRIBED_CEILING, "it still enters the graph");assert_eq!(origin_ceiling("described", 0.62), 0.62);for origin in ["stated", "ocr", "transcribed"] {assert_eq!(origin_ceiling(origin, 0.95), 0.95);@@ -3462,625 +363,11 @@ mod origin_ceiling_tests {}}-#[cfg(test)]-mod unit_for_tests {- use super::unit_for;- use serde_json::{json, Value};-- fn s(t: &str) -> Value {- Value::String(t.to_string())- }-- #[test]- fn a_unit_is_read_from_the_text_before_anything_else() {- assert_eq!(- unit_for(&s("EUR 30 million"), "number", None, Some("$")).as_deref(),- Some("€")- );- assert_eq!(- unit_for(&s("8.6亿元"), "number", None, Some("$")).as_deref(),- Some("¥")- );- assert_eq!(- unit_for(&s("12%"), "number", None, Some("¥")).as_deref(),- Some("%")- );- assert_eq!(- unit_for(&s("$5 billion"), "number", None, None).as_deref(),- Some("$")- );- }-- #[test]- fn a_sibling_currency_is_the_unit_of_a_bare_number() {- assert_eq!(- unit_for(&s("1500000000"), "number", Some("CNY"), Some("$")).as_deref(),- Some("CNY")- );- // 文本型属性没有币种可言- assert_eq!(unit_for(&s("B 轮"), "text", Some("CNY"), None), None);- // 文本值尾巴上的字也不是单位:"3年" 是期限的写法,不是 3 个「年」- assert_eq!(unit_for(&s("3年"), "text", None, Some("")), None);- assert_eq!(unit_for(&s("30日"), "text", None, None), None);- }-- #[test]- fn the_declared_unit_written_in_the_text_counts_as_read() {- assert_eq!(- unit_for(&s("4300 人"), "number", None, Some("人")).as_deref(),- Some("人")- );- }-- #[test]- fn an_unknown_unit_token_is_never_overwritten_by_the_default() {- // 宽松扫描把尾巴上的记号当单位读出来:记的是原文的单位,不是声明的 ¥- assert_eq!(- unit_for(&s("500 兆瓦"), "number", None, Some("¥")).as_deref(),- Some("兆瓦")- );- assert_eq!(- unit_for(&s("30 million francs"), "number", None, Some("$")).as_deref(),- Some("francs")- );- // 扫描读不出、原文却明明带着字:也不拿缺省顶上- assert_eq!(- unit_for(&s("about five hundred"), "number", None, Some("$")),- None- );- }-- #[test]- fn a_bare_figure_takes_the_declared_default() {- assert_eq!(- unit_for(&s("4300"), "number", None, Some("人")).as_deref(),- Some("人")- );- assert_eq!(- unit_for(&json!(4300), "number", None, Some("人")).as_deref(),- Some("人")- );- assert_eq!(unit_for(&s("4300"), "number", None, Some("")), None);- }-}--#[cfg(test)]-mod name_tests {- use super::{clause_suspect, is_entity_name};-- /// 样本全部取自实跑出来的库(ai-timeline-ends × schema.org),不是编的。- #[test]- fn a_clause_is_not_a_thing() {- for s in [- "thermal-imaging equipment used by volunteers flying over the site showed at least 33 generators giving off heat",- "about the same amount of power as the Tennessee Valley Authority's large gas-fired power plant nearby",- "removal was driven by growing discontent and distrust with Altman",- "a risk of developing cancer at four times the national average in 2013",- "745 of OpenAI's 770 employees",- ] {- assert!(!is_entity_name(s), "这是一句话,不该当成实体名:{s}");- }- }-- /// 一个数额不是一个东西。- ///- /// 样本取自实跑出来的库:`$5 billion`、`$1 billion`、`$30 billion` 各自成过节点,- /// 而且同名的会并成一个点——SSI Inc. 与 Nvidia 因为都出现过「$5 billion」- /// 在图上相连,那条路径没有任何含义。判据的窄处在**尾巴**:- /// 后面还有实词的一律放行,因为那时它说的就不再只是那个数。- #[test]- fn a_quantity_is_not_a_thing() {- for s in ["$5 billion", "€1.5 million", "52%", "3.5 million", "35,000"] {- assert!(!is_entity_name(s), "这是一个量,不该当成实体名:{s}");- }- // **以数字开头的真实体一个都不能误伤。** 量级词只认全写,所以 `3M` 解不动;- // 后面挂着实词的,尾巴那一条接住- for s in [- "3M",- "7-Eleven",- "23andMe",- "2025 Atlantic hurricane season",- "900 million weekly active users",- "1000 Islands",- "$10 billion investment",- ] {- assert!(is_entity_name(s), "这是真实体,不该被挡:{s}");- }- }-- /// **真实体会长,但不带谓语。** 判据是词数 + 限定动词,不是字符数——- /// 下面第一个 57 字符,比上面那条 65 字符的从句还短不了多少- #[test]- fn a_long_name_is_still_a_name() {- for s in [- "US District Court for the Northern District of California",- "United States District Court for the District of Delaware",- "OpenAI's board of directors",- "Safe Superintelligence Inc.",- "École Polytechnique",- "GPT-4",- ] {- assert!(is_entity_name(s), "这是真实体,不该被挡:{s}");- }- }-- /// #193:词表之外的句子,结构信号接住——句号结尾、情态动词。样本来自第二份语料- #[test]- fn a_sentence_is_caught_without_its_verb_on_the_list() {- for s in [- "The removal could slow down the artificial intelligence industry as a whole.",- "Shares in Microsoft fell nearly three percent following the announcement.",- "the board should reconsider its position",- ] {- assert!(!is_entity_name(s), "这是一句话,不该当成实体名:{s}");- }- // 大写缩写的句号、兼作名词的情态词,都不误伤- for s in [- "Safe Superintelligence Inc.",- "Theresa May",- "Trash Can Museum",- ] {- assert!(is_entity_name(s), "这是真实体,不该被挡:{s}");- }- }-- /// 弱信号只记不挡:像从句的照常落库,但留下样本- #[test]- fn a_suspect_is_recorded_not_rejected() {- let s = "The committee that reviewed the merger of the two companies";- assert!(is_entity_name(s));- assert_eq!(clause_suspect(s), Some("determiner_opens_a_long_string"));- assert_eq!(- clause_suspect("committee members who reviewed the merger"),- Some("relative_or_subordinate_clause")- );- for s in [- "US District Court for the Northern District of California",- "OpenAI's board of directors",- "The New York Times",- "The Men Who Stare",- ] {- assert_eq!(clause_suspect(s), None, "{s} 不该被怀疑");- }- }-- /// 分词在名词短语里完全正常,列进标志词会误伤。- #[test]- fn a_participle_is_not_a_predicate() {- assert!(is_entity_name("equipment used by volunteers"));- assert!(is_entity_name("Gas-Burning Turbines"));- }-- /// 短名字不做从句判断:`Is` 之类可能是专名的一部分。- #[test]- fn a_short_name_is_never_a_clause() {- assert!(is_entity_name("Was"));- assert!(is_entity_name("Is Elon"));- }-- #[test]- fn an_empty_name_is_not_a_name() {- assert!(!is_entity_name(""));- assert!(!is_entity_name(" "));- }-}-#[cfg(test)]mod tests {-- #[test]- fn two_ranked_lists_take_turns_and_the_longer_one_finishes() {- let ids: Vec<Uuid> = (0..5).map(|_| Uuid::now_v7()).collect();- let (a, b) = (vec![ids[0], ids[1], ids[2]], vec![ids[3]]);- assert_eq!(- super::interleave(a, b),- vec![ids[0], ids[3], ids[1], ids[2]]- );- assert!(super::interleave(Vec::new(), Vec::new()).is_empty());- }-- #[test]- fn a_name_declared_for_another_entity_is_not_an_alias() {- let (probe, project) = (Uuid::now_v7(), Uuid::now_v7());- let declared = HashMap::from([- ("海洋探测器1号".to_string(), probe),- ("海探1项目".to_string(), project),- ]);- assert!(name_claimed_elsewhere("海探1项目", probe, &declared));- assert!(- name_claimed_elsewhere(" 海探1项目 ", probe, &declared),- "空白不论"- );- assert!(- !name_claimed_elsewhere("海探1", probe, &declared),- "没人声明过的名字照收"- );- assert!(- !name_claimed_elsewhere("海洋探测器1号", probe, &declared),- "自己的名字不算撞"- );- }- use super::{name_claimed_elsewhere, slot_matches, span_in_quote, verify_span, SpanVerdict};- fn declared(names: &[&str]) -> HashMap<String, Uuid> {- names- .iter()- .map(|n| (n.to_string(), Uuid::now_v7()))- .collect()- }-- /// 片段核对:#578 的两句真实错例;改绑;头衔与指代只记;正常的放行- #[test]- fn a_span_that_names_a_description_is_not_the_entity() {- let d = declared(&[- "OpenAI",- "Anthropic",- "Sam Altman",- "OpenAI's board of directors",- "The Verge",- "Tasha McCauley",- ]);- let quote = "Former OpenAI personnel have founded competing AI companies Anthropic, SpaceXAI, Safe Superintelligence Inc., and Thinking Machines Lab";- // 名字后面还有词:名字只是修饰语- assert_eq!(- verify_span(Some("Former OpenAI personnel"), "OpenAI", "", quote, &d),- SpanVerdict::Described("Former OpenAI personnel".into())- );- assert_eq!(- verify_span(Some("Anthropic"), "Anthropic", "", quote, &d),- SpanVerdict::Ok- );- // 所有格:名字只是修饰语- let q3 = "Claude bypassed Anthropic's safeguards";- assert_eq!(- verify_span(Some("Anthropic's safeguards"), "Anthropic", "", q3, &d),- SpanVerdict::Described("Anthropic's safeguards".into())- );- // 另一个名字带着修饰,绑到了第三方- assert_eq!(- verify_span(- Some("The Verge reporter"),- "Sam Altman",- "",- "a The Verge reporter wrote",- &d- ),- SpanVerdict::Described("The Verge reporter".into())- );- // 名字前面带了词:头衔还是另一件东西分不开,只记- let q2 = "Over one hundred companies using OpenAI contacted Anthropic";- assert_eq!(- verify_span(Some("companies using OpenAI"), "OpenAI", "", q2, &d),- SpanVerdict::Prefixed("companies using OpenAI".into())- );- assert_eq!(- verify_span(- Some("entrepreneur Tasha McCauley"),- "Tasha McCauley",- "",- "with entrepreneur Tasha McCauley on the board",- &d- ),- SpanVerdict::Prefixed("entrepreneur Tasha McCauley".into())- );- // 片段点了另一个声明过的实体:改绑,不丢- assert_eq!(- verify_span(- Some("Sam Altman"),- "OpenAI",- "",- "Sam Altman announced the deal",- &d- ),- SpanVerdict::Rebind("Sam Altman".into())- );- // 单个词包在别的名字里不改绑(#595):句首大写不是专名的证据。"Altman" 绑错到- // OpenAI 时只记指代——改绑要么精确/词干命中,要么两个词以上- assert_eq!(- verify_span(- Some("Altman"),- "OpenAI",- "",- "Altman announced the deal",- &d- ),- SpanVerdict::Coreference("Altman".into())- );- assert_eq!(- verify_span(- Some("Stockholders"),- "NVIDIA",- "",- "Stockholders approved the election of each of our ten (10) director nominees",- &declared(&[- "NVIDIA",- "2026 Annual Meeting of Stockholders of NVIDIA Corporation"- ])- ),- SpanVerdict::Coreference("Stockholders".into())- );- assert_eq!(- verify_span(- Some("OpenAI's board"),- "OpenAI",- "",- "OpenAI's board removed Sam Altman as CEO",- &d- ),- SpanVerdict::Rebind("OpenAI's board of directors".into())- );- // 指代:没有任何名字,绑定照旧、只记- assert_eq!(- verify_span(- Some("him"),- "Sam Altman",- "",- "the board reinstated him",- &d- ),- SpanVerdict::Coreference("him".into())- );- // 单个普通词包在别的名字里也是指代,不改绑("the company" ≠ "for-profit company")- let d2 = declared(&[- "OpenAI",- "for-profit company",- "Satya Nadella",- "Jony Ive",- "Apple",- "Microsoft",- "Helen Toner",- "Fidji Simo",- ]);- assert_eq!(- verify_span(- Some("the company"),- "OpenAI",- "",- "the company took over",- &d2- ),- SpanVerdict::Coreference("the company".into())- );- // 片段以另一个名字结尾:不猜它是头(v4/v5 里猜错的多过猜对的),照描述处理- assert_eq!(- verify_span(- Some("Microsoft chief executive Satya Nadella"),- "Microsoft",- "OpenAI",- "convinced Microsoft chief executive Satya Nadella",- &d2- ),- SpanVerdict::Described("Microsoft chief executive Satya Nadella".into())- );- assert_eq!(- verify_span(- Some("Swisher and The Verge reporter Alex Heath"),- "Kara Swisher",- "",- "Swisher and The Verge reporter Alex Heath stated",- &declared(&["Kara Swisher", "Alex Heath", "The Verge"])- ),- SpanVerdict::Described("Swisher and The Verge reporter Alex Heath".into())- );- // 以另一侧的名字结尾:抄错了位置,绑定照旧、只记- assert_eq!(- verify_span(- Some("CEO of Applications: Fidji Simo"),- "OpenAI",- "Fidji Simo",- "CEO of Applications: Fidji Simo",- &d2- ),- SpanVerdict::Misplaced("CEO of Applications: Fidji Simo".into())- );- // 末尾的名字是介词的补语或名单的最后一项,不是头:不改绑(v4 的四个错例)- let d3 = declared(&[- "OpenAI",- "companies using OpenAI",- "TBPN",- "California",- "Pioneer Building",- "San Francisco",- "Anthropic",- ]);- assert_eq!(- verify_span(- Some("Over one hundred companies using OpenAI"),- "companies using OpenAI",- "Anthropic",- "Over one hundred companies using OpenAI contacted Anthropic",- &d3- ),- SpanVerdict::Prefixed("Over one hundred companies using OpenAI".into())- );- assert_eq!(- verify_span(- Some("his vested equity in OpenAI"),- "vested equity",- "Sam Altman",- "forfeited his vested equity in OpenAI",- &d3- ),- SpanVerdict::Described("his vested equity in OpenAI".into())- );- assert_eq!(- verify_span(- Some("TBPN, a media company in California"),- "TBPN",- "OpenAI",- "acquired TBPN, a media company in California",- &d3- ),- SpanVerdict::Ok- );- assert_eq!(- verify_span(- Some("the Pioneer Building in the Mission District, San Francisco"),- "Pioneer Building",- "OpenAI",- "located in the Pioneer Building in the Mission District, San Francisco",- &d3- ),- SpanVerdict::Described(- "the Pioneer Building in the Mission District, San Francisco".into()- )- );- // 名字后面紧跟逗号:同位语,还是它- assert_eq!(- verify_span(- Some("Helen Toner, strategy director for the Center for Security and Emerging Technology"),- "Helen Toner",- "OpenAI",- "and Helen Toner, strategy director for the Center for Security and Emerging Technology",- &d2- ),- SpanVerdict::Ok- );- // 没给片段:老行为- assert_eq!(verify_span(None, "OpenAI", "", quote, &d), SpanVerdict::Ok);- // 片段不在引文里:只记不拦- assert_eq!(- verify_span(Some("OpenAI staff"), "OpenAI", "", q2, &d),- SpanVerdict::NotInQuote- );- }-- /// 模型写的名字和它 ref 指的实体打架:写 "OpenAI employees" 却指向 OpenAI- #[test]- fn a_written_name_that_describes_its_reference_is_a_description() {- use super::written_verdict;- let d = declared(&[- "OpenAI",- "Anthropic",- "Sam Altman",- "OpenAI's board of directors",- ]);- // v5 的最后一条假边:片段 "eleven employees" 没有名字拦不住,写的名字拦得住- assert_eq!(- written_verdict("OpenAI employees", "OpenAI", "Anthropic", true, &d),- SpanVerdict::Described("OpenAI employees".into())- );- assert_eq!(- written_verdict("Former OpenAI personnel", "OpenAI", "Anthropic", true, &d),- SpanVerdict::Described("Former OpenAI personnel".into())- );- // 写的是另一个声明过的实体:改绑- assert_eq!(- written_verdict(- "OpenAI's board of directors",- "OpenAI",- "Sam Altman",- true,- &d- ),- SpanVerdict::Rebind("OpenAI's board of directors".into())- );- // 写的就是那个名字(含后缀、部分)、没有 ref、或是指代:无事- assert_eq!(- written_verdict("OpenAI, Inc.", "OpenAI", "", true, &d),- SpanVerdict::Ok- );- assert_eq!(- written_verdict("Altman", "Sam Altman", "", true, &d),- SpanVerdict::Ok- );- assert_eq!(- written_verdict("OpenAI employees", "OpenAI", "", false, &d),- SpanVerdict::Ok- );- assert_eq!(- written_verdict("the company", "OpenAI", "", true, &d),- SpanVerdict::Ok- );- }-- /// 名字后面接着 and 再接专名,是并列的一项,不是描述(#595);接着 and 再接小写- /// 的描述还是描述- #[test]- fn a_name_in_a_coordination_is_one_of_the_list() {- let d = declared(&["SB Energy", "SoftBank", "OpenAI"]);- let q = "SB Energy and SoftBank will build at least 10 GW of new energy generation";- assert_eq!(- verify_span(Some("SB Energy and SoftBank"), "SB Energy", "", q, &d),- SpanVerdict::Ok- );- assert_eq!(- verify_span(- Some("SB Energy & SoftBank"),- "SB Energy",- "",- "SB Energy & SoftBank will build",- &d- ),- SpanVerdict::Ok- );- // 绑在后一项上:名字前面带词,只记- assert_eq!(- verify_span(Some("SB Energy and SoftBank"), "SoftBank", "", q, &d),- SpanVerdict::Prefixed("SB Energy and SoftBank".into())- );- assert_eq!(- verify_span(- Some("OpenAI and its investors"),- "OpenAI",- "",- "OpenAI and its investors agreed",- &d- ),- SpanVerdict::Described("OpenAI and its investors".into())- );- }-- /// 名字后面接着专名样子的续词是同一个东西;接着小写的词就不是- #[test]- fn a_name_continued_in_capitals_is_the_same_thing() {- let d = declared(&["Anthropic", "OpenAI"]);- assert_eq!(- verify_span(- Some("Anthropic PBC"),- "Anthropic",- "",- "Anthropic PBC filed",- &d- ),- SpanVerdict::Ok- );- assert_eq!(- verify_span(- Some("OpenAI Global, LLC"),- "OpenAI",- "",- "OpenAI Global, LLC is the for-profit arm",- &d- ),- SpanVerdict::Ok- );- assert_eq!(- verify_span(- Some("OpenAI employees"),- "OpenAI",- "",- "OpenAI employees left",- &d- ),- SpanVerdict::Described("OpenAI employees".into())- );- }-- #[test]- fn a_slot_matches_its_name_by_stem_and_suffix() {- assert!(slot_matches("OpenAI", "OpenAI, Inc."));- assert!(slot_matches("Acme", "Acme Corp."));- assert!(slot_matches("openai", "OpenAI"));- assert!(slot_matches("OpenAI's", "OpenAI"));- assert!(slot_matches("Altman", "Sam Altman"));- assert!(slot_matches("Anthropic", "Anthropic, PBC"));- assert!(slot_matches(- "the Center for Security and Emerging Technology",- "Center for Security and Emerging Technology"- ));- assert!(!slot_matches("Former OpenAI personnel", "OpenAI"));- assert!(!slot_matches("Anthropic", "OpenAI"));- }+ use super::{incomplete_reason, resolve_handle, span_in_quote};+ use std::collections::HashMap;+ use uuid::Uuid;#[test]fn a_span_is_found_in_its_quote_regardless_of_case_and_spacing() {@@ -4095,109 +382,6 @@ mod tests {assert!(!span_in_quote("", "anything"));}- use super::{- incomplete_reason, looks_literal, no_ref_name_binding, referenced_entity, resolve_bare,- resolve_handle, validity_of, BoundEntity, NoRefNameBinding,- };- use std::collections::HashMap;- use uuid::Uuid;-- #[test]- fn quantities_and_dates_count_as_literals() {- // 认:这些出现在宾语位上时是值,不是实体- for yes in [- "2015",- "2023-03",- "2024-01-15",- "1200",- "62.5",- "-3",- // 带符号与量级词的量。从前这一档不认,于是图上长出一个叫- // 「$5 billion」的节点,同名的还并成一个,把毫不相干的两家公司连起来- "$5 billion",- "€1.5 million",- "52%",- "35,000",- // 合同照原文写的日期(#688)- "June 23, 2020",- "17 Mar. 2020",- "2020年3月17日",- ] {- assert!(looks_literal(yes), "{yes} 该认成字面值");- }- // 不认:判错的代价是把一个真实体降成一段文本,所以宁可漏- for no in [- "杭州",- "首席技术官",- "3M",- "V3",- "深蓝存储",- "",- " ",- "2015", // 全角数字:不是我们要处理的形态,交给实体路径- // 尾巴上还有实词:它说的不再只是那个数- "900 million weekly active users",- "2025 Atlantic hurricane season",- "$10 billion investment",- "8GW data center",- ] {- assert!(!looks_literal(no), "{no} 不该认成字面值");- }- }-- /// 区间两端照合同原文写(#688):读成日期,精度随写了几位;「unknown」仍是结束了不知哪天- #[test]- fn a_written_start_keeps_the_precision_it_was_written_with() {- let day = validity_of(Some("June 8, 2020"), None, None);- assert_eq!(- day.from.map(|t| t.date_naive().to_string()).as_deref(),- Some("2020-06-08")- );- assert_eq!(day.from_precision, Some("day"));- assert!(!day.has_ended());-- let month = validity_of(Some("March 2020"), Some("17 Mar. 2021"), None);- assert_eq!(month.from_precision, Some("month"));- assert_eq!(- month.to.map(|t| t.date_naive().to_string()).as_deref(),- Some("2021-03-17")- );- assert_eq!(month.to_precision, Some("day"));-- // 说不清几月几号的不当起点- let ambiguous = validity_of(Some("03/04/2020"), Some("unknown"), None);- assert_eq!((ambiguous.from, ambiguous.from_precision), (None, None));- assert_eq!(ambiguous.to_precision, Some("unknown"));- }-- #[test]- fn no_ref_names_bypass_legacy_map_only_after_two_handle_claims() {- let (a, b) = (Uuid::now_v7(), Uuid::now_v7());- let entity_ids = HashMap::from([("Zhang Wei".to_string(), b)]);- let normalized = utopia_store::resolution::normalize_name("Zhang Wei").to_lowercase();-- assert_eq!(- no_ref_name_binding(&entity_ids, &HashMap::new(), "Zhang Wei"),- NoRefNameBinding::Legacy(b)- );- assert_eq!(- no_ref_name_binding(- &entity_ids,- &HashMap::from([(normalized.clone(), vec![a])]),- "Zhang Wei",- ),- NoRefNameBinding::Legacy(b)- );- assert_eq!(- no_ref_name_binding(- &entity_ids,- &HashMap::from([(normalized, vec![a, b])]),- "Zhang Wei",- ),- NoRefNameBinding::AmbiguousHandled- );- }-/// 这条判据存在的理由:一次网络抖动让六篇文档 60 块里只抽成 12 块,/// 六篇**全部显示"抽取完成"**,八成的内容没进图,界面上没有任何东西说出来。#[test]@@ -4229,7 +413,7 @@ mod tests {}#[tokio::test]- async fn namesake_handles_keep_fact_attribution_and_bare_mentions_get_c() -> anyhow::Result<()>+ async fn namesake_handles_keep_fact_attribution_and_a_fresh_handle_gets_c() -> anyhow::Result<()>{let Some(url) = utopia_store::test_db::url() else {return Ok(());@@ -4264,32 +448,6 @@ mod tests {.await?;let run = async {- let mut legacy_cache = HashMap::new();- let mut legacy_needs_adjudication = false;- let legacy_first = super::resolve(- &pool,- kb,- Some(person),- "Alice",- None,- None,- &mut legacy_cache,- &mut legacy_needs_adjudication,- )- .await?;- let legacy_second = super::resolve(- &pool,- kb,- Some(person),- "Alice",- None,- None,- &mut legacy_cache,- &mut legacy_needs_adjudication,- )- .await?;- assert_eq!(legacy_first, legacy_second, "legacy name cache is unchanged");-let mut response_claims = HashMap::new();let mut document_claims = HashMap::new();let mut bare_cache = HashMap::new();@@ -4329,25 +487,8 @@ mod tests {"namesakes must not wake the LLM worker");- let refs = HashMap::from([- (- "e1".to_string(),- BoundEntity {- id: a,- type_id: Some(person),- },- ),- (- "e2".to_string(),- BoundEntity {- id: b,- type_id: Some(person),- },- ),- ]);- assert_eq!(referenced_entity(&refs, "e1").map(|x| x.id), Some(a));- assert_eq!(referenced_entity(&refs, "e2").map(|x| x.id), Some(b));- assert!(referenced_entity(&refs, "missing").is_none());+ // 同一回复里两个句柄各绑各的实体:事实跟着句柄走,不跟着名字走+ let refs = HashMap::from([("e1", a), ("e2", b)]);let (finance, platform) = (Uuid::now_v7(), Uuid::now_v7());let works_at = Uuid::now_v7();sqlx::query(@@ -4366,7 +507,7 @@ mod tests {.await?;}for (handle, object) in [("e1", finance), ("e2", platform)] {- let subject = referenced_entity(&refs, handle).expect("valid ref").id;+ let subject = refs[handle];utopia_store::graph::insert_fact(&pool,kb,@@ -4400,29 +541,25 @@ mod tests {assert!(labels.contains(&(a, Some("Finance".to_string()))));assert!(labels.contains(&(b, Some("Platform Engineering".to_string()))));- let mut doc_cache = HashMap::new();- // The legacy surface map still ends on B, but a no-ref fact must not use that- // last-write-wins value once handles have claimed two distinct Zhang Weis.- let entity_ids = HashMap::from([("Zhang Wei".to_string(), b)]);- assert_eq!(entity_ids.get("Zhang Wei"), Some(&b));- assert_eq!(- super::no_ref_name_binding(&entity_ids, &document_claims, "Zhang Wei"),- super::NoRefNameBinding::AmbiguousHandled- );- let c = resolve_bare(+ // 后一次回复给同一个名字开了一个新句柄,没挑 A 或 B:文本里没有挑的依据,+ // 不许猜。事实落到一个文档级的 C 上,对 A、对 B 各一条人工审核对+ let mut later_response_claims = HashMap::new();+ let c = resolve_handle(&pool,kb,Some(person),"Zhang Wei",None,None,- &mut doc_cache,- &document_claims,+ &mut later_response_claims,+ &mut document_claims,&mut bare_cache,&mut needs_adjudication,&mut human_reviews,).await?;+ assert_ne!(c, a);+ assert_ne!(c, b);utopia_store::graph::insert_fact(&pool,kb,@@ -4433,20 +570,26 @@ mod tests {0.9,).await?;- let c_again = resolve_bare(+ // 再往后的回复再开一个新句柄,仍是那个 C:一个新拼写不是猜 A 或 B 的许可+ let mut another_response_claims = HashMap::new();+ let c_again = resolve_handle(&pool,kb,None,"Zhang Wei",None,None,- &mut doc_cache,- &document_claims,+ &mut another_response_claims,+ &mut document_claims,&mut bare_cache,&mut needs_adjudication,&mut human_reviews,).await?;+ assert_eq!(+ c_again, c,+ "a later response cannot evade A/B ambiguity by inventing a new e-handle"+ );utopia_store::graph::insert_fact(&pool,kb,@@ -4457,12 +600,6 @@ mod tests {0.9,).await?;- assert_ne!(c, a);- assert_ne!(c, b);- assert_eq!(- c_again, c,- "later bare mentions must reuse document-local C"- );let c_objects: Vec<Uuid> = sqlx::query_scalar("SELECT object_id FROM factsWHERE kb_id = $1 AND subject_id = $2 AND object_id IS NOT NULL ORDER BY object_id",@@ -4474,25 +611,6 @@ mod tests {assert_eq!(c_objects.len(), 2);assert!(c_objects.contains(&finance));assert!(c_objects.contains(&platform));- let mut later_response_claims = HashMap::new();- let c_via_fresh_handle = resolve_handle(- &pool,- kb,- Some(person),- "Zhang Wei",- None,- None,- &mut later_response_claims,- &mut document_claims,- &mut bare_cache,- &mut needs_adjudication,- &mut human_reviews,- )- .await?;- assert_eq!(- c_via_fresh_handle, c,- "a later response cannot evade A/B ambiguity by inventing a new e-handle"- );let reviews = utopia_store::resolution::list_reviews(&pool,@@ -4520,11 +638,10 @@ mod tests {run}- /// #270:跨文档的裸 mention 同名并列——不靠 handle,靠画像相似度。库里已有两个- /// 「张伟」,画像质心一模一样(同一 chunk 播的种)。新 mention 对两人打出同一个分。- /// 旧路径在最高分 ≥ SIM_ATTACH 时静默 attach 到先遇到的那个;修好之后 `resolve`- /// 走的这条路要:新建第三个实体、两条**人工**审核对、且绝不唤醒 LLM 裁决器- /// (否则两条几乎相同的画像会被自动并掉,正是要防的)。+ /// #270:跨文档的同名并列——不靠句柄,靠画像相似度。库里已有两个「张伟」,画像质心+ /// 一模一样(同一 chunk 播的种)。新提及对两人打出同一个分。旧路径在最高分 ≥ SIM_ATTACH+ /// 时静默 attach 到先遇到的那个;修好之后走句柄这条路要:新建第三个实体、两条**人工**+ /// 审核对、且绝不唤醒 LLM 裁决器(否则两条几乎相同的画像会被自动并掉,正是要防的)。#[tokio::test]async fn a_profile_tie_across_documents_files_human_reviews_not_an_attach() -> anyhow::Result<()>{@@ -4578,17 +695,22 @@ mod tests {// 与两人画像都一致的上下文:打出的余弦相同 → 分不开let ctx: Vec<f32> = vec![1.0, 0.0, 0.0];- let mut doc_cache = HashMap::new();- let mut needs_adjudication = false;- let c = super::resolve(+ let mut response_claims = HashMap::new();+ let mut document_claims = HashMap::new();+ let mut bare_cache = HashMap::new();+ let (mut needs_adjudication, mut human_reviews) = (false, false);+ let c = resolve_handle(&pool,kb,Some(person),"Zhang Wei",Some(&ctx),None,- &mut doc_cache,+ &mut response_claims,+ &mut document_claims,+ &mut bare_cache,&mut needs_adjudication,+ &mut human_reviews,).await?;@@ -4629,36 +751,3 @@ mod tests {run}}--#[cfg(test)]-mod undeclared_beside_value_tests {- use super::undeclared_beside_value;- use std::collections::HashMap;- use uuid::Uuid;-- #[test]- fn a_phrase_beside_a_value_is_not_an_entity() {- let mut declared = HashMap::new();- declared.insert("SB Energy".to_string(), Uuid::nil());- declared.insert("Microsoft".to_string(), Uuid::nil());- // 回复里的真形状:没句柄、没声明 → 值落下,短语进谓词- assert!(undeclared_beside_value(- None,- "new energy generation",- &declared- ));- assert!(undeclared_beside_value(- Some(" "),- "new regional grid infrastructure",- &declared- ));- // 宾语有句柄,或者是认下的名字(大小写不计)→ 是实体,走边- assert!(!undeclared_beside_value(- Some("e2"),- "new energy generation",- &declared- ));- assert!(!undeclared_beside_value(None, "microsoft", &declared));- assert!(!undeclared_beside_value(None, " ", &declared));- }-}diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rsindex 92436221..c31f1517 100644--- a/crates/utopia-server/src/extraction_open.rs+++ b/crates/utopia-server/src/extraction_open.rs@@ -24,6 +24,7 @@ use crate::extraction::{span_in_quote,};use crate::state::AppState;+use sqlx::PgPool;use std::collections::{HashMap, HashSet};use utopia_core::models::{Document, KnowledgeBase, LlmSettings, Proposer};use utopia_store::extraction_drops::reason;@@ -76,6 +77,46 @@ fn name_key(name: &str) -> String {.to_lowercase()}+/// 陈述里写的名字 → 实体。先看这一块列出的,再看本文档前面认下的(提示词里的清单)。+///+/// **被描述的东西用到才建。** 模型会把段落里每个名词短语都列进 `e`:实测一个 25 篇的库里+/// 848 个实体有 616 个是被描述的,其中 181 个没挂任何陈述,还有 155 字的从句。没人指着的+/// 描述不是实体,只是一句话的一部分——所以描述先记在 `deferred` 里,第一条指到它的陈述+/// 才把它建出来;同一篇里同一段描述只建一次(`described`)+async fn place(+ pool: &PgPool,+ kb_id: Uuid,+ local: &mut HashMap<String, Uuid>,+ known: &HashMap<String, Uuid>,+ deferred: &HashMap<String, (String, String)>,+ described: &mut HashMap<String, Uuid>,+ name: &str,+) -> anyhow::Result<Option<Uuid>> {+ let key = name_key(name);+ if let Some(id) = local.get(&key).or_else(|| known.get(&key)) {+ return Ok(Some(*id));+ }+ let Some((text, kind)) = deferred.get(&key) else {+ return Ok(None);+ };+ let id = match described.get(&key) {+ Some(id) => *id,+ None => {+ let id = utopia_store::resolution::create_described(+ pool,+ kb_id,+ text,+ (!kind.is_empty()).then_some(kind.as_str()),+ )+ .await?;+ described.insert(key.clone(), id);+ id+ }+ };+ local.insert(key, id);+ Ok(Some(id))+}+/// `await_nod`:这是记忆日志(0015)——陈述不直接落库,原样进待确认表,人点头时才成为开放陈述。/// `proposer`:那句话是谁、经哪枚令牌说的(0026),随待确认项一起记#[allow(clippy::too_many_arguments)]@@ -195,6 +236,8 @@ pub(crate) async fn run_open(// ---- 东西:有名字的走身份消解,被描述的建成没有名字事实的实体 ----let mut response_claims: HashMap<String, Vec<Uuid>> = HashMap::new();let mut local: HashMap<String, Uuid> = HashMap::new();+ // 被描述的东西:先记下名字和类别词,等陈述指到它再建(见 `place`)+ let mut deferred: HashMap<String, (String, String)> = HashMap::new();for e in &extraction.entities {let name = e.name.trim();if name.is_empty() {@@ -248,28 +291,11 @@ pub(crate) async fn run_open(}id} else {- match described.get(&key) {- Some(id) => *id,- None => {- let id = utopia_store::resolution::create_described(- pool,- kb_id,- name,- (!kind.is_empty()).then_some(kind),- )- .await?;- described.insert(key.clone(), id);- id- }- }+ deferred.insert(key, (name.to_string(), kind.to_string()));+ continue;};local.insert(key, id);}- // 陈述里写的名字 → 实体:先看这一块列出的,再看本文档前面认下的(提示词里的清单)- let resolve_name = |name: &str| -> Option<Uuid> {- let key = name_key(name);- local.get(&key).or_else(|| known_by_name.get(&key)).copied()- };// ---- 陈述 ----for s in &extraction.statements {@@ -277,7 +303,17 @@ pub(crate) async fn run_open(if phrase.is_empty() {continue;}- let Some(subject) = resolve_name(&s.subject) else {+ let Some(subject) = place(+ pool,+ kb_id,+ &mut local,+ &known_by_name,+ &deferred,+ &mut described,+ &s.subject,+ )+ .await?+ else {drop_signal(state,kb_id,@@ -299,7 +335,17 @@ pub(crate) async fn run_open(.map(str::trim).filter(|v| v.chars().any(char::is_alphanumeric));let object = match s.object.as_deref().map(str::trim).filter(|o| !o.is_empty()) {- Some(name) => match resolve_name(name) {+ Some(name) => match place(+ pool,+ kb_id,+ &mut local,+ &known_by_name,+ &deferred,+ &mut described,+ name,+ )+ .await?+ {Some(id) => Some(id),None => {drop_signal(@@ -376,7 +422,17 @@ pub(crate) async fn run_open(if role.is_empty() || text.is_empty() {continue;}- match resolve_name(text) {+ match place(+ pool,+ kb_id,+ &mut local,+ &known_by_name,+ &deferred,+ &mut described,+ text,+ )+ .await?+ {Some(id) => qualifiers.push((role, None, Some(id))),None => qualifiers.push((role, Some(serde_json::json!(text)), None)),}@@ -500,7 +556,17 @@ pub(crate) async fn run_open(if name.is_empty() {continue;}- let Some(id) = resolve_name(&n.entity) else {+ let Some(id) = place(+ pool,+ kb_id,+ &mut local,+ &known_by_name,+ &deferred,+ &mut described,+ &n.entity,+ )+ .await?+ else {drop_signal(state,kb_id,diff --git a/crates/utopia-server/src/ontology_index.rs b/crates/utopia-server/src/ontology_index.rsindex b3e05bce..7b1ef762 100644--- a/crates/utopia-server/src/ontology_index.rs+++ b/crates/utopia-server/src/ontology_index.rs@@ -64,64 +64,6 @@ pub async fn refresh(state: &AppState, kb_id: Uuid) -> anyhow::Result<usize> {refresh_scoped(state, kb_id, None).await}-/// 抽取前的门控(#526)。**走这条路径的抽取器必须先问这个再继续**——-/// 不问而直接抽,要么抽出来的图是基于半个本体的、要么抽取 worker 在-/// `refresh` 锁上空转把 32 个并发槽占死。-///-/// 返回值语义:-/// - `Ok(false)`:本体小,全铺就行;本体大但已经在向量里了;或者没配嵌入模型-/// (照旧全铺)。继续抽。-/// - `Err(anyhow::Error::context(Deferred{30s}))`:本体超出预算且向量还没补齐,-/// 调用方应该把这个错误原样往上抛——`jobs::mark_failed` 认 `Deferred`,把任务-/// 挂回 `queued` 等 30s。worker 槽立刻空出来,下一轮再试。-pub async fn gate_required(state: &AppState, kb_id: Uuid) -> anyhow::Result<bool> {- let etypes = utopia_store::graph::entity_types(&state.pool, kb_id).await?;- let mut rtypes = utopia_store::graph::relation_types(&state.pool, kb_id).await?;- // 名字属性不进本体向量索引:检索出来就会被当成一条可抽的属性递给模型- rtypes.retain(|r| !utopia_store::names::is_name_attribute(r));- let budget = utopia_store::access::ontology_prompt_budget(&state.pool).await?;- let chars = crate::extraction::full_ontology_chars(&etypes, &rtypes);- if chars <= budget {- return Ok(false);- }- // 超出预算:需要检索候选。检索候选要先有向量——向量是否就绪取决于- // `types_needing_embedding`,而它要求一个具体的嵌入模型名。- //- // KB 拿不到就不再静默当成「没事」——`run()` 后面还会再读一次同样的字段,- // 让那次错误冒上来但归因不清。`Err` 直接归到这里,运维日志一眼能找到。- let kb = utopia_store::kbs::get(&state.pool, kb_id).await?;- let Some(settings) = utopia_store::settings::get(&state.pool, kb.workspace_id).await? else {- // 没设模型就走全铺那条路——和原行为一致。配置阶段不该在这里就报错- return Ok(false);- };- // 与 `refresh_scoped` 同一个判据:客户端与模型名缺一个,补齐就什么都不做,- // 等下去只会空等到期限- let (Some(_), Some(embed_model)) = (- llm_util::embed_client(&settings),- settings.embed_model.clone(),- ) else {- // 超预算 + 没配嵌入模型:照旧按全量本体抽。那种部署本来就没有检索,- // 没有什么可等;判成失败会让只配了对话模型的库一篇都抽不出来- return Ok(false);- };- let stale =- utopia_store::ontology::types_needing_embedding(&state.pool, kb_id, &embed_model, None)- .await?;- if stale.is_empty() {- // 超预算但所有类型都已嵌好——这个组合很罕见,意味着库被改大后又改小了,- // 但代码路径是合法的。继续抽- return Ok(false);- }- // 入队补齐任务——同库已排着的不重复(`enqueue_unless_queued` 的契约)- utopia_store::jobs::enqueue_unless_queued(- &state.pool,- "embed_ontology",- serde_json::json!({ "kb_id": kb_id }),- )- .await?;- Ok(true)-}-/// 只补一半。调用方清楚自己要哪一半时用它——类型消解只用类,/// 等关系嵌完是白等。补漏的那一半有后台任务兜着。pub async fn refresh_scoped(diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rsindex 0ba9cad0..15cf8021 100644--- a/crates/utopia-store/src/extraction_drops.rs+++ b/crates/utopia-store/src/extraction_drops.rs@@ -13,91 +13,24 @@ use utopia_core::AppResult;use uuid::Uuid;/// 原因码。前端按这个查文案,所以是稳定契约,不要改字面量。+/// 都由开放抽取那条路(0044)发出;类型化那条路的原因码随它一起退场了pub mod reason {- /// 主语没在 entities 里声明 → 类型不明,属性无法校验 domain- pub const SUBJECT_NOT_DECLARED: &str = "subject_not_declared";- /// 属性挂在了不该挂的类上(salary 挂到 Organization)- pub const ATTR_DOMAIN_MISMATCH: &str = "attr_domain_mismatch";- /// 属性事实既没给 value 也没给 object- pub const ATTR_NO_VALUE: &str = "attr_no_value";- /// 值不合 datatype,归一化失败- pub const ATTR_DATATYPE: &str = "attr_datatype";- /// 模型在边上写了一个这条关系没声明过的属性 key(0037)- pub const QUALIFIER_UNKNOWN: &str = "qualifier_unknown";- /// 边上属性的值换不成它声明的 datatype- pub const QUALIFIER_DATATYPE: &str = "qualifier_datatype";- /// 同一条边再听到一次,属性值与已记的不一致——先记下,不覆盖- pub const QUALIFIER_CONFLICT: &str = "qualifier_conflict";- /// 模型自报置信度低于阈值- pub const LOW_CONFIDENCE: &str = "low_confidence";- /// 关系事实缺宾语+ /// 陈述缺宾语也缺值pub const OBJECT_MISSING: &str = "object_missing";- /// 模型给的这一条不合结构(缺 predicate 之类)→ 只跳这一条,不牵连整块+ /// 模型给的这一条不合结构(缺短语之类)→ 只跳这一条,不牵连整块pub const MALFORMED_ITEM: &str = "malformed_item";- /// 主语的类型对不上关系声明的 domain,**且对调也不合法**——那是选错了关系- /// 或类型判错,不是方向问题。照原样落库 + 记信号,交给人看,不猜- pub const DOMAIN_MISMATCH: &str = "domain_mismatch";- /// 模型给的"实体名"其实是一整句话或从句——不是一个东西的名字。- /// 这类东西永远匹配不到别处的提及,在图上是孤点,还会拖累消解- pub const NOT_AN_ENTITY_NAME: &str = "not_an_entity_name";- /// 主语违反 domain 而宾语符合,已按本体声明的方向把主宾掰正。- /// **动作必须留痕**:自动的、看不见的改写才是 0001 反对的那种- pub const DIRECTION_CORRECTED: &str = "direction_corrected";/// 模型输出被截断(撞上 max_tokens)→ 已完整的那些留下,尾巴丢掉pub const TRUNCATED_REPLY: &str = "truncated_reply";- /// 守卫放行了、结构却像从句(限定词起头的长串、句中的关系词)。**只记不挡**:- /// 实体照常落库,例句留下来——#193 要的是一份跨语料的标注集,再决定哪条升成硬规则- pub const CLAUSE_SUSPECT: &str = "clause_suspect";- /// 主语是「跟 X 有关的一群人」而写成了 X(#578):"former OpenAI personnel" 不是- /// OpenAI。事实不落,例句是那个短语;这一类该由提示词的规则改写到有名字的一侧,- /// 这里只量它还错多少- pub const SUBJECT_SHORTENED: &str = "subject_shortened";- /// 主宾片段不在引文里(#582):模型没照抄。事实照旧处理,只记下来- pub const SPAN_NOT_IN_QUOTE: &str = "span_not_in_quote";/// 模型报的别名(或它的引文)不在这一块原文里(0041 决定 2):不记这个名字。/// 名字是召回的桥,一座凭空的桥会把两个不相干的实体接到一起pub const NAME_NOT_IN_TEXT: &str = "name_not_in_text";/// 模型报的别名,这次回复(或本文档前面几块)里已经是另一个实体的名字(0041 决定 2):/// 一个名字不会同时是两样东西的名字。「海探1项目」声明成了一个机构,就不是探测器的别名pub const NAME_CLAIMED_BY_ANOTHER: &str = "name_claimed_by_another";- /// 主语片段是个描述,不是任何声明过的实体的名字:事实不落(#582,取代 #578 的词表)- pub const SUBJECT_DESCRIBED: &str = "subject_described";- /// 宾语片段是个描述:事实照落,宾语落成字面值(#582)- pub const OBJECT_DESCRIBED: &str = "object_described";- /// 片段点的是另一个声明过的实体:改绑到它(#582)- pub const SPAN_REBOUND: &str = "span_rebound";- /// 片段是所绑名字前面带了别的词("entrepreneur Tasha McCauley" / "companies using- /// OpenAI"):头衔还是另一件东西,机器分不开,绑定照旧,只记(#582)- pub const SPAN_PREFIXED: &str = "span_prefixed";- /// 片段里没有任何声明过的名字("him" / "the company"):模型消解了指代,无从核对,- /// 绑定照旧,只记(#582)- pub const SPAN_COREFERENCE: &str = "span_coreference";- /// 片段抄的是事实**另一侧**的名字(宾语片段写成了主语):抄错了位置,不是绑错了- /// 实体。绑定照旧,只记(#582)- pub const SPAN_MISPLACED: &str = "span_misplaced";- /// 宾语既没在 entities 里声明、库里也没有叫这个名字的东西:事实照落,+ /// 宾语既没在回复里列出、库里也没有叫这个名字的东西:陈述照落,/// 宾语落成字面值而不是节点(#559)。记下来是为了量:这一类里有多少/// 本该是实体(模型漏报),有多少本来就是描述pub const OBJECT_UNDECLARED: &str = "object_undeclared";- /// 以下都来自 `utopia_extract::normalize`:只看结构、不看词的形状检查- /// 值只有破折号(`—`):表里的「无」,不落- pub const NO_VALUE: &str = "no_value";- /// 值后面有一截引文里没有的字:只留引文里有的那段。只记- pub const VALUE_TRIMMED: &str = "value_trimmed";- /// 没有宾语也没有值、只带边属性:属性落成主语上的值事实。只记- pub const QUALIFIERS_WITHOUT_OBJECT: &str = "qualifiers_without_object";- /// 宾语是契约格式的日期:数落成值、日期进有效期;没带数的把写出来的那段落成值。只记- pub const TIME_AS_OBJECT: &str = "time_as_object";- /// 主语是契约格式的日期:数是谁的回复里没说,不落- pub const TIME_AS_SUBJECT: &str = "time_as_subject";- /// 宾语名字包住另一个声明实体、同句已有指向本尊的边:可能是描述,也可能是另一个- /// 东西(每股收益包住了净利润)。只记,不删- pub const OBJECT_DESCRIBES_DECLARED: &str = "object_describes_declared";- /// 上面几条去掉事实后没人引用的声明:不建- pub const ORPHAN_DECLARATION: &str = "orphan_declaration";- /// 引文抄自提示词里附的文件开头、不在这一块:证据会挂错出处,不落- pub const QUOTE_FROM_OPENING: &str = "quote_from_opening";- /// 以下三条来自开放抽取那条路(0044 第一刀,#729)/// 模型的引文在这一块里找不到原样的一句:陈述照落、引文照记,只是没有偏移/// (`quote_start` / `quote_end` 留空)。这个信号数的是有多少条没定位到pub const QUOTE_NOT_IN_CHUNK: &str = "quote_not_in_chunk";diff --git a/crates/utopia-store/src/kbs.rs b/crates/utopia-store/src/kbs.rsindex 14077b09..af2baec8 100644--- a/crates/utopia-store/src/kbs.rs+++ b/crates/utopia-store/src/kbs.rs@@ -110,7 +110,6 @@ pub async fn update(auto_type_resolution: Option<bool>,governance: Option<bool>,data_conventions: Option<&str>,- open_extraction: Option<bool>,) -> AppResult<KnowledgeBase> {// 改语言不回头重写已有的类——它们已经是这个库的数据,可能有人手工调过。// 这一列往后管的是**新**描述(自动扩本体、AI 建议)写成什么语言@@ -152,8 +151,6 @@ pub async fn update(ELSE governance_since END,-- 人写的约定;清空要送空串,送 null 等于不改(与 description 同一约定)data_conventions = COALESCE($11, data_conventions),- -- 开放抽取(0044 第一刀):抽取只写开放图谱;关着走类型化那条路- open_extraction = COALESCE($12, open_extraction),updated_at = now()WHERE id = $1 RETURNING *",)@@ -168,7 +165,6 @@ pub async fn update(.bind(auto_type_resolution).bind(governance).bind(data_conventions)- .bind(open_extraction).fetch_optional(pool).await?.ok_or(AppError::NotFound)diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rsindex 514a29ce..10f7dd29 100644--- a/crates/utopia-store/src/resolution.rs+++ b/crates/utopia-store/src/resolution.rs@@ -261,7 +261,10 @@ pub async fn resolve_mention(WHERE (f.subject_id = e.id OR f.object_id = e.id)AND f.invalidated_at IS NULL AND {not_name}) AS degreeFROM entities e- WHERE e.kb_id = $1 AND e.type_id = $2 AND e.merged_into IS NULL+ -- IS NOT DISTINCT FROM 而不是 =(0009 的那个陷阱):开放图谱里的实体都没有类+ -- (类由对齐来定),`type_id = NULL` 永远不成立,同名的它就永远撞不上——+ -- 实测一个库里 Securities and Exchange Commission 与它的全大写写法成了两个实体+ WHERE e.kb_id = $1 AND e.type_id IS NOT DISTINCT FROM $2 AND e.merged_into IS NULL-- 被描述的东西没有名字(0044):它的 canonical_name 只是显示用的描述,-- 不是召回的桥——两篇文档里描述得一样的两个东西不能因此接到一起AND e.description IS NULLdiff --git a/crates/utopia-store/tests/an_untyped_name_meets_its_namesake.rs b/crates/utopia-store/tests/an_untyped_name_meets_its_namesake.rsnew file mode 100644index 00000000..afce5bed--- /dev/null+++ b/crates/utopia-store/tests/an_untyped_name_meets_its_namesake.rs@@ -0,0 +1,78 @@+//! 没有类的实体也要撞得上同名者(0044 第一刀之后的实测)。+//!+//! 开放图谱里的实体都没有类:类由对齐来定,抽取只记文档的字。`resolve_mention` 的候选+//! 查询从前写的是 `e.type_id = $2`,传 NULL 时这个条件永远不成立——于是同一个库里+//! 「Securities and Exchange Commission」和它的全大写写法成了两个实体,只有建实体时+//! 的精确同名匹配还兜得住。这里守的是:两次没有类的提及,写法只差大小写,落到同一个实体。+//!+//! 没有 `UTOPIA_DATABASE_URL` 时跳过。自建自拆,绝不碰已有的库。++use sqlx::PgPool;+use uuid::Uuid;++#[tokio::test]+async fn an_untyped_mention_attaches_to_its_untyped_namesake() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'untyped-namesake-test')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'untyped-namesake-test')",+ )+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'untyped-namesake-test')",+ )+ .bind(kb)+ .bind(ws)+ .execute(&pool)+ .await?;++ let first = utopia_store::resolution::resolve_mention(+ &pool,+ kb,+ None,+ "Securities and Exchange Commission",+ None,+ None,+ &[],+ )+ .await?;+ assert!(first.created, "第一次提及要建实体");+ let again = utopia_store::resolution::resolve_mention(+ &pool,+ kb,+ None,+ "SECURITIES AND EXCHANGE COMMISSION",+ None,+ None,+ &[],+ )+ .await?;+ assert_eq!(+ again.entity_id, first.entity_id,+ "只差大小写的同名提及要落到同一个没有类的实体上"+ );+ assert!(!again.created);+ let n: i64 = sqlx::query_scalar(+ "SELECT count(*) FROM entities WHERE kb_id = $1 AND merged_into IS NULL",+ )+ .bind(kb)+ .fetch_one(&pool)+ .await?;+ assert_eq!(n, 1);++ sqlx::query("DELETE FROM organizations WHERE id = $1")+ .bind(org)+ .execute(&pool)+ .await?;+ Ok(())+}diff --git a/crates/utopia-store/tests/exploration_describes_the_data.rs b/crates/utopia-store/tests/exploration_describes_the_data.rsindex fdf1dcc3..20d564ea 100644--- a/crates/utopia-store/tests/exploration_describes_the_data.rs+++ b/crates/utopia-store/tests/exploration_describes_the_data.rs@@ -51,7 +51,6 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> anNone,None,Some("Amounts are in cents.\nis_test = 1 is excluded."),- None,).await?;// 探索写描述与问题@@ -102,7 +101,6 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> anNone,None,Some("cents only"),- None,).await?;let got = utopia_store::kbs::get(&pool, kb).await?;@@ -121,7 +119,6 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> anNone,None,no,- None,).await?;assert_eq!(diff --git a/migrations/0063_extraction_reads_only_the_open_graph.sql b/migrations/0063_extraction_reads_only_the_open_graph.sqlnew file mode 100644index 00000000..a648c5ee--- /dev/null+++ b/migrations/0063_extraction_reads_only_the_open_graph.sql@@ -0,0 +1,10 @@+-- 抽取只写开放图谱(0044 决定 2):开关退场。+--+-- 0061 加这一列时给带本体的老路留了一个关掉的口子——按当时的决定 3,它是已批准本体下+-- 的可选第二路,等对齐追平就退场。现在定了:没有遗留层,也没有可选的类型化第二路。+-- 每篇文档(记忆日志也在内)都走 `extraction_open::run_open`,本体不进提示词;类型化的+-- 事实由对齐从开放陈述算出来,不由抽取写。一个再也没有第二个取值的开关不该留在表里:+-- 留着,读的人会以为还有一条路可以切回去。+--+-- 没有已发布的版本,直接删列;已有的 `layer = 'typed'` 行不动,它们是对齐的输入。+ALTER TABLE knowledge_bases DROP COLUMN open_extraction;diff --git a/scripts/bench/README.md b/scripts/bench/README.mdindex 7449ae1d..39a1cba2 100644--- a/scripts/bench/README.md+++ b/scripts/bench/README.md@@ -65,13 +65,12 @@ node scripts/bench/fetch-sec-filings.mjs # 一次就够node scripts/bench/recall.mjs --kb <id> # 一轮:清空 → 重抽 → 打分node scripts/bench/recall.mjs --kb <id> --score # 只打分node scripts/bench/recall.mjs --kb <id> --reprocess # 改了解析器:连分块一起重来-node scripts/bench/recall.mjs --kb <id> --typed # 带本体的老路(可选第二路);缺省走开放图谱(0044 第 1 刀)node scripts/bench/judge_open.mjs --kb <id> --sample 200 # 开放陈述有多少不是原文说的(门槛 2%)```-缺省(库的 `open_extraction` 开着)抽的是开放图谱:陈述按原文短语落成 `layer='open'` 的事实行,-限定挂在 `statement_qualifiers`,时间词进 `time_mentions`;打分口径不变(谓词取-`proposed_predicate`,限定也拼进那一行),所以 `--typed` 那条老路和它的分数直接可比。`judge_open.mjs` 是+抽取写的是开放图谱(0044 决定 2,没有带本体的第二条路):陈述按原文短语落成 `layer='open'` 的+事实行,限定挂在 `statement_qualifiers`,时间词进 `time_mentions`;打分口径照旧(谓词取+`proposed_predicate`,限定也拼进那一行),所以下表里带本体那些轮次的分数仍然可比。`judge_open.mjs` 是反面的尺子:抽样让一个裁判模型读原文判 stated / misworded / not_stated;裁判端点用`BENCH_JUDGE_BASE / BENCH_JUDGE_KEY / BENCH_JUDGE_MODEL`,不给就用工作区的对话模型(和抽取同一个模型,数字要打折看)。diff --git a/scripts/bench/recall.mjs b/scripts/bench/recall.mjsindex 18d93fc8..cc9f85b9 100644--- a/scripts/bench/recall.mjs+++ b/scripts/bench/recall.mjs@@ -16,7 +16,6 @@// node scripts/bench/recall.mjs --kb <id> # 已有库(本体向量已就绪)// node scripts/bench/recall.mjs --kb <id> --score # 只打分,不重抽// node scripts/bench/recall.mjs --kb <id> --reprocess # 改了解析器:连分块一起重来-// node scripts/bench/recall.mjs --kb <id> --typed # 带本体的老路(可选第二路);缺省是开放图谱(0044 第 1 刀)//// 环境变量:BENCH_BASE / BENCH_EMAIL / BENCH_PASSWORD / BENCH_PSQL(同 run.mjs)。//@@ -212,8 +211,7 @@ psql(`DELETE FROM ontology_misses WHERE kb_id='${KB}'`);if (packTs) psql(`DELETE FROM relation_types WHERE kb_id='${KB}' AND created_at > '${packTs}'::timestamptz + interval '1 second'`);// 会在抽取之后改本体、增派生事实的开关关掉,两轮的本体与打分口径才一样。治理开着:// 库生下来就开着它(0050),量的是产品本来的样子-psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE, materialize_inferences=FALSE, governance=TRUE, open_extraction=${args.typed ? "FALSE" : "TRUE"} WHERE id='${KB}'`);-console.log(args.typed ? "带本体的老路:本体进提示词" : "开放图谱:本体不进提示词,陈述按原文短语落库");+psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE, materialize_inferences=FALSE, governance=TRUE WHERE id='${KB}'`);psql(`UPDATE chunks SET extracted_at=NULL WHERE document_id IN (SELECT id FROM documents WHERE kb_id='${KB}' AND filename IN (${names}))`);console.log(`本体 ${num(`SELECT count(*) FROM relation_types WHERE kb_id='${KB}'`)} 个关系 / ${num(`SELECT count(*) FROM entity_types WHERE kb_id='${KB}'`)} 个类`);diff --git a/scripts/bench/temporal.mjs b/scripts/bench/temporal.mjsindex 9debb330..4a7dc0a1 100644--- a/scripts/bench/temporal.mjs+++ b/scripts/bench/temporal.mjs@@ -437,6 +437,7 @@ async function acceptUniqueness(kb, axioms) {const byId = new Map(relation_types.map((r) => [r.id, r]));const rows = [];for (const c of candidates) {+ // 候选来自类型化的事实(predicate_id);开放陈述带的是 `phrase`、没有 predicate_id,对不上的跳过const r = byId.get(c.predicate_id);if (!r) continue;const declared = wanted.get(c.key);diff --git a/web/src/api.ts b/web/src/api.tsindex a193ae77..9f4a110a 100644--- a/web/src/api.ts+++ b/web/src/api.ts@@ -102,9 +102,6 @@ export interface Kb {/** 抽取遇到本体外的说法时,是否允许系统自动补进本体并改写等它的事实。关掉不影响"留意":未匹配统计照常累积可见,只是变成你点一下的提案 */auto_extend_ontology: boolean;- /** 开放抽取(0044 第一刀):开着,抽取只写开放图谱——陈述照文档的字落库,- * 不读本体;关着走今天的类型化那条路。缺省关;这一刀没有开关界面 */- open_extraction: boolean;/** 把推出来的事实写进账本(R1)。**缺省关**——推理往图里加东西,* 而声明可能是错的,不该在用户没表态时就按它改图 */materialize_inferences: boolean;diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex ba0421cc..a2a48450 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -503,20 +503,9 @@ export const en = {"Each line says why, and how many.",dropsExample: "e.g.",dropReason: {- attr_domain_mismatch: "Attribute on the wrong class",- subject_not_declared: "Subject type unknown",- attr_no_value: "Attribute had no value",- attr_datatype: "Value did not match the datatype",- low_confidence: "Below the confidence threshold",object_missing: "Relation had no object",malformed_item: "The model's item did not fit the schema",truncated_reply: "The model's reply was cut off",- domain_mismatch:- "The subject does not fit the relation, and swapping would not help",- not_an_entity_name: "That name is a sentence, not a thing",- clause_suspect: "Kept, but the name reads like a clause: a sample for the guard",- direction_corrected:- "Subject and object were swapped to match the signature",quote_not_in_chunk: "Kept, but the quoted sentence is not in the text verbatim",time_not_in_quote: "A time mention's words are not in the text",unknown_ref: "The item points at nothing in the reply",@@ -2023,7 +2012,6 @@ export const en = {conflictReason: {no_time: "new fact has no date",simultaneous: "same start date",- low_confidence: "low confidence",} as Record<string, string>,conflictVs: "vs",conflictSince: (d: string) => `since ${d}`,diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 6b5c03ce..ec8fe671 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -454,18 +454,9 @@ export const zh: Strings = {"这些是从文档里抽出来的,但在写入途中被挡下了。每行说明原因与条数。",dropsExample: "例如",dropReason: {- attr_domain_mismatch: "属性挂在了错误的类上",- subject_not_declared: "主语类型未知",- attr_no_value: "属性没有取值",- attr_datatype: "取值与数据类型不符",- low_confidence: "低于置信度阈值",object_missing: "关系缺少宾语",malformed_item: "模型给的这一条结构不合",truncated_reply: "模型的输出被截断",- domain_mismatch: "主语对不上这个关系,对调也不行",- not_an_entity_name: "那个名字是一句话,不是一个东西",- clause_suspect: "已保留,但这个名字像从句:给守卫攒的样本",- direction_corrected: "已按签名把主宾掰正",quote_not_in_chunk: "已保留,但引文不是原文原样",time_not_in_quote: "时间词不在原文里",unknown_ref: "这一条指向的东西不在回复里",@@ -1764,7 +1755,6 @@ export const zh: Strings = {conflictReason: {no_time: "新事实没有日期",simultaneous: "起始日期相同",- low_confidence: "置信度低",},conflictVs: "对",conflictSince: (d: string) => `自 ${d} 起`,