World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A statement keeps the document's words (#731)
WaylandYang60df635d6924
diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rsindex 184dc2de..b4d38a66 100644--- a/crates/utopia-core/src/models.rs+++ b/crates/utopia-core/src/models.rs@@ -1077,6 +1077,10 @@ pub struct KnowledgeBase {/// 这次打开治理的时刻;保险丝只数它之后的撤回(0025 决定 9)。/// 新库生下来就开着治理,这一格于是等于建库的时刻(0050)pub governance_since: Option<DateTime<Utc>>,+ /// 开放抽取(0044 第一刀,#729):开着,抽取只写开放图谱——陈述照文档的字落库+ /// (`facts.layer = 'open'`),不读本体、不问日期。**缺省开**(0044 决定 2);关掉走+ /// 带本体的老路,它按决定 3 只是已批准本体下的可选第二路,等对齐追平就退场+ pub open_extraction: bool,/// 多久重推一次(分钟)。见 `knowledge_bases.inference_interval_minutes`pub inference_interval_minutes: i32,/// 上次推完的时间。**答的是「上次看过没有」,不是「上次改过没有」**diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rsindex 56706170..ad441573 100644--- a/crates/utopia-extract/src/lib.rs+++ b/crates/utopia-extract/src/lib.rs@@ -6,6 +6,7 @@ use serde::Deserialize;use utopia_llm::ChatMessage;pub mod governor;+pub mod open;pub mod normalize;pub use normalize::{drop_quotes_from_opening, normalize_facts, Normalization};@@ -451,7 +452,7 @@ pub fn build_messages_with_opening(/// 「这一块就是开头本身」由调用方判断,那时它传 `None`。////// 按字符截:不会截断一个字符,但会截在词中间——英文的最后一个词可能只剩半个-fn opening_block(opening: Option<&str>) -> String {+pub(crate) fn opening_block(opening: Option<&str>) -> String {let Some(text) = opening.map(str::trim).filter(|t| !t.is_empty()) else {return String::new();};@@ -491,7 +492,7 @@ fn temporal_mark(temporal: &str) -> Option<&'static str> {////// 超出就截断(保留先出现的)。中文商业文本先出全称、主角先出场,所以/// **首次出现顺序天然偏向那些后面会被简称的名字**。-const KNOWN_BUDGET_CHARS: usize = 1200;+pub(crate) const KNOWN_BUDGET_CHARS: usize = 1200;/// 把「本文档已经认下的实体」排版成提示词里的一段。空则返回空串。///@@ -539,8 +540,10 @@ fn known_block(known: &[KnownEntity]) -> String {)}-/// 从 LLM 回复中稳健地取出 JSON 块(容忍代码围栏与前后废话)。-pub fn json_block(raw: &str) -> anyhow::Result<String> {+/// 回复里可能是 JSON 的那段文字:切掉思考过程与代码围栏。前后的废话留给调用方按+/// 括号定位——[`json_block`] 取第一个 `{` 到最后一个 `}`;开放抽取的截断修补则从+/// 第一个 `{` 取到结尾,那边的记录是数组,最后一个 `}` 不是可靠的结尾+pub(crate) fn json_text(raw: &str) -> &str {let text = raw.trim();// 推理模型的思考过程(#690):`LlmClient::chat` 那边会先切,但取块这一层// 自己认得标记才是最后的保障——`chat_tools` 那条路就不经过 `chat`。@@ -550,11 +553,15 @@ pub fn json_block(raw: &str) -> anyhow::Result<String> {Some(pos) => text[pos + "</think>".len()..].trim(),None => text,};- let cleaned = text- .strip_prefix("```json")+ text.strip_prefix("```json").or_else(|| text.strip_prefix("```")).map(|s| s.trim_end_matches("```"))- .unwrap_or(text);+ .unwrap_or(text)+}++/// 从 LLM 回复中稳健地取出 JSON 块(容忍代码围栏与前后废话)。+pub fn json_block(raw: &str) -> anyhow::Result<String> {+ let cleaned = json_text(raw);let start = cleaned.find('{');let end = cleaned.rfind('}');match (start, end) {@@ -566,7 +573,7 @@ pub fn json_block(raw: &str) -> anyhow::Result<String> {/// 把 head 后面缺的括号补上。字符串字面量里的括号不算——`"a[b"` 不是一个开括号。////// 返回 None = 结构本身就不对(比如括号已经多了),不是"没写完"。-fn close_brackets(head: &str) -> Option<String> {+pub(crate) fn close_brackets(head: &str) -> Option<String> {let mut stack: Vec<char> = Vec::new();let (mut in_str, mut esc) = (false, false);for c in head.chars() {diff --git a/crates/utopia-extract/src/open.rs b/crates/utopia-extract/src/open.rsnew file mode 100644index 00000000..f96fa3c6--- /dev/null+++ b/crates/utopia-extract/src/open.rs@@ -0,0 +1,778 @@+//! 开放抽取(0044 第 1 刀,#729):模型读一块,用**文档自己的话**写下它陈述了什么。+//!+//! 与 lib.rs 里带本体的契约不同,这条路的提示词里**没有类型清单、没有关系清单、+//! 没有文档日期**:关系短语照原文写("acquired"、"修建"),时间照原文的字抄+//! ("去年冬天"),不算日期、不归一。本体是之后套上去的视图,不是抽取时的筛子。+//! 提示词里唯一逐块变化的是正文、开头与已知实体,所以系统消息是一个常量——+//! 前缀缓存能命中的就是它。+//!+//! 回复是紧凑 JSON——数组、短键——因为一块的输出动辄上百条,冗长的键名烧的是+//! max_tokens,截断一次丢的是整块的后半。但**陈述指东西用名字,引文每条自己抄一遍**,+//! 不编号:编号版在一块密集的财报上测过,deepseek-v4-flash 把 id 串了位("NVIDIA has+//! reached AI"、"tokens are tokens",原文说的是 "AI has reached its inflection point"),+//! 同一块的两次回复编号还不一致;原型的名字 + 逐条引文版 333 条里 0 条编造。+//! 多花的 token 买的是不串位。+//!+//! ```json+//! {"e": [["Harbor Bridge", "bridge", 1], ["farmland", "land", 0]],+//! "s": [["city council of Westbrook", "awarded", "paving contract", null, {"to": "Brightway Builders", "amount": "$2 million"}, "March 4, 2011", null, "The city council of Westbrook awarded the paving contract for the Harbor Bridge to Brightway Builders for $2 million on March 4, 2011."]],+//! "n": [["Brightway Builders", "Brightway", "Brightway Builders, known locally as Brightway, is based in Port Ellen."]]}+//! ```+//!+//! - `e`:`[名字或描述, 种类词, named]`。陈述提到的每样东西都在这里列一次,用的正是+//! 陈述里写的那个名字。named = 1 是原文点了名的东西(人、组织、产品、地点、文件、+//! 法律、事件),0 是只描述了的东西("the northern wing"、"farmland")。名字不带引用语+//! ("the Harbor Treaty signed on May 3, 1998" 叫 "Harbor Treaty"),也不带数字+//! ("about 40 hectares of farmland" 是 "farmland" 加一条 `area` = "about 40 hectares")。+//! 没有 id。+//! - `s`:`[主语名, 短语, 宾语名或 null, 字面值或 null, 限定词对象或 null, 何时或 null,+//! 何时终止或 null, 引文]`。主语与宾语是 `e` 里(或提示词已知清单里)的名字,拼写一模+//! 一样。宾语与字面值恰有一个(解析层不裁,两个都给调用方看)。一句话里超过两方参与、+//! 或者链接带着数额、头衔、条件、比较时,主要的一对进主宾,其余进限定词,键是原文里+//! 说明其角色的一两个词,值是列出的东西的名字或原文的话。何时/终止照原文的字抄,+//! 从不计算。引文是陈述它的那一句,逐字抄。+//! - `n`:`[e 里列的名字, 这一段用的另一个名字, 引文]`。只收原文真写了的名字。+//!+//! 主语、宾语的名字对不对得上 `e`(或已知清单)**这里不查**,交给服务端——那边手里有+//! 已知实体的句柄与库里的名字,这里只有字。+//!+//! 解析是逐项宽容的:一条坏记录计入 [`OpenExtraction::skipped`],不毁掉整块;调用方+//! **必须**把这个数报出去——不报就是一次静默丢弃(#108 那类错)。++use std::collections::HashSet;++use serde_json::Value;+use utopia_llm::ChatMessage;++use crate::{+ close_brackets, json_block, json_text, opening_block, KnownEntity, KNOWN_BUDGET_CHARS,+};++#[derive(Debug, Clone)]+pub struct OpenEntity {+ /// the name the statements use for it, exactly as listed+ pub name: String,+ /// 原文说它是个什么,两三个词。不校验、不入本体+ pub kind: String,+ /// 原文点了名(true)还是只描述了(false)+ pub named: bool,+}++#[derive(Debug, Clone)]+pub struct OpenStatement {+ /// the name of a thing, as listed in `e` or in the known-things list; not resolved here+ pub subject: String,+ /// the relation phrase as written, lowercase short phrase+ pub phrase: String,+ /// entity object by name, or+ pub object: Option<String>,+ /// literal value as written (figure with units, title, status word, date words)+ pub value: Option<String>,+ /// keyed by the document's role word ("to", "amount", "title", "compared to"); the value is+ /// text, which may be the name of a listed thing. The order is serde_json's map order+ /// (alphabetical by key) and carries no meaning+ pub qualifiers: Vec<(String, String)>,+ /// the words that say when it holds, happened or began, exactly as written+ pub when: Option<String>,+ /// the words that say when it stopped, exactly as written+ pub ended: Option<String>,+ /// the sentence of the passage that states it, verbatim+ pub quote: Option<String>,+}++#[derive(Debug, Clone)]+pub struct OpenName {+ /// the thing's name as listed in `e` or in the known-things list+ pub entity: String,+ pub name: String,+ pub quote: Option<String>,+}++#[derive(Debug, Default)]+pub struct OpenExtraction {+ pub entities: Vec<OpenEntity>,+ pub statements: Vec<OpenStatement>,+ pub names: Vec<OpenName>,+ /// items skipped because they were malformed (counted per array item; must be reported by the caller)+ pub skipped: usize,+ /// the reply was truncated and repaired to the last complete object+ pub truncated: bool,+}++/// 系统消息。规则 1–8 是原型上测过的措辞,改写成紧凑数组;例子是中性的,不出自任何+/// 测量语料。**没有**本体、关系清单、文档日期——这条路的全部意义就在这三个「没有」+const OPEN_SYSTEM: &str = "\+You read one passage of a document and write down what it states, in the passage's own words. \+Output one JSON object and nothing else, shaped like this:\n\+\n\+{\"e\": [[\"name or description\", \"kind word\", 1]],\n\+ \"s\": [[\"subject name\", \"relation phrase as written\", \"object name\", null, {\"to\": \"name of a listed thing\", \"amount\": \"$2 million\"}, \"when it holds or happened, as written\", \"when it ended, as written\", \"the sentence that states it, verbatim\"]],\n\+ \"n\": [[\"name as listed in e\", \"another name\", \"the sentence that uses it, verbatim\"]]}\n\+\n\+1. \"e\" lists the things the passage talks about, one entry each: [name, kind, named]. Every \+thing a statement refers to is listed here once, under exactly the name the statements use. A \+thing the passage names (people, organizations, products, places, documents, laws, events) has \+named = 1 and is named as the passage writes it, without citation words: \"the Harbor Treaty \+signed on May 3, 1998\" is named \"Harbor Treaty\". A thing the passage only describes has \+named = 0 and is named by the words that say what it is (\"the northern wing\", \"farmland\", \+\"beekeepers\"). kind is what the thing is, in two or three words, as the passage says it. A \+name never carries a figure: \"about 40 hectares of farmland\" is the thing \"farmland\" with a \+statement \"area\" = \"about 40 hectares\". List each thing once.\n\+2. \"s\" lists the statements, one entry each: [subject, phrase, object, value, qualifiers, \+when, ended, quote]. subject and object are names, written exactly as listed in \"e\" or in \+the list of things already recorded, with the same spelling every time. Exactly one of object \+and value is set; the other is null.\n\+ A statement with an object links two things. phrase is how the passage says the link, as a \+short lowercase phrase (\"acquired\", \"was designed by\", \"houses\", \"is the captain of\", \+\"修建\"); do not translate it into any vocabulary of your own.\n\+ A statement with a value is what the passage says about one thing by itself: a figure with \+its units, a percentage, an amount, a count, a title, a status. phrase is how the passage says \+what is measured or stated (\"area\", \"was completed\", \"joined\", \"占地面积\"); value is the \+literal as written. Wording that names or describes something is not a value: that something \+goes in \"e\" and is linked by a statement with an object. \"1,200 beekeepers joined\" is the \+thing \"beekeepers\" with \"joined\" = \"1,200\". Write every figure the passage states. In a \+table, a cell is a statement about its row's thing whose phrase is the column heading; leave \+out cells whose column heading is not in the passage.\n\+ A list is one statement per member, and so is a subject or object that joins several things \+(\"the city and the county funded the bridge\" is two statements).\n\+3. Nothing in a sentence is dropped. When more than two things take part, or the link carries \+a figure, a title, a condition or a comparison, put the main pair in subject and object and the \+rest in qualifiers: an object keyed by a word or two for each part's role, whose values are the \+name of a listed thing or the words as written. \"the city council awarded the paving contract \+to Brightway Builders for $2 million\" gives city council —awarded→ paving contract with \+{\"to\": \"Brightway Builders\", \"amount\": \"$2 million\"}; \"Jane Doe is interim headmaster \+of Hillside School\" gives Jane Doe —is headmaster of→ Hillside School with {\"title\": \+\"interim headmaster\"}. A statement with a value takes qualifiers the same way: \"the plant \+cut water use by 12% compared to 2019\" gives the plant \"cut water use\" = \"12%\" with \+{\"compared to\": \"2019\"}. qualifiers is null when there are none.\n\+4. When a description has another thing folded into it, also write the statement that unfolds \+it: \"hospitals accredited by the Joint Commission\" also gives Joint Commission —accredited→ \+hospitals. Do not unfold a description that only names what it belongs to or is about: \"the \+northern wing of the palace\" and \"the use of the coating\" give no statement, the palace and \+the coating are part of the description.\n\+5. when is the words that say when the statement holds, happened or began, and ended is the \+words that say when it stopped, each copied exactly as written (\"March 4, 2011\", \"去年冬天\", \+\"by the end of next season\"). Never compute, convert or normalise a date, and never write one \+the passage does not. Each is null when the passage gives none.\n\+6. quote is the sentence of the passage that states it, copied verbatim: one sentence, and when \+two sentences are needed, the one that carries the link. Every \"s\" and \"n\" entry carries its \+own quote.\n\+7. \"n\" lists other names, one entry each: [name as listed, other name, quote] — a short form, \+a former name, a spelling in another script that this passage uses for a thing in \"e\" or a \+thing already recorded. Only names actually written in the passage; never a pronoun or a \+description.\n\+8. State nothing the passage does not state, and state each thing once: with a value or with \+an object, not both. The Document line, the opening of the document and the list of things \+already recorded only say where the passage comes from; write nothing about them. If the \+passage states nothing, output {\"e\":[],\"s\":[],\"n\":[]}.\n\+\n\+Example. The passage \"The city council awarded the paving contract to Brightway Builders for \+$2 million on March 4, 2011.\" gives:\n\+{\"e\": [[\"city council\", \"council\", 0], [\"paving contract\", \"contract\", 0], [\"Brightway Builders\", \"builders\", 1]],\n\+ \"s\": [[\"city council\", \"awarded\", \"paving contract\", null, {\"to\": \"Brightway Builders\", \"amount\": \"$2 million\"}, \"March 4, 2011\", null, \"The city council awarded the paving contract to Brightway Builders for $2 million on March 4, 2011.\"]],\n\+ \"n\": []}";++/// 构造开放抽取的两条消息:常量系统消息 + `Document:` / 开头 / 已知实体 / `Passage:`。+///+/// 没有文档日期、没有类型与关系清单、没有属性——这些都不进提示词。开头与已知实体+/// 只说明这一段从哪来(模型被告知不从它们里面抽东西)。+pub fn build_open_messages(+ filename: &str,+ known: &[KnownEntity],+ opening: Option<&str>,+ chunk_text: &str,+) -> Vec<ChatMessage> {+ // 已知实体紧挨着正文:服从性靠位置,理由见 lib.rs 里 known_block 的注释+ let user = format!(+ "Document: {filename}\n{}{}\nPassage:\n{chunk_text}",+ opening_block(opening),+ known_block(known)+ );+ vec![+ ChatMessage {+ role: "system".into(),+ content: OPEN_SYSTEM.to_string(),+ },+ ChatMessage {+ role: "user".into(),+ content: user,+ },+ ]+}++/// 「本文档已经认下的实体」的开放版:`- <name> (<kind>)`,句柄不给模型看——这条路+/// 陈述里写的是名字,模型照这里的拼写写,对回哪个实体由服务端拿名字去查。lib.rs+/// 那版的指令讲的是 subject_ref/object_ref 与「给它同样的类型」,都是带本体那条路的+/// 字段,这里没有。预算与截断规则同那版(句柄不显示,所以不计入)+fn known_block(known: &[KnownEntity]) -> String {+ if known.is_empty() {+ return String::new();+ }+ let mut lines = Vec::new();+ let mut used = 0usize;+ for entity in known {+ used += entity.type_key.chars().count() + entity.name.chars().count() + 6;+ if used > KNOWN_BUDGET_CHARS {+ break;+ }+ let kind = entity.type_key.trim();+ if kind.is_empty() {+ lines.push(format!("- {}", entity.name));+ } else {+ lines.push(format!("- {} ({kind})", entity.name));+ }+ }+ if lines.is_empty() {+ return String::new();+ }+ format!(+ "\nAlready recorded from earlier parts of this same document:\n{}\n\+ \n\+ When the passage refers to one of these, write its name exactly as it is listed here \+ and do not list it again in \"e\"; a shortened form the passage uses for it goes in \+ \"n\". If it is a different thing, list it as the passage names it; do not force it \+ onto this list.\n",+ lines.join("\n")+ )+}++/// 截断的回复从第一个 `{` 取到**结尾**,交给修补自己退到最后一个完整条目。+///+/// `json_block` 取的是第一个 `{` 到最后一个 `}`,对紧凑回复这一刀切错两回:`}` 只在+/// 限定词对象与结尾出现,截断的回复要么一个 `}` 都没有(它报「没有 JSON」,修补+/// 根本没机会跑),要么最后一个 `}` 是半路某条陈述的限定词(它把后面完整的条目+/// 一并切掉)。围栏与思考过程的切法与它共用+fn json_tail(raw: &str) -> Option<&str> {+ let text = json_text(raw);+ text.find('{').map(|s| &text[s..])+}++/// 紧凑格式的截断修补:退到**最后一个完整的数组或对象**的结尾再把括号补齐。+///+/// lib.rs 的 `repair_truncated` 只认 `}` 作为回退点——那边每条记录都是对象,以 `}`+/// 结尾。这里每条记录是数组,限定词全是 null 的回复里可能一个 `}` 都没有,只认 `}`+/// 就是整块作废。所以回退点是最后一个 `]` 或 `}`,谁靠后用谁;括号是否在字符串里+/// 由 `close_brackets` 判断+fn repair_truncated_compact(json: &str) -> Option<String> {+ let mut cut = json.len();+ for _ in 0..64 {+ let idx = json[..cut].rfind([']', '}'])?;+ if let Some(closed) = close_brackets(&json[..=idx]) {+ if serde_json::from_str::<Value>(&closed).is_ok() {+ return Some(closed);+ }+ }+ cut = idx;+ }+ None+}++/// 顶层某个键下的数组;缺了或不是数组就当空——那不是一条坏记录,是整段没这一类+fn items<'a>(value: &'a Value, key: &str) -> &'a [Value] {+ value+ .get(key)+ .and_then(Value::as_array)+ .map_or(&[], Vec::as_slice)+}++/// 第 i 格的非空字符串(去掉首尾空白)——缺一不可的格子用它+fn text_at(arr: &[Value], i: usize) -> Option<&str> {+ arr.get(i)?+ .as_str()+ .map(str::trim)+ .filter(|s| !s.is_empty())+}++/// 可选的文字格:字符串去首尾空白;null、空串与放错的类型都是 None——那一格放错了+/// 东西不算整条坏,坏的只是那一格+fn opt_text(v: &Value) -> Option<String> {+ v.as_str()+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .map(str::to_string)+}++/// 字面值照原样:字符串去首尾空白;模型偶尔把数字写成 JSON 数字,照它的写法转回文本。+/// 限定词的值走同一条规则+fn literal(v: &Value) -> Option<String> {+ match v {+ Value::Number(n) => Some(n.to_string()),+ v => opt_text(v),+ }+}++/// 限定词:对象里值为字符串或数字的那些;键去首尾空白,空键与别的类型悄悄丢;+/// 不是对象当作没有限定词+fn qualifiers(v: &Value) -> Vec<(String, String)> {+ v.as_object()+ .map(|m| {+ m.iter()+ .filter_map(|(k, v)| {+ let k = k.trim();+ (!k.is_empty()).then(|| Some((k.to_string(), literal(v)?)))?+ })+ .collect()+ })+ .unwrap_or_default()+}++/// 去重键:首尾空白去掉、内部空白折成一个、小写。"Harbor Bridge" 与 "harbor bridge"+/// 是同一条+fn name_key(name: &str) -> String {+ name.split_whitespace()+ .collect::<Vec<_>>()+ .join(" ")+ .to_lowercase()+}++/// `[name, kind, named]`:name 缺不得;kind 缺了是空串;named 缺了当点了名+fn parse_entity(arr: &[Value]) -> Option<OpenEntity> {+ let name = text_at(arr, 0)?.to_string();+ let kind = arr+ .get(1)+ .and_then(Value::as_str)+ .map(str::trim)+ .unwrap_or_default()+ .to_string();+ let named = match arr.get(2) {+ Some(Value::Bool(b)) => *b,+ Some(Value::Number(n)) => n.as_f64() != Some(0.0),+ _ => true,+ };+ Some(OpenEntity { name, kind, named })+}++/// `[subject, phrase, object, value, qualifiers, when, ended, quote]`。八格都得在:短了的+/// 多半是截断修补切出来的半条,引文都没有,收下也没法核对。主语与短语缺不得;其余格+/// 放错了类型只是那一格为 None。宾语与字面值同时有则两个都留给调用方+fn parse_statement(arr: &[Value]) -> Option<OpenStatement> {+ if arr.len() < 8 {+ return None;+ }+ let subject = text_at(arr, 0)?.to_string();+ let phrase = text_at(arr, 1)?.to_string();+ Some(OpenStatement {+ subject,+ phrase,+ object: opt_text(&arr[2]),+ value: literal(&arr[3]),+ qualifiers: qualifiers(&arr[4]),+ when: opt_text(&arr[5]),+ ended: opt_text(&arr[6]),+ quote: opt_text(&arr[7]),+ })+}++/// `[entity name, other name, quote]`+fn parse_name(arr: &[Value]) -> Option<OpenName> {+ let entity = text_at(arr, 0)?.to_string();+ let name = text_at(arr, 1)?.to_string();+ let quote = arr.get(2).and_then(opt_text);+ Some(OpenName {+ entity,+ name,+ quote,+ })+}++/// **一条坏记录不该毁掉一整块**(同 lib.rs 的 `parse_response`)。+///+/// 先取 JSON 块解成 `Value`(截断就先退到最后一个完整条目补齐括号,并标 `truncated`),+/// 再逐项宽容地解:形状不对的条目计入 `skipped`,好的照收。`e` 里重名(去空白、+/// 不分大小写)留第一个,其余计入 `skipped`。主语、宾语的名字不在这里对回 `e`+pub fn parse_open_response(raw: &str) -> anyhow::Result<OpenExtraction> {+ // 先按常规取块(第一个 `{` 到最后一个 `}`):解得开就是完整回复,结尾之后+ // 哪怕跟着废话也不算截断。解不开才从第一个 `{` 取到结尾去修+ let block = json_block(raw)+ .and_then(|b| serde_json::from_str::<Value>(&b).map_err(anyhow::Error::from));+ let (value, truncated) = match block {+ Ok(v) => (v, false),+ Err(e) => {+ let fixed = json_tail(raw)+ .and_then(repair_truncated_compact)+ // 补不回来才是真解析失败:连一个完整条目都没有+ .ok_or_else(|| anyhow::anyhow!("Failed to parse open extraction JSON: {e}"))?;+ let v = serde_json::from_str::<Value>(&fixed)+ .map_err(|e| anyhow::anyhow!("Failed to parse open extraction JSON: {e}"))?;+ (v, true)+ }+ };++ let mut out = OpenExtraction {+ truncated,+ ..Default::default()+ };+ let mut seen = HashSet::new();+ for item in items(&value, "e") {+ match item.as_array().and_then(|a| parse_entity(a)) {+ Some(e) if seen.insert(name_key(&e.name)) => out.entities.push(e),+ _ => out.skipped += 1,+ }+ }+ for item in items(&value, "s") {+ match item.as_array().and_then(|a| parse_statement(a)) {+ Some(s) => out.statements.push(s),+ None => out.skipped += 1,+ }+ }+ for item in items(&value, "n") {+ match item.as_array().and_then(|a| parse_name(a)) {+ Some(n) => out.names.push(n),+ None => out.skipped += 1,+ }+ }+ Ok(out)+}++#[cfg(test)]+mod tests {+ use super::*;++ const AWARD: &str =+ "The city council awarded the paving contract to Brightway Builders for $2 million on March 4, 2011.";+ const LEASE: &str = "Nebula (formerly Starlight Labs) leased the northern wing from Harbor Estates from 2010 until the end of 2019.";++ /// 一份完整的紧凑回复:本块列出的与已知清单里的名字、两种限定词、起止时间、别名+ const FULL: &str = r#"{"e": [["city council", "council", 0], ["paving contract", "contract", 0],+ ["Brightway Builders", "builders", 1], ["northern wing", "wing", 0]],+ "s": [["city council", "awarded", "paving contract", null, {"to": "Brightway Builders", "amount": "$2 million"}, "March 4, 2011", null,+ "The city council awarded the paving contract to Brightway Builders for $2 million on March 4, 2011."],+ ["Nebula Technologies Inc.", "leased", "northern wing", null, {"from": "Harbor Estates", "term": "ten years"}, "from 2010", "until the end of 2019",+ "Nebula (formerly Starlight Labs) leased the northern wing from Harbor Estates from 2010 until the end of 2019."],+ ["paving contract", "worth", null, "$2 million", null, null, null,+ "The city council awarded the paving contract to Brightway Builders for $2 million on March 4, 2011."]],+ "n": [["Nebula Technologies Inc.", "Starlight Labs", "Nebula (formerly Starlight Labs) leased the northern wing from Harbor Estates from 2010 until the end of 2019."]]}"#;++ /// serde_json 的对象按键排序(没开 preserve_order),限定词的顺序不承载意义+ fn by_key(mut q: Vec<(String, String)>) -> Vec<(String, String)> {+ q.sort_by(|a, b| a.0.cmp(&b.0));+ q+ }++ fn pairs(q: &[(&str, &str)]) -> Vec<(String, String)> {+ by_key(+ q.iter()+ .map(|(k, v)| (k.to_string(), v.to_string()))+ .collect(),+ )+ }++ #[test]+ fn a_full_reply_parses_into_the_structs() {+ let x = parse_open_response(FULL).unwrap();+ assert!(!x.truncated);+ assert_eq!(x.skipped, 0);++ assert_eq!(x.entities.len(), 4);+ assert_eq!(x.entities[0].name, "city council");+ assert_eq!(x.entities[0].kind, "council");+ assert!(!x.entities[0].named);+ assert_eq!(x.entities[2].name, "Brightway Builders");+ assert!(x.entities[2].named);++ assert_eq!(x.statements.len(), 3);+ let s = &x.statements[0];+ assert_eq!(s.subject, "city council");+ assert_eq!(s.phrase, "awarded");+ assert_eq!(s.object.as_deref(), Some("paving contract"));+ assert_eq!(s.value, None);+ assert_eq!(+ by_key(s.qualifiers.clone()),+ pairs(&[("to", "Brightway Builders"), ("amount", "$2 million")])+ );+ assert_eq!(s.when.as_deref(), Some("March 4, 2011"));+ assert_eq!(s.ended, None);+ assert_eq!(s.quote.as_deref(), Some(AWARD));++ let s = &x.statements[1];+ assert_eq!(+ s.subject, "Nebula Technologies Inc.",+ "已知清单里的名字照字面收"+ );+ assert_eq!(s.object.as_deref(), Some("northern wing"));+ assert_eq!(+ by_key(s.qualifiers.clone()),+ pairs(&[("from", "Harbor Estates"), ("term", "ten years")])+ );+ assert_eq!(s.when.as_deref(), Some("from 2010"));+ assert_eq!(s.ended.as_deref(), Some("until the end of 2019"));+ assert_eq!(s.quote.as_deref(), Some(LEASE));++ let s = &x.statements[2];+ assert_eq!(s.object, None);+ assert_eq!(s.value.as_deref(), Some("$2 million"));+ assert!(s.qualifiers.is_empty());+ assert_eq!(s.when, None);+ assert_eq!(s.ended, None);+ assert_eq!(s.quote.as_deref(), Some(AWARD));++ assert_eq!(x.names.len(), 1);+ assert_eq!(x.names[0].entity, "Nebula Technologies Inc.");+ assert_eq!(x.names[0].name, "Starlight Labs");+ assert_eq!(x.names[0].quote.as_deref(), Some(LEASE));+ }++ /// 截在 `n` 数组中间:`s` 之前的全留下,`truncated` 标出来+ #[test]+ fn a_cut_off_reply_keeps_what_was_complete() {+ let marker = "\"n\": [[\"Nebula Technologies Inc.\", \"Starl";+ let cut = FULL.find(marker).unwrap() + marker.len();+ let x = parse_open_response(&FULL[..cut]).unwrap();+ assert!(x.truncated, "截断要标出来");+ assert_eq!(x.entities.len(), 4);+ assert_eq!(x.statements.len(), 3);+ assert_eq!(x.statements[2].quote.as_deref(), Some(AWARD));+ assert!(x.names.is_empty());+ assert_eq!(x.skipped, 0);+ }++ /// 截在一条陈述的限定词之后:修补退到那个 `}`,留下的半条不够八格,计入 skipped+ /// 而不是收成一条没引文的陈述+ #[test]+ fn a_half_statement_left_by_the_repair_is_counted() {+ let raw = r#"{"e": [["A", "thing", 1]],+ "s": [["A", "is", null, "b", null, null, null, "A is b."], ["A", "was", null, "c", {"at": "home"}, "in 20"#;+ let x = parse_open_response(raw).unwrap();+ assert!(x.truncated);+ assert_eq!(x.statements.len(), 1);+ assert_eq!(x.skipped, 1);+ }++ /// 限定词全是 null 的回复里一个 `}` 都没有:只认 `}` 的修补会把整块作废,+ /// 这里退到最后一个完整的 `]`+ #[test]+ fn a_cut_off_reply_without_any_closing_brace_is_still_repaired() {+ let raw = r#"{"e": [["A", "thing", 1]],+ "s": [["A", "is", null, "b", null, null, null, "A is b."], ["A", "was", null, "c", nu"#;+ let x = parse_open_response(raw).unwrap();+ assert!(x.truncated);+ assert_eq!(x.entities.len(), 1);+ assert_eq!(x.statements.len(), 1);+ assert_eq!(x.statements[0].value.as_deref(), Some("b"));+ assert_eq!(x.skipped, 0);+ }++ /// 连一个完整条目都没有时仍然报失败——容错不是把空结果说成成功+ #[test]+ fn a_reply_with_nothing_complete_still_fails() {+ assert!(parse_open_response(r#"{"e": [["a", "b"#).is_err());+ }++ /// 七格的陈述是坏条目:跳过并计数,八格的照收+ #[test]+ fn a_statement_with_seven_slots_is_skipped_and_counted() {+ let raw = r#"{"e": [["A", "thing", 1]],+ "s": [["A", "is", null, "b", null, null, null],+ ["A", "is", null, "b", null, null, null, "A is b."]]}"#;+ let x = parse_open_response(raw).unwrap();+ assert_eq!(x.statements.len(), 1);+ assert_eq!(x.statements[0].quote.as_deref(), Some("A is b."));+ assert_eq!(x.skipped, 1);+ assert!(!x.truncated);+ }++ /// 字面值写成 JSON 数字也照它的写法收成文本+ #[test]+ fn a_number_in_the_value_slot_becomes_text() {+ let raw = r#"{"e": [["a", "k", 1]],+ "s": [["a", "cut water use", null, 12, null, null, null, "q"],+ ["a", "ratio", null, 1.5, null, null, null, "q"]]}"#;+ let x = parse_open_response(raw).unwrap();+ assert_eq!(x.statements[0].value.as_deref(), Some("12"));+ assert_eq!(x.statements[1].value.as_deref(), Some("1.5"));+ assert_eq!(x.skipped, 0);+ }++ /// `e` 里重名(去首尾空白、内部空白折一、不分大小写)留第一个,其余计入 skipped+ #[test]+ fn a_duplicate_entity_name_keeps_the_first_and_is_counted() {+ let raw = r#"{"e": [["Harbor Bridge", "bridge", 1], [" harbor bridge ", "span", 0],+ ["HARBOR BRIDGE", "bridge", 1], ["Harbor Bridge Authority", "agency", 1]]}"#;+ let x = parse_open_response(raw).unwrap();+ assert_eq!(x.entities.len(), 2);+ assert_eq!(x.entities[0].name, "Harbor Bridge");+ assert_eq!(x.entities[0].kind, "bridge");+ assert!(x.entities[0].named);+ assert_eq!(x.entities[1].name, "Harbor Bridge Authority");+ assert_eq!(x.skipped, 2);+ }++ /// 坏条目逐个计数:没名字的实体、不是数组的实体、没主语的陈述、空短语的陈述、+ /// 少一格的陈述、不是数组的陈述、没别名的 n、实体格放了对象的 n、实体名为空的 n+ #[test]+ fn malformed_items_are_counted_not_fatal() {+ let raw = r#"{"e": [["A", "thing", 1], [""], [5, "x"], "not an array", []],+ "s": [["A", "is", null, "b", null, null, null, "A is b."],+ [null, "is", null, "b", null, null, null, "A is b."],+ ["A", " ", null, "b", null, null, null, "A is b."],+ ["A", "is", null, "b", null, null, null],+ {"subject": "A"}],+ "n": [["A", "Ay", "A is b."], ["A"], [{}, "Ay", "q"], ["", "Ay", "q"]]}"#;+ let x = parse_open_response(raw).unwrap();+ assert_eq!(x.entities.len(), 1);+ assert_eq!(x.statements.len(), 1);+ assert_eq!(x.names.len(), 1);+ assert_eq!(x.skipped, 4 + 4 + 3);+ assert!(!x.truncated);+ }++ /// 可选格放错了类型只是那一格为 None,整条照收:宾语放数字、时间放数组、终止放+ /// 对象、引文放数字、字面值放布尔、限定词放字符串+ #[test]+ fn a_wrong_type_in_an_optional_slot_is_null_not_malformed() {+ let raw = r#"{"e": [["A", "thing", 1]],+ "s": [["A", "is", 3, true, "not an object", [2019], {"y": 2020}, 7]]}"#;+ let x = parse_open_response(raw).unwrap();+ assert_eq!(x.statements.len(), 1);+ let s = &x.statements[0];+ assert_eq!(s.object, None);+ assert_eq!(s.value, None);+ assert!(s.qualifiers.is_empty());+ assert_eq!(s.when, None);+ assert_eq!(s.ended, None);+ assert_eq!(s.quote, None);+ assert_eq!(x.skipped, 0);+ }++ /// 限定词:字符串是名字或原文,数字照写法转文本,其余类型与空键悄悄丢+ #[test]+ fn qualifiers_take_strings_and_numbers_and_drop_the_rest() {+ let raw = r#"{"e": [["a", "k", 1]],+ "s": [["a", "cut water use", null, "12%", {"compared to": "2019", "at": "Harbor Estates", "by": 3, "ratio": 1.5, "flag": true, "who": null, "list": ["x"], "": "x", "blank": " "}, null, null, "q"]]}"#;+ let x = parse_open_response(raw).unwrap();+ assert_eq!(+ by_key(x.statements[0].qualifiers.clone()),+ pairs(&[+ ("compared to", "2019"),+ ("at", "Harbor Estates"),+ ("by", "3"),+ ("ratio", "1.5"),+ ])+ );+ assert_eq!(x.skipped, 0);+ }++ /// 每个字符串都去首尾空白:名字、种类、主语、短语、宾语、时间、引文、限定词的键与值+ #[test]+ fn every_string_is_trimmed() {+ let raw = r#"{"e": [[" A ", " thing ", 1]],+ "s": [[" A ", " is ", " B ", null, {" at ": " home "}, " in 2019 ", " until 2020 ", " A is B. "]],+ "n": [[" A ", " Ay ", " A is B. "]]}"#;+ let x = parse_open_response(raw).unwrap();+ assert_eq!(x.entities[0].name, "A");+ assert_eq!(x.entities[0].kind, "thing");+ let s = &x.statements[0];+ assert_eq!(s.subject, "A");+ assert_eq!(s.phrase, "is");+ assert_eq!(s.object.as_deref(), Some("B"));+ assert_eq!(s.qualifiers, vec![("at".to_string(), "home".to_string())]);+ assert_eq!(s.when.as_deref(), Some("in 2019"));+ assert_eq!(s.ended.as_deref(), Some("until 2020"));+ assert_eq!(s.quote.as_deref(), Some("A is B."));+ assert_eq!(x.names[0].entity, "A");+ assert_eq!(x.names[0].name, "Ay");+ assert_eq!(x.names[0].quote.as_deref(), Some("A is B."));+ }++ /// 宾语与字面值同时有:照收,两个都给调用方看,这里不裁+ #[test]+ fn a_statement_with_both_object_and_value_is_passed_through() {+ let raw = r#"{"e": [["A", "thing", 1], ["B", "thing", 1]],+ "s": [["A", "is", "B", "b", null, null, null, "A is b."]]}"#;+ let x = parse_open_response(raw).unwrap();+ assert_eq!(x.statements.len(), 1);+ assert_eq!(x.statements[0].object.as_deref(), Some("B"));+ assert_eq!(x.statements[0].value.as_deref(), Some("b"));+ assert_eq!(x.skipped, 0);+ }++ /// named 认 1/0 与 true/false,缺了当点了名;kind 缺了是空串+ #[test]+ fn named_takes_numbers_and_booleans_and_defaults_to_true() {+ let raw = r#"{"e": [["a", "k", 1], ["b", "k", 0], ["c", "k", true],+ ["d", "k", false], ["e"]]}"#;+ let x = parse_open_response(raw).unwrap();+ let named: Vec<bool> = x.entities.iter().map(|e| e.named).collect();+ assert_eq!(named, vec![true, false, true, false, true]);+ assert_eq!(x.entities[4].kind, "");+ assert_eq!(x.skipped, 0);+ }++ /// 围栏与前后废话照旧容忍(走的是同一个 json_block)+ #[test]+ fn a_fenced_reply_parses() {+ let raw = "Here you go:\n```json\n{\"e\": [[\"a\", \"k\", 1]], \"s\": [], \"n\": []}\n```";+ let x = parse_open_response(raw).unwrap();+ assert_eq!(x.entities.len(), 1);+ assert!(!x.truncated);+ }++ fn known(handle: &str, name: &str, type_key: &str) -> KnownEntity {+ KnownEntity {+ handle: handle.into(),+ type_key: type_key.into(),+ name: name.into(),+ }+ }++ /// 提示词里没有文档日期、没有类型与关系清单、没有编号与句柄;有文件名、开头、+ /// 已知名字与正文,并且按这个顺序+ #[test]+ fn the_prompt_carries_no_ontology_and_no_document_date() {+ let msgs = build_open_messages(+ "annual-report.txt",+ &[+ known("k1", "Nebula Technologies Inc.", "organization"),+ known("k2", "Harbor Estates", ""),+ ],+ Some("Nebula Technologies Inc. Annual Report 2019"),+ "Nebula leased the northern wing.",+ );+ assert_eq!(msgs.len(), 2);+ assert_eq!(msgs[0].role, "system");+ assert_eq!(msgs[1].role, "user");+ let system = &msgs[0].content;+ let user = &msgs[1].content;+ for s in [system, user] {+ assert!(!s.contains("Document date"), "{s}");+ assert!(!s.contains("Resolve relative time"), "{s}");+ assert!(!s.contains("Entity types"), "{s}");+ assert!(!s.contains("Relation types"), "{s}");+ assert!(!s.contains("Attributes ("), "{s}");+ assert!(!s.contains("\"q\""), "没有引文表: {s}");+ assert!(!s.contains("\"t\""), "没有时间表: {s}");+ assert!(!s.contains("k1"), "句柄不给模型看: {s}");+ }+ assert!(system.contains("Never compute, convert or normalise a date"));+ assert!(system.contains("do not translate it into any vocabulary of your own"));+ assert!(system.contains("written exactly as listed in \"e\""));+ assert!(system.contains("copied verbatim"));+ assert!(system.contains("write nothing about them"));+ assert!(system.contains(+ "\"s\": [[\"city council\", \"awarded\", \"paving contract\", null, {\"to\": \"Brightway Builders\""+ ));++ let doc = user.find("Document: annual-report.txt").unwrap();+ let opening = user.find("Opening of this document").unwrap();+ assert!(user.contains("Annual Report 2019"));+ let recorded = user.find("Already recorded from earlier parts").unwrap();+ assert!(user.contains("\n- Nebula Technologies Inc. (organization)\n"));+ assert!(user.contains("\n- Harbor Estates\n"), "空类型不留括号");+ assert!(user.contains("write its name exactly as it is listed here"));+ let passage = user+ .find("Passage:\nNebula leased the northern wing.")+ .unwrap();+ assert!(doc < opening && opening < recorded && recorded < passage);+ assert!(user.ends_with("Nebula leased the northern wing."));+ }++ /// 第一块:没有开头也没有已知实体,两段都不出现+ #[test]+ fn the_first_chunk_carries_neither_opening_nor_known_block() {+ let msgs = build_open_messages("a.txt", &[], None, "text");+ let user = &msgs[1].content;+ assert_eq!(user, "Document: a.txt\n\nPassage:\ntext");+ assert!(!user.contains("Opening of this document"));+ assert!(!user.contains("Already recorded"));+ }+}diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rsindex cb035664..45005f29 100644--- a/crates/utopia-llm/src/lib.rs+++ b/crates/utopia-llm/src/lib.rs@@ -307,11 +307,25 @@ impl LlmClient {req}- /// 非流式对话(连通性测试等轻量场景)。+ /// 非流式对话(连通性测试等轻量场景)。不传温度:请求体与从前一字不变,端点用它的缺省pub async fn chat(&self, messages: &[ChatMessage]) -> anyhow::Result<String> {+ self.chat_at(messages, None).await+ }++ /// 非流式对话,指定采样温度。抽取这类「照抄原文」的活要 0:端点缺省是 1.0,同一段+ /// 文字连问两次,一次给 6 条陈述一次给 19 条(#729 实测),密度全看运气+ pub async fn chat_at(+ &self,+ messages: &[ChatMessage],+ temperature: Option<f32>,+ ) -> anyhow::Result<String> {+ let mut body = json!({ "model": self.model, "messages": messages, "stream": false });+ if let Some(t) = temperature {+ body["temperature"] = json!(t);+ }let resp = self.request("/chat/completions")- .json(&json!({ "model": self.model, "messages": messages, "stream": false }))+ .json(&body).send().await.map_err(Unreachable)?;diff --git a/crates/utopia-server/src/api/kbs.rs b/crates/utopia-server/src/api/kbs.rsindex 89a4a71d..57f7c69c 100644--- a/crates/utopia-server/src/api/kbs.rs+++ b/crates/utopia-server/src/api/kbs.rs@@ -55,6 +55,10 @@ pub struct UpdateKbReq {/// 提示词都读它。探索生成的描述在另一个字段,PATCH 不了。见 #570#[serde(default)]pub data_conventions: Option<String>,+ /// 开放抽取(0044 第一刀,#729):开着,抽取只写开放图谱——陈述照文档的字落库,+ /// 不读本体;关着走今天的类型化那条路。缺省关+ #[serde(default)]+ pub open_extraction: Option<bool>,}/// 用户可见的 KB 列表(restricted 库仅矩阵成员与系统管理员可见)。@@ -115,6 +119,7 @@ pub async fn create(None,None,None,+ None,).await?;}@@ -197,6 +202,7 @@ pub async fn update(req.auto_type_resolution,req.governance,req.data_conventions.as_deref().map(str::trim),+ req.open_extraction,).await?;// 打开开关就自动开始处理:排一轮,同库已排着的不重复diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rsindex 59a028a0..f7a8931e 100644--- a/crates/utopia-server/src/extraction.rs+++ b/crates/utopia-server/src/extraction.rs@@ -43,18 +43,29 @@ fn jitter(base: Duration) -> Duration {/// 否则一个在等的分块会挡住本来可以通过的另一个。/// - **`Retry-After` 多数厂商不发**,所以它只是「有则更准」,判据是错误类型/// 本身;没有它就走指数退避。-async fn chat_retrying_rate_limits(+pub(crate) async fn chat_retrying_rate_limits(state: &AppState,settings: &utopia_core::models::LlmSettings,client: &utopia_llm::LlmClient,messages: &[utopia_llm::ChatMessage],+) -> anyhow::Result<String> {+ chat_retrying_rate_limits_at(state, settings, client, messages, None).await+}++/// 同上,指定采样温度(开放抽取要 0,见 `extraction_open`)。+pub(crate) async fn chat_retrying_rate_limits_at(+ state: &AppState,+ settings: &utopia_core::models::LlmSettings,+ client: &utopia_llm::LlmClient,+ messages: &[utopia_llm::ChatMessage],+ temperature: Option<f32>,) -> anyhow::Result<String> {let mut backoff = Duration::from_secs(2);for attempt in 1..=RATE_LIMIT_TRIES {// 许可只包住调用本身,出了这个块就还回去let outcome = {let _permit = llm_util::acquire_chat(state, settings).await;- client.chat(messages).await+ client.chat_at(messages, temperature).await};let err = match outcome {Ok(reply) => return Ok(reply),@@ -88,7 +99,7 @@ const MIN_CONFIDENCE: f32 = 0.6;const DESCRIBED_CEILING: f32 = utopia_store::temporal::AUTO_CLOSE_MIN_CONFIDENCE - 0.05;/// 这块文字的来源给事实置信度设的上限-fn origin_ceiling(origin: &str, confidence: f32) -> f32 {+pub(crate) fn origin_ceiling(origin: &str, confidence: f32) -> f32 {if origin == "described" {confidence.min(DESCRIBED_CEILING)} else {@@ -181,7 +192,7 @@ fn name_claimed_elsewhere(name: &str, bound: Uuid, declared: &HashMap<String, Uu}/// 片段在不在引文里:大小写、空白都不论-fn span_in_quote(span: &str, quote: &str) -> bool {+pub(crate) fn span_in_quote(span: &str, quote: &str) -> bool {let norm = |s: &str| {s.split_whitespace().collect::<Vec<_>>()@@ -479,7 +490,7 @@ fn is_entity_name(name: &str) -> bool {/// 记一条丢弃信号。抽取器有七处 `continue`,每一处都是"事实抽出来了、被挡掉、/// 什么都不说"。信号写失败绝不能带垮整篇文档的抽取,所以这里吞掉错误。-async fn drop_signal(+pub(crate) async fn drop_signal(state: &AppState,kb_id: Uuid,document_id: Uuid,@@ -506,7 +517,7 @@ async fn drop_signal(/// `attempted` 是**本轮取到的分块数**,不是文档总块数:重试只取/// `extracted_at IS NULL` 的块,所以第二轮的分母天然更小。措辞里说「本轮」,/// 别让读的人以为文档只有那么几块。-fn incomplete_reason(unextracted: &[(i32, String)], attempted: usize) -> Option<String> {+pub(crate) fn incomplete_reason(unextracted: &[(i32, String)], attempted: usize) -> Option<String> {if unextracted.is_empty() {return None;}@@ -761,7 +772,7 @@ fn no_ref_name_binding(}#[allow(clippy::too_many_arguments)]-async fn resolve_handle(+pub(crate) async fn resolve_handle(pool: &PgPool,kb_id: Uuid,type_id: Option<Uuid>,@@ -911,7 +922,8 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow://// 没配嵌入模型的库不等,照旧送完整本体——那种部署本来就没有检索。// 等也有期限(`jobs::DEFER_WINDOW_SECS`),补齐任务一直失败时这篇按失败处理。- if ontology_index::gate_required(state, doc.kb_id).await? {+ let kb = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;+ if !kb.open_extraction && ontology_index::gate_required(state, doc.kb_id).await? {// gate_required 已经把 `embed_ontology` 入队过了(如果应该入队的话)。// 这里只挂等待时长,不重复 enqueue。attempts 会被 mark_failed 退回去——// 同一个等待条件两次排队不应该消耗两次预算。@@ -919,7 +931,6 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:.context(utopia_core::Deferred::new(Duration::from_secs(30)));return Err(err);}- let kb = utopia_store::kbs::get(&state.pool, doc.kb_id).await?;let settings = utopia_store::settings::get(&state.pool, kb.workspace_id).await?.ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?;@@ -930,6 +941,14 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:let my_epoch = utopia_store::documents::extract_epoch(&state.pool, document_id).await?;utopia_store::documents::set_graph_status(&state.pool, document_id, "extracting").await?;state.emit_document(doc.kb_id, document_id);+ // **开放图谱**(0044 第 1 刀,#729):开关开着的库只写文档自己的话,本体不进提示词。+ // 记忆日志不走这条路——那里的事实要等人点头(0015),而待确认表里没有短语这一列+ if kb.open_extraction+ && !utopia_store::memory::is_memory_document(&state.pool, document_id).await?+ {+ return crate::extraction_open::run_open(state, &doc, &kb, &settings, &client, my_epoch)+ .await;+ }let etypes = utopia_store::graph::entity_types(&state.pool, doc.kb_id).await?;// 这一轮落过的事实(新建或重复观察):结尾对它们跑一遍签名检查let mut touched_facts: Vec<Uuid> = Vec::new();diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rsnew file mode 100644index 00000000..f7b3ea5f--- /dev/null+++ b/crates/utopia-server/src/extraction_open.rs@@ -0,0 +1,588 @@+//! 开放图谱的写入路径(0044 第 1 刀,#729):文档说了什么,就按它自己的话记下来。+//!+//! 和 `extraction::run` 的分别只有一处:**提示词里没有本体**,模型不选关系、不选类、+//! 不算日期。回复里是它提到的东西(`e`,有名字的和只被描述的)、它做的陈述(`s`,关系+//! 短语照抄,主宾按名字写,时间词照抄,引文整句照抄)和别名(`n`)。落库时陈述成+//! `layer = 'open'` 的事实行,短语留在行上;限定按文档自己的角色词挂在+//! `statement_qualifiers`;时间词原样进 `time_mentions`,谁也不把它算成日期——那是+//! 0045 的事。类型化的事实由对齐(第 2 刀)从这些行算出来,不在这里写。+//!+//! **陈述按名字指东西,不按编号。** 第一版让 `e` 带编号、陈述写编号、引文按句号索引,+//! 省的是输出 token;实测 deepseek-v4-flash 在密的段落里会把编号对错——同一块两次回复+//! 一次「NVIDIA has reached AI」一次「tokens are tokens」,原文说的是「AI has reached its+//! inflection point」。原型按名字写主宾、每条陈述抄引文,333 条里编造 0 条。忠实先于省钱+//!+//! 复用的是身份那一段:名字照旧走 `resolve_handle`(同一回复里两个同名的东西不会+//! 塌成一个,0041 的名字事实照记),被描述的东西建成没有名字事实的实体+//! (`create_described`),免得「一家医院」成了召回的桥。+//!+//! 两根时间轴(0022 / #714):`attested_from` 永远是此刻;`attested_at` 只在文档日期+//! 来自内容或来源系统时才用它——上传时刻与文件修改时间都不是文档说的日期。++use crate::extraction::{+ chat_retrying_rate_limits_at, drop_signal, incomplete_reason, origin_ceiling, resolve_handle,+ span_in_quote,+};+use crate::state::AppState;+use std::collections::{HashMap, HashSet};+use utopia_core::models::{Document, KnowledgeBase, LlmSettings};+use utopia_store::extraction_drops::reason;+use utopia_store::graph::FactObject;+use uuid::Uuid;++/// 文档日期只在它来自内容或来源系统时才算证据日期(与 `temporal::DATED_AT` 同一口径)。+fn dated_at(doc: &Document) -> Option<chrono::DateTime<chrono::Utc>> {+ if matches!(doc.doc_time_source.as_str(), "content" | "source") {+ doc.doc_time+ } else {+ None+ }+}++/// 一段字在块里的**字符**偏移(起、止)。字符不是字节:界面和 SQL 的 `substr` 都按字符数,+/// 中文一个字三个字节,按字节存的偏移到界面上就错位。找不到原样的就 `None`——+/// 偏移只能由服务端从原文算出来,模型报的数字不算数+fn locate(hay: &str, needle: &str) -> Option<(i32, i32)> {+ let needle = needle.trim();+ if needle.is_empty() {+ return None;+ }+ let byte = hay.find(needle)?;+ let start = hay[..byte].chars().count();+ let end = start + needle.chars().count();+ Some((start as i32, end as i32))+}++/// 时间词在块里的字符起点。**必须在这条陈述自己的那句引文里**:模型会把一个时间词挂到+/// 好几条陈述上(FDA 语料实测「week 4」挂到了「不应由过敏患者服用」上,六条带时间的陈述错了三条),+/// 整块里搜得到不等于这句说了它。引文里有、但引文本身没在块里定位到的,起点退回整块里的第一处+fn locate_time(chunk: &str, quote: Option<(&str, Option<(i32, i32)>)>, words: &str) -> Option<i32> {+ let (q, span) = quote?;+ let (inner, _) = locate(q, words)?;+ match span {+ Some((start, _)) => Some(start + inner),+ None => locate(chunk, words).map(|(s, _)| s),+ }+}++/// 名字的查找键:空白折叠、小写。陈述里写的名字和 `e` 里列的名字要一字不差,+/// 差的只许是空白和大小写+fn name_key(name: &str) -> String {+ name.split_whitespace()+ .collect::<Vec<_>>()+ .join(" ")+ .to_lowercase()+}++pub(crate) async fn run_open(+ state: &AppState,+ doc: &Document,+ kb: &KnowledgeBase,+ settings: &LlmSettings,+ client: &utopia_llm::LlmClient,+ my_epoch: i32,+) -> anyhow::Result<()> {+ let pool = &state.pool;+ let document_id = doc.id;+ let kb_id = kb.id;+ // 本轮从头讲一遍这篇文档的故事,旧信号先清掉(重抽自动作数)+ let _ = utopia_store::extraction_drops::clear_for_document(pool, document_id).await;+ let attested_at = dated_at(doc);+ let chunks = utopia_store::documents::chunks_for_extraction(pool, document_id).await?;+ let opening_chunk = utopia_store::documents::opening_chunk(pool, document_id).await?;++ // 本文档已认下的**有名字的**实体,按首次出现排序,送进后续分块的提示词;+ // 陈述按名字指它们。被描述的东西不进清单:「一家医院」在下一块里指的未必是同一家+ let mut doc_entities: Vec<(Uuid, String, String)> = Vec::new();+ let mut known_by_name: HashMap<String, Uuid> = HashMap::new();+ // 被描述的东西按描述文字在本文档内复用:同一块里「the northern wing」说了三次是一个东西+ let mut described: HashMap<String, Uuid> = HashMap::new();+ let mut handled_by_name: HashMap<String, Vec<Uuid>> = HashMap::new();+ let mut ambiguous_bare_cache: HashMap<String, Uuid> = HashMap::new();+ let mut touched_names: HashSet<String> = HashSet::new();+ let mut unextracted: Vec<(i32, String)> = Vec::new();+ let mut needs_adjudication = false;+ let mut human_reviews_found = false;+ let mut statement_count = 0usize;++ for chunk in chunks.iter() {+ // 被接管则安静退场(重抽自增 epoch):检查放在调用模型之前+ if utopia_store::documents::extract_epoch(pool, document_id).await? != my_epoch {+ tracing::info!(%document_id, "抽取任务已被新一轮接管,退出");+ return Ok(());+ }+ let ctx: Option<&[f32]> = chunk.embedding.as_ref().map(|v| v.as_slice());+ let known: Vec<utopia_extract::KnownEntity> = doc_entities+ .iter()+ .enumerate()+ .map(|(index, (_, kind, name))| utopia_extract::KnownEntity {+ handle: format!("k{}", index + 1),+ type_key: kind.clone(),+ name: name.clone(),+ })+ .collect();+ let opening = opening_chunk+ .as_ref()+ .filter(|(id, _)| *id != chunk.id)+ .map(|(_, text)| text.as_str());+ let messages =+ utopia_extract::open::build_open_messages(&doc.filename, &known, opening, &chunk.text);+ // 温度 0:照抄原文的活不该靠采样。端点缺省 1.0 时同一块两次回复密度差三倍+ let reply = match chat_retrying_rate_limits_at(+ state,+ settings,+ client,+ &messages,+ Some(0.0),+ )+ .await+ {+ Ok(r) => r,+ Err(e) => {+ tracing::warn!(%document_id, seq = chunk.seq, error = %e, "开放抽取调用失败,跳过该分块");+ unextracted.push((chunk.seq, format!("调用失败:{e}")));+ continue;+ }+ };+ tracing::debug!(%document_id, seq = chunk.seq, reply = %reply, "开放抽取的原始回复");+ let extraction = match utopia_extract::open::parse_open_response(&reply) {+ Ok(x) => x,+ Err(e) => {+ tracing::warn!(%document_id, seq = chunk.seq, error = %e, "开放抽取回复解析失败,跳过该分块");+ unextracted.push((chunk.seq, format!("回复解析失败:{e}")));+ continue;+ }+ };+ if extraction.truncated {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::TRUNCATED_REPLY,+ "the open reply was cut off; kept up to the last complete item",+ None,+ )+ .await;+ }+ if extraction.skipped > 0 {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::MALFORMED_ITEM,+ &format!(+ "{} items in the open reply were malformed",+ extraction.skipped+ ),+ None,+ )+ .await;+ }++ // ---- 东西:有名字的走身份消解,被描述的建成没有名字事实的实体 ----+ let mut response_claims: HashMap<String, Vec<Uuid>> = HashMap::new();+ let mut local: HashMap<String, Uuid> = HashMap::new();+ for e in &extraction.entities {+ let name = e.name.trim();+ if name.is_empty() {+ continue;+ }+ let kind = e.kind.trim();+ let key = name_key(name);+ if local.contains_key(&key) {+ continue;+ }+ let id = if e.named {+ let id = resolve_handle(+ pool,+ kb_id,+ None,+ name,+ ctx,+ Some(&chunk.text),+ &mut response_claims,+ &mut handled_by_name,+ &mut ambiguous_bare_cache,+ &mut needs_adjudication,+ &mut human_reviews_found,+ )+ .await?;+ // 模型说它是什么(「a British film」里的 film):存成它自己的说法,+ // 类的绑定等对齐来做+ if !kind.is_empty() {+ let _ = utopia_store::resolution::set_specific_type(pool, id, kind).await;+ }+ // 名字就在这一块原文里时,给名字事实补出处(0041)+ if span_in_quote(name, &chunk.text) {+ let _ = utopia_store::names::record(+ pool,+ kb_id,+ id,+ name,+ Some(utopia_store::names::NameSource {+ chunk_id: chunk.id,+ quote: name,+ }),+ attested_at,+ )+ .await;+ }+ touched_names.insert(utopia_store::resolution::normalize_name(name).to_lowercase());+ if !doc_entities.iter().any(|(x, _, _)| *x == id) {+ doc_entities.push((id, kind.to_string(), name.to_string()));+ known_by_name.entry(key.clone()).or_insert(id);+ }+ id+ } else {+ match described.get(&key) {+ Some(id) => *id,+ None => {+ let id = utopia_store::resolution::create_described(+ pool,+ kb_id,+ name,+ (!kind.is_empty()).then_some(kind),+ )+ .await?;+ described.insert(key.clone(), id);+ id+ }+ }+ };+ local.insert(key, id);+ }+ // 陈述里写的名字 → 实体:先看这一块列出的,再看本文档前面认下的(提示词里的清单)+ let resolve_name = |name: &str| -> Option<Uuid> {+ let key = name_key(name);+ local.get(&key).or_else(|| known_by_name.get(&key)).copied()+ };++ // ---- 陈述 ----+ for s in &extraction.statements {+ let phrase = s.phrase.trim();+ if phrase.is_empty() {+ continue;+ }+ let Some(subject) = resolve_name(&s.subject) else {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::UNKNOWN_REF,+ "a statement's subject is not a listed thing",+ Some(&s.subject),+ )+ .await;+ continue;+ };+ // 宾语写了名字但没在清单上:模型漏列了它。陈述照落,宾语落成字面值——+ // 不凭空建实体,也不丢这条话(#559 的那一档)+ // 一个字母数字都没有的值(表格里的「—」、空格)什么也没说:按没有值处理。+ // 这是结构判断,不是词表——看的是有没有内容,不是内容是什么+ let mut value = s+ .value+ .as_deref()+ .map(str::trim)+ .filter(|v| v.chars().any(char::is_alphanumeric));+ let object = match s.object.as_deref().map(str::trim).filter(|o| !o.is_empty()) {+ Some(name) => match resolve_name(name) {+ Some(id) => Some(id),+ None => {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::OBJECT_UNDECLARED,+ "a statement's object is not a listed thing; written as a value",+ Some(name),+ )+ .await;+ value = value.or(Some(name));+ None+ }+ },+ None => None,+ };+ let value_json;+ let fact_object = match (object, value) {+ (Some(o), _) if o == subject => {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::OBJECT_MISSING,+ "a statement points at its own subject",+ Some(phrase),+ )+ .await;+ continue;+ }+ (Some(o), _) => FactObject::Entity(o),+ (None, Some(v)) => {+ value_json = serde_json::json!({ "value": v });+ FactObject::Value(&value_json)+ }+ (None, None) => {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::OBJECT_MISSING,+ "a statement has neither an object nor a value",+ Some(phrase),+ )+ .await;+ continue;+ }+ };+ // 引文定位。定位不到的那句照样当证据文字记,只是没有偏移,并记一笔——+ // 模型没照抄的句子是「不是原文说的」那一类的苗头,量它+ let quote_text = s.quote.as_deref().map(str::trim).filter(|q| !q.is_empty());+ let quote: Option<(&str, Option<(i32, i32)>)> =+ quote_text.map(|q| (q, locate(&chunk.text, q)));+ if let Some((q, None)) = quote {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::QUOTE_NOT_IN_CHUNK,+ "a quoted sentence is not in the chunk verbatim",+ Some(q),+ )+ .await;+ }+ // 开放陈述没有模型自报的置信度:它说的是「文档这么说了」。看图描述出来的块+ // 照旧压上限(0040 决定 4)+ let confidence = origin_ceiling(&chunk.origin, 1.0);+ let (fact_id, _created) = utopia_store::graph::insert_open_statement(+ pool,+ kb_id,+ subject,+ phrase,+ fact_object,+ attested_at,+ confidence,+ )+ .await?;+ // 表层谓词也写短语:今天的读路径都从 `fact_surface_predicate` 取名字+ utopia_store::graph::add_evidence_located(+ pool,+ fact_id,+ chunk.id,+ quote.map(|(q, _)| q),+ Some(phrase),+ quote.and_then(|(_, span)| span),+ )+ .await?;+ // 限定:值是清单上某个东西的名字就挂实体,否则挂文字+ for (role, text) in &s.qualifiers {+ let (role, text) = (role.trim(), text.trim());+ if role.is_empty() || text.is_empty() {+ continue;+ }+ match resolve_name(text) {+ Some(id) => {+ utopia_store::graph::add_statement_qualifier(+ pool,+ fact_id,+ role,+ None,+ Some(id),+ )+ .await?+ }+ None => {+ utopia_store::graph::add_statement_qualifier(+ pool,+ fact_id,+ role,+ Some(&serde_json::json!(text)),+ None,+ )+ .await?+ }+ }+ }+ // 时间词:起与止各是一条提及,必须原样在这条陈述的引文里+ for words in [s.when.as_deref(), s.ended.as_deref()]+ .into_iter()+ .flatten()+ .map(str::trim)+ .filter(|w| !w.is_empty())+ {+ match locate_time(&chunk.text, quote, words) {+ Some(start) => {+ utopia_store::time_mentions::record(+ pool, kb_id, fact_id, chunk.id, words, start,+ )+ .await?;+ }+ None => {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::TIME_NOT_IN_QUOTE,+ "a time mention's words are not in the statement's own sentence",+ Some(words),+ )+ .await;+ }+ }+ }+ statement_count += 1;+ }++ // ---- 别名(0041 决定 2):服务端只核对名字确实在这一块原文里 ----+ for n in &extraction.names {+ let name = n.name.trim();+ if name.is_empty() {+ continue;+ }+ let Some(id) = resolve_name(&n.entity) else {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::UNKNOWN_REF,+ "a name's thing is not a listed thing",+ Some(&n.entity),+ )+ .await;+ continue;+ };+ if !span_in_quote(name, &chunk.text) {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::NAME_NOT_IN_TEXT,+ "an extra name is not in the chunk",+ Some(name),+ )+ .await;+ continue;+ }+ // 一个名字不会同时是两样东西的名字:这一块或本文档前面已经把它认成+ // 另一个实体的,不记+ let key = utopia_store::resolution::normalize_name(name).to_lowercase();+ let claimed = handled_by_name+ .get(&key)+ .is_some_and(|ids| ids.iter().any(|x| *x != id));+ if claimed {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::NAME_CLAIMED_BY_ANOTHER,+ "an extra name is already another thing's name",+ Some(name),+ )+ .await;+ continue;+ }+ let quote = n+ .quote+ .as_deref()+ .map(str::trim)+ .filter(|q| !q.is_empty())+ .unwrap_or(&chunk.text);+ let _ = utopia_store::names::record(+ pool,+ kb_id,+ id,+ name,+ Some(utopia_store::names::NameSource {+ chunk_id: chunk.id,+ quote,+ }),+ attested_at,+ )+ .await;+ touched_names.insert(key);+ }++ // 本块抽完即打标:更新时被认领的块携带标记跳过;中断的抽取可续跑+ utopia_store::documents::mark_chunk_extracted(pool, chunk.id).await?;+ }++ // 消歧后缀在实体创建时算会早于其事实写入——收尾时对本文档涉及的名字统一刷新+ for name in &touched_names {+ utopia_store::resolution::refresh_disambiguators(pool, kb_id, name).await?;+ }+ if utopia_store::documents::extract_epoch(pool, document_id).await? != my_epoch {+ tracing::info!(%document_id, "抽取任务已被新一轮接管,收尾时退出");+ return Ok(());+ }+ // 有分块没抽成就不许标 done(与类型化那条路同一条规矩)+ if let Some(msg) = incomplete_reason(&unextracted, chunks.len()) {+ return Err(anyhow::anyhow!(msg));+ }+ utopia_store::documents::set_graph_status(pool, document_id, "done").await?;+ state.emit_document(kb_id, document_id);+ state.emit_graph(kb_id);++ // 灰区对进了审核队列 → 治理 / 裁决任务,同库已排着的不重复。+ // 不排类型消解、不排自动扩本体:开放图谱里没有类也没有关系可扩,那是对齐的事+ if kb.governance {+ if needs_adjudication || human_reviews_found {+ utopia_store::jobs::enqueue_unless_queued(+ pool,+ "govern",+ serde_json::json!({ "kb_id": kb_id }),+ )+ .await?;+ }+ } else if needs_adjudication {+ utopia_store::jobs::enqueue_unless_queued(+ pool,+ "adjudicate_entities",+ serde_json::json!({ "kb_id": kb_id }),+ )+ .await?;+ }+ if needs_adjudication || human_reviews_found {+ state.emit_review(kb_id);+ }+ tracing::info!(%document_id, statements = statement_count, "开放图谱抽取完成");+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn offsets_are_in_characters_not_bytes() {+ let text = "北京的冬天很冷。去年冬天下了三场雪。";+ assert_eq!(locate(text, "去年冬天"), Some((8, 12)));+ assert_eq!(locate(text, " 去年冬天 "), Some((8, 12)));+ assert_eq!(locate(text, "前年冬天"), None);+ assert_eq!(locate(text, ""), None);+ }++ #[test]+ fn a_time_is_found_inside_its_own_sentence_first() {+ let text = "In 2019 the plant opened. In 2019 it closed again.";+ let quote = ("In 2019 it closed again.", Some((26, 50)));+ assert_eq!(locate_time(text, Some(quote), "2019"), Some(29));+ // 引文里有、引文自己没定位到:起点退回整块里的第一处+ assert_eq!(+ locate_time(text, Some(("it closed in 2019", None)), "2019"),+ Some(3)+ );+ // 不在这句引文里的时间不算这条陈述的,哪怕块里别处有+ assert_eq!(+ locate_time(text, Some(("the plant opened.", Some((8, 25)))), "2019"),+ None+ );+ assert_eq!(locate_time(text, None, "2019"), None);+ }++ #[test]+ fn names_match_up_to_whitespace_and_case() {+ assert_eq!(name_key(" Harbor Bridge "), "harbor bridge");+ assert_eq!(name_key("harbor bridge"), name_key("HARBOR BRIDGE"));+ assert_ne!(name_key("Harbor Bridge"), name_key("Harbour Bridge"));+ }+}diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex b1ea1bb2..194e134f 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -8,6 +8,7 @@ mod client_ctx;mod docs_corpus;mod error;mod extraction;+mod extraction_open;mod github_issues;mod governance;mod http_fetch;diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rsindex 54e7a037..0ba9cad0 100644--- a/crates/utopia-store/src/extraction_drops.rs+++ b/crates/utopia-store/src/extraction_drops.rs@@ -97,6 +97,15 @@ pub mod reason {pub const ORPHAN_DECLARATION: &str = "orphan_declaration";/// 引文抄自提示词里附的文件开头、不在这一块:证据会挂错出处,不落pub const QUOTE_FROM_OPENING: &str = "quote_from_opening";+ /// 以下三条来自开放抽取那条路(0044 第一刀,#729)+ /// 模型的引文在这一块里找不到原样的一句:陈述照落、引文照记,只是没有偏移+ /// (`quote_start` / `quote_end` 留空)。这个信号数的是有多少条没定位到+ pub const QUOTE_NOT_IN_CHUNK: &str = "quote_not_in_chunk";+ /// 时间词不在它的引文里、也不在这一块里:不记这条时间提及。时间词是照抄的字,+ /// 抄不出来的字就不是文档说的+ pub const TIME_NOT_IN_QUOTE: &str = "time_not_in_quote";+ /// 陈述、属性或名字指着一个回复里不存在的实体编号或已知句柄:跳过这一条+ pub const UNKNOWN_REF: &str = "unknown_ref";}pub async fn record(diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rsindex d7919f30..ff46538c 100644--- a/crates/utopia-store/src/graph.rs+++ b/crates/utopia-store/src/graph.rs@@ -201,6 +201,91 @@ pub async fn fact_qualifiers_for(Ok(out)}+/// 开放陈述上的一个属性(0044 第一刀):按文档的角色词记,不按本体属性。+/// 值或实体二选一;实体那一格带上它的显示名+#[derive(Debug, Clone, serde::Serialize)]+pub struct StatementQualifier {+ /// 文档自己的角色词("amount"、"buyer")+ pub role: String,+ pub value: Option<serde_json::Value>,+ pub entity_id: Option<Uuid>,+ pub entity_name: Option<String>,+}++/// 往一条开放陈述上写一个按角色词记的属性。**先写者留着**:同一条陈述同一个角色词+/// 再听到一次不覆盖(与 [`upsert_fact_qualifier`] 的 Conflict 一条规矩——两次观察+/// 不一致该另立一行加一条冲突,这一刀还没走到那一步)。值与实体必须恰好给一个+pub async fn add_statement_qualifier(+ pool: &PgPool,+ fact_id: Uuid,+ role: &str,+ value: Option<&serde_json::Value>,+ entity_id: Option<Uuid>,+) -> AppResult<()> {+ if value.is_some() == entity_id.is_some() {+ return Err(AppError::invalid(+ "qualifier_shape",+ "a statement qualifier carries exactly one of a value or an entity",+ ));+ }+ let role = role.trim();+ if role.is_empty() {+ return Err(AppError::invalid(+ "qualifier_role",+ "a statement qualifier needs the document's role word",+ ));+ }+ sqlx::query(+ "INSERT INTO statement_qualifiers (fact_id, role, value, entity_id)+ VALUES ($1, $2, $3, $4) ON CONFLICT (fact_id, role) DO NOTHING",+ )+ .bind(fact_id)+ .bind(role)+ .bind(value)+ .bind(entity_id)+ .execute(pool)+ .await?;+ Ok(())+}++/// 一批开放陈述各自的角色词属性,按事实 id 取回;实体那一格连上显示名+pub async fn statement_qualifiers_for(+ pool: &PgPool,+ fact_ids: &[Uuid],+) -> AppResult<HashMap<Uuid, Vec<StatementQualifier>>> {+ let mut out: HashMap<Uuid, Vec<StatementQualifier>> = HashMap::new();+ if fact_ids.is_empty() {+ return Ok(out);+ }+ #[derive(sqlx::FromRow)]+ struct Row {+ fact_id: Uuid,+ role: String,+ value: Option<serde_json::Value>,+ entity_id: Option<Uuid>,+ entity_name: Option<String>,+ }+ let rows: Vec<Row> = sqlx::query_as(+ "SELECT q.fact_id, q.role, q.value, q.entity_id, e.canonical_name AS entity_name+ FROM statement_qualifiers q+ LEFT JOIN entities e ON e.id = q.entity_id+ WHERE q.fact_id = ANY($1)+ ORDER BY q.fact_id, q.role",+ )+ .bind(fact_ids)+ .fetch_all(pool)+ .await?;+ for r in rows {+ out.entry(r.fact_id).or_default().push(StatementQualifier {+ role: r.role,+ value: r.value,+ entity_id: r.entity_id,+ entity_name: r.entity_name,+ });+ }+ Ok(out)+}+#[allow(clippy::too_many_arguments)]pub async fn insert_fact(pool: &PgPool,@@ -690,9 +775,11 @@ async fn insert_fact_inner(.execute(pool).await?;sqlx::query(- // 表层谓词随证据一起搬:精化的是时间,不是原文说了什么- "INSERT INTO fact_evidence (fact_id, chunk_id, quote, proposed_predicate, document_id, doc_version)- SELECT $1, chunk_id, quote, proposed_predicate, document_id, doc_version+ // 表层谓词随证据一起搬:精化的是时间,不是原文说了什么。引文的偏移一起搬+ "INSERT INTO fact_evidence (fact_id, chunk_id, quote, proposed_predicate, document_id, doc_version,+ quote_start, quote_end)+ SELECT $1, chunk_id, quote, proposed_predicate, document_id, doc_version,+ quote_start, quote_endFROM fact_evidence WHERE fact_id = $2ON CONFLICT DO NOTHING",)@@ -768,6 +855,93 @@ pub async fn insert_value_fact(.await}+/// 一条开放陈述(0044 第一刀,#729):文档说了什么,用它自己的话。+/// 返回 (事实 id, 是否新建)。+///+/// 落的是 `facts` 里 `layer = 'open'` 的一行:短语照写在 `phrase` 上、`predicate_id`+/// 为空(`facts_open_statement_shape` 在挡)。去重按 (库, 主语, 短语, 宾语) 在活着的+/// 开放行里找——今天空谓词的行按 (主, 谓, 宾) 去重、谓词是 NULL 就永远撞不上,每次+/// 观察都插一行;短语进了键之后同一句话再听到一次就是同一行,证据累积到它上面。+/// 撞上了只把 `attested_from` 往早挪([`attest_earlier`])。+///+/// **不走 [`insert_fact_inner`] 那道门**:不问 `predicate_temporal`,不过+/// `Validity::under`,不写任何 `valid_*`,也不碰时间线。一条开放陈述在世界轴上还+/// 没有位置——它提到的时间是照抄的字(`time_mentions`),把字读成日期是 0045+/// 后面那几刀的事。这里只有记录轴:`attested_from` 是文档日期(调用方只在+/// `doc_time_source IN ('content', 'source')` 时传,#714)或此刻。+///+/// 调用方随后要把 `proposed_predicate = phrase` 写到证据上([`add_evidence_located`]),+/// 于是所有已经容得下空谓词的读路径(`fact_surface_predicate`)不改一字就按短语显示它+pub async fn insert_open_statement(+ pool: &PgPool,+ kb_id: Uuid,+ subject_id: Uuid,+ phrase: &str,+ object: FactObject<'_>,+ attested_at: Option<chrono::DateTime<chrono::Utc>>,+ confidence: f32,+) -> AppResult<(Uuid, bool)> {+ let phrase = phrase.trim();+ if phrase.is_empty() {+ return Err(AppError::invalid(+ "phrase_missing",+ "an open statement keeps the document's relation phrase",+ ));+ }+ let same_sql = match object {+ FactObject::Entity(_) => {+ "SELECT id FROM facts+ WHERE layer = 'open' AND kb_id = $1 AND subject_id = $2 AND phrase = $3+ AND object_id = $4 AND invalidated_at IS NULL+ ORDER BY recorded_at LIMIT 1"+ }+ FactObject::Value(_) => {+ "SELECT id FROM facts+ WHERE layer = 'open' AND kb_id = $1 AND subject_id = $2 AND phrase = $3+ AND object_value = $4 AND object_id IS NULL AND invalidated_at IS NULL+ ORDER BY recorded_at LIMIT 1"+ }+ };+ let mut q = sqlx::query_scalar(same_sql)+ .bind(kb_id)+ .bind(subject_id)+ .bind(phrase);+ q = match object {+ FactObject::Entity(id) => q.bind(id),+ FactObject::Value(v) => q.bind(v),+ };+ let same: Option<Uuid> = q.fetch_optional(pool).await?;+ if let Some(existing) = same {+ attest_earlier(pool, existing, attested_at).await?;+ return Ok((existing, false));+ }++ let id = Uuid::now_v7();+ let insert_sql = match object {+ FactObject::Entity(_) => {+ "INSERT INTO facts (id, kb_id, subject_id, layer, phrase, object_id,+ confidence, attested_from)+ VALUES ($1, $2, $3, 'open', $4, $5, $6, COALESCE($7, now()))"+ }+ FactObject::Value(_) => {+ "INSERT INTO facts (id, kb_id, subject_id, layer, phrase, object_value,+ confidence, attested_from)+ VALUES ($1, $2, $3, 'open', $4, $5, $6, COALESCE($7, now()))"+ }+ };+ let mut ins = sqlx::query(insert_sql)+ .bind(id)+ .bind(kb_id)+ .bind(subject_id)+ .bind(phrase);+ ins = match object {+ FactObject::Entity(oid) => ins.bind(oid),+ FactObject::Value(v) => ins.bind(v),+ };+ ins.bind(confidence).bind(attested_at).execute(pool).await?;+ Ok((id, true))+}+/// `proposed`:模型在这一块里实际提议的谓词。命中本体时它等于 key,/// 本体外的谓词不落到关系上时,它是唯一还留着原意的东西——事实行上只剩/// "有关联",原文说的"runs on"就靠这里活下来。@@ -777,6 +951,22 @@ pub async fn add_evidence(chunk_id: Uuid,quote: Option<&str>,proposed: Option<&str>,+) -> AppResult<()> {+ add_evidence_located(pool, fact_id, chunk_id, quote, proposed, None).await+}++/// [`add_evidence`],外加引文在 `chunks.text` 里的位置(0044 第一刀)。+///+/// `span` 是字符偏移(不是字节)的 `(start, end)`,**由服务端搜文本算出来,从不取+/// 模型报的数**;没定位到就传 `None`,两列留空。同一对 (事实, 分块) 再写一次只在+/// 原值为空时补上偏移,与表层谓词一条规矩:第一次记下的就是它的+pub async fn add_evidence_located(+ pool: &PgPool,+ fact_id: Uuid,+ chunk_id: Uuid,+ quote: Option<&str>,+ proposed: Option<&str>,+ span: Option<(i32, i32)>,) -> AppResult<()> {// 证据落笔即记版本:出自哪份文档的第几版(S3 版本对账与"证据过期"判定的依据)// 冲突时补写表层谓词而非整行跳过:重抽命中的多是已有的 (事实, 分块) 对,@@ -811,15 +1001,21 @@ pub async fn add_evidence(}}sqlx::query(- "INSERT INTO fact_evidence (fact_id, chunk_id, quote, proposed_predicate, document_id, doc_version)- SELECT $1, $2, $3, left($4, 120), c.document_id, c.doc_version FROM chunks c WHERE c.id = $2+ "INSERT INTO fact_evidence (fact_id, chunk_id, quote, proposed_predicate, document_id, doc_version,+ quote_start, quote_end)+ SELECT $1, $2, $3, left($4, 120), c.document_id, c.doc_version, $5, $6+ FROM chunks c WHERE c.id = $2ON CONFLICT (fact_id, chunk_id) DO UPDATE- SET proposed_predicate = COALESCE(fact_evidence.proposed_predicate, EXCLUDED.proposed_predicate)",+ SET proposed_predicate = COALESCE(fact_evidence.proposed_predicate, EXCLUDED.proposed_predicate),+ quote_start = COALESCE(fact_evidence.quote_start, EXCLUDED.quote_start),+ quote_end = COALESCE(fact_evidence.quote_end, EXCLUDED.quote_end)",).bind(target).bind(chunk_id).bind(quote).bind(proposed)+ .bind(span.map(|(s, _)| s))+ .bind(span.map(|(_, e)| e)).execute(&mut *tx).await?;tx.commit().await?;@@ -1902,12 +2098,15 @@ pub async fn proposed_predicates(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<Pr//// 走 CTE 而不是相关子查询:后者每组重扫一遍证据表,同一份数据上// 360ms 对 7ms。这个函数每次 Suggest 和每次自动扩本体都要跑。+ // **只看类型化那一层。** 开放陈述(0044)的短语也写在证据的 proposed_predicate+ // 上,好让读路径按它显示;但它不是等着被采纳的说法——把一个开放短语自动+ // 采纳成关系类型,正是那一刀说不做的事(对齐与绑定是第二刀)"WITH spread AS (SELECT e.proposed_predicate AS form,count(DISTINCT e.document_id) AS doc_countFROM fact_evidence eJOIN facts ff ON ff.id = e.fact_id- WHERE ff.kb_id = $1 AND e.proposed_predicate IS NOT NULL+ WHERE ff.kb_id = $1 AND ff.layer = 'typed' AND e.proposed_predicate IS NOT NULLGROUP BY 1)SELECT fe.proposed_predicate AS form,@@ -1919,12 +2118,13 @@ pub async fn proposed_predicates(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<PrJOIN entities s ON s.id = f2.subject_idJOIN entities o ON o.id = f2.object_idWHERE e2.proposed_predicate = fe.proposed_predicate- AND f2.kb_id = $1 AND f2.predicate_id IS NULL AND f2.invalidated_at IS NULL+ AND f2.kb_id = $1 AND f2.layer = 'typed'+ AND f2.predicate_id IS NULL AND f2.invalidated_at IS NULLLIMIT 1) AS exampleFROM fact_evidence feJOIN facts f ON f.id = fe.fact_idJOIN spread sp ON sp.form = fe.proposed_predicate- WHERE f.kb_id = $1 AND f.predicate_id IS NULL+ WHERE f.kb_id = $1 AND f.layer = 'typed' AND f.predicate_id IS NULLAND f.invalidated_at IS NULL AND fe.proposed_predicate IS NOT NULL-- 字面值宾语的不算:它们同样没有谓词、也带原文说法,但要的是-- 一个属性而不是一个关系。混进来提案就会照着建关系,然后@@ -1965,7 +2165,7 @@ pub async fn proposed_predicate_documents("SELECT DISTINCT fe.proposed_predicate, fe.document_idFROM fact_evidence feJOIN facts f ON f.id = fe.fact_id- WHERE f.kb_id = $1+ WHERE f.kb_id = $1 AND f.layer = 'typed'AND fe.proposed_predicate IS NOT NULLAND fe.document_id IS NOT NULLAND NOT EXISTS (SELECT 1 FROM ontology_misses m@@ -2061,6 +2261,8 @@ async fn adopt("SELECT f.id, f.subject_id, f.object_id, f.object_valueFROM facts fWHERE f.kb_id = $1 AND f.predicate_id IS NULL AND f.invalidated_at IS NULL+ -- 开放陈述(0044)不采纳:短语挂上谓词是第二刀的对齐做的事+ AND f.layer = 'typed'-- **只碰宾语是实体的。** 同一个说法可能既有指向实体的事实-- 又有带字面值的(location 两种都用),后者归属性那条路:-- 把它改挂到一条关系上,那个值就再也不是值了@@ -2086,7 +2288,8 @@ async fn adopt(let ids: Vec<Uuid> = items.iter().map(|(id, _)| *id).collect();let live: Vec<(Uuid, Uuid)> = sqlx::query_as("SELECT id, subject_id FROM facts- WHERE kb_id = $1 AND id = ANY($2) AND invalidated_at IS NULL",+ WHERE kb_id = $1 AND id = ANY($2) AND invalidated_at IS NULL+ AND layer = 'typed'",).bind(kb_id).bind(&ids)@@ -2197,8 +2400,10 @@ async fn adopt(// 证据整体搬过去,表层谓词一并保留——它是这次改写的依据,不该在改写中丢失sqlx::query(- "INSERT INTO fact_evidence (fact_id, chunk_id, quote, proposed_predicate, document_id, doc_version)- SELECT $1, chunk_id, quote, proposed_predicate, document_id, doc_version+ "INSERT INTO fact_evidence (fact_id, chunk_id, quote, proposed_predicate, document_id, doc_version,+ quote_start, quote_end)+ SELECT $1, chunk_id, quote, proposed_predicate, document_id, doc_version,+ quote_start, quote_endFROM fact_evidence WHERE fact_id = $2ON CONFLICT DO NOTHING",)@@ -2322,12 +2527,13 @@ pub async fn proposed_attributes(Ok(sqlx::query_as(// 同 proposed_predicates:普遍程度从全量证据数,改写量从积压数;// 走 CTE 而不是相关子查询,后者每组重扫一遍证据表+ // 同 proposed_predicates:开放陈述(0044)的短语不是等着被采纳的说法"WITH spread AS (SELECT e.proposed_predicate AS form,count(DISTINCT e.document_id) AS doc_countFROM fact_evidence eJOIN facts ff ON ff.id = e.fact_id- WHERE ff.kb_id = $1 AND e.proposed_predicate IS NOT NULL+ WHERE ff.kb_id = $1 AND ff.layer = 'typed' AND e.proposed_predicate IS NOT NULLGROUP BY 1)SELECT fe.proposed_predicate AS form,@@ -2337,7 +2543,7 @@ pub async fn proposed_attributes(FROM fact_evidence e2JOIN facts f2 ON f2.id = e2.fact_idWHERE e2.proposed_predicate = fe.proposed_predicate- AND f2.kb_id = $1 AND f2.predicate_id IS NULL+ AND f2.kb_id = $1 AND f2.layer = 'typed' AND f2.predicate_id IS NULLAND f2.object_id IS NULL AND f2.invalidated_at IS NULLLIMIT 1) AS example,-- 主语实际是什么类:属性的 domain 从这里来,不靠猜@@ -2347,12 +2553,12 @@ pub async fn proposed_attributes(JOIN entities s ON s.id = f3.subject_idJOIN entity_types t ON t.id = s.type_idWHERE e3.proposed_predicate = fe.proposed_predicate- AND f3.kb_id = $1 AND f3.predicate_id IS NULL+ AND f3.kb_id = $1 AND f3.layer = 'typed' AND f3.predicate_id IS NULLAND f3.object_id IS NULL AND f3.invalidated_at IS NULL) AS domain_keysFROM fact_evidence feJOIN facts f ON f.id = fe.fact_idJOIN spread sp ON sp.form = fe.proposed_predicate- WHERE f.kb_id = $1 AND f.predicate_id IS NULL+ WHERE f.kb_id = $1 AND f.layer = 'typed' AND f.predicate_id IS NULLAND f.invalidated_at IS NULL AND fe.proposed_predicate IS NOT NULLAND f.object_id IS NULL-- 拒绝过的说法不再出现在候选里@@ -2391,7 +2597,8 @@ pub async fn value_facts_for_forms("SELECT DISTINCT f.id, s.type_id, f.object_valueFROM facts fJOIN entities s ON s.id = f.subject_id- WHERE f.kb_id = $1 AND f.predicate_id IS NULL AND f.invalidated_at IS NULL+ WHERE f.kb_id = $1 AND f.layer = 'typed'+ AND f.predicate_id IS NULL AND f.invalidated_at IS NULLAND f.object_id IS NULL AND f.object_value IS NOT NULLAND EXISTS (SELECT 1 FROM fact_evidence eWHERE e.fact_id = f.id AND e.proposed_predicate = ANY($2))",diff --git a/crates/utopia-store/src/kbs.rs b/crates/utopia-store/src/kbs.rsindex af2baec8..14077b09 100644--- a/crates/utopia-store/src/kbs.rs+++ b/crates/utopia-store/src/kbs.rs@@ -110,6 +110,7 @@ pub async fn update(auto_type_resolution: Option<bool>,governance: Option<bool>,data_conventions: Option<&str>,+ open_extraction: Option<bool>,) -> AppResult<KnowledgeBase> {// 改语言不回头重写已有的类——它们已经是这个库的数据,可能有人手工调过。// 这一列往后管的是**新**描述(自动扩本体、AI 建议)写成什么语言@@ -151,6 +152,8 @@ pub async fn update(ELSE governance_since END,-- 人写的约定;清空要送空串,送 null 等于不改(与 description 同一约定)data_conventions = COALESCE($11, data_conventions),+ -- 开放抽取(0044 第一刀):抽取只写开放图谱;关着走类型化那条路+ open_extraction = COALESCE($12, open_extraction),updated_at = now()WHERE id = $1 RETURNING *",)@@ -165,6 +168,7 @@ pub async fn update(.bind(auto_type_resolution).bind(governance).bind(data_conventions)+ .bind(open_extraction).fetch_optional(pool).await?.ok_or(AppError::NotFound)diff --git a/crates/utopia-store/src/lib.rs b/crates/utopia-store/src/lib.rsindex e25e3203..5301b4b0 100644--- a/crates/utopia-store/src/lib.rs+++ b/crates/utopia-store/src/lib.rs@@ -38,6 +38,7 @@ pub mod settings;pub mod sources;pub mod temporal;pub mod test_db;+pub mod time_mentions;pub mod tokens;pub mod vector_index;pub mod workspaces;diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rsindex 38f0e5de..514a29ce 100644--- a/crates/utopia-store/src/resolution.rs+++ b/crates/utopia-store/src/resolution.rs@@ -262,6 +262,9 @@ pub async fn resolve_mention(AND f.invalidated_at IS NULL AND {not_name}) AS degreeFROM entities eWHERE e.kb_id = $1 AND e.type_id = $2 AND e.merged_into IS NULL+ -- 被描述的东西没有名字(0044):它的 canonical_name 只是显示用的描述,+ -- 不是召回的桥——两篇文档里描述得一样的两个东西不能因此接到一起+ AND e.description IS NULLAND (lower(e.canonical_name) = ANY($3) OR {named})",not_name = crate::names::not_a_name("f"),named = crate::names::has_name_in("e", 1, 3),@@ -482,7 +485,12 @@ async fn corroborating_candidate<'a>(return Ok(None);}let ids: Vec<Uuid> = candidates.iter().map(|c| c.id).collect();- // 一次查完所有候选:每个候选取若干个「同伴不指向」的宾语名字+ // 一次查完所有候选:每个候选取若干个「同伴不指向」的宾语名字。+ //+ // **LEFT JOIN relation_types**(0044):开放陈述没有谓词、只有照抄的短语,+ // 它指着的宾语一样是这个候选的画像——「收购了 Beta」不因为本体没接住+ // 「收购了」就不算佐证。排序里关系的三个布尔位对开放行是 NULL,COALESCE+ // 成 false 排在类型化行之后:类型化行之间的次序一字不变let rows: Vec<(Uuid, String)> = sqlx::query_as("SELECT s.subject_id, s.name FROM (SELECT f.subject_id, o.canonical_name AS name,@@ -490,12 +498,12 @@ async fn corroborating_candidate<'a>(PARTITION BY f.subject_idORDER BY EXISTS (SELECT 1 FROM relation_type_ranges rrWHERE rr.relation_type_id = r.id) DESC,- (r.temporal = 'state') DESC,- r.functional DESC,+ COALESCE(r.temporal = 'state', FALSE) DESC,+ COALESCE(r.functional, FALSE) DESC,f.confidence DESC, f.recorded_at DESC) AS rnFROM facts f- JOIN relation_types r ON r.id = f.predicate_id+ LEFT JOIN relation_types r ON r.id = f.predicate_idJOIN entities o ON o.id = f.object_idWHERE f.kb_id = $1 AND f.subject_id = ANY($2)AND f.invalidated_at IS NULL AND f.object_id IS NOT NULL@@ -828,6 +836,8 @@ async fn resolve_type_drift(-- IS DISTINCT FROM 而不是 <>:后者遇 NULL 返回 NULL,被 WHERE 当假,-- 未分类实体会被整个漏掉(0009)WHERE e.kb_id = $1 AND e.type_id IS DISTINCT FROM $2 AND e.merged_into IS NULL+ -- 被描述的东西不参与名字召回(0044),同上+ AND e.description IS NULLAND (lower(e.canonical_name) = ANY($3) OR {named})",named = crate::names::has_name_in("e", 1, 3),))@@ -990,9 +1000,10 @@ async fn create_entity(.execute(&mut *tx).await?;let existing: Option<(Uuid,)> = sqlx::query_as(+ // 被描述的东西不算「同名的它」(0044):它的 canonical_name 只是显示用的描述"SELECT id FROM entitiesWHERE kb_id = $1 AND canonical_name = $2 AND type_id IS NOT DISTINCT FROM $3- AND merged_into IS NULL AND id <> ALL($4)+ AND merged_into IS NULL AND description IS NULL AND id <> ALL($4)ORDER BY id LIMIT 1",).bind(kb_id)@@ -1035,6 +1046,46 @@ async fn create_entity(Ok((id, true))}+/// 一个被描述、没有名字的东西(0044 第一刀,#729):"the buyer's parent company"、+/// 「那家上海的子公司」。它是一个实体——陈述要指着它——但它没有名字。+///+/// `canonical_name` 写成描述本身,**只为显示**;`description` 是它真正的身份。+/// **不写 `known_as`**:名字事实是召回的桥(0041),一段描述做桥会把两篇文档里+/// 碰巧描述得一样的两个东西接到一起。没有名字就没有桥,后来的提及找不到它——+/// 这是对的:一段描述指的是谁,要靠对齐(第二刀)而不是靠字面相同。+///+/// `kind_word` 是文档自己的类别词("company"、「子公司」),落在 `specific_type`+/// 上;`type_id` 留空,`type_source` 走缺省的 extracted。+///+/// 朴素插入:不加锁、不找候选、不去重——同一篇文档里同一段描述由调用方并成一个。+/// 两篇文档各描述一次,就是两个实体,等对齐来判它们是不是一个+pub async fn create_described(+ pool: &PgPool,+ kb_id: Uuid,+ description: &str,+ kind_word: Option<&str>,+) -> AppResult<Uuid> {+ let description = description.trim();+ if description.is_empty() {+ return Err(AppError::invalid(+ "description_missing",+ "a described thing needs the document's description",+ ));+ }+ let id = Uuid::now_v7();+ sqlx::query(+ "INSERT INTO entities (id, kb_id, type_id, canonical_name, description, specific_type)+ VALUES ($1, $2, NULL, $3, $3, left($4, 80))",+ )+ .bind(id)+ .bind(kb_id)+ .bind(description)+ .bind(kind_word.map(str::trim).filter(|k| !k.is_empty()))+ .execute(pool)+ .await?;+ Ok(id)+}+async fn touch_entity(pool: &PgPool, id: Uuid) -> AppResult<()> {sqlx::query("UPDATE entities SET updated_at = now() WHERE id = $1").bind(id)@@ -1117,6 +1168,8 @@ pub async fn existing_by_name(let id: Option<Uuid> = sqlx::query_scalar(&format!("SELECT e.id FROM entities eWHERE e.kb_id = $1 AND e.merged_into IS NULL+ -- 被描述的东西不参与名字召回(0044):描述不是桥+ AND e.description IS NULLAND (lower(e.canonical_name) = ANY($2) OR {named})ORDER BY (SELECT count(*) FROM facts fWHERE (f.subject_id = e.id OR f.object_id = e.id) AND {not_name}) DESC,@@ -1154,9 +1207,12 @@ pub async fn refresh_disambiguators(pool: &PgPool, kb_id: Uuid, name: &str) -> Alet peers: Vec<Uuid> = group.iter().map(|(id,)| *id).collect();for (id,) in &group {+ // LEFT JOIN(0044):开放陈述指着的宾语也能当消歧后缀——「张三 · 星云科技」+ // 不因为「任职于」没进本体就写不出来。关系的布尔位对开放行 COALESCE 成 false,+ // 类型化行之间的次序不变let label: Option<(String,)> = sqlx::query_as("SELECT o.canonical_name FROM facts f- JOIN relation_types r ON r.id = f.predicate_id+ LEFT JOIN relation_types r ON r.id = f.predicate_idJOIN entities o ON o.id = f.object_idWHERE f.kb_id = $1 AND f.subject_id = $2AND f.invalidated_at IS NULL AND f.object_id IS NOT NULL@@ -1168,8 +1224,8 @@ pub async fn refresh_disambiguators(pool: &PgPool, kb_id: Uuid, name: &str) -> AAND g.invalidated_at IS NULL)) DESC,EXISTS (SELECT 1 FROM relation_type_ranges rrWHERE rr.relation_type_id = r.id) DESC,- (r.temporal = 'state') DESC,- r.functional DESC,+ COALESCE(r.temporal = 'state', FALSE) DESC,+ COALESCE(r.functional, FALSE) DESC,f.confidence DESC, f.recorded_at DESCLIMIT 1",)@@ -1301,10 +1357,12 @@ pub async fn entity_fact_lines(valid_to: Option<DateTime<Utc>>,}// 名字不算审阅卡上的一条事实(0041):两个同名实体各有一条「known as 张伟」,- // 摆出来像是一条共同证据,其实它什么也分不出来+ // 摆出来像是一条共同证据,其实它什么也分不出来。+ // 开放陈述(0044)按它照抄的短语读:`phrase` 在关系标签之后、证据众数之前——+ // 类型化行的 phrase 是 NULL,它们的显示一字不变let rows: Vec<Line> = sqlx::query_as(&format!("SELECT CASE WHEN f.subject_id = $2 THEN 'out' ELSE 'in' END AS direction,- COALESCE(r.label, fact_surface_predicate(f.id)) AS predicate_label,+ COALESCE(r.label, f.phrase, fact_surface_predicate(f.id)) AS predicate_label,o.canonical_name AS other_name,f.valid_from, f.valid_toFROM facts f@@ -1313,7 +1371,7 @@ pub async fn entity_fact_lines(ON o.id = CASE WHEN f.subject_id = $2 THEN f.object_id ELSE f.subject_id ENDWHERE f.kb_id = $1 AND f.invalidated_at IS NULLAND (f.subject_id = $2 OR f.object_id = $2)- AND COALESCE(r.label, fact_surface_predicate(f.id)) IS NOT NULL+ AND COALESCE(r.label, f.phrase, fact_surface_predicate(f.id)) IS NOT NULLAND {not_name}ORDER BY f.confidence DESC, f.recorded_at DESCLIMIT $3",diff --git a/crates/utopia-store/src/temporal.rs b/crates/utopia-store/src/temporal.rsindex 3a91a824..bdf5bbdb 100644--- a/crates/utopia-store/src/temporal.rs+++ b/crates/utopia-store/src/temporal.rs@@ -1069,8 +1069,10 @@ async fn copy_evidence(to: Uuid,) -> AppResult<()> {sqlx::query(- "INSERT INTO fact_evidence (fact_id, chunk_id, quote, proposed_predicate, document_id, doc_version)- SELECT $1, chunk_id, quote, proposed_predicate, document_id, doc_version+ "INSERT INTO fact_evidence (fact_id, chunk_id, quote, proposed_predicate, document_id, doc_version,+ quote_start, quote_end)+ SELECT $1, chunk_id, quote, proposed_predicate, document_id, doc_version,+ quote_start, quote_endFROM fact_evidence WHERE fact_id = $2ON CONFLICT DO NOTHING",)diff --git a/crates/utopia-store/src/time_mentions.rs b/crates/utopia-store/src/time_mentions.rsnew file mode 100644index 00000000..9997d68b--- /dev/null+++ b/crates/utopia-store/src/time_mentions.rs@@ -0,0 +1,77 @@+//! 一条陈述提到的时间词(0044 第一刀,#729;时间记录 0045)。+//!+//! **一条时间提及是文档的字,永远不是算出来的日期。**「去年冬天」「合同签署后 30 日内」+//! 「Q3」——落库的是这几个字和它们在 `chunks.text` 里的字符偏移,不是某个+//! `TIMESTAMPTZ`。今天抽取把「2023 年上半年」读成 1 月 1 日,读错了只能删掉文档重抽;+//! 字留着,读法可以改。把字读成日期(形状、锚点、偏移、粒度)是 0045 后面几刀的事,+//! 那几列到时候加在这张表上,这里写下的行一行不动。+//!+//! 一条开放陈述于是不写任何 `valid_*`:它在世界轴上还没有位置。++use sqlx::PgPool;+use std::collections::HashMap;+use utopia_core::AppResult;+use uuid::Uuid;++/// 一条时间提及:哪条陈述、哪一块、照抄的字、在块里的字符偏移+#[derive(Debug, Clone, serde::Serialize, sqlx::FromRow)]+pub struct TimeMention {+ pub id: Uuid,+ pub fact_id: Uuid,+ pub chunk_id: Uuid,+ /// 照抄的字+ pub text: String,+ /// 在 `chunks.text` 里的字符偏移(不是字节)+ pub char_start: i32,+}++/// 记一条时间提及。`char_start` 是字符偏移,**由服务端在块里搜出来**,不取模型报的数。+/// 同一条陈述在同一块的同一位置只有一行;再记一次回的是那一行的 id+pub async fn record(+ pool: &PgPool,+ kb_id: Uuid,+ fact_id: Uuid,+ chunk_id: Uuid,+ text: &str,+ char_start: i32,+) -> AppResult<Uuid> {+ let id: Uuid = sqlx::query_scalar(+ "INSERT INTO time_mentions (id, kb_id, fact_id, chunk_id, text, char_start)+ VALUES ($1, $2, $3, $4, $5, $6)+ ON CONFLICT (fact_id, chunk_id, char_start) DO UPDATE SET text = time_mentions.text+ RETURNING id",+ )+ .bind(Uuid::now_v7())+ .bind(kb_id)+ .bind(fact_id)+ .bind(chunk_id)+ .bind(text)+ .bind(char_start)+ .fetch_one(pool)+ .await?;+ Ok(id)+}++/// 一批陈述各自提到的时间词,按事实 id 取回;块内按出现位置排+pub async fn for_facts(+ pool: &PgPool,+ fact_ids: &[Uuid],+) -> AppResult<HashMap<Uuid, Vec<TimeMention>>> {+ let mut out: HashMap<Uuid, Vec<TimeMention>> = HashMap::new();+ if fact_ids.is_empty() {+ return Ok(out);+ }+ let rows: Vec<TimeMention> = sqlx::query_as(+ "SELECT id, fact_id, chunk_id, text, char_start+ FROM time_mentions+ WHERE fact_id = ANY($1)+ ORDER BY fact_id, chunk_id, char_start",+ )+ .bind(fact_ids)+ .fetch_all(pool)+ .await?;+ for m in rows {+ out.entry(m.fact_id).or_default().push(m);+ }+ Ok(out)+}diff --git a/crates/utopia-store/tests/a_described_thing_is_an_entity_without_a_name.rs b/crates/utopia-store/tests/a_described_thing_is_an_entity_without_a_name.rsnew file mode 100644index 00000000..b7719202--- /dev/null+++ b/crates/utopia-store/tests/a_described_thing_is_an_entity_without_a_name.rs@@ -0,0 +1,132 @@+//! 一个被描述的东西是一个没有名字的实体(0044 第一刀,#729)。+//!+//! `create_described` 写 `description`、不写 `known_as`:名字事实是召回的桥(0041),+//! 一段描述不能做桥——两篇文档里碰巧描述得一样的两个东西不该因此接到一起。+//! 于是按名字召回找不到它,它的名字栏是空的,`facts` 里没有一条以它为主语的行。++use sqlx::PgPool;+use utopia_store::{names, resolution};+use uuid::Uuid;++const ORG: &str = "described-thing-test";++struct Fixture {+ org: Uuid,+ kb: Uuid,+}++async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, $2)")+ .bind(org)+ .bind(ORG)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, $3)")+ .bind(ws)+ .bind(org)+ .bind(ORG)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, $3)")+ .bind(kb)+ .bind(ws)+ .bind(ORG)+ .execute(pool)+ .await?;+ Ok(Fixture { org, kb })+}++#[tokio::test]+async fn a_described_thing_has_a_description_and_no_name() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ sqlx::query("DELETE FROM organizations WHERE name = $1")+ .bind(ORG)+ .execute(&pool)+ .await?;+ let f = seed(&pool).await?;++ let run = async {+ let description = "the buyer's Shanghai subsidiary";+ let id = resolution::create_described(&pool, f.kb, description, Some("subsidiary"))+ .await?;++ let (canonical, desc, type_id, specific): (+ String,+ Option<String>,+ Option<Uuid>,+ Option<String>,+ ) = sqlx::query_as(+ "SELECT canonical_name, description, type_id, specific_type FROM entities WHERE id = $1",+ )+ .bind(id)+ .fetch_one(&pool)+ .await?;+ assert_eq!(desc.as_deref(), Some(description), "描述是它的身份");+ assert_eq!(canonical, description, "显示名就是描述本身");+ assert!(type_id.is_none(), "文档的类别词不是本体的类");+ assert_eq!(specific.as_deref(), Some("subsidiary"));++ // 没有一条以它为主语的事实——尤其没有 known_as+ let facts: i64 = sqlx::query_scalar("SELECT count(*) FROM facts WHERE subject_id = $1")+ .bind(id)+ .fetch_one(&pool)+ .await?;+ assert_eq!(facts, 0, "描述不写成名字事实");+ assert!(+ names::for_entity(&pool, f.kb, id, None).await?.is_empty(),+ "名字栏是空的"+ );++ // 按名字召回找不到它:描述不是桥+ assert_eq!(+ resolution::existing_by_name(&pool, f.kb, description).await?,+ None,+ "一段描述不能把后来的提及接到它身上"+ );+ // 名字事实那条召回也找不到它:库里有一条同样小写值的名字事实才算+ let recalled: Option<Uuid> = sqlx::query_scalar(&format!(+ "SELECT e.id FROM entities e+ WHERE e.kb_id = $1 AND e.merged_into IS NULL AND {}",+ names::has_name_in("e", 1, 2)+ ))+ .bind(f.kb)+ .bind(vec![description.to_lowercase()])+ .fetch_optional(&pool)+ .await?;+ assert_eq!(recalled, None);+ // 后来一条同样措辞的提及走消解:另建一个,不归到被描述的那个身上+ let later =+ resolution::resolve_mention(&pool, f.kb, None, description, None, None, &[]).await?;+ assert!(later.created, "描述不是桥,提及不归到它身上");+ assert_ne!(later.entity_id, id);++ // 同一段描述再建一次是另一个实体:库里不去重,同一篇里并成一个是调用方的事+ let twin = resolution::create_described(&pool, f.kb, description, None).await?;+ assert_ne!(twin, id);+ let specific: Option<String> =+ sqlx::query_scalar("SELECT specific_type FROM entities WHERE id = $1")+ .bind(twin)+ .fetch_one(&pool)+ .await?;+ assert!(specific.is_none());++ assert!(+ resolution::create_described(&pool, f.kb, " ", None)+ .await+ .is_err(),+ "空描述不是一个东西"+ );+ Ok::<(), anyhow::Error>(())+ }+ .await;++ sqlx::query("DELETE FROM organizations WHERE id = $1")+ .bind(f.org)+ .execute(&pool)+ .await?;+ run+}diff --git a/crates/utopia-store/tests/a_time_mention_is_words_not_a_date.rs b/crates/utopia-store/tests/a_time_mention_is_words_not_a_date.rsnew file mode 100644index 00000000..503f21be--- /dev/null+++ b/crates/utopia-store/tests/a_time_mention_is_words_not_a_date.rs@@ -0,0 +1,171 @@+//! 一条时间提及是字,不是日期(0044 第一刀,#729;0045)。+//!+//! 「去年冬天」落库的是这四个字和它在 `chunks.text` 里的**字符**偏移(不是字节——+//! 中文一个字三个字节,按字节记偏移界面就指错地方)。把字读成日期是后面几刀的事:+//! 这条陈述没有 `valid_from`,也没有 `valid_to`。++use sqlx::PgPool;+use utopia_store::graph::{self, FactObject};+use utopia_store::time_mentions;+use uuid::Uuid;++const ORG: &str = "time-mention-test";+const TEXT: &str = "星云科技去年冬天收购了北斗软件,作价十二亿元。";++struct Fixture {+ org: Uuid,+ kb: Uuid,+ nebula: Uuid,+ beidou: Uuid,+ chunk: Uuid,+}++async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ let (nebula, beidou) = (Uuid::now_v7(), Uuid::now_v7());+ let (doc, chunk) = (Uuid::now_v7(), Uuid::now_v7());+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, $2)")+ .bind(org)+ .bind(ORG)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, $3)")+ .bind(ws)+ .bind(org)+ .bind(ORG)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, $3)")+ .bind(kb)+ .bind(ws)+ .bind(ORG)+ .execute(pool)+ .await?;+ for (id, name) in [(nebula, "星云科技"), (beidou, "北斗软件")] {+ sqlx::query("INSERT INTO entities (id, kb_id, canonical_name) VALUES ($1, $2, $3)")+ .bind(id)+ .bind(kb)+ .bind(name)+ .execute(pool)+ .await?;+ }+ sqlx::query(+ "INSERT INTO documents (id, kb_id, filename, sha256, status)+ VALUES ($1, $2, 'deal.txt', 'timemention', 'ready')",+ )+ .bind(doc)+ .bind(kb)+ .execute(pool)+ .await?;+ sqlx::query(+ "INSERT INTO chunks (id, kb_id, document_id, seq, text) VALUES ($1, $2, $3, 0, $4)",+ )+ .bind(chunk)+ .bind(kb)+ .bind(doc)+ .bind(TEXT)+ .execute(pool)+ .await?;+ Ok(Fixture {+ org,+ kb,+ nebula,+ beidou,+ chunk,+ })+}++#[tokio::test]+async fn a_time_mention_resolves_inside_the_chunk_and_sets_no_date() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ sqlx::query("DELETE FROM organizations WHERE name = $1")+ .bind(ORG)+ .execute(&pool)+ .await?;+ let f = seed(&pool).await?;++ let run = async {+ let (fact, _) = graph::insert_open_statement(+ &pool,+ f.kb,+ f.nebula,+ "收购了",+ FactObject::Entity(f.beidou),+ None,+ 0.9,+ )+ .await?;+ graph::add_evidence(&pool, fact, f.chunk, Some(TEXT), Some("收购了")).await?;++ // 服务端在块里搜出来的**字符**偏移+ let words = "去年冬天";+ let byte_start = TEXT.find(words).unwrap();+ let char_start = TEXT[..byte_start].chars().count() as i32;+ assert_eq!(char_start, 4, "「星云科技」四个字之后");+ assert_ne!(+ byte_start as i32, char_start,+ "中文里字节偏移与字符偏移不同"+ );+ let id = time_mentions::record(&pool, f.kb, fact, f.chunk, words, char_start).await?;++ // 字与偏移在块里对得上:按字符数到那里,取同样多的字,得到的就是记下的字+ let (stored_text, stored_start): (String, i32) =+ sqlx::query_as("SELECT text, char_start FROM time_mentions WHERE id = $1")+ .bind(id)+ .fetch_one(&pool)+ .await?;+ let chunk_text: String = sqlx::query_scalar("SELECT text FROM chunks WHERE id = $1")+ .bind(f.chunk)+ .fetch_one(&pool)+ .await?;+ let at: String = chunk_text+ .chars()+ .skip(stored_start as usize)+ .take(stored_text.chars().count())+ .collect();+ assert_eq!(at, words);+ assert_eq!(stored_text, words);+ // Postgres 的 substr 也按字符数:界面用它高亮时指的是同一处+ let by_sql: String = sqlx::query_scalar(+ "SELECT substr(c.text, m.char_start + 1, char_length(m.text))+ FROM time_mentions m JOIN chunks c ON c.id = m.chunk_id WHERE m.id = $1",+ )+ .bind(id)+ .fetch_one(&pool)+ .await?;+ assert_eq!(by_sql, words);++ // 字不是日期:陈述在世界轴上没有位置+ #[derive(sqlx::FromRow)]+ struct Span {+ valid_from: Option<chrono::DateTime<chrono::Utc>>,+ valid_to: Option<chrono::DateTime<chrono::Utc>>,+ valid_from_precision: Option<String>,+ valid_to_precision: Option<String>,+ }+ let span: Span = sqlx::query_as(+ "SELECT valid_from, valid_to, valid_from_precision, valid_to_precision+ FROM facts WHERE id = $1",+ )+ .bind(fact)+ .fetch_one(&pool)+ .await?;+ assert!(span.valid_from.is_none() && span.valid_from_precision.is_none());+ assert!(span.valid_to.is_none() && span.valid_to_precision.is_none());++ let mentions = time_mentions::for_facts(&pool, &[fact]).await?;+ assert_eq!(mentions[&fact].len(), 1);+ assert_eq!(mentions[&fact][0].chunk_id, f.chunk);+ Ok::<(), anyhow::Error>(())+ }+ .await;++ sqlx::query("DELETE FROM organizations WHERE id = $1")+ .bind(f.org)+ .execute(&pool)+ .await?;+ run+}diff --git a/crates/utopia-store/tests/an_open_statement_keeps_the_documents_words.rs b/crates/utopia-store/tests/an_open_statement_keeps_the_documents_words.rsnew file mode 100644index 00000000..42dd2ac0--- /dev/null+++ b/crates/utopia-store/tests/an_open_statement_keeps_the_documents_words.rs@@ -0,0 +1,380 @@+//! 一条陈述留着文档自己的字(0044 第一刀,#729)。+//!+//! 开放陈述是 `facts` 里 `layer = 'open'` 的一行:短语照写、没有谓词。所有已经容得下+//! 空谓词的读路径(图、实体面板、邻域、路径、导出)要按短语显示它;同一句话再听到+//! 一次是同一行、证据累积;它的短语永远不出现在等着被采纳的说法里;角色词属性与+//! 时间词各自取得回来;证据带着引文在块里的偏移;库拒绝没有短语的开放行。+//!+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。++use sqlx::PgPool;+use utopia_store::graph::{self, FactObject};+use utopia_store::time_mentions;+use uuid::Uuid;++const ORG: &str = "open-statement-test";+const TEXT_A: &str = "Acme acquired Beta for $1.2 billion last winter.";+const TEXT_B: &str = "Regulators later cleared the deal: Acme acquired Beta.";++struct Fixture {+ kb: Uuid,+ acme: Uuid,+ beta: Uuid,+ chunk_a: Uuid,+ chunk_b: Uuid,+}++async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ let etype = Uuid::now_v7();+ let (acme, beta) = (Uuid::now_v7(), Uuid::now_v7());+ let (src, doc, chunk_a, chunk_b) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, $2)")+ .bind(org)+ .bind(ORG)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, $3)")+ .bind(ws)+ .bind(org)+ .bind(ORG)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, $3)")+ .bind(kb)+ .bind(ws)+ .bind(ORG)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, 'org', '组织')")+ .bind(etype)+ .bind(kb)+ .execute(pool)+ .await?;+ for (id, name) in [(acme, "Acme"), (beta, "Beta")] {+ sqlx::query(+ "INSERT INTO entities (id, kb_id, type_id, canonical_name) VALUES ($1, $2, $3, $4)",+ )+ .bind(id)+ .bind(kb)+ .bind(etype)+ .bind(name)+ .execute(pool)+ .await?;+ }+ sqlx::query("INSERT INTO sources (id, kb_id, name) VALUES ($1, $2, '临时来源')")+ .bind(src)+ .bind(kb)+ .execute(pool)+ .await?;+ sqlx::query(+ "INSERT INTO documents (id, kb_id, source_id, filename, sha256, status)+ VALUES ($1, $2, $3, 'deal.md', 'openstatement', 'ready')",+ )+ .bind(doc)+ .bind(kb)+ .bind(src)+ .execute(pool)+ .await?;+ for (id, seq, text) in [(chunk_a, 0i32, TEXT_A), (chunk_b, 1i32, TEXT_B)] {+ sqlx::query(+ "INSERT INTO chunks (id, kb_id, document_id, seq, text) VALUES ($1, $2, $3, $4, $5)",+ )+ .bind(id)+ .bind(kb)+ .bind(doc)+ .bind(seq)+ .bind(text)+ .execute(pool)+ .await?;+ }+ Ok(Fixture {+ kb,+ acme,+ beta,+ chunk_a,+ chunk_b,+ })+}++/// 引文在块里的字符偏移:服务端搜文本算出来的那个数(这里的文本是 ASCII,字节即字符)+fn span_of(text: &str, quote: &str) -> (i32, i32) {+ let start = text.find(quote).expect("quote is in the chunk");+ let chars_before = text[..start].chars().count() as i32;+ (chars_before, chars_before + quote.chars().count() as i32)+}++#[tokio::test]+async fn an_open_statement_shows_under_its_phrase_and_reuses_its_row() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ sqlx::query("DELETE FROM organizations WHERE name = $1")+ .bind(ORG)+ .execute(&pool)+ .await?;+ let f = seed(&pool).await?;++ let run = async {+ // 1. 落一条开放陈述:Acme —acquired→ Beta,短语照写、谓词为空、没有 valid_*+ let (fact, created) = graph::insert_open_statement(+ &pool,+ f.kb,+ f.acme,+ "acquired",+ FactObject::Entity(f.beta),+ None,+ 0.9,+ )+ .await?;+ assert!(created);+ let quote = TEXT_A;+ graph::add_evidence_located(+ &pool,+ fact,+ f.chunk_a,+ Some(quote),+ Some("acquired"),+ Some(span_of(TEXT_A, quote)),+ )+ .await?;+ graph::add_statement_qualifier(+ &pool,+ fact,+ "amount",+ Some(&serde_json::json!({ "value": "$1.2 billion" })),+ None,+ )+ .await?;+ let (winter_start, _) = span_of(TEXT_A, "last winter");+ let mention =+ time_mentions::record(&pool, f.kb, fact, f.chunk_a, "last winter", winter_start)+ .await?;++ #[derive(sqlx::FromRow)]+ struct Row {+ layer: String,+ phrase: Option<String>,+ predicate_id: Option<Uuid>,+ valid_from: Option<chrono::DateTime<chrono::Utc>>,+ valid_to: Option<chrono::DateTime<chrono::Utc>>,+ }+ let row: Row = sqlx::query_as(+ "SELECT layer, phrase, predicate_id, valid_from, valid_to FROM facts WHERE id = $1",+ )+ .bind(fact)+ .fetch_one(&pool)+ .await?;+ assert_eq!(row.layer, "open");+ assert_eq!(row.phrase.as_deref(), Some("acquired"));+ assert!(row.predicate_id.is_none(), "开放陈述没有谓词");+ assert!(+ row.valid_from.is_none() && row.valid_to.is_none(),+ "开放陈述在世界轴上还没有位置"+ );++ // 2. 每条读路径都按短语显示它+ let (_, edges, _, _) = graph::overview(&pool, f.kb, 50, None, None).await?;+ let e = edges+ .iter()+ .find(|e| e.id == fact)+ .expect("overview shows it");+ assert_eq!(e.label.as_deref(), Some("acquired"));+ assert!(e.inferred, "名字来自原文,不是本体认下的关系");++ let (_, facts) = graph::entity_detail(&pool, f.kb, f.acme, None, None).await?;+ let d = facts+ .iter()+ .find(|x| x.id == fact)+ .expect("entity detail shows it");+ assert_eq!(d.predicate_label.as_deref(), Some("acquired"));+ assert_eq!(d.other_id, Some(f.beta));++ let (_, edges) = graph::neighborhood(&pool, f.kb, f.acme, 1, None, None).await?;+ let n = edges+ .iter()+ .find(|e| e.id == fact)+ .expect("neighborhood shows it");+ assert_eq!(n.label.as_deref(), Some("acquired"));++ let paths = utopia_store::paths::paths_between(+ &pool,+ f.kb,+ f.acme,+ f.beta,+ None,+ None,+ utopia_store::paths::Limits::default(),+ )+ .await?;+ let direct = paths+ .iter()+ .find(|p| p.edges.len() == 1 && p.edges[0].fact_id == fact)+ .expect("a path walks the open statement");+ assert_eq!(direct.edges[0].predicate.as_deref(), Some("acquired"));++ let exported = utopia_store::export::facts_page(&pool, f.kb, None).await?;+ let x = exported+ .iter()+ .find(|x| x.id == fact)+ .expect("export carries it");+ assert_eq!(x.surface_predicate.as_deref(), Some("acquired"));+ assert!(x.predicate_id.is_none());++ // 3. 同一句话再听到一次:同一行,证据累积+ let (again, created) = graph::insert_open_statement(+ &pool,+ f.kb,+ f.acme,+ "acquired",+ FactObject::Entity(f.beta),+ None,+ 0.8,+ )+ .await?;+ assert_eq!(again, fact, "同 (主语, 短语, 宾语) 复用那一行");+ assert!(!created);+ let quote_b = "Acme acquired Beta.";+ graph::add_evidence_located(+ &pool,+ fact,+ f.chunk_b,+ Some(quote_b),+ Some("acquired"),+ Some(span_of(TEXT_B, quote_b)),+ )+ .await?;+ let evidence: Vec<(Uuid, Option<i32>, Option<i32>)> = sqlx::query_as(+ "SELECT chunk_id, quote_start, quote_end FROM fact_evidence WHERE fact_id = $1+ ORDER BY quote_start",+ )+ .bind(fact)+ .fetch_all(&pool)+ .await?;+ assert_eq!(evidence.len(), 2, "第二块的证据累积到同一行上");+ assert_eq!(evidence[0], (f.chunk_a, Some(0), Some(TEXT_A.len() as i32)));+ let (b_start, b_end) = span_of(TEXT_B, quote_b);+ assert_eq!(evidence[1], (f.chunk_b, Some(b_start), Some(b_end)));+ let live: i64 = sqlx::query_scalar(+ "SELECT count(*) FROM facts+ WHERE kb_id = $1 AND layer = 'open' AND invalidated_at IS NULL",+ )+ .bind(f.kb)+ .fetch_one(&pool)+ .await?;+ assert_eq!(live, 1, "两次观察一行");++ // 另一个宾语是另一行:去重键里有宾语+ let (other, created) = graph::insert_open_statement(+ &pool,+ f.kb,+ f.acme,+ "acquired",+ FactObject::Value(&serde_json::json!({ "value": "a rival" })),+ None,+ 0.5,+ )+ .await?;+ assert!(created && other != fact);++ // 4. 短语不是等着被采纳的说法+ let proposed = graph::proposed_predicates(&pool, f.kb).await?;+ assert!(+ proposed.iter().all(|p| p.form != "acquired"),+ "开放短语不能进采纳候选:{proposed:?}"+ );++ // 5. 角色词属性与时间词各自取得回来+ let qualifiers = graph::statement_qualifiers_for(&pool, &[fact]).await?;+ let q = &qualifiers[&fact];+ assert_eq!(q.len(), 1);+ assert_eq!(q[0].role, "amount");+ assert_eq!(+ q[0].value,+ Some(serde_json::json!({ "value": "$1.2 billion" }))+ );+ assert!(q[0].entity_id.is_none());+ // 先写者留着+ graph::add_statement_qualifier(+ &pool,+ fact,+ "amount",+ Some(&serde_json::json!({ "value": "$2 billion" })),+ None,+ )+ .await?;+ let q = &graph::statement_qualifiers_for(&pool, &[fact]).await?[&fact];+ assert_eq!(+ q[0].value,+ Some(serde_json::json!({ "value": "$1.2 billion" }))+ );+ // 值与实体必须恰好一个+ assert!(+ graph::add_statement_qualifier(&pool, fact, "buyer", None, None)+ .await+ .is_err()+ );+ graph::add_statement_qualifier(&pool, fact, "buyer", None, Some(f.acme)).await?;+ let q = &graph::statement_qualifiers_for(&pool, &[fact]).await?[&fact];+ let buyer = q.iter().find(|x| x.role == "buyer").unwrap();+ assert_eq!(buyer.entity_id, Some(f.acme));+ assert_eq!(buyer.entity_name.as_deref(), Some("Acme"));++ let mentions = time_mentions::for_facts(&pool, &[fact]).await?;+ let m = &mentions[&fact];+ assert_eq!(m.len(), 1);+ assert_eq!(m[0].id, mention);+ assert_eq!(+ (m[0].text.as_str(), m[0].char_start),+ ("last winter", winter_start)+ );+ assert_eq!(+ time_mentions::record(&pool, f.kb, fact, f.chunk_a, "last winter", winter_start)+ .await?,+ mention,+ "同一位置再记一次回的是同一行"+ );++ // 6. 库拒绝没有短语的开放行+ let bad = sqlx::query(+ "INSERT INTO facts (id, kb_id, subject_id, layer, object_id)+ VALUES ($1, $2, $3, 'open', $4)",+ )+ .bind(Uuid::now_v7())+ .bind(f.kb)+ .bind(f.acme)+ .bind(f.beta)+ .execute(&pool)+ .await;+ assert!(bad.is_err(), "开放行必须带短语");+ let bad = sqlx::query(+ "INSERT INTO facts (id, kb_id, subject_id, layer, phrase, predicate_id, object_id)+ SELECT $1, $2, $3, 'open', 'acquired', r.id, $4+ FROM relation_types r WHERE r.kb_id = $2 LIMIT 1",+ )+ .bind(Uuid::now_v7())+ .bind(f.kb)+ .bind(f.acme)+ .bind(f.beta)+ .execute(&pool)+ .await;+ // 库里没有关系类型时这句什么也不插;有的话必须被约束挡下+ if let Ok(done) = &bad {+ assert_eq!(done.rows_affected(), 0);+ }+ Ok::<(), anyhow::Error>(())+ }+ .await;++ sqlx::query("DELETE FROM organizations WHERE name = $1")+ .bind(ORG)+ .execute(&pool)+ .await?;+ run+}diff --git a/crates/utopia-store/tests/exploration_describes_the_data.rs b/crates/utopia-store/tests/exploration_describes_the_data.rsindex 20d564ea..fdf1dcc3 100644--- a/crates/utopia-store/tests/exploration_describes_the_data.rs+++ b/crates/utopia-store/tests/exploration_describes_the_data.rs@@ -51,6 +51,7 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> anNone,None,Some("Amounts are in cents.\nis_test = 1 is excluded."),+ None,).await?;// 探索写描述与问题@@ -101,6 +102,7 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> anNone,None,Some("cents only"),+ None,).await?;let got = utopia_store::kbs::get(&pool, kb).await?;@@ -119,6 +121,7 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> anNone,None,no,+ None,).await?;assert_eq!(diff --git a/migrations/0061_a_statement_keeps_the_documents_words.sql b/migrations/0061_a_statement_keeps_the_documents_words.sqlnew file mode 100644index 00000000..7d4cce41--- /dev/null+++ b/migrations/0061_a_statement_keeps_the_documents_words.sql@@ -0,0 +1,79 @@+-- 一条陈述留着文档自己的字(0044 第一刀,#729)。+--+-- 开放陈述是文档说了什么、用它自己的话:主语、照抄的关系短语、一个宾语实体或一个字面值、+-- 按文档的角色词记的属性、它提到的时间词、指回分块的出处。今天一条本体没接住的关系是能+-- 表达的(0010:`predicate_id` 为空、说法留在 `fact_evidence.proposed_predicate`),但它+-- 去重去得很差(去重键对 NULL 视而不见,每次观察都插一行)、属性必须挂在本体的某个+-- 属性上(0037)、一个被描述而没有名字的东西成不了实体(`canonical_name` 与 `known_as`+-- 都是必填)、时间只以算出来的日期存在,而且没有任何一列标出这一层是哪一层。+--+-- 全部纯增量,不回填;没有已发布的版本。+--+-- facts.layer 'typed' 是今天的那条路;'open' 是照文档的字写的陈述。+-- facts.phrase 关系短语,照写。开放行必有它、且没有 predicate_id+-- (`facts_open_statement_shape`):短语就是它的名字,本体的+-- 关系是后一刀(对齐、绑定)才给它的。`proposed_predicate`+-- 仍照写在证据上,于是所有已经容得下空谓词的读路径+-- (图、实体面板、路径、导出、RDF、工具)不改一字就按短语显示它。+-- fact_evidence.quote_* 引文在 `chunks.text` 里的字符偏移;NULL = 没定位到。+-- 偏移由服务端搜文本算出,从不取模型报的数。+-- entities.description 一个被描述、没有名字的东西。它不写 `known_as`,于是永远不+-- 成为召回的桥(0041)。+-- statement_qualifiers 开放陈述上的属性,按文档的角色词("amount"、"buyer")记,+-- 不按本体属性;值或实体二选一,与 `fact_qualifiers` 同形。+-- time_mentions 陈述提到的时间词,**照抄,永远不是算出来的日期**(0045)。+-- 解释列(形状、锚点、偏移、粒度)等后一刀。开放陈述不写任何+-- `valid_*`:它在世界轴上的位置还没有人给。+--+-- 两个时钟:`attested_from`(记录轴)总是有;`attested_at` 只在 `doc_time_source IN+-- ('content', 'source')` 时来自 `doc_time`,从不取上传时间(#714)。++-- 缺省开:抽取只写开放图谱(0044 决定 2),升级后所有库都走这条路——没有已发布的版本,+-- 不留遗留层。关掉 = 走带本体的那条老路,它按 0044 决定 3 作为已批准本体下的可选第二路+-- 保留到对齐追平为止+ALTER TABLE knowledge_bases+ ADD COLUMN open_extraction BOOLEAN NOT NULL DEFAULT TRUE;++ALTER TABLE facts+ ADD COLUMN layer TEXT NOT NULL DEFAULT 'typed' CHECK (layer IN ('typed', 'open')),+ ADD COLUMN phrase TEXT,+ ADD CONSTRAINT facts_open_statement_shape+ CHECK (layer <> 'open' OR (phrase IS NOT NULL AND predicate_id IS NULL));++-- 开放陈述按 (库, 主语, 短语) 去重,只在活着的行里找+CREATE INDEX facts_open_by_subject ON facts (kb_id, subject_id, phrase)+ WHERE layer = 'open' AND invalidated_at IS NULL;++-- 引文在 chunks.text 里的字符偏移(不是字节);NULL = 没定位到+ALTER TABLE fact_evidence+ ADD COLUMN quote_start INT,+ ADD COLUMN quote_end INT;++-- 一个被描述、没有名字的东西+ALTER TABLE entities ADD COLUMN description TEXT;++CREATE TABLE statement_qualifiers (+ fact_id UUID NOT NULL REFERENCES facts(id) ON DELETE CASCADE,+ -- 文档自己的角色词+ role TEXT NOT NULL,+ value JSONB,+ entity_id UUID REFERENCES entities(id) ON DELETE CASCADE,+ PRIMARY KEY (fact_id, role),+ CHECK ((value IS NOT NULL) <> (entity_id IS NOT NULL))+);++CREATE TABLE time_mentions (+ id UUID PRIMARY KEY,+ kb_id UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,+ fact_id UUID NOT NULL REFERENCES facts(id) ON DELETE CASCADE,+ chunk_id UUID NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,+ -- 照抄的字,永远不是算出来的日期+ text TEXT NOT NULL,+ -- 在 chunks.text 里的字符偏移+ char_start INT NOT NULL,+ recorded_at TIMESTAMPTZ NOT NULL DEFAULT now(),+ UNIQUE (fact_id, chunk_id, char_start)+);++-- 读陈述时把它的时间词一起带出来,按事实取+CREATE INDEX time_mentions_fact_idx ON time_mentions (fact_id);diff --git a/scripts/bench/README.md b/scripts/bench/README.mdindex 6b93405e..7449ae1d 100644--- a/scripts/bench/README.md+++ b/scripts/bench/README.md@@ -65,8 +65,17 @@ node scripts/bench/fetch-sec-filings.mjs # 一次就够node scripts/bench/recall.mjs --kb <id> # 一轮:清空 → 重抽 → 打分node scripts/bench/recall.mjs --kb <id> --score # 只打分node scripts/bench/recall.mjs --kb <id> --reprocess # 改了解析器:连分块一起重来+node scripts/bench/recall.mjs --kb <id> --typed # 带本体的老路(可选第二路);缺省走开放图谱(0044 第 1 刀)+node scripts/bench/judge_open.mjs --kb <id> --sample 200 # 开放陈述有多少不是原文说的(门槛 2%)```+缺省(库的 `open_extraction` 开着)抽的是开放图谱:陈述按原文短语落成 `layer='open'` 的事实行,+限定挂在 `statement_qualifiers`,时间词进 `time_mentions`;打分口径不变(谓词取+`proposed_predicate`,限定也拼进那一行),所以 `--typed` 那条老路和它的分数直接可比。`judge_open.mjs` 是+反面的尺子:抽样让一个裁判模型读原文判 stated / misworded / not_stated;裁判端点用+`BENCH_JUDGE_BASE / BENCH_JUDGE_KEY / BENCH_JUDGE_MODEL`,不给就用工作区的对话模型+(和抽取同一个模型,数字要打折看)。+`--kb` 要一个**装了本体包、向量已补齐**的库:装一次 schema.org 要嵌 2500 条向量、二十分钟,而这个台子量的不是本体。库里除了本体没有跨轮状态——事实、实体、信号每轮都清空,自动扩本体上一轮长出来的关系也删掉(留着就进下一轮的提示词)。diff --git a/scripts/bench/judge_open.mjs b/scripts/bench/judge_open.mjsnew file mode 100644index 00000000..436113c3--- /dev/null+++ b/scripts/bench/judge_open.mjs@@ -0,0 +1,178 @@+#!/usr/bin/env node+// 开放陈述的裁判(0044 第 1 刀,#729):**图里的每一条陈述,文档到底说没说。**+//+// recall.mjs 量的是「该有的进没进」,这里量反面:进了的有多少不是原文说的。判据交给一个+// 模型读原文——先抄证据、再下结论(二分之前先引证据,原型上与人工标注一致 21/24)。+// 裁判模型和抽取模型该分开:BENCH_JUDGE_BASE / BENCH_JUDGE_KEY / BENCH_JUDGE_MODEL 指一个+// OpenAI 兼容端点;没给就读工作区的对话模型(llm_settings),那时裁判和抽取是同一个模型,+// 数字要打折看。+//+// 三档:stated(原文说了,陈述说对了)/ misworded(原文说了两者有关系,陈述的说法不对)/+// not_stated(原文没说)。0044 的门槛:not_stated 不超过 2%。+//+// 用法:+// node scripts/bench/judge_open.mjs --kb <id> [--sample 200] [--seed 7] [--out judged.json]+// node scripts/bench/judge_open.mjs --calibrate scripts/bench/truth/open-statements.json+// (人工标注的样本:[{"kb","fact","verdict"}],报裁判与人的一致率)+//+// 环境变量:BENCH_PSQL(同 recall.mjs)。++import fs from "node:fs";+import { execFileSync } from "node:child_process";++const args = Object.fromEntries(+ process.argv.slice(2).reduce((acc, cur, i, arr) => {+ if (cur.startsWith("--")) acc.push([cur.slice(2), arr[i + 1]?.startsWith("--") ? true : (arr[i + 1] ?? true)]);+ return acc;+ }, []),+);+const KB = args.kb;+if (!KB && !args.calibrate) {+ console.error("要一个 --kb <id>(或 --calibrate <标注文件>)");+ process.exit(1);+}++function psql(sql) {+ const cmd = process.env.BENCH_PSQL || "docker exec -e PGPASSWORD=utopia landscapebi-db-1 psql -U utopia -d utopia -tAc";+ const parts = cmd.split(" ");+ return execFileSync(parts[0], [...parts.slice(1), sql], { encoding: "utf8", maxBuffer: 64 << 20 }).trim();+}+const stamp = () => new Date().toISOString().slice(11, 19);++// 裁判端点:显式给的优先;否则读工作区的对话模型+function judgeEndpoint(kb) {+ if (process.env.BENCH_JUDGE_BASE) {+ return { base: process.env.BENCH_JUDGE_BASE, key: process.env.BENCH_JUDGE_KEY || "", model: process.env.BENCH_JUDGE_MODEL || "" };+ }+ const row = psql(`SELECT concat_ws(chr(31), s.chat_base_url, s.chat_api_key, s.chat_model)+ FROM llm_settings s JOIN knowledge_bases k ON k.workspace_id = s.workspace_id WHERE k.id = '${kb}'`);+ const [base, key, model] = row.split("");+ if (!base || !model) throw new Error("工作区没配对话模型,也没给 BENCH_JUDGE_*");+ console.error("裁判用的是工作区的对话模型——和抽取同一个模型,数字要打折看");+ return { base, key, model };+}++async function chat(ep, messages) {+ const r = await fetch(`${ep.base.replace(/\/$/, "")}/chat/completions`, {+ method: "POST",+ headers: { "content-type": "application/json", ...(ep.key ? { authorization: `Bearer ${ep.key}` } : {}) },+ body: JSON.stringify({ model: ep.model, messages, temperature: 0, max_tokens: 6000, response_format: { type: "json_object" } }),+ });+ const t = await r.text();+ if (!r.ok) throw new Error(`${r.status} ${t.slice(0, 200)}`);+ const content = JSON.parse(t).choices?.[0]?.message?.content || "";+ const s = content.indexOf("{"), e = content.lastIndexOf("}");+ return JSON.parse(content.slice(s, e + 1));+}++const JUDGE = `You check statements extracted from a document. Each numbered statement links a subject to an object or a value through a relation phrase taken from the document; some carry qualifiers (role: value) and the time words the document used.+For each, first copy the words of the document that bear on it ("evidence", "" if none). Then answer:+- "stated": the document states this, or a careful reader takes it directly from the document, and the statement describes it correctly;+- "misworded": the document does state a relationship between this subject and this object or value, but the statement describes it wrongly (the wrong phrase, the wrong direction, a qualifier or time that belongs to something else);+- "not_stated": the document does not state any such relationship between this subject and this object or value (it is invented, the object belongs to something else, or it goes beyond what the document says).+Judge only from the document. Output one JSON object: {"results":[{"i":0,"evidence":"...","verdict":"stated|misworded|not_stated"}]}`;++// 取样:现行的开放陈述,连它的证据块(原文)、限定与时间词+function loadStatements(kb, sample, seed) {+ const rows = psql(`+ SELECT concat_ws(chr(31), f.id, c.id, s.canonical_name, f.phrase,+ coalesce(o.canonical_name, f.object_value #>> '{value}', ''),+ coalesce((SELECT string_agg(sq.role || ': ' || coalesce(sq.value #>> '{}', qe.canonical_name, ''), '; ')+ FROM statement_qualifiers sq LEFT JOIN entities qe ON qe.id = sq.entity_id WHERE sq.fact_id = f.id), ''),+ coalesce((SELECT string_agg(tm.text, ' / ') FROM time_mentions tm WHERE tm.fact_id = f.id), ''),+ d.filename)+ FROM facts f+ JOIN fact_evidence fe ON fe.fact_id = f.id+ JOIN chunks c ON c.id = fe.chunk_id+ JOIN documents d ON d.id = c.document_id+ JOIN entities s ON s.id = f.subject_id+ LEFT JOIN entities o ON o.id = f.object_id+ WHERE f.kb_id = '${kb}' AND f.layer = 'open' AND f.invalidated_at IS NULL+ ORDER BY f.id`)+ .split("\n")+ .filter(Boolean)+ .map((l) => {+ const [fact, chunk, subj, phrase, obj, quals, times, file] = l.split("");+ return { fact, chunk, subj, phrase, obj, quals, times, file };+ });+ // 可重复的抽样:按种子打乱+ let x = Number(seed) || 7;+ const rnd = () => ((x = (x * 1103515245 + 12345) % 2147483648) / 2147483648);+ const picked = rows.map((r) => [rnd(), r]).sort((a, b) => a[0] - b[0]).slice(0, Number(sample) || rows.length).map(([, r]) => r);+ return { total: rows.length, picked };+}++function chunkText(chunkId) {+ return psql(`SELECT text FROM chunks WHERE id = '${chunkId}'`);+}++const render = (r) => `${r.subj} —${r.phrase}→ ${r.obj}${r.quals ? ` (${r.quals})` : ""}${r.times ? ` [${r.times}]` : ""}`;++async function judgeAll(kb, items) {+ const ep = judgeEndpoint(kb);+ const byChunk = new Map();+ for (const it of items) (byChunk.get(it.chunk) || byChunk.set(it.chunk, []).get(it.chunk)).push(it);+ const out = [];+ for (const [chunk, list] of byChunk) {+ const text = chunkText(chunk);+ for (let i = 0; i < list.length; i += 25) {+ const part = list.slice(i, i + 25);+ const body = `Document (${part[0].file}):\n${text}\n\nStatements:\n${part.map((x, j) => `${j}. ${render(x)}`).join("\n")}`;+ let res;+ try {+ res = await chat(ep, [{ role: "system", content: JUDGE }, { role: "user", content: body }]);+ } catch (e) {+ console.error(`${stamp()} 裁判调用失败:${e.message}`);+ continue;+ }+ for (const r of res.results || []) {+ const it = part[r.i];+ if (!it) continue;+ out.push({ ...it, statement: render(it), evidence: r.evidence || "", verdict: ["stated", "misworded", "not_stated"].includes(r.verdict) ? r.verdict : "unjudged" });+ }+ console.error(`${stamp()} 判了 ${out.length}/${items.length}`);+ }+ }+ return out;+}++function report(judged, total) {+ const n = judged.length;+ const count = (v) => judged.filter((j) => j.verdict === v).length;+ const pct = (a) => `${a}/${n}(${n ? ((100 * a) / n).toFixed(1) : 0}%)`;+ console.log(`\n开放陈述 ${total} 条,判了 ${n} 条:stated ${pct(count("stated"))},misworded ${pct(count("misworded"))},not_stated ${pct(count("not_stated"))},未判 ${count("unjudged")}`);+ console.log(`0044 的门槛:not_stated ≤ 2%${count("not_stated") / (n || 1) <= 0.02 ? ",过" : ",没过"}`);+ const bad = judged.filter((j) => j.verdict !== "stated").slice(0, 20);+ if (bad.length) {+ console.log("\n判不对的(样本):");+ for (const j of bad) console.log(` [${j.verdict}] ${j.statement}\n 证据:${j.evidence.slice(0, 160)}`);+ }+}++if (args.calibrate) {+ // 人工标注的样本:逐条按库取原文再判,报一致率+ const labels = JSON.parse(fs.readFileSync(args.calibrate, "utf8"));+ const byKb = new Map();+ for (const l of labels) (byKb.get(l.kb) || byKb.set(l.kb, []).get(l.kb)).push(l);+ let agree = 0, n = 0;+ for (const [kb, list] of byKb) {+ const { picked } = loadStatements(kb, Infinity, 1);+ const items = picked.filter((p) => list.some((l) => l.fact === p.fact));+ const judged = await judgeAll(kb, items);+ for (const j of judged) {+ const human = list.find((l) => l.fact === j.fact)?.verdict;+ if (!human) continue;+ n++;+ if ((human === "stated") === (j.verdict === "stated")) agree++;+ else console.log(` 不一致:人 ${human} / 裁判 ${j.verdict} — ${j.statement}`);+ }+ }+ console.log(`\n二分一致 ${agree}/${n}`);+ process.exit(0);+}++const { total, picked } = loadStatements(KB, args.sample, args.seed);+console.error(`${stamp()} 库里 ${total} 条开放陈述,抽 ${picked.length} 条`);+const judged = await judgeAll(KB, picked);+report(judged, total);+if (args.out) fs.writeFileSync(args.out, JSON.stringify(judged, null, 1));diff --git a/scripts/bench/recall.mjs b/scripts/bench/recall.mjsindex 6c89295a..18d93fc8 100644--- a/scripts/bench/recall.mjs+++ b/scripts/bench/recall.mjs@@ -16,6 +16,7 @@// node scripts/bench/recall.mjs --kb <id> # 已有库(本体向量已就绪)// node scripts/bench/recall.mjs --kb <id> --score # 只打分,不重抽// node scripts/bench/recall.mjs --kb <id> --reprocess # 改了解析器:连分块一起重来+// node scripts/bench/recall.mjs --kb <id> --typed # 带本体的老路(可选第二路);缺省是开放图谱(0044 第 1 刀)//// 环境变量:BENCH_BASE / BENCH_EMAIL / BENCH_PASSWORD / BENCH_PSQL(同 run.mjs)。//@@ -89,7 +90,13 @@ function score() {coalesce(to_char(f.valid_from,'YYYY-MM-DD'),''),coalesce((SELECT string_agg(q.key || '=' || coalesce(fq.value #>> '{value}', '') || ' ' || coalesce(fq.value #>> '{unit}', ''), ' ')FROM fact_qualifiers fq JOIN relation_types q ON q.id = fq.qualifier_type_id- WHERE fq.fact_id = f.id), ''))+ WHERE fq.fact_id = f.id), '')+ -- 开放陈述的限定(#729)按文档自己的角色词挂着:金额、职务在这里+ || ' ' || coalesce((SELECT string_agg(sq.role || '=' || coalesce(sq.value #>> '{}', qe.canonical_name, ''), ' ')+ FROM statement_qualifiers sq LEFT JOIN entities qe ON qe.id = sq.entity_id+ WHERE sq.fact_id = f.id), '')+ -- 开放陈述的时间是照抄的字(time_mentions),不在 valid_from 里:日期类真值靠它命中+ || ' ' || coalesce((SELECT string_agg(tm.text, ' ') FROM time_mentions tm WHERE tm.fact_id = f.id), ''))FROM facts fJOIN fact_evidence fe ON fe.fact_id = f.idJOIN documents d ON d.id = fe.document_id@@ -201,10 +208,12 @@ psql(`DELETE FROM entities WHERE kb_id='${KB}'`);psql(`DELETE FROM extraction_drops WHERE kb_id='${KB}'`);psql(`DELETE FROM ontology_misses WHERE kb_id='${KB}'`);// 自动扩本体上一轮长出来的关系:留着就进下一轮的提示词,两轮条件不同-psql(`DELETE FROM relation_types WHERE kb_id='${KB}' AND created_at > '${packTs}'::timestamptz + interval '1 second'`);+// 没装本体包的库(开放图谱那条路不需要本体):没有基准时刻,也就没有要删的+if (packTs) psql(`DELETE FROM relation_types WHERE kb_id='${KB}' AND created_at > '${packTs}'::timestamptz + interval '1 second'`);// 会在抽取之后改本体、增派生事实的开关关掉,两轮的本体与打分口径才一样。治理开着:// 库生下来就开着它(0050),量的是产品本来的样子-psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE, materialize_inferences=FALSE, governance=TRUE WHERE id='${KB}'`);+psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE, materialize_inferences=FALSE, governance=TRUE, open_extraction=${args.typed ? "FALSE" : "TRUE"} WHERE id='${KB}'`);+console.log(args.typed ? "带本体的老路:本体进提示词" : "开放图谱:本体不进提示词,陈述按原文短语落库");psql(`UPDATE chunks SET extracted_at=NULL WHERE document_id IN (SELECT id FROM documents WHERE kb_id='${KB}' AND filename IN (${names}))`);console.log(`本体 ${num(`SELECT count(*) FROM relation_types WHERE kb_id='${KB}'`)} 个关系 / ${num(`SELECT count(*) FROM entity_types WHERE kb_id='${KB}'`)} 个类`);diff --git a/web/src/api.ts b/web/src/api.tsindex 98547f31..a518841f 100644--- a/web/src/api.ts+++ b/web/src/api.ts@@ -102,6 +102,9 @@ export interface Kb {/** 抽取遇到本体外的说法时,是否允许系统自动补进本体并改写等它的事实。关掉不影响"留意":未匹配统计照常累积可见,只是变成你点一下的提案 */auto_extend_ontology: boolean;+ /** 开放抽取(0044 第一刀):开着,抽取只写开放图谱——陈述照文档的字落库,+ * 不读本体;关着走今天的类型化那条路。缺省关;这一刀没有开关界面 */+ open_extraction: boolean;/** 把推出来的事实写进账本(R1)。**缺省关**——推理往图里加东西,* 而声明可能是错的,不该在用户没表态时就按它改图 */materialize_inferences: boolean;diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex bddf240f..17c365f9 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -517,6 +517,9 @@ export const en = {clause_suspect: "Kept, but the name reads like a clause: a sample for the guard",direction_corrected:"Subject and object were swapped to match the signature",+ quote_not_in_chunk: "Kept, but the quoted sentence is not in the text verbatim",+ time_not_in_quote: "A time mention's words are not in the text",+ unknown_ref: "The item points at nothing in the reply",} as Record<string, string>,// 来源级重抽:不危险,只是费时费钱——轻确认,文案直说成本与保留项reExtractSource: "Re-extract",diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 0e2fcc9a..6337c6a7 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -466,6 +466,9 @@ export const zh: Strings = {not_an_entity_name: "那个名字是一句话,不是一个东西",clause_suspect: "已保留,但这个名字像从句:给守卫攒的样本",direction_corrected: "已按签名把主宾掰正",+ quote_not_in_chunk: "已保留,但引文不是原文原样",+ time_not_in_quote: "时间词不在原文里",+ unknown_ref: "这一条指向的东西不在回复里",},reExtractSource: "重新抽取",reExtractTitle: "重新抽取这个来源?",