Skip to content

World's first open-source enterprise world model.

An attribute's value is read in its datatype and unit, and a table cell or a figure written as an object is a value (#1252)

WaylandYang7393e9d026c6
diff --git a/Cargo.lock b/Cargo.lock
index 51e19ff9..f7a8637e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5564,6 +5564,7 @@ dependencies = [
"tracing",
"url",
"utopia-core",
+ "utopia-extract",
"utopia-ingest",
"utopia-reason",
"uuid",
diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs
index 97af7bf7..e2095939 100644
--- a/crates/utopia-extract/src/lib.rs
+++ b/crates/utopia-extract/src/lib.rs
@@ -416,7 +416,7 @@ pub(crate) fn escape_control_in_strings(text: &str) -> String {
/// **单位照抄符号,不猜币种。** `$` 可能是美元、加元、澳元,`¥` 可能是日元或
/// 人民币。猜出来的 "USD" 是一条没人负责的断言,而原文写的 `$` 是事实。
pub fn parse_quantity(s: &str) -> Option<(f64, Option<String>)> {
- scan_quantity(s, true)
+ scan_quantity(s, true).map(|q| (q.n, q.unit))
}
/// 开头是一个量、后面还挂着词的 → 那个量。`"1,250 people"` → (1250, "people")。
@@ -427,7 +427,7 @@ pub fn parse_quantity(s: &str) -> Option<(f64, Option<String>)> {
/// `datatype = number` 的属性——问的不再是「是不是数」,是「那个数是多少」,
/// 判错的代价只是一个值不对,量级差着好几档。
pub fn parse_leading_quantity(s: &str) -> Option<(f64, Option<String>)> {
- scan_quantity(s, false)
+ scan_quantity(s, false).map(|q| (q.n, q.unit))
}
/// 货币:符号、ISO 码、中英文单词,统一成符号。**只认这张表**,认不出的不猜。
@@ -466,7 +466,7 @@ fn magnitude(tok: &str) -> Option<u8> {
/// 把 `2亿美元`、`15亿元人民币`、`€30 million`、`30 million euros`、`USD 30m`(不认 m)
/// 这类写法拆成 [前缀货币] 数字 [量级] [后缀货币/单位] [其余]。
/// `strict` = 整体必须就是一个量:其余部分非空就不认。
-fn scan_quantity(s: &str, strict: bool) -> Option<(f64, Option<String>)> {
+fn scan_quantity(s: &str, strict: bool) -> Option<Quantity> {
let s = s.trim();
if s.is_empty() {
return None;
@@ -542,15 +542,99 @@ fn scan_quantity(s: &str, strict: bool) -> Option<(f64, Option<String>)> {
} else {
currency.map(str::to_string).or(unit)
};
- if strict {
- return tail.is_empty().then_some((n, unit));
+ if strict && !tail.is_empty() {
+ return None;
}
// 宽松:其余部分的第一个词当单位(`1,250 people` → people),没有货币时才用
- if unit.is_none() && !tail.is_empty() {
- let (tok, _) = next_token(tail);
- return Some((n, Some(tok.to_string())));
+ let unit = unit.or_else(|| (!tail.is_empty()).then(|| next_token(tail).0.to_string()));
+ Some(Quantity {
+ n,
+ unit,
+ scaled: ate_magnitude,
+ })
+}
+
+/// 读出来的一个量。`n` 已经乘上了量级词;`scaled` 记下原文写没写量级词——换算到属性
+/// 声明的单位时要知道 `5000` 是原文就这么写的数,还是 `5000万` 展开出来的
+struct Quantity {
+ n: f64,
+ unit: Option<String>,
+ scaled: bool,
+}
+
+/// 「约」「左右」这类只说数是个约数的词。去掉它们数还是那个数;「超过」「余」不在这里:
+/// 它们说的是一个界,把界当成值就是一条原文没说的断言
+fn strip_approx(s: &str) -> &str {
+ let mut s = s.trim();
+ for p in [
+ "大约",
+ "约合",
+ "将近",
+ "接近",
+ "约",
+ "近",
+ "about ",
+ "approximately ",
+ "around ",
+ "roughly ",
+ "nearly ",
+ "~",
+ "≈",
+ ] {
+ if let Some(rest) = s.strip_prefix(p) {
+ s = rest.trim_start();
+ break;
+ }
+ }
+ s.strip_suffix("左右").map_or(s, str::trim_end)
+}
+
+/// 计数的量词:`1200人`、`350名`、`32家`。数后面只跟这一个词的,整体还是一个数
+const COUNTERS: &[&str] = &[
+ "人",
+ "名",
+ "位",
+ "个",
+ "家",
+ "台",
+ "件",
+ "项",
+ "岁",
+ "次",
+ "座",
+ "所",
+ "户",
+ "辆",
+ "种",
+ " people",
+ " employees",
+ " staff",
+];
+
+/// 一个宾语的名字是不是其实是个值:一个量(带量级词、货币、百分号或计数量词)或一个
+/// 写法说得清的日期。开放抽取里模型有时把「成立于 2015年3月12日」「注册资本为
+/// 5000万元」的值列成东西写进宾语,签名于是成了两样东西之间的关系,永远对不上属性。
+///
+/// **光秃秃的一个数只在它不是点了名的东西时才算值**:`1200` 是个数,而被点了名的
+/// `1984`、`360` 多半是一本书、一家公司——吃掉一个真实体是不可逆的,留一条数
+/// 在宾语上只是少绑一个属性
+pub fn literal_object(name: &str, named: bool) -> bool {
+ let t = strip_approx(name);
+ if t.is_empty() {
+ return false;
+ }
+ if written_date(t).is_some() || t.contains(['-', '/', '.']) && parse_time(t).is_some() {
+ return true;
+ }
+ match scan_quantity(t, true) {
+ Some(q) if q.unit.is_some() || q.scaled => true,
+ Some(_) => !named,
+ None => COUNTERS.iter().any(|c| {
+ t.strip_suffix(c)
+ .and_then(|r| scan_quantity(r.trim_end(), true))
+ .is_some_and(|q| q.unit.is_none())
+ }),
}
- Some((n, unit))
}
/// 下一个记号:ASCII 按空白切;CJK 试最长三字、两字、一字里能认出的量级/货币词,
@@ -592,6 +676,8 @@ pub fn normalize_attr_value(datatype: &str, raw: &serde_json::Value) -> Option<s
.and_then(serde_json::Number::from_f64)
.map(serde_json::Value::Number),
serde_json::Value::String(s) => {
+ // 「约350名」:约数的那个字挡在数前面,宽的那套也解不动
+ let s = strip_approx(s);
let cleaned: String = s
.chars()
.filter(|c| !matches!(c, ',' | ' ' | '_'))
@@ -619,6 +705,7 @@ pub fn normalize_attr_value(datatype: &str, raw: &serde_json::Value) -> Option<s
let s = raw.as_str()?.trim();
match written_date(s) {
Some((date, precision)) => Some(serde_json::Value::String(match precision {
+ "year" => date.format("%Y").to_string(),
"month" => date.format("%Y-%m").to_string(),
_ => date.format("%Y-%m-%d").to_string(),
})),
@@ -645,6 +732,49 @@ pub fn normalize_attr_value(datatype: &str, raw: &serde_json::Value) -> Option<s
}
}
+/// 一条开放陈述的值落到一个属性上:按属性的 datatype 归一,数再换算到属性声明的单位。
+///
+/// 单位只换量级:属性写「万元」、原文写「2.3亿元」,存 23000;原文写「5000」这样光
+/// 秃秃的数,就当它已经是声明的单位。原文的货币和声明的货币对不上(「5000万美元」落到
+/// 「万元」上)返回 None——换汇是一条没人负责的断言。读不出来的也是 None:宁缺勿脏,
+/// 陈述留在开放图谱里,不把「约350名」这样的字当成一个数写进类型化图谱
+pub fn attribute_value(
+ datatype: &str,
+ unit: Option<&str>,
+ raw: &serde_json::Value,
+) -> Option<serde_json::Value> {
+ let declared = unit.map(str::trim).filter(|u| !u.is_empty());
+ let read = match (raw, datatype, declared) {
+ (serde_json::Value::String(s), "number", Some(d)) => {
+ scan_quantity(strip_approx(s), false).map(|q| (q, d))
+ }
+ _ => None,
+ };
+ let Some((q, declared)) = read else {
+ return normalize_attr_value(datatype, raw);
+ };
+ // 声明的单位拆成量级和其余:「万元」→ (4, 元),「million dollars」→ (6, dollars)
+ let (tok, rest) = next_token(declared);
+ let (exponent, base) = match magnitude(&tok.to_ascii_lowercase()) {
+ Some(m) => (m, rest.trim()),
+ None => (0, declared),
+ };
+ let written = q.unit.as_deref().and_then(currency_unit);
+ let wanted = currency_unit(base);
+ if written.is_some() && wanted.is_some() && written != wanted {
+ return None;
+ }
+ // 原文写了量级词,或写了和声明一样的单位(「50000000元」对「万元」),n 才是绝对量,
+ // 要除下来;光秃秃的数就是声明单位下的数。除法走一次十进制解析,同 `scan_quantity`
+ // 乘量级词的道理:先算成 f64 再除会留下 23000.000000000004
+ let n = if q.scaled || (written.is_some() && written == wanted) {
+ format!("{}e-{exponent}", q.n).parse().ok()?
+ } else {
+ q.n
+ };
+ serde_json::Number::from_f64(n).map(serde_json::Value::Number)
+}
+
/// 解析时间字符串 → (UTC 时间, 精度)。
///
/// 日期:YYYY / YYYY-MM / YYYY-MM-DD,精度随写了几位。带时区的时刻(0024):
@@ -774,6 +904,10 @@ fn year_first_date(s: &str) -> Option<Option<(NaiveDate, &'static str)>> {
[m] if marked && s.ends_with('月') && m.len() <= 2 => {
NaiveDate::from_ymd_opt(year, number(m)?, 1).map(|date| (date, "month"))
}
+ // 只到年:「1982年」。写了「年」才算,光秃秃的 `1982` 归 parse_time
+ [] if marked && s.ends_with('年') => {
+ NaiveDate::from_ymd_opt(year, 1, 1).map(|date| (date, "year"))
+ }
_ => None,
})
}
@@ -1289,3 +1423,69 @@ mod a_number_is_one_number {
assert_eq!(a.as_f64(), Some(65.0));
}
}
+
+#[cfg(test)]
+mod an_attribute_value_is_read_in_its_declared_unit {
+ use super::{attribute_value, literal_object};
+ use serde_json::json;
+
+ fn value(datatype: &str, unit: Option<&str>, raw: &str) -> Option<serde_json::Value> {
+ attribute_value(datatype, unit, &json!(raw))
+ }
+
+ /// 公司简介里最平常的几句:值照原文写,落到属性上要是那个数、那一天
+ #[test]
+ fn plain_company_figures_are_read() {
+ assert_eq!(
+ value("number", Some("万元"), "5000万元"),
+ Some(json!(5000.0))
+ );
+ assert_eq!(
+ value("number", Some("万元"), "2.3亿元"),
+ Some(json!(23000.0))
+ );
+ assert_eq!(value("number", Some("亿元"), "8.6亿元"), Some(json!(8.6)));
+ assert_eq!(
+ value("number", Some("万元"), "50000000元"),
+ Some(json!(5000.0))
+ );
+ // 光秃秃的数就是声明单位下的数
+ assert_eq!(value("number", Some("万元"), "5000"), Some(json!(5000.0)));
+ assert_eq!(value("number", Some("人"), "约350名"), Some(json!(350.0)));
+ // 只写到年的日期按年
+ assert_eq!(value("date", None, "1982年"), Some(json!("1982")));
+ }
+
+ /// 换不了的不换:货币对不上、读不出数的,宁缺勿脏
+ #[test]
+ fn what_cannot_be_read_is_not_written() {
+ assert_eq!(value("number", Some("万元"), "5000万美元"), None);
+ assert_eq!(value("number", Some("人"), "超过一千人"), None);
+ assert_eq!(value("date", None, "2023年底"), None);
+ assert_eq!(value("bool", None, "尚未上市"), None);
+ }
+
+ #[test]
+ fn a_figure_or_a_date_is_not_a_thing() {
+ for v in [
+ "5000万元",
+ "8.6亿元",
+ "1200人",
+ "约350名",
+ "2015年3月12日",
+ "1978年5月",
+ "2020年",
+ "2015-03-12",
+ "$5 billion",
+ "52%",
+ ] {
+ assert!(literal_object(v, true), "{v}");
+ }
+ // 光秃秃的数:没点名的是值,点了名的多半是个东西(《1984》、360)
+ assert!(literal_object("1200", false));
+ assert!(!literal_object("1984", true));
+ for name in ["3M", "7天", "2023年底", "星河科技", "董事长兼总经理", "K2"] {
+ assert!(!literal_object(name, false), "{name}");
+ }
+ }
+}
diff --git a/crates/utopia-extract/src/open.rs b/crates/utopia-extract/src/open.rs
index 10932994..3888a241 100644
--- a/crates/utopia-extract/src/open.rs
+++ b/crates/utopia-extract/src/open.rs
@@ -212,14 +212,17 @@ $2 million on March 4, 2011.\" gives:\n\
/// 用不上它,而它占系统消息的一成不到
const OPEN_TABLE_RULES: &str = "\
In a \
-table, a cell is a statement about its row's thing whose phrase is the column heading. When \
+table, a cell is a statement about its row's thing whose phrase is the column heading and \
+whose value is the cell as written; a cell is the object only when it names a thing of its own \
+(a person, an organization, a place). When \
the column heading names one time (a date, a quarter, a period), it is when instead, the \
statement is about the thing the table's caption names (the company of a financial statement) \
and the phrase is the row label as written, path included (\"Operating expenses › Research and \
development\"); a heading that names a comparison between periods (a change from the quarter \
before, a change from a year ago) is not a time, it stays the phrase; a unit the caption or a \
-heading gives (\"In millions\") is a qualifier on each such statement. Leave out cells whose \
-column heading is not in the passage.";
+heading gives (\"In millions\") is a qualifier on each such statement. Otherwise a row is about \
+the thing its first column (or its name, title or ID column) names, not the table or the \
+sheet. Leave out cells whose column heading is not in the passage.";
/// 这一段里有没有表格:Markdown 的竖线、HTML 的单元格、制表符,或者一行里用成串空格
/// 隔开的三栏以上。宁可多认:认错了只是多带一段规则,和从前一样
@@ -635,6 +638,56 @@ pub fn parse_open_response(raw: &str) -> anyhow::Result<OpenExtraction> {
Ok(out)
}
+/// 宾语写成了一个值的,改回值;返回挪了的那些宾语,调用方各记一笔(`object_is_value`)。
+///
+/// 规则 2 和表格规则说值写进 value,模型照样会把「成立于 2015年3月12日」「注册资本为
+/// 5000万元」、表格里的「待审核」写进宾语。签名于是成了两样东西之间的关系,对不上任何
+/// 属性。这里只挪格子,原文的字一个不改;挪完以后没有陈述再指着、也不是别名的那个
+/// 「东西」从 `e` 里拿掉,免得建出一个叫「2015年3月12日」的实体。
+///
+/// 只给模型的回复用:推送来的陈述是客户端的契约,它写成宾语的就是宾语
+pub fn values_are_not_things(x: &mut OpenExtraction) -> Vec<String> {
+ let listed: std::collections::HashMap<String, (bool, String)> = x
+ .entities
+ .iter()
+ .map(|e| (name_key(&e.name), (e.named, name_key(&e.kind))))
+ .collect();
+ let mut moved = Vec::new();
+ for s in &mut x.statements {
+ let (Some(object), None) = (s.object.as_deref(), s.value.as_deref()) else {
+ continue;
+ };
+ let (named, kind) = listed
+ .get(&name_key(object))
+ .map_or((false, ""), |(n, k)| (*n, k.as_str()));
+ // 表格里的一格:种类词就是列名、又写成以列名为短语的宾语,模型自己说了这是那一列
+ // 的值。点了名的只在名字里带数字时才算(编号、型号);只看表格行,散文里点了名的
+ // 「Model 3」照旧是东西
+ let a_cell = kind == name_key(&s.phrase)
+ && !kind.is_empty()
+ && s.quote.as_deref().is_some_and(has_table)
+ && (!named || object.chars().any(|c| c.is_ascii_digit()));
+ if a_cell || crate::literal_object(object, named) {
+ moved.extend(s.object.take());
+ s.value = moved.last().cloned();
+ }
+ }
+ let used: HashSet<String> = x
+ .statements
+ .iter()
+ .flat_map(|s| [Some(&s.subject), s.object.as_ref()])
+ .flatten()
+ .chain(x.names.iter().flat_map(|n| [&n.entity, &n.name]))
+ .map(|n| name_key(n))
+ .collect();
+ let moved_keys: HashSet<String> = moved.iter().map(|n| name_key(n)).collect();
+ x.entities.retain(|e| {
+ let key = name_key(&e.name);
+ used.contains(&key) || !moved_keys.contains(&key)
+ });
+ moved
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -1047,6 +1100,62 @@ mod tests {
assert_eq!(x.skipped, 0);
}
+ /// 值被列成东西写进了宾语:挪回值,那样「东西」不再列着。真东西(杭州)、散文里点了名
+ /// 的(Model 3,种类词恰好是短语)、被当成主语用着的(2020年)都不动
+ #[test]
+ fn a_value_written_as_an_object_becomes_the_value() {
+ let raw = r#"{"e": [["星河科技", "公司", 1], ["2015年3月12日", "日期", 1],
+ ["杭州", "城市", 1], ["Model 3", "竞品", 1], ["2020年", "年份", 1]],
+ "s": [["星河科技成立于2015年3月12日。", "星河科技", "成立于", "2015年3月12日", null, null, null, null],
+ ["总部位于杭州。", "星河科技", "总部位于", "杭州", null, null, null, null],
+ ["竞品是Model 3。", "星河科技", "竞品", "Model 3", null, null, null, null],
+ ["2020年是上市之年。", "2020年", "是", null, "上市之年", null, null, null]]}"#;
+ let mut x = parse_open_response(raw).unwrap();
+ assert_eq!(values_are_not_things(&mut x), vec!["2015年3月12日"]);
+ let objects: Vec<Option<&str>> = x.statements.iter().map(|s| s.object.as_deref()).collect();
+ assert_eq!(objects, vec![None, Some("杭州"), Some("Model 3"), None]);
+ assert_eq!(x.statements[0].value.as_deref(), Some("2015年3月12日"));
+ let names: Vec<&str> = x.entities.iter().map(|e| e.name.as_str()).collect();
+ assert_eq!(names, vec!["星河科技", "杭州", "Model 3", "2020年"]);
+ }
+
+ /// 实测的需求表(#987):一格被列成没点名的东西、种类词就是列名,再写成以列名为
+ /// 短语的宾语。那是这一格的值;点了名的、种类词和短语不同的照旧是宾语
+ #[test]
+ fn a_table_cell_listed_as_a_thing_becomes_the_value() {
+ let row = "| 修改体重时App出现bug | 待审核 | P1 | 个人资料 | 张伟 |";
+ let raw = format!(
+ r#"{{"e": [["修改体重时App出现bug", "需求", 0], ["待审核", "处理状态", 0],
+ ["P1", "优先级", 0], ["个人资料", "模块", 0], ["张伟", "负责人", 1],
+ ["REQ-0019", "需求编号", 1]],
+ "s": [["{row}", "修改体重时App出现bug", "处理状态", "待审核", null, null, null, null],
+ ["{row}", "修改体重时App出现bug", "优先级", "P1", null, null, null, null],
+ ["{row}", "修改体重时App出现bug", "需求编号", "REQ-0019", null, null, null, null],
+ ["{row}", "修改体重时App出现bug", "功能模块", "个人资料", null, null, null, null],
+ ["{row}", "修改体重时App出现bug", "负责人", "张伟", null, null, null, null]]}}"#
+ );
+ let mut x = parse_open_response(&raw).unwrap();
+ values_are_not_things(&mut x);
+ let shape: Vec<(Option<&str>, Option<&str>)> = x
+ .statements
+ .iter()
+ .map(|s| (s.object.as_deref(), s.value.as_deref()))
+ .collect();
+ assert_eq!(
+ shape,
+ vec![
+ (None, Some("待审核")),
+ (None, Some("P1")),
+ (None, Some("REQ-0019")),
+ (Some("个人资料"), None),
+ (Some("张伟"), None),
+ ]
+ );
+ // 挪成值的格子不再列着,免得建出一个叫 REQ-0019 的孤立实体
+ let names: Vec<&str> = x.entities.iter().map(|e| e.name.as_str()).collect();
+ assert_eq!(names, vec!["修改体重时App出现bug", "个人资料", "张伟"]);
+ }
+
/// named 认 1/0 与 true/false,缺了当点了名;kind 缺了是空串
#[test]
fn named_takes_numbers_and_booleans_and_defaults_to_true() {
diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs
index cf7e0302..0a4fa948 100644
--- a/crates/utopia-server/src/extraction_open.rs
+++ b/crates/utopia-server/src/extraction_open.rs
@@ -113,7 +113,7 @@ async fn call_and_parse_one_chunk(
let cut_by_ceiling = reply.hit_token_ceiling();
(reply.text, cut_by_ceiling)
};
- let extraction = match utopia_extract::open::parse_open_response(&reply_text) {
+ let mut extraction = match utopia_extract::open::parse_open_response(&reply_text) {
Ok(x) => x,
Err(e) => {
tracing::warn!(%document_id, seq = chunk.seq, error = %e, hit_token_ceiling = cut_by_ceiling, "开放抽取回复解析失败,跳过该分块");
@@ -143,6 +143,20 @@ async fn call_and_parse_one_chunk(
return Ok(None);
}
};
+ // 模型把值写成了宾语的,挪回值并各记一笔;推送来的是契约,不动(0054)
+ if !pushed {
+ for name in utopia_extract::open::values_are_not_things(&mut extraction) {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::OBJECT_IS_VALUE,
+ "kept, with the object read as a value",
+ Some(&name),
+ )
+ .await;
+ }
+ }
if extraction.truncated {
drop_signal(
state,
diff --git a/crates/utopia-store/Cargo.toml b/crates/utopia-store/Cargo.toml
index 2c81fb7e..d8a43736 100644
--- a/crates/utopia-store/Cargo.toml
+++ b/crates/utopia-store/Cargo.toml
@@ -8,6 +8,8 @@ license.workspace = true
utopia-core.workspace = true
utopia-ingest.workspace = true
utopia-reason.workspace = true
+# 物化把属性的值按 datatype 与单位读成数、日期(`attribute_value`)
+utopia-extract = { path = "../utopia-extract" }
serde.workspace = true
pgvector.workspace = true
futures-util.workspace = true
diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rs
index b68e613a..de55ec1c 100644
--- a/crates/utopia-store/src/extraction_drops.rs
+++ b/crates/utopia-store/src/extraction_drops.rs
@@ -58,6 +58,9 @@ pub mod reason {
/// 短语就是主语的名字(「实物商品网上零售额 —实物商品网上零售额→ 127878亿元」):
/// 同上,另一种写法,同样照落
pub const PHRASE_IS_SUBJECT: &str = "phrase_is_subject";
+ /// 宾语其实是个值(一个量、一个日期、表格的一格):挪进值那一格,不建实体
+ /// (`open::values_are_not_things`)。数的是规则 2 和表格规则还有多少次没管住模型
+ pub const OBJECT_IS_VALUE: &str = "object_is_value";
}
pub async fn record(
diff --git a/crates/utopia-store/src/materialize.rs b/crates/utopia-store/src/materialize.rs
index d6d3af49..cd81523b 100644
--- a/crates/utopia-store/src/materialize.rs
+++ b/crates/utopia-store/src/materialize.rs
@@ -23,6 +23,7 @@ use utopia_core::AppResult;
use uuid::Uuid;
use crate::graph::{insert_fact_on, FactObject, Validity};
+use serde_json::Value;
/// 陈述与绑定对得上的条件:短语归一后相等,两端的类相同(空也相同),宾语是不是字面值相同。
/// `s` 是开放陈述(facts),`se`/`oe` 是它两端的实体,`b` 是 phrase_bindings
@@ -44,6 +45,10 @@ const MOMENT_UNREAD: &str =
WHERE rt.id = b.relation_type_id AND rt.temporal = 'state')
AND b.marks IS DISTINCT FROM 'start' AND b.marks IS DISTINCT FROM 'end')";
+/// 属性的 datatype 与单位,值按它们读([`attribute_value`]);`rt` 是绑到或蕴含的那条属性,
+/// 是关系时 datatype 为 NULL,值原样写
+const SHAPE: &str = "CASE WHEN rt.kind = 'attribute' THEN rt.datatype END AS datatype, rt.unit";
+
/// 一轮重算写了什么。
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Outcome {
@@ -81,6 +86,9 @@ struct Due {
marks: Option<String>,
/// 绑到的属性的时间语义
temporal: Option<String>,
+ /// 绑到的是属性时它的 datatype 与单位,值按它们读([`attribute_value`])
+ datatype: Option<String>,
+ unit: Option<String>,
}
/// 对一个库重算一遍。
@@ -251,12 +259,12 @@ async fn materialize_in_tx(
s.valid_from, s.valid_from_precision, s.valid_from_grade,
s.valid_to, s.valid_to_precision,
s.attested_from, s.confidence, b.marks,
- (SELECT rt.temporal FROM relation_types rt WHERE rt.id = b.relation_type_id)
- AS temporal
+ rt.temporal, {SHAPE}
FROM facts s
JOIN entities se ON se.id = s.subject_id
LEFT JOIN entities oe ON oe.id = s.object_id
JOIN phrase_bindings b ON {MATCH}
+ JOIN relation_types rt ON rt.id = b.relation_type_id
WHERE s.kb_id = $1 AND s.layer = 'open' AND s.invalidated_at IS NULL
AND b.status = 'bound'
AND (b.direction = 'forward' OR s.object_id IS NOT NULL)
@@ -338,12 +346,17 @@ async fn materialize_in_tx(
.await?
}
(false, None, Some(value)) => {
+ // 读不成属性的 datatype:陈述留在开放图谱里,这一对不写
+ let Some(value) = attribute_value(value, d.datatype.as_deref(), d.unit.as_deref())
+ else {
+ continue;
+ };
insert_fact_on(
tx,
kb_id,
d.subject_id,
Some(d.property),
- FactObject::Value(value),
+ FactObject::Value(&value),
validity,
d.confidence,
)
@@ -502,6 +515,8 @@ struct Implied {
valid_to_precision: Option<String>,
attested_from: Option<chrono::DateTime<chrono::Utc>>,
confidence: f32,
+ datatype: Option<String>,
+ unit: Option<String>,
}
async fn imply_in_tx(
@@ -517,8 +532,9 @@ async fn imply_in_tx(
CASE WHEN r.reading IS NULL THEN s.object_id ELSE pr.entity_id END AS object_id,
CASE WHEN r.reading IS NULL THEN s.object_value ELSE pr.value END AS object_value,
s.valid_from, s.valid_from_precision, s.valid_from_grade, s.valid_to, s.valid_to_precision,
- s.attested_from, s.confidence
+ s.attested_from, s.confidence, {SHAPE}
FROM implication_rules r
+ JOIN relation_types rt ON rt.id = r.conclude_property_id
JOIN facts s ON s.kb_id = r.kb_id AND s.layer = 'open' AND s.invalidated_at IS NULL
JOIN entities se ON se.id = s.subject_id
LEFT JOIN entities oe ON oe.id = s.object_id
@@ -531,8 +547,7 @@ async fn imply_in_tx(
-- 规则没有 marks(#966):一刻的陈述蕴含一条状态属性时,读法未知,不算隐含行。
-- 用 IS NOT DISTINCT FROM:只有起点的陈述上 `=` 是 NULL,NOT 之后照样把它排掉
AND NOT (s.valid_from IS NOT NULL AND s.valid_to IS NOT DISTINCT FROM s.valid_from
- AND EXISTS (SELECT 1 FROM relation_types rt
- WHERE rt.id = r.conclude_property_id AND rt.temporal = 'state'))
+ AND rt.temporal = 'state')
AND NOT EXISTS (SELECT 1 FROM errata_actions ea
WHERE ea.statement_id = s.id AND ea.predicate_id = r.conclude_property_id
AND ea.status = 'applied' AND ea.action IN ('retract', 'revise'))
@@ -552,8 +567,9 @@ async fn imply_in_tx(
pr.entity_id AS object_id, pr.value AS object_value,
NULL::timestamptz AS valid_from, NULL::text AS valid_from_precision, NULL::text AS valid_from_grade,
NULL::timestamptz AS valid_to, NULL::text AS valid_to_precision,
- NULL::timestamptz AS attested_from, 0.9::real AS confidence
+ NULL::timestamptz AS attested_from, 0.9::real AS confidence, {SHAPE}
FROM implication_rules r
+ JOIN relation_types rt ON rt.id = r.conclude_property_id
JOIN entities e ON e.kb_id = r.kb_id AND e.merged_into IS NULL AND {KIND} = r.phrase
JOIN phrase_readings pr ON pr.kb_id = r.kb_id AND pr.reading = r.reading AND pr.phrase = r.phrase
WHERE r.kb_id = $1 AND r.status = 'approved' AND r.trigger = 'kind_word' AND r.reading IS NOT NULL
@@ -577,9 +593,16 @@ async fn imply_in_tx(
to_precision: d.valid_to_precision.as_deref(),
attested_at: d.attested_from,
};
+ let value;
let object = match (d.object_id, &d.object_value) {
(Some(o), _) => FactObject::Entity(o),
- (None, Some(v)) => FactObject::Value(v),
+ (None, Some(v)) => {
+ let Some(v) = attribute_value(v, d.datatype.as_deref(), d.unit.as_deref()) else {
+ continue;
+ };
+ value = v;
+ FactObject::Value(&value)
+ }
_ => continue,
};
// 一个东西不蕴含自己:读数把「Virginia」读成「United States」是对的,把「France」
@@ -634,6 +657,27 @@ async fn imply_in_tx(
Ok(implied)
}
+/// 开放陈述的值(`{"value": "2.3亿元"}`)落到属性上的样子:`value` 那一格按 datatype 与单位读成
+/// 数、日期、真假([`utopia_extract::attribute_value`]),其余键照抄;绑到关系上的值原样留着。
+///
+/// **读不出来就不写**(None)。从前照抄原文的字:声明了 number、单位万元的「注册资本」上躺着
+/// `"2.3亿元"`,导出成 `"2.3亿元"^^xsd:decimal`,比较、对账、规则全都对不上。读不出来的陈述
+/// 还在开放图谱里,原文的字一个没丢。只在这里读、不在 `insert_fact_on` 的门上读:人和勘误
+/// 会在日期属性上写相对的日期(`relative`),那是另一种值
+fn attribute_value(raw: &Value, datatype: Option<&str>, unit: Option<&str>) -> Option<Value> {
+ let Some(datatype) = datatype else {
+ return Some(raw.clone());
+ };
+ let read = utopia_extract::attribute_value(datatype, unit, raw.get("value").unwrap_or(raw))?;
+ let mut out = if raw.is_object() {
+ raw.clone()
+ } else {
+ serde_json::json!({})
+ };
+ out["value"] = read;
+ Some(out)
+}
+
/// 类别词的归一(同 type_bindings):这里要在 SQL 里对上实体的 specific_type
fn kind_word_sql(col: &str) -> String {
format!("lower(btrim(regexp_replace({col}, '\\s+', ' ', 'g')))")
diff --git a/crates/utopia-store/tests/store/an_attribute_value_is_read_in_its_unit.rs b/crates/utopia-store/tests/store/an_attribute_value_is_read_in_its_unit.rs
new file mode 100644
index 00000000..1a939b16
--- /dev/null
+++ b/crates/utopia-store/tests/store/an_attribute_value_is_read_in_its_unit.rs
@@ -0,0 +1,93 @@
+//! 绑到属性上的开放陈述算成类型化行时,值按属性的 datatype 与单位读(`materialize::attribute_value`)。
+//!
+//! 一家公司三条陈述:「注册资本 2.3亿元」绑到单位万元的 number 属性,「成立 2008年11月」
+//! 绑到 date 属性,「注册资本 5000万美元」货币对不上。前两条算出 23000 与 2008-11;第三条
+//! 不写——陈述还在开放图谱里,原文的字没丢。从前三条都照抄原文的字落进类型化图谱。
+//!
+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
+
+use serde_json::json;
+use sqlx::PgPool;
+use utopia_store::materialize::materialize;
+use uuid::Uuid;
+
+#[tokio::test]
+async fn an_attribute_value_is_read_in_its_unit() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'attribute-unit-test')")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'attribute-unit-test')")
+ .bind(ws)
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'attribute-unit-test')",
+ )
+ .bind(kb)
+ .bind(ws)
+ .execute(&pool)
+ .await?;
+
+ let run = async {
+ let id = Uuid::now_v7;
+ let (company, capital, founded, lanshan) = (id(), id(), id(), id());
+ let (in_yuan, in_dollars, month) = (id(), id(), id());
+ // 插进去的只有本地生成的 UUID 和写死的字
+ sqlx::raw_sql(&format!(
+ r#"
+ INSERT INTO entity_types (id, kb_id, key, label, color, shape) VALUES
+ ('{company}','{kb}','company','公司','#7fd0ff','circle');
+ INSERT INTO relation_types (id, kb_id, key, label, temporal, kind, datatype, unit) VALUES
+ ('{capital}','{kb}','registered_capital','注册资本','eternal','attribute','number','万元'),
+ ('{founded}','{kb}','founded_on','成立日期','eternal','attribute','date',NULL);
+ INSERT INTO entities (id, kb_id, type_id, canonical_name) VALUES
+ ('{lanshan}','{kb}','{company}','蓝山生物');
+ INSERT INTO facts (id, kb_id, subject_id, object_value, layer, phrase, confidence) VALUES
+ ('{in_yuan}','{kb}','{lanshan}','{{"value": "2.3亿元"}}','open','注册资本',0.9),
+ ('{in_dollars}','{kb}','{lanshan}','{{"value": "5000万美元"}}','open','注册资本',0.9),
+ ('{month}','{kb}','{lanshan}','{{"value": "2008年11月"}}','open','成立',0.9);
+ INSERT INTO phrase_bindings (id, kb_id, phrase, subject_type_id, object_type_id,
+ object_is_value, relation_type_id, direction, status) VALUES
+ ('{b1}','{kb}','注册资本','{company}',NULL,true,'{capital}','forward','bound'),
+ ('{b2}','{kb}','成立','{company}',NULL,true,'{founded}','forward','bound');
+ "#,
+ b1 = id(),
+ b2 = id(),
+ ))
+ .execute(&pool)
+ .await?;
+
+ materialize(&pool, kb).await?;
+ let typed: Vec<(Uuid, serde_json::Value)> = sqlx::query_as(
+ "SELECT predicate_id, object_value FROM facts
+ WHERE kb_id = $1 AND layer = 'typed' AND invalidated_at IS NULL
+ ORDER BY object_value::text",
+ )
+ .bind(kb)
+ .fetch_all(&pool)
+ .await?;
+ assert_eq!(
+ typed,
+ vec![
+ (founded, json!({"value": "2008-11"})),
+ (capital, json!({"value": 23000.0})),
+ ]
+ );
+
+ anyhow::Ok(())
+ }
+ .await;
+
+ sqlx::query("DELETE FROM organizations WHERE id = $1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ run
+}
diff --git a/crates/utopia-store/tests/store/main.rs b/crates/utopia-store/tests/store/main.rs
index b263d456..1f61b94b 100644
--- a/crates/utopia-store/tests/store/main.rs
+++ b/crates/utopia-store/tests/store/main.rs
@@ -103,6 +103,7 @@ mod a_wrong_time_can_be_corrected;
mod adopting_an_iri_adopts_the_shape;
mod an_admin_grants_system_admin;
mod an_agent_can_record;
+mod an_attribute_value_is_read_in_its_unit;
mod an_automatic_merge_is_gated_by_what_it_can_undo;
mod an_axiom_that_moved_closes_its_findings;
mod an_earlier_mention_keeps_the_stated_end;
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index 2798af29..efb8a52a 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -586,6 +586,7 @@ export const en = {
chunk_unextracted: "A passage the endpoint could not answer for",
phrase_is_value: "Kept, but the phrase is the value itself",
phrase_is_subject: "Kept, but the phrase is the subject's own name",
+ object_is_value: "Kept, with the object read as a value",
} as Record<string, string>,
// 来源级重抽:不危险,只是费时费钱——轻确认,文案直说成本与保留项
reExtractSource: "Re-extract",
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index 23eead08..17c16c7d 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -528,6 +528,7 @@ export const zh: Strings = {
chunk_unextracted: "这一段端点没能答上,没有进图",
phrase_is_value: "已保留,但短语就是那个值",
phrase_is_subject: "已保留,但短语就是主语的名字",
+ object_is_value: "已保留,宾语按值记下",
},
reExtractSource: "重新抽取",
reExtractTitle: "重新抽取这个来源?",