Skip to content

World's first open-source enterprise world model.

A time mention is resolved against its document (#740)

WaylandYang24a85df9dcb5
diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs
index d90fa4a8..0b0f19b7 100644
--- a/crates/utopia-core/src/models.rs
+++ b/crates/utopia-core/src/models.rs
@@ -93,7 +93,15 @@ pub struct Document {
pub status: String,
pub error: Option<String>,
pub doc_time: Option<DateTime<Utc>>,
+ /// `doc_time` 从哪来:`content`(正文里读出)/ `source`(来源系统给的:发布时间、
+ /// 归档日期)/ `none`(没有日期)。上传时刻与文件修改时刻都不是文档的日期
+ /// (0045 决定 3,#714):老值 `upload_time` / `file_mtime` 读作没有日期,见
+ /// [`Document::dated_at`]
pub doc_time_source: String,
+ /// 文档的时间语境(0045 决定 3):它自己的日期、它定义的期间与历法、叙述设下的锚点。
+ /// 服务端边抽取边填;`time_context_at` 是最近一次写下它的时刻
+ pub time_context: Option<serde_json::Value>,
+ pub time_context_at: Option<DateTime<Utc>>,
/// 图谱抽取状态:none → queued → extracting → done | failed
pub graph_status: String,
/// 抽取失败原因(失败时才有)。与 error 分列——那列归解析管道,
@@ -118,6 +126,17 @@ pub struct Document {
pub updated_at: DateTime<Utc>,
}
+impl Document {
+ /// 文档自己的日期:只认正文或来源系统给的(`content` / `source`)。上传、同步、
+ /// 抽取的时刻是记录时间,不是文档的日期(0045 决定 3,#714)——别的来源一律 `None`
+ pub fn dated_at(&self) -> Option<DateTime<Utc>> {
+ match self.doc_time_source.as_str() {
+ "content" | "source" => self.doc_time,
+ _ => None,
+ }
+ }
+}
+
/// 摄入来源("来源即文件夹":容器 + 定时同步)。
#[derive(Debug, Clone, Serialize, sqlx::FromRow)]
pub struct Source {
diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs
index 58fe1585..deb97afa 100644
--- a/crates/utopia-extract/src/lib.rs
+++ b/crates/utopia-extract/src/lib.rs
@@ -11,6 +11,7 @@ use utopia_llm::ChatMessage;
pub mod governor;
pub mod open;
+pub mod time;
/// Response-scoped reference to a persistent entity; database UUIDs must never enter prompts.
pub struct KnownEntity {
diff --git a/crates/utopia-extract/src/open.rs b/crates/utopia-extract/src/open.rs
index f96fa3c6..a04a0302 100644
--- a/crates/utopia-extract/src/open.rs
+++ b/crates/utopia-extract/src/open.rs
@@ -253,7 +253,7 @@ fn json_tail(raw: &str) -> Option<&str> {
/// 结尾。这里每条记录是数组,限定词全是 null 的回复里可能一个 `}` 都没有,只认 `}`
/// 就是整块作废。所以回退点是最后一个 `]` 或 `}`,谁靠后用谁;括号是否在字符串里
/// 由 `close_brackets` 判断
-fn repair_truncated_compact(json: &str) -> Option<String> {
+pub(crate) fn repair_truncated_compact(json: &str) -> Option<String> {
let mut cut = json.len();
for _ in 0..64 {
let idx = json[..cut].rfind([']', '}'])?;
diff --git a/crates/utopia-extract/src/time.rs b/crates/utopia-extract/src/time.rs
new file mode 100644
index 00000000..c7339789
--- /dev/null
+++ b/crates/utopia-extract/src/time.rs
@@ -0,0 +1,1066 @@
+//! 时间提法的解读(0045 决定 2、3、8):**模型读,代码算**。
+//!
+//! 两次调用,都只返回结构化的字段,模型从不写一个算出来的日期:
+//!
+//! 1. **给文档定日期**([`build_dating_messages`] / [`parse_dating_response`]):读文件开头,
+//! 抄下文档自己的日期(电头、申报日、公报的报告年份)、它命名的期间及其边界、财年
+//! 的截止日。上传时间永远不进来——那是记录时间轴(决定 3)。
+//! 2. **解读提法**([`build_interpretation_messages`] / [`parse_interpretation_response`]):
+//! 每条提法给形状、引用、粒度。绝对值按原文的字抄成数字部件;相对的说锚点(另一条
+//! 提法、文档本身、文档命名的期间)与偏移(数、单位、方向);期间报名字。日历算术、
+//! 财年到区间、按粒度截断、形状蕴含的区间,全在服务端算。
+//!
+//! 代码里**没有时间词**(决定 8):不认月份名、不认「去年」「上年末」,只解 JSON、验
+//! 部件范围、拼提示词。提示词是常量英文;例子是中性的(镇议会、桥、面包房),不出自
+//! 任何测量语料。
+//!
+//! 部件的键是短的:`y` 年、`m` 月、`d` 日、`h` 时、`min` 分、`s` 秒;只写原文说到的
+//! 部件(「2024 年」就是 `{"y": 2024}`)。细的部件必须带着粗的(有日必有月),这是
+//! 0024 的精度阶梯。
+//!
+//! 解析逐项宽容:坏的条目计数,不毁整批;调用方**必须**把计数报出去(#108 那类错)。
+//! 截断的回复退到最后一个完整条目补括号,与 `open.rs` 同一套修补。
+
+use std::collections::HashSet;
+use std::fmt;
+
+use serde::de::DeserializeOwned;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use utopia_llm::ChatMessage;
+
+use crate::open::repair_truncated_compact;
+use crate::{json_block, json_text};
+
+/// 原文说到的日期部件;只有原文写了的才是 `Some`。存成 JSONB 时用全名键
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DateParts {
+ pub year: i32,
+ pub month: Option<u32>,
+ pub day: Option<u32>,
+ pub hour: Option<u32>,
+ pub minute: Option<u32>,
+ pub second: Option<u32>,
+}
+
+impl DateParts {
+ /// 部件够到的那一级:有日就是日、只有年就是年。服务端拿它对照模型报的粒度——
+ /// 两者不合的提法进审核(记录「不做」的最后一条)
+ pub fn granularity(&self) -> Granularity {
+ if self.second.is_some() {
+ Granularity::Second
+ } else if self.minute.is_some() {
+ Granularity::Minute
+ } else if self.hour.is_some() {
+ Granularity::Hour
+ } else if self.day.is_some() {
+ Granularity::Day
+ } else if self.month.is_some() {
+ Granularity::Month
+ } else {
+ Granularity::Year
+ }
+ }
+}
+
+/// 只排版、不计算:`2011`、`2011-03`、`2011-03-04`、`2011-03-04 14:30`
+impl fmt::Display for DateParts {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "{}", self.year)?;
+ if let Some(m) = self.month {
+ write!(f, "-{m:02}")?;
+ }
+ if let Some(d) = self.day {
+ write!(f, "-{d:02}")?;
+ }
+ if let Some(h) = self.hour {
+ write!(f, " {h:02}")?;
+ }
+ if let Some(mi) = self.minute {
+ write!(f, ":{mi:02}")?;
+ }
+ if let Some(s) = self.second {
+ write!(f, ":{s:02}")?;
+ }
+ Ok(())
+ }
+}
+
+/// 文档命名的期间及其边界(「fiscal 2019」到 9 月 30 日;「2024年」作为报告年)
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NamedPeriod {
+ pub name: String,
+ pub from: DateParts,
+ pub to: DateParts,
+}
+
+/// 文档自己的时间语境(决定 3):从开头读出来、存在文档上、每块都带着
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[serde(default)]
+pub struct DocumentDating {
+ /// the document's own date as the text states it (a dateline, a filing date, a bulletin's
+ /// reporting year); None when the text gives none
+ pub date: Option<DateParts>,
+ /// the words that state it, verbatim (the server checks they occur in the opening)
+ pub date_words: Option<String>,
+ /// periods the document names with bounds it states ("fiscal 2027" with its end date;
+ /// "2024年" as a reporting year)
+ pub periods: Vec<NamedPeriod>,
+ /// (month, day) on which the document's fiscal year ends, when stated
+ pub fiscal_year_end: Option<(u32, u32)>,
+ /// items the reply wrote but that were malformed (a date with month 13, a period without
+ /// bounds); must be reported by the caller
+ pub skipped: usize,
+}
+
+/// 提法对它所定的陈述做了什么
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Shape {
+ Point,
+ Since,
+ Until,
+ Interval,
+ AsOf,
+ Duration,
+ EndedUnknown,
+}
+
+/// 原文的字够到阶梯的哪一级(0024),与引用分开记
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Granularity {
+ Year,
+ Month,
+ Day,
+ Hour,
+ Minute,
+ Second,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Unit {
+ Year,
+ Quarter,
+ Month,
+ Week,
+ Day,
+ Hour,
+ Minute,
+ Second,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Direction {
+ Before,
+ After,
+}
+
+/// 原文写的偏移:「two years later」= 2 年 after;「this quarter」= 0 quarter
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Offset {
+ pub count: i64,
+ pub unit: Unit,
+ pub direction: Direction,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum Anchor {
+ Mention { id: i64 },
+ Document,
+ Period { name: String },
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum Reference {
+ /// the words state the value; `to` only for an interval written out ("from March to May 2024")
+ Absolute {
+ from: DateParts,
+ to: Option<DateParts>,
+ },
+ /// relative to an anchor, with an offset when the words give one ("two years later"); no
+ /// offset = the anchor itself ("today")
+ Anchored {
+ anchor: Anchor,
+ offset: Option<Offset>,
+ },
+ /// a named period from the context or one whose bounds the text states here
+ Period {
+ name: String,
+ from: Option<DateParts>,
+ to: Option<DateParts>,
+ },
+ None,
+}
+
+/// 一条提法的解读;存成 JSONB,服务端从它算区间
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Interpretation {
+ pub id: i64,
+ pub shape: Shape,
+ pub reference: Reference,
+ pub granularity: Granularity,
+}
+
+/// 送去解读的一条提法:编号、原文的字、所在的句子(模型靠句子挑锚点)
+#[derive(Debug, Clone, Copy)]
+pub struct MentionInput<'a> {
+ pub id: i64,
+ pub text: &'a str,
+ pub sentence: &'a str,
+}
+
+/// 进提示词的时间语境:[`DocumentDating`] 的借用视图
+#[derive(Debug, Clone, Copy)]
+pub struct TimeContext<'a> {
+ pub date: Option<&'a DateParts>,
+ pub date_words: Option<&'a str>,
+ pub periods: &'a [NamedPeriod],
+ pub fiscal_year_end: Option<(u32, u32)>,
+}
+
+/// 定日期的系统消息。只抄开头说了的;文档自己的日期,从不是今天;部件是抄写的数字
+const DATING_SYSTEM: &str = "\
+You read the opening of a document and write down what it states about the document's own \
+time: its date, the periods it names with their bounds, and the day its fiscal year ends. You \
+transcribe what the words state; you never compute a date. Output one JSON object and nothing \
+else, shaped like this:\n\
+\n\
+{\"date\": {\"y\": 2011, \"m\": 3, \"d\": 4}, \"date_words\": \"March 4, 2011\", \"periods\": [[\"name as written\", {\"y\": 2010, \"m\": 10, \"d\": 1}, {\"y\": 2011, \"m\": 9, \"d\": 30}]], \"fiscal_year_end\": [9, 30]}\n\
+\n\
+1. date is the document's own date: the day it was written, issued, filed, published or signed \
+(a dateline, a filing date, a date under the title), or, for a report or bulletin that covers a \
+period, the period it reports on. It is never today's date, and never a date the opening \
+mentions for something else — an event it reports, an agreement it refers to, a deadline it \
+sets. null when the opening states none.\n\
+2. A date is written as parts, each a number transcribed from the words: y the year, m the \
+month (a month name becomes its number), d the day, and h, min, s for a clock time when one is \
+written. Write only the parts the words state: a bulletin for the year 2024 has {\"y\": 2024}; \
+a report for March 2024 has {\"y\": 2024, \"m\": 3}; a dateline \"4 March 2011\" has \
+{\"y\": 2011, \"m\": 3, \"d\": 4}.\n\
+3. date_words is the words that state the date, copied verbatim from the opening; null when \
+date is null.\n\
+4. periods lists the periods the opening names and bounds, one entry each, [name, from, to]: \
+a fiscal year or quarter, a reporting period, a term. name is the period as the opening writes \
+it (\"fiscal 2019\", \"the second quarter\", \"2024年\"); from and to are its first and last \
+day, month or year as parts, at the precision the words give. A reporting year \"2024\" runs \
+from {\"y\": 2024} to {\"y\": 2024}. A fiscal year or quarter the opening states by its end \
+date (\"the year ended September 30, 2019\") runs from the day after the previous one ended to \
+that end date. Leave out a period whose bounds the opening does not fix.\n\
+5. fiscal_year_end is [month, day] on which the document's fiscal year ends, when the opening \
+states it or a stated fiscal year end makes it plain; null otherwise.\n\
+6. Never compute a date from today, never resolve a relative expression, never fill in a part \
+the words do not state. When the opening states nothing about the document's date or periods, \
+output {\"date\": null, \"date_words\": null, \"periods\": [], \"fiscal_year_end\": null}.\n\
+\n\
+Example. The opening \"Westbrook Town Council — Minutes of the meeting held on March 4, 2011. \
+Present: the mayor and six councillors.\" gives {\"date\": {\"y\": 2011, \"m\": 3, \"d\": 4}, \
+\"date_words\": \"March 4, 2011\", \"periods\": [], \"fiscal_year_end\": null}. The opening \
+\"Harbor Bakery — Annual report for fiscal 2019, the year ended September 30, 2019. Issued \
+November 12, 2019.\" gives {\"date\": {\"y\": 2019, \"m\": 11, \"d\": 12}, \"date_words\": \
+\"November 12, 2019\", \"periods\": [[\"fiscal 2019\", {\"y\": 2018, \"m\": 10, \"d\": 1}, \
+{\"y\": 2019, \"m\": 9, \"d\": 30}]], \"fiscal_year_end\": [9, 30]}";
+
+/// 解读提法的系统消息。形状各一行;引用是字面说的,不是它蕴含的;从不计算
+const INTERPRETATION_SYSTEM: &str = "\
+You interpret the time expressions of a document. Each mention is given with its id, its words \
+exactly as written and the sentence it occurs in; before them, the document's own time context: \
+its date and the words that state it, the periods it names with their bounds, and the day its \
+fiscal year ends. For each mention you write what its words state — a shape, a reference and a \
+granularity — and never a date you computed: the reader computes dates from what you write. \
+Output one JSON object and nothing else, shaped like this:\n\
+\n\
+{\"m\": [[id, \"shape\", reference, \"granularity\"]]}\n\
+\n\
+1. shape is what the time does to the statement it dates, one of:\n\
+ \"point\" — the statement happened or holds at that time;\n\
+ \"since\" — it holds from that time on;\n\
+ \"until\" — it holds up to that time;\n\
+ \"interval\" — it holds between two bounds;\n\
+ \"as_of\" — the state observed at that time;\n\
+ \"duration\" — a length of time with no position;\n\
+ \"ended_unknown\" — the words say it ended but not when.\n\
+2. reference is what the words say, not what they imply, in one of four forms:\n\
+ {\"kind\": \"absolute\", \"from\": {\"y\": 2011, \"m\": 3, \"d\": 4}} — the words state the \
+value. Parts are transcribed digits: y the year, m the month (a month name becomes its number), \
+d the day, and h, min, s for a clock time; write only the parts the words state. A bare year \
+(\"2019\") is {\"y\": 2019} with granularity \"year\". \"to\" is added only for an interval the \
+words write out with both bounds (\"from March to May 2024\": from {\"y\": 2024, \"m\": 3}, to \
+{\"y\": 2024, \"m\": 5}).\n\
+ {\"kind\": \"anchored\", \"anchor\": {\"kind\": \"document\"}, \"offset\": {\"count\": 1, \
+\"unit\": \"year\", \"direction\": \"before\"}} — the words point at another time. The anchor is \
+the document itself, {\"kind\": \"document\"}, for \"today\", \"now\", \"this quarter\", \"last \
+year\", \"the prior year\"; the mention the sentence counts from, {\"kind\": \"mention\", \
+\"id\": 3}, for \"two years later\", \"three months earlier\", \"the following day\" when the \
+sentence names that time, and the document when it names none; a period from the context, \
+{\"kind\": \"period\", \"name\": \"fiscal 2019\"}, for \"the end of fiscal 2019\". offset is the \
+count, unit and direction the words state: \"two years later\" is count 2, unit \"year\", \
+direction \"after\"; \"this quarter\" and \"this year\" are count 0 with that unit; \"today\" \
+and \"now\" have no offset. unit is one of year, quarter, month, week, day, hour, minute, \
+second; direction is before or after.\n\
+ {\"kind\": \"period\", \"name\": \"Q2 fiscal 2019\"} — the words name a period. One listed in \
+the context is named exactly as listed. One whose bounds the words state here carries them: a \
+heading \"Three months ended June 30, 2019\" is {\"kind\": \"period\", \"name\": \"Three months \
+ended June 30, 2019\", \"to\": {\"y\": 2019, \"m\": 6, \"d\": 30}}, with \"from\" only when the \
+words state it. Words that name a period only relatively (the whole year, the end of the year, \
+the start of the month, this quarter) are not a period: they are anchored to the document with \
+count 0 and that unit; the end of it takes shape \"as_of\" or \"until\", the start \"since\", the \
+whole \"interval\". A count inside something the sentence names (a week of a trial, a month of a \
+programme) anchors to that thing's dated mention when the sentence gives one, otherwise it is \
+{\"kind\": \"none\"}: it is not counted from the document.\n\
+ {\"kind\": \"none\"} — the words give nothing to anchor to: a duration (\"ten years\"), an \
+ending without a date (\"formerly\", \"no longer\"), a vague time (\"recently\").\n\
+3. granularity is the rung the words reach: \"year\", \"month\", \"day\", \"hour\", \"minute\" \
+or \"second\". \"March 2024\" is month; \"two years later\" is year; \"today\" is day; a quarter \
+or a season is month.\n\
+4. Never compute: do not resolve \"last year\" to a year, do not add an offset to a date, do not \
+turn a period into dates, do not use today's date. Write one item per mention, with the id from \
+the list, and no item for an id not listed.\n\
+\n\
+Example. With the document dated March 4, 2011 and the mentions [id 1 \"March 4, 2011\" in \"The \
+council met on March 4, 2011.\", id 2 \"since 2005\" in \"Harbor Bakery has supplied the school \
+since 2005.\", id 3 \"2019\" and id 4 \"two years later\" in \"The bridge opened in 2019; two \
+years later the eastern span was widened.\", id 5 \"last year\" in \"Last year the bakery opened \
+a second shop.\"]:\n\
+{\"m\": [[1, \"point\", {\"kind\": \"absolute\", \"from\": {\"y\": 2011, \"m\": 3, \"d\": 4}}, \"day\"],\n\
+ [2, \"since\", {\"kind\": \"absolute\", \"from\": {\"y\": 2005}}, \"year\"],\n\
+ [3, \"point\", {\"kind\": \"absolute\", \"from\": {\"y\": 2019}}, \"year\"],\n\
+ [4, \"point\", {\"kind\": \"anchored\", \"anchor\": {\"kind\": \"mention\", \"id\": 3}, \"offset\": {\"count\": 2, \"unit\": \"year\", \"direction\": \"after\"}}, \"year\"],\n\
+ [5, \"point\", {\"kind\": \"anchored\", \"anchor\": {\"kind\": \"document\"}, \"offset\": {\"count\": 1, \"unit\": \"year\", \"direction\": \"before\"}}, \"year\"]]}";
+
+/// 定日期的两条消息:常量系统消息 + 文件名与开头。开头不在这里截——它就是这次
+/// 调用的正文,预算由调用方定
+pub fn build_dating_messages(filename: &str, opening: &str) -> Vec<ChatMessage> {
+ let user = format!("Document: {filename}\n\nOpening:\n\"\"\"\n{opening}\n\"\"\"");
+ vec![
+ ChatMessage {
+ role: "system".into(),
+ content: DATING_SYSTEM.to_string(),
+ },
+ ChatMessage {
+ role: "user".into(),
+ content: user,
+ },
+ ]
+}
+
+/// 解读提法的两条消息:常量系统消息 + 语境块 + 提法清单(编号、原文的字、句子)
+pub fn build_interpretation_messages(
+ context: &TimeContext<'_>,
+ mentions: &[MentionInput<'_>],
+) -> Vec<ChatMessage> {
+ let mut user = String::from("Document time context:\n");
+ match (context.date, context.date_words) {
+ (Some(date), Some(words)) => {
+ user.push_str(&format!(
+ "- date: {date}, stated by the words \"{words}\"\n"
+ ));
+ }
+ (Some(date), None) => user.push_str(&format!("- date: {date}\n")),
+ (None, _) => user.push_str(
+ "- date: not stated; still anchor to the document where the words point at it\n",
+ ),
+ }
+ if context.periods.is_empty() {
+ user.push_str("- periods: none named\n");
+ } else {
+ let listed: Vec<String> = context
+ .periods
+ .iter()
+ .map(|p| format!("\"{}\" from {} to {}", p.name, p.from, p.to))
+ .collect();
+ user.push_str(&format!("- periods: {}\n", listed.join("; ")));
+ }
+ match context.fiscal_year_end {
+ Some((m, d)) => user.push_str(&format!("- fiscal year end: month {m}, day {d}\n")),
+ None => user.push_str("- fiscal year end: not stated\n"),
+ }
+ user.push_str("\nMentions:\n");
+ for m in mentions {
+ user.push_str(&format!(
+ "- id {}, words \"{}\", in the sentence: \"{}\"\n",
+ m.id, m.text, m.sentence
+ ));
+ }
+ vec![
+ ChatMessage {
+ role: "system".into(),
+ content: INTERPRETATION_SYSTEM.to_string(),
+ },
+ ChatMessage {
+ role: "user".into(),
+ content: user,
+ },
+ ]
+}
+
+/// 先按常规取块解;解不开再从第一个 `{` 取到结尾,退到最后一个完整条目补括号
+/// (同 `open.rs`:紧凑回复的最后一个 `}` 不是可靠的结尾)
+fn reply_value(raw: &str, what: &str) -> anyhow::Result<Value> {
+ let block = json_block(raw)
+ .and_then(|b| serde_json::from_str::<Value>(&b).map_err(anyhow::Error::from));
+ match block {
+ Ok(v) => Ok(v),
+ Err(e) => {
+ let text = json_text(raw);
+ let fixed = text
+ .find('{')
+ .and_then(|s| repair_truncated_compact(&text[s..]))
+ .ok_or_else(|| anyhow::anyhow!("Failed to parse {what} JSON: {e}"))?;
+ serde_json::from_str::<Value>(&fixed)
+ .map_err(|e| anyhow::anyhow!("Failed to parse {what} JSON: {e}"))
+ }
+ }
+}
+
+/// 一个整数:JSON 数字,或模型偶尔写成字符串的数字
+fn int(v: &Value) -> Option<i64> {
+ match v {
+ Value::Number(n) => n.as_i64(),
+ Value::String(s) => s.trim().parse().ok(),
+ _ => None,
+ }
+}
+
+/// 可选的整数部件,带范围:缺或 null 是 `Some(None)`;有但出界或不是数是 `None`(坏)
+fn part(obj: &Value, key: &str, range: std::ops::RangeInclusive<i64>) -> Option<Option<u32>> {
+ match obj.get(key) {
+ None | Some(Value::Null) => Some(None),
+ Some(v) => {
+ let n = int(v)?;
+ range.contains(&n).then_some(Some(n as u32))
+ }
+ }
+}
+
+/// 日期部件:`y` 必有;细的部件必须带着粗的(有日必有月),月 1–12、日 1–31、
+/// 时 0–23、分秒 0–59;出界就是坏条目
+fn parts(v: &Value) -> Option<DateParts> {
+ if !v.is_object() {
+ return None;
+ }
+ let year = i32::try_from(int(v.get("y")?)?).ok()?;
+ let month = part(v, "m", 1..=12)?;
+ let day = part(v, "d", 1..=31)?;
+ let hour = part(v, "h", 0..=23)?;
+ let minute = part(v, "min", 0..=59)?;
+ let second = part(v, "s", 0..=59)?;
+ let ladder = [
+ month.is_some(),
+ day.is_some(),
+ hour.is_some(),
+ minute.is_some(),
+ second.is_some(),
+ ];
+ if ladder.windows(2).any(|w| w[1] && !w[0]) {
+ return None;
+ }
+ Some(DateParts {
+ year,
+ month,
+ day,
+ hour,
+ minute,
+ second,
+ })
+}
+
+/// 可选的部件:缺或 null 是 `Some(None)`;有但坏是 `None`
+fn opt_parts(v: Option<&Value>) -> Option<Option<DateParts>> {
+ match v {
+ None | Some(Value::Null) => Some(None),
+ Some(v) => parts(v).map(Some),
+ }
+}
+
+/// 非空文字,去首尾空白
+fn text(v: &Value) -> Option<String> {
+ v.as_str()
+ .map(str::trim)
+ .filter(|s| !s.is_empty())
+ .map(str::to_string)
+}
+
+/// 枚举词:借 serde 的 snake_case 名认,大小写、空格与连字符都宽容("As of" → as_of)
+fn word<T: DeserializeOwned>(v: &Value) -> Option<T> {
+ let s = v
+ .as_str()?
+ .trim()
+ .to_ascii_lowercase()
+ .replace([' ', '-'], "_");
+ serde_json::from_value(Value::String(s)).ok()
+}
+
+/// 期间条目 `[name, from, to]`
+fn named_period(v: &Value) -> Option<NamedPeriod> {
+ let arr = v.as_array()?;
+ Some(NamedPeriod {
+ name: text(arr.first()?)?,
+ from: parts(arr.get(1)?)?,
+ to: parts(arr.get(2)?)?,
+ })
+}
+
+/// 解定日期的回复。日期、财年截止日坏了就留空并计数;期间逐条计数
+pub fn parse_dating_response(raw: &str) -> anyhow::Result<DocumentDating> {
+ let value = reply_value(raw, "document dating")?;
+ let mut out = DocumentDating::default();
+ match value.get("date") {
+ None | Some(Value::Null) => {}
+ Some(v) => match parts(v) {
+ Some(d) => out.date = Some(d),
+ None => out.skipped += 1,
+ },
+ }
+ out.date_words = value.get("date_words").and_then(text);
+ if let Some(items) = value.get("periods").and_then(Value::as_array) {
+ for item in items {
+ match named_period(item) {
+ Some(p) => out.periods.push(p),
+ None => out.skipped += 1,
+ }
+ }
+ }
+ match value.get("fiscal_year_end") {
+ None | Some(Value::Null) => {}
+ Some(v) => {
+ let pair = v.as_array().and_then(|a| {
+ let m = int(a.first()?)?;
+ let d = int(a.get(1)?)?;
+ ((1..=12).contains(&m) && (1..=31).contains(&d)).then_some((m as u32, d as u32))
+ });
+ match pair {
+ Some(p) => out.fiscal_year_end = Some(p),
+ None => out.skipped += 1,
+ }
+ }
+ }
+ Ok(out)
+}
+
+/// 锚点:`{"kind": "mention", "id": N}` / `{"kind": "document"}` / `{"kind": "period", "name"}`;
+/// 光秃秃的 "document" 也认——只有它不带别的字段
+fn anchor(v: &Value) -> Option<Anchor> {
+ let kind = match v {
+ Value::String(s) => s.clone(),
+ v => v.get("kind")?.as_str()?.to_string(),
+ };
+ let kind = kind.trim().to_ascii_lowercase();
+ match kind.as_str() {
+ "mention" => Some(Anchor::Mention {
+ id: int(v.get("id")?)?,
+ }),
+ "document" => Some(Anchor::Document),
+ "period" => Some(Anchor::Period {
+ name: text(v.get("name")?)?,
+ }),
+ _ => None,
+ }
+}
+
+/// 偏移:数不为负(方向另说),单位与方向是枚举词
+fn offset(v: &Value) -> Option<Offset> {
+ let count = int(v.get("count")?)?;
+ if count < 0 {
+ return None;
+ }
+ Some(Offset {
+ count,
+ unit: word(v.get("unit")?)?,
+ direction: word(v.get("direction")?)?,
+ })
+}
+
+fn opt_offset(v: Option<&Value>) -> Option<Option<Offset>> {
+ match v {
+ None | Some(Value::Null) => Some(None),
+ Some(v) => offset(v).map(Some),
+ }
+}
+
+/// 引用:按 `kind` 分四种;JSON null 当 none
+fn reference(v: &Value) -> Option<Reference> {
+ if v.is_null() {
+ return Some(Reference::None);
+ }
+ let kind = v.get("kind")?.as_str()?.trim().to_ascii_lowercase();
+ match kind.as_str() {
+ "absolute" => Some(Reference::Absolute {
+ from: parts(v.get("from")?)?,
+ to: opt_parts(v.get("to"))?,
+ }),
+ "anchored" => Some(Reference::Anchored {
+ anchor: anchor(v.get("anchor")?)?,
+ offset: opt_offset(v.get("offset"))?,
+ }),
+ "period" => Some(Reference::Period {
+ name: text(v.get("name")?)?,
+ from: opt_parts(v.get("from"))?,
+ to: opt_parts(v.get("to"))?,
+ }),
+ "none" => Some(Reference::None),
+ _ => None,
+ }
+}
+
+/// 一条 `[id, shape, reference, granularity]`
+fn interpretation(v: &Value) -> Option<Interpretation> {
+ let arr = v.as_array()?;
+ Some(Interpretation {
+ id: int(arr.first()?)?,
+ shape: word(arr.get(1)?)?,
+ reference: reference(arr.get(2)?)?,
+ granularity: word(arr.get(3)?)?,
+ })
+}
+
+/// 解解读的回复:返回解开的解读与坏条目数。编号不在 `ids` 里的、同一编号第二次出现的
+/// 都算坏;截断的回复修补后少掉的条目不算坏——调用方拿 `ids` 对一下就知道谁没回来
+pub fn parse_interpretation_response(
+ raw: &str,
+ ids: &[i64],
+) -> anyhow::Result<(Vec<Interpretation>, usize)> {
+ let value = reply_value(raw, "time interpretation")?;
+ let known: HashSet<i64> = ids.iter().copied().collect();
+ let mut seen = HashSet::new();
+ let mut out = Vec::new();
+ let mut skipped = 0;
+ if let Some(items) = value.get("m").and_then(Value::as_array) {
+ for item in items {
+ match interpretation(item) {
+ Some(i) if known.contains(&i.id) && seen.insert(i.id) => out.push(i),
+ _ => skipped += 1,
+ }
+ }
+ }
+ Ok((out, skipped))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn ymd(y: i32, m: u32, d: u32) -> DateParts {
+ DateParts {
+ year: y,
+ month: Some(m),
+ day: Some(d),
+ hour: None,
+ minute: None,
+ second: None,
+ }
+ }
+
+ fn year(y: i32) -> DateParts {
+ DateParts {
+ year: y,
+ month: None,
+ day: None,
+ hour: None,
+ minute: None,
+ second: None,
+ }
+ }
+
+ /// 一份完整的定日期回复:带部件的日期、原话、两个期间、财年截止日
+ const DATING: &str = r#"{"date": {"y": 2019, "m": 11, "d": 12}, "date_words": "November 12, 2019",
+ "periods": [["fiscal 2019", {"y": 2018, "m": 10, "d": 1}, {"y": 2019, "m": 9, "d": 30}],
+ ["2024年", {"y": 2024}, {"y": 2024}]],
+ "fiscal_year_end": [9, 30]}"#;
+
+ #[test]
+ fn a_dating_reply_parses_into_the_context() {
+ let d = parse_dating_response(DATING).unwrap();
+ assert_eq!(d.skipped, 0);
+ assert_eq!(d.date, Some(ymd(2019, 11, 12)));
+ assert_eq!(d.date_words.as_deref(), Some("November 12, 2019"));
+ assert_eq!(d.periods.len(), 2);
+ assert_eq!(d.periods[0].name, "fiscal 2019");
+ assert_eq!(d.periods[0].from, ymd(2018, 10, 1));
+ assert_eq!(d.periods[0].to, ymd(2019, 9, 30));
+ assert_eq!(d.periods[1].name, "2024年");
+ assert_eq!(d.periods[1].from, year(2024));
+ assert_eq!(d.periods[1].to, year(2024));
+ assert_eq!(d.fiscal_year_end, Some((9, 30)));
+ }
+
+ #[test]
+ fn a_dating_reply_with_no_date_leaves_everything_empty() {
+ let d = parse_dating_response(
+ r#"{"date": null, "date_words": null, "periods": [], "fiscal_year_end": null}"#,
+ )
+ .unwrap();
+ assert_eq!(d.date, None);
+ assert_eq!(d.date_words, None);
+ assert!(d.periods.is_empty());
+ assert_eq!(d.fiscal_year_end, None);
+ assert_eq!(d.skipped, 0);
+ }
+
+ /// 坏的日期、坏的期间、坏的财年截止日各计一次;好的期间照收
+ #[test]
+ fn a_dating_reply_counts_its_malformed_items() {
+ let d = parse_dating_response(
+ r#"{"date": {"y": 2019, "m": 13}, "date_words": "Undecimber 2019",
+ "periods": [["fiscal 2019", {"y": 2018, "m": 10, "d": 1}, {"y": 2019, "m": 9, "d": 30}],
+ ["broken", {"y": 2018}],
+ ["day without month", {"y": 2018, "d": 5}, {"y": 2019}]],
+ "fiscal_year_end": [13, 1]}"#,
+ )
+ .unwrap();
+ assert_eq!(d.date, None);
+ assert_eq!(d.periods.len(), 1);
+ assert_eq!(d.fiscal_year_end, None);
+ assert_eq!(d.skipped, 4);
+ }
+
+ #[test]
+ fn a_clock_time_parses_and_prints_on_the_ladder() {
+ let d = parse_dating_response(
+ r#"{"date": {"y": 2011, "m": 3, "d": 4, "h": 14, "min": 30, "s": 5}, "date_words": "14:30:05 on March 4, 2011"}"#,
+ )
+ .unwrap();
+ let date = d.date.unwrap();
+ assert_eq!(date.granularity(), Granularity::Second);
+ assert_eq!(date.to_string(), "2011-03-04 14:30:05");
+ assert_eq!(year(2024).to_string(), "2024");
+ assert_eq!(year(2024).granularity(), Granularity::Year);
+ assert_eq!(ymd(2019, 9, 30).granularity(), Granularity::Day);
+ }
+
+ /// 四种引用、七种形状都在这一份回复里
+ const INTERPRETED: &str = r#"{"m": [
+ [1, "point", {"kind": "absolute", "from": {"y": 2011, "m": 3, "d": 4}}, "day"],
+ [2, "since", {"kind": "anchored", "anchor": {"kind": "document"}, "offset": {"count": 2, "unit": "year", "direction": "before"}}, "year"],
+ [3, "interval", {"kind": "period", "name": "fiscal 2019"}, "day"],
+ [4, "until", {"kind": "none"}, "day"],
+ [5, "as_of", {"kind": "anchored", "anchor": {"kind": "document"}}, "day"],
+ [6, "duration", {"kind": "none"}, "year"],
+ [7, "ended_unknown", null, "day"],
+ [8, "point", {"kind": "anchored", "anchor": {"kind": "mention", "id": 1}, "offset": {"count": 3, "unit": "month", "direction": "after"}}, "month"],
+ [9, "interval", {"kind": "absolute", "from": {"y": 2024, "m": 3}, "to": {"y": 2024, "m": 5}}, "month"],
+ [10, "interval", {"kind": "period", "name": "Three months ended June 30, 2019", "to": {"y": 2019, "m": 6, "d": 30}}, "day"],
+ [11, "point", {"kind": "anchored", "anchor": {"kind": "period", "name": "fiscal 2019"}, "offset": {"count": 0, "unit": "quarter", "direction": "after"}}, "month"]
+ ]}"#;
+
+ #[test]
+ fn an_interpretation_reply_parses_every_reference_kind_and_shape() {
+ let ids: Vec<i64> = (1..=11).collect();
+ let (items, skipped) = parse_interpretation_response(INTERPRETED, &ids).unwrap();
+ assert_eq!(skipped, 0);
+ assert_eq!(items.len(), 11);
+
+ assert_eq!(
+ items[0],
+ Interpretation {
+ id: 1,
+ shape: Shape::Point,
+ reference: Reference::Absolute {
+ from: ymd(2011, 3, 4),
+ to: None
+ },
+ granularity: Granularity::Day,
+ }
+ );
+ assert_eq!(
+ items[1].reference,
+ Reference::Anchored {
+ anchor: Anchor::Document,
+ offset: Some(Offset {
+ count: 2,
+ unit: Unit::Year,
+ direction: Direction::Before
+ }),
+ }
+ );
+ assert_eq!(items[1].shape, Shape::Since);
+ assert_eq!(items[1].granularity, Granularity::Year);
+ assert_eq!(
+ items[2].reference,
+ Reference::Period {
+ name: "fiscal 2019".into(),
+ from: None,
+ to: None
+ }
+ );
+ assert_eq!(items[2].shape, Shape::Interval);
+ assert_eq!(items[3].reference, Reference::None);
+ assert_eq!(items[3].shape, Shape::Until);
+ assert_eq!(
+ items[4].reference,
+ Reference::Anchored {
+ anchor: Anchor::Document,
+ offset: None
+ }
+ );
+ assert_eq!(items[4].shape, Shape::AsOf);
+ assert_eq!(items[5].shape, Shape::Duration);
+ assert_eq!(items[6].shape, Shape::EndedUnknown);
+ assert_eq!(
+ items[6].reference,
+ Reference::None,
+ "JSON null reads as none"
+ );
+ assert_eq!(
+ items[7].reference,
+ Reference::Anchored {
+ anchor: Anchor::Mention { id: 1 },
+ offset: Some(Offset {
+ count: 3,
+ unit: Unit::Month,
+ direction: Direction::After
+ }),
+ }
+ );
+ assert_eq!(
+ items[8].reference,
+ Reference::Absolute {
+ from: DateParts {
+ year: 2024,
+ month: Some(3),
+ day: None,
+ hour: None,
+ minute: None,
+ second: None
+ },
+ to: Some(DateParts {
+ year: 2024,
+ month: Some(5),
+ day: None,
+ hour: None,
+ minute: None,
+ second: None
+ }),
+ }
+ );
+ assert_eq!(
+ items[9].reference,
+ Reference::Period {
+ name: "Three months ended June 30, 2019".into(),
+ from: None,
+ to: Some(ymd(2019, 6, 30)),
+ }
+ );
+ assert_eq!(
+ items[10].reference,
+ Reference::Anchored {
+ anchor: Anchor::Period {
+ name: "fiscal 2019".into()
+ },
+ offset: Some(Offset {
+ count: 0,
+ unit: Unit::Quarter,
+ direction: Direction::After
+ }),
+ }
+ );
+ }
+
+ /// 坏形状、13 月、不在清单里的编号、重复的编号、坏单位各计一次;好的照收
+ #[test]
+ fn malformed_interpretation_items_are_counted_not_fatal() {
+ let raw = r#"{"m": [
+ [1, "point", {"kind": "absolute", "from": {"y": 2011, "m": 3, "d": 4}}, "day"],
+ [2, "sometime", {"kind": "none"}, "day"],
+ [3, "point", {"kind": "absolute", "from": {"y": 2011, "m": 13}}, "month"],
+ [99, "point", {"kind": "none"}, "day"],
+ [1, "since", {"kind": "none"}, "day"],
+ [4, "point", {"kind": "anchored", "anchor": {"kind": "document"}, "offset": {"count": 2, "unit": "fortnight", "direction": "after"}}, "day"],
+ [5, "point", {"kind": "elsewhere"}, "day"],
+ [6, "point", {"kind": "none"}, "fortnight"],
+ [7, "AS OF", {"kind": "anchored", "anchor": "document"}, "Day"]
+ ]}"#;
+ let (items, skipped) = parse_interpretation_response(raw, &[1, 2, 3, 4, 5, 6, 7]).unwrap();
+ assert_eq!(items.len(), 2);
+ assert_eq!(items[0].id, 1);
+ assert_eq!(items[1].id, 7, "case and spacing of the words are forgiven");
+ assert_eq!(items[1].shape, Shape::AsOf);
+ assert_eq!(items[1].granularity, Granularity::Day);
+ assert_eq!(
+ items[1].reference,
+ Reference::Anchored {
+ anchor: Anchor::Document,
+ offset: None
+ }
+ );
+ assert_eq!(skipped, 7);
+ }
+
+ /// 截在第三条中间:前两条留下,没有报错
+ #[test]
+ fn a_cut_off_interpretation_reply_keeps_the_complete_items() {
+ let marker = r#"[3, "interval", {"kind": "period", "name": "fis"#;
+ let cut = INTERPRETED.find(marker).unwrap() + marker.len();
+ let ids: Vec<i64> = (1..=11).collect();
+ let (items, skipped) = parse_interpretation_response(&INTERPRETED[..cut], &ids).unwrap();
+ assert_eq!(skipped, 0);
+ assert_eq!(items.len(), 2);
+ assert_eq!(items[1].id, 2);
+ }
+
+ #[test]
+ fn a_cut_off_dating_reply_keeps_the_complete_periods() {
+ let marker = r#"["2024年", {"y": 20"#;
+ let cut = DATING.find(marker).unwrap() + marker.len();
+ let d = parse_dating_response(&DATING[..cut]).unwrap();
+ assert_eq!(d.date, Some(ymd(2019, 11, 12)));
+ assert_eq!(d.periods.len(), 1);
+ assert_eq!(d.periods[0].name, "fiscal 2019");
+ }
+
+ #[test]
+ fn a_reply_without_json_is_an_error() {
+ assert!(parse_dating_response("I could not find a date.").is_err());
+ assert!(parse_interpretation_response("no", &[1]).is_err());
+ }
+
+ /// 语料里的字样:提示词里一个都不许有(例子是镇议会、桥、面包房)
+ const CORPUS_MARKS: &[&str] = &[
+ "NVIDIA",
+ "Food and Drug",
+ "FDA",
+ "January 25, 2027",
+ "January 26, 2026",
+ "July 26, 2026",
+ "fiscal 2027",
+ "上年末",
+ "统计公报",
+ "10-Q",
+ "10-K",
+ ];
+
+ #[test]
+ fn the_dating_prompt_carries_the_document_and_the_rules() {
+ let msgs = build_dating_messages(
+ "minutes.pdf",
+ "Westbrook Town Council — Minutes of the meeting held on March 4, 2011.",
+ );
+ assert_eq!(msgs.len(), 2);
+ assert_eq!(msgs[0].role, "system");
+ assert!(msgs[0].content.contains("Never compute"));
+ assert!(msgs[0].content.contains("never today's date"));
+ assert!(msgs[0].content.contains("fiscal_year_end"));
+ for mark in CORPUS_MARKS {
+ assert!(!msgs[0].content.contains(mark), "prompt carries {mark}");
+ }
+ assert_eq!(msgs[1].role, "user");
+ assert!(msgs[1].content.contains("Document: minutes.pdf"));
+ assert!(msgs[1].content.contains("held on March 4, 2011"));
+ }
+
+ #[test]
+ fn the_interpretation_prompt_carries_the_context_and_every_mention() {
+ let date = ymd(2019, 11, 12);
+ let periods = vec![
+ NamedPeriod {
+ name: "fiscal 2019".into(),
+ from: ymd(2018, 10, 1),
+ to: ymd(2019, 9, 30),
+ },
+ NamedPeriod {
+ name: "the fourth quarter".into(),
+ from: ymd(2019, 7, 1),
+ to: ymd(2019, 9, 30),
+ },
+ ];
+ let context = TimeContext {
+ date: Some(&date),
+ date_words: Some("November 12, 2019"),
+ periods: &periods,
+ fiscal_year_end: Some((9, 30)),
+ };
+ let mentions = [
+ MentionInput {
+ id: 7,
+ text: "two years later",
+ sentence:
+ "The bridge opened in 2019; two years later the eastern span was widened.",
+ },
+ MentionInput {
+ id: 8,
+ text: "this quarter",
+ sentence: "Sales this quarter rose at the bakery.",
+ },
+ ];
+ let msgs = build_interpretation_messages(&context, &mentions);
+ assert_eq!(msgs.len(), 2);
+ let system = &msgs[0].content;
+ assert!(system.contains("Never compute"));
+ for shape in [
+ "\"point\"",
+ "\"since\"",
+ "\"until\"",
+ "\"interval\"",
+ "\"as_of\"",
+ "\"duration\"",
+ "\"ended_unknown\"",
+ ] {
+ assert!(system.contains(shape), "shape {shape} is not explained");
+ }
+ for mark in CORPUS_MARKS {
+ assert!(!system.contains(mark), "prompt carries {mark}");
+ }
+ let user = &msgs[1].content;
+ assert!(user.contains("2019-11-12"));
+ assert!(user.contains("\"November 12, 2019\""));
+ assert!(user.contains("\"fiscal 2019\" from 2018-10-01 to 2019-09-30"));
+ assert!(user.contains("\"the fourth quarter\""));
+ assert!(user.contains("fiscal year end: month 9, day 30"));
+ assert!(user.contains("id 7, words \"two years later\""));
+ assert!(user.contains(mentions[0].sentence));
+ assert!(user.contains("id 8, words \"this quarter\""));
+ assert!(user.contains(mentions[1].sentence));
+ }
+
+ #[test]
+ fn an_undated_document_says_so_in_the_context() {
+ let context = TimeContext {
+ date: None,
+ date_words: None,
+ periods: &[],
+ fiscal_year_end: None,
+ };
+ let msgs = build_interpretation_messages(&context, &[]);
+ let user = &msgs[1].content;
+ assert!(user.contains("date: not stated"));
+ assert!(user.contains("periods: none named"));
+ assert!(user.contains("fiscal year end: not stated"));
+ }
+
+ /// 存成 JSONB 再读回来,一字不差
+ #[test]
+ fn interpretations_and_datings_round_trip_through_serde() {
+ let ids: Vec<i64> = (1..=11).collect();
+ let (items, _) = parse_interpretation_response(INTERPRETED, &ids).unwrap();
+ for item in &items {
+ let json = serde_json::to_string(item).unwrap();
+ let back: Interpretation = serde_json::from_str(&json).unwrap();
+ assert_eq!(&back, item);
+ }
+ let json = serde_json::to_value(&items[1]).unwrap();
+ assert_eq!(json["shape"], "since");
+ assert_eq!(json["reference"]["kind"], "anchored");
+ assert_eq!(json["reference"]["anchor"]["kind"], "document");
+ assert_eq!(json["reference"]["offset"]["unit"], "year");
+ assert_eq!(json["granularity"], "year");
+
+ let dating = parse_dating_response(DATING).unwrap();
+ let json = serde_json::to_string(&dating).unwrap();
+ let back: DocumentDating = serde_json::from_str(&json).unwrap();
+ assert_eq!(back.date, dating.date);
+ assert_eq!(back.date_words, dating.date_words);
+ assert_eq!(back.periods, dating.periods);
+ assert_eq!(back.fiscal_year_end, dating.fiscal_year_end);
+ assert_eq!(back.skipped, dating.skipped);
+ let stored = serde_json::to_value(&dating).unwrap();
+ assert_eq!(stored["date"]["year"], 2019, "stored parts use full names");
+ assert_eq!(stored["fiscal_year_end"], serde_json::json!([9, 30]));
+
+ let old: DocumentDating = serde_json::from_str("{}").unwrap();
+ assert_eq!(old.date, None, "a row stored without fields reads as empty");
+ }
+}
diff --git a/crates/utopia-server/src/api/documents_routes_tests.rs b/crates/utopia-server/src/api/documents_routes_tests.rs
index d58a2c46..682b524b 100644
--- a/crates/utopia-server/src/api/documents_routes_tests.rs
+++ b/crates/utopia-server/src/api/documents_routes_tests.rs
@@ -306,9 +306,10 @@ async fn fallback_restore_and_other_ingest_dates_do_not_change() -> anyhow::Resu
assert_eq!(status, StatusCode::OK, "{response}");
let docs = f.created_docs(&response).await?;
assert_eq!(docs.len(), 3);
+ // 上传时刻不是文档的日期(0045 决定 3,#714):没有日期就是没有日期
for doc in &docs {
- assert_eq!(doc.doc_time_source, "upload_time");
- assert_eq!(doc.doc_time, Some(doc.created_at));
+ assert_eq!(doc.doc_time_source, "none");
+ assert_eq!(doc.doc_time, None);
}
// A pre-existing, dated body with upload_time must retain that historical fallback on restoration.
let body = b"2024-02-29\nHistorical announcement";
diff --git a/crates/utopia-server/src/api/review_routes.rs b/crates/utopia-server/src/api/review_routes.rs
index a3b11427..3273833b 100644
--- a/crates/utopia-server/src/api/review_routes.rs
+++ b/crates/utopia-server/src/api/review_routes.rs
@@ -1116,6 +1116,14 @@ pub async fn decide_pending(
done.snapshot,
)
.await;
+ // 点头落下的开放陈述带着时间词:按文档重排一次时间解析(0045),
+ // 幂等,同一篇排着就不重复
+ let _ = utopia_store::jobs::enqueue_unless_queued(
+ &state.pool,
+ "resolve_time",
+ json!({ "document_id": done.document_id }),
+ )
+ .await;
state.emit_pending(kb_id);
state.emit_review(kb_id);
state.emit_graph(kb_id);
diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs
index c31f1517..4ba92617 100644
--- a/crates/utopia-server/src/extraction_open.rs
+++ b/crates/utopia-server/src/extraction_open.rs
@@ -34,7 +34,7 @@ use utopia_store::pending::{Outcome, Proposal};
use uuid::Uuid;
/// 文档日期只在它来自内容或来源系统时才算证据日期(与 `temporal::DATED_AT` 同一口径)。
-fn dated_at(doc: &Document) -> Option<chrono::DateTime<chrono::Utc>> {
+pub(crate) fn dated_at(doc: &Document) -> Option<chrono::DateTime<chrono::Utc>> {
if matches!(doc.doc_time_source.as_str(), "content" | "source") {
doc.doc_time
} else {
@@ -438,15 +438,14 @@ pub(crate) async fn run_open(
}
}
// 时间词:起与止各是一条提及,必须原样在这条陈述的引文里
- let mut time_words: Vec<(&str, i32)> = Vec::new();
- for words in [s.when.as_deref(), s.ended.as_deref()]
+ let mut time_words: Vec<(&str, i32, &str)> = Vec::new();
+ for (role, words) in [("when", s.when.as_deref()), ("ended", s.ended.as_deref())]
.into_iter()
- .flatten()
- .map(str::trim)
- .filter(|w| !w.is_empty())
+ .filter_map(|(role, words)| Some((role, words?.trim())))
+ .filter(|(_, w)| !w.is_empty())
{
match locate_time(&chunk.text, quote, words) {
- Some(start) => time_words.push((words, start)),
+ Some(start) => time_words.push((words, start, role)),
None => {
drop_signal(
state,
@@ -479,7 +478,9 @@ pub(crate) async fn run_open(
let time_json = serde_json::Value::Array(
time_words
.iter()
- .map(|(text, start)| serde_json::json!({ "text": text, "char_start": start }))
+ .map(|(text, start, role)| {
+ serde_json::json!({ "text": text, "char_start": start, "role": role })
+ })
.collect(),
);
let outcome = utopia_store::pending::propose(
@@ -542,9 +543,11 @@ pub(crate) async fn run_open(
)
.await?;
}
- for (words, start) in &time_words {
- utopia_store::time_mentions::record(pool, kb_id, fact_id, chunk.id, words, *start)
- .await?;
+ for (words, start, role) in &time_words {
+ utopia_store::time_mentions::record(
+ pool, kb_id, fact_id, chunk.id, words, *start, role,
+ )
+ .await?;
}
statement_count += 1;
}
@@ -648,6 +651,13 @@ pub(crate) async fn run_open(
utopia_store::documents::set_graph_status(pool, document_id, "done").await?;
state.emit_document(kb_id, document_id);
state.emit_graph(kb_id);
+ // 时间词的解析是它后面的任务(0045):文档时间上下文 + 每个提及的解释 + 算区间
+ utopia_store::jobs::enqueue_unless_queued(
+ pool,
+ "resolve_time",
+ serde_json::json!({ "document_id": document_id }),
+ )
+ .await?;
// 队列里多了东西才叫醒人:Review 的计数与对话里那张确认卡都靠这一声
if pending_count > 0 {
tracing::info!(%document_id, pending_count, "记忆抽出的陈述进了待确认队列");
diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs
index 194e134f..e23da3e8 100644
--- a/crates/utopia-server/src/main.rs
+++ b/crates/utopia-server/src/main.rs
@@ -32,6 +32,7 @@ mod readers;
mod retrieval;
mod rss_full_content;
mod state;
+mod time_resolution;
mod time_text;
mod type_resolution;
mod webdav;
@@ -493,6 +494,11 @@ async fn dispatch(st: &state::AppState, job: &utopia_store::jobs::Job) -> anyhow
);
Ok(())
}
+ // 时间提及按文档解析(0045):抽完一篇排一个,重排一次就是重新解析
+ "resolve_time" => {
+ let id = payload_document_id(&job.payload)?;
+ time_resolution::resolve_document(st, id).await
+ }
"resolve_types" => {
let kb_id: Uuid = job
.payload
diff --git a/crates/utopia-server/src/time_resolution.rs b/crates/utopia-server/src/time_resolution.rs
new file mode 100644
index 00000000..d516c99e
--- /dev/null
+++ b/crates/utopia-server/src/time_resolution.rs
@@ -0,0 +1,900 @@
+//! 时间提及按文档解析(0045 决定 2–5):模型读,代码算。
+//!
+//! 抽取只记时间**词**(`time_mentions`:原样的字、块、偏移、起/止)。这里是它后面的任务:
+//!
+//! 1. **文档时间上下文**(决定 3)。一篇文档只做一次:把开头交给模型,它转写文档自己的日期
+//! (落款、备案日、公报的报告年)和文档命名的期间(「fiscal 2027」及其起止);日期词要在
+//! 开头里核对得到才算数。结果存在 `documents.time_context`;文档自己的日期写进 `doc_time`
+//! (来源 `content`)。上传时间永远不进来(#714)。
+//! 2. **解释**(决定 2)。把文档的每个时间词连同它所在的那句话,和上下文一起交给模型;模型
+//! 只回形状、引用(写明的数字部分 / 锚点加偏移 / 命名的期间 / 没有)和粒度,**从不算日期**。
+//! 解释原样存回提及上,错了看得见、改得了。
+//! 3. **算**。日历算术、期间到区间、按粒度截断、形状蕴含的区间,都在这里用结构化字段算,
+//! 没有一个时间词写在代码里(决定 8)。写明的日期是 A 级,从文档自己给的锚点算出来的是
+//! B 级,锚不到的是 C 级——C 级什么也不写,等锚点(决定 4、6)。
+//! 4. 结果落到开放陈述的 `valid_from` / `valid_to`(决定 5 的第一根轴),时间轴上才有它。
+
+use crate::extraction::{chat_retrying_rate_limits_at, span_in_quote};
+use crate::llm_util;
+use crate::state::AppState;
+use chrono::{DateTime, Duration, Months, NaiveDate, NaiveTime, TimeZone, Utc};
+use std::collections::HashMap;
+use utopia_extract::time::{
+ build_dating_messages, build_interpretation_messages, parse_dating_response,
+ parse_interpretation_response, Anchor, DateParts, Direction, DocumentDating, Granularity,
+ Interpretation, MentionInput, Offset, Reference, Shape, TimeContext, Unit,
+};
+use utopia_store::graph::{truncate_to, ENDED_UNKNOWN, WORLD_PRECISIONS};
+use uuid::Uuid;
+
+/// 一批解释问多少个提及。
+const BATCH: usize = 40;
+/// 文档开头给模型看多少字:落款、报告年、财年定义都在前面
+const OPENING_CHARS: usize = 3000;
+
+/// 一次解析的结果:世界轴上的位置与等级。
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct Resolved {
+ from: Option<DateTime<Utc>>,
+ from_p: Option<&'static str>,
+ to: Option<DateTime<Utc>>,
+ to_p: Option<&'static str>,
+ /// A:写明的;B:从文档自己的锚点算出来的;C:锚不到
+ grade: &'static str,
+}
+
+const UNRESOLVED: Resolved = Resolved {
+ from: None,
+ from_p: None,
+ to: None,
+ to_p: None,
+ grade: "C",
+};
+
+/// 数字部分 → 时刻与精度。精度是字写到的那一档:只有年就是年。
+fn parts_to_time(p: &DateParts) -> Option<(DateTime<Utc>, &'static str)> {
+ let date = NaiveDate::from_ymd_opt(p.year, p.month.unwrap_or(1), p.day.unwrap_or(1))?;
+ let time = NaiveTime::from_hms_opt(
+ p.hour.unwrap_or(0),
+ p.minute.unwrap_or(0),
+ p.second.unwrap_or(0),
+ )?;
+ let precision = if p.month.is_none() {
+ "year"
+ } else if p.day.is_none() {
+ "month"
+ } else if p.hour.is_none() {
+ "day"
+ } else if p.minute.is_none() {
+ "hour"
+ } else if p.second.is_none() {
+ "minute"
+ } else {
+ "second"
+ };
+ Some((Utc.from_utc_datetime(&date.and_time(time)), precision))
+}
+
+fn granularity_str(g: Granularity) -> &'static str {
+ match g {
+ Granularity::Year => "year",
+ Granularity::Month => "month",
+ Granularity::Day => "day",
+ Granularity::Hour => "hour",
+ Granularity::Minute => "minute",
+ Granularity::Second => "second",
+ }
+}
+
+/// 两档精度取粗的那一档:字只写到年,模型说是「天」也只能是年——精度不能比字更细
+fn coarser(a: &'static str, b: &'static str) -> &'static str {
+ let rank = |p: &str| WORLD_PRECISIONS.iter().position(|x| *x == p).unwrap_or(0);
+ if rank(a) <= rank(b) {
+ a
+ } else {
+ b
+ }
+}
+
+/// 锚点加偏移(决定 2 的算术)。月、季、年按月份加减(月末自动收口);周和天以下按时长。
+fn shift(t: DateTime<Utc>, offset: &Offset) -> Option<DateTime<Utc>> {
+ let count = offset.count.unsigned_abs();
+ let forward = (offset.direction == Direction::After) == (offset.count >= 0);
+ let months = |n: u64| -> Option<DateTime<Utc>> {
+ let m = Months::new(u32::try_from(n).ok()?);
+ if forward {
+ t.checked_add_months(m)
+ } else {
+ t.checked_sub_months(m)
+ }
+ };
+ let span = |d: Duration| -> Option<DateTime<Utc>> {
+ if forward {
+ t.checked_add_signed(d)
+ } else {
+ t.checked_sub_signed(d)
+ }
+ };
+ match offset.unit {
+ Unit::Year => months(count.checked_mul(12)?),
+ Unit::Quarter => months(count.checked_mul(3)?),
+ Unit::Month => months(count),
+ Unit::Week => span(Duration::weeks(i64::try_from(count).ok()?)),
+ Unit::Day => span(Duration::days(i64::try_from(count).ok()?)),
+ Unit::Hour => span(Duration::hours(i64::try_from(count).ok()?)),
+ Unit::Minute => span(Duration::minutes(i64::try_from(count).ok()?)),
+ Unit::Second => span(Duration::seconds(i64::try_from(count).ok()?)),
+ }
+}
+
+/// 一个时刻和它的精度(世界轴的一端)。
+type Bound = (DateTime<Utc>, &'static str);
+
+/// 一个季度:包含 `t` 的那三个月(有财年结束日就按财年的季,否则按日历季),
+/// 起是季首月,止是季末月(终点精度到月)。「本季度」「上季度」都从这里来
+fn quarter_around(t: DateTime<Utc>, fiscal_year_end: Option<(u32, u32)>) -> Option<(Bound, Bound)> {
+ use chrono::Datelike;
+ // 财年从结束日的下一个月算起:结束于 1 月 25 日的财年,季首月是 2、5、8、11 月
+ let first_month = fiscal_year_end.map(|(m, _)| m % 12 + 1).unwrap_or(1);
+ let month0 = (t.month() as i32 - first_month as i32).rem_euclid(12); // 财年内第几个月(0 起)
+ let start_month0 = month0 - month0 % 3;
+ let start = t.checked_sub_months(Months::new(u32::try_from(month0 - start_month0).ok()?))?;
+ let start = truncate_to(start, Some("month"));
+ let end = start.checked_add_months(Months::new(2))?;
+ Some(((start, "month"), (end, "month")))
+}
+
+/// 一个期间的起止(写明的数字部分)。
+fn period_bounds(from: &DateParts, to: &DateParts) -> Option<(Bound, Bound)> {
+ Some((parts_to_time(from)?, parts_to_time(to)?))
+}
+
+/// 形状决定字在世界轴上占哪一段(决定 7:形状另存,这里只算区间)。
+/// 点:一刻,两端同值(0031 事件的写法);自:起点;至:终点;区间:两端;
+/// 截至:观察到的起点;时长:没有位置;结束但不知何时:终点精度 unknown
+fn place(
+ shape: Shape,
+ at: Option<(DateTime<Utc>, &'static str)>,
+ until: Option<(DateTime<Utc>, &'static str)>,
+ grade: &'static str,
+) -> Resolved {
+ let trunc = |(t, p): (DateTime<Utc>, &'static str)| (truncate_to(t, Some(p)), p);
+ let at = at.map(trunc);
+ let until = until.map(trunc);
+ let mut r = Resolved {
+ grade,
+ ..UNRESOLVED
+ };
+ match shape {
+ Shape::Point => {
+ if let Some((t, p)) = at {
+ r.from = Some(t);
+ r.from_p = Some(p);
+ r.to = Some(t);
+ r.to_p = Some(p);
+ }
+ }
+ Shape::Since | Shape::AsOf => {
+ if let Some((t, p)) = at {
+ r.from = Some(t);
+ r.from_p = Some(p);
+ }
+ }
+ Shape::Until => {
+ if let Some((t, p)) = until.or(at) {
+ r.to = Some(t);
+ r.to_p = Some(p);
+ }
+ }
+ Shape::Interval => {
+ if let Some((t, p)) = at {
+ r.from = Some(t);
+ r.from_p = Some(p);
+ }
+ if let Some((t, p)) = until {
+ r.to = Some(t);
+ r.to_p = Some(p);
+ }
+ }
+ Shape::Duration => return UNRESOLVED,
+ Shape::EndedUnknown => {
+ r.to_p = Some(ENDED_UNKNOWN);
+ }
+ }
+ if r.from.is_none() && r.to.is_none() && r.to_p.is_none() {
+ return UNRESOLVED;
+ }
+ r
+}
+
+/// 一个锚点在世界轴上的位置:文档日期、另一个提及、命名的期间。
+fn anchor_time(
+ anchor: &Anchor,
+ ctx: &DocumentDating,
+ earlier: &HashMap<i64, Resolved>,
+) -> Option<(Bound, Option<Bound>)> {
+ match anchor {
+ Anchor::Document => ctx.date.as_ref().and_then(parts_to_time).map(|t| (t, None)),
+ Anchor::Mention { id } => {
+ let r = earlier.get(id)?;
+ let from = r.from.zip(r.from_p)?;
+ Some((from, r.to.zip(r.to_p)))
+ }
+ Anchor::Period { name } => {
+ let p = ctx
+ .periods
+ .iter()
+ .find(|p| p.name.eq_ignore_ascii_case(name))?;
+ let (from, to) = period_bounds(&p.from, &p.to)?;
+ Some((from, Some(to)))
+ }
+ }
+}
+
+/// 一条解释 → 世界轴位置。`earlier` 是同一批里已经算出来的提及(锚点指着它们)。
+fn resolve_one(
+ interp: &Interpretation,
+ ctx: &DocumentDating,
+ earlier: &HashMap<i64, Resolved>,
+) -> Resolved {
+ let g = granularity_str(interp.granularity);
+ // 「结束了,不知哪天」是字明说的(#393 的那一档):没有日期可算,但结束本身是 A 级
+ if interp.shape == Shape::EndedUnknown {
+ return place(Shape::EndedUnknown, None, None, "A");
+ }
+ match &interp.reference {
+ Reference::Absolute { from, to } => {
+ let Some((t, p)) = parts_to_time(from) else {
+ return UNRESOLVED;
+ };
+ let until = to.as_ref().and_then(parts_to_time);
+ place(interp.shape, Some((t, coarser(p, g))), until, "A")
+ }
+ Reference::Anchored { anchor, offset } => {
+ let Some(((t, p), until)) = anchor_time(anchor, ctx, earlier) else {
+ return UNRESOLVED;
+ };
+ // 对着一个期间说「截至」「至」(「年末」对着报告年):说的是期间的末,不是首。
+ // 这是形状的语义,不是认词
+ if offset.is_none()
+ && matches!(anchor, Anchor::Period { .. })
+ && matches!(interp.shape, Shape::AsOf | Shape::Until)
+ {
+ if let Some(end) = until {
+ return place(interp.shape, Some(end), None, "B");
+ }
+ }
+ let (t, until) = match offset {
+ Some(o) => {
+ let Some(moved) = shift(t, o) else {
+ return UNRESOLVED;
+ };
+ // 以「年」「月」为单位的偏移(本年、去年、上个月)说的是那一整段:
+ // 起止同一个桶,精度就是那个单位
+ if matches!(o.unit, Unit::Year | Unit::Month)
+ && matches!(interp.shape, Shape::Point | Shape::Interval)
+ {
+ let unit_p = if o.unit == Unit::Year {
+ "year"
+ } else {
+ "month"
+ };
+ let start = truncate_to(moved, Some(unit_p));
+ return place(Shape::Point, Some((start, unit_p)), None, "B");
+ }
+ // 以「季」为单位的偏移(本季度、上季度)说的是那一整个季度,不是某一天
+ if o.unit == Unit::Quarter {
+ let Some((from, to)) = quarter_around(moved, ctx.fiscal_year_end) else {
+ return UNRESOLVED;
+ };
+ let shape = match interp.shape {
+ Shape::Point | Shape::Since | Shape::AsOf | Shape::Duration => {
+ Shape::Interval
+ }
+ s => s,
+ };
+ return place(shape, Some(from), Some(to), "B");
+ }
+ (moved, until.and_then(|(u, _)| shift(u, o).map(|m| (m, g))))
+ }
+ None => (t, until),
+ };
+ // 锚点算出来的时刻,精度由字说的粒度定,但不能比锚点自己更细
+ place(interp.shape, Some((t, coarser(p, g))), until, "B")
+ }
+ Reference::Period { name, from, to } => {
+ // 只写了一端的期间(「三个月截至 6 月 30 日」):写明的那一端照写,另一端不猜
+ match (from, to) {
+ (None, Some(t)) => {
+ return place(Shape::Until, None, parts_to_time(t), "A");
+ }
+ (Some(f), None) => {
+ return place(Shape::Since, parts_to_time(f), None, "A");
+ }
+ _ => {}
+ }
+ let (bounds, grade) = match (from, to) {
+ (Some(f), Some(t)) => (period_bounds(f, t), "A"),
+ _ => (
+ ctx.periods
+ .iter()
+ .find(|p| p.name.eq_ignore_ascii_case(name))
+ .and_then(|p| period_bounds(&p.from, &p.to)),
+ "B",
+ ),
+ };
+ let Some((from, to)) = bounds else {
+ return UNRESOLVED;
+ };
+ // 一个期间本身是区间;「截至期末」这类形状照形状算
+ let shape = match interp.shape {
+ Shape::Point | Shape::Since | Shape::Duration => Shape::Interval,
+ s => s,
+ };
+ place(shape, Some(from), Some(to), grade)
+ }
+ Reference::None => UNRESOLVED,
+ }
+}
+
+/// 一条陈述的起与止:`when` 给起(点与区间也给止),`ended` 给止。
+fn combine(when: Option<Resolved>, ended: Option<Resolved>) -> Resolved {
+ let mut r = when.unwrap_or(UNRESOLVED);
+ if let Some(e) = ended {
+ let to = e.to.or(e.from);
+ let to_p = e.to_p.or(e.from_p);
+ if to.is_some() || to_p.is_some() {
+ r.to = to;
+ r.to_p = to_p;
+ }
+ if r.grade == "C" {
+ r.grade = e.grade;
+ }
+ }
+ r
+}
+
+/// 解析一篇文档的全部时间提及(任务 `resolve_time`)。幂等:再跑一次覆盖上一次的结果,
+/// 所以锚点晚到(改了文档日期、后一块给了期间)时重排一次即可(决定 4 的第一步)
+pub async fn resolve_document(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
+ let pool = &state.pool;
+ let doc = utopia_store::documents::get(pool, document_id).await?;
+ if doc.deleted_at.is_some() {
+ return Ok(());
+ }
+ let kb = utopia_store::kbs::get(pool, doc.kb_id).await?;
+ let settings = utopia_store::settings::get(pool, kb.workspace_id)
+ .await?
+ .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot resolve time"))?;
+ let client = llm_util::chat_client(&settings)
+ .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot resolve time"))?;
+
+ // 1. 文档时间上下文:一篇只问一次
+ let context: DocumentDating = match doc.time_context.clone() {
+ Some(json) => serde_json::from_value(json)?,
+ None => {
+ let opening = utopia_store::documents::opening_chunk(pool, document_id)
+ .await?
+ .map(|(_, text)| text)
+ .unwrap_or_default();
+ let opening: String = opening.chars().take(OPENING_CHARS).collect();
+ let messages = build_dating_messages(&doc.filename, &opening);
+ let reply =
+ chat_retrying_rate_limits_at(state, &settings, &client, &messages, Some(0.0))
+ .await?;
+ let mut dating = parse_dating_response(&reply)?;
+ // 日期词要在开头里核对得到(与名字、引文同一条规矩);核不到的日期不算
+ match dating.date_words.as_deref() {
+ Some(words) if span_in_quote(words, &opening) => {}
+ _ => dating.date = None,
+ }
+ utopia_store::documents::set_time_context(
+ pool,
+ document_id,
+ &serde_json::to_value(&dating)?,
+ )
+ .await?;
+ // 文档自己的日期(决定 3):只在它还没有内容或来源给的日期时写
+ if crate::extraction_open::dated_at(&doc).is_none() {
+ if let Some((t, _)) = dating.date.as_ref().and_then(parts_to_time) {
+ utopia_store::documents::set_content_date(pool, document_id, t).await?;
+ }
+ }
+ dating
+ }
+ };
+
+ // 2. 提及:同一篇里同样的字在同样的句子里只解释一次
+ let mentions = utopia_store::time_mentions::for_document(pool, document_id).await?;
+ if mentions.is_empty() {
+ return Ok(());
+ }
+ let mut distinct: Vec<(String, String)> = Vec::new();
+ let mut index: HashMap<(String, String), usize> = HashMap::new();
+ for m in &mentions {
+ let key = (m.text.clone(), m.sentence.clone());
+ if !index.contains_key(&key) {
+ index.insert(key.clone(), distinct.len());
+ distinct.push(key);
+ }
+ }
+ let ctx = TimeContext {
+ date: context.date.as_ref(),
+ date_words: context.date_words.as_deref(),
+ periods: &context.periods,
+ fiscal_year_end: context.fiscal_year_end,
+ };
+ let mut resolved: HashMap<usize, (Interpretation, Resolved)> = HashMap::new();
+ let mut malformed = 0usize;
+ for (batch_no, batch) in distinct.chunks(BATCH).enumerate() {
+ let base = (batch_no * BATCH) as i64;
+ let inputs: Vec<MentionInput<'_>> = batch
+ .iter()
+ .enumerate()
+ .map(|(i, (text, sentence))| MentionInput {
+ id: base + i as i64,
+ text,
+ sentence,
+ })
+ .collect();
+ let ids: Vec<i64> = inputs.iter().map(|m| m.id).collect();
+ let messages = build_interpretation_messages(&ctx, &inputs);
+ let reply =
+ match chat_retrying_rate_limits_at(state, &settings, &client, &messages, Some(0.0))
+ .await
+ {
+ Ok(r) => r,
+ Err(e) => {
+ tracing::warn!(%document_id, error = %e, "时间解释调用失败,这一批留作未解析");
+ continue;
+ }
+ };
+ let (interps, skipped) = match parse_interpretation_response(&reply, &ids) {
+ Ok(x) => x,
+ Err(e) => {
+ tracing::warn!(%document_id, error = %e, "时间解释回复解析失败,这一批留作未解析");
+ continue;
+ }
+ };
+ malformed += skipped;
+ // 两遍:先算不靠别的提及的,再算指着别的提及的
+ let mut earlier: HashMap<i64, Resolved> = HashMap::new();
+ let anchored_to_mention = |i: &Interpretation| {
+ matches!(
+ &i.reference,
+ Reference::Anchored {
+ anchor: Anchor::Mention { .. },
+ ..
+ }
+ )
+ };
+ for pass in 0..2 {
+ for interp in interps
+ .iter()
+ .filter(|i| anchored_to_mention(i) == (pass == 1))
+ {
+ let r = resolve_one(interp, &context, &earlier);
+ earlier.insert(interp.id, r);
+ if let Ok(i) = usize::try_from(interp.id) {
+ resolved.insert(i, (interp.clone(), r));
+ }
+ }
+ }
+ }
+
+ // 3. 写回提及,再按陈述合起止
+ let mut per_fact: HashMap<Uuid, (Option<Resolved>, Option<Resolved>)> = HashMap::new();
+ let (mut a, mut b, mut c) = (0usize, 0usize, 0usize);
+ for m in &mentions {
+ let Some(&i) = index.get(&(m.text.clone(), m.sentence.clone())) else {
+ continue;
+ };
+ let Some((interp, r)) = resolved.get(&i) else {
+ // 这一批没答到:留着字,不写等级(下次重跑再问)
+ continue;
+ };
+ utopia_store::time_mentions::set_interpretation(
+ pool,
+ m.id,
+ serde_json::to_value(interp.shape)?
+ .as_str()
+ .unwrap_or("point"),
+ &serde_json::to_value(&interp.reference)?,
+ granularity_str(interp.granularity),
+ )
+ .await?;
+ utopia_store::time_mentions::set_resolution(
+ pool, m.id, r.grade, r.from, r.from_p, r.to, r.to_p,
+ )
+ .await?;
+ match r.grade {
+ "A" => a += 1,
+ "B" => b += 1,
+ _ => c += 1,
+ }
+ let slot = per_fact.entry(m.fact_id).or_insert((None, None));
+ if m.role == "ended" {
+ slot.1 = Some(*r);
+ } else {
+ slot.0 = Some(*r);
+ }
+ }
+ let mut dated = 0usize;
+ for (fact_id, (when, ended)) in per_fact {
+ let r = combine(when, ended);
+ if r.from.is_none() && r.to.is_none() && r.to_p.is_none() {
+ continue;
+ }
+ utopia_store::graph::set_open_validity(pool, fact_id, r.from, r.from_p, r.to, r.to_p)
+ .await?;
+ dated += 1;
+ }
+ tracing::info!(
+ %document_id,
+ mentions = mentions.len(),
+ grade_a = a,
+ grade_b = b,
+ grade_c = c,
+ malformed,
+ statements_dated = dated,
+ "时间提及解析完成"
+ );
+ if dated > 0 {
+ state.emit_graph(doc.kb_id);
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use utopia_extract::time::NamedPeriod;
+
+ fn parts(y: i32, m: Option<u32>, d: Option<u32>) -> DateParts {
+ DateParts {
+ year: y,
+ month: m,
+ day: d,
+ hour: None,
+ minute: None,
+ second: None,
+ }
+ }
+ fn at(s: &str) -> DateTime<Utc> {
+ DateTime::parse_from_rfc3339(s).unwrap().with_timezone(&Utc)
+ }
+ fn ctx_dated(y: i32, m: u32, d: u32) -> DocumentDating {
+ DocumentDating {
+ date: Some(parts(y, Some(m), Some(d))),
+ date_words: Some("x".into()),
+ periods: vec![NamedPeriod {
+ name: "fiscal 2027".into(),
+ from: parts(2026, Some(1), Some(26)),
+ to: parts(2027, Some(1), Some(25)),
+ }],
+ fiscal_year_end: Some((1, 25)),
+ skipped: 0,
+ }
+ }
+
+ #[test]
+ fn parts_give_the_precision_the_words_state() {
+ assert_eq!(parts_to_time(&parts(2024, None, None)).unwrap().1, "year");
+ assert_eq!(
+ parts_to_time(&parts(2024, Some(3), None)).unwrap().1,
+ "month"
+ );
+ let (t, p) = parts_to_time(&parts(2011, Some(3), Some(4))).unwrap();
+ assert_eq!((t, p), (at("2011-03-04T00:00:00Z"), "day"));
+ assert!(parts_to_time(&parts(2024, Some(13), None)).is_none());
+ }
+
+ #[test]
+ fn an_absolute_date_is_grade_a_and_a_point_has_two_equal_ends() {
+ let i = Interpretation {
+ id: 0,
+ shape: Shape::Point,
+ reference: Reference::Absolute {
+ from: parts(2011, Some(3), Some(4)),
+ to: None,
+ },
+ granularity: Granularity::Day,
+ };
+ let r = resolve_one(&i, &ctx_dated(2026, 9, 2), &HashMap::new());
+ assert_eq!(r.grade, "A");
+ assert_eq!(r.from, Some(at("2011-03-04T00:00:00Z")));
+ assert_eq!(r.to, r.from);
+ assert_eq!((r.from_p, r.to_p), (Some("day"), Some("day")));
+ }
+
+ #[test]
+ fn a_bare_year_stays_a_year_even_if_the_model_says_day() {
+ let i = Interpretation {
+ id: 0,
+ shape: Shape::Since,
+ reference: Reference::Absolute {
+ from: parts(1987, None, None),
+ to: None,
+ },
+ granularity: Granularity::Day,
+ };
+ let r = resolve_one(&i, &ctx_dated(2026, 9, 2), &HashMap::new());
+ assert_eq!(
+ (r.from, r.from_p, r.to),
+ (Some(at("1987-01-01T00:00:00Z")), Some("year"), None)
+ );
+ }
+
+ #[test]
+ fn today_and_last_year_anchor_to_the_document_as_grade_b() {
+ let today = Interpretation {
+ id: 0,
+ shape: Shape::Point,
+ reference: Reference::Anchored {
+ anchor: Anchor::Document,
+ offset: None,
+ },
+ granularity: Granularity::Day,
+ };
+ let r = resolve_one(&today, &ctx_dated(2026, 9, 2), &HashMap::new());
+ assert_eq!((r.grade, r.from), ("B", Some(at("2026-09-02T00:00:00Z"))));
+ let last_year = Interpretation {
+ id: 1,
+ shape: Shape::Interval,
+ reference: Reference::Anchored {
+ anchor: Anchor::Document,
+ offset: Some(Offset {
+ count: 1,
+ unit: Unit::Year,
+ direction: Direction::Before,
+ }),
+ },
+ granularity: Granularity::Year,
+ };
+ let r = resolve_one(&last_year, &ctx_dated(2026, 9, 2), &HashMap::new());
+ assert_eq!(
+ (r.grade, r.from, r.from_p),
+ ("B", Some(at("2025-01-01T00:00:00Z")), Some("year"))
+ );
+ // 没有文档日期就锚不到
+ let undated = DocumentDating::default();
+ assert_eq!(resolve_one(&today, &undated, &HashMap::new()), UNRESOLVED);
+ }
+
+ #[test]
+ fn a_named_period_is_an_interval_from_the_context() {
+ let i = Interpretation {
+ id: 0,
+ shape: Shape::Point,
+ reference: Reference::Period {
+ name: "Fiscal 2027".into(),
+ from: None,
+ to: None,
+ },
+ granularity: Granularity::Day,
+ };
+ let r = resolve_one(&i, &ctx_dated(2026, 9, 2), &HashMap::new());
+ assert_eq!(r.grade, "B");
+ assert_eq!(r.from, Some(at("2026-01-26T00:00:00Z")));
+ assert_eq!(r.to, Some(at("2027-01-25T00:00:00Z")));
+ // 文中自己写了起止的期间是 A 级
+ let stated = Interpretation {
+ reference: Reference::Period {
+ name: "three months ended July 26, 2026".into(),
+ from: Some(parts(2026, Some(4), Some(27))),
+ to: Some(parts(2026, Some(7), Some(26))),
+ },
+ ..i
+ };
+ assert_eq!(
+ resolve_one(&stated, &DocumentDating::default(), &HashMap::new()).grade,
+ "A"
+ );
+ }
+
+ #[test]
+ fn two_years_later_follows_the_mention_it_names() {
+ let mut earlier = HashMap::new();
+ earlier.insert(
+ 7,
+ Resolved {
+ from: Some(at("2019-01-01T00:00:00Z")),
+ from_p: Some("year"),
+ to: Some(at("2019-01-01T00:00:00Z")),
+ to_p: Some("year"),
+ grade: "A",
+ },
+ );
+ let i = Interpretation {
+ id: 8,
+ shape: Shape::Point,
+ reference: Reference::Anchored {
+ anchor: Anchor::Mention { id: 7 },
+ offset: Some(Offset {
+ count: 2,
+ unit: Unit::Year,
+ direction: Direction::After,
+ }),
+ },
+ granularity: Granularity::Year,
+ };
+ let r = resolve_one(&i, &DocumentDating::default(), &earlier);
+ assert_eq!(
+ (r.grade, r.from, r.from_p),
+ ("B", Some(at("2021-01-01T00:00:00Z")), Some("year"))
+ );
+ }
+
+ #[test]
+ fn an_ending_without_a_date_and_a_when_plus_ended_combine() {
+ let ended = Interpretation {
+ id: 0,
+ shape: Shape::EndedUnknown,
+ reference: Reference::None,
+ granularity: Granularity::Day,
+ };
+ let e = resolve_one(&ended, &DocumentDating::default(), &HashMap::new());
+ assert_eq!((e.to, e.to_p, e.grade), (None, Some(ENDED_UNKNOWN), "A"));
+ let since = Resolved {
+ from: Some(at("2020-01-01T00:00:00Z")),
+ from_p: Some("year"),
+ to: None,
+ to_p: None,
+ grade: "A",
+ };
+ let until = Resolved {
+ from: Some(at("2024-06-01T00:00:00Z")),
+ from_p: Some("month"),
+ to: None,
+ to_p: None,
+ grade: "A",
+ };
+ let r = combine(Some(since), Some(until));
+ assert_eq!(
+ (r.from, r.to, r.to_p),
+ (since.from, until.from, Some("month"))
+ );
+ assert_eq!(combine(None, Some(e)).to_p, Some(ENDED_UNKNOWN));
+ }
+
+ #[test]
+ fn this_quarter_is_the_whole_quarter_and_the_fiscal_calendar_moves_it() {
+ let this_quarter = Interpretation {
+ id: 0,
+ shape: Shape::Point,
+ reference: Reference::Anchored {
+ anchor: Anchor::Document,
+ offset: Some(Offset {
+ count: 0,
+ unit: Unit::Quarter,
+ direction: Direction::After,
+ }),
+ },
+ granularity: Granularity::Month,
+ };
+ // 日历季:9 月 2 日在第三季(7–9 月)
+ let calendar = DocumentDating {
+ fiscal_year_end: None,
+ ..ctx_dated(2026, 9, 2)
+ };
+ let r = resolve_one(&this_quarter, &calendar, &HashMap::new());
+ assert_eq!(
+ (r.from, r.to),
+ (
+ Some(at("2026-07-01T00:00:00Z")),
+ Some(at("2026-09-01T00:00:00Z"))
+ )
+ );
+ assert_eq!(
+ (r.from_p, r.to_p, r.grade),
+ (Some("month"), Some("month"), "B")
+ );
+ // 财年结束于 1 月 25 日:季首月是 2、5、8、11 月,9 月 2 日在 8–10 月那一季
+ let r = resolve_one(&this_quarter, &ctx_dated(2026, 9, 2), &HashMap::new());
+ assert_eq!(
+ (r.from, r.to),
+ (
+ Some(at("2026-08-01T00:00:00Z")),
+ Some(at("2026-10-01T00:00:00Z"))
+ )
+ );
+ // 上一季度
+ let last_quarter = Interpretation {
+ reference: Reference::Anchored {
+ anchor: Anchor::Document,
+ offset: Some(Offset {
+ count: 1,
+ unit: Unit::Quarter,
+ direction: Direction::Before,
+ }),
+ },
+ ..this_quarter
+ };
+ let r = resolve_one(&last_quarter, &calendar, &HashMap::new());
+ assert_eq!(r.from, Some(at("2026-04-01T00:00:00Z")));
+ }
+
+ #[test]
+ fn as_of_a_period_means_its_end_and_a_whole_year_is_one_bucket() {
+ let year_end = Interpretation {
+ id: 0,
+ shape: Shape::AsOf,
+ reference: Reference::Anchored {
+ anchor: Anchor::Period {
+ name: "fiscal 2027".into(),
+ },
+ offset: None,
+ },
+ granularity: Granularity::Day,
+ };
+ let r = resolve_one(&year_end, &ctx_dated(2026, 9, 2), &HashMap::new());
+ assert_eq!((r.from, r.grade), (Some(at("2027-01-25T00:00:00Z")), "B"));
+ let whole_year = Interpretation {
+ id: 1,
+ shape: Shape::Interval,
+ reference: Reference::Anchored {
+ anchor: Anchor::Document,
+ offset: Some(Offset {
+ count: 0,
+ unit: Unit::Year,
+ direction: Direction::After,
+ }),
+ },
+ granularity: Granularity::Year,
+ };
+ let r = resolve_one(&whole_year, &ctx_dated(2024, 1, 1), &HashMap::new());
+ assert_eq!(
+ (r.from, r.to, r.from_p, r.to_p),
+ (
+ Some(at("2024-01-01T00:00:00Z")),
+ Some(at("2024-01-01T00:00:00Z")),
+ Some("year"),
+ Some("year")
+ )
+ );
+ }
+
+ #[test]
+ fn a_period_with_one_stated_bound_writes_only_that_bound() {
+ let ended = Interpretation {
+ id: 0,
+ shape: Shape::Interval,
+ reference: Reference::Period {
+ name: "three months ended June 30, 2019".into(),
+ from: None,
+ to: Some(parts(2019, Some(6), Some(30))),
+ },
+ granularity: Granularity::Day,
+ };
+ let r = resolve_one(&ended, &DocumentDating::default(), &HashMap::new());
+ assert_eq!(
+ (r.from, r.to, r.grade),
+ (None, Some(at("2019-06-30T00:00:00Z")), "A")
+ );
+ }
+
+ #[test]
+ fn month_arithmetic_clamps_to_the_month_end() {
+ let t = at("2024-03-31T00:00:00Z");
+ let back = shift(
+ t,
+ &Offset {
+ count: 1,
+ unit: Unit::Month,
+ direction: Direction::Before,
+ },
+ )
+ .unwrap();
+ assert_eq!(back, at("2024-02-29T00:00:00Z"));
+ let q = shift(
+ t,
+ &Offset {
+ count: 1,
+ unit: Unit::Quarter,
+ direction: Direction::After,
+ },
+ )
+ .unwrap();
+ assert_eq!(q, at("2024-06-30T00:00:00Z"));
+ }
+}
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index fa2f4328..9044f9fe 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -89,7 +89,7 @@ async fn create_with_time_source(
sqlx::query_as(
"INSERT INTO documents (id, kb_id, filename, mime, size_bytes, sha256, source_id,
doc_time, doc_time_source, external_key)
- VALUES ($1, $2, $3, $4, $5, $6, $7, COALESCE($8, now()), $9, $10) RETURNING *",
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING *",
)
.bind(Uuid::now_v7())
.bind(kb_id)
@@ -99,10 +99,11 @@ async fn create_with_time_source(
.bind(sha256)
.bind(source_id)
.bind(doc_time)
+ // 没有日期就是没有日期:上传的时刻是记录时间,不是文档的日期(0045 决定 3,#714)
.bind(if doc_time.is_some() {
time_source
} else {
- "upload_time"
+ "none"
})
.bind(external_key)
.fetch_one(pool)
@@ -131,7 +132,7 @@ pub async fn create_with_version_and_processing(
let document: Document = sqlx::query_as(
"INSERT INTO documents (id, kb_id, filename, mime, size_bytes, sha256, source_id,
doc_time, doc_time_source, external_key)
- VALUES ($1, $2, $3, $4, $5, $6, $7, COALESCE($8, now()), $9, $10) RETURNING *",
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING *",
)
.bind(Uuid::now_v7())
.bind(kb_id)
@@ -141,11 +142,8 @@ pub async fn create_with_version_and_processing(
.bind(sha256)
.bind(source_id)
.bind(doc_time)
- .bind(if doc_time.is_some() {
- "source"
- } else {
- "upload_time"
- })
+ // 没有日期就是没有日期:同步的时刻是记录时间,不是文档的日期(0045 决定 3,#714)
+ .bind(if doc_time.is_some() { "source" } else { "none" })
.bind(external_key)
.fetch_one(&mut *tx)
.await
@@ -369,7 +367,7 @@ pub async fn upsert_source_document_tx(
let document: Document = sqlx::query_as(
"INSERT INTO documents (id, kb_id, filename, mime, size_bytes, sha256, source_id,
doc_time, doc_time_source, external_key)
- VALUES ($1, $2, $3, $4, $5, $6, $7, COALESCE($8, now()), $9, $10) RETURNING *",
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING *",
)
.bind(Uuid::now_v7())
.bind(kb_id)
@@ -379,11 +377,8 @@ pub async fn upsert_source_document_tx(
.bind(sha256)
.bind(source_id)
.bind(doc_time)
- .bind(if doc_time.is_some() {
- "source"
- } else {
- "upload_time"
- })
+ // 没有日期就是没有日期:同步的时刻是记录时间,不是文档的日期(0045 决定 3,#714)
+ .bind(if doc_time.is_some() { "source" } else { "none" })
.bind(external_key)
.fetch_one(&mut **tx)
.await
@@ -619,6 +614,39 @@ pub async fn adopt_external_key(pool: &PgPool, id: Uuid, external_key: &str) ->
Ok(())
}
+/// 文档的时间语境(0045 决定 3):它自己的日期、它定义的期间与历法、叙述设下的锚点。
+/// 服务端边抽取边填、整份覆盖;`time_context_at` 记最近一次写下它的时刻,后来的重新解算
+/// 拿它当输入
+pub async fn set_time_context(
+ pool: &PgPool,
+ id: Uuid,
+ context: &serde_json::Value,
+) -> AppResult<()> {
+ sqlx::query(
+ "UPDATE documents SET time_context = $2, time_context_at = now(), updated_at = now()
+ WHERE id = $1",
+ )
+ .bind(id)
+ .bind(context)
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
+/// 文档自己的日期,从正文里读出来的(0045 决定 3):`doc_time_source = 'content'`。
+/// 只有正文或来源系统给的日期算文档的日期([`Document::dated_at`]);上传时刻不进这里
+pub async fn set_content_date(pool: &PgPool, id: Uuid, date: DateTime<Utc>) -> AppResult<()> {
+ sqlx::query(
+ "UPDATE documents SET doc_time = $2, doc_time_source = 'content', updated_at = now()
+ WHERE id = $1",
+ )
+ .bind(id)
+ .bind(date)
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
/// 变更:原地替换文档内容(新 sha),状态回 pending 待重跑管道。
#[allow(clippy::too_many_arguments)]
pub async fn replace_content(
diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rs
index ff46538c..6db2cc05 100644
--- a/crates/utopia-store/src/graph.rs
+++ b/crates/utopia-store/src/graph.rs
@@ -942,6 +942,51 @@ pub async fn insert_open_statement(
Ok((id, true))
}
+/// 把解算结果写到一条开放陈述的世界轴上(0045 决定 2:模型读,代码算)。
+///
+/// 只改 `layer = 'open'` 且活着的行——WHERE 里写死,不靠调用方检查;类型化的行照旧走
+/// [`insert_fact_inner`] 那道门(谓词的时间语义、去重、时间线),作废的行是历史,不改。
+/// 值先按精度截断([`truncate_to`]),存的值与精度说同一句话,数据库的 CHECK 才放行。
+/// 结束端的三种状态与 [`Validity`] 同一张表:`(None, None)` 仍在持续、
+/// `(None, Some("unknown"))` 结束了不知哪天、`(Some(t), Some(精度))` 某时结束。
+/// 结束了不知哪天的要有自己的锚点(`facts_ended_unknown_has_anchor`,#393):说出结束的
+/// 就是这条陈述自己的文档,锚点取 `attested_from`;其余两种状态把 `attested_to` 清空
+///
+/// 没有这一行、它不是开放行、或它已作废:一行不改,返回 `not_an_open_statement`
+pub async fn set_open_validity(
+ pool: &PgPool,
+ fact_id: Uuid,
+ from: Option<chrono::DateTime<chrono::Utc>>,
+ from_precision: Option<&str>,
+ to: Option<chrono::DateTime<chrono::Utc>>,
+ to_precision: Option<&str>,
+) -> AppResult<()> {
+ let from = from.map(|t| truncate_to(t, from_precision));
+ let to = to.map(|t| truncate_to(t, to_precision));
+ let done = sqlx::query(
+ "UPDATE facts
+ SET valid_from = $2, valid_from_precision = $3,
+ valid_to = $4, valid_to_precision = $5,
+ attested_to = CASE WHEN $4 IS NULL AND $5 = 'unknown'
+ THEN COALESCE(attested_to, attested_from) END
+ WHERE id = $1 AND layer = 'open' AND invalidated_at IS NULL",
+ )
+ .bind(fact_id)
+ .bind(from)
+ .bind(from_precision)
+ .bind(to)
+ .bind(to_precision)
+ .execute(pool)
+ .await?;
+ if done.rows_affected() == 0 {
+ return Err(AppError::invalid(
+ "not_an_open_statement",
+ "only a live open statement takes its valid time from resolution",
+ ));
+ }
+ Ok(())
+}
+
/// `proposed`:模型在这一块里实际提议的谓词。命中本体时它等于 key,
/// 本体外的谓词不落到关系上时,它是唯一还留着原意的东西——事实行上只剩
/// "有关联",原文说的"runs on"就靠这里活下来。
diff --git a/crates/utopia-store/src/pending.rs b/crates/utopia-store/src/pending.rs
index eae5ebe0..383853fb 100644
--- a/crates/utopia-store/src/pending.rs
+++ b/crates/utopia-store/src/pending.rs
@@ -277,6 +277,8 @@ fn snapshot(v: &PendingFactView) -> serde_json::Value {
pub struct Confirmed {
pub fact_id: Uuid,
+ /// 那句记忆所在的文档:点头之后按它重排时间解析(0045)
+ pub document_id: Uuid,
/// false = 图上已有同一条活事实,这次只补了证据
pub created: bool,
/// 时态对账拿不准、进了 `fact_conflicts` 的条数。开放陈述不对账,恒为 0
@@ -395,6 +397,7 @@ pub async fn confirm(pool: &PgPool, kb_id: Uuid, id: Uuid) -> AppResult<Confirme
.await?;
Ok(Confirmed {
fact_id,
+ document_id: document_of(pool, v.chunk_id).await?,
created,
conflicts,
snapshot: snapshot(&v),
@@ -506,7 +509,13 @@ async fn confirm_open(
"a time word is the document's text and its char offset in the chunk",
));
};
- crate::time_mentions::record(pool, kb_id, fact_id, v.chunk_id, text, char_start).await?;
+ // 来自陈述的哪个槽;0062 之前记下的项没有这一字段,它们都是 when
+ let role = t
+ .get("role")
+ .and_then(serde_json::Value::as_str)
+ .unwrap_or("when");
+ crate::time_mentions::record(pool, kb_id, fact_id, v.chunk_id, text, char_start, role)
+ .await?;
}
sqlx::query("DELETE FROM pending_facts WHERE id = $1")
.bind(v.id)
@@ -514,6 +523,7 @@ async fn confirm_open(
.await?;
Ok(Confirmed {
fact_id,
+ document_id: document_of(pool, v.chunk_id).await?,
created,
conflicts: 0,
snapshot: snapshot(v),
@@ -535,6 +545,16 @@ fn json_items(v: Option<&serde_json::Value>) -> impl Iterator<Item = &serde_json
.iter()
}
+/// 一块所在的文档。
+async fn document_of(pool: &PgPool, chunk_id: Uuid) -> AppResult<Uuid> {
+ Ok(
+ sqlx::query_scalar("SELECT document_id FROM chunks WHERE id = $1")
+ .bind(chunk_id)
+ .fetch_one(pool)
+ .await?,
+ )
+}
+
/// 人拒绝:记进 `rejected_facts`(下一轮重抽先查它),从队列里拿掉。
/// 那句记忆本身留着——人确实说过那句话,只是没抽出可用的事实。
/// 开放陈述连短语一起记:拒的是「这句话」,不是这一对实体之间的一切
diff --git a/crates/utopia-store/src/time_mentions.rs b/crates/utopia-store/src/time_mentions.rs
index 9997d68b..187de205 100644
--- a/crates/utopia-store/src/time_mentions.rs
+++ b/crates/utopia-store/src/time_mentions.rs
@@ -1,19 +1,29 @@
-//! 一条陈述提到的时间词(0044 第一刀,#729;时间记录 0045)。
+//! 一条陈述提到的时间词(0044 第一刀,#729)和它的读法(0045)。
//!
-//! **一条时间提及是文档的字,永远不是算出来的日期。**「去年冬天」「合同签署后 30 日内」
-//! 「Q3」——落库的是这几个字和它们在 `chunks.text` 里的字符偏移,不是某个
-//! `TIMESTAMPTZ`。今天抽取把「2023 年上半年」读成 1 月 1 日,读错了只能删掉文档重抽;
-//! 字留着,读法可以改。把字读成日期(形状、锚点、偏移、粒度)是 0045 后面几刀的事,
-//! 那几列到时候加在这张表上,这里写下的行一行不动。
+//! **一条时间提及是文档的字,永远不是模型算出来的日期。**「去年冬天」「合同签署后 30 日内」
+//! 「Q3」——落库的是这几个字和它们在 `chunks.text` 里的字符偏移,不是某个模型写的
+//! `TIMESTAMPTZ`。字留着,读法可以改;读错了改读法,不用删掉文档重抽。
//!
-//! 一条开放陈述于是不写任何 `valid_*`:它在世界轴上还没有位置。
+//! 读法分两层(0045 决定 2:模型读,代码算):
+//!
+//! - **解释**(`shape` / `reference` / `granularity`)是模型给的:形状(点、区间、截至、
+//! 时长、没给日期的结束)、参照(照写的绝对值,或锚点 + 偏移,或没有)、字说到梯子的
+//! 哪一级。这里不认这些字段的取值——它们是抽取合同的词汇,库只照存。
+//! - **解算**(`grade` / `resolved_*`)是代码从解释算出来的:两端各自的时刻与精度,梯子与
+//! `facts` 同一张(0024),结束端多一个 `unknown`。A 是原文写明的绝对日期,B 是按文档
+//! 自己给的锚点算的,C 是没算出来(四列全空——算不出来的等着,决定 4)。`resolved_at`
+//! 是最近一次算的时刻:锚点后来到了要重算。
+//!
+//! 一条陈述的 `when` 与 `ended` 各是一条提及(`role`),同一处字可以两次出现。陈述在世界
+//! 轴上的位置(`facts.valid_*`)由 [`crate::graph::set_open_validity`] 按解算写,不由这里写。
+use chrono::{DateTime, Utc};
use sqlx::PgPool;
use std::collections::HashMap;
use utopia_core::AppResult;
use uuid::Uuid;
-/// 一条时间提及:哪条陈述、哪一块、照抄的字、在块里的字符偏移
+/// 一条时间提及:哪条陈述、哪一块、照抄的字、在块里的字符偏移,以及它的读法(有了的话)
#[derive(Debug, Clone, serde::Serialize, sqlx::FromRow)]
pub struct TimeMention {
pub id: Uuid,
@@ -23,10 +33,29 @@ pub struct TimeMention {
pub text: String,
/// 在 `chunks.text` 里的字符偏移(不是字节)
pub char_start: i32,
+ /// 来自陈述的哪个槽:`when` / `ended`
+ pub role: String,
+ /// 模型的解释:形状、参照、粒度。没读过为空
+ pub shape: Option<String>,
+ pub reference: Option<serde_json::Value>,
+ pub granularity: Option<String>,
+ /// 代码的解算:A 绝对、B 按锚点算出、C 没算出来。没算过为空
+ pub grade: Option<String>,
+ pub resolved_from: Option<DateTime<Utc>>,
+ pub resolved_from_precision: Option<String>,
+ pub resolved_to: Option<DateTime<Utc>>,
+ /// 结束端多一个 `unknown`:结束了,不知哪天
+ pub resolved_to_precision: Option<String>,
}
-/// 记一条时间提及。`char_start` 是字符偏移,**由服务端在块里搜出来**,不取模型报的数。
-/// 同一条陈述在同一块的同一位置只有一行;再记一次回的是那一行的 id
+const COLUMNS: &str =
+ "id, fact_id, chunk_id, text, char_start, role, shape, reference, granularity,
+ grade, resolved_from, resolved_from_precision, resolved_to,
+ resolved_to_precision";
+
+/// 记一条时间提及。`char_start` 是字符偏移,**由服务端在块里搜出来**,不取模型报的数;
+/// `role` 是它来自陈述的哪个槽(`when` / `ended`)。同一条陈述在同一块的同一位置、同一
+/// 个槽只有一行;再记一次回的是那一行的 id
pub async fn record(
pool: &PgPool,
kb_id: Uuid,
@@ -34,11 +63,12 @@ pub async fn record(
chunk_id: Uuid,
text: &str,
char_start: i32,
+ role: &str,
) -> AppResult<Uuid> {
let id: Uuid = sqlx::query_scalar(
- "INSERT INTO time_mentions (id, kb_id, fact_id, chunk_id, text, char_start)
- VALUES ($1, $2, $3, $4, $5, $6)
- ON CONFLICT (fact_id, chunk_id, char_start) DO UPDATE SET text = time_mentions.text
+ "INSERT INTO time_mentions (id, kb_id, fact_id, chunk_id, text, char_start, role)
+ VALUES ($1, $2, $3, $4, $5, $6, $7)
+ ON CONFLICT (fact_id, chunk_id, char_start, role) DO UPDATE SET text = time_mentions.text
RETURNING id",
)
.bind(Uuid::now_v7())
@@ -47,6 +77,7 @@ pub async fn record(
.bind(chunk_id)
.bind(text)
.bind(char_start)
+ .bind(role)
.fetch_one(pool)
.await?;
Ok(id)
@@ -61,12 +92,11 @@ pub async fn for_facts(
if fact_ids.is_empty() {
return Ok(out);
}
- let rows: Vec<TimeMention> = sqlx::query_as(
- "SELECT id, fact_id, chunk_id, text, char_start
- FROM time_mentions
- WHERE fact_id = ANY($1)
- ORDER BY fact_id, chunk_id, char_start",
- )
+ let rows: Vec<TimeMention> = sqlx::query_as(&format!(
+ "SELECT {COLUMNS} FROM time_mentions
+ WHERE fact_id = ANY($1)
+ ORDER BY fact_id, chunk_id, char_start, role"
+ ))
.bind(fact_ids)
.fetch_all(pool)
.await?;
@@ -75,3 +105,88 @@ pub async fn for_facts(
}
Ok(out)
}
+
+/// 一条等着读的提及:字、位置、槽,和它所在的那句话——解算按文档的语境走一遍时的输入
+#[derive(Debug, Clone, serde::Serialize, sqlx::FromRow)]
+pub struct MentionToResolve {
+ pub id: Uuid,
+ pub fact_id: Uuid,
+ pub chunk_id: Uuid,
+ pub text: String,
+ pub char_start: i32,
+ pub role: String,
+ /// 这条陈述在这一块里的引文(`fact_evidence.quote`);证据没留引文时退回整块的字
+ pub sentence: String,
+}
+
+/// 一篇文档里所有等着读的提及:每条活着的开放陈述在这篇文档的活分块上的每一条提及,
+/// 按分块顺序、块内位置排——解算沿着文档读下去,锚点才能从前一块带到后一块(0045 决定 3)。
+/// 被新版本替掉的分块不在内:它的字已经不是这篇文档的字
+pub async fn for_document(pool: &PgPool, document_id: Uuid) -> AppResult<Vec<MentionToResolve>> {
+ Ok(sqlx::query_as(
+ "SELECT m.id, m.fact_id, m.chunk_id, m.text, m.char_start, m.role,
+ COALESCE(fe.quote, c.text) AS sentence
+ FROM time_mentions m
+ JOIN chunks c ON c.id = m.chunk_id
+ JOIN facts f ON f.id = m.fact_id
+ LEFT JOIN fact_evidence fe ON fe.fact_id = m.fact_id AND fe.chunk_id = m.chunk_id
+ WHERE c.document_id = $1 AND c.superseded_at IS NULL
+ AND f.layer = 'open' AND f.invalidated_at IS NULL
+ ORDER BY c.seq, m.char_start, m.role",
+ )
+ .bind(document_id)
+ .fetch_all(pool)
+ .await?)
+}
+
+/// 模型对这条提及的解释(0045 决定 2):形状、参照、粒度,照存。取值是抽取合同的词汇,
+/// 库不认;日期算术不在这里
+pub async fn set_interpretation(
+ pool: &PgPool,
+ id: Uuid,
+ shape: &str,
+ reference: &serde_json::Value,
+ granularity: &str,
+) -> AppResult<()> {
+ sqlx::query(
+ "UPDATE time_mentions SET shape = $2, reference = $3, granularity = $4 WHERE id = $1",
+ )
+ .bind(id)
+ .bind(shape)
+ .bind(reference)
+ .bind(granularity)
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
+/// 代码对这条提及的解算:等级和两端各自的时刻与精度,`resolved_at` 记此刻。值先按精度截断
+/// ([`crate::graph::truncate_to`]),存的值与精度说同一句话,数据库的 CHECK 才放行。
+/// grade C 四列全空;结束端 `(None, Some("unknown"))` 是结束了不知哪天
+pub async fn set_resolution(
+ pool: &PgPool,
+ id: Uuid,
+ grade: &str,
+ from: Option<DateTime<Utc>>,
+ from_precision: Option<&str>,
+ to: Option<DateTime<Utc>>,
+ to_precision: Option<&str>,
+) -> AppResult<()> {
+ let from = from.map(|t| crate::graph::truncate_to(t, from_precision));
+ let to = to.map(|t| crate::graph::truncate_to(t, to_precision));
+ sqlx::query(
+ "UPDATE time_mentions
+ SET grade = $2, resolved_from = $3, resolved_from_precision = $4,
+ resolved_to = $5, resolved_to_precision = $6, resolved_at = now()
+ WHERE id = $1",
+ )
+ .bind(id)
+ .bind(grade)
+ .bind(from)
+ .bind(from_precision)
+ .bind(to)
+ .bind(to_precision)
+ .execute(pool)
+ .await?;
+ Ok(())
+}
diff --git a/crates/utopia-store/tests/a_time_mention_is_resolved_against_its_document.rs b/crates/utopia-store/tests/a_time_mention_is_resolved_against_its_document.rs
new file mode 100644
index 00000000..b2f4aa3a
--- /dev/null
+++ b/crates/utopia-store/tests/a_time_mention_is_resolved_against_its_document.rs
@@ -0,0 +1,477 @@
+//! 一条时间提及按它的文档来读(0045 第一、二刀的账本侧)。
+//!
+//! 一条开放陈述的 `when` 与 `ended` 各是一条提及;按文档取回时带着它在证据里的那句话,
+//! 按分块顺序、块内位置排;模型的解释与代码的解算各自落在提及上、取得回来;解算结果由
+//! `set_open_validity` 写到开放行的世界轴上——只写开放行,类型化的行一行不动;
+//! 结束了不知哪天的解算存得下;值与精度不一致的由库挡下。
+//!
+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
+
+use chrono::{DateTime, TimeZone, Utc};
+use sqlx::PgPool;
+use utopia_store::graph::{self, FactObject};
+use utopia_store::time_mentions;
+use uuid::Uuid;
+
+const ORG: &str = "time-resolution-test";
+const TEXT_A: &str = "Acme acquired Beta on March 4, 2011 and sold it in 2019.";
+const TEXT_B: &str = "Beta had been founded two years earlier by Carol.";
+
+struct Fixture {
+ kb: Uuid,
+ acme: Uuid,
+ beta: Uuid,
+ carol: Uuid,
+ doc: Uuid,
+ chunk_a: Uuid,
+ chunk_b: Uuid,
+}
+
+async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {
+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ let (acme, beta, carol) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ let (src, doc, chunk_a, chunk_b) = (
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ );
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, $2)")
+ .bind(org)
+ .bind(ORG)
+ .execute(pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, $3)")
+ .bind(ws)
+ .bind(org)
+ .bind(ORG)
+ .execute(pool)
+ .await?;
+ sqlx::query("INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, $3)")
+ .bind(kb)
+ .bind(ws)
+ .bind(ORG)
+ .execute(pool)
+ .await?;
+ for (id, name) in [(acme, "Acme"), (beta, "Beta"), (carol, "Carol")] {
+ sqlx::query("INSERT INTO entities (id, kb_id, canonical_name) VALUES ($1, $2, $3)")
+ .bind(id)
+ .bind(kb)
+ .bind(name)
+ .execute(pool)
+ .await?;
+ }
+ sqlx::query("INSERT INTO sources (id, kb_id, name) VALUES ($1, $2, '临时来源')")
+ .bind(src)
+ .bind(kb)
+ .execute(pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO documents (id, kb_id, source_id, filename, sha256, status)
+ VALUES ($1, $2, $3, 'deal.md', 'timeresolution', 'ready')",
+ )
+ .bind(doc)
+ .bind(kb)
+ .bind(src)
+ .execute(pool)
+ .await?;
+ for (id, seq, text) in [(chunk_a, 0i32, TEXT_A), (chunk_b, 1i32, TEXT_B)] {
+ sqlx::query(
+ "INSERT INTO chunks (id, kb_id, document_id, seq, text) VALUES ($1, $2, $3, $4, $5)",
+ )
+ .bind(id)
+ .bind(kb)
+ .bind(doc)
+ .bind(seq)
+ .bind(text)
+ .execute(pool)
+ .await?;
+ }
+ Ok(Fixture {
+ kb,
+ acme,
+ beta,
+ carol,
+ doc,
+ chunk_a,
+ chunk_b,
+ })
+}
+
+/// 服务端在块里搜出来的字符起点(这里的文本是 ASCII,字节即字符)
+fn start_of(text: &str, words: &str) -> i32 {
+ let byte = text.find(words).expect("words are in the chunk");
+ text[..byte].chars().count() as i32
+}
+
+fn day(y: i32, m: u32, d: u32) -> DateTime<Utc> {
+ Utc.with_ymd_and_hms(y, m, d, 0, 0, 0).unwrap()
+}
+
+#[derive(Debug, PartialEq, sqlx::FromRow)]
+struct Span {
+ valid_from: Option<DateTime<Utc>>,
+ valid_from_precision: Option<String>,
+ valid_to: Option<DateTime<Utc>>,
+ valid_to_precision: Option<String>,
+}
+
+async fn span_of(pool: &PgPool, fact: Uuid) -> anyhow::Result<Span> {
+ Ok(sqlx::query_as(
+ "SELECT valid_from, valid_from_precision, valid_to, valid_to_precision
+ FROM facts WHERE id = $1",
+ )
+ .bind(fact)
+ .fetch_one(pool)
+ .await?)
+}
+
+#[tokio::test]
+async fn a_mention_is_read_then_computed_then_written_to_the_open_row() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ sqlx::query("DELETE FROM organizations WHERE name = $1")
+ .bind(ORG)
+ .execute(&pool)
+ .await?;
+ let f = seed(&pool).await?;
+
+ let run = async {
+ // 一条开放陈述,证据带引文;起与止各一条提及
+ let (fact, _) = graph::insert_open_statement(
+ &pool,
+ f.kb,
+ f.acme,
+ "acquired",
+ FactObject::Entity(f.beta),
+ None,
+ 0.9,
+ )
+ .await?;
+ graph::add_evidence_located(
+ &pool,
+ fact,
+ f.chunk_a,
+ Some(TEXT_A),
+ Some("acquired"),
+ Some((0, TEXT_A.len() as i32)),
+ )
+ .await?;
+ let when_start = start_of(TEXT_A, "March 4, 2011");
+ let ended_start = start_of(TEXT_A, "2019");
+ let when = time_mentions::record(
+ &pool,
+ f.kb,
+ fact,
+ f.chunk_a,
+ "March 4, 2011",
+ when_start,
+ "when",
+ )
+ .await?;
+ let ended =
+ time_mentions::record(&pool, f.kb, fact, f.chunk_a, "2019", ended_start, "ended")
+ .await?;
+
+ // 1. 按文档取回:两条,带证据里的那句话,按位置排
+ let todo = time_mentions::for_document(&pool, f.doc).await?;
+ assert_eq!(todo.len(), 2, "{todo:?}");
+ assert_eq!(
+ (
+ todo[0].id,
+ todo[0].role.as_str(),
+ todo[0].text.as_str(),
+ todo[0].char_start
+ ),
+ (when, "when", "March 4, 2011", when_start)
+ );
+ assert_eq!(
+ (
+ todo[1].id,
+ todo[1].role.as_str(),
+ todo[1].text.as_str(),
+ todo[1].char_start
+ ),
+ (ended, "ended", "2019", ended_start)
+ );
+ assert!(todo
+ .iter()
+ .all(|m| m.sentence == TEXT_A && m.fact_id == fact && m.chunk_id == f.chunk_a));
+
+ // 第二块上的陈述排在后面;证据没留引文的退回整块的字
+ let (founded, _) = graph::insert_open_statement(
+ &pool,
+ f.kb,
+ f.beta,
+ "founded by",
+ FactObject::Entity(f.carol),
+ None,
+ 0.9,
+ )
+ .await?;
+ graph::add_evidence(&pool, founded, f.chunk_b, None, Some("founded by")).await?;
+ let earlier_start = start_of(TEXT_B, "two years earlier");
+ let earlier = time_mentions::record(
+ &pool,
+ f.kb,
+ founded,
+ f.chunk_b,
+ "two years earlier",
+ earlier_start,
+ "when",
+ )
+ .await?;
+ let todo = time_mentions::for_document(&pool, f.doc).await?;
+ assert_eq!(
+ todo.iter().map(|m| m.id).collect::<Vec<_>>(),
+ [when, ended, earlier],
+ "chunk order, then position"
+ );
+ assert_eq!(
+ todo[2].sentence, TEXT_B,
+ "no quote on the evidence: the chunk's text"
+ );
+
+ // 作废的陈述不在等着读的名单里
+ sqlx::query("UPDATE facts SET invalidated_at = now() WHERE id = $1")
+ .bind(founded)
+ .execute(&pool)
+ .await?;
+ let todo = time_mentions::for_document(&pool, f.doc).await?;
+ assert_eq!(todo.iter().map(|m| m.id).collect::<Vec<_>>(), [when, ended]);
+
+ // 2. 解释与解算各自落在提及上,取得回来
+ time_mentions::set_interpretation(
+ &pool,
+ when,
+ "point",
+ &serde_json::json!({ "absolute": "2011-03-04" }),
+ "day",
+ )
+ .await?;
+ // 解算给的值截到精度:15 点落成那一天的 0 点
+ time_mentions::set_resolution(
+ &pool,
+ when,
+ "A",
+ Some(Utc.with_ymd_and_hms(2011, 3, 4, 15, 0, 0).unwrap()),
+ Some("day"),
+ None,
+ None,
+ )
+ .await?;
+ time_mentions::set_interpretation(
+ &pool,
+ ended,
+ "ended",
+ &serde_json::json!({ "absolute": "2019" }),
+ "year",
+ )
+ .await?;
+ time_mentions::set_resolution(
+ &pool,
+ ended,
+ "A",
+ None,
+ None,
+ Some(day(2019, 6, 1)),
+ Some("year"),
+ )
+ .await?;
+ let mentions = time_mentions::for_facts(&pool, &[fact]).await?;
+ let m = &mentions[&fact];
+ assert_eq!(m.len(), 2);
+ let w = m.iter().find(|x| x.id == when).unwrap();
+ assert_eq!(
+ (
+ w.shape.as_deref(),
+ w.granularity.as_deref(),
+ w.grade.as_deref()
+ ),
+ (Some("point"), Some("day"), Some("A"))
+ );
+ assert_eq!(
+ w.reference,
+ Some(serde_json::json!({ "absolute": "2011-03-04" }))
+ );
+ assert_eq!(
+ (w.resolved_from, w.resolved_from_precision.as_deref()),
+ (Some(day(2011, 3, 4)), Some("day"))
+ );
+ assert!(w.resolved_to.is_none() && w.resolved_to_precision.is_none());
+ let e = m.iter().find(|x| x.id == ended).unwrap();
+ assert_eq!(
+ (e.shape.as_deref(), e.grade.as_deref()),
+ (Some("ended"), Some("A"))
+ );
+ assert_eq!(
+ (e.resolved_to, e.resolved_to_precision.as_deref()),
+ (Some(day(2019, 1, 1)), Some("year"))
+ );
+ let resolved_at: Vec<Option<DateTime<Utc>>> =
+ sqlx::query_scalar("SELECT resolved_at FROM time_mentions WHERE fact_id = $1")
+ .bind(fact)
+ .fetch_all(&pool)
+ .await?;
+ assert!(resolved_at.iter().all(Option::is_some));
+
+ // 结束了不知哪天:存得下
+ time_mentions::set_resolution(&pool, ended, "B", None, None, None, Some("unknown")).await?;
+ let e = &time_mentions::for_facts(&pool, &[fact]).await?[&fact]
+ .iter()
+ .find(|x| x.id == ended)
+ .cloned()
+ .unwrap();
+ assert_eq!(
+ (
+ e.grade.as_deref(),
+ e.resolved_to,
+ e.resolved_to_precision.as_deref()
+ ),
+ (Some("B"), None, Some("unknown"))
+ );
+ // 没算出来:四列全空
+ time_mentions::set_resolution(&pool, ended, "C", None, None, None, None).await?;
+ let e = &time_mentions::for_facts(&pool, &[fact]).await?[&fact]
+ .iter()
+ .find(|x| x.id == ended)
+ .cloned()
+ .unwrap();
+ assert_eq!(e.grade.as_deref(), Some("C"));
+ assert!(
+ e.resolved_from.is_none()
+ && e.resolved_from_precision.is_none()
+ && e.resolved_to.is_none()
+ && e.resolved_to_precision.is_none()
+ );
+ // 值与精度不一致的由库挡下:有日期的结束端不能说 unknown,等级只有 A / B / C
+ assert!(time_mentions::set_resolution(
+ &pool,
+ ended,
+ "A",
+ None,
+ None,
+ Some(day(2019, 1, 1)),
+ Some("unknown"),
+ )
+ .await
+ .is_err());
+ assert!(
+ time_mentions::set_resolution(&pool, ended, "D", None, None, None, None)
+ .await
+ .is_err()
+ );
+
+ // 3. 解算写到开放行的世界轴上,按精度截断
+ graph::set_open_validity(
+ &pool,
+ fact,
+ Some(Utc.with_ymd_and_hms(2011, 3, 4, 15, 0, 0).unwrap()),
+ Some("day"),
+ Some(day(2019, 6, 1)),
+ Some("year"),
+ )
+ .await?;
+ assert_eq!(
+ span_of(&pool, fact).await?,
+ Span {
+ valid_from: Some(day(2011, 3, 4)),
+ valid_from_precision: Some("day".into()),
+ valid_to: Some(day(2019, 1, 1)),
+ valid_to_precision: Some("year".into()),
+ }
+ );
+ // 结束了不知哪天
+ graph::set_open_validity(
+ &pool,
+ fact,
+ Some(day(2011, 3, 4)),
+ Some("day"),
+ None,
+ Some("unknown"),
+ )
+ .await?;
+ assert_eq!(
+ span_of(&pool, fact).await?,
+ Span {
+ valid_from: Some(day(2011, 3, 4)),
+ valid_from_precision: Some("day".into()),
+ valid_to: None,
+ valid_to_precision: Some("unknown".into()),
+ }
+ );
+ // 结束了不知哪天的有自己的锚点:说出结束的就是这条陈述自己的文档(#393)
+ let anchored: (bool,) = sqlx::query_as(
+ "SELECT attested_to IS NOT NULL AND attested_to = attested_from
+ FROM facts WHERE id = $1",
+ )
+ .bind(fact)
+ .fetch_one(&pool)
+ .await?;
+ assert!(
+ anchored.0,
+ "an unknown ending is anchored on its own document"
+ );
+ // 又算不出来了:清空,锚点也跟着走
+ graph::set_open_validity(&pool, fact, None, None, None, None).await?;
+ assert_eq!(
+ span_of(&pool, fact).await?,
+ Span {
+ valid_from: None,
+ valid_from_precision: None,
+ valid_to: None,
+ valid_to_precision: None,
+ }
+ );
+ let anchored: (bool,) =
+ sqlx::query_as("SELECT attested_to IS NOT NULL FROM facts WHERE id = $1")
+ .bind(fact)
+ .fetch_one(&pool)
+ .await?;
+ assert!(!anchored.0);
+
+ // 4. 类型化的行不归它写:一行不改
+ let typed = Uuid::now_v7();
+ sqlx::query("INSERT INTO facts (id, kb_id, subject_id, object_id) VALUES ($1, $2, $3, $4)")
+ .bind(typed)
+ .bind(f.kb)
+ .bind(f.acme)
+ .bind(f.beta)
+ .execute(&pool)
+ .await?;
+ let refused =
+ graph::set_open_validity(&pool, typed, Some(day(2011, 3, 4)), Some("day"), None, None)
+ .await;
+ assert!(refused.is_err(), "a typed row keeps to insert_fact_inner");
+ assert_eq!(
+ span_of(&pool, typed).await?,
+ Span {
+ valid_from: None,
+ valid_from_precision: None,
+ valid_to: None,
+ valid_to_precision: None,
+ }
+ );
+ // 作废的开放行是历史,也不改
+ assert!(graph::set_open_validity(
+ &pool,
+ founded,
+ Some(day(2009, 1, 1)),
+ Some("year"),
+ None,
+ None
+ )
+ .await
+ .is_err());
+ Ok::<(), anyhow::Error>(())
+ }
+ .await;
+
+ sqlx::query("DELETE FROM organizations WHERE name = $1")
+ .bind(ORG)
+ .execute(&pool)
+ .await?;
+ run
+}
diff --git a/crates/utopia-store/tests/a_time_mention_is_words_not_a_date.rs b/crates/utopia-store/tests/a_time_mention_is_words_not_a_date.rs
index 503f21be..232b0795 100644
--- a/crates/utopia-store/tests/a_time_mention_is_words_not_a_date.rs
+++ b/crates/utopia-store/tests/a_time_mention_is_words_not_a_date.rs
@@ -109,7 +109,8 @@ async fn a_time_mention_resolves_inside_the_chunk_and_sets_no_date() -> anyhow::
byte_start as i32, char_start,
"中文里字节偏移与字符偏移不同"
);
- let id = time_mentions::record(&pool, f.kb, fact, f.chunk, words, char_start).await?;
+ let id =
+ time_mentions::record(&pool, f.kb, fact, f.chunk, words, char_start, "when").await?;
// 字与偏移在块里对得上:按字符数到那里,取同样多的字,得到的就是记下的字
let (stored_text, stored_start): (String, i32) =
@@ -159,6 +160,35 @@ async fn a_time_mention_resolves_inside_the_chunk_and_sets_no_date() -> anyhow::
let mentions = time_mentions::for_facts(&pool, &[fact]).await?;
assert_eq!(mentions[&fact].len(), 1);
assert_eq!(mentions[&fact][0].chunk_id, f.chunk);
+ assert_eq!(mentions[&fact][0].role, "when");
+ // 刚记下的字还没有读法:解释与解算都空着
+ let m = &mentions[&fact][0];
+ assert!(m.shape.is_none() && m.reference.is_none() && m.granularity.is_none());
+ assert!(m.grade.is_none() && m.resolved_from.is_none() && m.resolved_to.is_none());
+
+ // 起与止各是一条提及:同一处字换一个槽是另一行,同一个槽再记一次是同一行
+ let ended =
+ time_mentions::record(&pool, f.kb, fact, f.chunk, words, char_start, "ended").await?;
+ assert_ne!(ended, id);
+ assert_eq!(
+ time_mentions::record(&pool, f.kb, fact, f.chunk, words, char_start, "when").await?,
+ id
+ );
+ let mentions = time_mentions::for_facts(&pool, &[fact]).await?;
+ assert_eq!(
+ mentions[&fact]
+ .iter()
+ .map(|m| m.role.as_str())
+ .collect::<Vec<_>>(),
+ ["ended", "when"]
+ );
+ // 槽只有这两个
+ assert!(
+ time_mentions::record(&pool, f.kb, fact, f.chunk, words, char_start, "as_of")
+ .await
+ .is_err(),
+ "the ledger takes only when / ended"
+ );
Ok::<(), anyhow::Error>(())
}
.await;
diff --git a/crates/utopia-store/tests/a_timeline_holds_whatever_the_order.rs b/crates/utopia-store/tests/a_timeline_holds_whatever_the_order.rs
index 50b2632a..f02f8324 100644
--- a/crates/utopia-store/tests/a_timeline_holds_whatever_the_order.rs
+++ b/crates/utopia-store/tests/a_timeline_holds_whatever_the_order.rs
@@ -92,7 +92,8 @@ async fn arrive_on(
subject: Uuid,
value: &str,
from: Option<&str>,
- doc: Option<(chrono::DateTime<chrono::Utc>, &str)>,
+ // (文档的日期, doc_time_source);没有日期的文档是 (None, "none")(0045 决定 3)
+ doc: Option<(Option<chrono::DateTime<chrono::Utc>>, &str)>,
confidence: f32,
reconcile: bool,
) -> anyhow::Result<Uuid> {
@@ -123,7 +124,7 @@ async fn arrive_on(
.bind(value)
.execute(pool)
.await?;
- validity = validity.attested(Some(doc_time));
+ validity = validity.attested(doc_time);
chunk = Some(c);
}
let object = json!({ "value": value });
@@ -255,7 +256,8 @@ async fn extracting_the_same_values_again_changes_nothing() -> anyhow::Result<()
Ok(())
}
-/// Undated upload: documents.create stores doc_time = now(), doc_time_source = 'upload_time'.
+/// Undated upload: the document writers store doc_time NULL, doc_time_source = 'none'
+/// (0045 decision 3); the upload time is recorded time and never a document date.
#[tokio::test]
async fn an_undated_upload_is_not_a_document_date() -> anyhow::Result<()> {
let Some(url) = utopia_store::test_db::url() else {
@@ -263,14 +265,13 @@ async fn an_undated_upload_is_not_a_document_date() -> anyhow::Result<()> {
};
let pool = PgPool::connect(&url).await?;
let f = seed(&pool).await?;
- let now = chrono::Utc::now();
let bare = arrive_on(
&pool,
&f,
f.lease,
"BBHQ1",
None,
- Some((now, "upload_time")),
+ Some((None, "none")),
0.9,
true,
)
@@ -281,7 +282,7 @@ async fn an_undated_upload_is_not_a_document_date() -> anyhow::Result<()> {
f.lease,
"HPBB1",
Some("2021-01-01"),
- Some((t("2021-01-01"), "content")),
+ Some((Some(t("2021-01-01")), "content")),
0.9,
true,
)
@@ -324,7 +325,7 @@ async fn a_dated_value_with_no_start_ends_at_the_next_start_after_its_date() ->
f.lease,
"BBHQ1",
None,
- Some((t("2020-08-13"), "content")),
+ Some((Some(t("2020-08-13")), "content")),
0.9,
true,
)
@@ -335,7 +336,7 @@ async fn a_dated_value_with_no_start_ends_at_the_next_start_after_its_date() ->
f.lease,
"HPBB1",
Some("2016-05-16"),
- Some((t("2016-05-16"), "content")),
+ Some((Some(t("2016-05-16")), "content")),
0.9,
true,
)
@@ -346,7 +347,7 @@ async fn a_dated_value_with_no_start_ends_at_the_next_start_after_its_date() ->
f.lease,
"NEWCO",
Some("2021-01-01"),
- Some((t("2021-01-01"), "content")),
+ Some((Some(t("2021-01-01")), "content")),
0.9,
true,
)
diff --git a/crates/utopia-store/tests/an_open_statement_keeps_the_documents_words.rs b/crates/utopia-store/tests/an_open_statement_keeps_the_documents_words.rs
index 42dd2ac0..a7c59e09 100644
--- a/crates/utopia-store/tests/an_open_statement_keeps_the_documents_words.rs
+++ b/crates/utopia-store/tests/an_open_statement_keeps_the_documents_words.rs
@@ -153,9 +153,16 @@ async fn an_open_statement_shows_under_its_phrase_and_reuses_its_row() -> anyhow
)
.await?;
let (winter_start, _) = span_of(TEXT_A, "last winter");
- let mention =
- time_mentions::record(&pool, f.kb, fact, f.chunk_a, "last winter", winter_start)
- .await?;
+ let mention = time_mentions::record(
+ &pool,
+ f.kb,
+ fact,
+ f.chunk_a,
+ "last winter",
+ winter_start,
+ "when",
+ )
+ .await?;
#[derive(sqlx::FromRow)]
struct Row {
@@ -335,8 +342,16 @@ async fn an_open_statement_shows_under_its_phrase_and_reuses_its_row() -> anyhow
("last winter", winter_start)
);
assert_eq!(
- time_mentions::record(&pool, f.kb, fact, f.chunk_a, "last winter", winter_start)
- .await?,
+ time_mentions::record(
+ &pool,
+ f.kb,
+ fact,
+ f.chunk_a,
+ "last winter",
+ winter_start,
+ "when",
+ )
+ .await?,
mention,
"同一位置再记一次回的是同一行"
);
diff --git a/crates/utopia-store/tests/an_upload_has_no_date.rs b/crates/utopia-store/tests/an_upload_has_no_date.rs
new file mode 100644
index 00000000..475b1a51
--- /dev/null
+++ b/crates/utopia-store/tests/an_upload_has_no_date.rs
@@ -0,0 +1,181 @@
+//! 一次上传没有日期(0045 决定 3,#714)。
+//!
+//! 上传、同步的时刻是记录时间,不是文档的日期:没读出日期的文档 `doc_time` 为空、
+//! 来源 'none',`dated_at` 是 None——每一个写入者都这样。正文里读出的日期落成
+//! 'content',来源系统给的落成 'source',只有这两种算文档的日期;老值
+//! 'upload_time' / 'file_mtime' 读作没有日期。时间语境原样进出。
+//!
+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
+
+use chrono::{TimeZone, Utc};
+use sqlx::PgPool;
+use utopia_store::documents;
+use uuid::Uuid;
+
+const ORG: &str = "upload-has-no-date-test";
+
+async fn seed(pool: &PgPool) -> anyhow::Result<(Uuid, Uuid)> {
+ let (org, ws, kb, src) = (
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ );
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, $2)")
+ .bind(org)
+ .bind(ORG)
+ .execute(pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, $3)")
+ .bind(ws)
+ .bind(org)
+ .bind(ORG)
+ .execute(pool)
+ .await?;
+ sqlx::query("INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, $3)")
+ .bind(kb)
+ .bind(ws)
+ .bind(ORG)
+ .execute(pool)
+ .await?;
+ sqlx::query("INSERT INTO sources (id, kb_id, name) VALUES ($1, $2, '临时来源')")
+ .bind(src)
+ .bind(kb)
+ .execute(pool)
+ .await?;
+ Ok((kb, src))
+}
+
+#[tokio::test]
+async fn an_undated_document_has_no_date_from_any_writer() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ sqlx::query("DELETE FROM organizations WHERE name = $1")
+ .bind(ORG)
+ .execute(&pool)
+ .await?;
+ let (kb, src) = seed(&pool).await?;
+
+ let run = async {
+ // 上传,正文里没读出日期
+ let doc =
+ documents::create_from_upload(&pool, kb, "a.txt", "text/plain", 1, "sha-a", None, None)
+ .await?;
+ assert_eq!((doc.doc_time, doc.doc_time_source.as_str()), (None, "none"));
+ assert!(doc.dated_at().is_none());
+ assert!(doc.time_context.is_none() && doc.time_context_at.is_none());
+
+ // 同步来的没给日期
+ let synced = documents::create(
+ &pool,
+ kb,
+ "b.txt",
+ "text/plain",
+ 1,
+ "sha-b",
+ Some(src),
+ None,
+ Some("key-b"),
+ )
+ .await?;
+ assert_eq!(
+ (synced.doc_time, synced.doc_time_source.as_str()),
+ (None, "none")
+ );
+ let queued = documents::create_with_version_and_processing(
+ &pool,
+ kb,
+ "c.txt",
+ "text/plain",
+ 1,
+ "sha-c",
+ Some(src),
+ None,
+ Some("key-c"),
+ )
+ .await?;
+ assert_eq!(
+ (queued.doc_time, queued.doc_time_source.as_str()),
+ (None, "none")
+ );
+ let mut tx = pool.begin().await?;
+ let upserted = documents::upsert_source_document_tx(
+ &mut tx,
+ kb,
+ src,
+ "key-d",
+ "d.txt",
+ "text/plain",
+ 1,
+ "sha-d",
+ None,
+ )
+ .await?;
+ tx.commit().await?;
+ assert_eq!(
+ (upserted.doc_time, upserted.doc_time_source.as_str()),
+ (None, "none")
+ );
+
+ // 来源系统给的日期留着它的来源
+ let published = Utc.with_ymd_and_hms(2011, 3, 4, 8, 0, 0).unwrap();
+ let dated = documents::create(
+ &pool,
+ kb,
+ "e.txt",
+ "text/plain",
+ 1,
+ "sha-e",
+ Some(src),
+ Some(published),
+ Some("key-e"),
+ )
+ .await?;
+ assert_eq!(
+ (dated.doc_time, dated.doc_time_source.as_str()),
+ (Some(published), "source")
+ );
+ assert_eq!(dated.dated_at(), Some(published));
+
+ // 正文里读出的日期
+ let written = Utc.with_ymd_and_hms(2024, 2, 29, 0, 0, 0).unwrap();
+ documents::set_content_date(&pool, doc.id, written).await?;
+ let doc = documents::get(&pool, doc.id).await?;
+ assert_eq!(
+ (doc.dated_at(), doc.doc_time_source.as_str()),
+ (Some(written), "content")
+ );
+
+ // 时间语境原样进出
+ let context = serde_json::json!({
+ "date": "2024-02-29",
+ "periods": [{ "name": "fiscal 2024", "from": "2023-01-29", "to": "2024-01-28" }],
+ "anchors": [{ "mention": "the acquisition", "date": "2011-03-04" }]
+ });
+ documents::set_time_context(&pool, doc.id, &context).await?;
+ let doc = documents::get(&pool, doc.id).await?;
+ assert_eq!(doc.time_context, Some(context));
+ assert!(doc.time_context_at.is_some());
+
+ // 老值读作没有日期
+ for old in ["upload_time", "file_mtime"] {
+ let mut d = doc.clone();
+ d.doc_time = Some(Utc::now());
+ d.doc_time_source = old.into();
+ assert!(
+ d.dated_at().is_none(),
+ "{old} is recorded time, not a document date"
+ );
+ }
+ Ok::<(), anyhow::Error>(())
+ }
+ .await;
+
+ sqlx::query("DELETE FROM organizations WHERE name = $1")
+ .bind(ORG)
+ .execute(&pool)
+ .await?;
+ run
+}
diff --git a/migrations/0064_a_time_mention_is_resolved_against_its_document.sql b/migrations/0064_a_time_mention_is_resolved_against_its_document.sql
new file mode 100644
index 00000000..2bbd248c
--- /dev/null
+++ b/migrations/0064_a_time_mention_is_resolved_against_its_document.sql
@@ -0,0 +1,78 @@
+-- 一条时间提及按它的文档来读(0045 第一、二刀的账本侧)。
+--
+-- 0061 把时间词照抄进 `time_mentions`,说好读法的列后一刀再加。这就是那一刀。0045 的
+-- 前五条决定,账本各落成什么:
+--
+-- 决定 1 时间提及是开放图谱的事实:字、块、字符偏移(0061 已有)。一条陈述的
+-- `when` 与 `ended` 各指一条提及——`role` 记它来自陈述的哪个槽,唯一键加上
+-- 它:起于「2019 年」、止于「2019 年」是同一处字的两条提及。
+-- 决定 2 模型读,代码算。模型给的只是解释——`shape`(点、区间、截至、时长、没给
+-- 日期的结束)、`reference`(照写的绝对值;或锚点 + 偏移;或没有)、
+-- `granularity`(字说到梯子的哪一级)——三列照存,取值是抽取合同的词汇,库不认。
+-- 日期算术在代码里做,结果落 `resolved_*`:两端各自的时刻与精度,梯子与 `facts`
+-- 同一张(0024:year … second),结束端多一个 'unknown'(结束了,不知哪天);
+-- 两条 CHECK 照抄 `facts_from_precision_matches_date` /
+-- `facts_to_precision_matches_date`(0033),存的值截到自己的精度。
+-- `grade`:A 原文写明的绝对日期,B 按文档自己给的锚点算出来的,C 没算出来。
+-- `resolved_at` 是最近一次算的时刻。
+-- 决定 3 文档把它的时间语境从一块带到下一块:`documents.time_context` 存它(文档自己
+-- 的日期、它定义的期间与历法、叙述设下的锚点),`time_context_at` 是最近一次
+-- 写下它的时刻。服务端边抽取边填。
+-- 文档的日期只来自正文或给它日期的来源系统。上传、同步、抽取的时刻是记录时间,
+-- 不进语境:`doc_time_source = 'upload_time'` 读作**没有日期**(#714)。文件的
+-- 修改时刻也不是文档的日期——它说的是文件系统上次写这个文件是什么时候,与文档
+-- 说的是哪一天无关,'file_mtime' 同样读作没有日期。两种都把 `doc_time` 清空、
+-- 来源改成 'none';列的默认值也改成 'none':从前默认 'file_mtime',一行没给
+-- 日期插进来就顶着一个它没有的来源。这一列没有 CHECK,代码里的取值是
+-- content / source / none(`Document::dated_at` 只认前两个)。
+-- 决定 4 算不出来的等着:grade C 的提及留着字和解释,`resolved_*` 全空,不给任何陈述
+-- 日期;锚点后来到了再算一次,`resolved_at` 跟着走。
+-- 决定 5 三个时间三列:valid 是算出来的(开放行的 `valid_*` 由 `graph::set_open_validity`
+-- 按解算写,不由模型写)、attested 是文档自己的日期(没有就空——服务端后续那
+-- 一刀)、recorded 是账本何时知道。
+--
+-- 纯增量,不回填解释。已有的提及全记成 role = 'when':0061 记 when 与 ended 时没分槽,
+-- 分不回去了;没有已发布的版本,不留遗留层。
+ALTER TABLE time_mentions
+ -- 来自陈述的哪个槽
+ ADD COLUMN role TEXT NOT NULL DEFAULT 'when' CHECK (role IN ('when', 'ended')),
+ -- 模型的解释:照存
+ ADD COLUMN shape TEXT,
+ ADD COLUMN reference JSONB,
+ ADD COLUMN granularity TEXT,
+ -- 代码的解算
+ ADD COLUMN grade CHAR(1) CHECK (grade IN ('A', 'B', 'C')),
+ ADD COLUMN resolved_from TIMESTAMPTZ,
+ ADD COLUMN resolved_from_precision TEXT,
+ ADD COLUMN resolved_to TIMESTAMPTZ,
+ ADD COLUMN resolved_to_precision TEXT,
+ ADD COLUMN resolved_at TIMESTAMPTZ,
+ -- 三值逻辑的坑同 0033:每个分支先写 IS NOT NULL
+ ADD CONSTRAINT time_mentions_from_precision_matches_date CHECK (
+ (resolved_from IS NULL AND resolved_from_precision IS NULL)
+ OR (resolved_from IS NOT NULL
+ AND resolved_from_precision IS NOT NULL
+ AND resolved_from_precision IN ('year', 'month', 'day', 'hour', 'minute', 'second')
+ AND resolved_from = date_trunc(resolved_from_precision, resolved_from, 'UTC'))),
+ ADD CONSTRAINT time_mentions_to_precision_matches_date CHECK (
+ (resolved_to IS NOT NULL
+ AND resolved_to_precision IS NOT NULL
+ AND resolved_to_precision IN ('year', 'month', 'day', 'hour', 'minute', 'second')
+ AND resolved_to = date_trunc(resolved_to_precision, resolved_to, 'UTC'))
+ OR (resolved_to IS NULL
+ AND (resolved_to_precision IS NULL OR resolved_to_precision = 'unknown')));
+
+-- 同一处字在起与止各一条
+ALTER TABLE time_mentions
+ DROP CONSTRAINT time_mentions_fact_id_chunk_id_char_start_key,
+ ADD CONSTRAINT time_mentions_fact_id_chunk_id_char_start_role_key
+ UNIQUE (fact_id, chunk_id, char_start, role);
+
+ALTER TABLE documents
+ ADD COLUMN time_context JSONB,
+ ADD COLUMN time_context_at TIMESTAMPTZ,
+ ALTER COLUMN doc_time_source SET DEFAULT 'none';
+
+-- 上传时刻与文件修改时刻都不是文档的日期
+UPDATE documents SET doc_time = NULL, doc_time_source = 'none'
+ WHERE doc_time_source IN ('upload_time', 'file_mtime');