World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A bench for time words: what each sentence of a written document should be dated (#990)
WaylandYang841b619fa602
diff --git a/scripts/bench/README.md b/scripts/bench/README.mdindex 42b58e9f..c1f4a037 100644--- a/scripts/bench/README.md+++ b/scripts/bench/README.md@@ -287,6 +287,40 @@ node scripts/bench/agent.mjs --kb <kb-id> --db utopia_bench4 --reset --arms trim- 提案旁边显示的"最近的已有元素"(按向量取两条)只罩住了被判重复的 16 条里的 6 条,是给人看的线索,不是防线。+## 时间词的测量台(0064)++`timewords.mjs` 把 `truth/timewords.json` 里五篇写好的文档(三篇中文、两篇英文,31 句)各传进一个新库,+抽取、解析之后逐句对:这句话抽出来没有、它的时间说法记成时间词没有、起点落在期望的区间里没有、+没写时间的句子有没有被硬给一个起点、「截至」对不对,以及几遍之间每句话的结果一不一样。每一遍建一个+新库,不删任何东西。++```+BENCH_BASE=http://127.0.0.1:1524 BENCH_PSQL="docker exec ... -d utopia_bench4 -tAc" \+BENCH_SERVER_LOG=/path/to/server.log node scripts/bench/timewords.mjs --db utopia_bench4 --runs 3+```++句子按种类分:`absolute`(写明的日期)、`relative_now`(相对文本说话的那一刻:三个月后、上周、today)、+`relative_event`(相对一件有日期的事:上市半年后)、`relative_undated_event`(相对一件没日期的事,期望+不写起点)、`section`(时间在节标题上)、`none`(没写时间,期望没有起点,「截至」来自上下文)。++**基线,2026-09-28,dev d7756a4,gemini-3.5-flash,三遍:**++| | 第 1 遍 | 第 2 遍 | 第 3 遍 |+|---|---|---|---|+| 句子抽到了 | 30/31 | 29/31 | 31/31 |+| 时间说法记成了时间词 | 16/21 | 16/21 | 16/21 |+| 起点对:写明的 | 4/4 | 4/4 | 4/4 |+| 起点对:相对此刻 | 7/11 | 7/11 | 7/11 |+| 起点对:相对有日期的事 | 3/3 | 3/3 | 3/3 |+| 起点对:节标题上的时间 | 0/2 | 0/2 | 0/2 |+| 没写时间的没被给起点 | 10/10 | 10/10 | 10/10 |+| 「截至」对 | 0/10 | 0/10 | 0/10 |+| 每块 token | 7,990 | 9,047 | 10,207 |++三遍里结果一致的句子 28/31。丢的是四种:「三个月后发布」「下个月启动」没记成时间词(词进了宾语);+「上周」记下了但没算出来;「in two months」三遍对一遍;节标题上的季度到不了它下面的句子。「截至」一栏+是 0:没写时间的陈述今天拿到的是处理文档的那一刻,不是文档里任何一个日期。+## 读数怎么算- `prompt_tokens_est` 是**本体段**的估算,不是整个提示词。实测 4.0 字符 ≈ 1 tokendiff --git a/scripts/bench/timewords.mjs b/scripts/bench/timewords.mjsnew file mode 100644index 00000000..4abcb94c--- /dev/null+++ b/scripts/bench/timewords.mjs@@ -0,0 +1,114 @@+#!/usr/bin/env node+// 时间词的测量台(ADR 0064):写好的文档里每句话带什么时间是已知的,量+// - 这句话抽出来了没有;它的时间说法记成时间词了没有;+// - 起点算对了没有(落在期望的区间里);没写时间的句子有没有被硬给一个起点;+// - 「截至」对不对(0064 第 3 刀之前这一栏是 0);+// - 同一份文档跑几遍,每句话的结果一不一样;+// - token(服务端日志的 llm usage 行)。+//+// BENCH_BASE=http://127.0.0.1:1524 BENCH_PSQL="docker exec ... -d utopia_bench4 -tAc" \+// BENCH_SERVER_LOG=/path/to/server.log \+// node scripts/bench/timewords.mjs --db utopia_bench4 [--runs 3] [--out file.json]+//+// 每一遍建一个新库;不删任何东西。+import fs from "node:fs";+import path from "node:path";+import { fileURLToPath } from "node:url";+import { BASE, api, login, cookieHeader, parseArgs, log, onDb, sleep } from "./lib.mjs";++const HERE = path.dirname(fileURLToPath(import.meta.url));+const args = parseArgs(process.argv);+const DB = args.db;+const RUNS = Number(args.runs || 3);+const LOG = process.env.BENCH_SERVER_LOG;+if (!DB) { console.error("--db <bench database> 是必须的"); process.exit(2); }+const corpus = JSON.parse(fs.readFileSync(path.join(HERE, "truth", "timewords.json"), "utf8"));+const q = (sql) => onDb(DB, sql);+const esc = (s) => String(s).replace(/'/g, "''");++async function run(n) {+ const ws = (await api("GET", "/api/v1/workspaces"))[0];+ const kb = await api("POST", `/api/v1/workspaces/${ws.id}/kbs`, { name: `timewords-${Date.now()}-${n}`, ontology_packs: [] });+ const KB = kb.id;+ const from = LOG ? fs.statSync(LOG).size : 0;+ for (const d of corpus.docs) {+ const fd = new FormData();+ fd.append("files", new Blob([d.text], { type: "text/markdown" }), d.filename);+ const r = await fetch(`${BASE}/api/v1/kbs/${KB}/documents`, { method: "POST", headers: { cookie: cookieHeader() }, body: fd });+ if (!r.ok) throw new Error(`upload ${d.filename} -> ${r.status} ${(await r.text()).slice(0, 200)}`);+ }+ for (let i = 0; i < 60; i++) { await sleep(3000); if (q(`SELECT count(*) FROM documents WHERE kb_id='${KB}' AND status<>'ready'`) === "0") break; }+ for (const d of (await api("GET", `/api/v1/kbs/${KB}/documents?limit=50`)).docs) await api("POST", `/api/v1/documents/${d.id}/extract`, {});+ for (let i = 0; i < 240; i++) {+ await sleep(5000);+ const pending = q(`SELECT count(*) FROM jobs WHERE status IN ('queued','running') AND kind IN ('extract_document','resolve_time','process_document') AND (payload->>'kb_id'='${KB}' OR payload->>'document_id' IN (SELECT id::text FROM documents WHERE kb_id='${KB}'))`);+ if (pending === "0" && i > 3) break;+ }+ // 每句话:这篇文档里出处引文含 key 的开放陈述+ const rows = [];+ for (const d of corpus.docs) for (const e of d.expect) {+ const facts = JSON.parse(q(`SELECT coalesce(json_agg(json_build_object(+ 'from', f.valid_from, 'to', f.valid_to, 'grade', f.valid_from_grade,+ 'attested', f.attested_from, 'recorded', f.recorded_at,+ 'mentions', (SELECT count(*) FROM time_mentions m WHERE m.fact_id = f.id),+ 'phrase', f.phrase)), '[]')+ FROM facts f JOIN fact_evidence fe ON fe.fact_id = f.id JOIN documents doc ON doc.id = fe.document_id+ WHERE f.kb_id='${KB}' AND f.layer='open' AND f.invalidated_at IS NULL+ AND doc.filename='${esc(d.filename)}' AND fe.quote LIKE '%${esc(e.key)}%'`));+ const day = (t) => (t ? String(t).slice(0, 10) : null);+ const within = (t, range) => !!t && !!range && day(t) >= range[0] && day(t) <= range[1];+ const timed = e.kind !== "none";+ const found = facts.length > 0;+ const recorded = facts.some((f) => f.mentions > 0);+ const dated = facts.some((f) => f.from);+ // 期望没有起点的(没写时间、或起算的事件没日期):给了起点就是错+ const right = !found ? false+ : e.from ? facts.some((f) => within(f.from, e.from))+ : !dated;+ // 「截至」:锚点落在期望的区间里,而且不是处理文档的那一刻+ const asOf = e.as_of ? facts.some((f) => within(f.attested, e.as_of) && day(f.attested) !== day(f.recorded)) : null;+ rows.push({ doc: d.filename, key: e.key, kind: e.kind, timed, found, recorded, dated, right, asOf,+ got: facts.map((f) => `${day(f.from) ?? "-"}${f.grade ? "/" + f.grade : ""}`).join(",") });+ }+ let tokens = null;+ if (LOG) {+ // eslint-disable-next-line no-control-regex+ const text = fs.readFileSync(LOG).subarray(from).toString("utf8").replace(/\x1b\[[0-9;]*m/g, "");+ const usage = [...text.matchAll(/llm usage .*prompt=(\d+) completion=(\d+)/g)];+ tokens = { calls: usage.length, prompt: usage.reduce((a, m) => a + +m[1], 0), completion: usage.reduce((a, m) => a + +m[2], 0) };+ }+ const chunks = Number(q(`SELECT count(*) FROM chunks c JOIN documents d ON d.id=c.document_id WHERE d.kb_id='${KB}' AND c.superseded_at IS NULL`));+ const statements = Number(q(`SELECT count(*) FROM facts WHERE kb_id='${KB}' AND layer='open' AND invalidated_at IS NULL`));+ return { kb: KB, rows, tokens, chunks, statements };+}++async function main() {+ await login();+ const runs = [];+ for (let n = 1; n <= RUNS; n++) { log(`run ${n} of ${RUNS}`); runs.push(await run(n)); }+ const pct = (a, b) => (b ? `${a}/${b} (${Math.round((100 * a) / b)}%)` : "–");+ const col = (f) => runs.map((r) => String(f(r)).padStart(16)).join(" ");+ const line = (label, f) => console.log(`${label.padEnd(52)} ${col(f)}`);+ console.log(`${"".padEnd(52)} ${runs.map((_, i) => `run ${i + 1}`.padStart(16)).join(" ")}`);+ line("statements in the base", (r) => r.statements);+ line("sentences found (any statement from the sentence)", (r) => pct(r.rows.filter((x) => x.found).length, r.rows.length));+ line("time expressions recorded as time words", (r) => pct(r.rows.filter((x) => x.timed && x.recorded).length, r.rows.filter((x) => x.timed).length));+ for (const kind of ["absolute", "relative_now", "relative_event", "section"])+ line(` start right: ${kind}`, (r) => pct(r.rows.filter((x) => x.kind === kind && x.right).length, r.rows.filter((x) => x.kind === kind).length));+ line(" undated event as anchor: no start written", (r) => pct(r.rows.filter((x) => x.kind === "relative_undated_event" && x.right).length, r.rows.filter((x) => x.kind === "relative_undated_event").length));+ line("no time words: no start written", (r) => pct(r.rows.filter((x) => x.kind === "none" && x.found && !x.dated).length, r.rows.filter((x) => x.kind === "none" && x.found).length));+ line("no time words: as-of right", (r) => pct(r.rows.filter((x) => x.asOf === true).length, r.rows.filter((x) => x.asOf !== null).length));+ if (LOG) {+ line("model calls", (r) => r.tokens.calls);+ line("tokens (prompt + completion)", (r) => r.tokens.prompt + r.tokens.completion);+ line("tokens per chunk", (r) => Math.round((r.tokens.prompt + r.tokens.completion) / Math.max(1, r.chunks)));+ }+ // 几遍之间一不一样:每句话的(抽到、记下、对)三样都相同才算一致+ const keys = runs[0].rows.map((x) => `${x.doc}|${x.key}`);+ const same = keys.filter((k, i) => runs.every((r) => ["found", "recorded", "right"].every((f) => r.rows[i][f] === runs[0].rows[i][f]))).length;+ console.log(`${"sentences with the same outcome in every run".padEnd(52)} ${pct(same, keys.length)}`);+ console.log("\nper sentence (runs left to right; ✓ start right, · not):");+ keys.forEach((k, i) => console.log(` ${runs.map((r) => (r.rows[i].right ? "✓" : "·") + (r.rows[i].recorded ? "w" : "-") + (r.rows[i].found ? "" : "?")).join(" ")} [${runs[0].rows[i].kind}] ${k} → ${runs.map((r) => r.rows[i].got || "none").join(" | ")}`));+ if (args.out) fs.writeFileSync(args.out, JSON.stringify(runs, null, 1));+}+main().catch((e) => { console.error(e); process.exit(1); });diff --git a/scripts/bench/truth/timewords.json b/scripts/bench/truth/timewords.jsonnew file mode 100644index 00000000..2b59a6b5--- /dev/null+++ b/scripts/bench/truth/timewords.json@@ -0,0 +1,66 @@+{+ "note": "Documents written for the time bench (ADR 0064). Each expectation names a sentence by a substring of it (key), the kind of time it carries, and the interval its start must fall in. kind none: the statement must have no start. as_of: the date the statement is known at, with the name of the context entry that gives it.",+ "docs": [+ {+ "filename": "zh-weekly.md",+ "text": "# 码表项目周报\n\n提报日期:2026年9月4日\n\n## 计划\n码表二代将在三个月后发布。\n上周,码表一代在日本市场开售。\n去年,码表一代进入北美市场。\n明年第一季度,码表三代启动立项。\n\n## 回顾\n码表一代于2024年3月在欧洲上市。上市半年后,码表一代销量突破十万台。\n东莞工厂完成验厂后两个月,码表二代开始试产。\n\n## 归属\n码表所属市场为全球。码表所属BU为户外BU。\n",+ "expect": [+ { "key": "三个月后发布", "kind": "relative_now", "from": ["2026-12-01", "2026-12-31"] },+ { "key": "在日本市场开售", "kind": "relative_now", "from": ["2026-08-24", "2026-09-03"] },+ { "key": "进入北美市场", "kind": "relative_now", "from": ["2025-01-01", "2025-12-31"] },+ { "key": "三代启动立项", "kind": "relative_now", "from": ["2027-01-01", "2027-03-31"] },+ { "key": "在欧洲上市", "kind": "absolute", "from": ["2024-03-01", "2024-03-31"] },+ { "key": "销量突破十万台", "kind": "relative_event", "from": ["2024-09-01", "2024-09-30"] },+ { "key": "开始试产", "kind": "relative_undated_event", "from": null },+ { "key": "所属市场为全球", "kind": "none", "as_of": ["2026-09-04", "2026-09-04"], "as_of_name": "提报日期" },+ { "key": "所属BU为户外BU", "kind": "none", "as_of": ["2026-09-04", "2026-09-04"], "as_of_name": "提报日期" }+ ]+ },+ {+ "filename": "zh-minutes.md",+ "text": "# 码表产品评审会议纪要\n\n会议时间:2026年8月20日\n纪要发出:2026年8月22日\n下次评审:2026年9月15日\n\n## 决议\n会议决定下个月启动码表二代试产。\n码表固件负责人为李雷。\n码表二代采用多模GNSS方案。\n\n## 背景\n两年前,码表一代完成立项。\n",+ "expect": [+ { "key": "下个月启动码表二代试产", "kind": "relative_now", "from": ["2026-09-01", "2026-09-30"] },+ { "key": "完成立项", "kind": "relative_now", "from": ["2024-01-01", "2024-12-31"] },+ { "key": "固件负责人为李雷", "kind": "none", "as_of": ["2026-08-20", "2026-08-20"], "as_of_name": "会议时间" },+ { "key": "采用多模GNSS方案", "kind": "none", "as_of": ["2026-08-20", "2026-08-20"], "as_of_name": "会议时间" }+ ]+ },+ {+ "filename": "zh-report.md",+ "text": "# 码表品类调研报告\n\n提报日期:2026年9月4日\n数据统计截止:2026年8月31日\n\n## 2025年第三季度\n码表销量为12万台。码表退货率为百分之二。\n\n## 供应链\n关键物料交期为8至12周。码表由东莞工厂代工。\n2026年第二季度起,码表主板改用国产芯片。\n",+ "expect": [+ { "key": "销量为12万台", "kind": "section", "from": ["2025-07-01", "2025-09-30"] },+ { "key": "退货率为百分之二", "kind": "section", "from": ["2025-07-01", "2025-09-30"] },+ { "key": "主板改用国产芯片", "kind": "absolute", "from": ["2026-04-01", "2026-06-30"] },+ { "key": "关键物料交期", "kind": "none", "as_of": ["2026-08-31", "2026-09-04"], "as_of_name": "数据统计截止" },+ { "key": "由东莞工厂代工", "kind": "none", "as_of": ["2026-08-31", "2026-09-04"], "as_of_name": "数据统计截止" }+ ]+ },+ {+ "filename": "en-release.md",+ "text": "# Acme to acquire Bolt\n\nAUSTIN, March 4, 2011\n\nAcme Corporation today announced the acquisition of Bolt Systems. The deal is expected to close in two months.\n\nBolt Systems was founded in 1998. Last year Acme opened an office in Lyon. Three years after its founding, Bolt Systems released its first compiler.\n\nAcme Corporation is headquartered in Austin. Bolt Systems employs 240 people.\n",+ "expect": [+ { "key": "announced the acquisition", "kind": "relative_now", "from": ["2011-03-04", "2011-03-04"] },+ { "key": "expected to close in two months", "kind": "relative_now", "from": ["2011-05-01", "2011-05-31"] },+ { "key": "was founded in 1998", "kind": "absolute", "from": ["1998-01-01", "1998-12-31"] },+ { "key": "opened an office in Lyon", "kind": "relative_now", "from": ["2010-01-01", "2010-12-31"] },+ { "key": "released its first compiler", "kind": "relative_event", "from": ["2001-01-01", "2001-12-31"] },+ { "key": "is headquartered in Austin", "kind": "none", "as_of": ["2011-03-04", "2011-03-04"], "as_of_name": "dateline" },+ { "key": "employs 240 people", "kind": "none", "as_of": ["2011-03-04", "2011-03-04"], "as_of_name": "dateline" }+ ]+ },+ {+ "filename": "en-report.md",+ "text": "# Orion programme status\n\nPublished: August 12, 2025\nData as of: June 30, 2025\n\n## Figures\nOrion has 1,200 active users. Orion runs in three regions.\n\n## Plan\nNext quarter the team will launch Orion Mobile. Two weeks ago the team completed the security audit.\n\n## History\nOrion entered beta in January 2024. Six months later Orion reached general availability.\n",+ "expect": [+ { "key": "will launch Orion Mobile", "kind": "relative_now", "from": ["2025-10-01", "2025-12-31"] },+ { "key": "completed the security audit", "kind": "relative_now", "from": ["2025-07-20", "2025-08-05"] },+ { "key": "entered beta in January 2024", "kind": "absolute", "from": ["2024-01-01", "2024-01-31"] },+ { "key": "reached general availability", "kind": "relative_event", "from": ["2024-07-01", "2024-07-31"] },+ { "key": "1,200 active users", "kind": "none", "as_of": ["2025-06-30", "2025-08-12"], "as_of_name": "Data as of" },+ { "key": "runs in three regions", "kind": "none", "as_of": ["2025-06-30", "2025-08-12"], "as_of_name": "Data as of" }+ ]+ }+ ]+}