Skip to content

World's first open-source enterprise world model.

The bench extracts each document once, and the table rules ride only with a passage that has a table (#1000)

WaylandYang701d6ea3fcd7
diff --git a/crates/utopia-extract/src/open.rs b/crates/utopia-extract/src/open.rs
index 6ba87705..eaa70da8 100644
--- a/crates/utopia-extract/src/open.rs
+++ b/crates/utopia-extract/src/open.rs
@@ -149,15 +149,7 @@ its units, a percentage, an amount, a count, a title, a status. phrase is how th
what is measured or stated (\"area\", \"was completed\", \"joined\", \"占地面积\"); value is the \
literal as written. Wording that names or describes something is not a value: that something \
goes in \"e\" and is linked by a statement with an object. \"1,200 beekeepers joined\" is the \
-thing \"beekeepers\" with \"joined\" = \"1,200\". Write every figure the passage states. In a \
-table, a cell is a statement about its row's thing whose phrase is the column heading. When \
-the column heading names one time (a date, a quarter, a period), it is when instead, the \
-statement is about the thing the table's caption names (the company of a financial statement) \
-and the phrase is the row label as written, path included (\"Operating expenses › Research and \
-development\"); a heading that names a comparison between periods (a change from the quarter \
-before, a change from a year ago) is not a time, it stays the phrase; a unit the caption or a \
-heading gives (\"In millions\") is a qualifier on each such statement. Leave out cells whose \
-column heading is not in the passage.\n\
+thing \"beekeepers\" with \"joined\" = \"1,200\". Write every figure the passage states.{TABLE}\n\
A list is one statement per member, and so is a subject or object that joins several things \
(\"the city and the county funded the bridge\" is two statements).\n\
3. Nothing in a sentence is dropped. When more than two things take part, or the link carries \
@@ -214,6 +206,47 @@ $2 million on March 4, 2011.\" gives:\n\
\"n\": [],\n\
\"t\": []}";
+/// 表格的读法,只在这一段里有表格时才进系统消息(见 [`has_table`]):没有表格的段落
+/// 用不上它,而它占系统消息的一成不到
+const OPEN_TABLE_RULES: &str = "\
+ In a \
+table, a cell is a statement about its row's thing whose phrase is the column heading. When \
+the column heading names one time (a date, a quarter, a period), it is when instead, the \
+statement is about the thing the table's caption names (the company of a financial statement) \
+and the phrase is the row label as written, path included (\"Operating expenses › Research and \
+development\"); a heading that names a comparison between periods (a change from the quarter \
+before, a change from a year ago) is not a time, it stays the phrase; a unit the caption or a \
+heading gives (\"In millions\") is a qualifier on each such statement. Leave out cells whose \
+column heading is not in the passage.";
+
+/// 这一段里有没有表格:Markdown 的竖线、HTML 的单元格、制表符,或者一行里用成串空格
+/// 隔开的三栏以上。宁可多认:认错了只是多带一段规则,和从前一样
+fn has_table(passage: &str) -> bool {
+ let lower = passage.to_ascii_lowercase();
+ passage.contains('|')
+ || passage.contains('\t')
+ || lower.contains("<td")
+ || lower.contains("<th")
+ || lower.contains("<tr")
+ || passage.lines().any(|l| {
+ l.trim()
+ .split(" ")
+ .filter(|c| !c.trim().is_empty())
+ .count()
+ >= 3
+ })
+}
+
+/// 这一段的系统消息:规则不变,只有表格的那几句按段落带或不带
+fn system_for(passage: &str) -> String {
+ let table = if has_table(passage) {
+ format!(" {OPEN_TABLE_RULES}")
+ } else {
+ String::new()
+ };
+ OPEN_SYSTEM.replace("{TABLE}", &table)
+}
+
/// 构造开放抽取的两条消息:常量系统消息 + `Document:` / 开头 / 已知实体 / `Passage:`。
///
/// 没有文档日期、没有类型与关系清单、没有属性——这些都不进提示词。开头与已知实体
@@ -233,7 +266,7 @@ pub fn build_open_messages(
vec![
ChatMessage {
role: "system".into(),
- content: OPEN_SYSTEM.to_string(),
+ content: system_for(chunk_text),
},
ChatMessage {
role: "user".into(),
@@ -765,6 +798,28 @@ mod tests {
/// 提示词里没有文档日期、没有类型与关系清单、没有编号与句柄;有文件名、开头、
/// 已知名字与正文,并且按这个顺序
+ #[test]
+ fn the_table_rules_ride_only_with_a_passage_that_has_a_table() {
+ let plain = "The council met on March 4, 2011. It awarded the paving contract.";
+ let system = &build_open_messages("a.md", &[], None, plain)[0].content;
+ assert!(!system.contains("In a table"));
+ assert!(!system.contains("{TABLE}"));
+ assert!(system.contains("Write every figure the passage states.\n"));
+ for table in [
+ "| Item | Q3 2025 |\n|---|---|\n| Revenue | 12 |",
+ "<table><tr><td>Revenue</td><td>12</td></tr></table>",
+ "Revenue\t12\t14",
+ "Revenue 12 14\nCosts 7 8",
+ ] {
+ let system = &build_open_messages("a.md", &[], None, table)[0].content;
+ assert!(
+ system.contains("Write every figure the passage states. In a table, a cell is"),
+ "{table}"
+ );
+ assert!(system.contains("column heading is not in the passage.\n"));
+ }
+ }
+
#[test]
fn the_prompt_carries_no_ontology_and_no_document_date() {
let msgs = build_open_messages(
diff --git a/scripts/bench/README.md b/scripts/bench/README.md
index 0f5aba1a..10b91137 100644
--- a/scripts/bench/README.md
+++ b/scripts/bench/README.md
@@ -864,3 +864,41 @@ bound / none / undecided。0044 的门槛:裁判精度不低于完整原型的
十条上下不再进人的队列,`votes.reason = "first_vote_none"` 查得到。
- 20 篇整轮(只有分批):合计 82.1 万 → 62.6 万,每篇 4.1 万 → 3.1 万;其中约一半来自分批,其余是
这一轮抽出的签名少、重问少。
+
+### 第七次真跑:每篇只抽一遍,表格的规则只随表格带(2026-09-28,dev @ 7d73ff9,gemini-3.5-flash,20 篇)
+
+**台子从前让每篇抽两遍。** 配了对话模型的库,文档解析完管线自己排抽取;`typed.mjs` 和 `timewords.mjs`
+随后又对每篇调一次手动抽取,而手动抽取是强制全量(解雇在跑的任务、从头再抽)。20 篇 24 个分块,抽取调用
+45 次。现在只补管线没排上的。这一节之前的每篇 token 都多算了将近一遍抽取,召回也可能受它影响(在跑的
+那一遍被中途顶替):这一轮同句召回 20.1%,实体对召回 18.0%,只有 20 篇,要在 100 篇上再量。
+
+| 调用 | 次数 | 每次输入 | 每次输出 | 每篇 |
+|---|---|---|---|---|
+| 短语对齐 | 38 | 3283 | 240 | 6695 |
+| 抽取 | 26 | 2517 | 1152 | 4770 |
+| 类别词对齐 | 24 | 2947 | 647 | 4314 |
+| 提规则 | 14 | 4061 | 122 | 2929 |
+| 时间解释 | 20 | 1770 | 141 | 1912 |
+| 本体代理 | 5 | 4966 | 2021 | 1747 |
+| 实体裁决 | 9 | 2504 | 503 | 1354 |
+| 提问题 | 1 | 7603 | 1466 | 453 |
+
+合计每篇 24175,输入占 85%。类型化 138,绑定 105 / 98 / 27,裁判精度 79.7%。
+
+**抽取的系统消息约 2380 token,一篇 Re-DocRED 文档约 270。** 试了三处删减,每处单独量:31 个没有表格
+的段落(这 20 篇的 24 个分块加时间台子的 7 篇),每种系统消息各抽三遍,回复用服务端的解析器读,对金标
+的实体对和时间台子的期望打分:
+
+| 系统消息 | 每次输入 | 金标实体对(三遍) | 同句(三遍) | 时间词记下 | 修复后只剩一条陈述的回复 |
+|---|---|---|---|---|---|
+| 原样 | 2488 | 27.0 / 27.6 / 28.6% | 42.6 / 44.6 / 47.0% | 28 / 28 / 27 | 0 / 93 |
+| 不带表格的规则 | 2339 | 28.6 / 28.4 / 28.6% | 47.0 / 46.2 / 46.2% | 27 / 27 / 27 | 0 / 93 |
+| 再去掉规则 6(引文的位置,规则 2 与 7 说过) | 2296 | 29.0 / 27.4 / 26.5% | 49.4 / 47.4 / 43.4% | 28 / 27 / 28 | 3 / 93 |
+| 再去掉结尾的例子 | 2182 | 27.9 / 23.3 / 27.6% | 44.2 / 41.4 / 45.0% | 28 / 29 / 28 | 0 / 93 |
+
+- 表格的规则只随有表格的段落带:没有表格的段落上分数不降,有表格的段落系统消息与从前逐字相同。落地。
+- 规则 6 重复,却在管格式:去掉之后 93 次里 3 次括号写错,解析器修到最后一个完整对象,整段只剩一条
+ 陈述。不去。
+- 结尾的例子:三遍里一遍实体对掉到 23.3%,解析没有出错。分不清是波动还是损失,不去。
+- 同一条系统消息、温度 0,两遍抽出的陈述(主语、短语、宾语)只有六成相同。逐条比对陈述量不出提示词的
+ 改动,要用有标准答案的指标。
diff --git a/scripts/bench/timewords.mjs b/scripts/bench/timewords.mjs
index 4abcb94c..11a8a77b 100644
--- a/scripts/bench/timewords.mjs
+++ b/scripts/bench/timewords.mjs
@@ -38,7 +38,10 @@ async function run(n) {
if (!r.ok) throw new Error(`upload ${d.filename} -> ${r.status} ${(await r.text()).slice(0, 200)}`);
}
for (let i = 0; i < 60; i++) { await sleep(3000); if (q(`SELECT count(*) FROM documents WHERE kb_id='${KB}' AND status<>'ready'`) === "0") break; }
- for (const d of (await api("GET", `/api/v1/kbs/${KB}/documents?limit=50`)).docs) await api("POST", `/api/v1/documents/${d.id}/extract`, {});
+ // 管线解析完自己排抽取;手动抽取是强制全量,只补没排上的,否则每篇抽两遍
+ for (const d of (await api("GET", `/api/v1/kbs/${KB}/documents?limit=50`)).docs) {
+ if (!["queued", "extracting", "done"].includes(d.graph_status)) await api("POST", `/api/v1/documents/${d.id}/extract`, {});
+ }
for (let i = 0; i < 240; i++) {
await sleep(5000);
const pending = q(`SELECT count(*) FROM jobs WHERE status IN ('queued','running') AND kind IN ('extract_document','resolve_time','process_document') AND (payload->>'kb_id'='${KB}' OR payload->>'document_id' IN (SELECT id::text FROM documents WHERE kb_id='${KB}'))`);
diff --git a/scripts/bench/typed.mjs b/scripts/bench/typed.mjs
index f1b1a01b..95c1bdc3 100644
--- a/scripts/bench/typed.mjs
+++ b/scripts/bench/typed.mjs
@@ -176,8 +176,12 @@ async function extract(KB) {
// 温库第二批:只排这一批的文档,老文档的图不动
const mine = new Set(corpus.docs.map((d) => d.filename));
const docs = (await api("GET", `/api/v1/kbs/${KB}/documents?limit=500`)).docs.filter((d) => !INTO || mine.has(d.filename));
- for (const d of docs) await api("POST", `/api/v1/documents/${d.id}/extract`, {});
- log(`排队抽取 ${docs.length} 篇`);
+ // 配了对话模型的库,文档解析完管线自己排抽取。这里只补没排上的:手动抽取是强制全量
+ // (解雇在跑的任务、从头再抽),对每篇都调一次就是每篇抽两遍——2026-09-28 之前的每篇
+ // token 都多算了一遍抽取
+ const idle = docs.filter((d) => !["queued", "extracting", "done"].includes(d.graph_status));
+ for (const d of idle) await api("POST", `/api/v1/documents/${d.id}/extract`, {});
+ log(`排队抽取 ${idle.length} 篇(管线已排 ${docs.length - idle.length} 篇)`);
const live = `SELECT count(*) FROM chunks c JOIN documents d ON d.id=c.document_id WHERE d.kb_id='${KB}' AND c.superseded_at IS NULL`;
await until(() => {
const done = num(live.replace("count(*)", "count(c.extracted_at)"));