Skip to content

World's first open-source enterprise world model.

The typed bench waits on the right jobs, scores what errata adds, and logs usage once per reply (#893)

WaylandYangf467f189b75b
diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rs
index a9a8d97f..e1bf5696 100644
--- a/crates/utopia-llm/src/lib.rs
+++ b/crates/utopia-llm/src/lib.rs
@@ -544,6 +544,14 @@ impl LlmClient {
if !saw_frame {
anyhow::bail!("LLM stream carried no frames");
}
+ if let Some(u) = usage {
+ tracing::info!(
+ model = %self.model,
+ prompt = u.prompt_tokens,
+ completion = u.completion_tokens,
+ "llm usage"
+ );
+ }
// 端点开口了又半路没了:拼到一半的回复长得像成功,不做成错误就会被当成
// 模型给的全部答案
if !ended {
@@ -593,8 +601,9 @@ impl LlmClient {
*finish_reason = Some(reason.to_string());
}
// 用量只在最后一帧(choices 为空)出现
+ // 有的网关每一帧都带累计用量:这里只记下来,流结束时记一次日志,否则一次调用
+ // 在日志里成了几百行「用量」,按行加总的人会把 token 高估几百倍
if !v["usage"].is_null() {
- log_usage(&self.model, &v);
let u = &v["usage"];
*usage = Some(Usage {
prompt_tokens: u["prompt_tokens"].as_u64().unwrap_or(0),
diff --git a/scripts/bench/README.md b/scripts/bench/README.md
index 0d8cae27..95f72bfc 100644
--- a/scripts/bench/README.md
+++ b/scripts/bench/README.md
@@ -517,3 +517,31 @@ node scripts/bench/typed.mjs --kb <id> --score # 只重新打
(裁判抽样;金标漏标严重,精度只信裁判)、entity-pair recall(开放陈述那一层)、绑定的
bound / none / undecided。0044 的门槛:裁判精度不低于完整原型的 75.3%,同句召回不低于完整原型,
两轮各报一次,每篇文档的 token 不到原型的五分之一。
+
+### 第一次真跑(2026-09-24,gemini-3.5-flash 经 apexin.net,两组各一个新库)
+
+裁判与抽取是同一个模型,精度要打折看。`--approve-rules` 替审核人把对齐器提的规则全批了,
+量的是蕴含机制的上限;真人会驳回一部分。
+
+| | 组 1 | 组 2 |
+|---|---|---|
+| 开放陈述 / 实体 | 1855 / 1854 | 1778 / — |
+| 签名 bound / none / undecided | 214 / 927 / 48 | 409 / 1074 / 70(打分中又跑了一轮对齐,从 312 涨到 409) |
+| 类型化(按绑定) | 358 | 411 → 629(第二轮对齐后) |
+| gold recall 勘误前(同句 / 跨句) | 6.0%(8.7% / 3.7%) | 6.5%(9.6% / 4.3%) |
+| 规则:提 / 批 → 读数 / 隐含行 | 112 / 112 → 88 / 75 | 159 / 159 → 146 / 164 |
+| gold recall 批规则后、勘误前 | 6.6%(9.4% / 4.1%) | 9.6%(13.7% / 6.2%) |
+| 裁判精度 勘误前(200 抽样) | 96.9% | 92.5% |
+| 勘误:看 / 撤 / 改 / 加 / 拒 | 429 / 53 / 24 / 463 / 49 | 963 / 135 / 97 / 1146 / 124(两遍累计) |
+| 撤改的行里裁判判 stated | 34 / 37 | 72 / 110(另 34 条 misworded) |
+| gold recall 勘误后 | 11.9%(17.7% / 7.0%) | 17.1%(24.6% / 10.7%) |
+| 裁判精度 勘误后(全量) | 96.7%(547 条) | 96.4%(700 条) |
+| entity-pair recall(开放层) | 15.0% | 13.7% |
+
+读数:精度远高于原型勘误前的 75.3%;召回远低于原型(同句 8.7% 与 9.6% 对比原型整体 15.3%)。短板在对齐的
+覆盖:1189 条签名里 927 条判「无」,P17(国家)565 条金标只中 1 条,P27 国籍、P150 下辖为 0——这些
+属性靠蕴含规则,而规则要人批;全批之后隐含 75 行,召回只涨 0.6 个点,因为读数只答出 36 / 88。勘误
+这一刀的账与原型相反:撤掉的 37 条里 34 条裁判认为原文说了(撤错),加的 300 多条精度与其余持平
+(全量 96.7%),召回因此从 6.0% 到 11.9%。每篇 token 这一轮量不出:apexin 的流每一帧都带累计用量,
+服务端按帧记了日志;从这个 PR 起每次回复只记一行,下一轮能按阶段加总。
+
diff --git a/scripts/bench/typed.mjs b/scripts/bench/typed.mjs
index 747a7428..b794cd0a 100644
--- a/scripts/bench/typed.mjs
+++ b/scripts/bench/typed.mjs
@@ -22,6 +22,7 @@
// node scripts/bench/typed.mjs --label dry --dry-run # 建库、装本体、灌语料、等解析,不抽取:验管线
// node scripts/bench/typed.mjs --label run1 --judge 200 --errata # 对齐之后再跑勘误 agent,报前后两份分与撤错多少
// node scripts/bench/typed.mjs --kb <id> --score --errata --judge 200 # 已有的库:跑勘误、打分
+// node scripts/bench/typed.mjs --kb <id> --score --approve-rules # 替审核人批下全部蕴含规则、读数、物化,再打分
// 环境:BENCH_BASE(默认 http://localhost:1516)、BENCH_EMAIL / BENCH_PASSWORD(lib.mjs)、
// BENCH_PSQL(指向应用库的 psql 命令行)、BENCH_JUDGE_BASE / _KEY / _MODEL(裁判端点)
@@ -143,7 +144,10 @@ async function extract(KB) {
await until(() => {
const done = num(live.replace("count(*)", "count(c.extracted_at)"));
const total = num(live);
- const left = num(`SELECT count(*) FROM jobs WHERE kind IN ('extract_document','process_document') AND status IN ('queued','running') AND payload->>'kb_id'='${KB}'`);
+ // 抽取任务的 payload 只带 document_id(没有 kb_id):按文档所属的库数,否则这里永远是 0,
+ // 台子会在第一篇抽完时就去排对齐,对齐跑在半截语料上
+ const left = num(`SELECT count(*) FROM jobs j WHERE j.kind IN ('extract_document','process_document') AND j.status IN ('queued','running')
+ AND (j.payload->>'kb_id'='${KB}' OR j.payload->>'document_id' IN (SELECT id::text FROM documents WHERE kb_id='${KB}'))`);
log(`抽取:${done}/${total} 块,${left} 个任务`);
return left === 0 && done > 0 ? true : done;
}, 15000, 15 * 60000);
@@ -154,16 +158,38 @@ async function align(KB) {
// 类别词先绑到类,短语签名才定型;类别词对齐结束时自己会排短语对齐,短语对齐结束时物化
psql(`INSERT INTO jobs (kind, payload) VALUES ('align_types', '{"kb_id":"${KB}"}')`);
await until(() => {
- const left = num(`SELECT count(*) FROM jobs WHERE kind IN ('align_types','align_phrases','materialize_typed') AND status IN ('queued','running') AND payload->>'kb_id'='${KB}'`);
+ const left = num(`SELECT count(*) FROM jobs WHERE kind IN ('align_types','align_phrases','materialize_typed','read_phrases') AND status IN ('queued','running') AND payload->>'kb_id'='${KB}'`);
const bound = num(`SELECT count(*) FROM phrase_bindings WHERE kb_id='${KB}' AND status='bound'`);
+ const decided = num(`SELECT count(*) FROM phrase_bindings WHERE kb_id='${KB}'`) + num(`SELECT count(*) FROM type_bindings WHERE kb_id='${KB}'`);
+ // 签名判完之后还有一段提规则(0044 决定 3 第五片):每条判定过的签名问一次,规则行慢慢增加,
+ // 绑定数却不再动——进展得把它也算上,否则这一段会被当成卡死
+ const rules = num(`SELECT count(*) FROM implication_rules WHERE kb_id='${KB}'`);
const typed = num(`SELECT count(*) FROM facts WHERE kb_id='${KB}' AND layer='typed' AND from_statement_id IS NOT NULL AND invalidated_at IS NULL`);
- log(`对齐:${left} 个任务在跑,绑定 ${bound},类型化 ${typed}`);
- return left === 0 && num(`SELECT count(*) FROM phrase_bindings WHERE kb_id='${KB}'`) > 0 ? true : bound * 100000 + typed;
- }, 15000, 20 * 60000);
+ log(`对齐:${left} 个任务在跑,判定 ${decided},绑定 ${bound},规则 ${rules},类型化 ${typed}`);
+ // `until` 只把数字当进展:四个数拼成一个单调的数
+ return left === 0 && num(`SELECT count(*) FROM phrase_bindings WHERE kb_id='${KB}'`) > 0 ? true : decided * 1e9 + bound * 1e6 + rules * 1e3 + typed;
+ }, 15000, 30 * 60000);
const failed = num(`SELECT count(*) FROM jobs WHERE kind IN ('align_types','align_phrases','materialize_typed') AND status='failed' AND payload->>'kb_id'='${KB}'`);
if (failed) log(`注意:${failed} 个对齐任务失败(看 jobs.last_error)`);
}
+// --approve-rules:替审核人把对齐器提的蕴含规则全批了(0044 决定 3 第五片的上限:真人会驳回一部分),
+// 排读数任务(读数完了自己排物化),等隐含行算出来。不带这个开关时提议只是躺在队列里,隐含行为 0
+async function approveRules(KB) {
+ const n = num(`SELECT count(*) FROM implication_rules WHERE kb_id='${KB}' AND status='proposed'`);
+ psql(`UPDATE implication_rules SET status='approved', decided_by='person', decided_at=now() WHERE kb_id='${KB}' AND status='proposed'`);
+ psql(`INSERT INTO jobs (kind, payload) VALUES ('read_phrases', '{"kb_id":"${KB}"}')`);
+ log(`批了 ${n} 条规则,排读数与物化`);
+ await until(() => {
+ const left = num(`SELECT count(*) FROM jobs WHERE kind IN ('read_phrases','materialize_typed') AND status IN ('queued','running') AND payload->>'kb_id'='${KB}'`);
+ const read = num(`SELECT count(*) FROM phrase_readings WHERE kb_id='${KB}'`);
+ const implied = num(`SELECT count(*) FROM facts WHERE kb_id='${KB}' AND implied AND invalidated_at IS NULL`);
+ log(`读数:${left} 个任务在跑,缓存 ${read} 条,隐含 ${implied} 条`);
+ return left === 0 ? true : read * 100000 + implied;
+ }, 15000, 30 * 60000);
+ return { approved: n, readings: num(`SELECT count(*) FROM phrase_readings WHERE kb_id='${KB}'`), implied: num(`SELECT count(*) FROM facts WHERE kb_id='${KB}' AND implied AND invalidated_at IS NULL`) };
+}
+
// 勘误(0044 决定 7):物化之后排一次 errata_review,等它把每篇文档看完。度量按 errata_runs 与
// errata_actions 报:撤了几条、改了几条、加了几条、留给人几条、拒了几条、花了多少 token
async function errata(KB) {
@@ -236,23 +262,37 @@ function valueMatches(gold, got) {
return gn.size > 0 && vn.size > 0 && [...gn].some((n) => vn.has(n));
}
-function score(KB) {
- // 每篇文档里的类型化事实:主语名集合、属性键、宾语名集合或值
- const typed = rows(`
+// scope:'after' 是库现在的样子(勘误撤掉的不算、勘误加上的算),'before' 是勘误之前
+// (撤掉的算回来、加上的不算)。0044 的门槛看勘误前,勘误的度量看两者之差
+function score(KB, scope = "after") {
+ // 每篇文档里的类型化事实:主语名集合、属性键、宾语名集合或值。
+ // 名字之间用 ¦ 接:psql -A 的列分隔符是 |,名字里再用 | 会把列切错(第一轮就是这么错到 0 的)
+ const rowsOf = rows(`
SELECT d.filename, r.key,
- s.canonical_name || '|' || coalesce((SELECT string_agg(n.object_value->>'value','|') FROM facts n JOIN relation_types nr ON nr.id=n.predicate_id
+ s.canonical_name || '¦' || coalesce((SELECT string_agg(n.object_value->>'value','¦') FROM facts n JOIN relation_types nr ON nr.id=n.predicate_id
WHERE n.subject_id=s.id AND nr.key='known_as' AND n.invalidated_at IS NULL AND n.object_value IS NOT NULL),''),
- coalesce(o.canonical_name || '|' || coalesce((SELECT string_agg(n.object_value->>'value','|') FROM facts n JOIN relation_types nr ON nr.id=n.predicate_id
+ coalesce(o.canonical_name || '¦' || coalesce((SELECT string_agg(n.object_value->>'value','¦') FROM facts n JOIN relation_types nr ON nr.id=n.predicate_id
WHERE n.subject_id=o.id AND nr.key='known_as' AND n.invalidated_at IS NULL AND n.object_value IS NOT NULL),''), ''),
- coalesce(f.object_value->>'value', f.object_value#>>'{}', '')
+ coalesce(f.object_value->>'value', f.object_value#>>'{}', ''),
+ EXISTS (SELECT 1 FROM errata_actions ea WHERE ea.fact_id=f.id AND ea.action IN ('retract','revise') AND ea.status='applied') AS removed,
+ -- 只算「只有勘误这一个来源」的行:加的事实撞上已有的行时 insert_fact_on 复用那一行,它仍是陈述算出来的
+ (f.from_statement_id IS NULL AND NOT f.implied
+ AND EXISTS (SELECT 1 FROM errata_actions ea WHERE ea.new_fact_id=f.id AND ea.status='applied')) AS added
FROM facts f
JOIN relation_types r ON r.id=f.predicate_id
JOIN entities s ON s.id=f.subject_id
LEFT JOIN entities o ON o.id=f.object_id
JOIN LATERAL (SELECT document_id FROM fact_evidence WHERE fact_id=f.id LIMIT 1) ev ON true
JOIN documents d ON d.id=ev.document_id
- WHERE f.kb_id='${KB}' AND f.layer='typed' AND f.from_statement_id IS NOT NULL AND f.invalidated_at IS NULL`
- ).map(([file, key, subj, obj, val]) => ({ file, key: key.toUpperCase(), subj: subj.split("|").map(norm).filter(Boolean), obj: obj.split("|").map(norm).filter(Boolean), val }));
+ WHERE f.kb_id='${KB}' AND f.layer='typed' AND NOT r.builtin
+ AND (f.from_statement_id IS NOT NULL OR f.implied
+ OR EXISTS (SELECT 1 FROM errata_actions ea WHERE ea.new_fact_id=f.id AND ea.status='applied'))
+ AND (f.invalidated_at IS NULL
+ OR EXISTS (SELECT 1 FROM errata_actions ea WHERE ea.fact_id=f.id AND ea.action IN ('retract','revise') AND ea.status='applied'))`
+ );
+ const typed = rowsOf
+ .map(([file, key, subj, obj, val, removed, added]) => ({ file, key: key.toUpperCase(), subj: subj.split("¦").map(norm).filter(Boolean), obj: obj.split("¦").map(norm).filter(Boolean), val, removed: removed === "t", added: added === "t" }))
+ .filter((t) => (scope === "before" ? !t.added : !t.removed));
// 开放陈述连上的实体对(抽取那一层)
const open = rows(`
SELECT d.filename, s.canonical_name, coalesce(o.canonical_name, f.object_value->>'value', '')
@@ -286,7 +326,7 @@ function score(KB) {
}
const pct = (a, b) => (b ? (100 * a / b).toFixed(1) + "%" : "-");
const result = {
- label: LABEL, kb: KB, corpus: CORPUS, at: new Date().toISOString(),
+ label: LABEL, kb: KB, corpus: CORPUS, scope, at: new Date().toISOString(),
typed_facts: typed.length, open_statements: open.length,
gold, gold_recall: hit / (gold || 1), gold_recall_same_sentence: hitSame / (goldSame || 1),
gold_recall_cross_sentence: (hit - hitSame) / ((gold - goldSame) || 1),
@@ -330,7 +370,9 @@ async function judge(KB, n, seed) {
SELECT f.id, d.filename, s.canonical_name, r.label, coalesce(o.canonical_name, f.object_value->>'value', f.object_value#>>'{}', '')
FROM facts f JOIN relation_types r ON r.id=f.predicate_id JOIN entities s ON s.id=f.subject_id LEFT JOIN entities o ON o.id=f.object_id
JOIN LATERAL (SELECT document_id FROM fact_evidence WHERE fact_id=f.id LIMIT 1) ev ON true JOIN documents d ON d.id=ev.document_id
- WHERE f.kb_id='${KB}' AND f.layer='typed' AND f.from_statement_id IS NOT NULL AND f.invalidated_at IS NULL ORDER BY f.id`);
+ WHERE f.kb_id='${KB}' AND f.layer='typed' AND NOT r.builtin AND f.invalidated_at IS NULL
+ AND (f.from_statement_id IS NOT NULL OR f.implied
+ OR EXISTS (SELECT 1 FROM errata_actions ea WHERE ea.new_fact_id=f.id AND ea.status='applied')) ORDER BY f.id`);
// 可复现抽样
let a = (seed >>> 0) || 1; const rnd = () => { a = (a * 1103515245 + 12345) >>> 0; return a / 4294967296; };
const sample = all.map((r) => [rnd(), r]).sort((x, y) => x[0] - y[0]).slice(0, n).map(([, r]) => r);
@@ -365,13 +407,17 @@ if (!KB) {
} else {
await login();
}
-const result = score(KB);
+let rules = null;
+if (args["approve-rules"]) rules = await approveRules(KB);
+// 勘误前的分:已经跑过勘误的库也能按 scope 算回来(撤掉的算回来、加上的不算)
+const result = score(KB, "before");
+if (rules) result.rules = rules;
if (args.judge) result.judge = await judge(KB, Number(args.judge) || 200, Number(args.seed || 1));
if (args.errata) {
// 勘误前的分留着,勘误后再打一次:0044 §7 的度量是两份分的差,与撤错了多少
result.before_errata = { gold_recall: result.gold_recall, gold_recall_same_sentence: result.gold_recall_same_sentence, typed_facts: result.typed_facts, judge: result.judge };
result.errata = await errata(KB);
- const after = score(KB);
+ const after = score(KB, "after");
result.after_errata = { gold_recall: after.gold_recall, gold_recall_same_sentence: after.gold_recall_same_sentence, typed_facts: after.typed_facts };
if (args.judge) {
result.after_errata.judge = await judge(KB, Number(args.judge) || 200, Number(args.seed || 1));