World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
The typed bench waits on the right jobs, scores what errata adds, and logs usage once per reply (#893)
WaylandYangf467f189b75b
diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rsindex a9a8d97f..e1bf5696 100644--- a/crates/utopia-llm/src/lib.rs+++ b/crates/utopia-llm/src/lib.rs@@ -544,6 +544,14 @@ impl LlmClient {if !saw_frame {anyhow::bail!("LLM stream carried no frames");}+ if let Some(u) = usage {+ tracing::info!(+ model = %self.model,+ prompt = u.prompt_tokens,+ completion = u.completion_tokens,+ "llm usage"+ );+ }// 端点开口了又半路没了:拼到一半的回复长得像成功,不做成错误就会被当成// 模型给的全部答案if !ended {@@ -593,8 +601,9 @@ impl LlmClient {*finish_reason = Some(reason.to_string());}// 用量只在最后一帧(choices 为空)出现+ // 有的网关每一帧都带累计用量:这里只记下来,流结束时记一次日志,否则一次调用+ // 在日志里成了几百行「用量」,按行加总的人会把 token 高估几百倍if !v["usage"].is_null() {- log_usage(&self.model, &v);let u = &v["usage"];*usage = Some(Usage {prompt_tokens: u["prompt_tokens"].as_u64().unwrap_or(0),diff --git a/scripts/bench/README.md b/scripts/bench/README.mdindex 0d8cae27..95f72bfc 100644--- a/scripts/bench/README.md+++ b/scripts/bench/README.md@@ -517,3 +517,31 @@ node scripts/bench/typed.mjs --kb <id> --score # 只重新打(裁判抽样;金标漏标严重,精度只信裁判)、entity-pair recall(开放陈述那一层)、绑定的bound / none / undecided。0044 的门槛:裁判精度不低于完整原型的 75.3%,同句召回不低于完整原型,两轮各报一次,每篇文档的 token 不到原型的五分之一。++### 第一次真跑(2026-09-24,gemini-3.5-flash 经 apexin.net,两组各一个新库)++裁判与抽取是同一个模型,精度要打折看。`--approve-rules` 替审核人把对齐器提的规则全批了,+量的是蕴含机制的上限;真人会驳回一部分。++| | 组 1 | 组 2 |+|---|---|---|+| 开放陈述 / 实体 | 1855 / 1854 | 1778 / — |+| 签名 bound / none / undecided | 214 / 927 / 48 | 409 / 1074 / 70(打分中又跑了一轮对齐,从 312 涨到 409) |+| 类型化(按绑定) | 358 | 411 → 629(第二轮对齐后) |+| gold recall 勘误前(同句 / 跨句) | 6.0%(8.7% / 3.7%) | 6.5%(9.6% / 4.3%) |+| 规则:提 / 批 → 读数 / 隐含行 | 112 / 112 → 88 / 75 | 159 / 159 → 146 / 164 |+| gold recall 批规则后、勘误前 | 6.6%(9.4% / 4.1%) | 9.6%(13.7% / 6.2%) |+| 裁判精度 勘误前(200 抽样) | 96.9% | 92.5% |+| 勘误:看 / 撤 / 改 / 加 / 拒 | 429 / 53 / 24 / 463 / 49 | 963 / 135 / 97 / 1146 / 124(两遍累计) |+| 撤改的行里裁判判 stated | 34 / 37 | 72 / 110(另 34 条 misworded) |+| gold recall 勘误后 | 11.9%(17.7% / 7.0%) | 17.1%(24.6% / 10.7%) |+| 裁判精度 勘误后(全量) | 96.7%(547 条) | 96.4%(700 条) |+| entity-pair recall(开放层) | 15.0% | 13.7% |++读数:精度远高于原型勘误前的 75.3%;召回远低于原型(同句 8.7% 与 9.6% 对比原型整体 15.3%)。短板在对齐的+覆盖:1189 条签名里 927 条判「无」,P17(国家)565 条金标只中 1 条,P27 国籍、P150 下辖为 0——这些+属性靠蕴含规则,而规则要人批;全批之后隐含 75 行,召回只涨 0.6 个点,因为读数只答出 36 / 88。勘误+这一刀的账与原型相反:撤掉的 37 条里 34 条裁判认为原文说了(撤错),加的 300 多条精度与其余持平+(全量 96.7%),召回因此从 6.0% 到 11.9%。每篇 token 这一轮量不出:apexin 的流每一帧都带累计用量,+服务端按帧记了日志;从这个 PR 起每次回复只记一行,下一轮能按阶段加总。+diff --git a/scripts/bench/typed.mjs b/scripts/bench/typed.mjsindex 747a7428..b794cd0a 100644--- a/scripts/bench/typed.mjs+++ b/scripts/bench/typed.mjs@@ -22,6 +22,7 @@// node scripts/bench/typed.mjs --label dry --dry-run # 建库、装本体、灌语料、等解析,不抽取:验管线// node scripts/bench/typed.mjs --label run1 --judge 200 --errata # 对齐之后再跑勘误 agent,报前后两份分与撤错多少// node scripts/bench/typed.mjs --kb <id> --score --errata --judge 200 # 已有的库:跑勘误、打分+// node scripts/bench/typed.mjs --kb <id> --score --approve-rules # 替审核人批下全部蕴含规则、读数、物化,再打分// 环境:BENCH_BASE(默认 http://localhost:1516)、BENCH_EMAIL / BENCH_PASSWORD(lib.mjs)、// BENCH_PSQL(指向应用库的 psql 命令行)、BENCH_JUDGE_BASE / _KEY / _MODEL(裁判端点)@@ -143,7 +144,10 @@ async function extract(KB) {await until(() => {const done = num(live.replace("count(*)", "count(c.extracted_at)"));const total = num(live);- const left = num(`SELECT count(*) FROM jobs WHERE kind IN ('extract_document','process_document') AND status IN ('queued','running') AND payload->>'kb_id'='${KB}'`);+ // 抽取任务的 payload 只带 document_id(没有 kb_id):按文档所属的库数,否则这里永远是 0,+ // 台子会在第一篇抽完时就去排对齐,对齐跑在半截语料上+ const left = num(`SELECT count(*) FROM jobs j WHERE j.kind IN ('extract_document','process_document') AND j.status IN ('queued','running')+ AND (j.payload->>'kb_id'='${KB}' OR j.payload->>'document_id' IN (SELECT id::text FROM documents WHERE kb_id='${KB}'))`);log(`抽取:${done}/${total} 块,${left} 个任务`);return left === 0 && done > 0 ? true : done;}, 15000, 15 * 60000);@@ -154,16 +158,38 @@ async function align(KB) {// 类别词先绑到类,短语签名才定型;类别词对齐结束时自己会排短语对齐,短语对齐结束时物化psql(`INSERT INTO jobs (kind, payload) VALUES ('align_types', '{"kb_id":"${KB}"}')`);await until(() => {- const left = num(`SELECT count(*) FROM jobs WHERE kind IN ('align_types','align_phrases','materialize_typed') AND status IN ('queued','running') AND payload->>'kb_id'='${KB}'`);+ const left = num(`SELECT count(*) FROM jobs WHERE kind IN ('align_types','align_phrases','materialize_typed','read_phrases') AND status IN ('queued','running') AND payload->>'kb_id'='${KB}'`);const bound = num(`SELECT count(*) FROM phrase_bindings WHERE kb_id='${KB}' AND status='bound'`);+ const decided = num(`SELECT count(*) FROM phrase_bindings WHERE kb_id='${KB}'`) + num(`SELECT count(*) FROM type_bindings WHERE kb_id='${KB}'`);+ // 签名判完之后还有一段提规则(0044 决定 3 第五片):每条判定过的签名问一次,规则行慢慢增加,+ // 绑定数却不再动——进展得把它也算上,否则这一段会被当成卡死+ const rules = num(`SELECT count(*) FROM implication_rules WHERE kb_id='${KB}'`);const typed = num(`SELECT count(*) FROM facts WHERE kb_id='${KB}' AND layer='typed' AND from_statement_id IS NOT NULL AND invalidated_at IS NULL`);- log(`对齐:${left} 个任务在跑,绑定 ${bound},类型化 ${typed}`);- return left === 0 && num(`SELECT count(*) FROM phrase_bindings WHERE kb_id='${KB}'`) > 0 ? true : bound * 100000 + typed;- }, 15000, 20 * 60000);+ log(`对齐:${left} 个任务在跑,判定 ${decided},绑定 ${bound},规则 ${rules},类型化 ${typed}`);+ // `until` 只把数字当进展:四个数拼成一个单调的数+ return left === 0 && num(`SELECT count(*) FROM phrase_bindings WHERE kb_id='${KB}'`) > 0 ? true : decided * 1e9 + bound * 1e6 + rules * 1e3 + typed;+ }, 15000, 30 * 60000);const failed = num(`SELECT count(*) FROM jobs WHERE kind IN ('align_types','align_phrases','materialize_typed') AND status='failed' AND payload->>'kb_id'='${KB}'`);if (failed) log(`注意:${failed} 个对齐任务失败(看 jobs.last_error)`);}+// --approve-rules:替审核人把对齐器提的蕴含规则全批了(0044 决定 3 第五片的上限:真人会驳回一部分),+// 排读数任务(读数完了自己排物化),等隐含行算出来。不带这个开关时提议只是躺在队列里,隐含行为 0+async function approveRules(KB) {+ const n = num(`SELECT count(*) FROM implication_rules WHERE kb_id='${KB}' AND status='proposed'`);+ psql(`UPDATE implication_rules SET status='approved', decided_by='person', decided_at=now() WHERE kb_id='${KB}' AND status='proposed'`);+ psql(`INSERT INTO jobs (kind, payload) VALUES ('read_phrases', '{"kb_id":"${KB}"}')`);+ log(`批了 ${n} 条规则,排读数与物化`);+ await until(() => {+ const left = num(`SELECT count(*) FROM jobs WHERE kind IN ('read_phrases','materialize_typed') AND status IN ('queued','running') AND payload->>'kb_id'='${KB}'`);+ const read = num(`SELECT count(*) FROM phrase_readings WHERE kb_id='${KB}'`);+ const implied = num(`SELECT count(*) FROM facts WHERE kb_id='${KB}' AND implied AND invalidated_at IS NULL`);+ log(`读数:${left} 个任务在跑,缓存 ${read} 条,隐含 ${implied} 条`);+ return left === 0 ? true : read * 100000 + implied;+ }, 15000, 30 * 60000);+ return { approved: n, readings: num(`SELECT count(*) FROM phrase_readings WHERE kb_id='${KB}'`), implied: num(`SELECT count(*) FROM facts WHERE kb_id='${KB}' AND implied AND invalidated_at IS NULL`) };+}+// 勘误(0044 决定 7):物化之后排一次 errata_review,等它把每篇文档看完。度量按 errata_runs 与// errata_actions 报:撤了几条、改了几条、加了几条、留给人几条、拒了几条、花了多少 tokenasync function errata(KB) {@@ -236,23 +262,37 @@ function valueMatches(gold, got) {return gn.size > 0 && vn.size > 0 && [...gn].some((n) => vn.has(n));}-function score(KB) {- // 每篇文档里的类型化事实:主语名集合、属性键、宾语名集合或值- const typed = rows(`+// scope:'after' 是库现在的样子(勘误撤掉的不算、勘误加上的算),'before' 是勘误之前+// (撤掉的算回来、加上的不算)。0044 的门槛看勘误前,勘误的度量看两者之差+function score(KB, scope = "after") {+ // 每篇文档里的类型化事实:主语名集合、属性键、宾语名集合或值。+ // 名字之间用 ¦ 接:psql -A 的列分隔符是 |,名字里再用 | 会把列切错(第一轮就是这么错到 0 的)+ const rowsOf = rows(`SELECT d.filename, r.key,- s.canonical_name || '|' || coalesce((SELECT string_agg(n.object_value->>'value','|') FROM facts n JOIN relation_types nr ON nr.id=n.predicate_id+ s.canonical_name || '¦' || coalesce((SELECT string_agg(n.object_value->>'value','¦') FROM facts n JOIN relation_types nr ON nr.id=n.predicate_idWHERE n.subject_id=s.id AND nr.key='known_as' AND n.invalidated_at IS NULL AND n.object_value IS NOT NULL),''),- coalesce(o.canonical_name || '|' || coalesce((SELECT string_agg(n.object_value->>'value','|') FROM facts n JOIN relation_types nr ON nr.id=n.predicate_id+ coalesce(o.canonical_name || '¦' || coalesce((SELECT string_agg(n.object_value->>'value','¦') FROM facts n JOIN relation_types nr ON nr.id=n.predicate_idWHERE n.subject_id=o.id AND nr.key='known_as' AND n.invalidated_at IS NULL AND n.object_value IS NOT NULL),''), ''),- coalesce(f.object_value->>'value', f.object_value#>>'{}', '')+ coalesce(f.object_value->>'value', f.object_value#>>'{}', ''),+ EXISTS (SELECT 1 FROM errata_actions ea WHERE ea.fact_id=f.id AND ea.action IN ('retract','revise') AND ea.status='applied') AS removed,+ -- 只算「只有勘误这一个来源」的行:加的事实撞上已有的行时 insert_fact_on 复用那一行,它仍是陈述算出来的+ (f.from_statement_id IS NULL AND NOT f.implied+ AND EXISTS (SELECT 1 FROM errata_actions ea WHERE ea.new_fact_id=f.id AND ea.status='applied')) AS addedFROM facts fJOIN relation_types r ON r.id=f.predicate_idJOIN entities s ON s.id=f.subject_idLEFT JOIN entities o ON o.id=f.object_idJOIN LATERAL (SELECT document_id FROM fact_evidence WHERE fact_id=f.id LIMIT 1) ev ON trueJOIN documents d ON d.id=ev.document_id- WHERE f.kb_id='${KB}' AND f.layer='typed' AND f.from_statement_id IS NOT NULL AND f.invalidated_at IS NULL`- ).map(([file, key, subj, obj, val]) => ({ file, key: key.toUpperCase(), subj: subj.split("|").map(norm).filter(Boolean), obj: obj.split("|").map(norm).filter(Boolean), val }));+ WHERE f.kb_id='${KB}' AND f.layer='typed' AND NOT r.builtin+ AND (f.from_statement_id IS NOT NULL OR f.implied+ OR EXISTS (SELECT 1 FROM errata_actions ea WHERE ea.new_fact_id=f.id AND ea.status='applied'))+ AND (f.invalidated_at IS NULL+ OR EXISTS (SELECT 1 FROM errata_actions ea WHERE ea.fact_id=f.id AND ea.action IN ('retract','revise') AND ea.status='applied'))`+ );+ const typed = rowsOf+ .map(([file, key, subj, obj, val, removed, added]) => ({ file, key: key.toUpperCase(), subj: subj.split("¦").map(norm).filter(Boolean), obj: obj.split("¦").map(norm).filter(Boolean), val, removed: removed === "t", added: added === "t" }))+ .filter((t) => (scope === "before" ? !t.added : !t.removed));// 开放陈述连上的实体对(抽取那一层)const open = rows(`SELECT d.filename, s.canonical_name, coalesce(o.canonical_name, f.object_value->>'value', '')@@ -286,7 +326,7 @@ function score(KB) {}const pct = (a, b) => (b ? (100 * a / b).toFixed(1) + "%" : "-");const result = {- label: LABEL, kb: KB, corpus: CORPUS, at: new Date().toISOString(),+ label: LABEL, kb: KB, corpus: CORPUS, scope, at: new Date().toISOString(),typed_facts: typed.length, open_statements: open.length,gold, gold_recall: hit / (gold || 1), gold_recall_same_sentence: hitSame / (goldSame || 1),gold_recall_cross_sentence: (hit - hitSame) / ((gold - goldSame) || 1),@@ -330,7 +370,9 @@ async function judge(KB, n, seed) {SELECT f.id, d.filename, s.canonical_name, r.label, coalesce(o.canonical_name, f.object_value->>'value', f.object_value#>>'{}', '')FROM facts f JOIN relation_types r ON r.id=f.predicate_id JOIN entities s ON s.id=f.subject_id LEFT JOIN entities o ON o.id=f.object_idJOIN LATERAL (SELECT document_id FROM fact_evidence WHERE fact_id=f.id LIMIT 1) ev ON true JOIN documents d ON d.id=ev.document_id- WHERE f.kb_id='${KB}' AND f.layer='typed' AND f.from_statement_id IS NOT NULL AND f.invalidated_at IS NULL ORDER BY f.id`);+ WHERE f.kb_id='${KB}' AND f.layer='typed' AND NOT r.builtin AND f.invalidated_at IS NULL+ AND (f.from_statement_id IS NOT NULL OR f.implied+ OR EXISTS (SELECT 1 FROM errata_actions ea WHERE ea.new_fact_id=f.id AND ea.status='applied')) ORDER BY f.id`);// 可复现抽样let a = (seed >>> 0) || 1; const rnd = () => { a = (a * 1103515245 + 12345) >>> 0; return a / 4294967296; };const sample = all.map((r) => [rnd(), r]).sort((x, y) => x[0] - y[0]).slice(0, n).map(([, r]) => r);@@ -365,13 +407,17 @@ if (!KB) {} else {await login();}-const result = score(KB);+let rules = null;+if (args["approve-rules"]) rules = await approveRules(KB);+// 勘误前的分:已经跑过勘误的库也能按 scope 算回来(撤掉的算回来、加上的不算)+const result = score(KB, "before");+if (rules) result.rules = rules;if (args.judge) result.judge = await judge(KB, Number(args.judge) || 200, Number(args.seed || 1));if (args.errata) {// 勘误前的分留着,勘误后再打一次:0044 §7 的度量是两份分的差,与撤错了多少result.before_errata = { gold_recall: result.gold_recall, gold_recall_same_sentence: result.gold_recall_same_sentence, typed_facts: result.typed_facts, judge: result.judge };result.errata = await errata(KB);- const after = score(KB);+ const after = score(KB, "after");result.after_errata = { gold_recall: after.gold_recall, gold_recall_same_sentence: after.gold_recall_same_sentence, typed_facts: after.typed_facts };if (args.judge) {result.after_errata.judge = await judge(KB, Number(args.judge) || 200, Number(args.seed || 1));