World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
The recall bench marks a round that did not extract every chunk and keeps it out of the table (#1014)
WaylandYang8a668802cff5
diff --git a/scripts/bench/README.md b/scripts/bench/README.mdindex 5ce94398..45d1afd3 100644--- a/scripts/bench/README.md+++ b/scripts/bench/README.md@@ -88,7 +88,9 @@ node scripts/bench/recall.mjs --table runs/588reason 前缀记;只算这一轮排队之后排的对,`--score` 没有排队时刻,算的是库里现有的——每轮开头删实体时它们随之删掉)、丢弃账(`extraction_drops`按原因)和排队到最后一块抽完的秒数。抽完之后还要等对齐、治理、时间消解、裁决这些任务收尾(它们还在补事实、裁对子),等到没有该跑的任务才打分,不然每轮记的是抽完那一刻的快照;等了多久记成 `settleSeconds`,卡住十五分钟没收尾的轮次-`--table` 会单独点名。`--table` 给每组的均值 ± 标准差,再给事实集合(去重的主/谓/宾)+`--table` 会单独点名。没抽完的轮次(端点整段不可用时抽取任务一个个失败、队列就空了,收尾等待立刻满足)记下来但+`complete: false`,`--table` 不把它算进任何一组,只在表下点名——它的分说的是端点那天的状态,不是这一轮的配置。+`--table` 给每组的均值 ± 标准差,再给事实集合(去重的主/谓/宾)的重合度:组内两两比是运行间的抖动,跨组两两比是 known 的影响——跨组落在组内的范围里,空 known 就没把输出挪出方差。diff --git a/scripts/bench/recall.mjs b/scripts/bench/recall.mjsindex 7ffff850..ec8e2919 100644--- a/scripts/bench/recall.mjs+++ b/scripts/bench/recall.mjs@@ -308,12 +308,16 @@ function record(result, perDoc, extra) {// 以及事实集合的重合度——组内两两比(运行间方差本身)与跨组两两比(known 的影响)。// 跨组的重合度落在组内的范围里,就是说空 known 挪动的没超出模型自己的抖动function table(dir) {- const runs = fs+ const all = fs.readdirSync(dir).filter((f) => f.endsWith(".json")).map((f) => JSON.parse(fs.readFileSync(path.join(dir, f), "utf8")));+ // 没抽完的轮次不进组:它的分、事实数、对子数说的都不是这一轮的配置,而是端点那天的状态。+ // 老记录没有 complete 字段,按每篇的块数看+ const incomplete = (r) => r.complete === false || Object.values(r.docs ?? {}).some((d) => d.extracted < d.chunks);+ const runs = all.filter((r) => !incomplete(r));if (!runs.length) {- console.error(`${dir} 里没有记录`);+ console.error(`${dir} 里没有抽完的记录`);process.exit(1);}const groups = ["shown", "empty"].map((k) => [k, runs.filter((r) => r.known === k)]).filter(([, rs]) => rs.length);@@ -345,6 +349,11 @@ function table(dir) {// 没等到收尾就记下的轮次(卡住十五分钟才停):数是快照,单独点名,别混进均值里看不出来const unsettled = runs.filter((r) => r.settled === false);if (unsettled.length) console.log(`\n没等到收尾的轮次:${unsettled.map((r) => `${r.at}(${r.known})`).join("、")}`);+ const skipped = all.filter(incomplete);+ if (skipped.length) {+ const blocks = (r) => `${Object.values(r.docs).reduce((s, d) => s + d.extracted, 0)}/${Object.values(r.docs).reduce((s, d) => s + d.chunks, 0)} 块`;+ console.log(`\n没抽完、不进表的轮次:${skipped.map((r) => `${r.at}(${r.known},${blocks(r)})`).join("、")}`);+ }// 事实集合的 Jaccard:每篇文档各算,再平均const jac = (a, b) => {@@ -434,6 +443,13 @@ for (;;) {last = done;await sleep(30000);}+// **抽完才算一轮。** 端点整段不可用时,抽取任务一个个失败、队列就空了,上面的循环照常退出,+// 后面的收尾等待也立刻满足——记下来的是一轮几乎没抽的数(0/52、几个块),却带着 settled=true,+// `--table` 会把它当成正常一轮拉进均值。所以按块数判完整:没抽完的轮次记下来但标明,排表时不进组+const extracted = num(`${live.replace("count(*)", "count(c.extracted_at)")}`);+const total = num(live);+const complete = total > 0 && extracted === total;+if (!complete) console.log(`只抽了 ${extracted}/${total} 块:这一轮不完整,记下来但不进表`);// **抽完不等于这一轮完了**:对齐、治理、时间消解、裁决还在后头跑,一边补事实一边裁对子// (同一篇财报,抽完那一刻 552 条事实,一小时后 565)。不等它们收尾,每轮记下的就是@@ -467,4 +483,4 @@ for (;;) {const settleSeconds = Math.round((Date.now() - settleFrom) / 1000);if (settled) console.log(`${stamp()} 收尾完了(抽完之后又等了 ${settleSeconds} 秒)`);-record(score(), stats(queuedAt), { endpoint, pairs: pairs(queuedAt), settled, settleSeconds });+record(score(), stats(queuedAt), { endpoint, pairs: pairs(queuedAt), complete, settled, settleSeconds });