World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Measure the agent's glossary on the same shapes, and carry the properties nearest in meaning (#965)
WaylandYangf1a36a583ba8
diff --git a/crates/utopia-extract/src/ontology_agent.rs b/crates/utopia-extract/src/ontology_agent.rsindex 84d3b4f0..51074946 100644--- a/crates/utopia-extract/src/ontology_agent.rs+++ b/crates/utopia-extract/src/ontology_agent.rs@@ -105,6 +105,15 @@ pub struct ProposalReply {pub existing: Vec<Existing>,/// 读不出的项数pub malformed: usize,+ /// 每个坏项为什么坏:测量台数它们,日志里也看得见+ pub reasons: Vec<String>,+}++impl ProposalReply {+ fn bad(&mut self, why: String) {+ self.malformed += 1;+ self.reasons.push(why);+ }}const SYSTEM: &str = "\@@ -130,6 +139,8 @@ instead of proposing, with dir: forward when the shape's subject is the propertywhen the shape reads the property backwards (\"X wrote Y\" is author read in reverse).\n\- Do not propose the inverse of an existing property: list the shape under that property with dir \reverse.\n\+- An existing entry holds shapes, not questions: every entry under existing lists at least one \+shape id in s or k. Do not add an entry only to say that a property serves a question.\n\Answer with one JSON object and nothing else:\n\{\"p\":[{\"kind\":\"property\",\"key\":\"...\",\"label\":\"...\",\"definition\":\"...\",\"value\":false,\\"datatype\":null,\"domains\":[\"class_key\"],\"ranges\":[\"class_key\"],\"s\":[ids],\"k\":[ids],\"q\":[ids]},\@@ -293,14 +304,14 @@ pub fn parse_proposal_response(Some(Value::Object(m)) => m.values().collect(),Some(Value::Null) | None => Vec::new(),Some(_) => {- reply.malformed += 1;+ reply.bad("`p` is not a list".to_string());Vec::new()}};let mut seen: HashSet<String> = HashSet::new();for it in items {let Some(obj) = it.as_object() else {- reply.malformed += 1;+ reply.bad("a proposal is not an object".to_string());continue;};let kind = obj@@ -318,7 +329,7 @@ pub fn parse_proposal_response(|| !valid_key(&key)|| !seen.insert(key.clone()){- reply.malformed += 1;+ reply.bad(format!("bad kind, bad key or repeated key: {kind} {key:?}"));continue;}let label = obj@@ -345,7 +356,9 @@ pub fn parse_proposal_response(strings(obj.get("ranges")),strings(obj.get("parents")),) else {- reply.malformed += 1;+ reply.bad(format!(+ "domains, ranges or parents are not lists of strings: {key}"+ ));continue;};let (Ok(signatures), Ok(kind_words), Ok(questions)) = (@@ -353,12 +366,12 @@ pub fn parse_proposal_response(ids(obj.get("k"), 'k', kind_word_ids),ids(obj.get("q"), 'q', question_ids),) else {- reply.malformed += 1;+ reply.bad(format!("a shape or question id outside the batch: {key}"));continue;};// 什么形状都不绑的提案不是从图谱里长出来的:不收if signatures.is_empty() && kind_words.is_empty() {- reply.malformed += 1;+ reply.bad(format!("binds no shape: {key}"));continue;}reply.proposals.push(Proposal {@@ -383,7 +396,7 @@ pub fn parse_proposal_response(if let Some(Value::Array(ex)) = v.get("existing") {for it in ex {let Some(obj) = it.as_object() else {- reply.malformed += 1;+ reply.bad("an existing entry is not an object".to_string());continue;};let key = obj@@ -395,18 +408,18 @@ pub fn parse_proposal_response(ids(obj.get("s"), 's', signature_ids),ids(obj.get("k"), 'k', kind_word_ids),) else {- reply.malformed += 1;+ reply.bad(format!("existing: a shape id outside the batch: {key}"));continue;};if key.is_empty() || (signatures.is_empty() && kind_words.is_empty()) {- reply.malformed += 1;+ reply.bad(format!("existing: no key or no shapes: {key:?}"));continue;}let direction = match obj.get("dir").and_then(Value::as_str).map(str::trim) {Some("reverse") => Some("reverse".to_string()),Some("forward") | None | Some("") => Some("forward".to_string()),Some(_) => {- reply.malformed += 1;+ reply.bad(format!("existing: unknown direction: {key}"));continue;}};diff --git a/crates/utopia-server/src/api/question_routes.rs b/crates/utopia-server/src/api/question_routes.rsindex 3b20f2c3..ca4fcc9c 100644--- a/crates/utopia-server/src/api/question_routes.rs+++ b/crates/utopia-server/src/api/question_routes.rs@@ -213,19 +213,28 @@ pub async fn report(})))}+#[derive(Deserialize, Default)]+pub struct ProposeReq {+ /// 测量用(bench/agent.mjs):`full` = 每批送全词表。缺省是按批裁过的+ #[serde(default)]+ pub glossary: Option<String>,+}+/// 叫代理来看一眼:排一份任务,马上回。结果落在 `ontology_proposals`,界面从那里读pub async fn propose(State(state): State<AppState>,AuthUser(user): AuthUser,Path(kb_id): Path<Uuid>,+ Json(req): Json<ProposeReq>,) -> ApiResult<Json<serde_json::Value>> {utopia_store::access::require_kb(&state.pool, &user, kb_id, Role::Editor).await?;- let queued = utopia_store::jobs::enqueue_unless_queued(- &state.pool,- "propose_ontology",- json!({ "kb_id": kb_id }),- )- .await?;+ let payload = if req.glossary.as_deref() == Some("full") {+ json!({ "kb_id": kb_id, "glossary": "full" })+ } else {+ json!({ "kb_id": kb_id })+ };+ let queued =+ utopia_store::jobs::enqueue_unless_queued(&state.pool, "propose_ontology", payload).await?;Ok(Json(json!({ "queued": queued.is_some() })))}diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex 3d7b7ea0..d3173377 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -644,7 +644,9 @@ async fn dispatch(st: &state::AppState, job: &utopia_store::jobs::Job) -> anyhow.and_then(|v| v.as_str()).and_then(|s| s.parse().ok()).ok_or_else(|| anyhow::anyhow!("payload 缺少 kb_id"))?;- ontology_agent::propose(st, kb_id).await+ // 测量用:payload 里 glossary = full 时每批送全词表(bench/agent.mjs)+ let full = job.payload.get("glossary").and_then(|v| v.as_str()) == Some("full");+ ontology_agent::propose_with(st, kb_id, full).await}"propose_questions" => {let kb_id: Uuid = jobdiff --git a/crates/utopia-server/src/ontology_agent.rs b/crates/utopia-server/src/ontology_agent.rsindex 47fa5b1c..526edf3f 100644--- a/crates/utopia-server/src/ontology_agent.rs+++ b/crates/utopia-server/src/ontology_agent.rs@@ -32,6 +32,8 @@ const TOP_ENTITIES: usize = 20;/// 一条形状结构上对得上的属性多过这个数,就按向量只留最近的这么多带定义(同对齐的短名单)const GLOSSARY_PER_SHAPE: usize = 10;+/// 每条形状按意思最近的这么多条属性,结构对不上也带定义+const GLOSSARY_NEAREST_ANYWAY: usize = 5;/// 一次调用交给模型的形状数(0044 决定 3 的成本教训:词表每批只带一次)pub(crate) const SIGNATURES_PER_CALL: usize = 12;@@ -53,7 +55,18 @@ fn shape_of(sig: &PhraseSignature) -> Value {}/// 任务入口:一个库同时只跑一份(同两种对齐,并行会把端点打出 502)+#[cfg(test)]pub async fn propose(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> {+ propose_with(state, kb_id, false).await+}++/// `full_glossary` 是测量用的旋钮(`scripts/bench/agent.mjs`):每批都送全部属性的定义,+/// 好和按批裁过的词表在同一批形状上比失败率、重复提案和 token。产品路径不开它+pub async fn propose_with(+ state: &AppState,+ kb_id: Uuid,+ full_glossary: bool,+) -> anyhow::Result<()> {let pool = &state.pool;let kb = utopia_store::kbs::get(pool, kb_id).await?;let settings = utopia_store::settings::get(pool, kb.workspace_id)@@ -73,7 +86,7 @@ pub async fn propose(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> {tracing::info!(%kb_id, "本体代理已有一份在跑,这次跳过");return Ok(());}- let result = propose_locked(state, kb_id, &settings, &client).await;+ let result = propose_locked(state, kb_id, &settings, &client, full_glossary).await;let _ = sqlx::query("SELECT pg_advisory_unlock(hashtext('propose_ontology'), hashtext($1))").bind(kb_id.to_string()).execute(&mut *guard)@@ -99,6 +112,7 @@ async fn propose_locked(kb_id: Uuid,settings: &utopia_core::models::LlmSettings,client: &utopia_llm::LlmClient,+ full_glossary: bool,) -> anyhow::Result<()> {let pool = &state.pool;let classes = utopia_store::graph::entity_types(pool, kb_id).await?;@@ -305,6 +319,9 @@ async fn propose_locked(};let glossary_for = |ids: std::ops::Range<usize>| -> agent::Glossary<'_> {let mut full: HashSet<Uuid> = HashSet::new();+ if full_glossary {+ full.extend(props.iter().map(|p| p.id));+ }for i in ids {let (Some(s), near) = (open.get(i), nearest.get(i)) else {continue;@@ -313,6 +330,12 @@ async fn propose_locked(.iter().filter(|p| crate::phrase_alignment::structurally_fits(p, s, &closure)).collect();+ // 意思最近的几条不管结构对不对得上都给定义:一端没类型、或宾语是字面值的形状,+ // 结构上对不上 positionHeld / workLocation,模型看不到定义就把它们又提了一遍+ // (受控对比里重复提案 1% → 6%,十条里七条是这样来的)+ if let Some(near) = near {+ full.extend(near.iter().take(GLOSSARY_NEAREST_ANYWAY).copied());+ }if fitting.len() > GLOSSARY_PER_SHAPE {if let Some(near) = near.filter(|n| !n.is_empty()) {full.extend(@@ -420,6 +443,9 @@ async fn propose_locked(}};malformed += parsed.malformed;+ for why in &parsed.reasons {+ tracing::warn!(%kb_id, batch = b, why = %why, "本体代理:坏项");+ }let mut sig_outcome: HashMap<i64, (&str, String)> = HashMap::new();let mut word_outcome: HashMap<i64, (&str, String)> = HashMap::new();// 「本体里已经有」:存成 map_to 提案,人点「用已有的」就是一次人的绑定判定@@ -430,6 +456,7 @@ async fn propose_locked("class".to_string()} else {malformed += 1;+ tracing::warn!(%kb_id, batch = b, why = %format!("existing: no such key in the ontology: {}", e.key), "本体代理:坏项");continue;};let slot = format!("map_to:{}", e.key);@@ -632,6 +659,48 @@ async fn propose_locked(.collect();// 同一个目标上一轮已经答过一批形状:并起来,不是盖掉(第一次真跑第三轮盖掉了第二轮的)let mut items = items;+ // 每条新元素带上本体里离它最近的两个已有元素(按向量,不问模型)。裁过的词表下模型看不到+ // 大部分属性的定义,受控对比里重复或反向已有属性的提案从 1% 升到 5%(awardReceived、+ // performer 这几条);审的人要在采纳前看见"它最像谁"+ for class_side in [false, true] {+ let idx: Vec<usize> = items+ .iter()+ .enumerate()+ .filter(|(_, it)| {+ it.section != "map_to" && (it.section == "entity_types") == class_side+ })+ .map(|(i, _)| i)+ .collect();+ let queries: Vec<String> = idx+ .iter()+ .map(|i| {+ let p = &items[*i].payload;+ format!(+ "{} · {}",+ p.get("label").and_then(Value::as_str).unwrap_or(""),+ p.get("description").and_then(Value::as_str).unwrap_or("")+ )+ })+ .collect();+ let target = if class_side {+ crate::ontology_index::Target::Class+ } else {+ crate::ontology_index::Target::Predicate(None)+ };+ match crate::ontology_index::nearest_for_each(state, kb_id, &queries, 2, target).await {+ Ok(found) => {+ for (i, cands) in idx.into_iter().zip(found) {+ items[i].payload["closest"] = json!(cands+ .iter()+ .map(|c| json!({ "key": c.key, "label": c.label, "distance": c.distance }))+ .collect::<Vec<_>>());+ }+ }+ Err(e) => {+ tracing::warn!(%kb_id, error = %e, "本体代理:最近的已有元素没算出来,提案不带这一格");+ }+ }+ }for it in items.iter_mut().filter(|it| it.section == "map_to") {if let Some(prev) =utopia_store::ontology::open_proposal(pool, kb_id, "map_to", &it.key).await?@@ -641,7 +710,7 @@ async fn propose_locked(}utopia_store::ontology::save_agent_proposals(pool, kb_id, &items).await?;agent_reviews::record(pool, kb_id, &reviews).await?;- tracing::info!(%kb_id, proposals = items.len(), failed, malformed, skipped_existing, "本体代理结束");+ tracing::info!(%kb_id, proposals = items.len(), failed, malformed, skipped_existing, calls, glossary = if full_glossary { "full" } else { "trimmed" }, "本体代理结束");if !items.is_empty() {state.emit_pending(kb_id);}diff --git a/scripts/bench/README.md b/scripts/bench/README.mdindex 923ef5c7..13347be8 100644--- a/scripts/bench/README.md+++ b/scripts/bench/README.md@@ -248,6 +248,41 @@ node scripts/bench/competency.mjs --kb <kb-id> --seed scripts/bench/truth/redocr- `truth/redocred-typed.questions.json` 是给 `typed.mjs` 那份 100 篇 Re-DocRED 库写的十一条:十条有期望答案(其中两条要代理采纳过的 `bordered_by`、`p40` 才答得上),一条只有形状。换一个库要另写。+## 本体代理的测量台(0061)++`agent.mjs` 在同一个库状态、同一批形状上各臂跑一轮:每臂开跑前把代理没采纳的提案和看过的形状记录删掉+(所以只在测量库上用,要显式给 `--reset`),从服务端日志读失败批次、坏项及原因、token,再让裁判拿着+**全词表**判每条新提案是不是已有元素的重复或反向,判每条「已有」答案对不对、方向对不对。+`POST /ontology/propose` 的 `glossary: full` 是给全词表那一臂用的旋钮,产品路径不开。++```+BENCH_BASE=http://127.0.0.1:1524 BENCH_PSQL="docker exec ... -d utopia_bench4 -tAc" \+BENCH_SERVER_LOG=/path/to/server.log BENCH_JUDGE_BASE=... BENCH_JUDGE_KEY=... BENCH_JUDGE_MODEL=... \+node scripts/bench/agent.mjs --kb <kb-id> --db utopia_bench4 --reset --arms trimmed,full,trimmed,full+```++**2026-09-27,bench4(100 篇 Re-DocRED,98 条属性),gemini-3.5-flash,每轮 120 条形状 + 69 个类别词、12 次调用:**++| | 全词表 | 按结构裁(#963) | 按结构裁 + 意思最近 5 条 |+|---|---|---|---|+| 轮数 | 4 | 7 | 4 |+| 失败批次 | 0 / 48 | 0 / 84 | 0 / 48 |+| 提示 token / 次 | 7.1k | 4.3k | 5.0k |+| 一轮 token | 111k | 78k | 87k |+| 新元素提案 | 135 | 292 | 178 |+| 其中重复或反向已有属性 | 1(0.7%) | 18(6.2%) | 6(3.4%) |+| 「已有」答案判对 | 44 / 65(68%) | 60 / 85(71%) | 44 / 56(79%) |++- 裁词表不让调用失败,也不让模型提已存在的键;让它**重提定义被裁掉的属性**。按结构裁时重的集中在+ positionHeld、workLocation、performer、awardReceived:形状一端没类型或宾语是字面值,结构上对不上,+ 定义就没送。再按意思带上最近的 5 条,重复降了一半,多花 0.7k token 一次。+- 每轮固定出现的十来个坏项是"existing 条目没有形状",两臂都有,数目等于库里的问题数:模型把"这条属性+ 服务某个问题"写进了 existing。解析时丢掉,无害;提示词里已补一句。+- **同一臂两轮之间的差别比两臂之间大**:同样的输入、温度 0,两轮只有一半提案键相同。单轮对比说明不了+ 质量,至少四轮合起来看。+- 提案旁边显示的"最近的已有元素"(按向量取两条)只罩住了被判重复的 16 条里的 6 条,是给人看的线索,+ 不是防线。+## 读数怎么算- `prompt_tokens_est` 是**本体段**的估算,不是整个提示词。实测 4.0 字符 ≈ 1 tokendiff --git a/scripts/bench/agent.mjs b/scripts/bench/agent.mjsnew file mode 100644index 00000000..10a8a8f1--- /dev/null+++ b/scripts/bench/agent.mjs@@ -0,0 +1,171 @@+#!/usr/bin/env node+// 本体代理的 A/B 测量台(ADR 0061):同一个库状态、同一批形状,各臂跑一轮,量+// - 失败:调用失败或读不出的批次、坏项、模型提了已有键;+// - 重复:提案其实是已有属性(或它的反向)——裁词表的主要风险,裁判拿着全词表判;+// - 「已有」答案的准确率:形状真的说的是那条属性、方向对不对;+// - token(服务端日志的 llm usage 行)。+//+// BENCH_BASE=http://127.0.0.1:1524 BENCH_PSQL="docker exec ... -d utopia_bench4 -tAc" \+// BENCH_SERVER_LOG=/path/to/server.log BENCH_JUDGE_BASE=... BENCH_JUDGE_KEY=... BENCH_JUDGE_MODEL=... \+// node scripts/bench/agent.mjs --kb <kb-id> --db utopia_bench4 --reset [--arms trimmed,full] [--out file.json]+//+// **--reset 会删这个库里代理的提案(没采纳的)和看过的形状记录**,每臂开跑前各删一次,好让两臂+// 看到同一批形状。只在测量库上用;不给 --reset 就不跑。+import fs from "node:fs";+import { api, login, parseArgs, log, onDb, sleep } from "./lib.mjs";++const args = parseArgs(process.argv);+const KB = args.kb, DB = args.db;+const ARMS = String(args.arms || "trimmed,full").split(",").map((s) => s.trim()).filter(Boolean);+const LOG = process.env.BENCH_SERVER_LOG;+if (!KB || !DB) { console.error("--kb <id> 与 --db <bench database> 是必须的"); process.exit(2); }+if (!args.reset) { console.error("这台子要重置代理的提案和记录才能让两臂看到同一批形状:确认是测量库后加 --reset"); process.exit(2); }+if (!LOG) { console.error("给 BENCH_SERVER_LOG:失败数和 token 从服务端日志里读"); process.exit(2); }+const q = (sql) => onDb(DB, sql);++const ep = { base: process.env.BENCH_JUDGE_BASE, key: process.env.BENCH_JUDGE_KEY || "", model: process.env.BENCH_JUDGE_MODEL };+if (!ep.base || !ep.model) { console.error("给 BENCH_JUDGE_BASE / _KEY / _MODEL"); process.exit(2); }+async function judge(system, user) {+ for (let attempt = 0; attempt < 3; attempt++) {+ const r = await fetch(`${ep.base.replace(/\/$/, "")}/chat/completions`, {+ method: "POST", headers: { "content-type": "application/json", ...(ep.key ? { authorization: `Bearer ${ep.key}` } : {}) },+ body: JSON.stringify({ model: ep.model, temperature: 0, messages: [{ role: "system", content: system }, { role: "user", content: user }] }),+ });+ if (r.ok) {+ const text = (await r.json()).choices?.[0]?.message?.content ?? "";+ const m = /\{[\s\S]*\}/.exec(text);+ if (m) { try { return JSON.parse(m[0]); } catch { /* 再问一次 */ } }+ } else if (r.status !== 429 && r.status < 500) throw new Error(`judge -> ${r.status} ${(await r.text()).slice(0, 200)}`);+ await sleep(3000 * (attempt + 1));+ }+ return null;+}++const DUP = `You review proposals to extend an ontology. You are given the full ontology (every class and property with its definition) and a list of proposed new elements. For each proposal decide whether the ontology ALREADY has it:+- "duplicate": an existing property or class means the same thing (name it);+- "inverse": an existing property is the same relation read in the other direction (name it);+- "new": nothing existing covers it.+Be strict about meaning, not wording. Output one JSON object: {"results":[{"i":0,"verdict":"duplicate|inverse|new","of":"existing_key or null"}]}`;+const EXIST = `You check decisions that a statement shape is an instance of an existing ontology property. Each item gives a phrase between a subject class and an object class (or a value), one example statement, the property with its definition, and the direction claimed: forward = the statement's subject is the property's subject; reverse = the statement reads the property backwards.+- "right": the phrase asserts this property in this direction;+- "wrong_direction": the property is right but the direction is not;+- "wrong": the phrase does not assert this property (too vague, or a different meaning).+Output one JSON object: {"results":[{"i":0,"verdict":"right|wrong_direction|wrong"}]}`;++function logWindow(from) {+ const buf = fs.readFileSync(LOG);+ // eslint-disable-next-line no-control-regex+ return buf.subarray(from).toString("utf8").replace(/\x1b\[[0-9;]*m/g, "");+}++async function arm(name) {+ log(`===== arm ${name}: reset`);+ q(`DELETE FROM ontology_agent_reviews WHERE kb_id='${KB}'`);+ q(`DELETE FROM ontology_proposals WHERE kb_id='${KB}' AND proposed_by='agent' AND status<>'adopted'`);+ q(`DELETE FROM jobs WHERE kind='propose_ontology' AND status='queued' AND payload->>'kb_id'='${KB}'`);+ const from = fs.statSync(LOG).size;+ const t0 = Date.now();+ await api("POST", `/api/v1/kbs/${KB}/ontology/propose`, name === "full" ? { glossary: "full" } : {});+ for (;;) {+ await sleep(10000);+ const st = q(`SELECT status FROM jobs WHERE kind='propose_ontology' AND payload->>'kb_id'='${KB}' ORDER BY id DESC LIMIT 1`);+ if (st !== "queued" && st !== "running") { if (st !== "done") log(`job ended as ${st}`); break; }+ }+ const text = logWindow(from);+ const started = /本体代理开始 .*shapes=(\d+) kind_words=(\d+) questions=(\d+) calls=(\d+)/.exec(text);+ const ended = /本体代理结束 .*proposals=(\d+) failed=(\d+) malformed=(\d+) skipped_existing=(\d+)/.exec(text);+ // 坏项各是为什么坏的(按原因归并,键去掉)+ const why = {};+ for (const m of text.matchAll(/本体代理:坏项 .*why=(.*)$/gm)) {+ const k = m[1].replace(/^"|"$/g, "").replace(/:\s[^:]*$/, "").trim();+ why[k] = (why[k] ?? 0) + 1;+ }+ const usage = [...text.matchAll(/llm usage .*prompt=(\d+) completion=(\d+)/g)];+ const prompt = usage.reduce((n, m) => n + Number(m[1]), 0), completion = usage.reduce((n, m) => n + Number(m[2]), 0);+ const rows = JSON.parse(q(`SELECT coalesce(json_agg(json_build_object('section',section,'key',key,'payload',payload,'signatures',signatures) ORDER BY section, key),'[]') FROM ontology_proposals WHERE kb_id='${KB}' AND proposed_by='agent' AND status='open'`));+ const proposals = rows.filter((r) => r.section !== "map_to");+ const maps = rows.filter((r) => r.section === "map_to");+ const reviews = Object.fromEntries(q(`SELECT kind||'/'||outcome||'|'||count(*) FROM ontology_agent_reviews WHERE kb_id='${KB}' GROUP BY kind, outcome`).split("\n").filter(Boolean).map((l) => l.split("|")).map(([k, n]) => [k, Number(n)]));++ // 裁判一:提案是不是已有的。拿全词表判+ const ontology = await api("GET", `/api/v1/kbs/${KB}/ontology`);+ const classKey = new Map(ontology.entity_types.map((c) => [c.id, c.key]));+ const glossary = [+ "Classes:", ...ontology.entity_types.map((c) => `- ${c.key} · ${c.label} · ${c.description ?? ""}`),+ "Properties:", ...ontology.relation_types.map((r) => `- ${r.key} · ${r.label} · ${r.kind} · ${(r.domains ?? []).map((d) => classKey.get(d)).join("|")} → ${(r.ranges ?? []).map((d) => classKey.get(d)).join("|")} · ${r.description ?? ""}`),+ ].join("\n");+ // shown = 被判重复/反向的里,服务端给的"最近的已有元素"里就有裁判说的那一个:审的人看得见+ const dup = { duplicate: 0, inverse: 0, new: 0, unjudged: 0, shown: 0, items: [] };+ for (let i = 0; i < proposals.length; i += 15) {+ const batch = proposals.slice(i, i + 15);+ const list = batch.map((p, j) => `${j}: [${p.section}] ${p.key} · ${p.payload.label} · ${(p.payload.domains ?? p.payload.parents ?? []).join("|")} → ${(p.payload.ranges ?? []).join("|")} · ${p.payload.description} · phrases: ${(p.payload.forms ?? []).join("; ")}${(p.payload.kind_words ?? []).length ? " · kind words: " + p.payload.kind_words.join("; ") : ""}`).join("\n");+ const out = await judge(DUP, `${glossary}\n\nProposals:\n${list}`);+ batch.forEach((p, j) => {+ const r = out?.results?.find((x) => Number(x.i) === j);+ const v = ["duplicate", "inverse", "new"].includes(r?.verdict) ? r.verdict : "unjudged";+ const closest = (p.payload.closest ?? []).map((c) => c.key);+ const shown = (v === "duplicate" || v === "inverse") && closest.includes(String(r?.of ?? ""));+ if (shown) dup.shown++;+ dup[v]++; dup.items.push({ key: p.key, verdict: v, of: r?.of ?? null, closest, shown });+ });+ }+ // 裁判二:「已有」答得对不对。每条形状配一条例句+ const propByKey = new Map(ontology.relation_types.map((r) => [r.key, r]));+ const shapes = [];+ for (const m of maps) {+ const prop = propByKey.get(m.key);+ if (!prop) continue;+ for (const s of m.signatures?.phrases ?? []) {+ const ex = q(`SELECT se.canonical_name||' — '||f.phrase||' → '||coalesce(oe.canonical_name, f.object_value->>'value', f.object_value#>>'{}', '?') FROM facts f JOIN entities se ON se.id=f.subject_id LEFT JOIN entities oe ON oe.id=f.object_id WHERE f.kb_id='${KB}' AND f.layer='open' AND f.phrase='${String(s.phrase).replace(/'/g, "''")}' LIMIT 1`);+ shapes.push({ key: m.key, shape: s, text: `${s.subject ?? "?"} — "${s.phrase}" → ${s.value ? "value" : (s.object ?? "?")} · e.g. ${ex || "(no example)"} · property ${prop.key} (${prop.label}): ${prop.description ?? ""} · direction claimed: ${s.direction ?? "forward"}` });+ }+ }+ const exist = { right: 0, wrong_direction: 0, wrong: 0, unjudged: 0, items: [] };+ for (let i = 0; i < shapes.length; i += 20) {+ const batch = shapes.slice(i, i + 20);+ const out = await judge(EXIST, batch.map((s, j) => `${j}: ${s.text}`).join("\n"));+ batch.forEach((s, j) => {+ const r = out?.results?.find((x) => Number(x.i) === j);+ const v = ["right", "wrong_direction", "wrong"].includes(r?.verdict) ? r.verdict : "unjudged";+ exist[v]++; exist.items.push({ key: s.key, phrase: s.shape.phrase, direction: s.shape.direction ?? "forward", verdict: v });+ });+ }+ return {+ arm: name, seconds: Math.round((Date.now() - t0) / 1000),+ offered: started ? { shapes: +started[1], kind_words: +started[2], questions: +started[3], calls: +started[4] } : null,+ run: ended ? { proposals_and_maps: +ended[1], failed_batches: +ended[2], malformed_items: +ended[3], proposed_existing_key: +ended[4] } : null,+ tokens: { calls: usage.length, prompt, completion, total: prompt + completion, prompt_per_call: usage.length ? Math.round(prompt / usage.length) : 0 },+ malformed_reasons: why,+ proposals: proposals.length, map_to: maps.length, map_to_shapes: shapes.length, reviews, duplicates: dup, existing: exist,+ };+}++async function main() {+ await login();+ const results = [];+ for (const a of ARMS) results.push(await arm(a));+ const pct = (n, d) => (d ? `${((100 * n) / d).toFixed(0)}%` : "–");+ const row = (label, f) => console.log(`${label.padEnd(44)} ${results.map((r) => String(f(r)).padStart(14)).join(" ")}`);+ console.log(`${"".padEnd(44)} ${results.map((r) => r.arm.padStart(14)).join(" ")}`);+ row("shapes / kind words offered", (r) => `${r.offered?.shapes}/${r.offered?.kind_words}`);+ row("calls", (r) => r.tokens.calls);+ row("failed batches (call or unreadable reply)", (r) => r.run?.failed_batches);+ row("malformed items", (r) => r.run?.malformed_items);+ row(" reasons", (r) => Object.entries(r.malformed_reasons).map(([k, n]) => `${n}×${k}`).join("; ").slice(0, 14) || "–");+ row("proposed a key that already exists", (r) => r.run?.proposed_existing_key);+ row("prompt tokens / call", (r) => r.tokens.prompt_per_call);+ row("tokens / round", (r) => r.tokens.total);+ row("proposals (new elements)", (r) => r.proposals);+ row(" judged duplicate of an existing element", (r) => `${r.duplicates.duplicate} (${pct(r.duplicates.duplicate, r.proposals)})`);+ row(" judged inverse of an existing property", (r) => `${r.duplicates.inverse} (${pct(r.duplicates.inverse, r.proposals)})`);+ row(" of those, the existing one is shown beside", (r) => `${r.duplicates.shown} of ${r.duplicates.duplicate + r.duplicates.inverse}`);+ row(" judged new", (r) => `${r.duplicates.new} (${pct(r.duplicates.new, r.proposals)})`);+ row("\"already in the ontology\" shapes", (r) => r.map_to_shapes);+ row(" judged right", (r) => `${r.existing.right} (${pct(r.existing.right, r.map_to_shapes)})`);+ row(" judged wrong direction", (r) => `${r.existing.wrong_direction} (${pct(r.existing.wrong_direction, r.map_to_shapes)})`);+ row(" judged wrong", (r) => `${r.existing.wrong} (${pct(r.existing.wrong, r.map_to_shapes)})`);+ row("shapes declined", (r) => r.reviews["phrase/declined"] ?? 0);+ for (const r of results) if (Object.keys(r.malformed_reasons).length) console.log(`malformed in ${r.arm}: ${JSON.stringify(r.malformed_reasons)}`);+ if (args.out) fs.writeFileSync(args.out, JSON.stringify(results, null, 1));+}+main().catch((e) => { console.error(e); process.exit(1); });diff --git a/web/src/api.ts b/web/src/api.tsindex bf23714d..55127b24 100644--- a/web/src/api.ts+++ b/web/src/api.ts@@ -1322,6 +1322,8 @@ export interface AgentProposalFields {kind_words?: string[];/** 原文里的几句,给人核对定义用 */examples?: string[];+ /** 本体里离这条提案最近的已有元素(按向量):采纳前看它是不是其实已经有了 */+ closest?: { key: string; label: string; distance: number }[];domains?: string[];ranges?: string[];parents?: string[];diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex ddc3e4b6..3b3a30a3 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -1758,6 +1758,7 @@ export const en = {n === 1 ? "binds 1 phrase shape" : `binds ${n} phrase shapes`,bindsKindWords: (n: number) =>n === 1 ? "1 kind word" : `${n} kind words`,+ closestExisting: (key: string, label: string) => `closest: ${key} ${label}`,rejectProposal: "Reject",rejectReasonPrompt:"Why not? (optional — the agent reads it before proposing again)",diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 0d4040d8..b3b67331 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -1528,6 +1528,7 @@ export const zh: Strings = {servesQuestions: (n: number) => `服务 ${n} 个问题`,bindsShapes: (n: number) => `会绑 ${n} 种短语形状`,bindsKindWords: (n: number) => `${n} 个类别词`,+ closestExisting: (key: string, label: string) => `最近的已有:${key} ${label}`,rejectProposal: "拒绝",rejectReasonPrompt: "为什么不要?(可不填——代理下次提案前会读)",rejectedProposal: "已拒绝",diff --git a/web/src/pages/Ontology.tsx b/web/src/pages/Ontology.tsxindex cb716745..38cb3a7f 100644--- a/web/src/pages/Ontology.tsx+++ b/web/src/pages/Ontology.tsx@@ -1887,6 +1887,16 @@ function AgentMeta({{!!p.serves?.length && (<Chip tone="info">{S.ontology.servesQuestions(p.serves.length)}</Chip>)}+ {!!p.closest?.length && (+ <span+ className="text-small text-ink-2 whitespace-nowrap"+ title={p.closest+ .map((c) => `${c.key} · ${c.label} · ${c.distance.toFixed(2)}`)+ .join("\n")}+ >+ {S.ontology.closestExisting(p.closest[0].key, p.closest[0].label)}+ </span>+ )}</>);}