Skip to content

World's first open-source enterprise world model.

Measure the agent's glossary on the same shapes, and carry the properties nearest in meaning (#965)

WaylandYangf1a36a583ba8
diff --git a/crates/utopia-extract/src/ontology_agent.rs b/crates/utopia-extract/src/ontology_agent.rs
index 84d3b4f0..51074946 100644
--- a/crates/utopia-extract/src/ontology_agent.rs
+++ b/crates/utopia-extract/src/ontology_agent.rs
@@ -105,6 +105,15 @@ pub struct ProposalReply {
pub existing: Vec<Existing>,
/// 读不出的项数
pub malformed: usize,
+ /// 每个坏项为什么坏:测量台数它们,日志里也看得见
+ pub reasons: Vec<String>,
+}
+
+impl ProposalReply {
+ fn bad(&mut self, why: String) {
+ self.malformed += 1;
+ self.reasons.push(why);
+ }
}
const SYSTEM: &str = "\
@@ -130,6 +139,8 @@ instead of proposing, with dir: forward when the shape's subject is the property
when the shape reads the property backwards (\"X wrote Y\" is author read in reverse).\n\
- Do not propose the inverse of an existing property: list the shape under that property with dir \
reverse.\n\
+- An existing entry holds shapes, not questions: every entry under existing lists at least one \
+shape id in s or k. Do not add an entry only to say that a property serves a question.\n\
Answer with one JSON object and nothing else:\n\
{\"p\":[{\"kind\":\"property\",\"key\":\"...\",\"label\":\"...\",\"definition\":\"...\",\"value\":false,\
\"datatype\":null,\"domains\":[\"class_key\"],\"ranges\":[\"class_key\"],\"s\":[ids],\"k\":[ids],\"q\":[ids]},\
@@ -293,14 +304,14 @@ pub fn parse_proposal_response(
Some(Value::Object(m)) => m.values().collect(),
Some(Value::Null) | None => Vec::new(),
Some(_) => {
- reply.malformed += 1;
+ reply.bad("`p` is not a list".to_string());
Vec::new()
}
};
let mut seen: HashSet<String> = HashSet::new();
for it in items {
let Some(obj) = it.as_object() else {
- reply.malformed += 1;
+ reply.bad("a proposal is not an object".to_string());
continue;
};
let kind = obj
@@ -318,7 +329,7 @@ pub fn parse_proposal_response(
|| !valid_key(&key)
|| !seen.insert(key.clone())
{
- reply.malformed += 1;
+ reply.bad(format!("bad kind, bad key or repeated key: {kind} {key:?}"));
continue;
}
let label = obj
@@ -345,7 +356,9 @@ pub fn parse_proposal_response(
strings(obj.get("ranges")),
strings(obj.get("parents")),
) else {
- reply.malformed += 1;
+ reply.bad(format!(
+ "domains, ranges or parents are not lists of strings: {key}"
+ ));
continue;
};
let (Ok(signatures), Ok(kind_words), Ok(questions)) = (
@@ -353,12 +366,12 @@ pub fn parse_proposal_response(
ids(obj.get("k"), 'k', kind_word_ids),
ids(obj.get("q"), 'q', question_ids),
) else {
- reply.malformed += 1;
+ reply.bad(format!("a shape or question id outside the batch: {key}"));
continue;
};
// 什么形状都不绑的提案不是从图谱里长出来的:不收
if signatures.is_empty() && kind_words.is_empty() {
- reply.malformed += 1;
+ reply.bad(format!("binds no shape: {key}"));
continue;
}
reply.proposals.push(Proposal {
@@ -383,7 +396,7 @@ pub fn parse_proposal_response(
if let Some(Value::Array(ex)) = v.get("existing") {
for it in ex {
let Some(obj) = it.as_object() else {
- reply.malformed += 1;
+ reply.bad("an existing entry is not an object".to_string());
continue;
};
let key = obj
@@ -395,18 +408,18 @@ pub fn parse_proposal_response(
ids(obj.get("s"), 's', signature_ids),
ids(obj.get("k"), 'k', kind_word_ids),
) else {
- reply.malformed += 1;
+ reply.bad(format!("existing: a shape id outside the batch: {key}"));
continue;
};
if key.is_empty() || (signatures.is_empty() && kind_words.is_empty()) {
- reply.malformed += 1;
+ reply.bad(format!("existing: no key or no shapes: {key:?}"));
continue;
}
let direction = match obj.get("dir").and_then(Value::as_str).map(str::trim) {
Some("reverse") => Some("reverse".to_string()),
Some("forward") | None | Some("") => Some("forward".to_string()),
Some(_) => {
- reply.malformed += 1;
+ reply.bad(format!("existing: unknown direction: {key}"));
continue;
}
};
diff --git a/crates/utopia-server/src/api/question_routes.rs b/crates/utopia-server/src/api/question_routes.rs
index 3b20f2c3..ca4fcc9c 100644
--- a/crates/utopia-server/src/api/question_routes.rs
+++ b/crates/utopia-server/src/api/question_routes.rs
@@ -213,19 +213,28 @@ pub async fn report(
})))
}
+#[derive(Deserialize, Default)]
+pub struct ProposeReq {
+ /// 测量用(bench/agent.mjs):`full` = 每批送全词表。缺省是按批裁过的
+ #[serde(default)]
+ pub glossary: Option<String>,
+}
+
/// 叫代理来看一眼:排一份任务,马上回。结果落在 `ontology_proposals`,界面从那里读
pub async fn propose(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path(kb_id): Path<Uuid>,
+ Json(req): Json<ProposeReq>,
) -> ApiResult<Json<serde_json::Value>> {
utopia_store::access::require_kb(&state.pool, &user, kb_id, Role::Editor).await?;
- let queued = utopia_store::jobs::enqueue_unless_queued(
- &state.pool,
- "propose_ontology",
- json!({ "kb_id": kb_id }),
- )
- .await?;
+ let payload = if req.glossary.as_deref() == Some("full") {
+ json!({ "kb_id": kb_id, "glossary": "full" })
+ } else {
+ json!({ "kb_id": kb_id })
+ };
+ let queued =
+ utopia_store::jobs::enqueue_unless_queued(&state.pool, "propose_ontology", payload).await?;
Ok(Json(json!({ "queued": queued.is_some() })))
}
diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs
index 3d7b7ea0..d3173377 100644
--- a/crates/utopia-server/src/main.rs
+++ b/crates/utopia-server/src/main.rs
@@ -644,7 +644,9 @@ async fn dispatch(st: &state::AppState, job: &utopia_store::jobs::Job) -> anyhow
.and_then(|v| v.as_str())
.and_then(|s| s.parse().ok())
.ok_or_else(|| anyhow::anyhow!("payload 缺少 kb_id"))?;
- ontology_agent::propose(st, kb_id).await
+ // 测量用:payload 里 glossary = full 时每批送全词表(bench/agent.mjs)
+ let full = job.payload.get("glossary").and_then(|v| v.as_str()) == Some("full");
+ ontology_agent::propose_with(st, kb_id, full).await
}
"propose_questions" => {
let kb_id: Uuid = job
diff --git a/crates/utopia-server/src/ontology_agent.rs b/crates/utopia-server/src/ontology_agent.rs
index 47fa5b1c..526edf3f 100644
--- a/crates/utopia-server/src/ontology_agent.rs
+++ b/crates/utopia-server/src/ontology_agent.rs
@@ -32,6 +32,8 @@ const TOP_ENTITIES: usize = 20;
/// 一条形状结构上对得上的属性多过这个数,就按向量只留最近的这么多带定义(同对齐的短名单)
const GLOSSARY_PER_SHAPE: usize = 10;
+/// 每条形状按意思最近的这么多条属性,结构对不上也带定义
+const GLOSSARY_NEAREST_ANYWAY: usize = 5;
/// 一次调用交给模型的形状数(0044 决定 3 的成本教训:词表每批只带一次)
pub(crate) const SIGNATURES_PER_CALL: usize = 12;
@@ -53,7 +55,18 @@ fn shape_of(sig: &PhraseSignature) -> Value {
}
/// 任务入口:一个库同时只跑一份(同两种对齐,并行会把端点打出 502)
+#[cfg(test)]
pub async fn propose(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> {
+ propose_with(state, kb_id, false).await
+}
+
+/// `full_glossary` 是测量用的旋钮(`scripts/bench/agent.mjs`):每批都送全部属性的定义,
+/// 好和按批裁过的词表在同一批形状上比失败率、重复提案和 token。产品路径不开它
+pub async fn propose_with(
+ state: &AppState,
+ kb_id: Uuid,
+ full_glossary: bool,
+) -> anyhow::Result<()> {
let pool = &state.pool;
let kb = utopia_store::kbs::get(pool, kb_id).await?;
let settings = utopia_store::settings::get(pool, kb.workspace_id)
@@ -73,7 +86,7 @@ pub async fn propose(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> {
tracing::info!(%kb_id, "本体代理已有一份在跑,这次跳过");
return Ok(());
}
- let result = propose_locked(state, kb_id, &settings, &client).await;
+ let result = propose_locked(state, kb_id, &settings, &client, full_glossary).await;
let _ = sqlx::query("SELECT pg_advisory_unlock(hashtext('propose_ontology'), hashtext($1))")
.bind(kb_id.to_string())
.execute(&mut *guard)
@@ -99,6 +112,7 @@ async fn propose_locked(
kb_id: Uuid,
settings: &utopia_core::models::LlmSettings,
client: &utopia_llm::LlmClient,
+ full_glossary: bool,
) -> anyhow::Result<()> {
let pool = &state.pool;
let classes = utopia_store::graph::entity_types(pool, kb_id).await?;
@@ -305,6 +319,9 @@ async fn propose_locked(
};
let glossary_for = |ids: std::ops::Range<usize>| -> agent::Glossary<'_> {
let mut full: HashSet<Uuid> = HashSet::new();
+ if full_glossary {
+ full.extend(props.iter().map(|p| p.id));
+ }
for i in ids {
let (Some(s), near) = (open.get(i), nearest.get(i)) else {
continue;
@@ -313,6 +330,12 @@ async fn propose_locked(
.iter()
.filter(|p| crate::phrase_alignment::structurally_fits(p, s, &closure))
.collect();
+ // 意思最近的几条不管结构对不对得上都给定义:一端没类型、或宾语是字面值的形状,
+ // 结构上对不上 positionHeld / workLocation,模型看不到定义就把它们又提了一遍
+ // (受控对比里重复提案 1% → 6%,十条里七条是这样来的)
+ if let Some(near) = near {
+ full.extend(near.iter().take(GLOSSARY_NEAREST_ANYWAY).copied());
+ }
if fitting.len() > GLOSSARY_PER_SHAPE {
if let Some(near) = near.filter(|n| !n.is_empty()) {
full.extend(
@@ -420,6 +443,9 @@ async fn propose_locked(
}
};
malformed += parsed.malformed;
+ for why in &parsed.reasons {
+ tracing::warn!(%kb_id, batch = b, why = %why, "本体代理:坏项");
+ }
let mut sig_outcome: HashMap<i64, (&str, String)> = HashMap::new();
let mut word_outcome: HashMap<i64, (&str, String)> = HashMap::new();
// 「本体里已经有」:存成 map_to 提案,人点「用已有的」就是一次人的绑定判定
@@ -430,6 +456,7 @@ async fn propose_locked(
"class".to_string()
} else {
malformed += 1;
+ tracing::warn!(%kb_id, batch = b, why = %format!("existing: no such key in the ontology: {}", e.key), "本体代理:坏项");
continue;
};
let slot = format!("map_to:{}", e.key);
@@ -632,6 +659,48 @@ async fn propose_locked(
.collect();
// 同一个目标上一轮已经答过一批形状:并起来,不是盖掉(第一次真跑第三轮盖掉了第二轮的)
let mut items = items;
+ // 每条新元素带上本体里离它最近的两个已有元素(按向量,不问模型)。裁过的词表下模型看不到
+ // 大部分属性的定义,受控对比里重复或反向已有属性的提案从 1% 升到 5%(awardReceived、
+ // performer 这几条);审的人要在采纳前看见"它最像谁"
+ for class_side in [false, true] {
+ let idx: Vec<usize> = items
+ .iter()
+ .enumerate()
+ .filter(|(_, it)| {
+ it.section != "map_to" && (it.section == "entity_types") == class_side
+ })
+ .map(|(i, _)| i)
+ .collect();
+ let queries: Vec<String> = idx
+ .iter()
+ .map(|i| {
+ let p = &items[*i].payload;
+ format!(
+ "{} · {}",
+ p.get("label").and_then(Value::as_str).unwrap_or(""),
+ p.get("description").and_then(Value::as_str).unwrap_or("")
+ )
+ })
+ .collect();
+ let target = if class_side {
+ crate::ontology_index::Target::Class
+ } else {
+ crate::ontology_index::Target::Predicate(None)
+ };
+ match crate::ontology_index::nearest_for_each(state, kb_id, &queries, 2, target).await {
+ Ok(found) => {
+ for (i, cands) in idx.into_iter().zip(found) {
+ items[i].payload["closest"] = json!(cands
+ .iter()
+ .map(|c| json!({ "key": c.key, "label": c.label, "distance": c.distance }))
+ .collect::<Vec<_>>());
+ }
+ }
+ Err(e) => {
+ tracing::warn!(%kb_id, error = %e, "本体代理:最近的已有元素没算出来,提案不带这一格");
+ }
+ }
+ }
for it in items.iter_mut().filter(|it| it.section == "map_to") {
if let Some(prev) =
utopia_store::ontology::open_proposal(pool, kb_id, "map_to", &it.key).await?
@@ -641,7 +710,7 @@ async fn propose_locked(
}
utopia_store::ontology::save_agent_proposals(pool, kb_id, &items).await?;
agent_reviews::record(pool, kb_id, &reviews).await?;
- tracing::info!(%kb_id, proposals = items.len(), failed, malformed, skipped_existing, "本体代理结束");
+ tracing::info!(%kb_id, proposals = items.len(), failed, malformed, skipped_existing, calls, glossary = if full_glossary { "full" } else { "trimmed" }, "本体代理结束");
if !items.is_empty() {
state.emit_pending(kb_id);
}
diff --git a/scripts/bench/README.md b/scripts/bench/README.md
index 923ef5c7..13347be8 100644
--- a/scripts/bench/README.md
+++ b/scripts/bench/README.md
@@ -248,6 +248,41 @@ node scripts/bench/competency.mjs --kb <kb-id> --seed scripts/bench/truth/redocr
- `truth/redocred-typed.questions.json` 是给 `typed.mjs` 那份 100 篇 Re-DocRED 库写的十一条:十条有期望答案
(其中两条要代理采纳过的 `bordered_by`、`p40` 才答得上),一条只有形状。换一个库要另写。
+## 本体代理的测量台(0061)
+
+`agent.mjs` 在同一个库状态、同一批形状上各臂跑一轮:每臂开跑前把代理没采纳的提案和看过的形状记录删掉
+(所以只在测量库上用,要显式给 `--reset`),从服务端日志读失败批次、坏项及原因、token,再让裁判拿着
+**全词表**判每条新提案是不是已有元素的重复或反向,判每条「已有」答案对不对、方向对不对。
+`POST /ontology/propose` 的 `glossary: full` 是给全词表那一臂用的旋钮,产品路径不开。
+
+```
+BENCH_BASE=http://127.0.0.1:1524 BENCH_PSQL="docker exec ... -d utopia_bench4 -tAc" \
+BENCH_SERVER_LOG=/path/to/server.log BENCH_JUDGE_BASE=... BENCH_JUDGE_KEY=... BENCH_JUDGE_MODEL=... \
+node scripts/bench/agent.mjs --kb <kb-id> --db utopia_bench4 --reset --arms trimmed,full,trimmed,full
+```
+
+**2026-09-27,bench4(100 篇 Re-DocRED,98 条属性),gemini-3.5-flash,每轮 120 条形状 + 69 个类别词、12 次调用:**
+
+| | 全词表 | 按结构裁(#963) | 按结构裁 + 意思最近 5 条 |
+|---|---|---|---|
+| 轮数 | 4 | 7 | 4 |
+| 失败批次 | 0 / 48 | 0 / 84 | 0 / 48 |
+| 提示 token / 次 | 7.1k | 4.3k | 5.0k |
+| 一轮 token | 111k | 78k | 87k |
+| 新元素提案 | 135 | 292 | 178 |
+| 其中重复或反向已有属性 | 1(0.7%) | 18(6.2%) | 6(3.4%) |
+| 「已有」答案判对 | 44 / 65(68%) | 60 / 85(71%) | 44 / 56(79%) |
+
+- 裁词表不让调用失败,也不让模型提已存在的键;让它**重提定义被裁掉的属性**。按结构裁时重的集中在
+ positionHeld、workLocation、performer、awardReceived:形状一端没类型或宾语是字面值,结构上对不上,
+ 定义就没送。再按意思带上最近的 5 条,重复降了一半,多花 0.7k token 一次。
+- 每轮固定出现的十来个坏项是"existing 条目没有形状",两臂都有,数目等于库里的问题数:模型把"这条属性
+ 服务某个问题"写进了 existing。解析时丢掉,无害;提示词里已补一句。
+- **同一臂两轮之间的差别比两臂之间大**:同样的输入、温度 0,两轮只有一半提案键相同。单轮对比说明不了
+ 质量,至少四轮合起来看。
+- 提案旁边显示的"最近的已有元素"(按向量取两条)只罩住了被判重复的 16 条里的 6 条,是给人看的线索,
+ 不是防线。
+
## 读数怎么算
- `prompt_tokens_est` 是**本体段**的估算,不是整个提示词。实测 4.0 字符 ≈ 1 token
diff --git a/scripts/bench/agent.mjs b/scripts/bench/agent.mjs
new file mode 100644
index 00000000..10a8a8f1
--- /dev/null
+++ b/scripts/bench/agent.mjs
@@ -0,0 +1,171 @@
+#!/usr/bin/env node
+// 本体代理的 A/B 测量台(ADR 0061):同一个库状态、同一批形状,各臂跑一轮,量
+// - 失败:调用失败或读不出的批次、坏项、模型提了已有键;
+// - 重复:提案其实是已有属性(或它的反向)——裁词表的主要风险,裁判拿着全词表判;
+// - 「已有」答案的准确率:形状真的说的是那条属性、方向对不对;
+// - token(服务端日志的 llm usage 行)。
+//
+// BENCH_BASE=http://127.0.0.1:1524 BENCH_PSQL="docker exec ... -d utopia_bench4 -tAc" \
+// BENCH_SERVER_LOG=/path/to/server.log BENCH_JUDGE_BASE=... BENCH_JUDGE_KEY=... BENCH_JUDGE_MODEL=... \
+// node scripts/bench/agent.mjs --kb <kb-id> --db utopia_bench4 --reset [--arms trimmed,full] [--out file.json]
+//
+// **--reset 会删这个库里代理的提案(没采纳的)和看过的形状记录**,每臂开跑前各删一次,好让两臂
+// 看到同一批形状。只在测量库上用;不给 --reset 就不跑。
+import fs from "node:fs";
+import { api, login, parseArgs, log, onDb, sleep } from "./lib.mjs";
+
+const args = parseArgs(process.argv);
+const KB = args.kb, DB = args.db;
+const ARMS = String(args.arms || "trimmed,full").split(",").map((s) => s.trim()).filter(Boolean);
+const LOG = process.env.BENCH_SERVER_LOG;
+if (!KB || !DB) { console.error("--kb <id> 与 --db <bench database> 是必须的"); process.exit(2); }
+if (!args.reset) { console.error("这台子要重置代理的提案和记录才能让两臂看到同一批形状:确认是测量库后加 --reset"); process.exit(2); }
+if (!LOG) { console.error("给 BENCH_SERVER_LOG:失败数和 token 从服务端日志里读"); process.exit(2); }
+const q = (sql) => onDb(DB, sql);
+
+const ep = { base: process.env.BENCH_JUDGE_BASE, key: process.env.BENCH_JUDGE_KEY || "", model: process.env.BENCH_JUDGE_MODEL };
+if (!ep.base || !ep.model) { console.error("给 BENCH_JUDGE_BASE / _KEY / _MODEL"); process.exit(2); }
+async function judge(system, user) {
+ for (let attempt = 0; attempt < 3; attempt++) {
+ const r = await fetch(`${ep.base.replace(/\/$/, "")}/chat/completions`, {
+ method: "POST", headers: { "content-type": "application/json", ...(ep.key ? { authorization: `Bearer ${ep.key}` } : {}) },
+ body: JSON.stringify({ model: ep.model, temperature: 0, messages: [{ role: "system", content: system }, { role: "user", content: user }] }),
+ });
+ if (r.ok) {
+ const text = (await r.json()).choices?.[0]?.message?.content ?? "";
+ const m = /\{[\s\S]*\}/.exec(text);
+ if (m) { try { return JSON.parse(m[0]); } catch { /* 再问一次 */ } }
+ } else if (r.status !== 429 && r.status < 500) throw new Error(`judge -> ${r.status} ${(await r.text()).slice(0, 200)}`);
+ await sleep(3000 * (attempt + 1));
+ }
+ return null;
+}
+
+const DUP = `You review proposals to extend an ontology. You are given the full ontology (every class and property with its definition) and a list of proposed new elements. For each proposal decide whether the ontology ALREADY has it:
+- "duplicate": an existing property or class means the same thing (name it);
+- "inverse": an existing property is the same relation read in the other direction (name it);
+- "new": nothing existing covers it.
+Be strict about meaning, not wording. Output one JSON object: {"results":[{"i":0,"verdict":"duplicate|inverse|new","of":"existing_key or null"}]}`;
+const EXIST = `You check decisions that a statement shape is an instance of an existing ontology property. Each item gives a phrase between a subject class and an object class (or a value), one example statement, the property with its definition, and the direction claimed: forward = the statement's subject is the property's subject; reverse = the statement reads the property backwards.
+- "right": the phrase asserts this property in this direction;
+- "wrong_direction": the property is right but the direction is not;
+- "wrong": the phrase does not assert this property (too vague, or a different meaning).
+Output one JSON object: {"results":[{"i":0,"verdict":"right|wrong_direction|wrong"}]}`;
+
+function logWindow(from) {
+ const buf = fs.readFileSync(LOG);
+ // eslint-disable-next-line no-control-regex
+ return buf.subarray(from).toString("utf8").replace(/\x1b\[[0-9;]*m/g, "");
+}
+
+async function arm(name) {
+ log(`===== arm ${name}: reset`);
+ q(`DELETE FROM ontology_agent_reviews WHERE kb_id='${KB}'`);
+ q(`DELETE FROM ontology_proposals WHERE kb_id='${KB}' AND proposed_by='agent' AND status<>'adopted'`);
+ q(`DELETE FROM jobs WHERE kind='propose_ontology' AND status='queued' AND payload->>'kb_id'='${KB}'`);
+ const from = fs.statSync(LOG).size;
+ const t0 = Date.now();
+ await api("POST", `/api/v1/kbs/${KB}/ontology/propose`, name === "full" ? { glossary: "full" } : {});
+ for (;;) {
+ await sleep(10000);
+ const st = q(`SELECT status FROM jobs WHERE kind='propose_ontology' AND payload->>'kb_id'='${KB}' ORDER BY id DESC LIMIT 1`);
+ if (st !== "queued" && st !== "running") { if (st !== "done") log(`job ended as ${st}`); break; }
+ }
+ const text = logWindow(from);
+ const started = /本体代理开始 .*shapes=(\d+) kind_words=(\d+) questions=(\d+) calls=(\d+)/.exec(text);
+ const ended = /本体代理结束 .*proposals=(\d+) failed=(\d+) malformed=(\d+) skipped_existing=(\d+)/.exec(text);
+ // 坏项各是为什么坏的(按原因归并,键去掉)
+ const why = {};
+ for (const m of text.matchAll(/本体代理:坏项 .*why=(.*)$/gm)) {
+ const k = m[1].replace(/^"|"$/g, "").replace(/:\s[^:]*$/, "").trim();
+ why[k] = (why[k] ?? 0) + 1;
+ }
+ const usage = [...text.matchAll(/llm usage .*prompt=(\d+) completion=(\d+)/g)];
+ const prompt = usage.reduce((n, m) => n + Number(m[1]), 0), completion = usage.reduce((n, m) => n + Number(m[2]), 0);
+ const rows = JSON.parse(q(`SELECT coalesce(json_agg(json_build_object('section',section,'key',key,'payload',payload,'signatures',signatures) ORDER BY section, key),'[]') FROM ontology_proposals WHERE kb_id='${KB}' AND proposed_by='agent' AND status='open'`));
+ const proposals = rows.filter((r) => r.section !== "map_to");
+ const maps = rows.filter((r) => r.section === "map_to");
+ const reviews = Object.fromEntries(q(`SELECT kind||'/'||outcome||'|'||count(*) FROM ontology_agent_reviews WHERE kb_id='${KB}' GROUP BY kind, outcome`).split("\n").filter(Boolean).map((l) => l.split("|")).map(([k, n]) => [k, Number(n)]));
+
+ // 裁判一:提案是不是已有的。拿全词表判
+ const ontology = await api("GET", `/api/v1/kbs/${KB}/ontology`);
+ const classKey = new Map(ontology.entity_types.map((c) => [c.id, c.key]));
+ const glossary = [
+ "Classes:", ...ontology.entity_types.map((c) => `- ${c.key} · ${c.label} · ${c.description ?? ""}`),
+ "Properties:", ...ontology.relation_types.map((r) => `- ${r.key} · ${r.label} · ${r.kind} · ${(r.domains ?? []).map((d) => classKey.get(d)).join("|")} → ${(r.ranges ?? []).map((d) => classKey.get(d)).join("|")} · ${r.description ?? ""}`),
+ ].join("\n");
+ // shown = 被判重复/反向的里,服务端给的"最近的已有元素"里就有裁判说的那一个:审的人看得见
+ const dup = { duplicate: 0, inverse: 0, new: 0, unjudged: 0, shown: 0, items: [] };
+ for (let i = 0; i < proposals.length; i += 15) {
+ const batch = proposals.slice(i, i + 15);
+ const list = batch.map((p, j) => `${j}: [${p.section}] ${p.key} · ${p.payload.label} · ${(p.payload.domains ?? p.payload.parents ?? []).join("|")} → ${(p.payload.ranges ?? []).join("|")} · ${p.payload.description} · phrases: ${(p.payload.forms ?? []).join("; ")}${(p.payload.kind_words ?? []).length ? " · kind words: " + p.payload.kind_words.join("; ") : ""}`).join("\n");
+ const out = await judge(DUP, `${glossary}\n\nProposals:\n${list}`);
+ batch.forEach((p, j) => {
+ const r = out?.results?.find((x) => Number(x.i) === j);
+ const v = ["duplicate", "inverse", "new"].includes(r?.verdict) ? r.verdict : "unjudged";
+ const closest = (p.payload.closest ?? []).map((c) => c.key);
+ const shown = (v === "duplicate" || v === "inverse") && closest.includes(String(r?.of ?? ""));
+ if (shown) dup.shown++;
+ dup[v]++; dup.items.push({ key: p.key, verdict: v, of: r?.of ?? null, closest, shown });
+ });
+ }
+ // 裁判二:「已有」答得对不对。每条形状配一条例句
+ const propByKey = new Map(ontology.relation_types.map((r) => [r.key, r]));
+ const shapes = [];
+ for (const m of maps) {
+ const prop = propByKey.get(m.key);
+ if (!prop) continue;
+ for (const s of m.signatures?.phrases ?? []) {
+ const ex = q(`SELECT se.canonical_name||' — '||f.phrase||' → '||coalesce(oe.canonical_name, f.object_value->>'value', f.object_value#>>'{}', '?') FROM facts f JOIN entities se ON se.id=f.subject_id LEFT JOIN entities oe ON oe.id=f.object_id WHERE f.kb_id='${KB}' AND f.layer='open' AND f.phrase='${String(s.phrase).replace(/'/g, "''")}' LIMIT 1`);
+ shapes.push({ key: m.key, shape: s, text: `${s.subject ?? "?"} — "${s.phrase}" → ${s.value ? "value" : (s.object ?? "?")} · e.g. ${ex || "(no example)"} · property ${prop.key} (${prop.label}): ${prop.description ?? ""} · direction claimed: ${s.direction ?? "forward"}` });
+ }
+ }
+ const exist = { right: 0, wrong_direction: 0, wrong: 0, unjudged: 0, items: [] };
+ for (let i = 0; i < shapes.length; i += 20) {
+ const batch = shapes.slice(i, i + 20);
+ const out = await judge(EXIST, batch.map((s, j) => `${j}: ${s.text}`).join("\n"));
+ batch.forEach((s, j) => {
+ const r = out?.results?.find((x) => Number(x.i) === j);
+ const v = ["right", "wrong_direction", "wrong"].includes(r?.verdict) ? r.verdict : "unjudged";
+ exist[v]++; exist.items.push({ key: s.key, phrase: s.shape.phrase, direction: s.shape.direction ?? "forward", verdict: v });
+ });
+ }
+ return {
+ arm: name, seconds: Math.round((Date.now() - t0) / 1000),
+ offered: started ? { shapes: +started[1], kind_words: +started[2], questions: +started[3], calls: +started[4] } : null,
+ run: ended ? { proposals_and_maps: +ended[1], failed_batches: +ended[2], malformed_items: +ended[3], proposed_existing_key: +ended[4] } : null,
+ tokens: { calls: usage.length, prompt, completion, total: prompt + completion, prompt_per_call: usage.length ? Math.round(prompt / usage.length) : 0 },
+ malformed_reasons: why,
+ proposals: proposals.length, map_to: maps.length, map_to_shapes: shapes.length, reviews, duplicates: dup, existing: exist,
+ };
+}
+
+async function main() {
+ await login();
+ const results = [];
+ for (const a of ARMS) results.push(await arm(a));
+ const pct = (n, d) => (d ? `${((100 * n) / d).toFixed(0)}%` : "–");
+ const row = (label, f) => console.log(`${label.padEnd(44)} ${results.map((r) => String(f(r)).padStart(14)).join(" ")}`);
+ console.log(`${"".padEnd(44)} ${results.map((r) => r.arm.padStart(14)).join(" ")}`);
+ row("shapes / kind words offered", (r) => `${r.offered?.shapes}/${r.offered?.kind_words}`);
+ row("calls", (r) => r.tokens.calls);
+ row("failed batches (call or unreadable reply)", (r) => r.run?.failed_batches);
+ row("malformed items", (r) => r.run?.malformed_items);
+ row(" reasons", (r) => Object.entries(r.malformed_reasons).map(([k, n]) => `${n}×${k}`).join("; ").slice(0, 14) || "–");
+ row("proposed a key that already exists", (r) => r.run?.proposed_existing_key);
+ row("prompt tokens / call", (r) => r.tokens.prompt_per_call);
+ row("tokens / round", (r) => r.tokens.total);
+ row("proposals (new elements)", (r) => r.proposals);
+ row(" judged duplicate of an existing element", (r) => `${r.duplicates.duplicate} (${pct(r.duplicates.duplicate, r.proposals)})`);
+ row(" judged inverse of an existing property", (r) => `${r.duplicates.inverse} (${pct(r.duplicates.inverse, r.proposals)})`);
+ row(" of those, the existing one is shown beside", (r) => `${r.duplicates.shown} of ${r.duplicates.duplicate + r.duplicates.inverse}`);
+ row(" judged new", (r) => `${r.duplicates.new} (${pct(r.duplicates.new, r.proposals)})`);
+ row("\"already in the ontology\" shapes", (r) => r.map_to_shapes);
+ row(" judged right", (r) => `${r.existing.right} (${pct(r.existing.right, r.map_to_shapes)})`);
+ row(" judged wrong direction", (r) => `${r.existing.wrong_direction} (${pct(r.existing.wrong_direction, r.map_to_shapes)})`);
+ row(" judged wrong", (r) => `${r.existing.wrong} (${pct(r.existing.wrong, r.map_to_shapes)})`);
+ row("shapes declined", (r) => r.reviews["phrase/declined"] ?? 0);
+ for (const r of results) if (Object.keys(r.malformed_reasons).length) console.log(`malformed in ${r.arm}: ${JSON.stringify(r.malformed_reasons)}`);
+ if (args.out) fs.writeFileSync(args.out, JSON.stringify(results, null, 1));
+}
+main().catch((e) => { console.error(e); process.exit(1); });
diff --git a/web/src/api.ts b/web/src/api.ts
index bf23714d..55127b24 100644
--- a/web/src/api.ts
+++ b/web/src/api.ts
@@ -1322,6 +1322,8 @@ export interface AgentProposalFields {
kind_words?: string[];
/** 原文里的几句,给人核对定义用 */
examples?: string[];
+ /** 本体里离这条提案最近的已有元素(按向量):采纳前看它是不是其实已经有了 */
+ closest?: { key: string; label: string; distance: number }[];
domains?: string[];
ranges?: string[];
parents?: string[];
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index ddc3e4b6..3b3a30a3 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -1758,6 +1758,7 @@ export const en = {
n === 1 ? "binds 1 phrase shape" : `binds ${n} phrase shapes`,
bindsKindWords: (n: number) =>
n === 1 ? "1 kind word" : `${n} kind words`,
+ closestExisting: (key: string, label: string) => `closest: ${key} ${label}`,
rejectProposal: "Reject",
rejectReasonPrompt:
"Why not? (optional — the agent reads it before proposing again)",
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index 0d4040d8..b3b67331 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -1528,6 +1528,7 @@ export const zh: Strings = {
servesQuestions: (n: number) => `服务 ${n} 个问题`,
bindsShapes: (n: number) => `会绑 ${n} 种短语形状`,
bindsKindWords: (n: number) => `${n} 个类别词`,
+ closestExisting: (key: string, label: string) => `最近的已有:${key} ${label}`,
rejectProposal: "拒绝",
rejectReasonPrompt: "为什么不要?(可不填——代理下次提案前会读)",
rejectedProposal: "已拒绝",
diff --git a/web/src/pages/Ontology.tsx b/web/src/pages/Ontology.tsx
index cb716745..38cb3a7f 100644
--- a/web/src/pages/Ontology.tsx
+++ b/web/src/pages/Ontology.tsx
@@ -1887,6 +1887,16 @@ function AgentMeta({
{!!p.serves?.length && (
<Chip tone="info">{S.ontology.servesQuestions(p.serves.length)}</Chip>
)}
+ {!!p.closest?.length && (
+ <span
+ className="text-small text-ink-2 whitespace-nowrap"
+ title={p.closest
+ .map((c) => `${c.key} · ${c.label} · ${c.distance.toFixed(2)}`)
+ .join("\n")}
+ >
+ {S.ontology.closestExisting(p.closest[0].key, p.closest[0].label)}
+ </span>
+ )}
</>
);
}