Skip to content

World's first open-source enterprise world model.

Every arm gets its own knowledge base (#83)

WaylandYang116c56805c13
diff --git a/.gitignore b/.gitignore
index e0efb2b7..4e23c096 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,7 @@ web/dist/
/docs/*
!/docs/decisions/
.claude/
+
+# 测量台的运行产物
+_bench_*.json
+_bench.err
diff --git a/crates/utopia-server/src/ontology_index.rs b/crates/utopia-server/src/ontology_index.rs
index a830a822..eb94c5ba 100644
--- a/crates/utopia-server/src/ontology_index.rs
+++ b/crates/utopia-server/src/ontology_index.rs
@@ -28,7 +28,11 @@ pub async fn refresh_scoped(
kb_id: Uuid,
only: Option<utopia_store::ontology::TypeKind>,
) -> anyhow::Result<usize> {
- let kb = utopia_store::kbs::get(&state.pool, kb_id).await?;
+ // 库可能已经被删了——任务是导入时排的,中间隔着几分钟。
+ // 这不是失败,是没事可做;当成错误的话它还要重试三次才放弃
+ let Ok(kb) = utopia_store::kbs::get(&state.pool, kb_id).await else {
+ return Ok(0);
+ };
let Some(settings) = utopia_store::settings::get(&state.pool, kb.workspace_id).await? else {
return Ok(0);
};
diff --git a/scripts/bench/README.md b/scripts/bench/README.md
new file mode 100644
index 00000000..fec70095
--- /dev/null
+++ b/scripts/bench/README.md
@@ -0,0 +1,46 @@
+# 类型消解的测量台
+
+**每一组一个新库。** 这条规则是整个目录存在的理由。
+
+此前连着三轮在同一个库上调检索,而那个库带着前几轮的改类结果——容易的实体早已
+精化,拒绝理由里直接写着 `already correctly typed as pharmacy`。后两轮的数字跟
+第一轮根本不可比,却被当成依据改了两次代码。复用一个库省下的几分钟,换来的是
+一整段无效的结论。
+
+## 跑一组
+
+```
+node scripts/bench/run.mjs --corpus pharma --label seeds-only
+node scripts/bench/run.mjs --corpus pharma --ontology /tmp/schemaorg.ttl --label schemaorg
+```
+
+前置:`utopia-server` 已启动、连着一个能写的库、工作区配好了对话与嵌入模型。
+环境变量见 `run.mjs` 头部。
+
+## 目录
+
+- `corpora/*.json` —— 固定语料。**实体跨文档反复出现**是刻意的:单篇语料里每个
+ 实体只有一两条事实,画像基本只剩名字,量不出消解的真实水平。
+- `truth/*.json` —— 每个实体期望落到哪个类。key 是名字的一个足以认出它的片段
+ (抽取给的名字每次略有出入,全等匹配会把这种变化算成失败);值是可接受的类,
+ 任一命中算对。**空数组 = 本体里没有对得上的类,此时正确行为是不动它**。
+- `run.mjs` —— 一组:新建库 → 灌语料 → 可选导入本体 → 跑消解 → 打分。
+
+## 读数怎么算
+
+- `prompt_tokens_est` 是**本体段**的估算,不是整个提示词。实测 4.0 字符 ≈ 1 token
+ (377,735↔81,855、396,716↔99,041)。真实 token 数在 LLM 客户端里,穿出来要改
+ 一路签名;这里要量的是"本体规模",比例稳定就够用。
+- `for_review` 按**没改**算进 miss。它确实还没改——算成命中就是把人的活记在机器账上。
+- `absent` = 标准答案里有、但抽取压根没抽出这个实体。它不是消解的错,单独一栏。
+
+## 标准答案会写错
+
+第一次跑就写窄了一个:`心血管健康论坛` 只写了 `business_event|event_series`,
+而系统给的 `conference_event` 是对的。**答案错了要改答案**——但要在结果出来之后
+才改、且写清楚为什么,否则这份答案就变成了"系统这次答了什么"的记录,量不出任何东西。
+
+## 加一个语料
+
+两个文件:`corpora/x.json` 与 `truth/x.json`。语料换行业是有意的——同一套判断在
+两个领域上都成立,才谈得上不是过拟合到某一批词上。
diff --git a/scripts/bench/corpora/pharma.json b/scripts/bench/corpora/pharma.json
new file mode 100644
index 00000000..89ecd959
--- /dev/null
+++ b/scripts/bench/corpora/pharma.json
@@ -0,0 +1,11 @@
+{
+ "name": "pharma",
+ "note": "医药领域,5 篇。实体反复出现,跨文档积累事实——单篇语料里每个实体只有一两条事实,画像基本只剩名字,量不出消解的真实水平。",
+ "docs": [
+ ["m1.txt", "仁和堂连锁药房在杭州拱墅区开设了第 40 家门店,由仁和医药集团控股。\n集团旗下的仁和制药厂位于绍兴,主要生产降压药苯磺酸氨氯地平片。\n集团董事长陈国华曾任浙江大学药学院教授。"],
+ ["m2.txt", "苯磺酸氨氯地平片是一种钙通道阻滞剂,用于治疗原发性高血压。\n该药由仁和制药厂生产,2022 年通过国家药监局的仿制药一致性评价。\n常见不良反应包括踝部水肿和头痛。"],
+ ["m3.txt", "浙江大学医学院附属第一医院是仁和医药集团的长期临床合作方。\n该院心内科主任周敏参与了苯磺酸氨氯地平片的三期临床试验。\n试验于 2021 年在杭州启动,入组 800 例高血压患者。"],
+ ["m4.txt", "《中华心血管病杂志》刊登了这项三期临床试验的结果。\n周敏是该论文的通讯作者,陈国华为共同作者。\n杂志由中华医学会主办,为月刊。"],
+ ["m5.txt", "仁和医药集团 2023 年在乌镇举办了首届心血管健康论坛。\n论坛由浙江大学医学院附属第一医院协办,周敏做了主题报告。\n仁和堂连锁药房在论坛上发布了慢病管理小程序。"]
+ ]
+}
diff --git a/scripts/bench/corpora/tech.json b/scripts/bench/corpora/tech.json
new file mode 100644
index 00000000..25ff088e
--- /dev/null
+++ b/scripts/bench/corpora/tech.json
@@ -0,0 +1,11 @@
+{
+ "name": "tech",
+ "note": "科技公司领域,5 篇。与 pharma 换了一个完全不同的行业——同一套判断在两个领域上都成立,才谈得上不是过拟合。",
+ "docs": [
+ ["d1.txt", "星云科技(上海)有限公司注册于上海浦东新区,是一家股份有限公司。\n创始人兼首席技术官张伟毕业于清华大学计算机系。\n公司的核心产品是深蓝向量数据库,一款部署在服务器上的检索软件。\n2023 年公司营收 4.2 亿元。"],
+ ["d2.txt", "深蓝向量数据库支持十亿级向量的近似检索,运行在 Linux 服务器上。\n该软件由星云科技研发并对外发布,当前版本为 3.0。\n深蓝的主要竞品是开源的 Milvus。\n张伟在多个场合介绍过深蓝的架构设计。"],
+ ["d3.txt", "2024 年 7 月,世界人工智能大会在上海举办,为期三天。\n星云科技在大会上发布了深蓝 3.0。\n张伟在大会主论坛发表了主题演讲。\n大会由上海市政府与多家企业联合主办。"],
+ ["d4.txt", "《中国数据智能》是一本季度出版的学术期刊,创刊于 2018 年。\n张伟在该期刊上发表过关于向量检索的论文。\n该期刊由清华大学计算机系的多位教授担任编委。"],
+ ["d5.txt", "星云科技在杭州西湖区设有上海研究院。\n上海研究院的负责人是李娜博士,她此前在清华大学计算机系任教。\n研究院隶属于星云科技。"]
+ ]
+}
diff --git a/scripts/bench/run.mjs b/scripts/bench/run.mjs
new file mode 100644
index 00000000..f7dacab7
--- /dev/null
+++ b/scripts/bench/run.mjs
@@ -0,0 +1,290 @@
+#!/usr/bin/env node
+// 类型消解的测量台:**每一组一个新库**。
+//
+// 存在的理由是一次踩过的坑:连着三轮在同一个库上调检索,而那个库带着前几轮的
+// 改类结果——容易的实体早已精化,拒绝理由里写着 "already correctly typed as
+// pharmacy"。后两轮的数字跟第一轮根本不可比,我却拿它们当依据改了两次代码。
+//
+// 一组 = 新建知识库 → 灌固定语料 → 可选导入本体 → 跑类型消解 → 对标准答案打分。
+// 语料与标准答案都在库里(scripts/bench/),所以任何人重跑得到同一批数字。
+//
+// 用法:
+// node scripts/bench/run.mjs --corpus pharma --label seeds-only
+// node scripts/bench/run.mjs --corpus pharma --ontology /tmp/schemaorg.ttl --label schemaorg
+//
+// 环境变量:BENCH_BASE(默认 http://localhost:18080)、BENCH_EMAIL / BENCH_PASSWORD、
+// BENCH_PSQL(默认 docker exec … psql;本体段字符数要直接查库)。
+
+import fs from "node:fs";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const BASE = process.env.BENCH_BASE || "http://localhost:18080";
+const EMAIL = process.env.BENCH_EMAIL || "[email protected]";
+const PASSWORD = process.env.BENCH_PASSWORD || "benchbench123";
+
+// 实测 4.0 字符 ≈ 1 token(377,735↔81,855、396,716↔99,041,两次都是 4.0)。
+//
+// 抽取提示词的真实 token 数拿不到——它在 LLM 客户端里,穿出来要改一路签名。
+// 本体段字符数与它稳定成比例,而这里要量的正是"本体规模",够用且不动客户端。
+const CHARS_PER_TOKEN = 4.0;
+
+// 种子本体的类。判断"本不该改的有没有被改"要用它:留在种子类上就是没动。
+const SEED = [
+ "concept",
+ "dimension",
+ "event",
+ "location",
+ "metric",
+ "organization",
+ "person",
+ "product",
+ "project",
+];
+
+const args = Object.fromEntries(
+ process.argv.slice(2).reduce((acc, cur, i, arr) => {
+ if (cur.startsWith("--")) acc.push([cur.slice(2), arr[i + 1]]);
+ return acc;
+ }, []),
+);
+const corpusName = args.corpus || "pharma";
+const label = args.label || corpusName;
+
+let cookie = "";
+async function api(method, url, body, isForm) {
+ const init = { method, headers: {} };
+ if (cookie) init.headers.cookie = cookie;
+ if (isForm) init.body = body;
+ else if (body !== undefined) {
+ init.headers["content-type"] = "application/json";
+ init.body = JSON.stringify(body);
+ }
+ const r = await fetch(BASE + url, init);
+ for (const c of r.headers.getSetCookie?.() ?? []) cookie = c.split(";")[0];
+ const text = await r.text();
+ if (!r.ok) throw new Error(method + " " + url + " -> " + r.status + " " + text.slice(0, 200));
+ return text ? JSON.parse(text) : null;
+}
+
+function psql(sql) {
+ const cmd =
+ process.env.BENCH_PSQL ||
+ "docker exec -e PGPASSWORD=utopia landscapebi-db-1 psql -U utopia -d utopia -tAc";
+ const parts = cmd.split(" ");
+ return execFileSync(parts[0], [...parts.slice(1), sql], { encoding: "utf8" }).trim();
+}
+const num = (sql) => Number(psql(sql) || 0);
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+async function until(fn, everyMs, capMs) {
+ const deadline = Date.now() + (capMs || 900000);
+ for (;;) {
+ if (await fn()) return;
+ if (Date.now() > deadline) throw new Error("等超时");
+ await sleep(everyMs || 5000);
+ }
+}
+
+async function main() {
+ const corpus = JSON.parse(
+ fs.readFileSync(path.join(HERE, "corpora", corpusName + ".json"), "utf8"),
+ );
+ const truth = JSON.parse(
+ fs.readFileSync(path.join(HERE, "truth", corpusName + ".json"), "utf8"),
+ ).expect;
+
+ try {
+ await api("POST", "/api/v1/auth/register", {
+ email: EMAIL,
+ display_name: "bench",
+ password: PASSWORD,
+ });
+ } catch {
+ // 已经注册过,走登录
+ }
+ await api("POST", "/api/v1/auth/login", { email: EMAIL, password: PASSWORD });
+ psql("UPDATE users SET is_admin=TRUE WHERE email='" + EMAIL + "'");
+ await api("POST", "/api/v1/auth/login", { email: EMAIL, password: PASSWORD });
+
+ const ws = (await api("GET", "/api/v1/workspaces"))[0].id;
+ // **每组一个新库**:这一条是整个脚本存在的理由,别为了省几分钟去复用
+ const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+ const kb = (
+ await api("POST", "/api/v1/workspaces/" + ws + "/kbs", {
+ name: "bench " + label + " " + stamp,
+ })
+ ).id;
+ // 自动扩本体会在测量中途改本体,那样两组比的就不是同一件事了
+ psql("UPDATE knowledge_bases SET auto_extend_ontology=FALSE WHERE id='" + kb + "'");
+ await sleep(6000);
+
+ const t0 = Date.now();
+ for (const [filename, content] of corpus.docs) {
+ await api("POST", "/api/v1/kbs/" + kb + "/ingest", { filename, content });
+ }
+ await until(
+ async () =>
+ num(
+ "SELECT count(*) FROM documents WHERE kb_id='" +
+ kb +
+ "' AND graph_status='done'",
+ ) >= corpus.docs.length,
+ );
+ const extractMs = Date.now() - t0;
+
+ let importMs = 0;
+ if (args.ontology) {
+ const t1 = Date.now();
+ const form = new FormData();
+ form.append(
+ "file",
+ new Blob([fs.readFileSync(args.ontology)]),
+ path.basename(args.ontology),
+ );
+ await api("POST", "/api/v1/kbs/" + kb + "/ontology/imports", form, true);
+ // 类向量建完才谈得上检索。关系那一半有后台任务补,类型消解用不到。
+ //
+ // 一千个类的冷启动要几分钟到几十分钟——跟别的库的补齐任务抢同一个嵌入
+ // 并发信号量。所以放宽到 40 分钟,并把剩余数打到 stderr:静默地等二十
+ // 分钟,分不清是在跑还是卡死了。
+ await until(
+ async () => {
+ const left = num(
+ "SELECT count(*) FILTER (WHERE embedding IS NULL) FROM entity_types WHERE kb_id='" +
+ kb +
+ "'",
+ );
+ if (left) process.stderr.write(" 类向量还差 " + left + "\n");
+ return left === 0;
+ },
+ 10000,
+ 2400000,
+ );
+ importMs = Date.now() - t1;
+ }
+
+ const classChars = num(
+ "SELECT coalesce(sum(length('- '||key||coalesce(': '||nullif(description,''),''))+1),0)" +
+ " FROM entity_types WHERE kb_id='" +
+ kb +
+ "'",
+ );
+ const relChars = num(
+ "SELECT coalesce(sum(length('- '||key||' (x)'||coalesce(': '||nullif(description,''),''))+1),0)" +
+ " FROM relation_types WHERE kb_id='" +
+ kb +
+ "' AND kind<>'attribute'",
+ );
+ const attrChars = num(
+ "SELECT coalesce(sum((length('- '||r.key||' (text)'||coalesce(': '||nullif(r.description,''),''))+1)" +
+ " * greatest(1,(SELECT count(*) FROM relation_type_domains d WHERE d.relation_type_id=r.id))),0)" +
+ " FROM relation_types r WHERE r.kb_id='" +
+ kb +
+ "' AND r.kind='attribute'",
+ );
+
+ const t2 = Date.now();
+ const outcome = await api("POST", "/api/v1/kbs/" + kb + "/ontology/type-resolution");
+ const resolveMs = Date.now() - t2;
+
+ // 打分。**待人工的按"没改"算**——它确实还没改,算成命中就是把人的活记在机器账上。
+ const rows = psql(
+ "SELECT e.canonical_name || '|' || t.key FROM entities e" +
+ " JOIN entity_types t ON t.id=e.type_id" +
+ " WHERE e.kb_id='" +
+ kb +
+ "' AND e.merged_into IS NULL",
+ )
+ .split("\n")
+ .filter(Boolean)
+ .map((l) => {
+ const i = l.lastIndexOf("|");
+ return [l.slice(0, i), l.slice(i + 1)];
+ });
+
+ let hit = 0;
+ let miss = 0;
+ let correctlyLeft = 0;
+ let wronglyChanged = 0;
+ let absent = 0;
+ const notes = [];
+ for (const [frag, accept] of Object.entries(truth)) {
+ // 按片段匹配而不是全等:抽取给的名字每次略有出入
+ //("星云科技" / "星云科技(上海)有限公司"),全等会把这种变化算成失败
+ const found = rows.filter(([name]) => name.includes(frag));
+ if (found.length === 0) {
+ absent += 1;
+ continue;
+ }
+ const keys = found.map((r) => r[1]);
+ if (accept.length === 0) {
+ // 本体里没有对得上的类:正确行为是**不动**,动了才算错
+ if (keys.some((k) => !SEED.includes(k))) {
+ wronglyChanged += 1;
+ notes.push(frag + ":本不该改,却成了 " + keys.join("/"));
+ } else correctlyLeft += 1;
+ } else if (keys.some((k) => accept.includes(k))) {
+ hit += 1;
+ } else {
+ miss += 1;
+ notes.push(frag + ":期望 " + accept.join("|") + ",实得 " + keys.join("/"));
+ }
+ }
+
+ console.log(
+ JSON.stringify(
+ {
+ label,
+ corpus: corpusName,
+ ontology: args.ontology ? path.basename(args.ontology) : null,
+ kb_id: kb,
+ ontology_size: {
+ classes: num("SELECT count(*) FROM entity_types WHERE kb_id='" + kb + "'"),
+ relations: num(
+ "SELECT count(*) FROM relation_types WHERE kb_id='" +
+ kb +
+ "' AND kind<>'attribute'",
+ ),
+ attributes: num(
+ "SELECT count(*) FROM relation_types WHERE kb_id='" +
+ kb +
+ "' AND kind='attribute'",
+ ),
+ prompt_chars: classChars + relChars + attrChars,
+ prompt_tokens_est: Math.round(
+ (classChars + relChars + attrChars) / CHARS_PER_TOKEN,
+ ),
+ },
+ graph: {
+ entities: num(
+ "SELECT count(*) FROM entities WHERE kb_id='" +
+ kb +
+ "' AND merged_into IS NULL",
+ ),
+ facts: num(
+ "SELECT count(*) FROM facts WHERE kb_id='" +
+ kb +
+ "' AND invalidated_at IS NULL",
+ ),
+ },
+ resolution: {
+ retyped: outcome.retyped,
+ for_review: outcome.for_review.length,
+ left_alone: outcome.left_alone.length,
+ },
+ score: { hit, miss, correctlyLeft, wronglyChanged, absent, notes },
+ ms: { extract: extractMs, import: importMs, resolve: resolveMs },
+ },
+ null,
+ 2,
+ ),
+ );
+}
+
+main().catch((e) => {
+ console.error(e);
+ process.exit(1);
+});
diff --git a/scripts/bench/truth/pharma.json b/scripts/bench/truth/pharma.json
new file mode 100644
index 00000000..9b6389c6
--- /dev/null
+++ b/scripts/bench/truth/pharma.json
@@ -0,0 +1,28 @@
+{
+ "note": "期望的 schema.org 类。key 是实体名的一个足以认出它的片段(抽取给的名字每次略有出入,全等匹配会把变化算成失败);值是可接受的类 key,任一命中即算对。",
+ "note2": "空数组 = 本体里没有对得上的类,此时**正确行为是不动它**——改了反而算错。",
+ "expect": {
+ "仁和医药集团": ["corporation", "organization"],
+ "仁和制药厂": ["corporation", "organization"],
+ "仁和堂连锁药房": ["pharmacy"],
+ "浙江大学医学院附属第一医院": ["hospital"],
+ "国家药监局": ["government_organization"],
+ "中华医学会": ["medical_organization"],
+ "苯磺酸氨氯地平片": ["drug"],
+ "浙江大学药学院": ["college_or_university", "educational_organization"],
+ "中华心血管病杂志": ["periodical"],
+ "三期临床试验": ["medical_trial"],
+ "原发性高血压": ["medical_condition"],
+ "头痛": ["medical_symptom", "medical_sign_or_symptom"],
+ "踝部水肿": ["medical_symptom", "medical_sign_or_symptom"],
+ "杭州": ["city"],
+ "绍兴": ["city"],
+ "乌镇": ["city"],
+ "杭州拱墅区": ["administrative_area"],
+ "心血管健康论坛": ["business_event", "event_series", "conference_event"],
+ "慢病管理小程序": ["software_application", "mobile_application"],
+ "周敏": [],
+ "陈国华": [],
+ "仿制药一致性评价": []
+ }
+}
diff --git a/scripts/bench/truth/tech.json b/scripts/bench/truth/tech.json
new file mode 100644
index 00000000..937e8208
--- /dev/null
+++ b/scripts/bench/truth/tech.json
@@ -0,0 +1,21 @@
+{
+ "note": "见 pharma.json 的说明。",
+ "expect": {
+ "星云科技": ["corporation"],
+ "上海研究院": ["research_organization"],
+ "清华大学计算机系": ["college_or_university", "educational_organization"],
+ "上海市政府": ["government_organization"],
+ "深蓝向量数据库": ["software_application"],
+ "Milvus": ["software_application"],
+ "深蓝 3.0": ["software_application"],
+ "Linux": ["operating_system", "software_application"],
+ "中国数据智能": ["periodical"],
+ "世界人工智能大会": ["business_event", "event_series", "conference_event"],
+ "大会主论坛": ["conference_event", "event"],
+ "上海": ["city"],
+ "上海浦东新区": ["administrative_area"],
+ "杭州西湖区": ["administrative_area"],
+ "张伟": [],
+ "李娜": []
+ }
+}