World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Every arm gets its own knowledge base (#83)
WaylandYang116c56805c13
diff --git a/.gitignore b/.gitignoreindex e0efb2b7..4e23c096 100644--- a/.gitignore+++ b/.gitignore@@ -11,3 +11,7 @@ web/dist//docs/*!/docs/decisions/.claude/++# 测量台的运行产物+_bench_*.json+_bench.errdiff --git a/crates/utopia-server/src/ontology_index.rs b/crates/utopia-server/src/ontology_index.rsindex a830a822..eb94c5ba 100644--- a/crates/utopia-server/src/ontology_index.rs+++ b/crates/utopia-server/src/ontology_index.rs@@ -28,7 +28,11 @@ pub async fn refresh_scoped(kb_id: Uuid,only: Option<utopia_store::ontology::TypeKind>,) -> anyhow::Result<usize> {- let kb = utopia_store::kbs::get(&state.pool, kb_id).await?;+ // 库可能已经被删了——任务是导入时排的,中间隔着几分钟。+ // 这不是失败,是没事可做;当成错误的话它还要重试三次才放弃+ let Ok(kb) = utopia_store::kbs::get(&state.pool, kb_id).await else {+ return Ok(0);+ };let Some(settings) = utopia_store::settings::get(&state.pool, kb.workspace_id).await? else {return Ok(0);};diff --git a/scripts/bench/README.md b/scripts/bench/README.mdnew file mode 100644index 00000000..fec70095--- /dev/null+++ b/scripts/bench/README.md@@ -0,0 +1,46 @@+# 类型消解的测量台++**每一组一个新库。** 这条规则是整个目录存在的理由。++此前连着三轮在同一个库上调检索,而那个库带着前几轮的改类结果——容易的实体早已+精化,拒绝理由里直接写着 `already correctly typed as pharmacy`。后两轮的数字跟+第一轮根本不可比,却被当成依据改了两次代码。复用一个库省下的几分钟,换来的是+一整段无效的结论。++## 跑一组++```+node scripts/bench/run.mjs --corpus pharma --label seeds-only+node scripts/bench/run.mjs --corpus pharma --ontology /tmp/schemaorg.ttl --label schemaorg+```++前置:`utopia-server` 已启动、连着一个能写的库、工作区配好了对话与嵌入模型。+环境变量见 `run.mjs` 头部。++## 目录++- `corpora/*.json` —— 固定语料。**实体跨文档反复出现**是刻意的:单篇语料里每个+ 实体只有一两条事实,画像基本只剩名字,量不出消解的真实水平。+- `truth/*.json` —— 每个实体期望落到哪个类。key 是名字的一个足以认出它的片段+ (抽取给的名字每次略有出入,全等匹配会把这种变化算成失败);值是可接受的类,+ 任一命中算对。**空数组 = 本体里没有对得上的类,此时正确行为是不动它**。+- `run.mjs` —— 一组:新建库 → 灌语料 → 可选导入本体 → 跑消解 → 打分。++## 读数怎么算++- `prompt_tokens_est` 是**本体段**的估算,不是整个提示词。实测 4.0 字符 ≈ 1 token+ (377,735↔81,855、396,716↔99,041)。真实 token 数在 LLM 客户端里,穿出来要改+ 一路签名;这里要量的是"本体规模",比例稳定就够用。+- `for_review` 按**没改**算进 miss。它确实还没改——算成命中就是把人的活记在机器账上。+- `absent` = 标准答案里有、但抽取压根没抽出这个实体。它不是消解的错,单独一栏。++## 标准答案会写错++第一次跑就写窄了一个:`心血管健康论坛` 只写了 `business_event|event_series`,+而系统给的 `conference_event` 是对的。**答案错了要改答案**——但要在结果出来之后+才改、且写清楚为什么,否则这份答案就变成了"系统这次答了什么"的记录,量不出任何东西。++## 加一个语料++两个文件:`corpora/x.json` 与 `truth/x.json`。语料换行业是有意的——同一套判断在+两个领域上都成立,才谈得上不是过拟合到某一批词上。diff --git a/scripts/bench/corpora/pharma.json b/scripts/bench/corpora/pharma.jsonnew file mode 100644index 00000000..89ecd959--- /dev/null+++ b/scripts/bench/corpora/pharma.json@@ -0,0 +1,11 @@+{+ "name": "pharma",+ "note": "医药领域,5 篇。实体反复出现,跨文档积累事实——单篇语料里每个实体只有一两条事实,画像基本只剩名字,量不出消解的真实水平。",+ "docs": [+ ["m1.txt", "仁和堂连锁药房在杭州拱墅区开设了第 40 家门店,由仁和医药集团控股。\n集团旗下的仁和制药厂位于绍兴,主要生产降压药苯磺酸氨氯地平片。\n集团董事长陈国华曾任浙江大学药学院教授。"],+ ["m2.txt", "苯磺酸氨氯地平片是一种钙通道阻滞剂,用于治疗原发性高血压。\n该药由仁和制药厂生产,2022 年通过国家药监局的仿制药一致性评价。\n常见不良反应包括踝部水肿和头痛。"],+ ["m3.txt", "浙江大学医学院附属第一医院是仁和医药集团的长期临床合作方。\n该院心内科主任周敏参与了苯磺酸氨氯地平片的三期临床试验。\n试验于 2021 年在杭州启动,入组 800 例高血压患者。"],+ ["m4.txt", "《中华心血管病杂志》刊登了这项三期临床试验的结果。\n周敏是该论文的通讯作者,陈国华为共同作者。\n杂志由中华医学会主办,为月刊。"],+ ["m5.txt", "仁和医药集团 2023 年在乌镇举办了首届心血管健康论坛。\n论坛由浙江大学医学院附属第一医院协办,周敏做了主题报告。\n仁和堂连锁药房在论坛上发布了慢病管理小程序。"]+ ]+}diff --git a/scripts/bench/corpora/tech.json b/scripts/bench/corpora/tech.jsonnew file mode 100644index 00000000..25ff088e--- /dev/null+++ b/scripts/bench/corpora/tech.json@@ -0,0 +1,11 @@+{+ "name": "tech",+ "note": "科技公司领域,5 篇。与 pharma 换了一个完全不同的行业——同一套判断在两个领域上都成立,才谈得上不是过拟合。",+ "docs": [+ ["d1.txt", "星云科技(上海)有限公司注册于上海浦东新区,是一家股份有限公司。\n创始人兼首席技术官张伟毕业于清华大学计算机系。\n公司的核心产品是深蓝向量数据库,一款部署在服务器上的检索软件。\n2023 年公司营收 4.2 亿元。"],+ ["d2.txt", "深蓝向量数据库支持十亿级向量的近似检索,运行在 Linux 服务器上。\n该软件由星云科技研发并对外发布,当前版本为 3.0。\n深蓝的主要竞品是开源的 Milvus。\n张伟在多个场合介绍过深蓝的架构设计。"],+ ["d3.txt", "2024 年 7 月,世界人工智能大会在上海举办,为期三天。\n星云科技在大会上发布了深蓝 3.0。\n张伟在大会主论坛发表了主题演讲。\n大会由上海市政府与多家企业联合主办。"],+ ["d4.txt", "《中国数据智能》是一本季度出版的学术期刊,创刊于 2018 年。\n张伟在该期刊上发表过关于向量检索的论文。\n该期刊由清华大学计算机系的多位教授担任编委。"],+ ["d5.txt", "星云科技在杭州西湖区设有上海研究院。\n上海研究院的负责人是李娜博士,她此前在清华大学计算机系任教。\n研究院隶属于星云科技。"]+ ]+}diff --git a/scripts/bench/run.mjs b/scripts/bench/run.mjsnew file mode 100644index 00000000..f7dacab7--- /dev/null+++ b/scripts/bench/run.mjs@@ -0,0 +1,290 @@+#!/usr/bin/env node+// 类型消解的测量台:**每一组一个新库**。+//+// 存在的理由是一次踩过的坑:连着三轮在同一个库上调检索,而那个库带着前几轮的+// 改类结果——容易的实体早已精化,拒绝理由里写着 "already correctly typed as+// pharmacy"。后两轮的数字跟第一轮根本不可比,我却拿它们当依据改了两次代码。+//+// 一组 = 新建知识库 → 灌固定语料 → 可选导入本体 → 跑类型消解 → 对标准答案打分。+// 语料与标准答案都在库里(scripts/bench/),所以任何人重跑得到同一批数字。+//+// 用法:+// node scripts/bench/run.mjs --corpus pharma --label seeds-only+// node scripts/bench/run.mjs --corpus pharma --ontology /tmp/schemaorg.ttl --label schemaorg+//+// 环境变量:BENCH_BASE(默认 http://localhost:18080)、BENCH_EMAIL / BENCH_PASSWORD、+// BENCH_PSQL(默认 docker exec … psql;本体段字符数要直接查库)。++import fs from "node:fs";+import path from "node:path";+import { execFileSync } from "node:child_process";+import { fileURLToPath } from "node:url";++const HERE = path.dirname(fileURLToPath(import.meta.url));+const BASE = process.env.BENCH_BASE || "http://localhost:18080";+const EMAIL = process.env.BENCH_EMAIL || "[email protected]";+const PASSWORD = process.env.BENCH_PASSWORD || "benchbench123";++// 实测 4.0 字符 ≈ 1 token(377,735↔81,855、396,716↔99,041,两次都是 4.0)。+//+// 抽取提示词的真实 token 数拿不到——它在 LLM 客户端里,穿出来要改一路签名。+// 本体段字符数与它稳定成比例,而这里要量的正是"本体规模",够用且不动客户端。+const CHARS_PER_TOKEN = 4.0;++// 种子本体的类。判断"本不该改的有没有被改"要用它:留在种子类上就是没动。+const SEED = [+ "concept",+ "dimension",+ "event",+ "location",+ "metric",+ "organization",+ "person",+ "product",+ "project",+];++const args = Object.fromEntries(+ process.argv.slice(2).reduce((acc, cur, i, arr) => {+ if (cur.startsWith("--")) acc.push([cur.slice(2), arr[i + 1]]);+ return acc;+ }, []),+);+const corpusName = args.corpus || "pharma";+const label = args.label || corpusName;++let cookie = "";+async function api(method, url, body, isForm) {+ const init = { method, headers: {} };+ if (cookie) init.headers.cookie = cookie;+ if (isForm) init.body = body;+ else if (body !== undefined) {+ init.headers["content-type"] = "application/json";+ init.body = JSON.stringify(body);+ }+ const r = await fetch(BASE + url, init);+ for (const c of r.headers.getSetCookie?.() ?? []) cookie = c.split(";")[0];+ const text = await r.text();+ if (!r.ok) throw new Error(method + " " + url + " -> " + r.status + " " + text.slice(0, 200));+ return text ? JSON.parse(text) : null;+}++function psql(sql) {+ const cmd =+ process.env.BENCH_PSQL ||+ "docker exec -e PGPASSWORD=utopia landscapebi-db-1 psql -U utopia -d utopia -tAc";+ const parts = cmd.split(" ");+ return execFileSync(parts[0], [...parts.slice(1), sql], { encoding: "utf8" }).trim();+}+const num = (sql) => Number(psql(sql) || 0);++const sleep = (ms) => new Promise((r) => setTimeout(r, ms));+async function until(fn, everyMs, capMs) {+ const deadline = Date.now() + (capMs || 900000);+ for (;;) {+ if (await fn()) return;+ if (Date.now() > deadline) throw new Error("等超时");+ await sleep(everyMs || 5000);+ }+}++async function main() {+ const corpus = JSON.parse(+ fs.readFileSync(path.join(HERE, "corpora", corpusName + ".json"), "utf8"),+ );+ const truth = JSON.parse(+ fs.readFileSync(path.join(HERE, "truth", corpusName + ".json"), "utf8"),+ ).expect;++ try {+ await api("POST", "/api/v1/auth/register", {+ email: EMAIL,+ display_name: "bench",+ password: PASSWORD,+ });+ } catch {+ // 已经注册过,走登录+ }+ await api("POST", "/api/v1/auth/login", { email: EMAIL, password: PASSWORD });+ psql("UPDATE users SET is_admin=TRUE WHERE email='" + EMAIL + "'");+ await api("POST", "/api/v1/auth/login", { email: EMAIL, password: PASSWORD });++ const ws = (await api("GET", "/api/v1/workspaces"))[0].id;+ // **每组一个新库**:这一条是整个脚本存在的理由,别为了省几分钟去复用+ const stamp = new Date().toISOString().replace(/[:.]/g, "-");+ const kb = (+ await api("POST", "/api/v1/workspaces/" + ws + "/kbs", {+ name: "bench " + label + " " + stamp,+ })+ ).id;+ // 自动扩本体会在测量中途改本体,那样两组比的就不是同一件事了+ psql("UPDATE knowledge_bases SET auto_extend_ontology=FALSE WHERE id='" + kb + "'");+ await sleep(6000);++ const t0 = Date.now();+ for (const [filename, content] of corpus.docs) {+ await api("POST", "/api/v1/kbs/" + kb + "/ingest", { filename, content });+ }+ await until(+ async () =>+ num(+ "SELECT count(*) FROM documents WHERE kb_id='" ++ kb ++ "' AND graph_status='done'",+ ) >= corpus.docs.length,+ );+ const extractMs = Date.now() - t0;++ let importMs = 0;+ if (args.ontology) {+ const t1 = Date.now();+ const form = new FormData();+ form.append(+ "file",+ new Blob([fs.readFileSync(args.ontology)]),+ path.basename(args.ontology),+ );+ await api("POST", "/api/v1/kbs/" + kb + "/ontology/imports", form, true);+ // 类向量建完才谈得上检索。关系那一半有后台任务补,类型消解用不到。+ //+ // 一千个类的冷启动要几分钟到几十分钟——跟别的库的补齐任务抢同一个嵌入+ // 并发信号量。所以放宽到 40 分钟,并把剩余数打到 stderr:静默地等二十+ // 分钟,分不清是在跑还是卡死了。+ await until(+ async () => {+ const left = num(+ "SELECT count(*) FILTER (WHERE embedding IS NULL) FROM entity_types WHERE kb_id='" ++ kb ++ "'",+ );+ if (left) process.stderr.write(" 类向量还差 " + left + "\n");+ return left === 0;+ },+ 10000,+ 2400000,+ );+ importMs = Date.now() - t1;+ }++ const classChars = num(+ "SELECT coalesce(sum(length('- '||key||coalesce(': '||nullif(description,''),''))+1),0)" ++ " FROM entity_types WHERE kb_id='" ++ kb ++ "'",+ );+ const relChars = num(+ "SELECT coalesce(sum(length('- '||key||' (x)'||coalesce(': '||nullif(description,''),''))+1),0)" ++ " FROM relation_types WHERE kb_id='" ++ kb ++ "' AND kind<>'attribute'",+ );+ const attrChars = num(+ "SELECT coalesce(sum((length('- '||r.key||' (text)'||coalesce(': '||nullif(r.description,''),''))+1)" ++ " * greatest(1,(SELECT count(*) FROM relation_type_domains d WHERE d.relation_type_id=r.id))),0)" ++ " FROM relation_types r WHERE r.kb_id='" ++ kb ++ "' AND r.kind='attribute'",+ );++ const t2 = Date.now();+ const outcome = await api("POST", "/api/v1/kbs/" + kb + "/ontology/type-resolution");+ const resolveMs = Date.now() - t2;++ // 打分。**待人工的按"没改"算**——它确实还没改,算成命中就是把人的活记在机器账上。+ const rows = psql(+ "SELECT e.canonical_name || '|' || t.key FROM entities e" ++ " JOIN entity_types t ON t.id=e.type_id" ++ " WHERE e.kb_id='" ++ kb ++ "' AND e.merged_into IS NULL",+ )+ .split("\n")+ .filter(Boolean)+ .map((l) => {+ const i = l.lastIndexOf("|");+ return [l.slice(0, i), l.slice(i + 1)];+ });++ let hit = 0;+ let miss = 0;+ let correctlyLeft = 0;+ let wronglyChanged = 0;+ let absent = 0;+ const notes = [];+ for (const [frag, accept] of Object.entries(truth)) {+ // 按片段匹配而不是全等:抽取给的名字每次略有出入+ //("星云科技" / "星云科技(上海)有限公司"),全等会把这种变化算成失败+ const found = rows.filter(([name]) => name.includes(frag));+ if (found.length === 0) {+ absent += 1;+ continue;+ }+ const keys = found.map((r) => r[1]);+ if (accept.length === 0) {+ // 本体里没有对得上的类:正确行为是**不动**,动了才算错+ if (keys.some((k) => !SEED.includes(k))) {+ wronglyChanged += 1;+ notes.push(frag + ":本不该改,却成了 " + keys.join("/"));+ } else correctlyLeft += 1;+ } else if (keys.some((k) => accept.includes(k))) {+ hit += 1;+ } else {+ miss += 1;+ notes.push(frag + ":期望 " + accept.join("|") + ",实得 " + keys.join("/"));+ }+ }++ console.log(+ JSON.stringify(+ {+ label,+ corpus: corpusName,+ ontology: args.ontology ? path.basename(args.ontology) : null,+ kb_id: kb,+ ontology_size: {+ classes: num("SELECT count(*) FROM entity_types WHERE kb_id='" + kb + "'"),+ relations: num(+ "SELECT count(*) FROM relation_types WHERE kb_id='" ++ kb ++ "' AND kind<>'attribute'",+ ),+ attributes: num(+ "SELECT count(*) FROM relation_types WHERE kb_id='" ++ kb ++ "' AND kind='attribute'",+ ),+ prompt_chars: classChars + relChars + attrChars,+ prompt_tokens_est: Math.round(+ (classChars + relChars + attrChars) / CHARS_PER_TOKEN,+ ),+ },+ graph: {+ entities: num(+ "SELECT count(*) FROM entities WHERE kb_id='" ++ kb ++ "' AND merged_into IS NULL",+ ),+ facts: num(+ "SELECT count(*) FROM facts WHERE kb_id='" ++ kb ++ "' AND invalidated_at IS NULL",+ ),+ },+ resolution: {+ retyped: outcome.retyped,+ for_review: outcome.for_review.length,+ left_alone: outcome.left_alone.length,+ },+ score: { hit, miss, correctlyLeft, wronglyChanged, absent, notes },+ ms: { extract: extractMs, import: importMs, resolve: resolveMs },+ },+ null,+ 2,+ ),+ );+}++main().catch((e) => {+ console.error(e);+ process.exit(1);+});diff --git a/scripts/bench/truth/pharma.json b/scripts/bench/truth/pharma.jsonnew file mode 100644index 00000000..9b6389c6--- /dev/null+++ b/scripts/bench/truth/pharma.json@@ -0,0 +1,28 @@+{+ "note": "期望的 schema.org 类。key 是实体名的一个足以认出它的片段(抽取给的名字每次略有出入,全等匹配会把变化算成失败);值是可接受的类 key,任一命中即算对。",+ "note2": "空数组 = 本体里没有对得上的类,此时**正确行为是不动它**——改了反而算错。",+ "expect": {+ "仁和医药集团": ["corporation", "organization"],+ "仁和制药厂": ["corporation", "organization"],+ "仁和堂连锁药房": ["pharmacy"],+ "浙江大学医学院附属第一医院": ["hospital"],+ "国家药监局": ["government_organization"],+ "中华医学会": ["medical_organization"],+ "苯磺酸氨氯地平片": ["drug"],+ "浙江大学药学院": ["college_or_university", "educational_organization"],+ "中华心血管病杂志": ["periodical"],+ "三期临床试验": ["medical_trial"],+ "原发性高血压": ["medical_condition"],+ "头痛": ["medical_symptom", "medical_sign_or_symptom"],+ "踝部水肿": ["medical_symptom", "medical_sign_or_symptom"],+ "杭州": ["city"],+ "绍兴": ["city"],+ "乌镇": ["city"],+ "杭州拱墅区": ["administrative_area"],+ "心血管健康论坛": ["business_event", "event_series", "conference_event"],+ "慢病管理小程序": ["software_application", "mobile_application"],+ "周敏": [],+ "陈国华": [],+ "仿制药一致性评价": []+ }+}diff --git a/scripts/bench/truth/tech.json b/scripts/bench/truth/tech.jsonnew file mode 100644index 00000000..937e8208--- /dev/null+++ b/scripts/bench/truth/tech.json@@ -0,0 +1,21 @@+{+ "note": "见 pharma.json 的说明。",+ "expect": {+ "星云科技": ["corporation"],+ "上海研究院": ["research_organization"],+ "清华大学计算机系": ["college_or_university", "educational_organization"],+ "上海市政府": ["government_organization"],+ "深蓝向量数据库": ["software_application"],+ "Milvus": ["software_application"],+ "深蓝 3.0": ["software_application"],+ "Linux": ["operating_system", "software_application"],+ "中国数据智能": ["periodical"],+ "世界人工智能大会": ["business_event", "event_series", "conference_event"],+ "大会主论坛": ["conference_event", "event"],+ "上海": ["city"],+ "上海浦东新区": ["administrative_area"],+ "杭州西湖区": ["administrative_area"],+ "张伟": [],+ "李娜": []+ }+}