World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
The competency bench records whether an answer went through the graph: 0061 cut 3 second slice (#962)
WaylandYang6ef840af0be9
diff --git a/crates/utopia-server/src/api/question_routes.rs b/crates/utopia-server/src/api/question_routes.rsindex 4efad3c5..3b20f2c3 100644--- a/crates/utopia-server/src/api/question_routes.rs+++ b/crates/utopia-server/src/api/question_routes.rs@@ -153,6 +153,9 @@ pub struct QuestionResultReq {/// expected | shape:按期望答案判的,还是只查了它需要的形状#[serde(default)]pub judged_by: Option<String>,+ /// graph | text | both | none:回答从哪里来的(图谱工具拿到过事实,还是只读了正文)+ #[serde(default)]+ pub via: Option<String>,#[serde(default)]pub detail: Option<serde_json::Value>,}@@ -167,10 +170,16 @@ pub async fn record_result() -> ApiResult<Json<serde_json::Value>> {utopia_store::access::require_kb(&state.pool, &user, kb_id, Role::Editor).await?;let answer: Option<String> = req.answer.map(|a| a.chars().take(4000).collect());+ let via = req+ .via+ .as_deref()+ .filter(|v| matches!(*v, "graph" | "text" | "both" | "none"))+ .unwrap_or("none");let result = json!({"answered": req.answered,"answer": answer,"judged_by": req.judged_by.unwrap_or_else(|| "expected".into()),+ "via": via,"detail": req.detail,});if !questions::record_result(&state.pool, kb_id, qid, &result).await? {diff --git a/crates/utopia-store/src/competency_questions.rs b/crates/utopia-store/src/competency_questions.rsindex 6876ebd9..b876ff56 100644--- a/crates/utopia-store/src/competency_questions.rs+++ b/crates/utopia-store/src/competency_questions.rs@@ -209,6 +209,10 @@ pub struct QuestionReport {pub proposed: i64,pub checked: i64,pub answered: i64,+ /// 答上了、而且回答途中从图谱工具拿到过事实(via graph 或 both)+ pub answered_with_graph: i64,+ /// 答上了、而且只走了图谱没读正文:本体本身答的+ pub answered_graph_only: i64,}pub async fn report(pool: &PgPool, kb_id: Uuid) -> AppResult<QuestionReport> {@@ -216,7 +220,11 @@ pub async fn report(pool: &PgPool, kb_id: Uuid) -> AppResult<QuestionReport> {"SELECT count(*) FILTER (WHERE status = 'accepted') AS accepted,count(*) FILTER (WHERE status = 'proposed') AS proposed,count(*) FILTER (WHERE status = 'accepted' AND last_checked_at IS NOT NULL) AS checked,- count(*) FILTER (WHERE status = 'accepted' AND (last_result->>'answered')::boolean) AS answered+ count(*) FILTER (WHERE status = 'accepted' AND (last_result->>'answered')::boolean) AS answered,+ count(*) FILTER (WHERE status = 'accepted' AND (last_result->>'answered')::boolean+ AND last_result->>'via' IN ('graph', 'both')) AS answered_with_graph,+ count(*) FILTER (WHERE status = 'accepted' AND (last_result->>'answered')::boolean+ AND last_result->>'via' = 'graph') AS answered_graph_onlyFROM competency_questions WHERE kb_id = $1",).bind(kb_id)diff --git a/docs/decisions/0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md b/docs/decisions/0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.mdindex 2950f9b8..a49ce800 100644--- a/docs/decisions/0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md+++ b/docs/decisions/0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md@@ -1,6 +1,6 @@# 0061 · The ontology is proposed from the open graph and judged by its questions-- **Status**: Accepted 2026-09-26 · 0044 cut 5 · cut 1 built (#947, migration 0077): competency questions are rows, the agent proposes from unbound signatures and kind words into `ontology_proposals`, adoption writes the element and re-decides its shapes through alignment · cut 1.1 (#950, migration 0078): the agent records every shape it saw, "already in the ontology" becomes a map_to proposal adopted as a person's binding per shape, adoption embeds the element before alignment, and the editor's create and update handlers queue the same refresh ahead of alignment, which waits for it · measured on a 100-document base: ~9k tokens a call, 12 calls a round, about half of a round's proposals adoptable as-is · cut 3 first slice (#955, migration 0079): the agent proposes questions for a base with none, `competency.mjs` asks the accepted ones through chat and the two numbers are reported (11 of 11 on the bench base) · cut 2, the rest of cut 3 (FDA and statistics corpora) and cut 4 open · the ontology agent reads the open graph's unbound signatures and kind words against the base's competency questions and proposes types, properties and rules with definitions, examples and the signatures they would bind; approval writes the ontology and re-decides those signatures; every approved element carries regression cases; the ontology is measured by the questions it answers+- **Status**: Accepted 2026-09-26 · 0044 cut 5 · cut 1 built (#947, migration 0077): competency questions are rows, the agent proposes from unbound signatures and kind words into `ontology_proposals`, adoption writes the element and re-decides its shapes through alignment · cut 1.1 (#950, migration 0078): the agent records every shape it saw, "already in the ontology" becomes a map_to proposal adopted as a person's binding per shape, adoption embeds the element before alignment, and the editor's create and update handlers queue the same refresh ahead of alignment, which waits for it · measured on a 100-document base: ~9k tokens a call, 12 calls a round, about half of a round's proposals adoptable as-is · cut 3 first slice (#955, migration 0079): the agent proposes questions for a base with none, `competency.mjs` asks the accepted ones through chat and the two numbers are reported (11 of 11 on the bench base) · second slice (#962): each answer records whether it went through the graph; on the bench base 10 of 11 answered, 8 with graph facts, 6 from the graph alone · cut 2, the rest of cut 3 (FDA and statistics corpora) and cut 4 open · the ontology agent reads the open graph's unbound signatures and kind words against the base's competency questions and proposes types, properties and rules with definitions, examples and the signatures they would bind; approval writes the ontology and re-decides those signatures; every approved element carries regression cases; the ontology is measured by the questions it answers- **Written**: 2026-09-26 (conventions in the [README](README.md))- **Related**: [0044](0044-the-ontology-is-a-view-over-what-documents-say.md) §4 decided the workbench and named this cut; [0003](0003-ontology-growth-loop.md) grew the ontology from `ontology_misses` and surface predicates, which [0044](0044-the-ontology-is-a-view-over-what-documents-say.md) cut 1 left idle and #725 retires; [0012](0012-the-ontology-is-a-contract-not-a-suggestion.md) makes every element a contract a person signed, which this record keeps; [0053](0053-a-phrase-decision-records-the-inputs-it-considered.md) is what makes approval re-decide signatures without a new mechanism; [0008](0008-ontology-packs-as-cold-start.md) is the second of the three sources; [0043](0043-every-review-queue-is-governed.md) and #725 give the queue its shape. From #725's "Ontology proposals" row and 0044's open question about a base with no questions.diff --git a/docs/decisions/README.md b/docs/decisions/README.mdindex 547167e2..f3db49e0 100644--- a/docs/decisions/README.md+++ b/docs/decisions/README.md@@ -81,7 +81,7 @@ The test for writing one: if someone (including us) looks at a piece of code in| 0053 | [A phrase decision records the inputs it considered](0053-a-phrase-decision-records-the-inputs-it-considered.md) | Implemented 2026-09-23 · a decision stores a fingerprint of the ancestor closures and admitted candidates it saw; stale means the fingerprint of the current inputs differs, which is what timestamps could not see (#807, #795): inheritance, parent edges, edits during the request; no-candidate and overflow become recorded outcomes; requeue reads live signatures only, so orphaned rows stop looping · revised 2026-09-26: kind words fingerprinted too, read from one snapshot and compared again when a reply is accepted| 0054 | [A source may push statements in the open contract](0054-a-source-may-push-statements-in-the-open-contract.md) | Proposed 2026-09-23 · cut 1 in its PR · a `statements` source accepts the open extraction contract (`e`/`s`/`n`) verbatim on `POST /sources/{id}/statements` with the `api` push's identity, versions and tombstones; the payload is stored as one chunk and extraction parses it instead of prompting a model, then runs the unchanged path, so a pushed statement is an open statement and reaches the typed graph only through alignment; there is no slot for a property or class; an update marks earlier statements stale, it does not close them; tables stay on the mount (0036)| 0060 | [A rule's definition has a history](0060-a-rule-definition-has-a-history.md) | Implemented 2026-09-25 (#912, migration 0076) · A business rule was edited in place and a derivation pointed at the row, so an invalidated conclusion pointed at a rule that now said something else. Every edit that changes what a rule says opens a **version**, a full snapshot with a record time; a derivation names the version it was drawn under, a kept conclusion moves to the new one, and the proof, the rules panel and a versions endpoint read the history. Name, description and the switch open nothing. Exporting rule bodies per version is now honest and stays #902's second cut |-| 0061 | [The ontology is proposed from the open graph and judged by its questions](0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md) | Accepted · cut 1 built (#947): questions, agent proposals, adoption · cut 1.1 (#950): the agent remembers what it saw, existing answers become map_to · cut 3 first slice (#955): proposed questions, the two numbers, `competency.mjs` · cut 2, rest of 3, 4 open · A base's unbound signatures and classless kind words are what its documents say and its ontology cannot hold, and nothing says what the ontology is *for*. **Competency questions** become rows of the base and the standard an element is judged by; an **ontology agent** reads the unbound signatures and kind words against them and proposes types, properties and rules with a definition, the statements they would bind and the questions they serve; **approval** writes the element with its structure and re-decides those signatures through 0053's basis, and every approved element carries **regression cases** rerun when its definition changes. Measured by questions answered correctly and by the share of proposals people change. No automatic adoption; the 0003 `Suggest` retires once this replaces it |+| 0061 | [The ontology is proposed from the open graph and judged by its questions](0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md) | Accepted · cut 1 built (#947): questions, agent proposals, adoption · cut 1.1 (#950): the agent remembers what it saw, existing answers become map_to · cut 3 first slice (#955): proposed questions, the two numbers, `competency.mjs`; second slice (#962): answers record whether they went through the graph · cut 2, rest of 3, 4 open · A base's unbound signatures and classless kind words are what its documents say and its ontology cannot hold, and nothing says what the ontology is *for*. **Competency questions** become rows of the base and the standard an element is judged by; an **ontology agent** reads the unbound signatures and kind words against them and proposes types, properties and rules with a definition, the statements they would bind and the questions they serve; **approval** writes the element with its structure and re-decides those signatures through 0053's basis, and every approved element carries **regression cases** rerun when its definition changes. Measured by questions answered correctly and by the share of proposals people change. No automatic adoption; the 0003 `Suggest` retires once this replaces it || 0062 | [The export says what is contested](0062-the-export-says-what-is-contested.md) | Proposed 2026-09-25 · #564: the RDF export carries `fact_conflicts` and `axiom_violations` as two distinct resource classes — conflicts point at the prior and incoming statements, violations at the statements they are about and (open rows only) the governing criterion and its relation; an ordered `rdf:List` keeps cycle paths; precondition is the current-ontology invariant: `status = 'open'` means reconciled against the current ontology, so criterion-changing ontology edits settle superseded open rows with `criterion_changed` and re-detect in the same transaction| | Record | Domain | Status |diff --git a/scripts/bench/README.md b/scripts/bench/README.mdindex f883b862..923ef5c7 100644--- a/scripts/bench/README.md+++ b/scripts/bench/README.md@@ -240,6 +240,9 @@ node scripts/bench/competency.mjs --kb <kb-id> --seed scripts/bench/truth/redocr- 有期望答案的问题交给裁判模型判"回答里说了期望答案没有";没有的只查它需要的形状(`needs` 里的类与属性)是否都在、属性是否有类型化事实。后者弱得多——一条开放图谱就能答的问题不需要本体,这是 0061 的开放问题。+- 回答经没经过图谱也记下来(`via`:graph / text / both / none,看 chat 的 step 事件里 facts、neighbors、+ timeline、path 这些图谱工具有没有拿到事实,search、document 有没有读正文)。报告里"答上"之外另给+ "经过图谱"和"只靠图谱"两个数:只靠正文答上的不需要本体,这个数才是本体自己的。- 第二个数从 `ontology_proposals` 算:代理提的里人表过态的,有几条被拒或改过再采纳。- `--seed` 把写好的问题灌进库(同一句跳过),`--only-report` 只读数不问。- `truth/redocred-typed.questions.json` 是给 `typed.mjs` 那份 100 篇 Re-DocRED 库写的十一条:十条有期望答案diff --git a/scripts/bench/competency.mjs b/scripts/bench/competency.mjsindex 447ece06..6afa6e64 100644--- a/scripts/bench/competency.mjs+++ b/scripts/bench/competency.mjs@@ -10,6 +10,10 @@// 而判分口径会改(同 ask.mjs)。// 有期望答案的问题交给裁判模型判;没有的只查它需要的形状(needs 里的类与属性)是否存在并且有事实。// 服务端不自己问:chat 还没有进程内入口,而"按人问的方式问"正是走接口的意思。+//+// 答案经没经过图谱也记下来(`via`):chat 的 step 事件里 facts / neighbors / timeline / path 是图谱+// 工具,search / document 是正文。答对但只走了正文的问题不需要本体——这是 0061 的开放问题,+// 现在至少能数出来。import fs from "node:fs";import { api, login, askChat, parseArgs, log, psql } from "./lib.mjs";@@ -17,6 +21,18 @@ const args = parseArgs(process.argv);const KB = args.kb;if (!KB) { console.error("--kb <knowledge base id> 是必须的"); process.exit(2); }+// 一轮回答走了哪条路。graph = 只从图谱工具拿到过事实;text = 只读了正文;both;none+// 服务端写的 kind:paths_between 那一步叫 "path"(tools_graph.rs)+const GRAPH_KINDS = new Set(["facts", "neighbors", "timeline", "path"]);+const TEXT_KINDS = new Set(["search", "document"]);+function viaOf(steps) {+ const count = (s) => Number((/^(\d+)/.exec(String(s.count ?? s.detail ?? "")) ?? [])[1] ?? 0);+ const graph = steps.filter((s) => GRAPH_KINDS.has(s.kind) && count(s) > 0).length;+ const text = steps.filter((s) => TEXT_KINDS.has(s.kind) && count(s) > 0).length;+ return { via: graph && text ? "both" : graph ? "graph" : text ? "text" : "none", graph_steps: graph, text_steps: text,+ steps: steps.map((s) => `${s.kind}:${s.label ?? ""}:${s.count ?? s.detail ?? ""}`).slice(0, 20) };+}+function judgeEndpoint() {if (process.env.BENCH_JUDGE_BASE) return { base: process.env.BENCH_JUDGE_BASE, key: process.env.BENCH_JUDGE_KEY || "", model: process.env.BENCH_JUDGE_MODEL || "" };const [base, key, model] = psql(`SELECT s.chat_base_url, s.chat_api_key, s.chat_model FROM llm_settings s JOIN knowledge_bases k ON k.workspace_id=s.workspace_id WHERE k.id='${KB}'`).split("|");@@ -59,15 +75,19 @@ async function main() {const ep = judgeEndpoint();for (const q of questions) {const t0 = Date.now();- let answered = false, judged_by, detail = {}, answer = "";+ let answered = false, judged_by, detail = {}, answer = "", via = q.last_result?.via ?? "none";try {if (args.rejudge) {answer = q.last_result?.answer ?? "";if (!answer) throw new Error("no stored answer to rejudge");+ detail = { ...(q.last_result?.detail ?? {}) };} else {const r = await askChat(KB, q.question);answer = r.text;if (r.error) detail.error = String(r.error).slice(0, 300);+ const v = viaOf(r.steps);+ via = v.via;+ detail = { graph_steps: v.graph_steps, text_steps: v.text_steps, steps: v.steps };}if (q.expected_answer) {judged_by = "expected";@@ -85,7 +105,7 @@ async function main() {const missingClasses = (needs.classes ?? []).filter((k) => !classKeys.has(k));const missingProps = (needs.properties ?? []).filter((k) => !propByKey.has(k));const emptyProps = (needs.properties ?? []).filter((k) => propByKey.has(k) && !(propByKey.get(k).usage > 0));- detail = { missing_classes: missingClasses, missing_properties: missingProps, empty_properties: emptyProps };+ detail = { ...detail, missing_classes: missingClasses, missing_properties: missingProps, empty_properties: emptyProps };answered = missingClasses.length === 0 && missingProps.length === 0 && emptyProps.length === 0&& ((needs.classes ?? []).length + (needs.properties ?? []).length > 0);}@@ -93,13 +113,13 @@ async function main() {detail.error = String(e.message || e).slice(0, 300);}detail.seconds = Math.round((Date.now() - t0) / 1000);- await api("POST", `/api/v1/kbs/${KB}/questions/${q.id}/result`, { answered, answer, judged_by, detail });- log(`${answered ? "✓" : "✗"} [${judged_by}] ${q.question} — ${detail.why ?? JSON.stringify(detail)}`);+ await api("POST", `/api/v1/kbs/${KB}/questions/${q.id}/result`, { answered, answer, judged_by, via, detail });+ log(`${answered ? "✓" : "✗"} [${judged_by}, via ${via}] ${q.question} — ${detail.why ?? JSON.stringify(detail).slice(0, 200)}`);}}const report = await api("GET", `/api/v1/kbs/${KB}/questions/report`);const qs = report.questions, ps = report.proposals;- console.log(`questions: ${qs.answered}/${qs.checked} answered (${qs.accepted} accepted, ${qs.proposed} proposed)`);+ console.log(`questions: ${qs.answered}/${qs.checked} answered (${qs.accepted} accepted, ${qs.proposed} proposed); with graph facts ${qs.answered_with_graph}, graph only ${qs.answered_graph_only}`);console.log(`proposals: ${ps.changed}/${ps.decided} changed before adoption or rejected` + (ps.changed_share != null ? ` (${(ps.changed_share * 100).toFixed(0)}%)` : "") + `, ${ps.open} open`);}main().catch((e) => { console.error(e); process.exit(1); });diff --git a/web/src/api.ts b/web/src/api.tsindex 64913d3c..bf23714d 100644--- a/web/src/api.ts+++ b/web/src/api.ts@@ -1250,7 +1250,16 @@ export interface CompetencyQuestion {/** 本体的两个数(0061 决定 5) */export interface QuestionReport {- questions: { accepted: number; proposed: number; checked: number; answered: number };+ questions: {+ accepted: number;+ proposed: number;+ checked: number;+ answered: number;+ /** 答上了且途中从图谱拿到过事实 */+ answered_with_graph: number;+ /** 答上了且只走了图谱:本体本身答的 */+ answered_graph_only: number;+ };proposals: {open: number;adopted: number;diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex 154e2ad9..ddc3e4b6 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -1780,10 +1780,16 @@ export const en = {acceptQuestion: "Accept",questionAnswered: "answered",questionUnanswered: "not answered",+ /* 回答从哪里来的(competency.mjs 记的 via) */+ questionVia: { graph: "from the graph", text: "from text", both: "graph and text" } as Record<string, string>,questionsScore: (answered: number, checked: number) =>`${answered} of ${checked} answered`,questionsScoreHint:"Accepted questions the competency bench asked over the graph, and how many it answered.",+ viaGraph: (withGraph: number, graphOnly: number) =>+ `${withGraph} with graph facts · ${graphOnly} graph only`,+ viaGraphHint:+ "Of the answered questions, how many were answered after fetching facts from the graph, and how many without reading document text at all. An answer read from text alone did not need the ontology.",proposalsChanged: (changed: number, decided: number) =>`${changed} of ${decided} proposals changed`,proposalsChangedHint:diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 9854eb8b..0d4040d8 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -1547,9 +1547,14 @@ export const zh: Strings = {acceptQuestion: "接受",questionAnswered: "答上了",questionUnanswered: "没答上",+ questionVia: { graph: "来自图谱", text: "来自正文", both: "图谱与正文" },questionsScore: (answered: number, checked: number) =>`${checked} 条问过,${answered} 条答上`,questionsScoreHint: "competency bench 对图谱问过的已接受问题,答上了几条。",+ viaGraph: (withGraph: number, graphOnly: number) =>+ `${withGraph} 条经过图谱 · ${graphOnly} 条只靠图谱`,+ viaGraphHint:+ "答上的问题里,几条途中从图谱拿到过事实,几条根本没读正文。只靠正文答上的不需要本体。",proposalsChanged: (changed: number, decided: number) =>`${decided} 条提案表过态,${changed} 条被改或被拒`,proposalsChangedHint: "代理的提案里人表过态的,有几条被拒或改过再采纳。",diff --git a/web/src/pages/Ontology.tsx b/web/src/pages/Ontology.tsxindex 47410fad..cb716745 100644--- a/web/src/pages/Ontology.tsx+++ b/web/src/pages/Ontology.tsx@@ -1978,6 +1978,14 @@ function QuestionsPanel({)}</Chip>)}+ {report.data && report.data.questions.answered > 0 && (+ <Chip tone="neutral" title={S.ontology.viaGraphHint}>+ {S.ontology.viaGraph(+ report.data.questions.answered_with_graph,+ report.data.questions.answered_graph_only,+ )}+ </Chip>+ )}{report.data && report.data.proposals.decided > 0 && (<Chip tone="neutral" title={S.ontology.proposalsChangedHint}>{S.ontology.proposalsChanged(@@ -2045,6 +2053,8 @@ function QuestionsPanel({{(q.last_result as { answered?: boolean }).answered? S.ontology.questionAnswered: S.ontology.questionUnanswered}+ {S.ontology.questionVia[(q.last_result as { via?: string }).via ?? ""] &&+ ` · ${S.ontology.questionVia[(q.last_result as { via?: string }).via ?? ""]}`}</Chip>)}{q.status === "proposed" && (