World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
The agent proposes questions for a base that has none, and the ontology is measured by two numbers: 0061 cut 3 first slice (#955)
WaylandYang6f74a08f901e
diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rsindex bc58d3be..097802f7 100644--- a/crates/utopia-cli/src/main.rs+++ b/crates/utopia-cli/src/main.rs@@ -82,7 +82,7 @@ struct ManifestDataDir {/// not a side effect of a code change.// 是迁移文件的**个数**,不是最大的编号(守卫 `schema_version_policy_compares_against_current`// 按个数比):编号有空缺时两者不同——0071 由一个开放 PR 占着,0072 先落,个数是 71-const CURRENT_SCHEMA_VERSION: u32 = 83;+const CURRENT_SCHEMA_VERSION: u32 = 84;fn main() -> anyhow::Result<()> {dotenvy::dotenv().ok();diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rsindex 93c95c25..b7461e56 100644--- a/crates/utopia-extract/src/lib.rs+++ b/crates/utopia-extract/src/lib.rs@@ -16,6 +16,7 @@ pub mod implication;pub mod ontology_agent;pub mod open;pub mod phrase_align;+pub mod question_agent;pub mod time;/// Response-scoped reference to a persistent entity; database UUIDs must never enter prompts.diff --git a/crates/utopia-extract/src/question_agent.rs b/crates/utopia-extract/src/question_agent.rsnew file mode 100644index 00000000..bac7dcb4--- /dev/null+++ b/crates/utopia-extract/src/question_agent.rs@@ -0,0 +1,272 @@+//! 给没有能力问题的库提问题(0061 决定 1):从开放图谱说得最多的东西——陈述最多的形状、+//! 最多的类别词、连接最多的实体——出发,让模型写出这个库该答得上来的问题。提出来的是+//! `proposed`,人接受、改或拒;代理不替人决定库是干什么的。++use std::collections::HashSet;++use serde_json::Value;+use utopia_llm::ChatMessage;++use crate::align::parse_value;+use crate::ontology_agent::Glossary;++/// 一条说得多的形状;`property` 是对齐绑上的属性键(绑上了才有)+#[derive(Debug, Clone)]+pub struct TopSignature<'a> {+ pub phrase: &'a str,+ pub subject_class: Option<&'a str>,+ pub object_class: Option<&'a str>,+ pub object_is_value: bool,+ pub statement_count: i64,+ pub property: Option<&'a str>,+ pub example: Option<&'a str>,+}++#[derive(Debug, Clone)]+pub struct TopKindWord<'a> {+ pub kind_word: &'a str,+ pub count: i64,+}++#[derive(Debug, Clone)]+pub struct TopEntity<'a> {+ pub name: &'a str,+ pub class: Option<&'a str>,+ pub degree: i64,+}++/// 一条提出来的问题:原话,加上答它要走的类与属性(都是现有的键;模型编的键丢掉)+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct ProposedQuestion {+ pub question: String,+ pub classes: Vec<String>,+ pub properties: Vec<String>,+}++const SYSTEM: &str = "You write competency questions for a knowledge base built from documents. \+A competency question is a question a person would actually ask this base and expect it to answer \+from its graph; the ontology (its classes and properties) is judged by whether such questions can be \+answered. You are given what the documents say most: the most frequent statement shapes (a phrase \+between a subject class and an object class or a value, with the property it binds to if any), the \+most frequent kind words, and the most connected entities, plus the current ontology.\n\+Rules:\n\+- Write questions in the language of the documents, as a person would phrase them, specific enough \+to have a checkable answer (name the kind of thing asked for; a concrete entity from the list may \+be used when it makes the question natural).\n\+- Prefer questions that need the graph to answer: several statements, or a join across two \+properties. Do not ask what a single sentence answers.\n\+- Each question lists the classes and properties an answer must traverse, as keys from the \+ontology; leave the lists empty when the ontology has no element for it yet (that gap is the point).\n\+- No duplicates, no yes/no questions, at most the number asked for.\n\+Answer with one JSON object and nothing else:\n\+{\"q\":[{\"question\":\"...\",\"classes\":[\"class_key\"],\"properties\":[\"property_key\"]}]}";++pub fn build_question_messages(+ signatures: &[TopSignature<'_>],+ kind_words: &[TopKindWord<'_>],+ entities: &[TopEntity<'_>],+ glossary: &Glossary<'_>,+ existing: &[&str],+ want: usize,+) -> Vec<ChatMessage> {+ let mut user = String::new();+ user.push_str("Ontology classes:\n");+ if glossary.classes.is_empty() {+ user.push_str(" (none)\n");+ }+ for (key, label, description) in &glossary.classes {+ user.push_str(&format!("- {key} · {label}"));+ if !description.is_empty() {+ user.push_str(&format!(" · {description}"));+ }+ user.push('\n');+ }+ user.push_str("Ontology properties:\n");+ if glossary.properties.is_empty() {+ user.push_str(" (none)\n");+ }+ for (key, label, kind, domains, ranges, description) in &glossary.properties {+ user.push_str(&format!("- {key} · {label} · {kind}"));+ if !domains.is_empty() || !ranges.is_empty() {+ user.push_str(&format!(" · {} → {}", domains.join("|"), ranges.join("|")));+ }+ if !description.is_empty() {+ user.push_str(&format!(" · {description}"));+ }+ user.push('\n');+ }+ user.push_str("\nMost frequent statement shapes:\n");+ for s in signatures {+ user.push_str(&format!(+ "- {} — \"{}\" → {} · {} statements",+ s.subject_class.unwrap_or("?"),+ s.phrase,+ if s.object_is_value {+ "value"+ } else {+ s.object_class.unwrap_or("?")+ },+ s.statement_count+ ));+ match s.property {+ Some(p) => user.push_str(&format!(" · bound to {p}")),+ None => user.push_str(" · not bound to any property"),+ }+ if let Some(e) = s.example {+ user.push_str(&format!(" · e.g. {e}"));+ }+ user.push('\n');+ }+ user.push_str("\nMost frequent kind words:\n");+ for k in kind_words {+ user.push_str(&format!("- {} · {} entities\n", k.kind_word, k.count));+ }+ user.push_str("\nMost connected entities:\n");+ for e in entities {+ user.push_str(&format!(+ "- {} ({}) · {} statements\n",+ e.name,+ e.class.unwrap_or("?"),+ e.degree+ ));+ }+ if !existing.is_empty() {+ user.push_str("\nQuestions the base already has (do not repeat them):\n");+ for q in existing {+ user.push_str(&format!("- {q}\n"));+ }+ }+ user.push_str(&format!("\nWrite up to {want} competency questions."));+ vec![+ ChatMessage {+ role: "system".into(),+ content: SYSTEM.into(),+ },+ ChatMessage {+ role: "user".into(),+ content: user,+ },+ ]+}++/// 读回复。`class_keys` / `property_keys` 是本体现有的键:答案里不存在的键丢掉(不算坏项——+/// 模型常把它想要的属性写成键,那正是缺口)。返回 (问题, 坏项数)+pub fn parse_question_response(+ raw: &str,+ class_keys: &HashSet<&str>,+ property_keys: &HashSet<&str>,+) -> anyhow::Result<(Vec<ProposedQuestion>, usize)> {+ let v = parse_value(raw)?;+ let items: Vec<&Value> = match v.get("q") {+ Some(Value::Array(a)) => a.iter().collect(),+ Some(Value::Null) | None => Vec::new(),+ Some(_) => anyhow::bail!("`q` is not a list"),+ };+ let mut out: Vec<ProposedQuestion> = Vec::new();+ let mut malformed = 0usize;+ let keys = |v: Option<&Value>, allowed: &HashSet<&str>| -> Vec<String> {+ v.and_then(Value::as_array)+ .into_iter()+ .flatten()+ .filter_map(Value::as_str)+ .map(str::trim)+ .filter(|k| allowed.contains(k))+ .map(str::to_string)+ .collect()+ };+ for it in items {+ let Some(obj) = it.as_object() else {+ malformed += 1;+ continue;+ };+ let question = obj+ .get("question")+ .and_then(Value::as_str)+ .map(|q| q.trim().trim_end_matches('\n').to_string())+ .unwrap_or_default();+ if question.chars().count() < 8 {+ malformed += 1;+ continue;+ }+ if out+ .iter()+ .any(|q| q.question.eq_ignore_ascii_case(&question))+ {+ continue;+ }+ out.push(ProposedQuestion {+ question,+ classes: keys(obj.get("classes"), class_keys),+ properties: keys(obj.get("properties"), property_keys),+ });+ }+ Ok((out, malformed))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn the_prompt_lists_shapes_kind_words_entities_and_existing_questions() {+ let glossary = Glossary {+ classes: vec![("person", "Person", "")],+ properties: vec![(+ "p20",+ "placeOfDeath",+ "relation",+ vec!["person"],+ vec!["location"],+ "",+ )],+ };+ let sigs = [TopSignature {+ phrase: "died in",+ subject_class: Some("person"),+ object_class: Some("location"),+ object_is_value: false,+ statement_count: 7,+ property: Some("p20"),+ example: Some("Orlov died in Moscow"),+ }];+ let words = [TopKindWord {+ kind_word: "district",+ count: 12,+ }];+ let ents = [TopEntity {+ name: "Pedro II",+ class: Some("person"),+ degree: 9,+ }];+ let m = build_question_messages(&sigs, &words, &ents, &glossary, &["Who died where?"], 10);+ let user = &m[1].content;+ assert!(user.contains("\"died in\" → location · 7 statements · bound to p20"));+ assert!(user.contains("district · 12 entities"));+ assert!(user.contains("Pedro II (person) · 9 statements"));+ assert!(user.contains("do not repeat them"));+ assert!(user.contains("Who died where?"));+ assert!(user.contains("up to 10"));+ }++ #[test]+ fn keys_the_ontology_lacks_are_dropped_and_short_or_duplicate_questions_are_not_kept() {+ let classes: HashSet<&str> = ["person"].into();+ let props: HashSet<&str> = ["p20"].into();+ let raw = r#"{"q":[+ {"question":"Where did each person die?","classes":["person","ghost"],"properties":["p20","died_in"]},+ {"question":"where did each PERSON die?","classes":[],"properties":[]},+ {"question":"Why?","classes":[],"properties":[]},+ "not an object"+ ]}"#;+ let (qs, malformed) = parse_question_response(raw, &classes, &props).unwrap();+ assert_eq!(+ qs,+ vec![ProposedQuestion {+ question: "Where did each person die?".into(),+ classes: vec!["person".into()],+ properties: vec!["p20".into()],+ }]+ );+ assert_eq!(malformed, 2);+ }+}diff --git a/crates/utopia-server/src/api/mod.rs b/crates/utopia-server/src/api/mod.rsindex 0143ec46..a1ab68d5 100644--- a/crates/utopia-server/src/api/mod.rs+++ b/crates/utopia-server/src/api/mod.rs@@ -370,6 +370,15 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router {"/kbs/{id}/questions/{qid}",patch(question_routes::update).delete(question_routes::delete),)+ .route(+ "/kbs/{id}/questions/propose",+ post(question_routes::propose_questions),+ )+ .route("/kbs/{id}/questions/report", get(question_routes::report))+ .route(+ "/kbs/{id}/questions/{qid}/result",+ post(question_routes::record_result),+ ).route("/kbs/{id}/ontology/propose", post(question_routes::propose)).route("/kbs/{id}/ontology/proposals/adopt",diff --git a/crates/utopia-server/src/api/question_routes.rs b/crates/utopia-server/src/api/question_routes.rsindex e9fb4f32..4efad3c5 100644--- a/crates/utopia-server/src/api/question_routes.rs+++ b/crates/utopia-server/src/api/question_routes.rs@@ -128,6 +128,82 @@ pub async fn delete(Ok(Json(json!({ "ok": true })))}+/// 让代理给这个库提问题:排一份任务。结果是 status = proposed 的问题,人接受了才算+pub async fn propose_questions(+ State(state): State<AppState>,+ AuthUser(user): AuthUser,+ Path(kb_id): Path<Uuid>,+) -> ApiResult<Json<serde_json::Value>> {+ utopia_store::access::require_kb(&state.pool, &user, kb_id, Role::Editor).await?;+ let queued = utopia_store::jobs::enqueue_unless_queued(+ &state.pool,+ "propose_questions",+ json!({ "kb_id": kb_id }),+ )+ .await?;+ Ok(Json(json!({ "queued": queued.is_some() })))+}++#[derive(Deserialize)]+pub struct QuestionResultReq {+ pub answered: bool,+ /// 拿到的回答(截断存)+ #[serde(default)]+ pub answer: Option<String>,+ /// expected | shape:按期望答案判的,还是只查了它需要的形状+ #[serde(default)]+ pub judged_by: Option<String>,+ #[serde(default)]+ pub detail: Option<serde_json::Value>,+}++/// 一条问题问过了(决定 5)。问的是 competency bench,它按人在 chat 里问的方式问;+/// 服务端只记结果,不自己问——chat 还没有一个进程内的入口+pub async fn record_result(+ State(state): State<AppState>,+ AuthUser(user): AuthUser,+ Path((kb_id, qid)): Path<(Uuid, Uuid)>,+ Json(req): Json<QuestionResultReq>,+) -> ApiResult<Json<serde_json::Value>> {+ utopia_store::access::require_kb(&state.pool, &user, kb_id, Role::Editor).await?;+ let answer: Option<String> = req.answer.map(|a| a.chars().take(4000).collect());+ let result = json!({+ "answered": req.answered,+ "answer": answer,+ "judged_by": req.judged_by.unwrap_or_else(|| "expected".into()),+ "detail": req.detail,+ });+ if !questions::record_result(&state.pool, kb_id, qid, &result).await? {+ return Err(AppError::NotFound.into());+ }+ Ok(Json(json!({ "ok": true })))+}++/// 两个数(决定 5):问题答对了几条;代理的提案人改过或拒掉的占几成+pub async fn report(+ State(state): State<AppState>,+ AuthUser(user): AuthUser,+ Path(kb_id): Path<Uuid>,+) -> ApiResult<Json<serde_json::Value>> {+ utopia_store::access::require_kb(&state.pool, &user, kb_id, Role::Viewer).await?;+ let q = questions::report(&state.pool, kb_id).await?;+ let p = utopia_store::ontology::agent_proposal_report(&state.pool, kb_id).await?;+ let decided = p.adopted + p.rejected;+ let changed = p.adopted_edited + p.rejected;+ Ok(Json(json!({+ "questions": q,+ "proposals": {+ "open": p.open,+ "adopted": p.adopted,+ "adopted_edited": p.adopted_edited,+ "rejected": p.rejected,+ "decided": decided,+ "changed": changed,+ "changed_share": if decided > 0 { Some(changed as f64 / decided as f64) } else { None },+ }+ })))+}+/// 叫代理来看一眼:排一份任务,马上回。结果落在 `ontology_proposals`,界面从那里读pub async fn propose(State(state): State<AppState>,diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex 65e8e7e8..3d7b7ea0 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -646,6 +646,17 @@ async fn dispatch(st: &state::AppState, job: &utopia_store::jobs::Job) -> anyhow.ok_or_else(|| anyhow::anyhow!("payload 缺少 kb_id"))?;ontology_agent::propose(st, kb_id).await}+ "propose_questions" => {+ let kb_id: Uuid = job+ .payload+ .get("kb_id")+ .and_then(|v| v.as_str())+ .and_then(|s| s.parse().ok())+ .ok_or_else(|| anyhow::anyhow!("payload 缺少 kb_id"))?;+ ontology_agent::propose_questions(st, kb_id)+ .await+ .map(|_| ())+ }"sync_source" => {let source_id: Uuid = job.payloaddiff --git a/crates/utopia-server/src/ontology_agent.rs b/crates/utopia-server/src/ontology_agent.rsindex b7ac1dbc..9b8da2b1 100644--- a/crates/utopia-server/src/ontology_agent.rs+++ b/crates/utopia-server/src/ontology_agent.rs@@ -23,6 +23,13 @@ use crate::extraction::chat_retrying_rate_limits_at;use crate::llm_util;use crate::state::AppState;+/// 给没问题的库一次提这么多问题+pub(crate) const QUESTIONS_PER_ROUND: usize = 10;+/// 提问题时看多少条说得最多的形状 / 类别词 / 实体+const TOP_SIGNATURES: usize = 40;+const TOP_KIND_WORDS: usize = 20;+const TOP_ENTITIES: usize = 20;+/// 一次调用交给模型的形状数(0044 决定 3 的成本教训:词表每批只带一次)pub(crate) const SIGNATURES_PER_CALL: usize = 12;/// 每批跟着的类别词至多这么多@@ -239,6 +246,12 @@ async fn propose_locked(}open_words.truncate(calls * KIND_WORDS_PER_CALL);+ // 一条问题都没有的库(决定 1):先让代理提问题,人接受了下一轮才算数;这一轮照旧按说法提+ if competency_questions::list(pool, kb_id).await?.is_empty() {+ if let Err(e) = propose_questions_with(state, kb_id, settings, client).await {+ tracing::warn!(%kb_id, error = %e, "本体代理:提问题失败,这一轮不带问题");+ }+ }let questions = competency_questions::accepted(pool, kb_id).await?;let q_items: Vec<agent::QuestionItem<'_>> = questions.iter()@@ -715,6 +728,13 @@ pub async fn adopt(let Some(p) = utopia_store::ontology::open_proposal(pool, kb_id, section, key).await? else {return Err(AppError::NotFound);};+ // 人改过几格:决定 5 的第二个数要它+ let edited = edits.label.is_some()+ || edits.description.is_some()+ || edits.datatype.is_some()+ || edits.domains.is_some()+ || edits.ranges.is_some()+ || edits.parents.is_some();let classes = utopia_store::graph::entity_types(pool, kb_id).await?;let by_key: HashMap<&str, Uuid> = classes.iter().map(|c| (c.key.as_str(), c.id)).collect();let resolve = |keys: Vec<String>| -> Result<Vec<Uuid>, AppError> {@@ -845,6 +865,9 @@ pub async fn adopt(}}utopia_store::ontology::decide_proposal(pool, kb_id, section, key, "adopted", actor).await?;+ if edited {+ utopia_store::ontology::mark_proposal_edited(pool, kb_id, section, key).await?;+ }let _ = utopia_store::audit::record(pool,Some(kb_id),@@ -933,6 +956,174 @@ async fn adopt_map_to(Ok(property)}+/// 任务入口:给库提问题(决定 1 的"没有问题的库不被卡住")。界面上点、或本体代理发现库里+/// 一条问题都没有时走这里+pub async fn propose_questions(state: &AppState, kb_id: Uuid) -> anyhow::Result<usize> {+ let pool = &state.pool;+ let kb = utopia_store::kbs::get(pool, kb_id).await?;+ let settings = utopia_store::settings::get(pool, kb.workspace_id)+ .await?+ .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot propose questions"))?;+ let client = llm_util::chat_client_thinking(&settings)+ .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot propose questions"))?;+ propose_questions_with(state, kb_id, &settings, &client).await+}++#[derive(sqlx::FromRow)]+struct EntityDegree {+ name: String,+ class: Option<String>,+ degree: i64,+}++async fn propose_questions_with(+ state: &AppState,+ kb_id: Uuid,+ settings: &utopia_core::models::LlmSettings,+ client: &utopia_llm::LlmClient,+) -> anyhow::Result<usize> {+ use utopia_extract::question_agent as qa;+ let pool = &state.pool;+ let classes = utopia_store::graph::entity_types(pool, kb_id).await?;+ let props = utopia_store::ontology::relation_type_views(pool, kb_id).await?;+ let class_key: HashMap<Uuid, &str> = classes.iter().map(|c| (c.id, c.key.as_str())).collect();+ let prop_key: HashMap<Uuid, &str> = props.iter().map(|p| (p.id, p.key.as_str())).collect();+ let keys_of = |ids: &[Uuid]| -> Vec<&str> {+ ids.iter()+ .filter_map(|id| class_key.get(id).copied())+ .collect()+ };+ let glossary = agent::Glossary {+ classes: classes+ .iter()+ .map(|c| (c.key.as_str(), c.label.as_str(), c.description.as_str()))+ .collect(),+ properties: props+ .iter()+ .map(|p| {+ (+ p.key.as_str(),+ p.label.as_str(),+ p.kind.as_str(),+ keys_of(&p.domains),+ keys_of(&p.ranges),+ p.description.as_str(),+ )+ })+ .collect(),+ };+ // 说得最多的形状,带上对齐绑到的属性+ let mut sigs = phrase_bindings::signatures(pool, kb_id).await?;+ sigs.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.phrase.cmp(&b.phrase)));+ sigs.truncate(TOP_SIGNATURES);+ let bindings: HashMap<_, _> = phrase_bindings::bindings(pool, kb_id)+ .await?+ .into_iter()+ .map(|b| (b.key(), b))+ .collect();+ let top_sigs: Vec<qa::TopSignature<'_>> = sigs+ .iter()+ .map(|s| qa::TopSignature {+ phrase: &s.phrase,+ subject_class: s.subject_type_key.as_deref(),+ object_class: s.object_type_key.as_deref(),+ object_is_value: s.object_is_value,+ statement_count: s.count,+ property: bindings+ .get(&s.key())+ .filter(|b| b.status == "bound")+ .and_then(|b| b.relation_type_id)+ .and_then(|id| prop_key.get(&id).copied()),+ example: s.quotes.first().map(String::as_str),+ })+ .collect();+ let mut words = type_bindings::signatures(pool, kb_id).await?;+ words.sort_by(|a, b| {+ b.count+ .cmp(&a.count)+ .then_with(|| a.kind_word.cmp(&b.kind_word))+ });+ words.truncate(TOP_KIND_WORDS);+ let top_words: Vec<qa::TopKindWord<'_>> = words+ .iter()+ .map(|w| qa::TopKindWord {+ kind_word: &w.kind_word,+ count: w.count,+ })+ .collect();+ let degrees: Vec<EntityDegree> = sqlx::query_as(+ "SELECT e.canonical_name AS name, t.key AS class, count(f.id) AS degree+ FROM entities e+ LEFT JOIN entity_types t ON t.id = e.type_id+ JOIN facts f ON f.kb_id = e.kb_id AND (f.subject_id = e.id OR f.object_id = e.id)+ AND f.invalidated_at IS NULL+ WHERE e.kb_id = $1+ GROUP BY e.id, e.canonical_name, t.key+ ORDER BY count(f.id) DESC, e.canonical_name+ LIMIT $2",+ )+ .bind(kb_id)+ .bind(TOP_ENTITIES as i64)+ .fetch_all(pool)+ .await?;+ let top_entities: Vec<qa::TopEntity<'_>> = degrees+ .iter()+ .map(|d| qa::TopEntity {+ name: &d.name,+ class: d.class.as_deref(),+ degree: d.degree,+ })+ .collect();+ let have = competency_questions::list(pool, kb_id).await?;+ let existing: Vec<&str> = have+ .iter()+ .filter(|q| q.status != "rejected")+ .map(|q| q.question.as_str())+ .collect();+ if top_sigs.is_empty() && top_words.is_empty() && top_entities.is_empty() {+ tracing::info!(%kb_id, "提问题:库里还没有东西可问");+ return Ok(0);+ }+ let messages = qa::build_question_messages(+ &top_sigs,+ &top_words,+ &top_entities,+ &glossary,+ &existing,+ QUESTIONS_PER_ROUND,+ );+ let reply = chat_retrying_rate_limits_at(state, settings, client, &messages, Some(0.0)).await?;+ let class_keys: HashSet<&str> = classes.iter().map(|c| c.key.as_str()).collect();+ let prop_keys: HashSet<&str> = props.iter().map(|p| p.key.as_str()).collect();+ let (proposed, malformed) = qa::parse_question_response(&reply.text, &class_keys, &prop_keys)?;+ let mut written = 0usize;+ for q in proposed.iter().take(QUESTIONS_PER_ROUND) {+ if competency_questions::exists_text(pool, kb_id, &q.question).await? {+ continue;+ }+ let needs = json!({ "classes": q.classes, "properties": q.properties });+ competency_questions::create(+ pool,+ kb_id,+ competency_questions::NewQuestion {+ question: &q.question,+ expected_answer: None,+ needs: Some(&needs),+ origin: "agent",+ status: "proposed",+ created_by: None,+ },+ )+ .await?;+ written += 1;+ }+ tracing::info!(%kb_id, proposed = proposed.len(), written, malformed, "提问题结束");+ if written > 0 {+ state.emit_pending(kb_id);+ }+ Ok(written)+}+#[cfg(test)]#[path = "ontology_agent_tests.rs"]mod tests;diff --git a/crates/utopia-server/src/ontology_agent_tests.rs b/crates/utopia-server/src/ontology_agent_tests.rsindex 9be6d562..bf3e04c8 100644--- a/crates/utopia-server/src/ontology_agent_tests.rs+++ b/crates/utopia-server/src/ontology_agent_tests.rs@@ -155,6 +155,23 @@ impl Fx {Ok(())}+ /// 库里有一条人写的问题:本体代理就不先去提问题(那是没问题的库才走的一步)+ async fn seed_question(&self) -> anyhow::Result<Uuid> {+ Ok(competency_questions::create(+ &self.pool,+ self.kb,+ competency_questions::NewQuestion {+ question: "Where was each organization founded?",+ expected_answer: None,+ needs: None,+ origin: "person",+ status: "accepted",+ created_by: None,+ },+ )+ .await?)+ }+async fn requests(&self) -> usize {self.model.requests.lock().unwrap().len()}@@ -211,19 +228,7 @@ async fn an_unbound_shape_becomes_a_proposal_that_serves_a_question_and_adoptionreturn Ok(());};f.decide_none().await?;- let qid = competency_questions::create(- &f.pool,- f.kb,- competency_questions::NewQuestion {- question: "Where was each organization founded?",- expected_answer: None,- needs: None,- origin: "person",- status: "accepted",- created_by: None,- },- )- .await?;+ let qid = f.seed_question().await?;propose(&f.state, f.kb).await?;@@ -375,6 +380,7 @@ async fn a_proposal_for_an_existing_key_or_binding_nothing_is_dropped() -> anyhoreturn Ok(());};f.decide_none().await?;+ f.seed_question().await?;propose(&f.state, f.kb).await?;let open = utopia_store::ontology::open_proposals(&f.pool, f.kb).await?;assert_eq!(open.len(), 1, "only the attribute survives: {open:?}");@@ -399,6 +405,7 @@ async fn a_declined_shape_returns_when_its_statements_double_and_an_existing_ansreturn Ok(());};f.decide_none().await?;+ f.seed_question().await?;propose(&f.state, f.kb).await?;assert_eq!(f.requests().await, 1);@@ -521,3 +528,91 @@ async fn a_declined_shape_returns_when_its_statements_double_and_an_existing_ansassert_eq!(f.requests().await, 2);f.cleanup().await}++#[tokio::test]+async fn a_base_with_no_questions_gets_them_proposed_and_the_report_counts_them(+) -> anyhow::Result<()> {+ let Some(f) = Fx::new(vec![+ // 本体代理发现库里一条问题都没有:先提问题(这一轮的第一次调用)+ json!({ "q": [+ { "question": "Where is each organization located?", "classes": ["organization", "place"], "properties": ["located_in", "made_up"] },+ { "question": "Why?", "classes": [], "properties": [] }+ ] }),+ // 然后照旧按说法提本体+ json!({ "p": [], "existing": [] }),+ // 界面上再叫一次提问题:同一句不重复,新的进来+ json!({ "q": [+ { "question": "where is each ORGANIZATION located?", "classes": [], "properties": [] },+ { "question": "Which organizations were founded in each place?", "classes": ["organization"], "properties": [] }+ ] }),+ ])+ .await?+ else {+ return Ok(());+ };+ f.decide_none().await?;++ propose(&f.state, f.kb).await?;+ assert_eq!(+ f.requests().await,+ 2,+ "questions first, then the ontology round"+ );+ let text = user_text(&f.model.requests.lock().unwrap()[0]);+ assert!(text.contains("Most frequent statement shapes"), "{text}");+ assert!(+ text.contains("\"founded in\""),+ "the shapes are offered: {text}"+ );+ assert!(+ text.contains("Acme (organization)"),+ "the connected entities are offered: {text}"+ );+ let qs = competency_questions::list(&f.pool, f.kb).await?;+ assert_eq!(qs.len(), 1, "the short one is dropped: {qs:?}");+ assert_eq!(qs[0].origin, "agent");+ assert_eq!(qs[0].status, "proposed");+ assert_eq!(+ qs[0].needs,+ Some(json!({ "classes": ["organization", "place"], "properties": ["located_in"] })),+ "keys the ontology lacks are dropped from needs"+ );+ // 提出来的还没接受:本体那一轮没有问题可用+ let text = user_text(&f.model.requests.lock().unwrap()[1]);+ assert!(text.contains("(none written yet"), "{text}");++ let written = propose_questions(&f.state, f.kb).await?;+ assert_eq!(written, 1, "the duplicate is skipped, the new one written");+ assert_eq!(f.requests().await, 3);+ let text = user_text(&f.model.requests.lock().unwrap()[2]);+ assert!(text.contains("do not repeat them"), "{text}");+ assert!(+ text.contains("Where is each organization located?"),+ "{text}"+ );++ // 接受一条、问过一条:报告数得出来+ let qs = competency_questions::list(&f.pool, f.kb).await?;+ competency_questions::set_status(&f.pool, f.kb, qs[0].id, "accepted").await?;+ competency_questions::set_status(&f.pool, f.kb, qs[1].id, "accepted").await?;+ assert!(+ competency_questions::record_result(+ &f.pool,+ f.kb,+ qs[0].id,+ &json!({ "answered": true, "judged_by": "shape" })+ )+ .await?+ );+ let r = competency_questions::report(&f.pool, f.kb).await?;+ assert_eq!(+ (r.accepted, r.proposed, r.checked, r.answered),+ (2, 0, 1, 1)+ );+ let p = utopia_store::ontology::agent_proposal_report(&f.pool, f.kb).await?;+ assert_eq!(+ (p.open, p.adopted, p.adopted_edited, p.rejected),+ (0, 0, 0, 0)+ );+ f.cleanup().await+}diff --git a/crates/utopia-store/src/competency_questions.rs b/crates/utopia-store/src/competency_questions.rsindex 65b5d2b5..6876ebd9 100644--- a/crates/utopia-store/src/competency_questions.rs+++ b/crates/utopia-store/src/competency_questions.rs@@ -170,3 +170,56 @@ pub async fn delete(pool: &PgPool, kb_id: Uuid, id: Uuid) -> AppResult<()> {}Ok(())}++/// 一条问题问过了:什么时候、答成了什么(0061 决定 5)。`result` 至少带 `answered: bool`+pub async fn record_result(+ pool: &PgPool,+ kb_id: Uuid,+ id: Uuid,+ result: &serde_json::Value,+) -> AppResult<bool> {+ let res = sqlx::query(+ "UPDATE competency_questions SET last_checked_at = now(), last_result = $3+ WHERE kb_id = $1 AND id = $2",+ )+ .bind(kb_id)+ .bind(id)+ .bind(result)+ .execute(pool)+ .await?;+ Ok(res.rows_affected() > 0)+}++/// 同一句话(不分大小写、去首尾空白)已经在库里了吗——代理提问题不重复人的+pub async fn exists_text(pool: &PgPool, kb_id: Uuid, question: &str) -> AppResult<bool> {+ Ok(sqlx::query_scalar(+ "SELECT EXISTS (SELECT 1 FROM competency_questions+ WHERE kb_id = $1 AND lower(btrim(question)) = lower(btrim($2)))",+ )+ .bind(kb_id)+ .bind(question)+ .fetch_one(pool)+ .await?)+}++/// 两个数里的第一个:接受了的问题里,问过的、答对的各几条+#[derive(Debug, Clone, Serialize, sqlx::FromRow)]+pub struct QuestionReport {+ pub accepted: i64,+ pub proposed: i64,+ pub checked: i64,+ pub answered: i64,+}++pub async fn report(pool: &PgPool, kb_id: Uuid) -> AppResult<QuestionReport> {+ Ok(sqlx::query_as(+ "SELECT count(*) FILTER (WHERE status = 'accepted') AS accepted,+ count(*) FILTER (WHERE status = 'proposed') AS proposed,+ count(*) FILTER (WHERE status = 'accepted' AND last_checked_at IS NOT NULL) AS checked,+ count(*) FILTER (WHERE status = 'accepted' AND (last_result->>'answered')::boolean) AS answered+ FROM competency_questions WHERE kb_id = $1",+ )+ .bind(kb_id)+ .fetch_one(pool)+ .await?)+}diff --git a/crates/utopia-store/src/ontology.rs b/crates/utopia-store/src/ontology.rsindex 7e6d63d5..225cabcf 100644--- a/crates/utopia-store/src/ontology.rs+++ b/crates/utopia-store/src/ontology.rs@@ -2486,3 +2486,44 @@ mod name_shape_tests {}}}++/// 两个数里的第二个(0061 决定 5):代理的提案里,人表过态的有几条,其中拒掉的、改过再采纳的各几条+#[derive(Debug, Clone, serde::Serialize, sqlx::FromRow)]+pub struct AgentProposalReport {+ pub open: i64,+ pub adopted: i64,+ pub adopted_edited: i64,+ pub rejected: i64,+}++pub async fn agent_proposal_report(pool: &PgPool, kb_id: Uuid) -> AppResult<AgentProposalReport> {+ Ok(sqlx::query_as(+ "SELECT count(*) FILTER (WHERE status = 'open') AS open,+ count(*) FILTER (WHERE status = 'adopted') AS adopted,+ count(*) FILTER (WHERE status = 'adopted' AND edited) AS adopted_edited,+ count(*) FILTER (WHERE status = 'rejected') AS rejected+ FROM ontology_proposals WHERE kb_id = $1 AND proposed_by = 'agent'",+ )+ .bind(kb_id)+ .fetch_one(pool)+ .await?)+}++/// 采纳时人改过:记下来,占比才算得出+pub async fn mark_proposal_edited(+ pool: &PgPool,+ kb_id: Uuid,+ section: &str,+ key: &str,+) -> AppResult<()> {+ sqlx::query(+ "UPDATE ontology_proposals SET edited = true+ WHERE kb_id = $1 AND section = $2 AND key = $3",+ )+ .bind(kb_id)+ .bind(section)+ .bind(key)+ .execute(pool)+ .await?;+ Ok(())+}diff --git a/docs/decisions/0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md b/docs/decisions/0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.mdindex 9773c297..c8c543a9 100644--- a/docs/decisions/0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md+++ b/docs/decisions/0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md@@ -1,6 +1,6 @@# 0061 · The ontology is proposed from the open graph and judged by its questions-- **Status**: Accepted 2026-09-26 · 0044 cut 5 · cut 1 built (#947, migration 0077): competency questions are rows, the agent proposes from unbound signatures and kind words into `ontology_proposals`, adoption writes the element and re-decides its shapes through alignment · cut 1.1 (#950, migration 0078): the agent records every shape it saw, "already in the ontology" becomes a map_to proposal adopted as a person's binding per shape, adoption embeds the element before alignment · measured on a 100-document base: ~9k tokens a call, 12 calls a round, about half of a round's proposals adoptable as-is · cuts 2–4 open · the ontology agent reads the open graph's unbound signatures and kind words against the base's competency questions and proposes types, properties and rules with definitions, examples and the signatures they would bind; approval writes the ontology and re-decides those signatures; every approved element carries regression cases; the ontology is measured by the questions it answers+- **Status**: Accepted 2026-09-26 · 0044 cut 5 · cut 1 built (#947, migration 0077): competency questions are rows, the agent proposes from unbound signatures and kind words into `ontology_proposals`, adoption writes the element and re-decides its shapes through alignment · cut 1.1 (#950, migration 0078): the agent records every shape it saw, "already in the ontology" becomes a map_to proposal adopted as a person's binding per shape, adoption embeds the element before alignment · measured on a 100-document base: ~9k tokens a call, 12 calls a round, about half of a round's proposals adoptable as-is · cut 3 first slice (#955, migration 0079): the agent proposes questions for a base with none, `competency.mjs` asks the accepted ones through chat and the two numbers are reported (11 of 11 on the bench base) · cut 2, the rest of cut 3 (FDA and statistics corpora) and cut 4 open · the ontology agent reads the open graph's unbound signatures and kind words against the base's competency questions and proposes types, properties and rules with definitions, examples and the signatures they would bind; approval writes the ontology and re-decides those signatures; every approved element carries regression cases; the ontology is measured by the questions it answers- **Written**: 2026-09-26 (conventions in the [README](README.md))- **Related**: [0044](0044-the-ontology-is-a-view-over-what-documents-say.md) §4 decided the workbench and named this cut; [0003](0003-ontology-growth-loop.md) grew the ontology from `ontology_misses` and surface predicates, which [0044](0044-the-ontology-is-a-view-over-what-documents-say.md) cut 1 left idle and #725 retires; [0012](0012-the-ontology-is-a-contract-not-a-suggestion.md) makes every element a contract a person signed, which this record keeps; [0053](0053-a-phrase-decision-records-the-inputs-it-considered.md) is what makes approval re-decide signatures without a new mechanism; [0008](0008-ontology-packs-as-cold-start.md) is the second of the three sources; [0043](0043-every-review-queue-is-governed.md) and #725 give the queue its shape. From #725's "Ontology proposals" row and 0044's open question about a base with no questions.diff --git a/docs/decisions/README.md b/docs/decisions/README.mdindex 9fd0a958..547167e2 100644--- a/docs/decisions/README.md+++ b/docs/decisions/README.md@@ -81,7 +81,7 @@ The test for writing one: if someone (including us) looks at a piece of code in| 0053 | [A phrase decision records the inputs it considered](0053-a-phrase-decision-records-the-inputs-it-considered.md) | Implemented 2026-09-23 · a decision stores a fingerprint of the ancestor closures and admitted candidates it saw; stale means the fingerprint of the current inputs differs, which is what timestamps could not see (#807, #795): inheritance, parent edges, edits during the request; no-candidate and overflow become recorded outcomes; requeue reads live signatures only, so orphaned rows stop looping · revised 2026-09-26: kind words fingerprinted too, read from one snapshot and compared again when a reply is accepted| 0054 | [A source may push statements in the open contract](0054-a-source-may-push-statements-in-the-open-contract.md) | Proposed 2026-09-23 · cut 1 in its PR · a `statements` source accepts the open extraction contract (`e`/`s`/`n`) verbatim on `POST /sources/{id}/statements` with the `api` push's identity, versions and tombstones; the payload is stored as one chunk and extraction parses it instead of prompting a model, then runs the unchanged path, so a pushed statement is an open statement and reaches the typed graph only through alignment; there is no slot for a property or class; an update marks earlier statements stale, it does not close them; tables stay on the mount (0036)| 0060 | [A rule's definition has a history](0060-a-rule-definition-has-a-history.md) | Implemented 2026-09-25 (#912, migration 0076) · A business rule was edited in place and a derivation pointed at the row, so an invalidated conclusion pointed at a rule that now said something else. Every edit that changes what a rule says opens a **version**, a full snapshot with a record time; a derivation names the version it was drawn under, a kept conclusion moves to the new one, and the proof, the rules panel and a versions endpoint read the history. Name, description and the switch open nothing. Exporting rule bodies per version is now honest and stays #902's second cut |-| 0061 | [The ontology is proposed from the open graph and judged by its questions](0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md) | Accepted · cut 1 built (#947): questions, agent proposals, adoption · cut 1.1 (#950): the agent remembers what it saw, existing answers become map_to · cuts 2–4 open · A base's unbound signatures and classless kind words are what its documents say and its ontology cannot hold, and nothing says what the ontology is *for*. **Competency questions** become rows of the base and the standard an element is judged by; an **ontology agent** reads the unbound signatures and kind words against them and proposes types, properties and rules with a definition, the statements they would bind and the questions they serve; **approval** writes the element with its structure and re-decides those signatures through 0053's basis, and every approved element carries **regression cases** rerun when its definition changes. Measured by questions answered correctly and by the share of proposals people change. No automatic adoption; the 0003 `Suggest` retires once this replaces it |+| 0061 | [The ontology is proposed from the open graph and judged by its questions](0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md) | Accepted · cut 1 built (#947): questions, agent proposals, adoption · cut 1.1 (#950): the agent remembers what it saw, existing answers become map_to · cut 3 first slice (#955): proposed questions, the two numbers, `competency.mjs` · cut 2, rest of 3, 4 open · A base's unbound signatures and classless kind words are what its documents say and its ontology cannot hold, and nothing says what the ontology is *for*. **Competency questions** become rows of the base and the standard an element is judged by; an **ontology agent** reads the unbound signatures and kind words against them and proposes types, properties and rules with a definition, the statements they would bind and the questions they serve; **approval** writes the element with its structure and re-decides those signatures through 0053's basis, and every approved element carries **regression cases** rerun when its definition changes. Measured by questions answered correctly and by the share of proposals people change. No automatic adoption; the 0003 `Suggest` retires once this replaces it || 0062 | [The export says what is contested](0062-the-export-says-what-is-contested.md) | Proposed 2026-09-25 · #564: the RDF export carries `fact_conflicts` and `axiom_violations` as two distinct resource classes — conflicts point at the prior and incoming statements, violations at the statements they are about and (open rows only) the governing criterion and its relation; an ordered `rdf:List` keeps cycle paths; precondition is the current-ontology invariant: `status = 'open'` means reconciled against the current ontology, so criterion-changing ontology edits settle superseded open rows with `criterion_changed` and re-detect in the same transaction| | Record | Domain | Status |diff --git a/migrations/0079_a_proposal_remembers_whether_a_person_changed_it.sql b/migrations/0079_a_proposal_remembers_whether_a_person_changed_it.sqlnew file mode 100644index 00000000..ae004d81--- /dev/null+++ b/migrations/0079_a_proposal_remembers_whether_a_person_changed_it.sql@@ -0,0 +1,3 @@+-- 0061 · cut 3:本体用两个数来量(决定 5)。其中一个是"人改过或拒掉的提案占比"——+-- 采纳时人改过标签、定义、域值域,此前只在审计里留痕,算不出比例。+ALTER TABLE ontology_proposals ADD COLUMN edited BOOLEAN NOT NULL DEFAULT false;diff --git a/scripts/bench/README.md b/scripts/bench/README.mdindex 1bdec573..f883b862 100644--- a/scripts/bench/README.md+++ b/scripts/bench/README.md@@ -226,6 +226,25 @@ node scripts/bench/ask.mjs --kb <id> --recall 8 # 只量检索:那条对`lib.mjs` 是两个测量台共用的地基——**判等必须是同一份**,各写一份 `same()`迟早漂移,而一旦漂移,「提议对了几条」与「答案对了几条」就不是同一把尺子量出来的。+## 本体的两个数(0061 决定 5)++`competency.mjs` 把一个库**接受了的**能力问题按人在 chat 里问的方式问一遍(走 `/kbs/{id}/chat`,+不另造引擎),判答没答上,把结果写回问题(`last_result`),再读服务端算的两个数:++```+BENCH_BASE=http://127.0.0.1:1524 BENCH_PSQL="docker exec ... -d utopia_bench4 -tAc" \+BENCH_JUDGE_BASE=... BENCH_JUDGE_KEY=... BENCH_JUDGE_MODEL=... \+node scripts/bench/competency.mjs --kb <kb-id> --seed scripts/bench/truth/redocred-typed.questions.json+```++- 有期望答案的问题交给裁判模型判"回答里说了期望答案没有";没有的只查它需要的形状+ (`needs` 里的类与属性)是否都在、属性是否有类型化事实。后者弱得多——一条开放图谱就能答的问题+ 不需要本体,这是 0061 的开放问题。+- 第二个数从 `ontology_proposals` 算:代理提的里人表过态的,有几条被拒或改过再采纳。+- `--seed` 把写好的问题灌进库(同一句跳过),`--only-report` 只读数不问。+- `truth/redocred-typed.questions.json` 是给 `typed.mjs` 那份 100 篇 Re-DocRED 库写的十一条:十条有期望答案+ (其中两条要代理采纳过的 `bordered_by`、`p40` 才答得上),一条只有形状。换一个库要另写。+## 读数怎么算- `prompt_tokens_est` 是**本体段**的估算,不是整个提示词。实测 4.0 字符 ≈ 1 tokendiff --git a/scripts/bench/competency.mjs b/scripts/bench/competency.mjsnew file mode 100644index 00000000..447ece06--- /dev/null+++ b/scripts/bench/competency.mjs@@ -0,0 +1,105 @@+#!/usr/bin/env node+// competency bench(ADR 0061 决定 5):把一个库接受了的能力问题按人在 chat 里问的方式问一遍,+// 判答没答上,把结果写回问题(last_result),再读服务端算的两个数。+//+// BENCH_BASE=http://127.0.0.1:1524 BENCH_PSQL="docker exec ... -d utopia_bench4 -tAc" \+// BENCH_JUDGE_BASE=... BENCH_JUDGE_KEY=... BENCH_JUDGE_MODEL=... \+// node scripts/bench/competency.mjs --kb <kb-id> [--seed truth/redocred-typed.questions.json] [--rejudge] [--only-report]+//+// --rejudge:不再问 chat,拿上次存下的回答(last_result.answer)重判。问一轮几分钟、还有模型的账,+// 而判分口径会改(同 ask.mjs)。+// 有期望答案的问题交给裁判模型判;没有的只查它需要的形状(needs 里的类与属性)是否存在并且有事实。+// 服务端不自己问:chat 还没有进程内入口,而"按人问的方式问"正是走接口的意思。+import fs from "node:fs";+import { api, login, askChat, parseArgs, log, psql } from "./lib.mjs";++const args = parseArgs(process.argv);+const KB = args.kb;+if (!KB) { console.error("--kb <knowledge base id> 是必须的"); process.exit(2); }++function judgeEndpoint() {+ if (process.env.BENCH_JUDGE_BASE) return { base: process.env.BENCH_JUDGE_BASE, key: process.env.BENCH_JUDGE_KEY || "", model: process.env.BENCH_JUDGE_MODEL || "" };+ const [base, key, model] = psql(`SELECT s.chat_base_url, s.chat_api_key, s.chat_model FROM llm_settings s JOIN knowledge_bases k ON k.workspace_id=s.workspace_id WHERE k.id='${KB}'`).split("|");+ if (!base || !model) throw new Error("工作区没配对话模型,也没给 BENCH_JUDGE_*");+ if (key.startsWith("enc:")) throw new Error("库里的 chat_api_key 是封印过的密文,裁判读不了它:给 BENCH_JUDGE_BASE / _KEY / _MODEL");+ return { base, key, model };+}+async function judgeChat(ep, messages) {+ const r = await fetch(`${ep.base.replace(/\/$/, "")}/chat/completions`, {+ method: "POST", headers: { "content-type": "application/json", ...(ep.key ? { authorization: `Bearer ${ep.key}` } : {}) },+ body: JSON.stringify({ model: ep.model, temperature: 0, messages }),+ });+ if (!r.ok) throw new Error(`judge -> ${r.status} ${(await r.text()).slice(0, 200)}`);+ const j = await r.json();+ return j.choices?.[0]?.message?.content ?? "";+}+const JUDGE = `You grade an answer a knowledge base gave to a question, against the expected answer a person wrote.+Judge only whether the answer states the expected answer (the same entities, values or dates; wording and extra correct detail do not matter). An answer that says the base does not know, or names something else, is wrong.+Output one JSON object: {"correct": true|false, "why": "one short sentence"}`;++async function main() {+ await login();+ // --seed:把写好的问题灌进库(已有同一句的跳过),status = accepted+ if (args.seed) {+ const have = await api("GET", `/api/v1/kbs/${KB}/questions`);+ const texts = new Set(have.map((q) => q.question.trim().toLowerCase()));+ let n = 0;+ for (const q of JSON.parse(fs.readFileSync(args.seed, "utf8"))) {+ if (texts.has(q.question.trim().toLowerCase())) continue;+ await api("POST", `/api/v1/kbs/${KB}/questions`, { question: q.question, expected_answer: q.expected_answer ?? null, needs: q.needs ?? null });+ n++;+ }+ log(`seeded ${n} questions`);+ }+ const questions = (await api("GET", `/api/v1/kbs/${KB}/questions`)).filter((q) => q.status === "accepted");+ if (!args["only-report"]) {+ const ontology = await api("GET", `/api/v1/kbs/${KB}/ontology`);+ const classKeys = new Set(ontology.entity_types.map((c) => c.key));+ const propByKey = new Map(ontology.relation_types.map((r) => [r.key, r]));+ const ep = judgeEndpoint();+ for (const q of questions) {+ const t0 = Date.now();+ let answered = false, judged_by, detail = {}, answer = "";+ try {+ if (args.rejudge) {+ answer = q.last_result?.answer ?? "";+ if (!answer) throw new Error("no stored answer to rejudge");+ } else {+ const r = await askChat(KB, q.question);+ answer = r.text;+ if (r.error) detail.error = String(r.error).slice(0, 300);+ }+ if (q.expected_answer) {+ judged_by = "expected";+ const verdict = await judgeChat(ep, [+ { role: "system", content: JUDGE },+ { role: "user", content: `Question: ${q.question}\nExpected answer: ${q.expected_answer}\nAnswer given:\n${answer.slice(0, 4000)}` },+ ]);+ const m = /\{[\s\S]*\}/.exec(verdict);+ const j = m ? JSON.parse(m[0]) : { correct: false, why: "judge gave no JSON" };+ answered = !!j.correct; detail.why = j.why;+ } else {+ // 没有期望答案:查它需要的形状——类都在、属性都在并且各有类型化事实+ judged_by = "shape";+ const needs = q.needs ?? {};+ const missingClasses = (needs.classes ?? []).filter((k) => !classKeys.has(k));+ const missingProps = (needs.properties ?? []).filter((k) => !propByKey.has(k));+ const emptyProps = (needs.properties ?? []).filter((k) => propByKey.has(k) && !(propByKey.get(k).usage > 0));+ detail = { missing_classes: missingClasses, missing_properties: missingProps, empty_properties: emptyProps };+ answered = missingClasses.length === 0 && missingProps.length === 0 && emptyProps.length === 0+ && ((needs.classes ?? []).length + (needs.properties ?? []).length > 0);+ }+ } catch (e) {+ detail.error = String(e.message || e).slice(0, 300);+ }+ detail.seconds = Math.round((Date.now() - t0) / 1000);+ await api("POST", `/api/v1/kbs/${KB}/questions/${q.id}/result`, { answered, answer, judged_by, detail });+ log(`${answered ? "✓" : "✗"} [${judged_by}] ${q.question} — ${detail.why ?? JSON.stringify(detail)}`);+ }+ }+ const report = await api("GET", `/api/v1/kbs/${KB}/questions/report`);+ const qs = report.questions, ps = report.proposals;+ console.log(`questions: ${qs.answered}/${qs.checked} answered (${qs.accepted} accepted, ${qs.proposed} proposed)`);+ console.log(`proposals: ${ps.changed}/${ps.decided} changed before adoption or rejected` + (ps.changed_share != null ? ` (${(ps.changed_share * 100).toFixed(0)}%)` : "") + `, ${ps.open} open`);+}+main().catch((e) => { console.error(e); process.exit(1); });diff --git a/scripts/bench/lib.mjs b/scripts/bench/lib.mjsindex 2a26cc0c..2e44c321 100644--- a/scripts/bench/lib.mjs+++ b/scripts/bench/lib.mjs@@ -136,3 +136,39 @@ export async function until(fn, everyMs, stallMs) {await sleep(everyMs);}}++/// 一轮问答,按人在界面里问的方式走 chat 接口。SSE 帧是 `event: X\ndata: {...}\n\n`。+/// ask.mjs 里有一份同样的;competency.mjs 也要,所以搬到这里+export async function askChat(kb, message) {+ const res = await fetch(`${BASE}/api/v1/kbs/${kb}/chat`, {+ method: "POST",+ headers: { "content-type": "application/json", cookie: cookieHeader() },+ body: JSON.stringify({ message }),+ });+ if (!res.ok) throw new Error(`chat -> ${res.status} ${(await res.text()).slice(0, 200)}`);+ const reader = res.body.getReader();+ const dec = new TextDecoder();+ let buf = "", text = "", conversation = null, error = null;+ const steps = [];+ for (;;) {+ const { done, value: chunk } = await reader.read();+ if (done) break;+ buf += dec.decode(chunk, { stream: true });+ let i;+ while ((i = buf.indexOf("\n\n")) >= 0) {+ const frame = buf.slice(0, i);+ buf = buf.slice(i + 2);+ const ev = /^event: ?(.*)$/m.exec(frame)?.[1];+ const data = frame.split("\n").filter((l) => l.startsWith("data:")).map((l) => l.slice(5).replace(/^ /, "")).join("\n");+ try {+ if (ev === "conversation") conversation = JSON.parse(data).id;+ else if (ev === "delta") text += JSON.parse(data).text ?? "";+ else if (ev === "step") steps.push(JSON.parse(data));+ else if (ev === "error") error = data;+ } catch {+ /* 半帧或非 JSON:下一帧再说 */+ }+ }+ }+ return { conversation, text, steps, error };+}diff --git a/scripts/bench/truth/redocred-typed.questions.json b/scripts/bench/truth/redocred-typed.questions.jsonnew file mode 100644index 00000000..f9f411ef--- /dev/null+++ b/scripts/bench/truth/redocred-typed.questions.json@@ -0,0 +1,13 @@+[+ { "question": "Which districts border Chicualacuala District?", "expected_answer": "Massangena District and Chigubo District (also Mabalane District)", "needs": { "classes": ["location"], "properties": ["bordered_by"] } },+ { "question": "Who are the children of Pedro II?", "expected_answer": "Afonso and Isabel", "needs": { "classes": ["person"], "properties": ["p40"] } },+ { "question": "Who is the father of Abdullah I bin Sabah Al-Sabah?", "expected_answer": "Sabah bin Jaber", "needs": { "classes": ["person"], "properties": ["p40"] } },+ { "question": "Who directed The Umbrellas of Cherbourg?", "expected_answer": "Jacques Demy", "needs": { "properties": ["p57"] } },+ { "question": "Who wrote Natural Selection?", "expected_answer": "Bill Dare", "needs": { "properties": ["p50"] } },+ { "question": "Where is the Foundling Museum located?", "expected_answer": "Brunswick Square", "needs": { "properties": ["p131"] } },+ { "question": "When was Anna Karenina published?", "expected_answer": "1878", "needs": { "properties": ["p577"] } },+ { "question": "Which award did Andrew Alexander Cole receive?", "expected_answer": "PFA Young Player of the Year", "needs": { "properties": ["p166"] } },+ { "question": "Where did Allen Francis Moore die?", "expected_answer": "Monticello Cemetery", "needs": { "properties": ["p20"] } },+ { "question": "What position did Allen Francis Moore hold?", "expected_answer": "a seat in the Sixty-ninth Congress", "needs": { "properties": ["p39"] } },+ { "question": "Which organizations does each place house?", "expected_answer": null, "needs": { "classes": ["location", "organization"], "properties": ["houses"] } }+]diff --git a/web/src/api.ts b/web/src/api.tsindex d36e5267..ab33ad74 100644--- a/web/src/api.ts+++ b/web/src/api.ts@@ -1248,6 +1248,20 @@ export interface CompetencyQuestion {last_result: unknown | null;}+/** 本体的两个数(0061 决定 5) */+export interface QuestionReport {+ questions: { accepted: number; proposed: number; checked: number; answered: number };+ proposals: {+ open: number;+ adopted: number;+ adopted_edited: number;+ rejected: number;+ decided: number;+ changed: number;+ changed_share: number | null;+ };+}+export interface OntologyMiss {kind: "entity_type" | "relation_type";key: string;@@ -2271,6 +2285,15 @@ export const api = {request<{ ok: boolean }>(`/api/v1/kbs/${kbId}/questions/${qid}`, {method: "DELETE",}),+ /** 让代理给库提问题(0061 决定 1):排任务,结果是 proposed 的问题 */+ proposeQuestions: (kbId: string) =>+ request<{ queued: boolean }>(`/api/v1/kbs/${kbId}/questions/propose`, {+ method: "POST",+ body: "{}",+ }),+ /** 两个数(0061 决定 5) */+ questionReport: (kbId: string) =>+ request<QuestionReport>(`/api/v1/kbs/${kbId}/questions/report`),dismissMiss: (kbId: string, kind: string, key: string) =>request<{ ok: boolean }>(`/api/v1/kbs/${kbId}/ontology/misses/dismiss`, {method: "POST",diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex a33820b3..6db266e4 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -1717,6 +1717,21 @@ export const en = {deleteQuestion: "Delete",questionAdded: "Question added",questionRetired: "Question retired",+ askQuestions: "Ask the agent for questions",+ askQuestionsHint:+ "Proposes questions from what the documents say most. They wait here until you accept them.",+ questionsQueued: "The agent is writing questions — they appear here when it is done.",+ acceptQuestion: "Accept",+ questionAnswered: "answered",+ questionUnanswered: "not answered",+ questionsScore: (answered: number, checked: number) =>+ `${answered} of ${checked} answered`,+ questionsScoreHint:+ "Accepted questions the competency bench asked over the graph, and how many it answered.",+ proposalsChanged: (changed: number, decided: number) =>+ `${changed} of ${decided} proposals changed`,+ proposalsChangedHint:+ "Of the agent's proposals people decided on, how many were rejected or edited before adoption.",/* ---- Schema diagram ---- */schemaDiagram: "Schema diagram",/* 从前这句把「先加个类或导入 OWL 文件」说成了开始的前提,而本体本来就diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex fa02e1c0..2061ecaf 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -1505,6 +1505,18 @@ export const zh: Strings = {deleteQuestion: "删除",questionAdded: "已添加问题",questionRetired: "问题已退役",+ askQuestions: "请代理提问题",+ askQuestionsHint: "从文档说得最多的东西里提问题,等你接受了才算数。",+ questionsQueued: "代理在写问题——写完了会出现在这里。",+ acceptQuestion: "接受",+ questionAnswered: "答上了",+ questionUnanswered: "没答上",+ questionsScore: (answered: number, checked: number) =>+ `${checked} 条问过,${answered} 条答上`,+ questionsScoreHint: "competency bench 对图谱问过的已接受问题,答上了几条。",+ proposalsChanged: (changed: number, decided: number) =>+ `${decided} 条提案表过态,${changed} 条被改或被拒`,+ proposalsChangedHint: "代理的提案里人表过态的,有几条被拒或改过再采纳。",/* ---- 模式图 ---- */schemaDiagram: "模式图",schemaEmpty: "还没有类。文档进来时会自动补上。",diff --git a/web/src/pages/Ontology.tsx b/web/src/pages/Ontology.tsxindex 9afa5673..47410fad 100644--- a/web/src/pages/Ontology.tsx+++ b/web/src/pages/Ontology.tsx@@ -1900,12 +1900,44 @@ function QuestionsPanel({onError: (e: unknown) => void;}) {const qc = useQueryClient();+ // 叫了代理之后等它提问题:隔几秒看一眼,多出 proposed 的就停+ const [awaiting, setAwaiting] = useState<{ since: number; had: number } | null>(null);const questions = useQuery({queryKey: ["questions", kbId],queryFn: () => api.listQuestions(kbId),+ refetchInterval: awaiting ? 4000 : false,});+ const report = useQuery({+ queryKey: ["questionReport", kbId],+ queryFn: () => api.questionReport(kbId),+ });+ useEffect(() => {+ if (!awaiting || !questions.data) return;+ const now = questions.data.filter((q) => q.status === "proposed").length;+ if (now > awaiting.had || Date.now() - awaiting.since > 120_000) setAwaiting(null);+ }, [awaiting, questions.data]);const [draft, setDraft] = useState("");- const refresh = () => qc.invalidateQueries({ queryKey: ["questions", kbId] });+ const refresh = () => {+ qc.invalidateQueries({ queryKey: ["questions", kbId] });+ qc.invalidateQueries({ queryKey: ["questionReport", kbId] });+ };+ const askForQuestions = useMutation({+ mutationFn: () => api.proposeQuestions(kbId),+ onSuccess: () => {+ toast.info(S.ontology.questionsQueued);+ setAwaiting({+ since: Date.now(),+ had: (questions.data ?? []).filter((q) => q.status === "proposed").length,+ });+ },+ onError,+ });+ const decide = useMutation({+ mutationFn: ({ q, status }: { q: CompetencyQuestion; status: "accepted" | "rejected" }) =>+ api.updateQuestion(kbId, q.id, { status }),+ onSuccess: refresh,+ onError,+ });const add = useMutation({mutationFn: (question: string) => api.createQuestion(kbId, { question }),onSuccess: () => {@@ -1936,7 +1968,35 @@ function QuestionsPanel({};return (<div className="mt-6 border-t border-line pt-3">- <h4 className="text-small font-semibold text-ink-2">{S.ontology.questions}</h4>+ <div className="flex items-center gap-2">+ <h4 className="text-small font-semibold text-ink-2">{S.ontology.questions}</h4>+ {report.data && report.data.questions.checked > 0 && (+ <Chip tone="info" title={S.ontology.questionsScoreHint}>+ {S.ontology.questionsScore(+ report.data.questions.answered,+ report.data.questions.checked,+ )}+ </Chip>+ )}+ {report.data && report.data.proposals.decided > 0 && (+ <Chip tone="neutral" title={S.ontology.proposalsChangedHint}>+ {S.ontology.proposalsChanged(+ report.data.proposals.changed,+ report.data.proposals.decided,+ )}+ </Chip>+ )}+ <Button+ size="sm"+ variant="secondary"+ className="ml-auto"+ title={S.ontology.askQuestionsHint}+ onClick={() => askForQuestions.mutate()}+ disabled={askForQuestions.isPending || awaiting !== null}+ >+ {awaiting ? S.ontology.agentWaiting : S.ontology.askQuestions}+ </Button>+ </div><p className="mb-2 text-small text-ink-2">{S.ontology.questionsHint}</p><div className="mb-2 flex items-center gap-2"><Input@@ -1976,6 +2036,37 @@ function QuestionsPanel({{q.origin === "agent" && (<Chip tone="success">{S.ontology.agentChip}</Chip>)}+ {q.last_result != null &&+ typeof q.last_result === "object" &&+ "answered" in q.last_result && (+ <Chip+ tone={(q.last_result as { answered?: boolean }).answered ? "success" : "warn"}+ >+ {(q.last_result as { answered?: boolean }).answered+ ? S.ontology.questionAnswered+ : S.ontology.questionUnanswered}+ </Chip>+ )}+ {q.status === "proposed" && (+ <>+ <Button+ size="sm"+ variant="primary"+ onClick={() => decide.mutate({ q, status: "accepted" })}+ disabled={decide.isPending}+ >+ {S.ontology.acceptQuestion}+ </Button>+ <Button+ size="sm"+ variant="secondary"+ onClick={() => decide.mutate({ q, status: "rejected" })}+ disabled={decide.isPending}+ >+ {S.ontology.rejectProposal}+ </Button>+ </>+ )}{q.status === "accepted" && (<Buttonsize="sm"