World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
The governor decides three clusters at once (#461)
WaylandYangf16127032f75
diff --git a/crates/utopia-server/src/governance.rs b/crates/utopia-server/src/governance.rsindex e554b20f..f55d9362 100644--- a/crates/utopia-server/src/governance.rs+++ b/crates/utopia-server/src/governance.rs@@ -17,8 +17,9 @@use crate::llm_util;use crate::state::AppState;use chrono::{Duration, Utc};+use futures_util::future::join_all;use serde_json::{json, Value};-use std::collections::HashMap;+use std::collections::{HashMap, HashSet};use utopia_core::models::{LlmSettings, ReviewItem, Role};use utopia_core::AppError;use utopia_extract::governor::{self, Step};@@ -132,90 +133,196 @@ pub async fn govern(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> {Ok(())}+/// 一轮里并行裁几簇。只有互不共享实体、也不共享名字的簇才能同一轮:一簇里的合并+/// 会改另一簇看到的东西。模型那边的并发由 llm_util 的闸门限,这里只决定一次挂几个+const CLUSTERS_PER_ROUND: usize = 3;+/// 挑簇时从队头往后看多少对+const HEADS_TO_SCAN: i64 = 60;++/// 一簇:等人的对、各自的先例、给模型看的样子、已经有的看法(人定过的快路)+struct Cluster {+ items: Vec<ReviewItem>,+ precedents: Vec<Precedents>,+ pairs: Vec<utopia_extract::AdjudicationPair>,+ looks: Vec<Option<Look>>,+}+/// 一轮轮走到队列空、开关关或轮数用完。回 true = 轮数用完还有积压async fn rounds(ctx: &Ctx<'_>) -> anyhow::Result<bool> {let state = ctx.state;let kb_id = ctx.kb_id;- let client = ctx.client;+ let pool = &state.pool;for _ in 0..MAX_ROUNDS {- // 关闭后队列自动终止:每一簇之前看一眼- if !utopia_store::kbs::get(&state.pool, kb_id).await?.governance {+ // 关闭后队列自动终止:每一轮之前看一眼+ if !utopia_store::kbs::get(pool, kb_id).await?.governance {tracing::info!(%kb_id, "治理:开关已关,停在这里");return Ok(false);}- let Some(head) = gov::queue(&state.pool, kb_id, 1).await?.into_iter().next() else {++ // 从队头起挑互不重叠的簇,最多 K 簇;和已取的簇有交集的头留到下一轮+ let heads = gov::queue(pool, kb_id, HEADS_TO_SCAN).await?;+ if heads.is_empty() {return Ok(false);- };- let mut items = vec![head];- let siblings = gov::cluster_of(&state.pool, kb_id, &items[0], BATCH_SIZE - 1).await?;- items.extend(siblings);- // 这一簇归 agent 了:人在这几分钟里不能裁它们,界面上看得见- let ids: Vec<Uuid> = items.iter().map(|i| i.id).collect();- gov::lock(&state.pool, kb_id, &ids).await?;+ }+ let mut clusters: Vec<Vec<ReviewItem>> = Vec::new();+ let mut taken_ids: HashSet<Uuid> = HashSet::new();+ let mut taken_entities: HashSet<Uuid> = HashSet::new();+ let mut taken_names: HashSet<String> = HashSet::new();+ for head in heads {+ if clusters.len() >= CLUSTERS_PER_ROUND {+ break;+ }+ if taken_ids.contains(&head.id) {+ continue;+ }+ let mut items = vec![head];+ let siblings = gov::cluster_of(pool, kb_id, &items[0], BATCH_SIZE - 1).await?;+ items.extend(siblings);+ let overlaps = items.iter().any(|i| {+ taken_ids.contains(&i.id)+ || taken_entities.contains(&i.left.id)+ || taken_entities.contains(&i.right.id)+ || taken_names.contains(&i.left.name.to_lowercase())+ || taken_names.contains(&i.right.name.to_lowercase())+ });+ if overlaps {+ continue;+ }+ for i in &items {+ taken_ids.insert(i.id);+ taken_entities.insert(i.left.id);+ taken_entities.insert(i.right.id);+ taken_names.insert(i.left.name.to_lowercase());+ taken_names.insert(i.right.name.to_lowercase());+ }+ clusters.push(items);+ }+ // 这几簇归 agent 了:人在这几分钟里不能裁它们,界面上看得见+ let ids: Vec<Uuid> = taken_ids.into_iter().collect();+ gov::lock(pool, kb_id, &ids).await?;state.emit_review(kb_id);- let mut precedents = Vec::with_capacity(items.len());- for item in &items {- precedents.push(gov::precedents_for(&state.pool, kb_id, item).await?);- }- let pairs: Vec<utopia_extract::AdjudicationPair> = items- .iter()- .zip(&precedents)- .map(|(item, p)| pair_of(item, p))- .collect();-- // 人定过的直接照办,不问模型;剩下的才攒批- let mut asked: Vec<usize> = Vec::new();- for (idx, item) in items.iter().enumerate() {- let prior =- gov::decided_before(&state.pool, kb_id, item.left.id, item.right.id).await?;- match gov::settled_by_people(&item.left.name, &item.right.name, &precedents[idx], prior)- {- Some(merged) => {- settle(- ctx,- item,- &pairs[idx],+ // 先例与快路(库操作,顺序做)+ let mut round: Vec<Cluster> = Vec::with_capacity(clusters.len());+ for items in clusters {+ let mut precedents = Vec::with_capacity(items.len());+ for item in &items {+ precedents.push(gov::precedents_for(pool, kb_id, item).await?);+ }+ let pairs: Vec<utopia_extract::AdjudicationPair> = items+ .iter()+ .zip(&precedents)+ .map(|(item, p)| pair_of(item, p))+ .collect();+ let mut looks = Vec::with_capacity(items.len());+ for (idx, item) in items.iter().enumerate() {+ let prior = gov::decided_before(pool, kb_id, item.left.id, item.right.id).await?;+ looks.push(+ gov::settled_by_people(+ &item.left.name,+ &item.right.name,&precedents[idx],- Look::from_people(merged),+ prior,)- .await?+ .map(Look::from_people),+ );+ }+ round.push(Cluster {+ items,+ precedents,+ pairs,+ looks,+ });+ }++ // 攒批:每簇一次调用,几簇同时挂。任何一次失败整轮失败,任务按退避重试;+ // 重试耗尽后这些对留在队列里,人照样能裁+ let batches = join_all(round.iter().map(|c| batch_looks(ctx, c))).await;+ for (c, looks) in round.iter_mut().zip(batches) {+ for (idx, look) in looks? {+ c.looks[idx] = Some(look);+ }+ }++ // 第二层:判不定的、同名却说不同的,带工具再看一遍——几对同时看,+ // 模型那边由闸门限并发+ let mut second: Vec<(usize, usize)> = Vec::new();+ for (ci, c) in round.iter().enumerate() {+ for (idx, look) in c.looks.iter().enumerate() {+ if let Some(look) = look {+ if wants_second_look(&c.items[idx], &c.precedents[idx], look) {+ second.push((ci, idx));+ }}- None => asked.push(idx),}}- if !asked.is_empty() {- let batch: Vec<utopia_extract::AdjudicationPair> =- asked.iter().map(|&i| pairs[i].clone()).collect();- let messages = utopia_extract::build_adjudication_messages(&batch);- let reply = {- let _permit = permit(ctx).await;- // 调用/解析失败 → 任务按退避重试;重试耗尽后这些对留在队列里,人照样能裁- client.chat(&messages).await?- };- let verdicts = utopia_extract::parse_adjudication(&reply)?;- let by_i: HashMap<usize, &utopia_extract::AdjudicationVerdict> =- verdicts.iter().map(|v| (v.i, v)).collect();-- for (n, &idx) in asked.iter().enumerate() {- let look = match by_i.get(&n) {- Some(v) => Look::from_batch(- match v.verdict.as_str() {- "same" => Some(true),- "different" => Some(false),- _ => None,- },- v.confidence.unwrap_or(0.5).clamp(0.0, 1.0),- v.why.clone(),- ),- None => Look::from_batch(None, 0.0, None),- };- settle(ctx, &items[idx], &pairs[idx], &precedents[idx], look).await?;+ let seen = join_all(second.iter().map(|&(ci, idx)| {+ let c = &round[ci];+ second_look(+ ctx,+ &c.items[idx],+ &c.pairs[idx],+ c.looks[idx].as_ref().expect("a look"),+ )+ }))+ .await;+ for (&(ci, idx), look) in second.iter().zip(seen) {+ if let Some(look) = look {+ round[ci].looks[idx] = Some(look);+ }+ }++ // 落地:按簇、按对顺序写库——连锁合并要按顺序+ for c in round.iter_mut() {+ for idx in 0..c.items.len() {+ let look = c.looks[idx]+ .take()+ .unwrap_or_else(|| Look::from_batch(None, 0.0, None));+ apply(ctx, &c.items[idx], &c.precedents[idx], look).await?;}}state.emit_review(kb_id);}- Ok(!gov::queue(&state.pool, kb_id, 1).await?.is_empty())+ Ok(!gov::queue(pool, kb_id, 1).await?.is_empty())+}++/// 一簇的攒批调用:人定过的不问,其余一次问完;回每对的看法+async fn batch_looks(ctx: &Ctx<'_>, c: &Cluster) -> anyhow::Result<Vec<(usize, Look)>> {+ let asked: Vec<usize> = (0..c.items.len())+ .filter(|&i| c.looks[i].is_none())+ .collect();+ if asked.is_empty() {+ return Ok(Vec::new());+ }+ let batch: Vec<utopia_extract::AdjudicationPair> =+ asked.iter().map(|&i| c.pairs[i].clone()).collect();+ let messages = utopia_extract::build_adjudication_messages(&batch);+ let reply = {+ let _permit = permit(ctx).await;+ ctx.client.chat(&messages).await?+ };+ let verdicts = utopia_extract::parse_adjudication(&reply)?;+ let by_i: HashMap<usize, &utopia_extract::AdjudicationVerdict> =+ verdicts.iter().map(|v| (v.i, v)).collect();+ Ok(asked+ .iter()+ .enumerate()+ .map(|(n, &idx)| {+ let look = match by_i.get(&n) {+ Some(v) => Look::from_batch(+ match v.verdict.as_str() {+ "same" => Some(true),+ "different" => Some(false),+ _ => None,+ },+ v.confidence.unwrap_or(0.5).clamp(0.0, 1.0),+ v.why.clone(),+ ),+ None => Look::from_batch(None, 0.0, None),+ };+ (idx, look)+ })+ .collect())}async fn permit(ctx: &Ctx<'_>) -> Option<tokio::sync::OwnedSemaphorePermit> {@@ -260,48 +367,63 @@ fn pair_of(item: &ReviewItem, p: &Precedents) -> utopia_extract::AdjudicationPai}}-/// 一对的判决落地:第一层判不定的先进第二层再看;过闸就动手并记 applied,-/// 否则转人工并记 proposed(带着问题与轨迹)-async fn settle(- ctx: &Ctx<'_>,- item: &ReviewItem,- pair: &utopia_extract::AdjudicationPair,- p: &Precedents,- mut look: Look,-) -> anyhow::Result<()> {- let pool = &ctx.state.pool;- let kb_id = ctx.kb_id;+/// 第一层没定(没判决、置信度不到线),或者同名、大类不冲突、模型却说不同——那是它+/// 最爱错的一种:都让第二层带着全部事实与原文再看一遍。硬规则拦下的不进:再看也不会改规则+fn wants_second_look(item: &ReviewItem, p: &Precedents, look: &Look) -> bool {let types_conflict = gov::types_conflict(item.left.type_label.as_deref(),item.right.type_label.as_deref(),);let shape = gov::name_shape(&item.left.name, &item.right.name);-- // 第二层:只接判不定的,硬规则拦下的不进。预算用完了照第一刀写建议- // 同名、大类不冲突、模型却说不同:这是它最爱错的一种,先别采纳,让第二层带着- // 全部事实与原文再看一遍let doubted_split = shape == gov::NameShape::Identical&& !types_conflict&& look.same == Some(false)&& look.calls == 0;- if (gov::gate(look.same, look.conf, types_conflict, shape, p) == Gate::Propose- && look.uncertain()+ ((gov::gate(look.same, look.conf, types_conflict, shape, p) == Gate::Propose+ && look.uncertain())|| doubted_split)&& p.reverts.is_empty()- {- let spent = gov::loop_calls_today(pool, kb_id).await?;- if spent < LOOP_DAILY_CALLS {- match investigate(ctx, item, pair, &look).await {- Ok(l) => look = l,- // 第二层失败不拖累第一层:这一对照第一层的看法写成建议- Err(e) => {- tracing::warn!(%kb_id, review = %item.id, error = %e, "治理:第二层没跑完")- }- }- } else {- tracing::info!(%kb_id, spent, "治理:今天的循环预算用完,判不定的只写建议");+}++/// 第二层看一对:预算够就看,看完的看法替掉第一层的;看不成(预算用完、模型出错)回 None,+/// 照第一层的看法办+async fn second_look(+ ctx: &Ctx<'_>,+ item: &ReviewItem,+ pair: &utopia_extract::AdjudicationPair,+ look: &Look,+) -> Option<Look> {+ let kb_id = ctx.kb_id;+ let spent = match gov::loop_calls_today(&ctx.state.pool, kb_id).await {+ Ok(n) => n,+ Err(e) => {+ tracing::warn!(%kb_id, error = %e, "治理:数循环预算失败");+ return None;+ }+ };+ if spent >= LOOP_DAILY_CALLS {+ tracing::info!(%kb_id, spent, "治理:今天的循环预算用完,判不定的只写建议");+ return None;+ }+ match investigate(ctx, item, pair, look).await {+ Ok(l) => Some(l),+ // 第二层失败不拖累第一层:这一对照第一层的看法写成建议+ Err(e) => {+ tracing::warn!(%kb_id, review = %item.id, error = %e, "治理:第二层没跑完");+ None}}+}++/// 一对的判决落地:过闸就动手并记 applied,否则转人工并记 proposed(带着问题与轨迹)+async fn apply(ctx: &Ctx<'_>, item: &ReviewItem, p: &Precedents, look: Look) -> anyhow::Result<()> {+ let pool = &ctx.state.pool;+ let kb_id = ctx.kb_id;+ let types_conflict = gov::types_conflict(+ item.left.type_label.as_deref(),+ item.right.type_label.as_deref(),+ );+ let shape = gov::name_shape(&item.left.name, &item.right.name);let action = look.action();let precedents = gov::precedents_json(p);diff --git a/crates/utopia-store/src/governance.rs b/crates/utopia-store/src/governance.rsindex ca35f98f..d44ea7ef 100644--- a/crates/utopia-store/src/governance.rs+++ b/crates/utopia-store/src/governance.rs@@ -319,10 +319,17 @@ pub fn name_shape(a: &str, b: &str) -> NameShape {.chars().last().is_some_and(|c| c.is_alphanumeric());- let after_ok = long[i + short.len()..]- .chars()- .next()- .is_none_or(|c| !c.is_alphabetic());+ // 后面跟着字母不算整词——除非只多一个复数的 s:sorting algorithms 含着+ // sorting algorithm,Times 含着 Time+ let rest = &long[i + short.len()..];+ let mut rest_chars = rest.chars();+ let after_ok = match (rest_chars.next(), rest_chars.next()) {+ (None, _) => true,+ (Some(c), _) if !c.is_alphabetic() => true,+ (Some('s'), None) => true,+ (Some('s'), Some(d)) => !d.is_alphanumeric(),+ _ => false,+ };if before_ok && after_ok {return Some(i);}@@ -366,7 +373,21 @@ pub fn name_shape(a: &str, b: &str) -> NameShape {if long.matches(", ").count() >= 2 {return NameShape::Phrase;}- if pos + short.len() == long.len() {+ // 短的那个在长的里到哪里结束:复数的 s 算在里面+ let end = {+ let rest = &long[pos + short.len()..];+ if rest.starts_with('s')+ && rest[1..]+ .chars()+ .next()+ .is_none_or(|c| !c.is_alphanumeric())+ {+ pos + short.len() + 1+ } else {+ pos + short.len()+ }+ };+ if end == long.len() {let front = long[..pos].trim().trim_end_matches([',', '-', ':']);if front.chars().next().is_some_and(|c| c.is_ascii_digit()) {return NameShape::Version;@@ -377,7 +398,7 @@ pub fn name_shape(a: &str, b: &str) -> NameShape {return NameShape::Abbreviation;}if pos == 0 {- let tail = long[short.len()..]+ let tail = long[end..].trim().trim_start_matches([',', '-', ':', '.', 'v']);let tail = tail.trim();@@ -1307,7 +1328,15 @@ mod tests {name_shape("MuZero, AlphaStar, AlphaGeometry", "MuZero"),Phrase);- assert_eq!(name_shape("Time", "Financial Times"), Unrelated);+ assert_eq!(name_shape("Time", "Financial Times"), Abbreviation);+ assert_eq!(+ name_shape(+ "C++ Standard Library sorting algorithms",+ "sorting algorithm"+ ),+ Phrase+ );+ assert_eq!(name_shape("Timeline", "Time"), Unrelated);assert_eq!(name_shape("Altimeter", "Altimeter Capital"), Suffix);}diff --git a/docs/decisions/0025-governance-reads-the-ledger-before-it-decides.md b/docs/decisions/0025-governance-reads-the-ledger-before-it-decides.mdindex 047dd86f..8a87b30a 100644--- a/docs/decisions/0025-governance-reads-the-ledger-before-it-decides.md+++ b/docs/decisions/0025-governance-reads-the-ledger-before-it-decides.md@@ -1,6 +1,6 @@# 0025 · Governance reads the ledger before it decides-- **Status**: decision 10 added 2026-09-07 (identity rules, name shapes, type families, the labeled set and `govern.mjs`) · decision 4 revised 2026-09-06 (the agent's own confidence decides, history moves the bar or blocks) · cut 1 implemented (#434) · migration 0035 adds `knowledge_bases.governance` and `agent_decisions`; `governance` in the store holds the precedent families, the first-in-first-out queue with its clusters, the gate and the table; the `govern` job in the server reads the switch, calls the model with precedents and applies or proposes; `?queue=agent`, `ReviewCounts.agent` and `POST /kbs/{id}/review/agent/{decision_id}` on the API · cut 3 (UI, #437) implemented: the switch in base settings, the Agent queue with its rows and answers, the proposal chip on a duplicate card whose Merge / Keep answers the proposal, the Agent section on the Overview · cut 2 implemented: migration 0036 adds `question`, `trace` and `calls` to `agent_decisions`; `governor` in the extract crate holds the tools, the opening and the step reader; `investigate` in the server job runs the loop for pairs the batch could not settle (decision 8) · cut 4 implemented: migration 0037 adds `knowledge_bases.governance_since`; `fuse` in the server job turns the switch off after two reverts of the agent's merges since it was turned on, within seven days, raises `governance.tripped` and writes the ledger (decision 9)+- **Status**: decision 3 revised 2026-09-07 (three clusters a round, #458) · decision 10 added 2026-09-07 (identity rules, name shapes, type families, the labeled set and `govern.mjs`) · decision 4 revised 2026-09-06 (the agent's own confidence decides, history moves the bar or blocks) · cut 1 implemented (#434) · migration 0035 adds `knowledge_bases.governance` and `agent_decisions`; `governance` in the store holds the precedent families, the first-in-first-out queue with its clusters, the gate and the table; the `govern` job in the server reads the switch, calls the model with precedents and applies or proposes; `?queue=agent`, `ReviewCounts.agent` and `POST /kbs/{id}/review/agent/{decision_id}` on the API · cut 3 (UI, #437) implemented: the switch in base settings, the Agent queue with its rows and answers, the proposal chip on a duplicate card whose Merge / Keep answers the proposal, the Agent section on the Overview · cut 2 implemented: migration 0036 adds `question`, `trace` and `calls` to `agent_decisions`; `governor` in the extract crate holds the tools, the opening and the step reader; `investigate` in the server job runs the loop for pairs the batch could not settle (decision 8) · cut 4 implemented: migration 0037 adds `knowledge_bases.governance_since`; `fuse` in the server job turns the switch off after two reverts of the agent's merges since it was turned on, within seven days, raises `governance.tripped` and writes the ledger (decision 9)- **Written**: 2026-09-06 (conventions in the [README](README.md))- **Related**: [0016](0016-close-the-open-seams-before-cutting-new-ones.md) C2 gave a base its first automation switch, `auto_type_resolution`, and this record copies its shape; #428 asked for bulk and automatic handling of same-name pairs and got the batch path (#429, #430) this builds on; [0020](0020-an-auditor-reads-it-without-us.md) made the ledger complete enough to be read back; [0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) keeps a person's own sentences out of any machine's reach.@@ -28,6 +28,8 @@ Two decisions folded into this one. The record first had three states, off / sug### 3. First in, first out, and the head brings its cluster+> **Revised 2026-09-07 (#458).** A round now takes up to three clusters instead of one, chosen from the head of the queue outward and only where they share no entity and no name, since a merge in one cluster changes what the other sees. Their batch calls go out together, the uncertain pairs of the round get their second look together, and the results are applied in order, cluster by cluster, because chained merges must stay sequential. On the labeled base this took a run from 25 minutes to 13 with the same score within the model's run-to-run noise (97.7% decided on its own, 96.7% agreed, 6 wrong merges of which 4 are one pair the model flips on, 12 left for people; 16 pairs and 28 calls in the loop). The model's concurrency limit is the ceiling, not the code: the bench account allows three calls at once. The text below describes the one-cluster round it replaces.+The queue is every pending pair without an open proposal, oldest first. The head's cluster is every other queued pair that shares a name or an entity with it, up to twelve in all, and the whole cluster goes into one model call. Oldest first because the oldest has waited longest and because the order is one a person can predict; the cluster because a name should be judged consistently in one sitting, not once per pair across separate calls, and because the cluster is the unit #428 asked about: three "Zhang Wei" pairs a person kept apart and nine more that should follow.### 4. Merging needs precedent, keeping apart needs confidence, hard rules come first@@ -75,6 +77,7 @@ The same 589 pairs, reset and re-run after each change:| acronyms, surnames, two-word tails | 98.5% | 96.3% | 6 | 12 | 8 | 22 min · 12 || one label for identical names, second look on a split | 98.2% | 97.3% | 4 | 9 | 9 | 22 min · 15 || one label only within one family (the version shipped) | 97.9% | 96.9% | 4 | 11 | 11 | 25 min · 46 |+| the same, three clusters a round (#458) | 97.7% | 96.7% | 6 | 10 | 12 | 13 min · 28 |Wall clock is for 589 pairs with the model capped at three concurrent calls (the endpoint rate-limits above that), so about twenty-five pairs a minute; each run makes one batch call per cluster of up to twelve pairs, roughly sixty, plus the loop calls shown. The loop budget of 2000 calls a day was never approached.@@ -101,5 +104,5 @@ Duplicates that the agent proposed on carry the reason code `proposed`; pairs it- **Other queues.** Conflicts have three human actions to learn from (`conflict.close_old`, `keep_both`, `reject_new`); unconfirmed and low-confidence facts have `fact.confirm` and `fact.reject`. Nods (`pending_facts`, 0015) stay out: the person who said it nods.- **Cross-base precedent.** A workspace's other bases may hold the same names. Kept to one base until someone asks.-- **A budget for the batch.** The loop has a daily budget (decision 8); the batch does not. A job runs twenty clusters and re-enqueues itself while backlog remains, which is what the adjudicator always did. Whether a base needs a cap on that too is a question for a large corpus.+- **A budget for the batch.** The loop has a daily budget (decision 8); the batch does not, and since #458 a round makes up to three batch calls at once. A job runs twenty clusters and re-enqueues itself while backlog remains, which is what the adjudicator always did. Whether a base needs a cap on that too is a question for a large corpus.- **Tools the loop does not have yet.** The graph around a side beyond its direct facts, the entity's disambiguator history, a full-text search of the corpus. Add one when a deferred question keeps asking for it.