Skip to content

World's first open-source enterprise world model.

The agent's glossary is trimmed per batch the way alignment picks its candidates (#963)

WaylandYang071a37099964
diff --git a/crates/utopia-extract/src/ontology_agent.rs b/crates/utopia-extract/src/ontology_agent.rs
index 32888169..84d3b4f0 100644
--- a/crates/utopia-extract/src/ontology_agent.rs
+++ b/crates/utopia-extract/src/ontology_agent.rs
@@ -44,6 +44,9 @@ pub struct Glossary<'a> {
pub classes: Vec<(&'a str, &'a str, &'a str)>,
/// (key, label, kind, domains, ranges, description)
pub properties: Vec<GlossaryProperty<'a>>,
+ /// 这一批用不上、只报个名字的属性:(key, label)。模型要知道它们存在,才不会重提;
+ /// 定义只给结构对得上、或按向量最近的那些(同对齐的短名单,0044 决定 3 的成本教训)
+ pub others: Vec<(&'a str, &'a str)>,
}
/// 词表里的一条属性:(key, label, kind, domains, ranges, description)
@@ -168,6 +171,15 @@ pub fn build_proposal_messages(
}
user.push('\n');
}
+ if !glossary.others.is_empty() {
+ user.push_str(
+ "Other existing properties (key · label only; none of them fits these shapes by \
+ structure or meaning, but do not propose a key already here):\n",
+ );
+ for (key, label) in &glossary.others {
+ user.push_str(&format!("- {key} · {label}\n"));
+ }
+ }
user.push_str("\nCompetency questions:\n");
if questions.is_empty() {
user.push_str(
@@ -491,6 +503,7 @@ mod tests {
vec!["location"],
"where a thing is",
)],
+ others: vec![("p999", "somethingElse")],
};
let qs = vec![QuestionItem {
id: 1,
@@ -499,6 +512,8 @@ mod tests {
let m = build_proposal_messages(&sigs, &kws, &glossary, &qs);
assert_eq!(m.len(), 2);
let user = &m[1].content;
+ assert!(user.contains("Other existing properties"), "{user}");
+ assert!(user.contains("- p999 · somethingElse"), "{user}");
for needle in [
"s0: phrase \"supplies\"",
"k0: kind word \"supplier\"",
diff --git a/crates/utopia-extract/src/question_agent.rs b/crates/utopia-extract/src/question_agent.rs
index bac7dcb4..6b7221e8 100644
--- a/crates/utopia-extract/src/question_agent.rs
+++ b/crates/utopia-extract/src/question_agent.rs
@@ -219,6 +219,7 @@ mod tests {
vec!["location"],
"",
)],
+ others: vec![("p999", "somethingElse")],
};
let sigs = [TopSignature {
phrase: "died in",
diff --git a/crates/utopia-server/src/ontology_agent.rs b/crates/utopia-server/src/ontology_agent.rs
index 9b8da2b1..47fa5b1c 100644
--- a/crates/utopia-server/src/ontology_agent.rs
+++ b/crates/utopia-server/src/ontology_agent.rs
@@ -10,7 +10,7 @@ use std::collections::{HashMap, HashSet};
use serde::Deserialize;
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
-use utopia_core::models::RelationAxioms;
+use utopia_core::models::{RelationAxioms, RelationTypeView};
use utopia_core::AppError;
use utopia_extract::ontology_agent as agent;
use utopia_store::agent_reviews::{self, Review};
@@ -30,6 +30,9 @@ const TOP_SIGNATURES: usize = 40;
const TOP_KIND_WORDS: usize = 20;
const TOP_ENTITIES: usize = 20;
+/// 一条形状结构上对得上的属性多过这个数,就按向量只留最近的这么多带定义(同对齐的短名单)
+const GLOSSARY_PER_SHAPE: usize = 10;
+
/// 一次调用交给模型的形状数(0044 决定 3 的成本教训:词表每批只带一次)
pub(crate) const SIGNATURES_PER_CALL: usize = 12;
/// 每批跟着的类别词至多这么多
@@ -267,24 +270,87 @@ async fn propose_locked(
.filter_map(|id| class_key.get(id).copied())
.collect()
};
- let glossary = agent::Glossary {
- classes: classes
+ // 词表按批裁(第一次真跑:98 条属性带定义每次调用 4k token,占一次调用的一半多)。
+ // 每条形状只给结构对得上的属性定义;对得上的太多就按向量留最近的几条——判据和对齐的
+ // 候选、短名单是同一套。其余属性只报键和标签,让模型知道它们在
+ let closure =
+ crate::phrase_alignment::closures(classes.iter().map(|c| (c.id, c.parents.as_slice())));
+ let nearest: Vec<Vec<Uuid>> = {
+ let queries: Vec<String> = open
.iter()
- .map(|c| (c.key.as_str(), c.label.as_str(), c.description.as_str()))
- .collect(),
- properties: props
- .iter()
- .map(|p| {
- (
- p.key.as_str(),
- p.label.as_str(),
- p.kind.as_str(),
- keys_of(&p.domains),
- keys_of(&p.ranges),
- p.description.as_str(),
- )
+ .map(|s| match (s.examples.first(), s.quotes.first()) {
+ (Some(e), Some(q)) => format!("{} · {e} · {q}", s.phrase),
+ (Some(e), None) => format!("{} · {e}", s.phrase),
+ _ => s.phrase.clone(),
})
- .collect(),
+ .collect();
+ match crate::ontology_index::nearest_for_each(
+ state,
+ kb_id,
+ &queries,
+ GLOSSARY_PER_SHAPE as i64,
+ crate::ontology_index::Target::Predicate(None),
+ )
+ .await
+ {
+ Ok(v) => v
+ .into_iter()
+ .map(|cands| cands.into_iter().map(|c| c.id).collect())
+ .collect(),
+ Err(e) => {
+ tracing::warn!(%kb_id, error = %e, "本体代理:形状向量没算出来,词表按结构裁");
+ vec![Vec::new(); open.len()]
+ }
+ }
+ };
+ let glossary_for = |ids: std::ops::Range<usize>| -> agent::Glossary<'_> {
+ let mut full: HashSet<Uuid> = HashSet::new();
+ for i in ids {
+ let (Some(s), near) = (open.get(i), nearest.get(i)) else {
+ continue;
+ };
+ let fitting: Vec<&RelationTypeView> = props
+ .iter()
+ .filter(|p| crate::phrase_alignment::structurally_fits(p, s, &closure))
+ .collect();
+ if fitting.len() > GLOSSARY_PER_SHAPE {
+ if let Some(near) = near.filter(|n| !n.is_empty()) {
+ full.extend(
+ fitting
+ .iter()
+ .filter(|p| near.contains(&p.id))
+ .map(|p| p.id),
+ );
+ continue;
+ }
+ }
+ full.extend(fitting.iter().map(|p| p.id));
+ }
+ agent::Glossary {
+ classes: classes
+ .iter()
+ .map(|c| (c.key.as_str(), c.label.as_str(), c.description.as_str()))
+ .collect(),
+ properties: props
+ .iter()
+ .filter(|p| full.contains(&p.id))
+ .map(|p| {
+ (
+ p.key.as_str(),
+ p.label.as_str(),
+ p.kind.as_str(),
+ keys_of(&p.domains),
+ keys_of(&p.ranges),
+ p.description.as_str(),
+ )
+ })
+ .collect(),
+ others: props
+ .iter()
+ .filter(|p| !full.contains(&p.id))
+ .map(|p| (p.key.as_str(), p.label.as_str()))
+ .collect(),
+ }
};
let sig_items: Vec<agent::OpenSignature<'_>> = open
.iter()
@@ -329,6 +395,7 @@ async fn propose_locked(
.unwrap_or(&[]);
let sig_ids: HashSet<i64> = sig_batch.iter().map(|s| s.id).collect();
let word_ids: HashSet<i64> = word_batch.iter().map(|w| w.id).collect();
+ let glossary = glossary_for(b * SIGNATURES_PER_CALL..(b + 1) * SIGNATURES_PER_CALL);
let messages = agent::build_proposal_messages(sig_batch, word_batch, &glossary, &q_items);
let reply =
match chat_retrying_rate_limits_at(state, settings, client, &messages, Some(0.0)).await
@@ -1011,6 +1078,7 @@ async fn propose_questions_with(
)
})
.collect(),
+ others: Vec::new(),
};
// 说得最多的形状,带上对齐绑到的属性
let mut sigs = phrase_bindings::signatures(pool, kb_id).await?;
diff --git a/crates/utopia-server/src/phrase_alignment.rs b/crates/utopia-server/src/phrase_alignment.rs
index 1094ec79..15724298 100644
--- a/crates/utopia-server/src/phrase_alignment.rs
+++ b/crates/utopia-server/src/phrase_alignment.rs
@@ -35,9 +35,9 @@ type Vote = Option<(String, Direction)>;
/// 一个类连同它的全部祖先。候选按它命中:属性的定义域声明在 legal_entity 上,
/// organization 是它的子类,这条属性对 organization 的签名就是候选(#807 第一条)。
-type Closure = HashMap<Uuid, Vec<Uuid>>;
+pub(crate) type Closure = HashMap<Uuid, Vec<Uuid>>;
-fn closures<'a>(classes: impl IntoIterator<Item = (Uuid, &'a [Uuid])>) -> Closure {
+pub(crate) fn closures<'a>(classes: impl IntoIterator<Item = (Uuid, &'a [Uuid])>) -> Closure {
let classes: Vec<(Uuid, &[Uuid])> = classes.into_iter().collect();
let parents: HashMap<Uuid, &[Uuid]> = classes.iter().copied().collect();
classes
@@ -87,6 +87,15 @@ fn within(
.map(|d| Some((*d, c)))
}
+/// 结构上对得上(本体代理给每批裁词表时用的判据,同对齐的候选)
+pub(crate) fn structurally_fits(
+ p: &RelationTypeView,
+ sig: &PhraseSignature,
+ closure: &Closure,
+) -> bool {
+ fits(p, sig, closure).is_some()
+}
+
/// 签名两端的类落在属性声明的域/值域里,经继承也算;正反两个方向都算。
fn fits(p: &RelationTypeView, sig: &PhraseSignature, closure: &Closure) -> Option<Fit> {
let mut via = Vec::new();