Skip to content

World's first open-source enterprise world model.

Proposals can now say the ontology already has one (#75)

WaylandYang0e55ba073db4
diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs
index 35b35f28..ed274de2 100644
--- a/crates/utopia-core/src/models.rs
+++ b/crates/utopia-core/src/models.rs
@@ -669,3 +669,18 @@ pub struct DataSourceView {
pub last_test_at: Option<DateTime<Utc>>,
pub last_test_ok: Option<bool>,
}
+
+/// 向量检索出来的一个候选本体行(类 / 关系 / 属性)。
+///
+/// `distance` 是余弦距离,越小越近。原样带给调用方而不是先折成"相似度":
+/// 阈值该定在哪由消费者按自己的数据定,这里不替它归一化。
+#[derive(Debug, Clone, Serialize)]
+pub struct TypeCandidate {
+ pub id: Uuid,
+ pub key: String,
+ pub label: String,
+ pub description: String,
+ /// 关系行才有:`relation` 或 `attribute`
+ pub kind: Option<String>,
+ pub distance: f32,
+}
diff --git a/crates/utopia-server/src/api/ontology_routes.rs b/crates/utopia-server/src/api/ontology_routes.rs
index df2e5d64..a8ac107d 100644
--- a/crates/utopia-server/src/api/ontology_routes.rs
+++ b/crates/utopia-server/src/api/ontology_routes.rs
@@ -355,6 +355,12 @@ pub async fn suggest(
/// 生成本体扩展提案。人工点 Suggest 与冷启动自动扩本体走同一条路——
/// 自动那条不该是另一套判断,只是少了点头那一步。
+/// 每个说法检索几个候选。
+///
+/// 小是刻意的:候选是给模型判断"是不是已经有了"用的,不是让它挑一个最像的凑合。
+/// 开大了只会让它在一堆勉强相关的条目里硬选一个映射过去。
+const CANDIDATES_PER_PROBE: i64 = 5;
+
pub async fn build_proposals(
state: &AppState,
kb_id: Uuid,
@@ -367,8 +373,6 @@ pub async fn build_proposals(
let client = llm_util::chat_client(&settings)
.ok_or_else(|| AppError::invalid("no_chat_model", "Chat model not configured"))?;
- let entity_types = utopia_store::ontology::entity_type_views(&state.pool, kb_id).await?;
- let relation_types = utopia_store::ontology::relation_type_views(&state.pool, kb_id).await?;
// 只取这一遍**产得出提案**的那两类。attribute_type 的信号(未知谓词带字面值)
// 也在这张表里,但下面的 JSON 骨架只有 entity_types / relation_types——
// 把属性信号喂进去,模型只会照着建一个关系,而那正是它不该是的东西。
@@ -384,8 +388,95 @@ pub async fn build_proposals(
return Ok(json!({ "entity_types": [], "relation_types": [] }));
}
- let current_et: Vec<String> = entity_types.iter().map(|t| t.key.clone()).collect();
- let current_rt: Vec<String> = relation_types.iter().map(|r| r.key.clone()).collect();
+ // **本体的相关切片,不是它的全文。**
+ //
+ // 从前这里内联两串全量 key。两个毛病:一份 965 类的本体就是 1949 个 key
+ // 的提示词;而且只有 key 没有描述,模型据此判断不了"这个说法是不是某个
+ // 已有类型的同义",于是它只会新建,本体里就稳定长出重复。
+ //
+ // 现在按每个说法各检索几个最近的候选,带着描述给它看。提示词大小从此
+ // 与本体规模无关,而"已经有一个了"这件事第一次变得可判断。
+ let _ = crate::ontology_index::refresh(state, kb_id).await;
+ // 谓词那一路:表层说法 + 关系类的 miss。样例给向量更多着落——
+ // 光一个 acquires 太短,带上"星云科技 → 深蓝存储"就有了语境
+ let pred_probes: Vec<String> = forms
+ .iter()
+ .map(|f| match f.example.as_deref() {
+ Some(ex) if !ex.is_empty() => format!("{} — {ex}", f.form),
+ _ => f.form.clone(),
+ })
+ .chain(
+ misses
+ .iter()
+ .filter(|m| m.kind == "relation_type")
+ .map(|m| m.key.clone()),
+ )
+ .collect();
+ // 类那一路。**必须分开检索**:两张表的描述写的是完全不同的东西,
+ // 拿一个词表外的类名去关系里找,回来的一定是勉强相关的关系
+ let class_probes: Vec<String> = misses
+ .iter()
+ .filter(|m| m.kind == "entity_type")
+ .map(|m| m.key.clone())
+ .collect();
+ let per_pred = crate::ontology_index::nearest_for_each(
+ state,
+ kb_id,
+ &pred_probes,
+ CANDIDATES_PER_PROBE,
+ crate::ontology_index::Target::Predicate(None),
+ )
+ .await
+ .unwrap_or_default();
+ let per_class = crate::ontology_index::nearest_for_each(
+ state,
+ kb_id,
+ &class_probes,
+ CANDIDATES_PER_PROBE,
+ crate::ontology_index::Target::Class,
+ )
+ .await
+ .unwrap_or_default();
+ // 并集去重:几个说法常常指向同一个候选,逐个说法各列一遍是白费令牌
+ let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
+ let mut candidate_lines: Vec<String> = Vec::new();
+ // 模型抄回来的 key 要能对回本体:候选表同时按 key、归一化 key、标签建索引
+ let mut by_name: std::collections::HashMap<String, String> = std::collections::HashMap::new();
+ for c in per_pred.iter().chain(per_class.iter()).flatten() {
+ if !seen.insert(c.key.clone()) {
+ continue;
+ }
+ by_name.insert(normalize_name(&c.key), c.key.clone());
+ by_name.insert(normalize_name(&c.label), c.key.clone());
+ // 关系行自带 relation / attribute,类行没有 kind
+ let kind = c.kind.as_deref().unwrap_or("entity type");
+ let d = c.description.trim();
+ // **标签只在它确实多说了点什么的时候才写。**
+ // 导进来的本体里 label 常常只是 key 的驼峰写法(acquired_from /
+ // acquiredFrom),两个几乎一样的名字并排摆着,模型抄走的就是错的那个。
+ // 中文标签配英文 key 那种才是标签有信息量的情形,那时才留
+ let name = if normalize_name(&c.label) == normalize_name(&c.key) {
+ String::new()
+ } else {
+ format!(" ({})", c.label)
+ };
+ candidate_lines.push(if d.is_empty() {
+ format!("- {} [{kind}]{name}", c.key)
+ } else {
+ format!("- {} [{kind}]{name}: {d}", c.key)
+ });
+ }
+ // 一个候选都没有(没配嵌入模型,或本体是空的)时如实说,别让模型
+ // 以为"本体里什么都没有"从而放开手建
+ let candidates_block = if candidate_lines.is_empty() {
+ "(no candidates retrieved — the ontology may be empty, or embeddings are unavailable)"
+ .to_string()
+ } else {
+ candidate_lines.join(
+ "
+",
+ )
+ };
let miss_lines: Vec<String> = misses
.iter()
.map(|m| {
@@ -414,16 +505,27 @@ pub async fn build_proposals(
.collect();
let prompt = format!(
- "You are an ontology engineer. A knowledge graph has this ontology:\n\
- Entity type keys: {}\n\
- Relation type keys: {}\n\
+ "You are an ontology engineer.\n\
+ \n\
+ Below are the ontology entries closest in meaning to the unmatched wordings that \
+ follow. This is a retrieved slice, not the whole ontology, so \"not in this list\" \
+ does not mean \"not in the ontology\" — it means nothing close to it was found:\n{}\n\
\n\
During extraction, the LLM repeatedly produced types/relations OUTSIDE this ontology:\n{}\n\
\n\
These predicates were taken from the source text because nothing in the ontology fit. \
Their facts are currently filed under \"related_to\", which says nothing:\n{}\n\
\n\
- Propose ontology extensions. Rules:\n\
+ For each wording, decide one of two things.\n\
+ \n\
+ **If one of the candidates above already means it, map to it** — name that \
+ candidate's key in \"map_to\". Do this whenever the meaning matches even though the \
+ spelling differs (\"founding date\" is founding_date; \"headquartered in\" is \
+ location). Adding a second entry for a meaning the ontology already carries is the \
+ worst outcome available here: it splits the same facts across two keys permanently, \
+ and nothing downstream can tell they were the same.\n\
+ \n\
+ **Otherwise propose a new entry.** Rules:\n\
- Merge near-duplicates into ONE relation and list every spelling it covers in \
\"forms\" (e.g. available_on / available_from / \"available through\" are one relation).\n\
- Skip generic verbs that carry no domain meaning (is, has, includes, provides, brings).\n\
@@ -441,15 +543,15 @@ pub async fn build_proposals(
\n\
Output exactly one JSON object:\n\
{{\"entity_types\":[{{\"key\":\"snake_case\",\"label\":\"Display Name\",\"description\":\"what belongs here, and what does not\",\"reason\":\"why add it\"}}],\n\
- \"relation_types\":[{{\"key\":\"snake_case\",\"label\":\"display label\",\"temporal\":\"state|event|eternal\",\"functional\":false,\"forms\":[\"surface spellings this covers\"],\"description\":\"what this relation asserts, and what it does not\",\"reason\":\"why add it\"}}]}}\n\
+ \"relation_types\":[{{\"key\":\"snake_case\",\"label\":\"display label\",\"temporal\":\"state|event|eternal\",\"functional\":false,\"forms\":[\"surface spellings this covers\"],\"description\":\"what this relation asserts, and what it does not\",\"reason\":\"why add it\"}}],\n\
+ \"map_to\":[{{\"key\":\"an existing key, copied from the candidate list above\",\"forms\":[\"surface spellings that mean it\"],\"reason\":\"why these are the same thing\"}}]}}\n\
\n\
Language, and it overrides the skeleton above — that skeleton is written in English \
only because these instructions are. Write every \"label\" and \"description\" in {}: \
they become this knowledge base's own ontology, and the description is read by the \
extraction model while it reads documents in that language. Write every \"reason\" \
in {}: a person reads it. \"key\" and \"forms\" stay lowercase ASCII either way.",
- current_et.join(", "),
- current_rt.join(", "),
+ candidates_block,
miss_lines.join("\n"),
form_lines.join("\n"),
lang_name(&kb.ontology_lang),
@@ -464,14 +566,67 @@ pub async fn build_proposals(
.await
.map_err(AppError::Other)?;
let block = utopia_extract::json_block(&reply).map_err(AppError::Other)?;
- let proposals: serde_json::Value =
+ let mut proposals: serde_json::Value =
serde_json::from_str(&block).map_err(|e| AppError::Other(e.into()))?;
+ resolve_map_targets(&mut proposals, &by_name);
Ok(proposals)
}
+/// 把 `map_to` 里的 key 对回本体真正的 key,对不上的整条丢掉。
+///
+/// 模型抄错很常见,而且抄的往往是候选行里挨着的那个标签——`acquiredFrom`
+/// 而不是 `acquired_from`。**这一步必须在服务端做**:界面上那个"用已有的"
+/// 按钮承诺的是把一批事实挂到某个已有谓词上,key 对不上时它只会报错,
+/// 而承诺已经说出去了。对不上就不该显示这条。
+fn resolve_map_targets(
+ proposals: &mut serde_json::Value,
+ by_name: &std::collections::HashMap<String, String>,
+) {
+ let Some(items) = proposals.get_mut("map_to").and_then(|v| v.as_array_mut()) else {
+ return;
+ };
+ items.retain_mut(|m| {
+ let Some(raw) = m.get("key").and_then(|v| v.as_str()) else {
+ return false;
+ };
+ match by_name.get(&normalize_name(raw)) {
+ Some(real) => {
+ m["key"] = json!(real);
+ true
+ }
+ None => {
+ tracing::debug!(key = raw, "map_to 指向了候选之外的 key,丢弃");
+ false
+ }
+ }
+ });
+}
+
+/// key 与标签的比较形式:小写,只留字母数字。
+///
+/// **分隔符整个扔掉**,因为要对齐的正是分隔符的差异:`acquiredFrom`、
+/// `acquired_from`、`Acquired From` 都归到 `acquiredfrom`。折成下划线是不够的
+/// ——驼峰里根本没有分隔符可折。
+///
+/// 只做写法对齐,不做同义判断——那是检索与模型的活。
+fn normalize_name(s: &str) -> String {
+ s.chars()
+ .filter(|c| c.is_alphanumeric())
+ .flat_map(|c| c.to_lowercase())
+ .collect()
+}
+
#[derive(Deserialize)]
pub struct AdoptReq {
pub key: String,
+ /// true = 这个 key 指的是**已有**的关系/属性,别再建一个。
+ ///
+ /// 这一位是本体消解的落点:检索告诉模型本体里已经有 `founding_date` 了,
+ /// 模型说"这些说法就是它",采纳时只做改写那一半。少了这一位,同一个意思
+ /// 会长出第二个 key,往后这批事实就永久分在两处。
+ #[serde(default)]
+ pub existing: bool,
+ #[serde(default)]
pub label: String,
#[serde(default)]
pub description: Option<String>,
@@ -492,6 +647,9 @@ pub struct AdoptReq {
/// 与单纯 create 的区别就在后半句。只建类型的话本体长大了、图没变好——
/// 那 57 条事实会继续是"有关联"。改写走追加(新行 + supersedes),
/// 实体历史里读得到"先记成 related to,后精化成 available on"。
+///
+/// `existing` 为真时跳过前半句:本体里已经有这个意思了,这一次只做改写。
+/// 这是消解与增长的分界——同一个入口,因为对图做的事完全一样。
pub async fn adopt_predicate(
State(state): State<AppState>,
AuthUser(user): AuthUser,
@@ -503,23 +661,33 @@ pub async fn adopt_predicate(
if req.forms.is_empty() {
return Err(AppError::invalid("forms_required", "forms cannot be empty").into());
}
- let predicate_id = utopia_store::ontology::create_relation_type(
- &state.pool,
- kb_id,
- key,
- req.label.trim(),
- &req.temporal,
- req.functional,
- req.inverse_functional,
- req.description.as_deref().unwrap_or("").trim(),
- "relation",
- // 提案与冷启动只建关系,不声明 domain/range —— 留空 = 不限主宾类型
- &[],
- &[],
- None,
- None,
- )
- .await?;
+ let predicate_id = if req.existing {
+ // 按 key 找已有的那一条。找不到就报错而不是退回新建——
+ // 前端说的是"映射到已有的",静默改成新建就是它最不想要的结果
+ utopia_store::ontology::relation_type_id_by_key(&state.pool, kb_id, key)
+ .await?
+ .ok_or_else(|| {
+ AppError::invalid("unknown_relation_key", "no relation type with that key")
+ })?
+ } else {
+ utopia_store::ontology::create_relation_type(
+ &state.pool,
+ kb_id,
+ key,
+ req.label.trim(),
+ &req.temporal,
+ req.functional,
+ req.inverse_functional,
+ req.description.as_deref().unwrap_or("").trim(),
+ "relation",
+ // 提案与冷启动只建关系,不声明 domain/range —— 留空 = 不限主宾类型
+ &[],
+ &[],
+ None,
+ None,
+ )
+ .await?
+ };
let (batch_id, remapped) = utopia_store::graph::adopt_proposed_predicates(
&state.pool,
kb_id,
@@ -743,3 +911,59 @@ fn lang_name(code: &str) -> &'static str {
_ => "English",
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::{normalize_name, resolve_map_targets};
+ use serde_json::json;
+ use std::collections::HashMap;
+
+ #[test]
+ fn names_align_across_spellings() {
+ // 导进来的本体里 label 常常只是 key 的驼峰写法,两者必须归到一起
+ assert_eq!(
+ normalize_name("acquiredFrom"),
+ normalize_name("acquired_from")
+ );
+ assert_eq!(
+ normalize_name("Acquired From"),
+ normalize_name("acquired_from")
+ );
+ // 不同的名字仍然不同——这一步只对齐写法,不做同义判断
+ assert_ne!(normalize_name("acquired_from"), normalize_name("acquires"));
+ // 中文标签原样保留(去不掉也不该去)
+ assert_eq!(normalize_name("员工数"), "员工数");
+ }
+
+ #[test]
+ fn a_map_target_outside_the_candidates_is_dropped() {
+ let by_name: HashMap<String, String> = [
+ ("acquiredfrom".to_string(), "acquired_from".to_string()),
+ ("员工数".to_string(), "headcount".to_string()),
+ ]
+ .into_iter()
+ .collect();
+ let mut p = json!({"map_to": [
+ // 抄的是标签而不是 key —— 实测里模型就是这么干的,要能对回去
+ {"key": "acquiredFrom", "forms": ["acquired from"]},
+ // 按中文标签抄的,同样对得回去
+ {"key": "员工数", "forms": ["员工总数"]},
+ // 候选表里没有:界面上那个按钮会承诺一件做不到的事,所以整条丢掉
+ {"key": "invented_key", "forms": ["whatever"]},
+ // 连 key 都没有
+ {"forms": ["x"]},
+ ]});
+ resolve_map_targets(&mut p, &by_name);
+ let items = p["map_to"].as_array().unwrap();
+ assert_eq!(items.len(), 2);
+ assert_eq!(items[0]["key"], "acquired_from");
+ assert_eq!(items[1]["key"], "headcount");
+ }
+
+ #[test]
+ fn proposals_without_a_map_to_section_are_left_alone() {
+ let mut p = json!({"entity_types": [], "relation_types": []});
+ resolve_map_targets(&mut p, &HashMap::new());
+ assert!(p.get("map_to").is_none());
+ }
+}
diff --git a/crates/utopia-server/src/bootstrap_ontology.rs b/crates/utopia-server/src/bootstrap_ontology.rs
index 30269cec..ff33071e 100644
--- a/crates/utopia-server/src/bootstrap_ontology.rs
+++ b/crates/utopia-server/src/bootstrap_ontology.rs
@@ -197,6 +197,58 @@ pub async fn bootstrap_ontology(state: &AppState, kb_id: Uuid) -> anyhow::Result
}
}
+ // **映射到已有类型**:本体里已经有这个意思了,只改写事实、不动本体。
+ //
+ // 自动执行是安全的一档:它不让本体长大,只把一批 related_to 挂到一个
+ // 早就存在的谓词上,而且跟新建那条路走同一个批次机制,同样可撤销。
+ // 反过来说,漏掉这一档才是危险的——检索告诉模型"已经有 founding_date 了",
+ // 模型答"这些说法就是它",我们却什么都不做,那批事实继续是"有关联"。
+ let mapped = proposals
+ .get("map_to")
+ .and_then(|v| v.as_array())
+ .cloned()
+ .unwrap_or_default();
+ for m in &mapped {
+ let Some(key) = str_of(m, "key") else {
+ continue;
+ };
+ let forms: Vec<String> = m
+ .get("forms")
+ .and_then(|v| v.as_array())
+ .map(|a| {
+ a.iter()
+ .filter_map(|s| s.as_str().map(String::from))
+ .collect()
+ })
+ .unwrap_or_default();
+ if forms.is_empty() {
+ continue;
+ }
+ // 模型偶尔会把候选清单之外的 key 抄进来(或者干脆编一个)。
+ // 找不到就跳过——**不新建**:这条路的前提就是"它已经存在"
+ let Some(predicate_id) =
+ utopia_store::ontology::relation_type_id_by_key(&state.pool, kb_id, key).await?
+ else {
+ tracing::warn!(%kb_id, key, "映射目标不在本体里,跳过");
+ continue;
+ };
+ let (batch, moved) = utopia_store::graph::adopt_proposed_predicates(
+ &state.pool,
+ kb_id,
+ predicate_id,
+ &forms,
+ )
+ .await?;
+ moved_total += moved;
+ if moved > 0 {
+ batches.push(batch);
+ }
+ for form in &forms {
+ let _ =
+ utopia_store::ontology::clear_miss(&state.pool, kb_id, "relation_type", form).await;
+ }
+ }
+
// 类先建好、实体后抽出来是常态:把等着已存在类型的那些也收走
match utopia_store::resolution::sweep_proposed_types(&state.pool, kb_id).await {
Ok(swept) => {
diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs
index 62fa4f71..d94a7b08 100644
--- a/crates/utopia-server/src/main.rs
+++ b/crates/utopia-server/src/main.rs
@@ -10,6 +10,7 @@ mod extraction;
mod ingest_sources;
mod llm_util;
mod mappings;
+mod ontology_index;
mod owl_import;
mod pipeline;
mod query_engine;
diff --git a/crates/utopia-server/src/ontology_index.rs b/crates/utopia-server/src/ontology_index.rs
new file mode 100644
index 00000000..34a8ad74
--- /dev/null
+++ b/crates/utopia-server/src/ontology_index.rs
@@ -0,0 +1,111 @@
+//! 本体的向量索引:把类、关系、属性的 `label + description` 嵌成向量。
+//!
+//! 它服务的不是某一个功能,而是所有要问"这个说法对应本体里的哪一个"的地方:
+//! 本体提案(今天把 1949 个 key 全量内联,且只有 key 没有描述)、谓词消解、
+//! 类型消解。检索出候选之后再交给模型裁决,提示词才与本体规模脱钩。
+//!
+//! **自愈,不挂钩子。** 填充按"当时嵌的原文与模型名跟现在对不对得上"来判,
+//! 所以任何改了描述的写入点都不需要记得通知这里——漏挂一个钩子会悄悄烂掉,
+//! 而对比原文不会。
+
+use crate::{llm_util, AppState};
+use utopia_store::ontology::TypeToEmbed;
+use uuid::Uuid;
+
+/// 一次送多少条去嵌入。跟文档分块那边同量级,够摊平往返又不至于把请求撑爆。
+const BATCH: usize = 64;
+
+/// 把这个库里陈掉的本体向量补齐。没配嵌入模型就直接返回 `Ok(0)`——
+/// 检索是增强,不该拦住没配模型的部署。
+pub async fn refresh(state: &AppState, kb_id: Uuid) -> anyhow::Result<usize> {
+ let kb = utopia_store::kbs::get(&state.pool, kb_id).await?;
+ let Some(settings) = utopia_store::settings::get(&state.pool, kb.workspace_id).await? else {
+ return Ok(0);
+ };
+ let (Some(client), Some(model)) = (
+ llm_util::embed_client(&settings),
+ settings.embed_model.as_deref(),
+ ) else {
+ return Ok(0);
+ };
+
+ let stale = utopia_store::ontology::types_needing_embedding(&state.pool, kb_id, model).await?;
+ if stale.is_empty() {
+ return Ok(0);
+ }
+ let mut done = 0usize;
+ for batch in stale.chunks(BATCH) {
+ let texts: Vec<String> = batch.iter().map(|t| t.text.clone()).collect();
+ let _permit = llm_util::acquire_embed(state, &settings).await;
+ let vectors = client.embed(&texts).await?;
+ // 数量对不上就整批放弃:按位置配对,错位会把 person 的向量写到
+ // organization 上,而这种错一旦落库就再也看不出来了
+ if vectors.len() != batch.len() {
+ anyhow::bail!("嵌入返回 {} 条,送去的是 {} 条", vectors.len(), batch.len());
+ }
+ let items: Vec<(TypeToEmbed, Vec<f32>)> = batch.iter().cloned().zip(vectors).collect();
+ utopia_store::ontology::set_type_embeddings(&state.pool, model, &items).await?;
+ done += batch.len();
+ }
+ tracing::info!(%kb_id, count = done, "本体向量已补齐");
+ Ok(done)
+}
+
+/// 一批说法各自最像本体里的哪几个关系/属性。
+///
+/// **一次嵌完再逐个查库**,不是每个说法各发一次嵌入请求:本体提案一轮要处理
+/// 十几到几十个说法,逐个发就是十几到几十次往返。
+pub async fn nearest_for_each(
+ state: &AppState,
+ kb_id: Uuid,
+ queries: &[String],
+ limit: i64,
+ target: Target,
+) -> anyhow::Result<Vec<Vec<utopia_core::models::TypeCandidate>>> {
+ let empty = || vec![Vec::new(); queries.len()];
+ if queries.is_empty() {
+ return Ok(Vec::new());
+ }
+ let kb = utopia_store::kbs::get(&state.pool, kb_id).await?;
+ let Some(settings) = utopia_store::settings::get(&state.pool, kb.workspace_id).await? else {
+ return Ok(empty());
+ };
+ let Some(client) = llm_util::embed_client(&settings) else {
+ return Ok(empty());
+ };
+ let mut vectors = Vec::with_capacity(queries.len());
+ for batch in queries.chunks(BATCH) {
+ let _permit = llm_util::acquire_embed(state, &settings).await;
+ let got = client.embed(batch).await?;
+ if got.len() != batch.len() {
+ anyhow::bail!("嵌入返回 {} 条,送去的是 {} 条", got.len(), batch.len());
+ }
+ vectors.extend(got);
+ }
+ let mut out = Vec::with_capacity(vectors.len());
+ for v in &vectors {
+ out.push(match target {
+ Target::Class => {
+ utopia_store::ontology::nearest_entity_types(&state.pool, kb_id, v, limit).await?
+ }
+ Target::Predicate(kind) => {
+ utopia_store::ontology::nearest_relation_types(&state.pool, kb_id, v, limit, kind)
+ .await?
+ }
+ });
+ }
+ Ok(out)
+}
+
+/// 检索的是本体的哪一半。
+///
+/// **必须分开。** 类进 `entity_types`、关系与属性进 `relation_types`,两边的
+/// 描述写的是完全不同的东西("什么样的实体属于这里" vs "这条断言说了什么")。
+/// 拿一个词表外的类名去关系里检索,回来的一定是勉强相关的关系。
+#[derive(Debug, Clone, Copy)]
+pub enum Target {
+ Class,
+ /// `Some("attribute")` 只找属性、`Some("relation")` 只找关系、`None` 都找。
+ /// 字面值宾语的事实要的是属性,实体宾语的要的是关系
+ Predicate(Option<&'static str>),
+}
diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rs
index f1a8c43c..cb1db556 100644
--- a/crates/utopia-store/src/graph.rs
+++ b/crates/utopia-store/src/graph.rs
@@ -1305,8 +1305,8 @@ pub async fn adopt_proposed_predicates(
if forms.is_empty() {
return Ok((batch_id, 0));
}
- let targets: Vec<(Uuid, Uuid, Option<Uuid>)> = sqlx::query_as(
- "SELECT f.id, f.subject_id, f.object_id
+ let targets: Vec<(Uuid, Uuid, Option<Uuid>, Option<serde_json::Value>)> = sqlx::query_as(
+ "SELECT f.id, f.subject_id, f.object_id, f.object_value
FROM facts f
JOIN relation_types rt ON rt.id = f.predicate_id
WHERE f.kb_id = $1 AND rt.key = $2 AND f.invalidated_at IS NULL
@@ -1324,18 +1324,25 @@ pub async fn adopt_proposed_predicates(
.await?;
let mut moved = 0u32;
- for (old_id, subject_id, object_id) in targets {
+ for (old_id, subject_id, object_id, object_value) in targets {
let mut tx = pool.begin().await?;
- // 目标断言可能已存在(同主宾已有一条真关系):那就并进去,别造重复
+ // 目标断言可能已存在(同主宾已有一条真关系):那就并进去,别造重复。
+ //
+ // **宾语两侧都要比。** 字面值事实的 object_id 都是 NULL,只比它就等于
+ // 把同主同谓的所有值当成同一条断言——(星云科技, founding_date, 2015) 与
+ // (星云科技, founding_date, 2016) 会被并成一条,后一个值静默消失
let existing: Option<(Uuid,)> = sqlx::query_as(
"SELECT id FROM facts
WHERE kb_id = $1 AND subject_id = $2 AND predicate_id = $3
- AND object_id IS NOT DISTINCT FROM $4 AND invalidated_at IS NULL",
+ AND object_id IS NOT DISTINCT FROM $4
+ AND object_value IS NOT DISTINCT FROM $5
+ AND invalidated_at IS NULL",
)
.bind(kb_id)
.bind(subject_id)
.bind(predicate_id)
.bind(object_id)
+ .bind(&object_value)
.fetch_optional(&mut *tx)
.await?;
diff --git a/crates/utopia-store/src/ontology.rs b/crates/utopia-store/src/ontology.rs
index 789d9ed1..84f26cdd 100644
--- a/crates/utopia-store/src/ontology.rs
+++ b/crates/utopia-store/src/ontology.rs
@@ -1,8 +1,10 @@
//! 本体编辑器仓储:类型/关系 CRUD(带使用量与删除保护)+ 未匹配统计。
+use pgvector::Vector;
use sqlx::PgPool;
use utopia_core::models::{
EntityInstance, EntityTypeView, OntologyImportView, OntologyMiss, RelationTypeView,
+ TypeCandidate,
};
use utopia_core::{AppError, AppResult};
use uuid::Uuid;
@@ -775,3 +777,199 @@ pub async fn list_imports(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<OntologyI
.fetch_all(pool)
.await?)
}
+
+/// 一条待嵌入的本体行。`text` 是要送去嵌入的那段字,`kind` 决定回写哪张表。
+#[derive(Debug, Clone)]
+pub struct TypeToEmbed {
+ pub id: Uuid,
+ pub kind: TypeKind,
+ pub text: String,
+}
+
+/// 本体行属于哪张表。类进 `entity_types`,关系与属性同住 `relation_types`。
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TypeKind {
+ Entity,
+ Relation,
+}
+
+/// 嵌入用的那段字:标签在前、描述在后。
+///
+/// **key 不进去。** key 是模型读写的令牌(`founding_date`),描述才是这个类型
+/// 的意思所在。把 key 混进来,检索会被"两个 key 长得像"带偏——而
+/// `position`(列表位次)与 `position`(职位)正是长得一模一样的两回事。
+fn embed_text(label: &str, description: &str) -> String {
+ let d = description.trim();
+ if d.is_empty() {
+ label.trim().to_string()
+ } else {
+ format!("{}\n{}", label.trim(), d)
+ }
+}
+
+/// 哪些本体行的向量是陈的(没嵌过、描述改了、或换了嵌入模型)。
+///
+/// 判据是**比对当时嵌的原文与模型名**,不是看时间戳:描述改了、模型换了,
+/// 时间戳一样看不出来。这样也不必在每个改描述的写入点挂钩子——漏一个就悄悄烂掉。
+pub async fn types_needing_embedding(
+ pool: &PgPool,
+ kb_id: Uuid,
+ model: &str,
+) -> AppResult<Vec<TypeToEmbed>> {
+ let mut out = Vec::new();
+ let ents: Vec<(Uuid, String, String)> = sqlx::query_as(
+ "SELECT id, label, coalesce(description, '') FROM entity_types
+ WHERE kb_id = $1
+ AND (embedding IS NULL
+ OR embedded_model IS DISTINCT FROM $2
+ OR embedded_text IS DISTINCT FROM
+ CASE WHEN coalesce(btrim(description), '') = '' THEN btrim(label)
+ ELSE btrim(label) || E'\n' || btrim(description) END)",
+ )
+ .bind(kb_id)
+ .bind(model)
+ .fetch_all(pool)
+ .await?;
+ out.extend(ents.into_iter().map(|(id, label, desc)| TypeToEmbed {
+ id,
+ kind: TypeKind::Entity,
+ text: embed_text(&label, &desc),
+ }));
+ let rels: Vec<(Uuid, String, String)> = sqlx::query_as(
+ "SELECT id, label, coalesce(description, '') FROM relation_types
+ WHERE kb_id = $1
+ AND (embedding IS NULL
+ OR embedded_model IS DISTINCT FROM $2
+ OR embedded_text IS DISTINCT FROM
+ CASE WHEN coalesce(btrim(description), '') = '' THEN btrim(label)
+ ELSE btrim(label) || E'\n' || btrim(description) END)",
+ )
+ .bind(kb_id)
+ .bind(model)
+ .fetch_all(pool)
+ .await?;
+ out.extend(rels.into_iter().map(|(id, label, desc)| TypeToEmbed {
+ id,
+ kind: TypeKind::Relation,
+ text: embed_text(&label, &desc),
+ }));
+ Ok(out)
+}
+
+/// 回写向量,连同"嵌的是哪段字、用的哪个模型"。三者必须同一次写入——
+/// 只写向量而不写来源,下一轮就会认为它还是陈的,从此每轮重嵌。
+pub async fn set_type_embeddings(
+ pool: &PgPool,
+ model: &str,
+ items: &[(TypeToEmbed, Vec<f32>)],
+) -> AppResult<()> {
+ let mut tx = pool.begin().await?;
+ for (item, emb) in items {
+ let table = match item.kind {
+ TypeKind::Entity => "entity_types",
+ TypeKind::Relation => "relation_types",
+ };
+ sqlx::query(&format!(
+ "UPDATE {table} SET embedding = $2, embedded_text = $3, embedded_model = $4
+ WHERE id = $1"
+ ))
+ .bind(item.id)
+ .bind(Vector::from(emb.clone()))
+ .bind(&item.text)
+ .bind(model)
+ .execute(&mut *tx)
+ .await?;
+ }
+ tx.commit().await?;
+ Ok(())
+}
+
+/// 与给定向量最近的 k 个实体类型。
+pub async fn nearest_entity_types(
+ pool: &PgPool,
+ kb_id: Uuid,
+ embedding: &[f32],
+ limit: i64,
+) -> AppResult<Vec<TypeCandidate>> {
+ let rows: Vec<(Uuid, String, String, String, f64)> = sqlx::query_as(
+ "SELECT id, key, label, coalesce(description, ''), (embedding <=> $2)::float8
+ FROM entity_types
+ WHERE kb_id = $1 AND embedding IS NOT NULL
+ ORDER BY embedding <=> $2
+ LIMIT $3",
+ )
+ .bind(kb_id)
+ .bind(Vector::from(embedding.to_vec()))
+ .bind(limit)
+ .fetch_all(pool)
+ .await?;
+ Ok(rows
+ .into_iter()
+ .map(|(id, key, label, description, distance)| TypeCandidate {
+ id,
+ key,
+ label,
+ description,
+ kind: None,
+ distance: distance as f32,
+ })
+ .collect())
+}
+
+/// 与给定向量最近的 k 个关系/属性。
+///
+/// `only_kind` 分道:字面值宾语的事实要找的是属性,实体宾语的要找的是关系。
+/// 不分道就会把 `founding_date` 这种属性推给一条关系事实,反过来也一样。
+pub async fn nearest_relation_types(
+ pool: &PgPool,
+ kb_id: Uuid,
+ embedding: &[f32],
+ limit: i64,
+ only_kind: Option<&str>,
+) -> AppResult<Vec<TypeCandidate>> {
+ let rows: Vec<(Uuid, String, String, String, String, f64)> = sqlx::query_as(
+ "SELECT id, key, label, coalesce(description, ''), kind, (embedding <=> $2)::float8
+ FROM relation_types
+ WHERE kb_id = $1 AND embedding IS NOT NULL
+ AND ($4::text IS NULL OR kind = $4)
+ ORDER BY embedding <=> $2
+ LIMIT $3",
+ )
+ .bind(kb_id)
+ .bind(Vector::from(embedding.to_vec()))
+ .bind(limit)
+ .bind(only_kind)
+ .fetch_all(pool)
+ .await?;
+ Ok(rows
+ .into_iter()
+ .map(
+ |(id, key, label, description, kind, distance)| TypeCandidate {
+ id,
+ key,
+ label,
+ description,
+ kind: Some(kind),
+ distance: distance as f32,
+ },
+ )
+ .collect())
+}
+
+/// 按 key 找关系/属性的 id。给"映射到已有类型"那条路用。
+///
+/// 不区分 kind:属性与关系同住一张表且共用 key 命名空间,调用方拿到 id 之后
+/// 该怎么用它自己清楚(改写事实时谓词就是谓词)。
+pub async fn relation_type_id_by_key(
+ pool: &PgPool,
+ kb_id: Uuid,
+ key: &str,
+) -> AppResult<Option<Uuid>> {
+ let row: Option<(Uuid,)> =
+ sqlx::query_as("SELECT id FROM relation_types WHERE kb_id = $1 AND key = $2")
+ .bind(kb_id)
+ .bind(key)
+ .fetch_optional(pool)
+ .await?;
+ Ok(row.map(|(id,)| id))
+}
diff --git a/migrations/0038_ontology_embeddings.sql b/migrations/0038_ontology_embeddings.sql
new file mode 100644
index 00000000..6ea04863
--- /dev/null
+++ b/migrations/0038_ontology_embeddings.sql
@@ -0,0 +1,18 @@
+-- 本体自己的向量。类、关系、属性各一条,用 label + description 嵌出来。
+--
+-- 为什么要它:今天凡是要问"这个说法对应本体里的哪一个"的地方,都是把整个本体
+-- 内联进提示词——build_proposals 一次塞 1949 个 key(还只有 key 没有描述,
+-- 模型据此根本判断不了"这是不是已有类型的同义说法")。有了向量就能只取候选。
+--
+-- 维度不定:跟 chunks.embedding 一样随工作区所选模型走,所以不建 HNSW,顺扫。
+-- 本体行数以千计,顺扫完全够。
+ALTER TABLE entity_types ADD COLUMN embedding vector;
+ALTER TABLE relation_types ADD COLUMN embedding vector;
+
+-- **存"当时嵌的是什么"而不是一个时间戳。**
+-- 时间戳只能回答"嵌过没有",回答不了"嵌的还是不是现在这段文字"——描述改了、
+-- 嵌入模型换了,向量就是陈的,而时间戳看不出来。存原文与模型名,填充任务
+-- 一比对就知道该重嵌谁,也就不必去每一个改描述的写入点挂钩子(漏一个就悄悄烂掉)。
+-- 本体只有几千行,存原文的代价可以忽略,而排查检索质量时它值钱。
+ALTER TABLE entity_types ADD COLUMN embedded_text text, ADD COLUMN embedded_model text;
+ALTER TABLE relation_types ADD COLUMN embedded_text text, ADD COLUMN embedded_model text;
diff --git a/web/src/api.ts b/web/src/api.ts
index a540ed98..48b4130b 100644
--- a/web/src/api.ts
+++ b/web/src/api.ts
@@ -445,6 +445,17 @@ export interface OntologyProposals {
/** 这条关系归并了哪些表层说法。有它才谈得上把等待的事实改写过去 */
forms?: string[];
}[];
+ /**
+ * 本体里**已经有**这个意思,只需把说法挂过去。
+ *
+ * 跟 relation_types 的区别是不建东西:同一个意思长出第二个 key,
+ * 这批事实就永久分在两处,谁也认不出它们本是一回事。
+ */
+ map_to?: {
+ key: string;
+ forms?: string[];
+ reason?: string;
+ }[];
}
/** 原文说过、本体没有、事实降级成了 related_to 的谓词。 */
@@ -898,7 +909,9 @@ export const api = {
kbId: string,
body: {
key: string;
- label: string;
+ /** true = key 指的是已有的关系/属性,只改写事实,不建新类型 */
+ existing?: boolean;
+ label?: string;
temporal?: string;
functional?: boolean;
description?: string;
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index d0778429..4a2bda22 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -778,6 +778,9 @@ export const en = {
suggesting: "Analyzing…",
noMisses: "No unmatched types — the ontology covers your corpus.",
approve: "Add",
+ /* 映射那一档的按钮。刻意不叫 Add——它不加东西,本体里已经有了。
+ 两个按钮都写 Add 的话,"已经有了"这件事在界面上就消失了 */
+ mapOver: "Use existing",
/* 影响面:采纳一个提案会把多少条 related_to 事实改写过去。
没有这一句,"Add" 只是凭空多一个空关系 */
willRemap: (n: number) =>
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index 1e043280..73f7f52c 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -712,6 +712,7 @@ export const zh: Strings = {
suggesting: "分析中…",
noMisses: "没有未匹配的类型——本体覆盖了你的语料。",
approve: "加入",
+ mapOver: "用已有的",
willRemap: (n: number) => `将重新归类 ${n} 条事实`,
adopted: (n: number) => `已加入——${n} 条事实已重新归类`,
undoAdopt: (key: string, n: number) =>
diff --git a/web/src/pages/Ontology.tsx b/web/src/pages/Ontology.tsx
index c63d657b..8d244709 100644
--- a/web/src/pages/Ontology.tsx
+++ b/web/src/pages/Ontology.tsx
@@ -1403,6 +1403,33 @@ function MissesPanel({
// 逐条串行而不是加个批量端点:每个谓词各有自己的批次和撤销粒度,
// 而且部分失败能如实报告("5 个成功,1 个 key 已存在")而不是整批回滚
+ // 映射到已有类型:不建东西,只把这些说法的事实挂过去。
+ // 跟新建走同一个采纳入口,因为它对图做的事一模一样——也因此同样可撤销
+ const approveMapping = useMutation({
+ mutationFn: (p: { key: string; forms?: string[] }) =>
+ api.adoptPredicate(kbId, {
+ key: p.key,
+ existing: true,
+ forms: p.forms ?? [],
+ }),
+ onSuccess: (data, p) => {
+ const moved = data.remapped ?? 0;
+ toast.success(moved > 0 ? S.ontology.adopted(moved) : S.toast.saved);
+ if (moved > 0 && data.batch)
+ setLastAdopt({ batches: [data.batch], key: p.key, moved });
+ setProposals(
+ (prev) =>
+ prev && {
+ ...prev,
+ map_to: (prev.map_to ?? []).filter((x) => x.key !== p.key),
+ },
+ );
+ for (const form of p.forms ?? [])
+ api.dismissMiss(kbId, "relation_type", form).catch(() => {});
+ onChanged();
+ },
+ onError,
+ });
const addAll = useMutation({
mutationFn: async (all: OntologyProposals) => {
const batches: string[] = [];
@@ -1441,6 +1468,20 @@ function MissesPanel({
failed.push(p.key);
}
}
+ for (const p of all.map_to ?? []) {
+ if (!p.forms?.length) continue;
+ try {
+ const r = await api.adoptPredicate(kbId, {
+ key: p.key,
+ existing: true,
+ forms: p.forms,
+ });
+ moved += r.remapped;
+ if (r.remapped > 0) batches.push(r.batch);
+ } catch {
+ failed.push(p.key);
+ }
+ }
return { batches, moved, failed };
},
onSuccess: (r) => {
@@ -1600,7 +1641,9 @@ function MissesPanel({
{S.ontology.proposals}
</h4>
{/* 常见情形是"这些都对"——一条条点是把一个决定拆成八个 */}
- {proposals.relation_types.length + proposals.entity_types.length >
+ {proposals.relation_types.length +
+ proposals.entity_types.length +
+ (proposals.map_to?.length ?? 0) >
1 && (
<Button
size="sm"
@@ -1613,12 +1656,47 @@ function MissesPanel({
? S.ontology.addingAll
: S.ontology.addAll(
proposals.relation_types.length +
- proposals.entity_types.length,
+ proposals.entity_types.length +
+ (proposals.map_to?.length ?? 0),
)}
</Button>
)}
</div>
<div className="space-y-1.5">
+ {/* 排在最前:它说的是"本体已经有了",而这正是最该先看见的一句。
+ 排在新建后面的话,人一路点下来就把重复建出来了 */}
+ {(proposals.map_to ?? []).map((p) => (
+ <div key={`map-${p.key}`} className="flex items-center gap-2 text-sm">
+ <Chip tone="success">=</Chip>
+ <span className="font-mono text-neutral-300">{p.key}</span>
+ {!!p.forms?.length && (
+ <span
+ className="text-xs text-neutral-400 truncate"
+ title={p.forms.join(" · ")}
+ >
+ {p.forms.join(" · ")}
+ </span>
+ )}
+ {!!p.forms?.length && (
+ <span className="text-xs text-[var(--u-accent)]">
+ {S.ontology.willRemap(factsWaiting(p.forms))}
+ </span>
+ )}
+ {p.reason && (
+ <span className="text-xs text-neutral-500 truncate">
+ {p.reason}
+ </span>
+ )}
+ <Button
+ size="sm"
+ className="ml-auto"
+ onClick={() => approveMapping.mutate(p)}
+ disabled={approveMapping.isPending}
+ >
+ {S.ontology.mapOver}
+ </Button>
+ </div>
+ ))}
{proposals.entity_types.map((p) => (
<div key={p.key} className="flex items-center gap-2 text-sm">
<Chip tone="info">C</Chip>
@@ -1671,7 +1749,8 @@ function MissesPanel({
</div>
))}
{proposals.entity_types.length === 0 &&
- proposals.relation_types.length === 0 && (
+ proposals.relation_types.length === 0 &&
+ !proposals.map_to?.length && (
<p className="text-sm text-neutral-500">—</p>
)}
</div>