World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Proposals can now say the ontology already has one (#75)
WaylandYang0e55ba073db4
diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rsindex 35b35f28..ed274de2 100644--- a/crates/utopia-core/src/models.rs+++ b/crates/utopia-core/src/models.rs@@ -669,3 +669,18 @@ pub struct DataSourceView {pub last_test_at: Option<DateTime<Utc>>,pub last_test_ok: Option<bool>,}++/// 向量检索出来的一个候选本体行(类 / 关系 / 属性)。+///+/// `distance` 是余弦距离,越小越近。原样带给调用方而不是先折成"相似度":+/// 阈值该定在哪由消费者按自己的数据定,这里不替它归一化。+#[derive(Debug, Clone, Serialize)]+pub struct TypeCandidate {+ pub id: Uuid,+ pub key: String,+ pub label: String,+ pub description: String,+ /// 关系行才有:`relation` 或 `attribute`+ pub kind: Option<String>,+ pub distance: f32,+}diff --git a/crates/utopia-server/src/api/ontology_routes.rs b/crates/utopia-server/src/api/ontology_routes.rsindex df2e5d64..a8ac107d 100644--- a/crates/utopia-server/src/api/ontology_routes.rs+++ b/crates/utopia-server/src/api/ontology_routes.rs@@ -355,6 +355,12 @@ pub async fn suggest(/// 生成本体扩展提案。人工点 Suggest 与冷启动自动扩本体走同一条路——/// 自动那条不该是另一套判断,只是少了点头那一步。+/// 每个说法检索几个候选。+///+/// 小是刻意的:候选是给模型判断"是不是已经有了"用的,不是让它挑一个最像的凑合。+/// 开大了只会让它在一堆勉强相关的条目里硬选一个映射过去。+const CANDIDATES_PER_PROBE: i64 = 5;+pub async fn build_proposals(state: &AppState,kb_id: Uuid,@@ -367,8 +373,6 @@ pub async fn build_proposals(let client = llm_util::chat_client(&settings).ok_or_else(|| AppError::invalid("no_chat_model", "Chat model not configured"))?;- let entity_types = utopia_store::ontology::entity_type_views(&state.pool, kb_id).await?;- let relation_types = utopia_store::ontology::relation_type_views(&state.pool, kb_id).await?;// 只取这一遍**产得出提案**的那两类。attribute_type 的信号(未知谓词带字面值)// 也在这张表里,但下面的 JSON 骨架只有 entity_types / relation_types——// 把属性信号喂进去,模型只会照着建一个关系,而那正是它不该是的东西。@@ -384,8 +388,95 @@ pub async fn build_proposals(return Ok(json!({ "entity_types": [], "relation_types": [] }));}- let current_et: Vec<String> = entity_types.iter().map(|t| t.key.clone()).collect();- let current_rt: Vec<String> = relation_types.iter().map(|r| r.key.clone()).collect();+ // **本体的相关切片,不是它的全文。**+ //+ // 从前这里内联两串全量 key。两个毛病:一份 965 类的本体就是 1949 个 key+ // 的提示词;而且只有 key 没有描述,模型据此判断不了"这个说法是不是某个+ // 已有类型的同义",于是它只会新建,本体里就稳定长出重复。+ //+ // 现在按每个说法各检索几个最近的候选,带着描述给它看。提示词大小从此+ // 与本体规模无关,而"已经有一个了"这件事第一次变得可判断。+ let _ = crate::ontology_index::refresh(state, kb_id).await;+ // 谓词那一路:表层说法 + 关系类的 miss。样例给向量更多着落——+ // 光一个 acquires 太短,带上"星云科技 → 深蓝存储"就有了语境+ let pred_probes: Vec<String> = forms+ .iter()+ .map(|f| match f.example.as_deref() {+ Some(ex) if !ex.is_empty() => format!("{} — {ex}", f.form),+ _ => f.form.clone(),+ })+ .chain(+ misses+ .iter()+ .filter(|m| m.kind == "relation_type")+ .map(|m| m.key.clone()),+ )+ .collect();+ // 类那一路。**必须分开检索**:两张表的描述写的是完全不同的东西,+ // 拿一个词表外的类名去关系里找,回来的一定是勉强相关的关系+ let class_probes: Vec<String> = misses+ .iter()+ .filter(|m| m.kind == "entity_type")+ .map(|m| m.key.clone())+ .collect();+ let per_pred = crate::ontology_index::nearest_for_each(+ state,+ kb_id,+ &pred_probes,+ CANDIDATES_PER_PROBE,+ crate::ontology_index::Target::Predicate(None),+ )+ .await+ .unwrap_or_default();+ let per_class = crate::ontology_index::nearest_for_each(+ state,+ kb_id,+ &class_probes,+ CANDIDATES_PER_PROBE,+ crate::ontology_index::Target::Class,+ )+ .await+ .unwrap_or_default();+ // 并集去重:几个说法常常指向同一个候选,逐个说法各列一遍是白费令牌+ let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();+ let mut candidate_lines: Vec<String> = Vec::new();+ // 模型抄回来的 key 要能对回本体:候选表同时按 key、归一化 key、标签建索引+ let mut by_name: std::collections::HashMap<String, String> = std::collections::HashMap::new();+ for c in per_pred.iter().chain(per_class.iter()).flatten() {+ if !seen.insert(c.key.clone()) {+ continue;+ }+ by_name.insert(normalize_name(&c.key), c.key.clone());+ by_name.insert(normalize_name(&c.label), c.key.clone());+ // 关系行自带 relation / attribute,类行没有 kind+ let kind = c.kind.as_deref().unwrap_or("entity type");+ let d = c.description.trim();+ // **标签只在它确实多说了点什么的时候才写。**+ // 导进来的本体里 label 常常只是 key 的驼峰写法(acquired_from /+ // acquiredFrom),两个几乎一样的名字并排摆着,模型抄走的就是错的那个。+ // 中文标签配英文 key 那种才是标签有信息量的情形,那时才留+ let name = if normalize_name(&c.label) == normalize_name(&c.key) {+ String::new()+ } else {+ format!(" ({})", c.label)+ };+ candidate_lines.push(if d.is_empty() {+ format!("- {} [{kind}]{name}", c.key)+ } else {+ format!("- {} [{kind}]{name}: {d}", c.key)+ });+ }+ // 一个候选都没有(没配嵌入模型,或本体是空的)时如实说,别让模型+ // 以为"本体里什么都没有"从而放开手建+ let candidates_block = if candidate_lines.is_empty() {+ "(no candidates retrieved — the ontology may be empty, or embeddings are unavailable)"+ .to_string()+ } else {+ candidate_lines.join(+ "+",+ )+ };let miss_lines: Vec<String> = misses.iter().map(|m| {@@ -414,16 +505,27 @@ pub async fn build_proposals(.collect();let prompt = format!(- "You are an ontology engineer. A knowledge graph has this ontology:\n\- Entity type keys: {}\n\- Relation type keys: {}\n\+ "You are an ontology engineer.\n\+ \n\+ Below are the ontology entries closest in meaning to the unmatched wordings that \+ follow. This is a retrieved slice, not the whole ontology, so \"not in this list\" \+ does not mean \"not in the ontology\" — it means nothing close to it was found:\n{}\n\\n\During extraction, the LLM repeatedly produced types/relations OUTSIDE this ontology:\n{}\n\\n\These predicates were taken from the source text because nothing in the ontology fit. \Their facts are currently filed under \"related_to\", which says nothing:\n{}\n\\n\- Propose ontology extensions. Rules:\n\+ For each wording, decide one of two things.\n\+ \n\+ **If one of the candidates above already means it, map to it** — name that \+ candidate's key in \"map_to\". Do this whenever the meaning matches even though the \+ spelling differs (\"founding date\" is founding_date; \"headquartered in\" is \+ location). Adding a second entry for a meaning the ontology already carries is the \+ worst outcome available here: it splits the same facts across two keys permanently, \+ and nothing downstream can tell they were the same.\n\+ \n\+ **Otherwise propose a new entry.** Rules:\n\- Merge near-duplicates into ONE relation and list every spelling it covers in \\"forms\" (e.g. available_on / available_from / \"available through\" are one relation).\n\- Skip generic verbs that carry no domain meaning (is, has, includes, provides, brings).\n\@@ -441,15 +543,15 @@ pub async fn build_proposals(\n\Output exactly one JSON object:\n\{{\"entity_types\":[{{\"key\":\"snake_case\",\"label\":\"Display Name\",\"description\":\"what belongs here, and what does not\",\"reason\":\"why add it\"}}],\n\- \"relation_types\":[{{\"key\":\"snake_case\",\"label\":\"display label\",\"temporal\":\"state|event|eternal\",\"functional\":false,\"forms\":[\"surface spellings this covers\"],\"description\":\"what this relation asserts, and what it does not\",\"reason\":\"why add it\"}}]}}\n\+ \"relation_types\":[{{\"key\":\"snake_case\",\"label\":\"display label\",\"temporal\":\"state|event|eternal\",\"functional\":false,\"forms\":[\"surface spellings this covers\"],\"description\":\"what this relation asserts, and what it does not\",\"reason\":\"why add it\"}}],\n\+ \"map_to\":[{{\"key\":\"an existing key, copied from the candidate list above\",\"forms\":[\"surface spellings that mean it\"],\"reason\":\"why these are the same thing\"}}]}}\n\\n\Language, and it overrides the skeleton above — that skeleton is written in English \only because these instructions are. Write every \"label\" and \"description\" in {}: \they become this knowledge base's own ontology, and the description is read by the \extraction model while it reads documents in that language. Write every \"reason\" \in {}: a person reads it. \"key\" and \"forms\" stay lowercase ASCII either way.",- current_et.join(", "),- current_rt.join(", "),+ candidates_block,miss_lines.join("\n"),form_lines.join("\n"),lang_name(&kb.ontology_lang),@@ -464,14 +566,67 @@ pub async fn build_proposals(.await.map_err(AppError::Other)?;let block = utopia_extract::json_block(&reply).map_err(AppError::Other)?;- let proposals: serde_json::Value =+ let mut proposals: serde_json::Value =serde_json::from_str(&block).map_err(|e| AppError::Other(e.into()))?;+ resolve_map_targets(&mut proposals, &by_name);Ok(proposals)}+/// 把 `map_to` 里的 key 对回本体真正的 key,对不上的整条丢掉。+///+/// 模型抄错很常见,而且抄的往往是候选行里挨着的那个标签——`acquiredFrom`+/// 而不是 `acquired_from`。**这一步必须在服务端做**:界面上那个"用已有的"+/// 按钮承诺的是把一批事实挂到某个已有谓词上,key 对不上时它只会报错,+/// 而承诺已经说出去了。对不上就不该显示这条。+fn resolve_map_targets(+ proposals: &mut serde_json::Value,+ by_name: &std::collections::HashMap<String, String>,+) {+ let Some(items) = proposals.get_mut("map_to").and_then(|v| v.as_array_mut()) else {+ return;+ };+ items.retain_mut(|m| {+ let Some(raw) = m.get("key").and_then(|v| v.as_str()) else {+ return false;+ };+ match by_name.get(&normalize_name(raw)) {+ Some(real) => {+ m["key"] = json!(real);+ true+ }+ None => {+ tracing::debug!(key = raw, "map_to 指向了候选之外的 key,丢弃");+ false+ }+ }+ });+}++/// key 与标签的比较形式:小写,只留字母数字。+///+/// **分隔符整个扔掉**,因为要对齐的正是分隔符的差异:`acquiredFrom`、+/// `acquired_from`、`Acquired From` 都归到 `acquiredfrom`。折成下划线是不够的+/// ——驼峰里根本没有分隔符可折。+///+/// 只做写法对齐,不做同义判断——那是检索与模型的活。+fn normalize_name(s: &str) -> String {+ s.chars()+ .filter(|c| c.is_alphanumeric())+ .flat_map(|c| c.to_lowercase())+ .collect()+}+#[derive(Deserialize)]pub struct AdoptReq {pub key: String,+ /// true = 这个 key 指的是**已有**的关系/属性,别再建一个。+ ///+ /// 这一位是本体消解的落点:检索告诉模型本体里已经有 `founding_date` 了,+ /// 模型说"这些说法就是它",采纳时只做改写那一半。少了这一位,同一个意思+ /// 会长出第二个 key,往后这批事实就永久分在两处。+ #[serde(default)]+ pub existing: bool,+ #[serde(default)]pub label: String,#[serde(default)]pub description: Option<String>,@@ -492,6 +647,9 @@ pub struct AdoptReq {/// 与单纯 create 的区别就在后半句。只建类型的话本体长大了、图没变好——/// 那 57 条事实会继续是"有关联"。改写走追加(新行 + supersedes),/// 实体历史里读得到"先记成 related to,后精化成 available on"。+///+/// `existing` 为真时跳过前半句:本体里已经有这个意思了,这一次只做改写。+/// 这是消解与增长的分界——同一个入口,因为对图做的事完全一样。pub async fn adopt_predicate(State(state): State<AppState>,AuthUser(user): AuthUser,@@ -503,23 +661,33 @@ pub async fn adopt_predicate(if req.forms.is_empty() {return Err(AppError::invalid("forms_required", "forms cannot be empty").into());}- let predicate_id = utopia_store::ontology::create_relation_type(- &state.pool,- kb_id,- key,- req.label.trim(),- &req.temporal,- req.functional,- req.inverse_functional,- req.description.as_deref().unwrap_or("").trim(),- "relation",- // 提案与冷启动只建关系,不声明 domain/range —— 留空 = 不限主宾类型- &[],- &[],- None,- None,- )- .await?;+ let predicate_id = if req.existing {+ // 按 key 找已有的那一条。找不到就报错而不是退回新建——+ // 前端说的是"映射到已有的",静默改成新建就是它最不想要的结果+ utopia_store::ontology::relation_type_id_by_key(&state.pool, kb_id, key)+ .await?+ .ok_or_else(|| {+ AppError::invalid("unknown_relation_key", "no relation type with that key")+ })?+ } else {+ utopia_store::ontology::create_relation_type(+ &state.pool,+ kb_id,+ key,+ req.label.trim(),+ &req.temporal,+ req.functional,+ req.inverse_functional,+ req.description.as_deref().unwrap_or("").trim(),+ "relation",+ // 提案与冷启动只建关系,不声明 domain/range —— 留空 = 不限主宾类型+ &[],+ &[],+ None,+ None,+ )+ .await?+ };let (batch_id, remapped) = utopia_store::graph::adopt_proposed_predicates(&state.pool,kb_id,@@ -743,3 +911,59 @@ fn lang_name(code: &str) -> &'static str {_ => "English",}}++#[cfg(test)]+mod tests {+ use super::{normalize_name, resolve_map_targets};+ use serde_json::json;+ use std::collections::HashMap;++ #[test]+ fn names_align_across_spellings() {+ // 导进来的本体里 label 常常只是 key 的驼峰写法,两者必须归到一起+ assert_eq!(+ normalize_name("acquiredFrom"),+ normalize_name("acquired_from")+ );+ assert_eq!(+ normalize_name("Acquired From"),+ normalize_name("acquired_from")+ );+ // 不同的名字仍然不同——这一步只对齐写法,不做同义判断+ assert_ne!(normalize_name("acquired_from"), normalize_name("acquires"));+ // 中文标签原样保留(去不掉也不该去)+ assert_eq!(normalize_name("员工数"), "员工数");+ }++ #[test]+ fn a_map_target_outside_the_candidates_is_dropped() {+ let by_name: HashMap<String, String> = [+ ("acquiredfrom".to_string(), "acquired_from".to_string()),+ ("员工数".to_string(), "headcount".to_string()),+ ]+ .into_iter()+ .collect();+ let mut p = json!({"map_to": [+ // 抄的是标签而不是 key —— 实测里模型就是这么干的,要能对回去+ {"key": "acquiredFrom", "forms": ["acquired from"]},+ // 按中文标签抄的,同样对得回去+ {"key": "员工数", "forms": ["员工总数"]},+ // 候选表里没有:界面上那个按钮会承诺一件做不到的事,所以整条丢掉+ {"key": "invented_key", "forms": ["whatever"]},+ // 连 key 都没有+ {"forms": ["x"]},+ ]});+ resolve_map_targets(&mut p, &by_name);+ let items = p["map_to"].as_array().unwrap();+ assert_eq!(items.len(), 2);+ assert_eq!(items[0]["key"], "acquired_from");+ assert_eq!(items[1]["key"], "headcount");+ }++ #[test]+ fn proposals_without_a_map_to_section_are_left_alone() {+ let mut p = json!({"entity_types": [], "relation_types": []});+ resolve_map_targets(&mut p, &HashMap::new());+ assert!(p.get("map_to").is_none());+ }+}diff --git a/crates/utopia-server/src/bootstrap_ontology.rs b/crates/utopia-server/src/bootstrap_ontology.rsindex 30269cec..ff33071e 100644--- a/crates/utopia-server/src/bootstrap_ontology.rs+++ b/crates/utopia-server/src/bootstrap_ontology.rs@@ -197,6 +197,58 @@ pub async fn bootstrap_ontology(state: &AppState, kb_id: Uuid) -> anyhow::Result}}+ // **映射到已有类型**:本体里已经有这个意思了,只改写事实、不动本体。+ //+ // 自动执行是安全的一档:它不让本体长大,只把一批 related_to 挂到一个+ // 早就存在的谓词上,而且跟新建那条路走同一个批次机制,同样可撤销。+ // 反过来说,漏掉这一档才是危险的——检索告诉模型"已经有 founding_date 了",+ // 模型答"这些说法就是它",我们却什么都不做,那批事实继续是"有关联"。+ let mapped = proposals+ .get("map_to")+ .and_then(|v| v.as_array())+ .cloned()+ .unwrap_or_default();+ for m in &mapped {+ let Some(key) = str_of(m, "key") else {+ continue;+ };+ let forms: Vec<String> = m+ .get("forms")+ .and_then(|v| v.as_array())+ .map(|a| {+ a.iter()+ .filter_map(|s| s.as_str().map(String::from))+ .collect()+ })+ .unwrap_or_default();+ if forms.is_empty() {+ continue;+ }+ // 模型偶尔会把候选清单之外的 key 抄进来(或者干脆编一个)。+ // 找不到就跳过——**不新建**:这条路的前提就是"它已经存在"+ let Some(predicate_id) =+ utopia_store::ontology::relation_type_id_by_key(&state.pool, kb_id, key).await?+ else {+ tracing::warn!(%kb_id, key, "映射目标不在本体里,跳过");+ continue;+ };+ let (batch, moved) = utopia_store::graph::adopt_proposed_predicates(+ &state.pool,+ kb_id,+ predicate_id,+ &forms,+ )+ .await?;+ moved_total += moved;+ if moved > 0 {+ batches.push(batch);+ }+ for form in &forms {+ let _ =+ utopia_store::ontology::clear_miss(&state.pool, kb_id, "relation_type", form).await;+ }+ }+// 类先建好、实体后抽出来是常态:把等着已存在类型的那些也收走match utopia_store::resolution::sweep_proposed_types(&state.pool, kb_id).await {Ok(swept) => {diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex 62fa4f71..d94a7b08 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -10,6 +10,7 @@ mod extraction;mod ingest_sources;mod llm_util;mod mappings;+mod ontology_index;mod owl_import;mod pipeline;mod query_engine;diff --git a/crates/utopia-server/src/ontology_index.rs b/crates/utopia-server/src/ontology_index.rsnew file mode 100644index 00000000..34a8ad74--- /dev/null+++ b/crates/utopia-server/src/ontology_index.rs@@ -0,0 +1,111 @@+//! 本体的向量索引:把类、关系、属性的 `label + description` 嵌成向量。+//!+//! 它服务的不是某一个功能,而是所有要问"这个说法对应本体里的哪一个"的地方:+//! 本体提案(今天把 1949 个 key 全量内联,且只有 key 没有描述)、谓词消解、+//! 类型消解。检索出候选之后再交给模型裁决,提示词才与本体规模脱钩。+//!+//! **自愈,不挂钩子。** 填充按"当时嵌的原文与模型名跟现在对不对得上"来判,+//! 所以任何改了描述的写入点都不需要记得通知这里——漏挂一个钩子会悄悄烂掉,+//! 而对比原文不会。++use crate::{llm_util, AppState};+use utopia_store::ontology::TypeToEmbed;+use uuid::Uuid;++/// 一次送多少条去嵌入。跟文档分块那边同量级,够摊平往返又不至于把请求撑爆。+const BATCH: usize = 64;++/// 把这个库里陈掉的本体向量补齐。没配嵌入模型就直接返回 `Ok(0)`——+/// 检索是增强,不该拦住没配模型的部署。+pub async fn refresh(state: &AppState, kb_id: Uuid) -> anyhow::Result<usize> {+ let kb = utopia_store::kbs::get(&state.pool, kb_id).await?;+ let Some(settings) = utopia_store::settings::get(&state.pool, kb.workspace_id).await? else {+ return Ok(0);+ };+ let (Some(client), Some(model)) = (+ llm_util::embed_client(&settings),+ settings.embed_model.as_deref(),+ ) else {+ return Ok(0);+ };++ let stale = utopia_store::ontology::types_needing_embedding(&state.pool, kb_id, model).await?;+ if stale.is_empty() {+ return Ok(0);+ }+ let mut done = 0usize;+ for batch in stale.chunks(BATCH) {+ let texts: Vec<String> = batch.iter().map(|t| t.text.clone()).collect();+ let _permit = llm_util::acquire_embed(state, &settings).await;+ let vectors = client.embed(&texts).await?;+ // 数量对不上就整批放弃:按位置配对,错位会把 person 的向量写到+ // organization 上,而这种错一旦落库就再也看不出来了+ if vectors.len() != batch.len() {+ anyhow::bail!("嵌入返回 {} 条,送去的是 {} 条", vectors.len(), batch.len());+ }+ let items: Vec<(TypeToEmbed, Vec<f32>)> = batch.iter().cloned().zip(vectors).collect();+ utopia_store::ontology::set_type_embeddings(&state.pool, model, &items).await?;+ done += batch.len();+ }+ tracing::info!(%kb_id, count = done, "本体向量已补齐");+ Ok(done)+}++/// 一批说法各自最像本体里的哪几个关系/属性。+///+/// **一次嵌完再逐个查库**,不是每个说法各发一次嵌入请求:本体提案一轮要处理+/// 十几到几十个说法,逐个发就是十几到几十次往返。+pub async fn nearest_for_each(+ state: &AppState,+ kb_id: Uuid,+ queries: &[String],+ limit: i64,+ target: Target,+) -> anyhow::Result<Vec<Vec<utopia_core::models::TypeCandidate>>> {+ let empty = || vec![Vec::new(); queries.len()];+ if queries.is_empty() {+ return Ok(Vec::new());+ }+ let kb = utopia_store::kbs::get(&state.pool, kb_id).await?;+ let Some(settings) = utopia_store::settings::get(&state.pool, kb.workspace_id).await? else {+ return Ok(empty());+ };+ let Some(client) = llm_util::embed_client(&settings) else {+ return Ok(empty());+ };+ let mut vectors = Vec::with_capacity(queries.len());+ for batch in queries.chunks(BATCH) {+ let _permit = llm_util::acquire_embed(state, &settings).await;+ let got = client.embed(batch).await?;+ if got.len() != batch.len() {+ anyhow::bail!("嵌入返回 {} 条,送去的是 {} 条", got.len(), batch.len());+ }+ vectors.extend(got);+ }+ let mut out = Vec::with_capacity(vectors.len());+ for v in &vectors {+ out.push(match target {+ Target::Class => {+ utopia_store::ontology::nearest_entity_types(&state.pool, kb_id, v, limit).await?+ }+ Target::Predicate(kind) => {+ utopia_store::ontology::nearest_relation_types(&state.pool, kb_id, v, limit, kind)+ .await?+ }+ });+ }+ Ok(out)+}++/// 检索的是本体的哪一半。+///+/// **必须分开。** 类进 `entity_types`、关系与属性进 `relation_types`,两边的+/// 描述写的是完全不同的东西("什么样的实体属于这里" vs "这条断言说了什么")。+/// 拿一个词表外的类名去关系里检索,回来的一定是勉强相关的关系。+#[derive(Debug, Clone, Copy)]+pub enum Target {+ Class,+ /// `Some("attribute")` 只找属性、`Some("relation")` 只找关系、`None` 都找。+ /// 字面值宾语的事实要的是属性,实体宾语的要的是关系+ Predicate(Option<&'static str>),+}diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rsindex f1a8c43c..cb1db556 100644--- a/crates/utopia-store/src/graph.rs+++ b/crates/utopia-store/src/graph.rs@@ -1305,8 +1305,8 @@ pub async fn adopt_proposed_predicates(if forms.is_empty() {return Ok((batch_id, 0));}- let targets: Vec<(Uuid, Uuid, Option<Uuid>)> = sqlx::query_as(- "SELECT f.id, f.subject_id, f.object_id+ let targets: Vec<(Uuid, Uuid, Option<Uuid>, Option<serde_json::Value>)> = sqlx::query_as(+ "SELECT f.id, f.subject_id, f.object_id, f.object_valueFROM facts fJOIN relation_types rt ON rt.id = f.predicate_idWHERE f.kb_id = $1 AND rt.key = $2 AND f.invalidated_at IS NULL@@ -1324,18 +1324,25 @@ pub async fn adopt_proposed_predicates(.await?;let mut moved = 0u32;- for (old_id, subject_id, object_id) in targets {+ for (old_id, subject_id, object_id, object_value) in targets {let mut tx = pool.begin().await?;- // 目标断言可能已存在(同主宾已有一条真关系):那就并进去,别造重复+ // 目标断言可能已存在(同主宾已有一条真关系):那就并进去,别造重复。+ //+ // **宾语两侧都要比。** 字面值事实的 object_id 都是 NULL,只比它就等于+ // 把同主同谓的所有值当成同一条断言——(星云科技, founding_date, 2015) 与+ // (星云科技, founding_date, 2016) 会被并成一条,后一个值静默消失let existing: Option<(Uuid,)> = sqlx::query_as("SELECT id FROM factsWHERE kb_id = $1 AND subject_id = $2 AND predicate_id = $3- AND object_id IS NOT DISTINCT FROM $4 AND invalidated_at IS NULL",+ AND object_id IS NOT DISTINCT FROM $4+ AND object_value IS NOT DISTINCT FROM $5+ AND invalidated_at IS NULL",).bind(kb_id).bind(subject_id).bind(predicate_id).bind(object_id)+ .bind(&object_value).fetch_optional(&mut *tx).await?;diff --git a/crates/utopia-store/src/ontology.rs b/crates/utopia-store/src/ontology.rsindex 789d9ed1..84f26cdd 100644--- a/crates/utopia-store/src/ontology.rs+++ b/crates/utopia-store/src/ontology.rs@@ -1,8 +1,10 @@//! 本体编辑器仓储:类型/关系 CRUD(带使用量与删除保护)+ 未匹配统计。+use pgvector::Vector;use sqlx::PgPool;use utopia_core::models::{EntityInstance, EntityTypeView, OntologyImportView, OntologyMiss, RelationTypeView,+ TypeCandidate,};use utopia_core::{AppError, AppResult};use uuid::Uuid;@@ -775,3 +777,199 @@ pub async fn list_imports(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<OntologyI.fetch_all(pool).await?)}++/// 一条待嵌入的本体行。`text` 是要送去嵌入的那段字,`kind` 决定回写哪张表。+#[derive(Debug, Clone)]+pub struct TypeToEmbed {+ pub id: Uuid,+ pub kind: TypeKind,+ pub text: String,+}++/// 本体行属于哪张表。类进 `entity_types`,关系与属性同住 `relation_types`。+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum TypeKind {+ Entity,+ Relation,+}++/// 嵌入用的那段字:标签在前、描述在后。+///+/// **key 不进去。** key 是模型读写的令牌(`founding_date`),描述才是这个类型+/// 的意思所在。把 key 混进来,检索会被"两个 key 长得像"带偏——而+/// `position`(列表位次)与 `position`(职位)正是长得一模一样的两回事。+fn embed_text(label: &str, description: &str) -> String {+ let d = description.trim();+ if d.is_empty() {+ label.trim().to_string()+ } else {+ format!("{}\n{}", label.trim(), d)+ }+}++/// 哪些本体行的向量是陈的(没嵌过、描述改了、或换了嵌入模型)。+///+/// 判据是**比对当时嵌的原文与模型名**,不是看时间戳:描述改了、模型换了,+/// 时间戳一样看不出来。这样也不必在每个改描述的写入点挂钩子——漏一个就悄悄烂掉。+pub async fn types_needing_embedding(+ pool: &PgPool,+ kb_id: Uuid,+ model: &str,+) -> AppResult<Vec<TypeToEmbed>> {+ let mut out = Vec::new();+ let ents: Vec<(Uuid, String, String)> = sqlx::query_as(+ "SELECT id, label, coalesce(description, '') FROM entity_types+ WHERE kb_id = $1+ AND (embedding IS NULL+ OR embedded_model IS DISTINCT FROM $2+ OR embedded_text IS DISTINCT FROM+ CASE WHEN coalesce(btrim(description), '') = '' THEN btrim(label)+ ELSE btrim(label) || E'\n' || btrim(description) END)",+ )+ .bind(kb_id)+ .bind(model)+ .fetch_all(pool)+ .await?;+ out.extend(ents.into_iter().map(|(id, label, desc)| TypeToEmbed {+ id,+ kind: TypeKind::Entity,+ text: embed_text(&label, &desc),+ }));+ let rels: Vec<(Uuid, String, String)> = sqlx::query_as(+ "SELECT id, label, coalesce(description, '') FROM relation_types+ WHERE kb_id = $1+ AND (embedding IS NULL+ OR embedded_model IS DISTINCT FROM $2+ OR embedded_text IS DISTINCT FROM+ CASE WHEN coalesce(btrim(description), '') = '' THEN btrim(label)+ ELSE btrim(label) || E'\n' || btrim(description) END)",+ )+ .bind(kb_id)+ .bind(model)+ .fetch_all(pool)+ .await?;+ out.extend(rels.into_iter().map(|(id, label, desc)| TypeToEmbed {+ id,+ kind: TypeKind::Relation,+ text: embed_text(&label, &desc),+ }));+ Ok(out)+}++/// 回写向量,连同"嵌的是哪段字、用的哪个模型"。三者必须同一次写入——+/// 只写向量而不写来源,下一轮就会认为它还是陈的,从此每轮重嵌。+pub async fn set_type_embeddings(+ pool: &PgPool,+ model: &str,+ items: &[(TypeToEmbed, Vec<f32>)],+) -> AppResult<()> {+ let mut tx = pool.begin().await?;+ for (item, emb) in items {+ let table = match item.kind {+ TypeKind::Entity => "entity_types",+ TypeKind::Relation => "relation_types",+ };+ sqlx::query(&format!(+ "UPDATE {table} SET embedding = $2, embedded_text = $3, embedded_model = $4+ WHERE id = $1"+ ))+ .bind(item.id)+ .bind(Vector::from(emb.clone()))+ .bind(&item.text)+ .bind(model)+ .execute(&mut *tx)+ .await?;+ }+ tx.commit().await?;+ Ok(())+}++/// 与给定向量最近的 k 个实体类型。+pub async fn nearest_entity_types(+ pool: &PgPool,+ kb_id: Uuid,+ embedding: &[f32],+ limit: i64,+) -> AppResult<Vec<TypeCandidate>> {+ let rows: Vec<(Uuid, String, String, String, f64)> = sqlx::query_as(+ "SELECT id, key, label, coalesce(description, ''), (embedding <=> $2)::float8+ FROM entity_types+ WHERE kb_id = $1 AND embedding IS NOT NULL+ ORDER BY embedding <=> $2+ LIMIT $3",+ )+ .bind(kb_id)+ .bind(Vector::from(embedding.to_vec()))+ .bind(limit)+ .fetch_all(pool)+ .await?;+ Ok(rows+ .into_iter()+ .map(|(id, key, label, description, distance)| TypeCandidate {+ id,+ key,+ label,+ description,+ kind: None,+ distance: distance as f32,+ })+ .collect())+}++/// 与给定向量最近的 k 个关系/属性。+///+/// `only_kind` 分道:字面值宾语的事实要找的是属性,实体宾语的要找的是关系。+/// 不分道就会把 `founding_date` 这种属性推给一条关系事实,反过来也一样。+pub async fn nearest_relation_types(+ pool: &PgPool,+ kb_id: Uuid,+ embedding: &[f32],+ limit: i64,+ only_kind: Option<&str>,+) -> AppResult<Vec<TypeCandidate>> {+ let rows: Vec<(Uuid, String, String, String, String, f64)> = sqlx::query_as(+ "SELECT id, key, label, coalesce(description, ''), kind, (embedding <=> $2)::float8+ FROM relation_types+ WHERE kb_id = $1 AND embedding IS NOT NULL+ AND ($4::text IS NULL OR kind = $4)+ ORDER BY embedding <=> $2+ LIMIT $3",+ )+ .bind(kb_id)+ .bind(Vector::from(embedding.to_vec()))+ .bind(limit)+ .bind(only_kind)+ .fetch_all(pool)+ .await?;+ Ok(rows+ .into_iter()+ .map(+ |(id, key, label, description, kind, distance)| TypeCandidate {+ id,+ key,+ label,+ description,+ kind: Some(kind),+ distance: distance as f32,+ },+ )+ .collect())+}++/// 按 key 找关系/属性的 id。给"映射到已有类型"那条路用。+///+/// 不区分 kind:属性与关系同住一张表且共用 key 命名空间,调用方拿到 id 之后+/// 该怎么用它自己清楚(改写事实时谓词就是谓词)。+pub async fn relation_type_id_by_key(+ pool: &PgPool,+ kb_id: Uuid,+ key: &str,+) -> AppResult<Option<Uuid>> {+ let row: Option<(Uuid,)> =+ sqlx::query_as("SELECT id FROM relation_types WHERE kb_id = $1 AND key = $2")+ .bind(kb_id)+ .bind(key)+ .fetch_optional(pool)+ .await?;+ Ok(row.map(|(id,)| id))+}diff --git a/migrations/0038_ontology_embeddings.sql b/migrations/0038_ontology_embeddings.sqlnew file mode 100644index 00000000..6ea04863--- /dev/null+++ b/migrations/0038_ontology_embeddings.sql@@ -0,0 +1,18 @@+-- 本体自己的向量。类、关系、属性各一条,用 label + description 嵌出来。+--+-- 为什么要它:今天凡是要问"这个说法对应本体里的哪一个"的地方,都是把整个本体+-- 内联进提示词——build_proposals 一次塞 1949 个 key(还只有 key 没有描述,+-- 模型据此根本判断不了"这是不是已有类型的同义说法")。有了向量就能只取候选。+--+-- 维度不定:跟 chunks.embedding 一样随工作区所选模型走,所以不建 HNSW,顺扫。+-- 本体行数以千计,顺扫完全够。+ALTER TABLE entity_types ADD COLUMN embedding vector;+ALTER TABLE relation_types ADD COLUMN embedding vector;++-- **存"当时嵌的是什么"而不是一个时间戳。**+-- 时间戳只能回答"嵌过没有",回答不了"嵌的还是不是现在这段文字"——描述改了、+-- 嵌入模型换了,向量就是陈的,而时间戳看不出来。存原文与模型名,填充任务+-- 一比对就知道该重嵌谁,也就不必去每一个改描述的写入点挂钩子(漏一个就悄悄烂掉)。+-- 本体只有几千行,存原文的代价可以忽略,而排查检索质量时它值钱。+ALTER TABLE entity_types ADD COLUMN embedded_text text, ADD COLUMN embedded_model text;+ALTER TABLE relation_types ADD COLUMN embedded_text text, ADD COLUMN embedded_model text;diff --git a/web/src/api.ts b/web/src/api.tsindex a540ed98..48b4130b 100644--- a/web/src/api.ts+++ b/web/src/api.ts@@ -445,6 +445,17 @@ export interface OntologyProposals {/** 这条关系归并了哪些表层说法。有它才谈得上把等待的事实改写过去 */forms?: string[];}[];+ /**+ * 本体里**已经有**这个意思,只需把说法挂过去。+ *+ * 跟 relation_types 的区别是不建东西:同一个意思长出第二个 key,+ * 这批事实就永久分在两处,谁也认不出它们本是一回事。+ */+ map_to?: {+ key: string;+ forms?: string[];+ reason?: string;+ }[];}/** 原文说过、本体没有、事实降级成了 related_to 的谓词。 */@@ -898,7 +909,9 @@ export const api = {kbId: string,body: {key: string;- label: string;+ /** true = key 指的是已有的关系/属性,只改写事实,不建新类型 */+ existing?: boolean;+ label?: string;temporal?: string;functional?: boolean;description?: string;diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex d0778429..4a2bda22 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -778,6 +778,9 @@ export const en = {suggesting: "Analyzing…",noMisses: "No unmatched types — the ontology covers your corpus.",approve: "Add",+ /* 映射那一档的按钮。刻意不叫 Add——它不加东西,本体里已经有了。+ 两个按钮都写 Add 的话,"已经有了"这件事在界面上就消失了 */+ mapOver: "Use existing",/* 影响面:采纳一个提案会把多少条 related_to 事实改写过去。没有这一句,"Add" 只是凭空多一个空关系 */willRemap: (n: number) =>diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 1e043280..73f7f52c 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -712,6 +712,7 @@ export const zh: Strings = {suggesting: "分析中…",noMisses: "没有未匹配的类型——本体覆盖了你的语料。",approve: "加入",+ mapOver: "用已有的",willRemap: (n: number) => `将重新归类 ${n} 条事实`,adopted: (n: number) => `已加入——${n} 条事实已重新归类`,undoAdopt: (key: string, n: number) =>diff --git a/web/src/pages/Ontology.tsx b/web/src/pages/Ontology.tsxindex c63d657b..8d244709 100644--- a/web/src/pages/Ontology.tsx+++ b/web/src/pages/Ontology.tsx@@ -1403,6 +1403,33 @@ function MissesPanel({// 逐条串行而不是加个批量端点:每个谓词各有自己的批次和撤销粒度,// 而且部分失败能如实报告("5 个成功,1 个 key 已存在")而不是整批回滚+ // 映射到已有类型:不建东西,只把这些说法的事实挂过去。+ // 跟新建走同一个采纳入口,因为它对图做的事一模一样——也因此同样可撤销+ const approveMapping = useMutation({+ mutationFn: (p: { key: string; forms?: string[] }) =>+ api.adoptPredicate(kbId, {+ key: p.key,+ existing: true,+ forms: p.forms ?? [],+ }),+ onSuccess: (data, p) => {+ const moved = data.remapped ?? 0;+ toast.success(moved > 0 ? S.ontology.adopted(moved) : S.toast.saved);+ if (moved > 0 && data.batch)+ setLastAdopt({ batches: [data.batch], key: p.key, moved });+ setProposals(+ (prev) =>+ prev && {+ ...prev,+ map_to: (prev.map_to ?? []).filter((x) => x.key !== p.key),+ },+ );+ for (const form of p.forms ?? [])+ api.dismissMiss(kbId, "relation_type", form).catch(() => {});+ onChanged();+ },+ onError,+ });const addAll = useMutation({mutationFn: async (all: OntologyProposals) => {const batches: string[] = [];@@ -1441,6 +1468,20 @@ function MissesPanel({failed.push(p.key);}}+ for (const p of all.map_to ?? []) {+ if (!p.forms?.length) continue;+ try {+ const r = await api.adoptPredicate(kbId, {+ key: p.key,+ existing: true,+ forms: p.forms,+ });+ moved += r.remapped;+ if (r.remapped > 0) batches.push(r.batch);+ } catch {+ failed.push(p.key);+ }+ }return { batches, moved, failed };},onSuccess: (r) => {@@ -1600,7 +1641,9 @@ function MissesPanel({{S.ontology.proposals}</h4>{/* 常见情形是"这些都对"——一条条点是把一个决定拆成八个 */}- {proposals.relation_types.length + proposals.entity_types.length >+ {proposals.relation_types.length ++ proposals.entity_types.length ++ (proposals.map_to?.length ?? 0) >1 && (<Buttonsize="sm"@@ -1613,12 +1656,47 @@ function MissesPanel({? S.ontology.addingAll: S.ontology.addAll(proposals.relation_types.length +- proposals.entity_types.length,+ proposals.entity_types.length ++ (proposals.map_to?.length ?? 0),)}</Button>)}</div><div className="space-y-1.5">+ {/* 排在最前:它说的是"本体已经有了",而这正是最该先看见的一句。+ 排在新建后面的话,人一路点下来就把重复建出来了 */}+ {(proposals.map_to ?? []).map((p) => (+ <div key={`map-${p.key}`} className="flex items-center gap-2 text-sm">+ <Chip tone="success">=</Chip>+ <span className="font-mono text-neutral-300">{p.key}</span>+ {!!p.forms?.length && (+ <span+ className="text-xs text-neutral-400 truncate"+ title={p.forms.join(" · ")}+ >+ {p.forms.join(" · ")}+ </span>+ )}+ {!!p.forms?.length && (+ <span className="text-xs text-[var(--u-accent)]">+ {S.ontology.willRemap(factsWaiting(p.forms))}+ </span>+ )}+ {p.reason && (+ <span className="text-xs text-neutral-500 truncate">+ {p.reason}+ </span>+ )}+ <Button+ size="sm"+ className="ml-auto"+ onClick={() => approveMapping.mutate(p)}+ disabled={approveMapping.isPending}+ >+ {S.ontology.mapOver}+ </Button>+ </div>+ ))}{proposals.entity_types.map((p) => (<div key={p.key} className="flex items-center gap-2 text-sm"><Chip tone="info">C</Chip>@@ -1671,7 +1749,8 @@ function MissesPanel({</div>))}{proposals.entity_types.length === 0 &&- proposals.relation_types.length === 0 && (+ proposals.relation_types.length === 0 &&+ !proposals.map_to?.length && (<p className="text-sm text-neutral-500">—</p>)}</div>