World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Ask the model what it would call the thing (#79)
WaylandYangf2ad2b9ab078
diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rsindex 8ce85ab9..fa3cd6a7 100644--- a/crates/utopia-extract/src/lib.rs+++ b/crates/utopia-extract/src/lib.rs@@ -18,6 +18,14 @@ pub struct ExtractedEntity {pub name: String,#[serde(rename = "type")]pub type_key: String,+ /// 模型自己的说法:它认为这最具体是个什么。**不校验、不入本体**。+ ///+ /// 存在的理由是清单里总有个"差不多"的:本体有 product,模型觉得够用就选了,+ /// 心里那个"向量数据库软件"就此丢失。实测 17 个实体的 proposed_type+ /// 全是空的,正是这个原因——而事后消解最需要的恰是这个名字:+ /// 短名字对短标签,比拿一段中文散文去匹配 "A software application." 近得多。+ #[serde(default)]+ pub specific_type: Option<String>,}#[derive(Debug, Deserialize)]@@ -163,7 +171,7 @@ pub fn build_messages({attr_section}\\n\Output format:\n\- {{\"entities\":[{{\"name\":\"entity name\",\"type\":\"type key\"}}],\n\+ {{\"entities\":[{{\"name\":\"entity name\",\"type\":\"type key\",\"specific_type\":\"what you would call it\"}}],\n\\"facts\":[{{\"subject\":\"subject entity name\",\"predicate\":\"relation key\",\"object\":\"object entity name\",\n\\"valid_from\":\"2023-01\",\"valid_to\":null,\"confidence\":0.9,\"quote\":\"verbatim supporting quote\"}}]}}\n\\n\@@ -186,7 +194,12 @@ pub fn build_messages(9. The same holds for entity types: if none of the listed types fits, write the type \the text implies, in snake_case (e.g. \"model\", \"technology\"). Do not fall back \to a broad listed type such as \"concept\" merely because nothing specific matched \- — that hides the gap instead of reporting it.\+ — that hides the gap instead of reporting it.\n\+ 10. specific_type is required on every entity and is never checked against the list. \+ Name the most specific kind the thing is, in the words you would use for it. Write \+ it even when \"type\" already fits, and make it narrower than \"type\" wherever the \+ text supports it — type \"product\", specific_type \"vector database software\". \+ Repeat the listed type only when the text genuinely says nothing more precise.\{attr_rules}");@@ -618,4 +631,20 @@ mod tests {assert_eq!(e.entities.len(), 1);assert_eq!(e.entities[0].type_key, "person");}++ /// specific_type 在骨架里、也在规则里,且两处都说"永远要填"。+ ///+ /// 只写进骨架是不够的:**规则与骨架冲突时骨架赢**(语言那条就栽过一次)。+ /// 这里两边一致,所以要一起钉住。+ #[test]+ fn every_entity_is_asked_for_its_own_words() {+ let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text");+ let sys = &msgs[0].content;+ assert!(sys.contains("\"specific_type\":\"what you would call it\""));+ assert!(sys.contains("required on every entity"));+ // 关键的一句:不校验。校验它就等于又造了一个词表+ assert!(sys.contains("never checked against the list"));+ // 与 type 的关系必须说清楚,否则模型会把粗类抄一遍+ assert!(sys.contains("narrower than"));+ }}diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rsindex ebf5e488..872eef78 100644--- a/crates/utopia-server/src/extraction.rs+++ b/crates/utopia-server/src/extraction.rs@@ -324,6 +324,18 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {if let Some(p) = proposed {let _ = utopia_store::resolution::set_proposed_type(&state.pool, id, p).await;}+ // 模型自己的说法。**跟 proposed_type 分开存**:那一列的含义是+ // "本体里没有",增长回路靠它的稀有性设门槛;这一列每个实体都有。+ // 与粗类同名的不记——那不是更具体的说法,只是把清单抄了一遍+ if let Some(st) = e+ .specific_type+ .as_deref()+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .filter(|s| !s.eq_ignore_ascii_case(&e.type_key))+ {+ let _ = utopia_store::resolution::set_specific_type(&state.pool, id, st).await;+ }// 只记模型自己声明过类型的:主宾兜底那条路一律按 concept 消解,// 把一个猜出来的类型放进清单等于让后续分块照着猜的抄if !doc_entities.iter().any(|(eid, _, _)| *eid == id) {diff --git a/crates/utopia-server/src/type_resolution.rs b/crates/utopia-server/src/type_resolution.rsindex d2249d12..d060c00a 100644--- a/crates/utopia-server/src/type_resolution.rs+++ b/crates/utopia-server/src/type_resolution.rs@@ -11,6 +11,11 @@//! 2. **实体画像**——它参与的全部谓词,跨文档累积。//! 3. **证据引文**——同一批句子,但聚在一起,且不必跟几十个别的实体抢注意力。//!+//! 候选**两路来**,取并集不合分数:一路拿画像搜类的描述,一路拿语境向量搜+//! 已定类的实体、把它们的类当票投。第二路绕开了第一路的软肋(中文画像对+//! 英文样板描述),而且库越大越准。两路的距离一个在类空间一个在实体空间,+//! 合成一个排序是自欺——何况实测同一路的距离都不能跨实体比。+//!//! 粗类的后代**排在前面**,但不是唯一能选的——两套本体的分类轴常常不重合//! (schema.org 把软件挂在 CreativeWork 下,而抽取给的粗类是 product)。//! 该说不的是裁决那一步,它看得到描述也看得到粗类。@@ -25,6 +30,8 @@ const DUMPING_GROUND: &str = "concept";const BATCH: i64 = 60;/// 每个实体检索几个候选。给裁决看的,不是让它在一堆勉强相关里硬挑一个。const CANDIDATES: i64 = 8;+/// 看几个已定类的近邻。少了凑不出票数,多了尾巴上全是噪音。+const NEIGHBOURS: i64 = 10;/// 一个实体的消解建议(preview 用,不写库)。#[derive(Debug, serde::Serialize)]@@ -33,11 +40,36 @@ pub struct TypeSuggestion {pub name: String,pub coarse: String,pub proposed_type: Option<String>,+ pub specific_type: Option<String>,pub fact_count: i64,/// 送去检索的那段字。**回给调用方**——检索找不着的时候,/// 第一个要看的就是"我们拿什么去找的"pub profile: String,+ /// 第一路候选:画像 → 类的描述pub candidates: Vec<utopia_core::models::TypeCandidate>,+ /// 第二路候选:语境相似的已定类实体,按类投票+ pub neighbours: Vec<NeighbourVote>,+}++/// 近邻投出来的一个类。+///+/// **不跟 `candidates` 合成一个排序**:两路的距离一个在类空间、一个在实体空间,+/// 本来就不可比——何况实测同一路的距离都不能跨实体比。并集交给裁决,+/// 各自标明来源。附带的好处是这一路的证据人能读懂:+///「像 Milvus,而 Milvus 标的是 software_application」比「余弦 0.49」有用得多。+#[derive(Debug, serde::Serialize)]+pub struct NeighbourVote {+ pub key: String,+ /// 有几个近邻是这个类+ pub votes: usize,+ /// 最近的那个近邻有多近+ pub best_distance: f64,+ /// 投票的实体名,给人看的证据+ pub examples: Vec<String>,+ /// 这些近邻是不是**全部**来自同一批文档。+ /// 是的话这一票要打折:只出现一次的实体,语境向量就是那一块的向量,+ /// 同文档的实体自然互相成为近邻,而那不是类型证据+ pub same_document_only: bool,}/// 只算不写:每个待消解实体的画像与候选。@@ -94,14 +126,57 @@ pub async fn preview(state: &AppState, kb_id: Uuid) -> AppResult<Vec<TypeSuggest// 稳定排序:后代提前,同一档内保持检索给的距离序ranked.sort_by_key(|c| !descendants.contains(&c.id));let candidates: Vec<_> = ranked.into_iter().take(CANDIDATES as usize).collect();+ // 第二路:语境相似的已定类实体,按类投票+ let raw = utopia_store::resolution::nearest_typed_entities(+ &state.pool,+ kb_id,+ s.id,+ DUMPING_GROUND,+ NEIGHBOURS,+ )+ .await?;+ let mut votes: std::collections::BTreeMap<String, (usize, f64, Vec<String>, bool)> =+ std::collections::BTreeMap::new();+ for (name, _tid, key, distance, same_doc) in raw {+ let slot = votes+ .entry(key)+ .or_insert_with(|| (0, distance, Vec::new(), true));+ slot.0 += 1;+ slot.1 = slot.1.min(distance);+ if slot.2.len() < 3 {+ slot.2.push(name);+ }+ slot.3 &= same_doc;+ }+ let mut neighbours: Vec<NeighbourVote> = votes+ .into_iter()+ .filter(|(key, _)| *key != s.coarse_key)+ .map(+ |(key, (votes, best_distance, examples, same_document_only))| NeighbourVote {+ key,+ votes,+ best_distance,+ examples,+ same_document_only,+ },+ )+ .collect();+ neighbours.sort_by(|a, b| {+ b.votes+ .cmp(&a.votes)+ .then(a.best_distance.total_cmp(&b.best_distance))+ });+out.push(TypeSuggestion {entity_id: s.id,name: s.canonical_name.clone(),coarse: s.coarse_key.clone(),proposed_type: s.proposed_type.clone(),+ specific_type: s.specific_type.clone(),fact_count: s.fact_count,profile: profiles[i].clone(),candidates,+ neighbours,});}Ok(out)@@ -115,13 +190,12 @@ pub async fn preview(state: &AppState, kb_id: Uuid) -> AppResult<Vec<TypeSuggest/// 名字、别名其次;谓词与引文垫后当语境。fn profile_of(s: &utopia_store::resolution::TypeCandidateSubject) -> String {let mut parts: Vec<String> = Vec::new();- if let Some(p) = s- .proposed_type- .as_deref()- .map(str::trim)- .filter(|x| !x.is_empty())- {- parts.push(p.to_string());+ // 模型自己的说法放最前面,两个都有就都写。它们是**名字**,而检索的目标+ //(类的 label)也是名字——名字对名字,正是这个索引擅长的形状+ for named in [s.specific_type.as_deref(), s.proposed_type.as_deref()] {+ if let Some(p) = named.map(str::trim).filter(|x| !x.is_empty()) {+ parts.push(p.to_string());+ }}parts.push(s.canonical_name.clone());if !s.aliases.is_empty() {diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rsindex ec9c1834..a2cd3979 100644--- a/crates/utopia-store/src/resolution.rs+++ b/crates/utopia-store/src/resolution.rs@@ -1681,9 +1681,14 @@ pub struct TypeCandidateSubject {/// 现在挂着的粗类。**它是地板**:精化只往它的后代走,够不着就原地不动pub coarse_key: String,pub coarse_id: Uuid,- /// 抽取时模型自己报的类型名。词表里没有才会留在这儿——- /// 也正因如此它是最强的信号:不是"读懂这是什么",是"本体里哪个类叫这个"+ /// 抽取时模型自己报的类型名,**词表里没有**才会留在这儿pub proposed_type: Option<String>,+ /// 模型对它自己的说法,每个实体都有。+ ///+ /// 这是最强的信号,因为它把任务从"读懂这是什么"换回"本体里哪个类叫这个"+ /// ——短名字对短标签。实测的失败正出在另一头:拿一段中文画像去匹配+ /// schema.org 的 "A software application.",两边形状根本不对等+ pub specific_type: Option<String>,/// 它参与的谓词,连着对方的名字(`produces 深蓝`、`leads by 张伟`)。/// **跨文档累积**,这正是抽取当场没有的东西pub roles: Vec<String>,@@ -1708,7 +1713,7 @@ pub async fn entities_for_type_resolution() -> AppResult<Vec<TypeCandidateSubject>> {Ok(sqlx::query_as("SELECT e.id, e.canonical_name, e.aliases, t.key AS coarse_key, t.id AS coarse_id,- e.proposed_type,+ e.proposed_type, e.specific_type,-- 对方写**名字**,不写它的类型 key。-- 写 key 是自毁:查询里出现 organization / product / event 这些词,-- 而检索的目标正是这些类自己,于是候选就是画像里那几个词。@@ -1744,7 +1749,7 @@ pub async fn entities_for_type_resolution(-- **以及粗类本身还有子类的**。第三种是主力:抽取只认基类之后,-- organization 下面挂着 167 个更具体的类,那才是导进来的本体-- 唯一的用武之地。只看前两种就把它整个漏掉了- AND (t.key = $2 OR e.proposed_type IS NOT NULL+ AND (t.key = $2 OR e.proposed_type IS NOT NULL OR e.specific_type IS NOT NULLOR EXISTS (SELECT 1 FROM entity_type_parents p WHERE p.parent_id = t.id))ORDER BY fact_count DESC, e.created_atLIMIT $3",@@ -1775,3 +1780,73 @@ pub async fn descendants_of(pool: &PgPool, kb_id: Uuid, root: Uuid) -> AppResult.await?;Ok(rows.into_iter().map(|(id,)| id).collect())}++/// 记下模型对这个实体自己的说法。+///+/// **后写者不覆盖先写者**(`IS NULL` 才写),与 `set_proposed_type` 同一条:+/// 同一个实体会被多个分块提到,第一次的说法通常出自最完整的那句话,+/// 后面的分块往往只是顺带一提。+pub async fn set_specific_type(pool: &PgPool, entity_id: Uuid, value: &str) -> AppResult<()> {+ sqlx::query(+ "UPDATE entities SET specific_type = left($2, 80)+ WHERE id = $1 AND specific_type IS NULL",+ )+ .bind(entity_id)+ .bind(value)+ .execute(pool)+ .await?;+ Ok(())+}++/// 跟这个实体最像的**已定类**实体,连同它们的类。+///+/// 类型消解的第二个候选来源。第一个是拿画像去搜类的描述,它的软肋实测很清楚:+/// 中文画像对 schema.org 的 "A software application.",两边形状根本不对等。+/// 这一条绕开了那道坎——**名字对名字、同语言**,而且库越大越准:+/// 「深蓝向量数据库」像「Milvus」,而 Milvus 已经标成 software_application。+///+/// 比的是双方的 `profile_embedding`(出现语境的滑动平均),**同一种向量对同一种+/// 向量**,不是拿文本画像去比语境向量。代价是零——这个向量实体消解本来就在维护。+///+/// 已知弱点:只出现在一篇文档里的实体,语境向量就是那一块的向量,同文档的实体+/// 会互相成为近邻。调用方要看得到 `same_document`,别把它当成类型证据。+pub async fn nearest_typed_entities(+ pool: &PgPool,+ kb_id: Uuid,+ entity_id: Uuid,+ dumping_ground: &str,+ limit: i64,+) -> AppResult<Vec<(String, Uuid, String, f64, bool)>> {+ Ok(sqlx::query_as(+ "WITH me AS (+ SELECT profile_embedding AS v FROM entities WHERE id = $2 AND kb_id = $1+ ),+ my_docs AS (+ SELECT DISTINCT ev.document_id FROM fact_evidence ev+ JOIN facts f ON f.id = ev.fact_id+ WHERE f.kb_id = $1 AND (f.subject_id = $2 OR f.object_id = $2)+ )+ SELECT e.canonical_name, t.id, t.key,+ (e.profile_embedding <=> (SELECT v FROM me))::float8 AS distance,+ EXISTS (SELECT 1 FROM fact_evidence ev2+ JOIN facts f2 ON f2.id = ev2.fact_id+ WHERE f2.kb_id = $1 AND (f2.subject_id = e.id OR f2.object_id = e.id)+ AND ev2.document_id IN (SELECT document_id FROM my_docs))+ AS same_document+ FROM entities e+ JOIN entity_types t ON t.id = e.type_id+ WHERE e.kb_id = $1 AND e.merged_into IS NULL AND e.id <> $2+ AND e.profile_embedding IS NOT NULL+ AND (SELECT v FROM me) IS NOT NULL+ -- 倾倒场类不是答案,拿它当邻居的证据只会把没判出来的传染开+ AND t.key <> $3+ ORDER BY e.profile_embedding <=> (SELECT v FROM me)+ LIMIT $4",+ )+ .bind(kb_id)+ .bind(entity_id)+ .bind(dumping_ground)+ .bind(limit)+ .fetch_all(pool)+ .await?)+}diff --git a/migrations/0039_specific_type.sql b/migrations/0039_specific_type.sqlnew file mode 100644index 00000000..c206611c--- /dev/null+++ b/migrations/0039_specific_type.sql@@ -0,0 +1,11 @@+-- 模型对每个实体自己的说法:它认为这最具体是个什么。+--+-- 跟 proposed_type 是两回事,不能合用一列:那一列的含义是"模型要的东西本体里+-- 没有",0003 的增长回路靠它的**稀有性**设门槛。每个实体都填的话,回路会给+-- 每一个实体提议建一个新类。+--+-- 为什么需要它:清单里总有个"差不多"的。本体有 product,模型觉得够用就选了,+-- 心里那个"向量数据库软件"就此丢失——实测 17 个实体的 proposed_type 全空。+-- 而类型消解最需要的正是这个名字:短名字对短标签,比拿一段中文散文去匹配+-- schema.org 的 "A software application." 近得多。+ALTER TABLE entities ADD COLUMN specific_type text;