World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Short type names get an index shaped like them, and responses arrive compressed (#116)
WaylandYangb16b54fc640c
diff --git a/Cargo.lock b/Cargo.lockindex ab52085e..9b919538 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -116,6 +116,18 @@ dependencies = ["password-hash",]+[[package]]+name = "async-compression"+version = "0.4.43"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8"+dependencies = [+ "compression-codecs",+ "compression-core",+ "pin-project-lite",+ "tokio",+]+[[package]]name = "async-stream"version = "0.3.6"@@ -587,6 +599,23 @@ dependencies = ["memchr",]+[[package]]+name = "compression-codecs"+version = "0.4.38"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf"+dependencies = [+ "compression-core",+ "flate2",+ "memchr",+]++[[package]]+name = "compression-core"+version = "0.4.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789"+[[package]]name = "const-oid"version = "0.9.6"@@ -4466,6 +4495,7 @@ version = "0.6.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"dependencies = [+ "async-compression","bitflags","bytes","futures-core",diff --git a/Cargo.toml b/Cargo.tomlindex 82d78b86..c68e3974 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -55,7 +55,11 @@ text-splitter = "0.32"pdf-extract = "0.12"calamine = "0.36"pgvector = { version = "0.4", features = ["sqlx"] }-reqwest = { version = "0.13", default-features = false, features = ["json", "stream", "http2", "rustls", "webpki-roots", "charset", "system-proxy"] }+# `gzip` 不是可选项:一次嵌入回 64×1024 个浮点数,写成 JSON 文本每个数字占+# 21 字节而信息量只有 4 字节,一次响应 1.33 MB。实测同一条链路上不压缩 62 秒、+# gzip 21 秒(429 KB)——服务端算完只用 1 秒,其余全在传。+# `default-features = false` 把它一并关掉了,这里补回来。+reqwest = { version = "0.13", default-features = false, features = ["json", "stream", "http2", "rustls", "webpki-roots", "charset", "system-proxy", "gzip"] }feed-rs = "2"cron = "0.12"diff --git a/crates/utopia-server/src/ontology_index.rs b/crates/utopia-server/src/ontology_index.rsindex eb94c5ba..b2ca146a 100644--- a/crates/utopia-server/src/ontology_index.rs+++ b/crates/utopia-server/src/ontology_index.rs@@ -9,11 +9,31 @@//! 而对比原文不会。use crate::{llm_util, AppState};+use futures_util::{stream, StreamExt};use utopia_store::ontology::TypeToEmbed;use uuid::Uuid;-/// 一次送多少条去嵌入。跟文档分块那边同量级,够摊平往返又不至于把请求撑爆。+/// 一次送多少条去嵌入。+///+/// **64 是实测站得住的那个数。** 曾经改成 16,理由是拿合成短文本量出来的+/// "每条更快"——而真实的类文本平均 151 字符,那组数据不作数。改小之后请求数+/// 翻四倍、每个都付约 2.5 秒固定开销,整体吞吐反而掉了一半。+///+/// 换批大小之前先拿**真实的 `embedded_text`** 去量,别拿造出来的短句。const BATCH: usize = 64;+/// 同时在飞的嵌入批数上限。+///+/// **关键不是快,是不能把闸门占死。** 这是个后台补齐任务,而嵌入模型的并发闸门+/// (`model_concurrency`,缺省 10)是全局共享的——文档分块入库也要走它。+///+/// 从前这里串行,一次一张许可证、每批之间释放,文档处理插得进来。改成+/// `join_all` 一次挂 31 个批次之后,它变成一个常驻的 10 深队列,每个批次握着+/// 许可证 60 秒:**5 篇文档全卡在 `embedding` 状态,抽取 15 分钟零进展**。+/// 不是 API 挂了,是这个任务把共享资源占满了。+///+/// 所以这里的上限必须**明显小于**闸门,给别的活留出位置。4 是实测能跑满+/// 吞吐又留有余量的:4 个 16 条并发,墙钟 18.5 秒(串行要 50 秒)。+const EMBED_JOBS: usize = 4;/// 把这个库里陈掉的本体向量补齐。没配嵌入模型就直接返回 `Ok(0)`——/// 检索是增强,不该拦住没配模型的部署。@@ -38,31 +58,60 @@ pub async fn refresh_scoped(};let (Some(client), Some(model)) = (llm_util::embed_client(&settings),- settings.embed_model.as_deref(),+ // 取拥有的一份:下面那些 future 要把 settings 挪进 Arc,借着它就动不了了+ settings.embed_model.clone(),) else {return Ok(0);};let stale =- utopia_store::ontology::types_needing_embedding(&state.pool, kb_id, model, only).await?;+ utopia_store::ontology::types_needing_embedding(&state.pool, kb_id, &model, only).await?;if stale.is_empty() {return Ok(0);}let mut done = 0usize;- for batch in stale.chunks(BATCH) {- let texts: Vec<String> = batch.iter().map(|t| t.text.clone()).collect();- let _permit = llm_util::acquire_embed(state, &settings).await;- let vectors = client.embed(&texts).await?;- // 数量对不上就整批放弃:按位置配对,错位会把 person 的向量写到- // organization 上,而这种错一旦落库就再也看不出来了- if vectors.len() != batch.len() {- anyhow::bail!("嵌入返回 {} 条,送去的是 {} 条", vectors.len(), batch.len());+ let mut failed = 0usize;+ // **全部用拥有的数据**:这些 future 要跨 await 被并发驱动,借用过不了 Send 边界+ let client = std::sync::Arc::new(client);+ let model = std::sync::Arc::new(model);+ let settings = std::sync::Arc::new(settings);+ let batches: Vec<Vec<TypeToEmbed>> = stale.chunks(BATCH).map(<[_]>::to_vec).collect();+ let mut jobs = stream::iter(batches.into_iter().map(|batch| {+ let (client, model, settings) = (client.clone(), model.clone(), settings.clone());+ async move {+ let texts: Vec<String> = batch.iter().map(|t| t.text.clone()).collect();+ let _permit = llm_util::acquire_embed(state, &settings).await;+ let vectors = client.embed(&texts).await?;+ // 数量对不上就整批放弃:按位置配对,错位会把 person 的向量写到+ // organization 上,而这种错一旦落库就再也看不出来了+ if vectors.len() != batch.len() {+ anyhow::bail!("嵌入返回 {} 条,送去的是 {} 条", vectors.len(), batch.len());+ }+ let n = batch.len();+ let items: Vec<(TypeToEmbed, Vec<f32>)> = batch.into_iter().zip(vectors).collect();+ utopia_store::ontology::set_type_embeddings(&state.pool, &model, &items).await?;+ Ok::<usize, anyhow::Error>(n)}- let items: Vec<(TypeToEmbed, Vec<f32>)> = batch.iter().cloned().zip(vectors).collect();- utopia_store::ontology::set_type_embeddings(&state.pool, model, &items).await?;- done += batch.len();+ }))+ .buffer_unordered(EMBED_JOBS);+ // **一批失败不拖垮其余**:这是补齐任务,缺的下一轮自愈(判据是"当时嵌的+ // 原文对不对得上",不是时间戳)。整个 bail 掉等于把已经嵌好的也白跑一遍+ while let Some(r) = jobs.next().await {+ match r {+ Ok(n) => done += n,+ Err(e) => {+ failed += 1;+ tracing::warn!(%kb_id, error = %e, "一批本体向量没嵌上,留给下一轮");+ }+ }+ }+ // **失败数要出现在这一行。** 从前它只报补齐了多少,于是 24 批全挂的那次+ // 日志照样写着"已补齐",看日志的人以为没事+ if failed > 0 {+ tracing::warn!(%kb_id, count = done, failed, "本体向量补了一部分,其余留给下一轮");+ } else {+ tracing::info!(%kb_id, count = done, "本体向量已补齐");}- tracing::info!(%kb_id, count = done, "本体向量已补齐");Ok(done)}@@ -101,7 +150,12 @@ pub async fn nearest_for_each(for v in &vectors {out.push(match target {Target::Class => {- utopia_store::ontology::nearest_entity_types(&state.pool, kb_id, v, limit).await?+ utopia_store::ontology::nearest_entity_types(&state.pool, kb_id, v, limit, false)+ .await?+ }+ Target::ClassLabel => {+ utopia_store::ontology::nearest_entity_types(&state.pool, kb_id, v, limit, true)+ .await?}Target::Predicate(kind) => {utopia_store::ontology::nearest_relation_types(&state.pool, kb_id, v, limit, kind)@@ -119,7 +173,12 @@ pub async fn nearest_for_each(/// 拿一个词表外的类名去关系里检索,回来的一定是勉强相关的关系。#[derive(Debug, Clone, Copy)]pub enum Target {+ /// 整段索引(label + 描述)。长画像走这一路Class,+ /// **只有 label 的索引**(0050)。短说法(`district. place`)走这一路。+ /// 短说法比整段索引会被同义反复的类接管——`Map` 那一行的描述就是+ /// "A map.",赢在长度不在语义+ ClassLabel,/// `Some("attribute")` 只找属性、`Some("relation")` 只找关系、`None` 都找。/// 字面值宾语的事实要的是属性,实体宾语的要的是关系Predicate(Option<&'static str>),diff --git a/crates/utopia-server/src/type_resolution.rs b/crates/utopia-server/src/type_resolution.rsindex 19ed622c..165cceb0 100644--- a/crates/utopia-server/src/type_resolution.rs+++ b/crates/utopia-server/src/type_resolution.rs@@ -39,8 +39,14 @@ use uuid::Uuid;/// 一轮看多少个实体。够一次人工过目,也够看出检索准不准。const BATCH: i64 = 60;-/// 每个实体检索几个候选。给裁决看的,不是让它在一堆勉强相关里硬挑一个。-const CANDIDATES: i64 = 8;+/// 每个实体检索几个候选。+///+/// **检索的活是端候选,说不是裁决的活。** 所以宁可多端两个——裁决看得到描述、+/// 看得到现类,能说出"这些都不是";而检索漏掉的,裁决再好也够不着。0001 P3a+/// 实测的失败正是全在检索一侧(`administrative_area`、`periodical` 从没被端上来)。+///+/// 从 8 加到 10:两路交替各占五席。代价是提示词里多两行类定义,而漏检没有退路。+const CANDIDATES: i64 = 10;/// 看几个已定类的近邻。少了凑不出票数,多了尾巴上全是噪音。const NEIGHBOURS: i64 = 10;@@ -123,28 +129,42 @@ pub async fn preview(state: &AppState, kb_id: Uuid) -> AppResult<Vec<TypeSuggest// 两次结果取并集——多一次嵌入,换掉一整类漏检。let profiles: Vec<String> = subjects.iter().map(profile_of).collect();let names: Vec<Option<String>> = subjects.iter().map(name_query_of).collect();- let mut queries: Vec<String> = Vec::with_capacity(subjects.len() * 2);- // 每个实体在 queries 里占的下标:(画像, 名字)+ // **两路各查各的索引**(0050)。从前两路共用整段索引,于是短说法那一路+ // 被同义反复的类接管:`district. place` 回来的是 Map / Park / Country /+ // Museum,四个赢家的嵌入原文全是 `X\nAn X.` 一行话,而带一百字定义的+ // AdministrativeArea 排不进前八。短对短、长对长,这才是可比的+ let name_queries: Vec<String> = names.iter().flatten().cloned().collect();+ let (profile_hits, name_hits) = tokio::join!(+ ontology_index::nearest_for_each(+ state,+ kb_id,+ &profiles,+ // 多取一些再按祖先过滤:过滤在检索之后,所以要留出被滤掉的余量+ CANDIDATES * 4,+ ontology_index::Target::Class,+ ),+ ontology_index::nearest_for_each(+ state,+ kb_id,+ &name_queries,+ CANDIDATES * 4,+ ontology_index::Target::ClassLabel,+ )+ );+ let profile_hits = profile_hits.unwrap_or_default();+ let name_hits = name_hits.unwrap_or_default();+ // 每个实体在两份结果里各占的下标:(画像, 名字)。名字那一路只有给得出+ // 说法的实体才发了查询,所以下标要单独数let mut slots: Vec<(usize, Option<usize>)> = Vec::with_capacity(subjects.len());- for (p, n) in profiles.iter().zip(&names) {- let pi = queries.len();- queries.push(p.clone());- let ni = n.as_ref().map(|q| {- queries.push(q.clone());- queries.len() - 1+ let mut ni = 0usize;+ for (pi, n) in names.iter().enumerate() {+ let slot = n.as_ref().map(|_| {+ let k = ni;+ ni += 1;+ k});- slots.push((pi, ni));+ slots.push((pi, slot));}- let hits = ontology_index::nearest_for_each(- state,- kb_id,- &queries,- // 多取一些再按祖先过滤:过滤在检索之后,所以要留出被滤掉的余量- CANDIDATES * 4,- ontology_index::Target::Class,- )- .await- .unwrap_or_default();let mut out = Vec::with_capacity(subjects.len());for (i, s) in subjects.iter().enumerate() {@@ -176,11 +196,10 @@ pub async fn preview(state: &AppState, kb_id: Uuid) -> AppResult<Vec<TypeSuggest//// 交替之后两路各占一半席位,谁的距离数值大小不再影响谁被看见。let (pi, ni) = slots[i];- let lists: Vec<Vec<_>> = [Some(pi), ni]- .into_iter()- .flatten()- .map(|idx| hits.get(idx).cloned().unwrap_or_default())- .collect();+ let mut lists: Vec<Vec<_>> = vec![profile_hits.get(pi).cloned().unwrap_or_default()];+ if let Some(k) = ni {+ lists.push(name_hits.get(k).cloned().unwrap_or_default());+ }let mut seen_ids: std::collections::HashSet<Uuid> = std::collections::HashSet::new();let mut ranked: Vec<utopia_core::models::TypeCandidate> = Vec::new();let longest = lists.iter().map(Vec::len).max().unwrap_or(0);diff --git a/crates/utopia-store/src/ontology.rs b/crates/utopia-store/src/ontology.rsindex 57f161bf..c2cdfcb5 100644--- a/crates/utopia-store/src/ontology.rs+++ b/crates/utopia-store/src/ontology.rs@@ -826,6 +826,17 @@ pub struct TypeToEmbed {pub id: Uuid,pub kind: TypeKind,pub text: String,+ /// 写进哪一组列。类有两份向量:整段(label + 描述)与只有 label 的那份,+ /// 分别服务长画像与短说法两种查询(见迁移 0050)+ pub field: EmbedField,+}++/// 同一行的两份向量。**短查询比 Label,长画像比 Full**——查询分了两种形状,+/// 文档也得分两种,否则短查询会被同义反复的类接管(`Map\nA map.` 那一类)。+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum EmbedField {+ Full,+ Label,}/// 本体行属于哪张表。类进 `entity_types`,关系与属性同住 `relation_types`。@@ -884,6 +895,26 @@ pub async fn types_needing_embedding(id,kind: TypeKind::Entity,text: embed_text(&label, &desc),+ field: EmbedField::Full,+ }));+ // 只嵌 label 的那一份(0050)。短查询走这个索引——查询分了两种形状,+ // 文档也得分两种,否则短查询被同义反复的类接管+ let labels: Vec<(Uuid, String)> = sqlx::query_as(+ "SELECT id, label FROM entity_types+ WHERE kb_id = $1+ AND (label_embedding IS NULL+ OR label_embedded_model IS DISTINCT FROM $2+ OR label_embedded_text IS DISTINCT FROM btrim(label))",+ )+ .bind(kb_id)+ .bind(model)+ .fetch_all(pool)+ .await?;+ out.extend(labels.into_iter().map(|(id, label)| TypeToEmbed {+ id,+ kind: TypeKind::Entity,+ text: label.trim().to_string(),+ field: EmbedField::Label,}));if only == Some(TypeKind::Entity) {return Ok(out);@@ -915,6 +946,8 @@ async fn relations_needing_embedding(id,kind: TypeKind::Relation,text: embed_text(&label, &desc),+ // 关系没有短查询那一路,只有整段这一份+ field: EmbedField::Full,}));Ok(out)}@@ -932,8 +965,17 @@ pub async fn set_type_embeddings(TypeKind::Entity => "entity_types",TypeKind::Relation => "relation_types",};+ // 两份向量各写各的列(0050)。列名前缀不同,其余一模一样+ let (vec_col, text_col, model_col) = match item.field {+ EmbedField::Full => ("embedding", "embedded_text", "embedded_model"),+ EmbedField::Label => (+ "label_embedding",+ "label_embedded_text",+ "label_embedded_model",+ ),+ };sqlx::query(&format!(- "UPDATE {table} SET embedding = $2, embedded_text = $3, embedded_model = $4+ "UPDATE {table} SET {vec_col} = $2, {text_col} = $3, {model_col} = $4WHERE id = $1")).bind(item.id)@@ -967,17 +1009,25 @@ pub async fn nearest_entity_types(kb_id: Uuid,embedding: &[f32],limit: i64,+ // true = 比只有 label 的那份向量(0050)。短说法走这一路:**短对短**,+ // 否则 `district. place` 会被 `Map\nA map.` 这类同义反复的一行话赢过去+ by_label: bool,) -> AppResult<Vec<TypeCandidate>> {- let rows: Vec<(Uuid, String, String, String, f64)> = sqlx::query_as(- "SELECT id, key, label, coalesce(description, ''), (embedding <=> $2)::float8+ let col = if by_label {+ "label_embedding"+ } else {+ "embedding"+ };+ let rows: Vec<(Uuid, String, String, String, f64)> = sqlx::query_as(&format!(+ "SELECT id, key, label, coalesce(description, ''), ({col} <=> $2)::float8FROM entity_types t- WHERE t.kb_id = $1 AND t.embedding IS NOT NULL+ WHERE t.kb_id = $1 AND t.{col} IS NOT NULLAND (coalesce(btrim(t.description), '') <> ''OR EXISTS (SELECT 1 FROM entity_type_parents pWHERE p.child_id = t.id OR p.parent_id = t.id))- ORDER BY embedding <=> $2- LIMIT $3",- )+ ORDER BY {col} <=> $2+ LIMIT $3"+ )).bind(kb_id).bind(Vector::from(embedding.to_vec())).bind(limit)diff --git a/migrations/0050_label_embedding.sql b/migrations/0050_label_embedding.sqlnew file mode 100644index 00000000..5afdab26--- /dev/null+++ b/migrations/0050_label_embedding.sql@@ -0,0 +1,30 @@+-- 类的**第二个**向量:只嵌 label,不带描述。+--+-- 类型消解对每个实体发两种查询(见 `type_resolution.rs`):一种是模型自己给的+-- 说法(`district. place`,短),一种是整段画像(长)。两种此前比的是同一批+-- `label + description` 向量——**查询分了两种形状,文档只有一种**。+--+-- 后果是短查询被同义反复的类接管。实测「杭州拱墅区」(specific_type = district)+-- 的短查询回来的是:+--+-- Map(0.308) / Park(0.345) / Country(0.347) / Museum(0.356)+--+-- 四个赢家的嵌入原文分别是 `Map\nA map.`、`Park\nA park.`、`Country\nA country.`、+-- `Museum\nA museum.`——全是一行同义反复。而正确答案 AdministrativeArea 的原文是+-- `AdministrativeArea\nA geographical region, typically under the jurisdiction of a+-- particular government.`,一百字,排不进前八。+--+-- 不是语义更近,是**长度对上了**。整体测得:被检索出来的类,嵌入原文长度中位数+-- 44,而全部 965 个类的中位数是 89——检索系统性地偏好短文本。+--+-- 「短的那一侧距离系统性地更小」这条规律,本仓库已经栽过四次(跨实体不可比、+-- 两路之间不可比、同一路两个查询之间不可比、空描述的悬空类占便宜)。前四次修的+-- 都是查询一侧,这一次是文档一侧:**两种查询就该有两份文档**,短对短、长对长。+ALTER TABLE entity_types ADD COLUMN label_embedding vector;++-- 与 `embedded_text` / `embedded_model` 同一套自愈判据(见 0038):比对「当时嵌的+-- 原文与模型名跟现在对不对得上」,而不是看时间戳。改了 label、换了嵌入模型都会+-- 当场对不上,于是自动重嵌——不必在每个改 label 的写入点挂钩子,漏挂一个就悄悄烂掉。+ALTER TABLE entity_types+ ADD COLUMN label_embedded_text text,+ ADD COLUMN label_embedded_model text;diff --git a/scripts/bench/run.mjs b/scripts/bench/run.mjsindex 614028bf..97ac42bf 100644--- a/scripts/bench/run.mjs+++ b/scripts/bench/run.mjs@@ -178,8 +178,12 @@ async function main() {// 分钟,分不清是在跑还是卡死了。await until(async () => {+ // **两份向量都要等**(0050)。只等 `embedding` 的话,label 那份还没补完+ // 就开跑,短说法那一路一条都检索不到——测出来的是个半成品,而且看不出来const left = num(- "SELECT count(*) FILTER (WHERE embedding IS NULL) FROM entity_types WHERE kb_id='" ++ "SELECT count(*) FILTER (WHERE embedding IS NULL)" ++ " + count(*) FILTER (WHERE label_embedding IS NULL)" ++ " FROM entity_types WHERE kb_id='" +kb +"'",);