Skip to content

World's first open-source enterprise world model.

Short type names get an index shaped like them, and responses arrive compressed (#116)

WaylandYangb16b54fc640c
diff --git a/Cargo.lock b/Cargo.lock
index ab52085e..9b919538 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -116,6 +116,18 @@ dependencies = [
"password-hash",
]
+[[package]]
+name = "async-compression"
+version = "0.4.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8"
+dependencies = [
+ "compression-codecs",
+ "compression-core",
+ "pin-project-lite",
+ "tokio",
+]
+
[[package]]
name = "async-stream"
version = "0.3.6"
@@ -587,6 +599,23 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "compression-codecs"
+version = "0.4.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf"
+dependencies = [
+ "compression-core",
+ "flate2",
+ "memchr",
+]
+
+[[package]]
+name = "compression-core"
+version = "0.4.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789"
+
[[package]]
name = "const-oid"
version = "0.9.6"
@@ -4466,6 +4495,7 @@ version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
dependencies = [
+ "async-compression",
"bitflags",
"bytes",
"futures-core",
diff --git a/Cargo.toml b/Cargo.toml
index 82d78b86..c68e3974 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -55,7 +55,11 @@ text-splitter = "0.32"
pdf-extract = "0.12"
calamine = "0.36"
pgvector = { version = "0.4", features = ["sqlx"] }
-reqwest = { version = "0.13", default-features = false, features = ["json", "stream", "http2", "rustls", "webpki-roots", "charset", "system-proxy"] }
+# `gzip` 不是可选项:一次嵌入回 64×1024 个浮点数,写成 JSON 文本每个数字占
+# 21 字节而信息量只有 4 字节,一次响应 1.33 MB。实测同一条链路上不压缩 62 秒、
+# gzip 21 秒(429 KB)——服务端算完只用 1 秒,其余全在传。
+# `default-features = false` 把它一并关掉了,这里补回来。
+reqwest = { version = "0.13", default-features = false, features = ["json", "stream", "http2", "rustls", "webpki-roots", "charset", "system-proxy", "gzip"] }
feed-rs = "2"
cron = "0.12"
diff --git a/crates/utopia-server/src/ontology_index.rs b/crates/utopia-server/src/ontology_index.rs
index eb94c5ba..b2ca146a 100644
--- a/crates/utopia-server/src/ontology_index.rs
+++ b/crates/utopia-server/src/ontology_index.rs
@@ -9,11 +9,31 @@
//! 而对比原文不会。
use crate::{llm_util, AppState};
+use futures_util::{stream, StreamExt};
use utopia_store::ontology::TypeToEmbed;
use uuid::Uuid;
-/// 一次送多少条去嵌入。跟文档分块那边同量级,够摊平往返又不至于把请求撑爆。
+/// 一次送多少条去嵌入。
+///
+/// **64 是实测站得住的那个数。** 曾经改成 16,理由是拿合成短文本量出来的
+/// "每条更快"——而真实的类文本平均 151 字符,那组数据不作数。改小之后请求数
+/// 翻四倍、每个都付约 2.5 秒固定开销,整体吞吐反而掉了一半。
+///
+/// 换批大小之前先拿**真实的 `embedded_text`** 去量,别拿造出来的短句。
const BATCH: usize = 64;
+/// 同时在飞的嵌入批数上限。
+///
+/// **关键不是快,是不能把闸门占死。** 这是个后台补齐任务,而嵌入模型的并发闸门
+/// (`model_concurrency`,缺省 10)是全局共享的——文档分块入库也要走它。
+///
+/// 从前这里串行,一次一张许可证、每批之间释放,文档处理插得进来。改成
+/// `join_all` 一次挂 31 个批次之后,它变成一个常驻的 10 深队列,每个批次握着
+/// 许可证 60 秒:**5 篇文档全卡在 `embedding` 状态,抽取 15 分钟零进展**。
+/// 不是 API 挂了,是这个任务把共享资源占满了。
+///
+/// 所以这里的上限必须**明显小于**闸门,给别的活留出位置。4 是实测能跑满
+/// 吞吐又留有余量的:4 个 16 条并发,墙钟 18.5 秒(串行要 50 秒)。
+const EMBED_JOBS: usize = 4;
/// 把这个库里陈掉的本体向量补齐。没配嵌入模型就直接返回 `Ok(0)`——
/// 检索是增强,不该拦住没配模型的部署。
@@ -38,31 +58,60 @@ pub async fn refresh_scoped(
};
let (Some(client), Some(model)) = (
llm_util::embed_client(&settings),
- settings.embed_model.as_deref(),
+ // 取拥有的一份:下面那些 future 要把 settings 挪进 Arc,借着它就动不了了
+ settings.embed_model.clone(),
) else {
return Ok(0);
};
let stale =
- utopia_store::ontology::types_needing_embedding(&state.pool, kb_id, model, only).await?;
+ utopia_store::ontology::types_needing_embedding(&state.pool, kb_id, &model, only).await?;
if stale.is_empty() {
return Ok(0);
}
let mut done = 0usize;
- for batch in stale.chunks(BATCH) {
- let texts: Vec<String> = batch.iter().map(|t| t.text.clone()).collect();
- let _permit = llm_util::acquire_embed(state, &settings).await;
- let vectors = client.embed(&texts).await?;
- // 数量对不上就整批放弃:按位置配对,错位会把 person 的向量写到
- // organization 上,而这种错一旦落库就再也看不出来了
- if vectors.len() != batch.len() {
- anyhow::bail!("嵌入返回 {} 条,送去的是 {} 条", vectors.len(), batch.len());
+ let mut failed = 0usize;
+ // **全部用拥有的数据**:这些 future 要跨 await 被并发驱动,借用过不了 Send 边界
+ let client = std::sync::Arc::new(client);
+ let model = std::sync::Arc::new(model);
+ let settings = std::sync::Arc::new(settings);
+ let batches: Vec<Vec<TypeToEmbed>> = stale.chunks(BATCH).map(<[_]>::to_vec).collect();
+ let mut jobs = stream::iter(batches.into_iter().map(|batch| {
+ let (client, model, settings) = (client.clone(), model.clone(), settings.clone());
+ async move {
+ let texts: Vec<String> = batch.iter().map(|t| t.text.clone()).collect();
+ let _permit = llm_util::acquire_embed(state, &settings).await;
+ let vectors = client.embed(&texts).await?;
+ // 数量对不上就整批放弃:按位置配对,错位会把 person 的向量写到
+ // organization 上,而这种错一旦落库就再也看不出来了
+ if vectors.len() != batch.len() {
+ anyhow::bail!("嵌入返回 {} 条,送去的是 {} 条", vectors.len(), batch.len());
+ }
+ let n = batch.len();
+ let items: Vec<(TypeToEmbed, Vec<f32>)> = batch.into_iter().zip(vectors).collect();
+ utopia_store::ontology::set_type_embeddings(&state.pool, &model, &items).await?;
+ Ok::<usize, anyhow::Error>(n)
}
- let items: Vec<(TypeToEmbed, Vec<f32>)> = batch.iter().cloned().zip(vectors).collect();
- utopia_store::ontology::set_type_embeddings(&state.pool, model, &items).await?;
- done += batch.len();
+ }))
+ .buffer_unordered(EMBED_JOBS);
+ // **一批失败不拖垮其余**:这是补齐任务,缺的下一轮自愈(判据是"当时嵌的
+ // 原文对不对得上",不是时间戳)。整个 bail 掉等于把已经嵌好的也白跑一遍
+ while let Some(r) = jobs.next().await {
+ match r {
+ Ok(n) => done += n,
+ Err(e) => {
+ failed += 1;
+ tracing::warn!(%kb_id, error = %e, "一批本体向量没嵌上,留给下一轮");
+ }
+ }
+ }
+ // **失败数要出现在这一行。** 从前它只报补齐了多少,于是 24 批全挂的那次
+ // 日志照样写着"已补齐",看日志的人以为没事
+ if failed > 0 {
+ tracing::warn!(%kb_id, count = done, failed, "本体向量补了一部分,其余留给下一轮");
+ } else {
+ tracing::info!(%kb_id, count = done, "本体向量已补齐");
}
- tracing::info!(%kb_id, count = done, "本体向量已补齐");
Ok(done)
}
@@ -101,7 +150,12 @@ pub async fn nearest_for_each(
for v in &vectors {
out.push(match target {
Target::Class => {
- utopia_store::ontology::nearest_entity_types(&state.pool, kb_id, v, limit).await?
+ utopia_store::ontology::nearest_entity_types(&state.pool, kb_id, v, limit, false)
+ .await?
+ }
+ Target::ClassLabel => {
+ utopia_store::ontology::nearest_entity_types(&state.pool, kb_id, v, limit, true)
+ .await?
}
Target::Predicate(kind) => {
utopia_store::ontology::nearest_relation_types(&state.pool, kb_id, v, limit, kind)
@@ -119,7 +173,12 @@ pub async fn nearest_for_each(
/// 拿一个词表外的类名去关系里检索,回来的一定是勉强相关的关系。
#[derive(Debug, Clone, Copy)]
pub enum Target {
+ /// 整段索引(label + 描述)。长画像走这一路
Class,
+ /// **只有 label 的索引**(0050)。短说法(`district. place`)走这一路。
+ /// 短说法比整段索引会被同义反复的类接管——`Map` 那一行的描述就是
+ /// "A map.",赢在长度不在语义
+ ClassLabel,
/// `Some("attribute")` 只找属性、`Some("relation")` 只找关系、`None` 都找。
/// 字面值宾语的事实要的是属性,实体宾语的要的是关系
Predicate(Option<&'static str>),
diff --git a/crates/utopia-server/src/type_resolution.rs b/crates/utopia-server/src/type_resolution.rs
index 19ed622c..165cceb0 100644
--- a/crates/utopia-server/src/type_resolution.rs
+++ b/crates/utopia-server/src/type_resolution.rs
@@ -39,8 +39,14 @@ use uuid::Uuid;
/// 一轮看多少个实体。够一次人工过目,也够看出检索准不准。
const BATCH: i64 = 60;
-/// 每个实体检索几个候选。给裁决看的,不是让它在一堆勉强相关里硬挑一个。
-const CANDIDATES: i64 = 8;
+/// 每个实体检索几个候选。
+///
+/// **检索的活是端候选,说不是裁决的活。** 所以宁可多端两个——裁决看得到描述、
+/// 看得到现类,能说出"这些都不是";而检索漏掉的,裁决再好也够不着。0001 P3a
+/// 实测的失败正是全在检索一侧(`administrative_area`、`periodical` 从没被端上来)。
+///
+/// 从 8 加到 10:两路交替各占五席。代价是提示词里多两行类定义,而漏检没有退路。
+const CANDIDATES: i64 = 10;
/// 看几个已定类的近邻。少了凑不出票数,多了尾巴上全是噪音。
const NEIGHBOURS: i64 = 10;
@@ -123,28 +129,42 @@ pub async fn preview(state: &AppState, kb_id: Uuid) -> AppResult<Vec<TypeSuggest
// 两次结果取并集——多一次嵌入,换掉一整类漏检。
let profiles: Vec<String> = subjects.iter().map(profile_of).collect();
let names: Vec<Option<String>> = subjects.iter().map(name_query_of).collect();
- let mut queries: Vec<String> = Vec::with_capacity(subjects.len() * 2);
- // 每个实体在 queries 里占的下标:(画像, 名字)
+ // **两路各查各的索引**(0050)。从前两路共用整段索引,于是短说法那一路
+ // 被同义反复的类接管:`district. place` 回来的是 Map / Park / Country /
+ // Museum,四个赢家的嵌入原文全是 `X\nAn X.` 一行话,而带一百字定义的
+ // AdministrativeArea 排不进前八。短对短、长对长,这才是可比的
+ let name_queries: Vec<String> = names.iter().flatten().cloned().collect();
+ let (profile_hits, name_hits) = tokio::join!(
+ ontology_index::nearest_for_each(
+ state,
+ kb_id,
+ &profiles,
+ // 多取一些再按祖先过滤:过滤在检索之后,所以要留出被滤掉的余量
+ CANDIDATES * 4,
+ ontology_index::Target::Class,
+ ),
+ ontology_index::nearest_for_each(
+ state,
+ kb_id,
+ &name_queries,
+ CANDIDATES * 4,
+ ontology_index::Target::ClassLabel,
+ )
+ );
+ let profile_hits = profile_hits.unwrap_or_default();
+ let name_hits = name_hits.unwrap_or_default();
+ // 每个实体在两份结果里各占的下标:(画像, 名字)。名字那一路只有给得出
+ // 说法的实体才发了查询,所以下标要单独数
let mut slots: Vec<(usize, Option<usize>)> = Vec::with_capacity(subjects.len());
- for (p, n) in profiles.iter().zip(&names) {
- let pi = queries.len();
- queries.push(p.clone());
- let ni = n.as_ref().map(|q| {
- queries.push(q.clone());
- queries.len() - 1
+ let mut ni = 0usize;
+ for (pi, n) in names.iter().enumerate() {
+ let slot = n.as_ref().map(|_| {
+ let k = ni;
+ ni += 1;
+ k
});
- slots.push((pi, ni));
+ slots.push((pi, slot));
}
- let hits = ontology_index::nearest_for_each(
- state,
- kb_id,
- &queries,
- // 多取一些再按祖先过滤:过滤在检索之后,所以要留出被滤掉的余量
- CANDIDATES * 4,
- ontology_index::Target::Class,
- )
- .await
- .unwrap_or_default();
let mut out = Vec::with_capacity(subjects.len());
for (i, s) in subjects.iter().enumerate() {
@@ -176,11 +196,10 @@ pub async fn preview(state: &AppState, kb_id: Uuid) -> AppResult<Vec<TypeSuggest
//
// 交替之后两路各占一半席位,谁的距离数值大小不再影响谁被看见。
let (pi, ni) = slots[i];
- let lists: Vec<Vec<_>> = [Some(pi), ni]
- .into_iter()
- .flatten()
- .map(|idx| hits.get(idx).cloned().unwrap_or_default())
- .collect();
+ let mut lists: Vec<Vec<_>> = vec![profile_hits.get(pi).cloned().unwrap_or_default()];
+ if let Some(k) = ni {
+ lists.push(name_hits.get(k).cloned().unwrap_or_default());
+ }
let mut seen_ids: std::collections::HashSet<Uuid> = std::collections::HashSet::new();
let mut ranked: Vec<utopia_core::models::TypeCandidate> = Vec::new();
let longest = lists.iter().map(Vec::len).max().unwrap_or(0);
diff --git a/crates/utopia-store/src/ontology.rs b/crates/utopia-store/src/ontology.rs
index 57f161bf..c2cdfcb5 100644
--- a/crates/utopia-store/src/ontology.rs
+++ b/crates/utopia-store/src/ontology.rs
@@ -826,6 +826,17 @@ pub struct TypeToEmbed {
pub id: Uuid,
pub kind: TypeKind,
pub text: String,
+ /// 写进哪一组列。类有两份向量:整段(label + 描述)与只有 label 的那份,
+ /// 分别服务长画像与短说法两种查询(见迁移 0050)
+ pub field: EmbedField,
+}
+
+/// 同一行的两份向量。**短查询比 Label,长画像比 Full**——查询分了两种形状,
+/// 文档也得分两种,否则短查询会被同义反复的类接管(`Map\nA map.` 那一类)。
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EmbedField {
+ Full,
+ Label,
}
/// 本体行属于哪张表。类进 `entity_types`,关系与属性同住 `relation_types`。
@@ -884,6 +895,26 @@ pub async fn types_needing_embedding(
id,
kind: TypeKind::Entity,
text: embed_text(&label, &desc),
+ field: EmbedField::Full,
+ }));
+ // 只嵌 label 的那一份(0050)。短查询走这个索引——查询分了两种形状,
+ // 文档也得分两种,否则短查询被同义反复的类接管
+ let labels: Vec<(Uuid, String)> = sqlx::query_as(
+ "SELECT id, label FROM entity_types
+ WHERE kb_id = $1
+ AND (label_embedding IS NULL
+ OR label_embedded_model IS DISTINCT FROM $2
+ OR label_embedded_text IS DISTINCT FROM btrim(label))",
+ )
+ .bind(kb_id)
+ .bind(model)
+ .fetch_all(pool)
+ .await?;
+ out.extend(labels.into_iter().map(|(id, label)| TypeToEmbed {
+ id,
+ kind: TypeKind::Entity,
+ text: label.trim().to_string(),
+ field: EmbedField::Label,
}));
if only == Some(TypeKind::Entity) {
return Ok(out);
@@ -915,6 +946,8 @@ async fn relations_needing_embedding(
id,
kind: TypeKind::Relation,
text: embed_text(&label, &desc),
+ // 关系没有短查询那一路,只有整段这一份
+ field: EmbedField::Full,
}));
Ok(out)
}
@@ -932,8 +965,17 @@ pub async fn set_type_embeddings(
TypeKind::Entity => "entity_types",
TypeKind::Relation => "relation_types",
};
+ // 两份向量各写各的列(0050)。列名前缀不同,其余一模一样
+ let (vec_col, text_col, model_col) = match item.field {
+ EmbedField::Full => ("embedding", "embedded_text", "embedded_model"),
+ EmbedField::Label => (
+ "label_embedding",
+ "label_embedded_text",
+ "label_embedded_model",
+ ),
+ };
sqlx::query(&format!(
- "UPDATE {table} SET embedding = $2, embedded_text = $3, embedded_model = $4
+ "UPDATE {table} SET {vec_col} = $2, {text_col} = $3, {model_col} = $4
WHERE id = $1"
))
.bind(item.id)
@@ -967,17 +1009,25 @@ pub async fn nearest_entity_types(
kb_id: Uuid,
embedding: &[f32],
limit: i64,
+ // true = 比只有 label 的那份向量(0050)。短说法走这一路:**短对短**,
+ // 否则 `district. place` 会被 `Map\nA map.` 这类同义反复的一行话赢过去
+ by_label: bool,
) -> AppResult<Vec<TypeCandidate>> {
- let rows: Vec<(Uuid, String, String, String, f64)> = sqlx::query_as(
- "SELECT id, key, label, coalesce(description, ''), (embedding <=> $2)::float8
+ let col = if by_label {
+ "label_embedding"
+ } else {
+ "embedding"
+ };
+ let rows: Vec<(Uuid, String, String, String, f64)> = sqlx::query_as(&format!(
+ "SELECT id, key, label, coalesce(description, ''), ({col} <=> $2)::float8
FROM entity_types t
- WHERE t.kb_id = $1 AND t.embedding IS NOT NULL
+ WHERE t.kb_id = $1 AND t.{col} IS NOT NULL
AND (coalesce(btrim(t.description), '') <> ''
OR EXISTS (SELECT 1 FROM entity_type_parents p
WHERE p.child_id = t.id OR p.parent_id = t.id))
- ORDER BY embedding <=> $2
- LIMIT $3",
- )
+ ORDER BY {col} <=> $2
+ LIMIT $3"
+ ))
.bind(kb_id)
.bind(Vector::from(embedding.to_vec()))
.bind(limit)
diff --git a/migrations/0050_label_embedding.sql b/migrations/0050_label_embedding.sql
new file mode 100644
index 00000000..5afdab26
--- /dev/null
+++ b/migrations/0050_label_embedding.sql
@@ -0,0 +1,30 @@
+-- 类的**第二个**向量:只嵌 label,不带描述。
+--
+-- 类型消解对每个实体发两种查询(见 `type_resolution.rs`):一种是模型自己给的
+-- 说法(`district. place`,短),一种是整段画像(长)。两种此前比的是同一批
+-- `label + description` 向量——**查询分了两种形状,文档只有一种**。
+--
+-- 后果是短查询被同义反复的类接管。实测「杭州拱墅区」(specific_type = district)
+-- 的短查询回来的是:
+--
+-- Map(0.308) / Park(0.345) / Country(0.347) / Museum(0.356)
+--
+-- 四个赢家的嵌入原文分别是 `Map\nA map.`、`Park\nA park.`、`Country\nA country.`、
+-- `Museum\nA museum.`——全是一行同义反复。而正确答案 AdministrativeArea 的原文是
+-- `AdministrativeArea\nA geographical region, typically under the jurisdiction of a
+-- particular government.`,一百字,排不进前八。
+--
+-- 不是语义更近,是**长度对上了**。整体测得:被检索出来的类,嵌入原文长度中位数
+-- 44,而全部 965 个类的中位数是 89——检索系统性地偏好短文本。
+--
+-- 「短的那一侧距离系统性地更小」这条规律,本仓库已经栽过四次(跨实体不可比、
+-- 两路之间不可比、同一路两个查询之间不可比、空描述的悬空类占便宜)。前四次修的
+-- 都是查询一侧,这一次是文档一侧:**两种查询就该有两份文档**,短对短、长对长。
+ALTER TABLE entity_types ADD COLUMN label_embedding vector;
+
+-- 与 `embedded_text` / `embedded_model` 同一套自愈判据(见 0038):比对「当时嵌的
+-- 原文与模型名跟现在对不对得上」,而不是看时间戳。改了 label、换了嵌入模型都会
+-- 当场对不上,于是自动重嵌——不必在每个改 label 的写入点挂钩子,漏挂一个就悄悄烂掉。
+ALTER TABLE entity_types
+ ADD COLUMN label_embedded_text text,
+ ADD COLUMN label_embedded_model text;
diff --git a/scripts/bench/run.mjs b/scripts/bench/run.mjs
index 614028bf..97ac42bf 100644
--- a/scripts/bench/run.mjs
+++ b/scripts/bench/run.mjs
@@ -178,8 +178,12 @@ async function main() {
// 分钟,分不清是在跑还是卡死了。
await until(
async () => {
+ // **两份向量都要等**(0050)。只等 `embedding` 的话,label 那份还没补完
+ // 就开跑,短说法那一路一条都检索不到——测出来的是个半成品,而且看不出来
const left = num(
- "SELECT count(*) FILTER (WHERE embedding IS NULL) FROM entity_types WHERE kb_id='" +
+ "SELECT count(*) FILTER (WHERE embedding IS NULL)" +
+ " + count(*) FILTER (WHERE label_embedding IS NULL)" +
+ " FROM entity_types WHERE kb_id='" +
kb +
"'",
);