World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Nearest neighbours break distance ties by id (#654)
WaylandYangb7162480953b
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rsindex 652e9214..0bafb544 100644--- a/crates/utopia-store/src/documents.rs+++ b/crates/utopia-store/src/documents.rs@@ -1222,7 +1222,11 @@ pub async fn set_embeddings(pool: &PgPool, items: &[(Uuid, Vec<f32>)]) -> AppRes}/// 向量近邻检索(余弦距离)。维度写成字面量、两侧 cast、`relaxed_order`——三条-/// 规矩见 `vector_index`;走不走索引由规划器定+/// 规矩见 `vector_index`;走不走索引由规划器定。+///+/// 次序在外层再排一遍(`vector_index::RESORT`):`relaxed_order` 下索引给的次序+/// 只是大致按距离,而距离并列时精确路径和 HNSW 各排各的——同一问两种计划回的+/// 居首不同(#652)。并列由 id 定pub async fn vector_search(pool: &PgPool,kb_id: Uuid,@@ -1238,14 +1242,18 @@ pub async fn vector_search(let mut tx = pool.begin().await?;crate::vector_index::relaxed_order(pool, &mut tx).await?;let rows: Vec<(Uuid,)> = sqlx::query_as(&format!(- "SELECT id FROM chunks c- WHERE c.kb_id = $1 AND c.embedding IS NOT NULL AND {live}- AND {same_dims}- ORDER BY {distance}- LIMIT $3",+ "WITH nearest AS MATERIALIZED (+ SELECT id, {distance} AS distance FROM chunks c+ WHERE c.kb_id = $1 AND c.embedding IS NOT NULL AND {live}+ AND {same_dims}+ ORDER BY {distance}+ LIMIT $3+ )+ SELECT id FROM nearest ORDER BY {resort}",live = crate::record_axis::chunk_live_at("c", 4),same_dims = crate::vector_index::same_dims("c.embedding", dims),distance = crate::vector_index::distance("c.embedding", 2, dims),+ resort = crate::vector_index::RESORT,)).bind(kb_id).bind(&query_vec)diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rsindex eee15853..d68c1c2d 100644--- a/crates/utopia-store/src/resolution.rs+++ b/crates/utopia-store/src/resolution.rs@@ -2659,23 +2659,29 @@ pub async fn nearest_typed_entities(SELECT DISTINCT ev.document_id FROM fact_evidence evJOIN facts f ON f.id = ev.fact_idWHERE f.kb_id = $1 AND (f.subject_id = $2 OR f.object_id = $2)+ ),+ nearest AS MATERIALIZED (+ SELECT e.id, e.canonical_name, t.id AS type_id, t.key,+ ({distance})::float8 AS distance,+ EXISTS (SELECT 1 FROM fact_evidence ev2+ JOIN facts f2 ON f2.id = ev2.fact_id+ WHERE f2.kb_id = $1 AND (f2.subject_id = e.id OR f2.object_id = e.id)+ AND ev2.document_id IN (SELECT document_id FROM my_docs))+ AS same_document+ -- 内连接就是那道门:没判出类型的实体(type_id IS NULL)不是答案,+ -- 拿它当邻居的证据只会把「没判出来」传染开+ FROM entities e+ JOIN entity_types t ON t.id = e.type_id+ WHERE e.kb_id = $1 AND e.merged_into IS NULL AND e.id <> $2+ AND e.profile_embedding IS NOT NULL AND {same_dims}+ ORDER BY {distance}+ LIMIT $4)- SELECT e.canonical_name, t.id, t.key,- ({distance})::float8 AS distance,- EXISTS (SELECT 1 FROM fact_evidence ev2- JOIN facts f2 ON f2.id = ev2.fact_id- WHERE f2.kb_id = $1 AND (f2.subject_id = e.id OR f2.object_id = e.id)- AND ev2.document_id IN (SELECT document_id FROM my_docs))- AS same_document- -- 内连接就是那道门:没判出类型的实体(type_id IS NULL)不是答案,- -- 拿它当邻居的证据只会把「没判出来」传染开- FROM entities e- JOIN entity_types t ON t.id = e.type_id- WHERE e.kb_id = $1 AND e.merged_into IS NULL AND e.id <> $2- AND e.profile_embedding IS NOT NULL AND {same_dims}- ORDER BY {distance}- LIMIT $4",+ -- 次序在外层再排一遍,并列由实体 id 定(`vector_index::RESORT`,#652)+ SELECT canonical_name, type_id, key, distance, same_document+ FROM nearest ORDER BY {resort}",distance = crate::vector_index::distance("e.profile_embedding", 3, dims),+ resort = crate::vector_index::RESORT,same_dims = crate::vector_index::same_dims("e.profile_embedding", dims),)).bind(kb_id)diff --git a/crates/utopia-store/src/vector_index.rs b/crates/utopia-store/src/vector_index.rsindex 0c5ec49f..e114eb2a 100644--- a/crates/utopia-store/src/vector_index.rs+++ b/crates/utopia-store/src/vector_index.rs@@ -93,6 +93,14 @@ pub fn distance(column: &str, param: usize, dims: usize) -> String {format!("{column}::vector({dims}) <=> ${param}::vector({dims})")}+/// 近邻查询的外层次序(#652)。里层照规矩 2 写 `ORDER BY {distance} LIMIT n` 让索引+/// 接住,套进 `MATERIALIZED` 的 CTE(列名 `distance`、`id`),外层再按这个排:+/// - `relaxed_order` 下索引回的次序只是大致按距离,外层补一次真排序+/// - 距离并列时两种计划各排各的(一模一样的两条向量,精确路径和 HNSW 居首不同),id 定下来+/// - `+ 0` 不能省:没有它规划器认 CTE 的次序为已排(`Presorted Key: distance`),+/// 只在并列的组里排 id,乱序的那部分原样漏出去+pub const RESORT: &str = "distance + 0, id";+/// 索引现在的状态:`None` 没有;`Some(valid)` 有,`false` 是上次建到一半留下的pub async fn status(pool: &PgPool, target: Target, dims: usize) -> AppResult<Option<bool>> {let row: Option<(bool,)> = sqlx::query_as(diff --git a/crates/utopia-store/tests/the_nearest_chunk_is_found_however_it_is_reached.rs b/crates/utopia-store/tests/the_nearest_chunk_is_found_however_it_is_reached.rsindex e31558ba..8cf1de50 100644--- a/crates/utopia-store/tests/the_nearest_chunk_is_found_however_it_is_reached.rs+++ b/crates/utopia-store/tests/the_nearest_chunk_is_found_however_it_is_reached.rs@@ -236,8 +236,13 @@ async fn the_answers_agree_with_and_without_the_index() -> anyhow::Result<()> {);assert!(!exact.big_top_10_has_superseded, "顶掉的不是命中");assert!(!exact.big_top_10_has_other_dims, "7 维的查询看不见 5 维的");- // 三月一日:顶掉的那条还活着,它与查询重合,所以它或活着的那条居首都对——- // 两条向量一样,并列由 id 定。只钉「顶掉的那条回来了」这件事+ // 三月一日:顶掉的那条还活着,它与查询重合——两条向量一样,距离并列,由 id 定。+ // 不定的话精确路径和 HNSW 各排各的,下面「索引改了答案」就会时红时绿(#652)+ assert_eq!(+ exact.as_of_march_first,+ Some(f.planted.min(f.superseded)),+ "距离并列由 id 定"+ );let then = utopia_store::documents::vector_search(&pool,f.big,