Skip to content

World's first open-source enterprise world model.

Nearest neighbours break distance ties by id (#654)

WaylandYangb7162480953b
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index 652e9214..0bafb544 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -1222,7 +1222,11 @@ pub async fn set_embeddings(pool: &PgPool, items: &[(Uuid, Vec<f32>)]) -> AppRes
}
/// 向量近邻检索(余弦距离)。维度写成字面量、两侧 cast、`relaxed_order`——三条
-/// 规矩见 `vector_index`;走不走索引由规划器定
+/// 规矩见 `vector_index`;走不走索引由规划器定。
+///
+/// 次序在外层再排一遍(`vector_index::RESORT`):`relaxed_order` 下索引给的次序
+/// 只是大致按距离,而距离并列时精确路径和 HNSW 各排各的——同一问两种计划回的
+/// 居首不同(#652)。并列由 id 定
pub async fn vector_search(
pool: &PgPool,
kb_id: Uuid,
@@ -1238,14 +1242,18 @@ pub async fn vector_search(
let mut tx = pool.begin().await?;
crate::vector_index::relaxed_order(pool, &mut tx).await?;
let rows: Vec<(Uuid,)> = sqlx::query_as(&format!(
- "SELECT id FROM chunks c
- WHERE c.kb_id = $1 AND c.embedding IS NOT NULL AND {live}
- AND {same_dims}
- ORDER BY {distance}
- LIMIT $3",
+ "WITH nearest AS MATERIALIZED (
+ SELECT id, {distance} AS distance FROM chunks c
+ WHERE c.kb_id = $1 AND c.embedding IS NOT NULL AND {live}
+ AND {same_dims}
+ ORDER BY {distance}
+ LIMIT $3
+ )
+ SELECT id FROM nearest ORDER BY {resort}",
live = crate::record_axis::chunk_live_at("c", 4),
same_dims = crate::vector_index::same_dims("c.embedding", dims),
distance = crate::vector_index::distance("c.embedding", 2, dims),
+ resort = crate::vector_index::RESORT,
))
.bind(kb_id)
.bind(&query_vec)
diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs
index eee15853..d68c1c2d 100644
--- a/crates/utopia-store/src/resolution.rs
+++ b/crates/utopia-store/src/resolution.rs
@@ -2659,23 +2659,29 @@ pub async fn nearest_typed_entities(
SELECT DISTINCT ev.document_id FROM fact_evidence ev
JOIN facts f ON f.id = ev.fact_id
WHERE f.kb_id = $1 AND (f.subject_id = $2 OR f.object_id = $2)
+ ),
+ nearest AS MATERIALIZED (
+ SELECT e.id, e.canonical_name, t.id AS type_id, t.key,
+ ({distance})::float8 AS distance,
+ EXISTS (SELECT 1 FROM fact_evidence ev2
+ JOIN facts f2 ON f2.id = ev2.fact_id
+ WHERE f2.kb_id = $1 AND (f2.subject_id = e.id OR f2.object_id = e.id)
+ AND ev2.document_id IN (SELECT document_id FROM my_docs))
+ AS same_document
+ -- 内连接就是那道门:没判出类型的实体(type_id IS NULL)不是答案,
+ -- 拿它当邻居的证据只会把「没判出来」传染开
+ FROM entities e
+ JOIN entity_types t ON t.id = e.type_id
+ WHERE e.kb_id = $1 AND e.merged_into IS NULL AND e.id <> $2
+ AND e.profile_embedding IS NOT NULL AND {same_dims}
+ ORDER BY {distance}
+ LIMIT $4
)
- SELECT e.canonical_name, t.id, t.key,
- ({distance})::float8 AS distance,
- EXISTS (SELECT 1 FROM fact_evidence ev2
- JOIN facts f2 ON f2.id = ev2.fact_id
- WHERE f2.kb_id = $1 AND (f2.subject_id = e.id OR f2.object_id = e.id)
- AND ev2.document_id IN (SELECT document_id FROM my_docs))
- AS same_document
- -- 内连接就是那道门:没判出类型的实体(type_id IS NULL)不是答案,
- -- 拿它当邻居的证据只会把「没判出来」传染开
- FROM entities e
- JOIN entity_types t ON t.id = e.type_id
- WHERE e.kb_id = $1 AND e.merged_into IS NULL AND e.id <> $2
- AND e.profile_embedding IS NOT NULL AND {same_dims}
- ORDER BY {distance}
- LIMIT $4",
+ -- 次序在外层再排一遍,并列由实体 id 定(`vector_index::RESORT`,#652)
+ SELECT canonical_name, type_id, key, distance, same_document
+ FROM nearest ORDER BY {resort}",
distance = crate::vector_index::distance("e.profile_embedding", 3, dims),
+ resort = crate::vector_index::RESORT,
same_dims = crate::vector_index::same_dims("e.profile_embedding", dims),
))
.bind(kb_id)
diff --git a/crates/utopia-store/src/vector_index.rs b/crates/utopia-store/src/vector_index.rs
index 0c5ec49f..e114eb2a 100644
--- a/crates/utopia-store/src/vector_index.rs
+++ b/crates/utopia-store/src/vector_index.rs
@@ -93,6 +93,14 @@ pub fn distance(column: &str, param: usize, dims: usize) -> String {
format!("{column}::vector({dims}) <=> ${param}::vector({dims})")
}
+/// 近邻查询的外层次序(#652)。里层照规矩 2 写 `ORDER BY {distance} LIMIT n` 让索引
+/// 接住,套进 `MATERIALIZED` 的 CTE(列名 `distance`、`id`),外层再按这个排:
+/// - `relaxed_order` 下索引回的次序只是大致按距离,外层补一次真排序
+/// - 距离并列时两种计划各排各的(一模一样的两条向量,精确路径和 HNSW 居首不同),id 定下来
+/// - `+ 0` 不能省:没有它规划器认 CTE 的次序为已排(`Presorted Key: distance`),
+/// 只在并列的组里排 id,乱序的那部分原样漏出去
+pub const RESORT: &str = "distance + 0, id";
+
/// 索引现在的状态:`None` 没有;`Some(valid)` 有,`false` 是上次建到一半留下的
pub async fn status(pool: &PgPool, target: Target, dims: usize) -> AppResult<Option<bool>> {
let row: Option<(bool,)> = sqlx::query_as(
diff --git a/crates/utopia-store/tests/the_nearest_chunk_is_found_however_it_is_reached.rs b/crates/utopia-store/tests/the_nearest_chunk_is_found_however_it_is_reached.rs
index e31558ba..8cf1de50 100644
--- a/crates/utopia-store/tests/the_nearest_chunk_is_found_however_it_is_reached.rs
+++ b/crates/utopia-store/tests/the_nearest_chunk_is_found_however_it_is_reached.rs
@@ -236,8 +236,13 @@ async fn the_answers_agree_with_and_without_the_index() -> anyhow::Result<()> {
);
assert!(!exact.big_top_10_has_superseded, "顶掉的不是命中");
assert!(!exact.big_top_10_has_other_dims, "7 维的查询看不见 5 维的");
- // 三月一日:顶掉的那条还活着,它与查询重合,所以它或活着的那条居首都对——
- // 两条向量一样,并列由 id 定。只钉「顶掉的那条回来了」这件事
+ // 三月一日:顶掉的那条还活着,它与查询重合——两条向量一样,距离并列,由 id 定。
+ // 不定的话精确路径和 HNSW 各排各的,下面「索引改了答案」就会时红时绿(#652)
+ assert_eq!(
+ exact.as_of_march_first,
+ Some(f.planted.min(f.superseded)),
+ "距离并列由 id 定"
+ );
let then = utopia_store::documents::vector_search(
&pool,
f.big,