Skip to content

World's first open-source enterprise world model.

fix(store): serialize vector index drops with builds (#908)

WaylandYanga9da69c94df9
diff --git a/crates/utopia-store/src/vector_index.rs b/crates/utopia-store/src/vector_index.rs
index 23f86e1f..c22180f9 100644
--- a/crates/utopia-store/src/vector_index.rs
+++ b/crates/utopia-store/src/vector_index.rs
@@ -27,7 +27,7 @@
//! 占表大头的库才走 HNSW(实测 6 万行:20 行和 1 万行的库走精确,5 万的走索引)。
//! 应用侧不设阈值——阈值是对规划器的猜测,猜错了两边都慢。
-use sqlx::{Executor, PgPool, Postgres, Transaction};
+use sqlx::{Executor, PgConnection, PgPool, Postgres, Transaction};
use std::collections::HashSet;
use std::sync::{Mutex, OnceLock};
use utopia_core::{AppError, AppResult};
@@ -45,6 +45,25 @@ const BUILD_UNLOCK: &str = "SELECT pg_advisory_unlock(hashtextextended('vector_i
/// 没抢到锁时隔多久再探。建一次索引几十秒到几分钟,四分之一秒的粒度够了
const BUILD_LOCK_POLL: std::time::Duration = std::time::Duration::from_millis(250);
+async fn lock_build(conn: &mut PgConnection) -> AppResult<()> {
+ loop {
+ let got: bool = sqlx::query_scalar(BUILD_TRY_LOCK)
+ .fetch_one(&mut *conn)
+ .await?;
+ if got {
+ return Ok(());
+ }
+ tokio::time::sleep(BUILD_LOCK_POLL).await;
+ }
+}
+
+async fn unlock_build(mut conn: sqlx::pool::PoolConnection<Postgres>) {
+ let unlocked: Result<bool, _> = sqlx::query_scalar(BUILD_UNLOCK).fetch_one(&mut *conn).await;
+ if !matches!(unlocked, Ok(true)) {
+ let _ = conn.close().await;
+ }
+}
+
/// pgvector 的 HNSW 对 `vector` 类型的上限。超过的维度(text-embedding-3-large
/// 是 3072)不建索引,查询照常走精确路径。`halfvec` 能到 4000,但那是另一种
/// 精度,等有人用到再说
@@ -209,15 +228,7 @@ pub async fn build(pool: &PgPool, target: Target, dims: usize) -> AppResult<Buil
// 构建任务被同时认领就是这个局面。锁是会话级的:CONCURRENTLY 不能进事务,事务级
// 咨询锁没处放;跟着这条连接走,跨 worker、跨实例都排队。建索引一次几十秒到几分钟,
// 排队比死锁后重试便宜
- loop {
- let got: bool = sqlx::query_scalar(BUILD_TRY_LOCK)
- .fetch_one(&mut *conn)
- .await?;
- if got {
- break;
- }
- tokio::time::sleep(BUILD_LOCK_POLL).await;
- }
+ lock_build(&mut conn).await?;
let outcome = async {
conn.execute("SET max_parallel_maintenance_workers = 0")
.await?;
@@ -250,11 +261,11 @@ pub async fn build(pool: &PgPool, target: Target, dims: usize) -> AppResult<Buil
// 还回池子:带着锁回池,之后所有构建都会卡在它后面
let _ = conn.execute("RESET max_parallel_maintenance_workers").await;
let _ = conn.execute("RESET maintenance_work_mem").await;
- if conn.execute(BUILD_UNLOCK).await.is_err() {
- let _ = conn.close().await;
+ if outcome.is_ok() {
+ remember(&name);
}
+ unlock_build(conn).await;
let created = outcome?;
- remember(&name);
Ok(Built {
name,
created,
@@ -265,10 +276,18 @@ pub async fn build(pool: &PgPool, target: Target, dims: usize) -> AppResult<Buil
/// 删掉(测试与手工维护用;写路径不会走到这里)
pub async fn drop(pool: &PgPool, target: Target, dims: usize) -> AppResult<()> {
let name = index_name(target, dims);
- forget(&name);
let mut conn = pool.acquire().await?;
- conn.execute(format!("DROP INDEX CONCURRENTLY IF EXISTS {name}").as_str())
- .await?;
+ // Dropping an index must take the same lock as building one: separate
+ // concurrent index operations on the table can deadlock each other.
+ lock_build(&mut conn).await?;
+ let outcome = conn
+ .execute(format!("DROP INDEX CONCURRENTLY IF EXISTS {name}").as_str())
+ .await;
+ if outcome.is_ok() {
+ forget(&name);
+ }
+ unlock_build(conn).await;
+ outcome?;
Ok(())
}
diff --git a/crates/utopia-store/tests/a_vector_index_is_built_by_a_job.rs b/crates/utopia-store/tests/a_vector_index_is_built_by_a_job.rs
index fc2838a4..33f9235d 100644
--- a/crates/utopia-store/tests/a_vector_index_is_built_by_a_job.rs
+++ b/crates/utopia-store/tests/a_vector_index_is_built_by_a_job.rs
@@ -23,6 +23,41 @@ async fn queued(pool: &PgPool, dims: usize) -> anyhow::Result<i64> {
.await?)
}
+#[tokio::test]
+async fn dropping_an_index_waits_for_a_build() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let mut holder = pool.acquire().await?;
+ let lock = "SELECT pg_try_advisory_lock(hashtextextended('vector_index:build', 0))";
+ loop {
+ if sqlx::query_scalar::<_, bool>(lock)
+ .fetch_one(&mut *holder)
+ .await?
+ {
+ break;
+ }
+ tokio::time::sleep(std::time::Duration::from_millis(10)).await;
+ }
+
+ // An absent index makes DROP CONCURRENTLY quick unless it takes the build lock.
+ let blocked = tokio::time::timeout(
+ std::time::Duration::from_millis(100),
+ vector_index::drop(&pool, Target::Chunks, 17),
+ )
+ .await
+ .is_err();
+ let unlocked: bool =
+ sqlx::query_scalar("SELECT pg_advisory_unlock(hashtextextended('vector_index:build', 0))")
+ .fetch_one(&mut *holder)
+ .await?;
+ assert!(unlocked);
+ assert!(blocked, "DROP must wait for an in-progress index build");
+ vector_index::drop(&pool, Target::Chunks, 17).await?;
+ Ok(())
+}
+
#[tokio::test]
async fn the_first_write_of_a_dimension_queues_one_build() -> anyhow::Result<()> {
let Some(url) = utopia_store::test_db::url() else {