World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Serialize vector index builds so two concurrent CREATE INDEX CONCURRENTLY cannot deadlock (#887)
WaylandYangfb3c4c1d468d
diff --git a/crates/utopia-store/src/vector_index.rs b/crates/utopia-store/src/vector_index.rsindex 8a3a7d15..17f9d5fc 100644--- a/crates/utopia-store/src/vector_index.rs+++ b/crates/utopia-store/src/vector_index.rs@@ -35,6 +35,16 @@ use utopia_core::{AppError, AppResult};/// 任务种类,`main.rs` 的分发按这个名字认pub const JOB_KIND: &str = "build_vector_index";+/// 建索引的会话级咨询锁(见 [`build`])。key 用字符串哈希,和 temporal 里的时间线锁同一套写法。+/// 只用 **try** 版本:阻塞的 `pg_advisory_lock` 等锁时那条语句自己就是一个带快照的事务,+/// 而 CONCURRENTLY 建到最后一步要等所有比它老的快照结束——建的等排队的、排队的等建的,+/// 换了个地方死锁(本地复现每轮必中)。探一下就返回、不留快照,等待放在客户端+const BUILD_TRY_LOCK: &str =+ "SELECT pg_try_advisory_lock(hashtextextended('vector_index:build', 0))";+const BUILD_UNLOCK: &str = "SELECT pg_advisory_unlock(hashtextextended('vector_index:build', 0))";+/// 没抢到锁时隔多久再探。建一次索引几十秒到几分钟,四分之一秒的粒度够了+const BUILD_LOCK_POLL: std::time::Duration = std::time::Duration::from_millis(250);+/// pgvector 的 HNSW 对 `vector` 类型的上限。超过的维度(text-embedding-3-large/// 是 3072)不建索引,查询照常走精确路径。`halfvec` 能到 4000,但那是另一种/// 精度,等有人用到再说@@ -188,6 +198,21 @@ pub async fn build(pool: &PgPool, target: Target, dims: usize) -> AppResult<Buillet name = index_name(target, dims);let started = std::time::Instant::now();let mut conn = pool.acquire().await?;+ // 一次只建一条。同一张表上两条 CONCURRENTLY 各自要等表上其他事务结束、也各自算+ // 对方要等的事务,旁边再有摄取往 chunks 写就凑成死锁(40P01;复现在+ // a_vector_index_is_built_by_a_job,一轮约一半概率)。worker 并发默认 64,两个维度的+ // 构建任务被同时认领就是这个局面。锁是会话级的:CONCURRENTLY 不能进事务,事务级+ // 咨询锁没处放;跟着这条连接走,跨 worker、跨实例都排队。建索引一次几十秒到几分钟,+ // 排队比死锁后重试便宜+ loop {+ let got: bool = sqlx::query_scalar(BUILD_TRY_LOCK)+ .fetch_one(&mut *conn)+ .await?;+ if got {+ break;+ }+ tokio::time::sleep(BUILD_LOCK_POLL).await;+ }let outcome = async {conn.execute("SET max_parallel_maintenance_workers = 0").await?;@@ -216,9 +241,13 @@ pub async fn build(pool: &PgPool, target: Target, dims: usize) -> AppResult<BuilOk::<bool, AppError>(!existed)}.await;- // 会话级 SET 跟着连接回池,成败都复位+ // 会话级 SET 跟着连接回池,成败都复位。锁也一样——解不掉就把这条连接关掉而不是+ // 还回池子:带着锁回池,之后所有构建都会卡在它后面let _ = conn.execute("RESET max_parallel_maintenance_workers").await;let _ = conn.execute("RESET maintenance_work_mem").await;+ if conn.execute(BUILD_UNLOCK).await.is_err() {+ let _ = conn.close().await;+ }let created = outcome?;remember(&name);Ok(Built {diff --git a/crates/utopia-store/tests/a_vector_index_is_built_by_a_job.rs b/crates/utopia-store/tests/a_vector_index_is_built_by_a_job.rsindex 04d9dfb3..fc2838a4 100644--- a/crates/utopia-store/tests/a_vector_index_is_built_by_a_job.rs+++ b/crates/utopia-store/tests/a_vector_index_is_built_by_a_job.rs@@ -88,3 +88,57 @@ async fn the_first_write_of_a_dimension_queues_one_build() -> anyhow::Result<()>.await?;run}++/// 两个维度同时建,旁边还有事务在碰 `chunks`。两条 `CREATE INDEX CONCURRENTLY` 各自要等+/// 表上其他事务结束,也各自算对方要等的事务,于是 Postgres 报 deadlock detected:本地复现+/// 一轮约一半概率,错开 10ms 以上就不会(#886 把测试合成一个进程时 3/3 撞上;server 的+/// worker 并发认领两条 build_vector_index 任务是同一件事,并发默认 64)。+/// `build` 里的会话级咨询锁让后到的那条等前一条建完。连做八轮,不加锁时几乎必红+#[tokio::test]+async fn two_dimensions_built_at_once_take_turns() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ // 这个测试独占的两个维度+ const A: usize = 11;+ const B: usize = 13;+ let mut outcome = Ok(());+ for round in 0..8 {+ for dims in [A, B] {+ vector_index::drop(&pool, Target::Chunks, dims).await?;+ }+ // 摄取还在往 chunks 写:CIC 要等这些事务,死锁就靠它凑齐+ let writer = async {+ for _ in 0..20 {+ let mut tx = pool.begin().await?;+ sqlx::query("SELECT count(*) FROM chunks")+ .execute(&mut *tx)+ .await?;+ tokio::time::sleep(std::time::Duration::from_millis(5)).await;+ tx.commit().await?;+ }+ Ok::<_, anyhow::Error>(())+ };+ let (a, b, w) = tokio::join!(+ vector_index::build(&pool, Target::Chunks, A),+ vector_index::build(&pool, Target::Chunks, B),+ writer+ );+ w?;+ match (a, b) {+ (Ok(a), Ok(b)) => assert!(+ a.created && b.created,+ "第 {round} 轮两条都建成:{a:?} {b:?}"+ ),+ (a, b) => {+ outcome = Err(anyhow::anyhow!("第 {round} 轮:{a:?} / {b:?}"));+ break;+ }+ }+ }+ for dims in [A, B] {+ let _ = vector_index::drop(&pool, Target::Chunks, dims).await;+ }+ outcome+}