World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Keep overlapping and superseded document reads from corrupting chunks (#780)
Maya Kid68178f9287e3
diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rsindex 54053f62..04b58f41 100644--- a/crates/utopia-server/src/pipeline.rs+++ b/crates/utopia-server/src/pipeline.rs@@ -151,9 +151,20 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {}};let text_len = text.chars().count() as i32;- let chunk_pairs =- utopia_store::documents::replace_chunks(&state.pool, doc.kb_id, document_id, &pieces)- .await?;+ let Some(chunk_pairs) = utopia_store::documents::replace_chunks_if_current(+ &state.pool,+ doc.kb_id,+ document_id,+ &pieces,+ &doc.sha256,+ )+ .await?+ else {+ // 读取期间源文档可能已更新或删除,丢弃过期结果,+ // 不再改写新任务的索引、状态和抽取队列。+ tracing::info!(%document_id, "discarding a superseded document read");+ return Ok(());+ };let chunk_count = chunk_pairs.len() as i32;// 3. 全文索引(Tantivy)diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rsindex 23ae9719..1e2f3705 100644--- a/crates/utopia-server/src/pipeline_tests.rs+++ b/crates/utopia-server/src/pipeline_tests.rs@@ -700,6 +700,155 @@ async fn with_transcriber(f: &Fx, fake: &FakeTranscriber, model: &str) -> anyhowconst MP3: [u8; 12] = [b'I', b'D', b'3', 4, 0, 0, 0, 0, 0, 0, 0xFF, 0xFB];+async fn pause_transcription(+ f: &Fx,+ doc: Uuid,+) -> anyhow::Result<(+ Arc<tokio::sync::Notify>,+ tokio::task::JoinHandle<anyhow::Result<()>>,+ tokio::task::JoinHandle<()>,+)> {+ let entered = Arc::new(tokio::sync::Notify::new());+ let resume = Arc::new(tokio::sync::Notify::new());+ let app = axum::Router::new().route(+ "/audio/transcriptions",+ axum::routing::post({+ let entered = entered.clone();+ let resume = resume.clone();+ move || {+ let entered = entered.clone();+ let resume = resume.clone();+ async move {+ entered.notify_one();+ resume.notified().await;+ axum::Json(serde_json::json!({"segments": [{+ "speaker": "A", "start": 0.0, "end": 1.0,+ "text": "The OLD budget is 100."+ }]}))+ }+ }+ }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;+ let base = format!("http://{}", listener.local_addr()?);+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ utopia_store::settings::upsert_transcribe(+ &f.pool,+ f.ws,+ Some(&base),+ None,+ Some("gpt-4o-transcribe-diarize"),+ )+ .await?;+ let state = f.state.clone();+ let processing = tokio::spawn(async move { super::process_document(&state, doc).await });+ tokio::time::timeout(Duration::from_secs(10), entered.notified()).await?;+ Ok((resume, processing, server))+}++#[tokio::test]+async fn a_late_reader_preserves_a_newer_processed_revision() -> anyhow::Result<()> {+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(5))).await? else {+ return Ok(());+ };+ let doc = f.document_with_bytes("budget.mp3", &MP3).await?;+ let source = Uuid::now_v7();+ sqlx::query("INSERT INTO sources (id, kb_id, kind, name) VALUES ($1,$2,'api','audit')")+ .bind(source)+ .bind(f.kb)+ .execute(&f.pool)+ .await?;+ sqlx::query("UPDATE documents SET source_id=$2, external_key='budget' WHERE id=$1")+ .bind(doc)+ .bind(source)+ .execute(&f.pool)+ .await?;+ let (resume, old, server) = pause_transcription(&f, doc).await?;+ let text = "The NEW budget is 200.";+ use sha2::{Digest, Sha256};+ let sha: String = Sha256::digest(text.as_bytes())+ .iter()+ .map(|b| format!("{b:02x}"))+ .collect();+ f.state.blob.put(&sha, text.as_bytes()).await?;+ let mut tx = f.pool.begin().await?;+ let updated = utopia_store::documents::upsert_source_document_tx(+ &mut tx,+ f.kb,+ source,+ "budget",+ "budget.txt",+ "text/plain",+ text.len() as i64,+ &sha,+ None,+ )+ .await?;+ tx.commit().await?;+ assert_eq!(updated.id, doc);+ super::process_document(&f.state, doc).await?;+ // 新版本任务已经完成;旧读取若覆盖其分块,不会再有后续重试修复。+ let job: i64 = sqlx::query_scalar(+ "SELECT id FROM jobs WHERE kind = 'process_document' AND payload->>'document_id' = $1",+ )+ .bind(doc.to_string())+ .fetch_one(&f.pool)+ .await?;+ sqlx::query("UPDATE jobs SET status = 'done', updated_at = now() WHERE id = $1")+ .bind(job)+ .execute(&f.pool)+ .await?;+ let completed = utopia_store::documents::get(&f.pool, doc).await?;+ resume.notify_one();+ let old_result = old.await?;+ server.abort();+ let live: Vec<String> = sqlx::query_scalar(+ "SELECT text FROM chunks WHERE document_id=$1 AND superseded_at IS NULL ORDER BY seq",+ )+ .bind(doc)+ .fetch_all(&f.pool)+ .await?;+ let current = utopia_store::documents::get(&f.pool, doc).await?;+ f.cleanup().await?;+ old_result?;+ assert_eq!(current.sha256, sha);+ assert_eq!(current.status, "ready");+ assert_eq!(current.text_len, completed.text_len);+ assert_eq!(+ live,+ vec![text.to_string()],+ "a completed newer revision must not be replaced by an old read"+ );+ Ok(())+}++#[tokio::test]+async fn a_late_reader_does_not_repopulate_a_deleted_document() -> anyhow::Result<()> {+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(5))).await? else {+ return Ok(());+ };+ let doc = f.document_with_bytes("budget.mp3", &MP3).await?;+ let (resume, processing, server) = pause_transcription(&f, doc).await?;+ utopia_store::documents::delete(&f.pool, f.kb, doc, None).await?;+ let deleted = utopia_store::documents::get(&f.pool, doc).await?;+ resume.notify_one();+ let result = processing.await?;+ server.abort();+ let live: i64 = sqlx::query_scalar(+ "SELECT count(*) FROM chunks WHERE document_id=$1 AND superseded_at IS NULL",+ )+ .bind(doc)+ .fetch_one(&f.pool)+ .await?;+ let current = utopia_store::documents::get(&f.pool, doc).await?;+ f.cleanup().await?;+ result?;+ assert!(current.deleted_at.is_some());+ assert_eq!(current.status, deleted.status);+ assert_eq!(live, 0);+ Ok(())+}+/// 0040 第三刀:录音交给会标说话人的转写模型。说话人写进正文,每块记着起止时刻和说话人#[tokio::test]async fn a_recording_is_read_with_who_said_what() -> anyhow::Result<()> {diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rsindex 6aa35efb..c315c6fa 100644--- a/crates/utopia-store/src/documents.rs+++ b/crates/utopia-store/src/documents.rs@@ -1289,7 +1289,51 @@ pub async fn replace_chunks(document_id: Uuid,pieces: &[ChunkPiece],) -> AppResult<Vec<(String, String)>> {+ Ok(+ replace_chunks_for_snapshot(pool, kb_id, document_id, pieces, None)+ .await?+ .unwrap_or_default(),+ )+}++/// 慢读取不能覆盖新版本分块,也不能重新填入已删除文档。+/// 返回 None 表示输入已过期,调用方应结束本次处理。+pub async fn replace_chunks_if_current(+ pool: &PgPool,+ kb_id: Uuid,+ document_id: Uuid,+ pieces: &[ChunkPiece],+ sha256: &str,+) -> AppResult<Option<Vec<(String, String)>>> {+ replace_chunks_for_snapshot(pool, kb_id, document_id, pieces, Some(sha256)).await+}++async fn replace_chunks_for_snapshot(+ pool: &PgPool,+ kb_id: Uuid,+ document_id: Uuid,+ pieces: &[ChunkPiece],+ sha256: Option<&str>,+) -> AppResult<Option<Vec<(String, String)>>> {let mut tx = pool.begin().await?;+ // 两个任务可能同时处理同一文档。先锁父记录,即使还没有分块,+ // 后一个任务也能认领前一个任务写入的行,避免重复插入。+ //+ // **`FOR NO KEY UPDATE`,不是 `FOR UPDATE`**:后者与外键检查要的 `FOR KEY SHARE`+ // 冲突,于是这个事务活着的时候,这份文档所有子表的插入都被挡住(chunks、+ // document_versions、memory::append),而这个事务是每个分块一个来回——四千块的+ // 文档要锁四秒。两者对另一个 `replace_chunks` 的互斥是一样的+ let current: Option<(String, bool)> = sqlx::query_as(+ "SELECT sha256, deleted_at IS NOT NULL FROM documents WHERE id = $1 FOR NO KEY UPDATE",+ )+ .bind(document_id)+ .fetch_optional(&mut *tx)+ .await?;+ if let Some(expected) = sha256 {+ if !matches!(current, Some((ref sha, false)) if sha == expected) {+ return Ok(None);+ }+ }let (version,): (i32,) = sqlx::query_as("SELECT COALESCE(MAX(version), 1) FROM document_versions WHERE document_id = $1",)@@ -1412,7 +1456,7 @@ pub async fn replace_chunks(.await?;}tx.commit().await?;- Ok(out)+ Ok(Some(out))}/// 抽取完成一个分块即打标(认领的块携带标记跳过重抽;也让中断的抽取可续跑)。diff --git a/crates/utopia-store/tests/concurrent_chunk_replacement.rs b/crates/utopia-store/tests/concurrent_chunk_replacement.rsnew file mode 100644index 00000000..db8a948d--- /dev/null+++ b/crates/utopia-store/tests/concurrent_chunk_replacement.rs@@ -0,0 +1,94 @@+use sqlx::{postgres::PgPoolOptions, PgPool};+use std::time::Duration;+use utopia_ingest::{ChunkPiece, Provenance};+use uuid::Uuid;++fn pieces() -> Vec<ChunkPiece> {+ vec![ChunkPiece {+ seq: 0,+ text: "Revenue was 100.".into(),+ char_start: 0,+ char_end: 16,+ heading: None,+ provenance: Provenance::stated(),+ }]+}++#[tokio::test]+async fn overlapping_reprocessing_keeps_one_live_copy_of_each_chunk() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ let (org, ws, kb, doc) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'chunk-race')")+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'chunk-race')")+ .bind(ws)+ .bind(org)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'chunk-race')",+ )+ .bind(kb)+ .bind(ws)+ .execute(&pool)+ .await?;+ sqlx::query("INSERT INTO documents (id, kb_id, filename, sha256, status) VALUES ($1, $2, 'report.txt', $1::text, 'pending')").bind(doc).bind(kb).execute(&pool).await?;++ let result = async {+ let mut gate = pool.begin().await?;+ sqlx::query("SELECT id FROM documents WHERE id = $1 FOR UPDATE")+ .bind(doc).execute(&mut *gate).await?;+ // Separate one-connection pools let us observe both writers at a real DB+ // lock, rather than assuming that a sleep has produced the interleaving.+ let mut tasks = Vec::new();+ let mut pids = Vec::new();+ for _ in 0..2 {+ let writer = PgPoolOptions::new().max_connections(1).connect(&url).await?;+ pids.push(sqlx::query_scalar::<_, i32>("SELECT pg_backend_pid()").fetch_one(&writer).await?);+ tasks.push(tokio::spawn(async move {+ utopia_store::documents::replace_chunks(&writer, kb, doc, &pieces()).await+ }));+ }+ let waiting = tokio::time::timeout(Duration::from_secs(10), async {+ loop {+ let count: i64 = sqlx::query_scalar("SELECT count(*) FROM pg_stat_activity WHERE pid = ANY($1) AND wait_event_type = 'Lock'")+ .bind(&pids).fetch_one(&pool).await?;+ if count == 2 { break Ok::<_, sqlx::Error>(()); }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ }).await;+ // Before the fix both have already read an empty chunk set and are+ // blocked by the insert's FK check. With the fix they wait before reading.+ gate.commit().await?;+ let mut returned = Vec::new();+ for task in tasks { returned.push(task.await??); }+ waiting??;+ let live: i64 = sqlx::query_scalar("SELECT count(*) FROM chunks WHERE document_id = $1 AND superseded_at IS NULL")+ .bind(doc).fetch_one(&pool).await?;+ Ok::<_, anyhow::Error>((live, returned))+ }.await;+ sqlx::query("DELETE FROM organizations WHERE id = $1")+ .bind(org)+ .execute(&pool)+ .await?;+ let (live, returned) = result?;+ assert_eq!(+ live, 1,+ "overlapping processing must not duplicate the document's live text"+ );+ assert_eq!(+ returned[0], returned[1],+ "both runs must return the adopted chunk id for indexing"+ );+ Ok(())+}