Skip to content

World's first open-source enterprise world model.

Keep overlapping and superseded document reads from corrupting chunks (#780)

Maya Kid68178f9287e3
diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rs
index 54053f62..04b58f41 100644
--- a/crates/utopia-server/src/pipeline.rs
+++ b/crates/utopia-server/src/pipeline.rs
@@ -151,9 +151,20 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
}
};
let text_len = text.chars().count() as i32;
- let chunk_pairs =
- utopia_store::documents::replace_chunks(&state.pool, doc.kb_id, document_id, &pieces)
- .await?;
+ let Some(chunk_pairs) = utopia_store::documents::replace_chunks_if_current(
+ &state.pool,
+ doc.kb_id,
+ document_id,
+ &pieces,
+ &doc.sha256,
+ )
+ .await?
+ else {
+ // 读取期间源文档可能已更新或删除,丢弃过期结果,
+ // 不再改写新任务的索引、状态和抽取队列。
+ tracing::info!(%document_id, "discarding a superseded document read");
+ return Ok(());
+ };
let chunk_count = chunk_pairs.len() as i32;
// 3. 全文索引(Tantivy)
diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rs
index 23ae9719..1e2f3705 100644
--- a/crates/utopia-server/src/pipeline_tests.rs
+++ b/crates/utopia-server/src/pipeline_tests.rs
@@ -700,6 +700,155 @@ async fn with_transcriber(f: &Fx, fake: &FakeTranscriber, model: &str) -> anyhow
const MP3: [u8; 12] = [b'I', b'D', b'3', 4, 0, 0, 0, 0, 0, 0, 0xFF, 0xFB];
+async fn pause_transcription(
+ f: &Fx,
+ doc: Uuid,
+) -> anyhow::Result<(
+ Arc<tokio::sync::Notify>,
+ tokio::task::JoinHandle<anyhow::Result<()>>,
+ tokio::task::JoinHandle<()>,
+)> {
+ let entered = Arc::new(tokio::sync::Notify::new());
+ let resume = Arc::new(tokio::sync::Notify::new());
+ let app = axum::Router::new().route(
+ "/audio/transcriptions",
+ axum::routing::post({
+ let entered = entered.clone();
+ let resume = resume.clone();
+ move || {
+ let entered = entered.clone();
+ let resume = resume.clone();
+ async move {
+ entered.notify_one();
+ resume.notified().await;
+ axum::Json(serde_json::json!({"segments": [{
+ "speaker": "A", "start": 0.0, "end": 1.0,
+ "text": "The OLD budget is 100."
+ }]}))
+ }
+ }
+ }),
+ );
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
+ let base = format!("http://{}", listener.local_addr()?);
+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ utopia_store::settings::upsert_transcribe(
+ &f.pool,
+ f.ws,
+ Some(&base),
+ None,
+ Some("gpt-4o-transcribe-diarize"),
+ )
+ .await?;
+ let state = f.state.clone();
+ let processing = tokio::spawn(async move { super::process_document(&state, doc).await });
+ tokio::time::timeout(Duration::from_secs(10), entered.notified()).await?;
+ Ok((resume, processing, server))
+}
+
+#[tokio::test]
+async fn a_late_reader_preserves_a_newer_processed_revision() -> anyhow::Result<()> {
+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(5))).await? else {
+ return Ok(());
+ };
+ let doc = f.document_with_bytes("budget.mp3", &MP3).await?;
+ let source = Uuid::now_v7();
+ sqlx::query("INSERT INTO sources (id, kb_id, kind, name) VALUES ($1,$2,'api','audit')")
+ .bind(source)
+ .bind(f.kb)
+ .execute(&f.pool)
+ .await?;
+ sqlx::query("UPDATE documents SET source_id=$2, external_key='budget' WHERE id=$1")
+ .bind(doc)
+ .bind(source)
+ .execute(&f.pool)
+ .await?;
+ let (resume, old, server) = pause_transcription(&f, doc).await?;
+ let text = "The NEW budget is 200.";
+ use sha2::{Digest, Sha256};
+ let sha: String = Sha256::digest(text.as_bytes())
+ .iter()
+ .map(|b| format!("{b:02x}"))
+ .collect();
+ f.state.blob.put(&sha, text.as_bytes()).await?;
+ let mut tx = f.pool.begin().await?;
+ let updated = utopia_store::documents::upsert_source_document_tx(
+ &mut tx,
+ f.kb,
+ source,
+ "budget",
+ "budget.txt",
+ "text/plain",
+ text.len() as i64,
+ &sha,
+ None,
+ )
+ .await?;
+ tx.commit().await?;
+ assert_eq!(updated.id, doc);
+ super::process_document(&f.state, doc).await?;
+ // 新版本任务已经完成;旧读取若覆盖其分块,不会再有后续重试修复。
+ let job: i64 = sqlx::query_scalar(
+ "SELECT id FROM jobs WHERE kind = 'process_document' AND payload->>'document_id' = $1",
+ )
+ .bind(doc.to_string())
+ .fetch_one(&f.pool)
+ .await?;
+ sqlx::query("UPDATE jobs SET status = 'done', updated_at = now() WHERE id = $1")
+ .bind(job)
+ .execute(&f.pool)
+ .await?;
+ let completed = utopia_store::documents::get(&f.pool, doc).await?;
+ resume.notify_one();
+ let old_result = old.await?;
+ server.abort();
+ let live: Vec<String> = sqlx::query_scalar(
+ "SELECT text FROM chunks WHERE document_id=$1 AND superseded_at IS NULL ORDER BY seq",
+ )
+ .bind(doc)
+ .fetch_all(&f.pool)
+ .await?;
+ let current = utopia_store::documents::get(&f.pool, doc).await?;
+ f.cleanup().await?;
+ old_result?;
+ assert_eq!(current.sha256, sha);
+ assert_eq!(current.status, "ready");
+ assert_eq!(current.text_len, completed.text_len);
+ assert_eq!(
+ live,
+ vec![text.to_string()],
+ "a completed newer revision must not be replaced by an old read"
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn a_late_reader_does_not_repopulate_a_deleted_document() -> anyhow::Result<()> {
+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(5))).await? else {
+ return Ok(());
+ };
+ let doc = f.document_with_bytes("budget.mp3", &MP3).await?;
+ let (resume, processing, server) = pause_transcription(&f, doc).await?;
+ utopia_store::documents::delete(&f.pool, f.kb, doc, None).await?;
+ let deleted = utopia_store::documents::get(&f.pool, doc).await?;
+ resume.notify_one();
+ let result = processing.await?;
+ server.abort();
+ let live: i64 = sqlx::query_scalar(
+ "SELECT count(*) FROM chunks WHERE document_id=$1 AND superseded_at IS NULL",
+ )
+ .bind(doc)
+ .fetch_one(&f.pool)
+ .await?;
+ let current = utopia_store::documents::get(&f.pool, doc).await?;
+ f.cleanup().await?;
+ result?;
+ assert!(current.deleted_at.is_some());
+ assert_eq!(current.status, deleted.status);
+ assert_eq!(live, 0);
+ Ok(())
+}
+
/// 0040 第三刀:录音交给会标说话人的转写模型。说话人写进正文,每块记着起止时刻和说话人
#[tokio::test]
async fn a_recording_is_read_with_who_said_what() -> anyhow::Result<()> {
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index 6aa35efb..c315c6fa 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -1289,7 +1289,51 @@ pub async fn replace_chunks(
document_id: Uuid,
pieces: &[ChunkPiece],
) -> AppResult<Vec<(String, String)>> {
+ Ok(
+ replace_chunks_for_snapshot(pool, kb_id, document_id, pieces, None)
+ .await?
+ .unwrap_or_default(),
+ )
+}
+
+/// 慢读取不能覆盖新版本分块,也不能重新填入已删除文档。
+/// 返回 None 表示输入已过期,调用方应结束本次处理。
+pub async fn replace_chunks_if_current(
+ pool: &PgPool,
+ kb_id: Uuid,
+ document_id: Uuid,
+ pieces: &[ChunkPiece],
+ sha256: &str,
+) -> AppResult<Option<Vec<(String, String)>>> {
+ replace_chunks_for_snapshot(pool, kb_id, document_id, pieces, Some(sha256)).await
+}
+
+async fn replace_chunks_for_snapshot(
+ pool: &PgPool,
+ kb_id: Uuid,
+ document_id: Uuid,
+ pieces: &[ChunkPiece],
+ sha256: Option<&str>,
+) -> AppResult<Option<Vec<(String, String)>>> {
let mut tx = pool.begin().await?;
+ // 两个任务可能同时处理同一文档。先锁父记录,即使还没有分块,
+ // 后一个任务也能认领前一个任务写入的行,避免重复插入。
+ //
+ // **`FOR NO KEY UPDATE`,不是 `FOR UPDATE`**:后者与外键检查要的 `FOR KEY SHARE`
+ // 冲突,于是这个事务活着的时候,这份文档所有子表的插入都被挡住(chunks、
+ // document_versions、memory::append),而这个事务是每个分块一个来回——四千块的
+ // 文档要锁四秒。两者对另一个 `replace_chunks` 的互斥是一样的
+ let current: Option<(String, bool)> = sqlx::query_as(
+ "SELECT sha256, deleted_at IS NOT NULL FROM documents WHERE id = $1 FOR NO KEY UPDATE",
+ )
+ .bind(document_id)
+ .fetch_optional(&mut *tx)
+ .await?;
+ if let Some(expected) = sha256 {
+ if !matches!(current, Some((ref sha, false)) if sha == expected) {
+ return Ok(None);
+ }
+ }
let (version,): (i32,) = sqlx::query_as(
"SELECT COALESCE(MAX(version), 1) FROM document_versions WHERE document_id = $1",
)
@@ -1412,7 +1456,7 @@ pub async fn replace_chunks(
.await?;
}
tx.commit().await?;
- Ok(out)
+ Ok(Some(out))
}
/// 抽取完成一个分块即打标(认领的块携带标记跳过重抽;也让中断的抽取可续跑)。
diff --git a/crates/utopia-store/tests/concurrent_chunk_replacement.rs b/crates/utopia-store/tests/concurrent_chunk_replacement.rs
new file mode 100644
index 00000000..db8a948d
--- /dev/null
+++ b/crates/utopia-store/tests/concurrent_chunk_replacement.rs
@@ -0,0 +1,94 @@
+use sqlx::{postgres::PgPoolOptions, PgPool};
+use std::time::Duration;
+use utopia_ingest::{ChunkPiece, Provenance};
+use uuid::Uuid;
+
+fn pieces() -> Vec<ChunkPiece> {
+ vec![ChunkPiece {
+ seq: 0,
+ text: "Revenue was 100.".into(),
+ char_start: 0,
+ char_end: 16,
+ heading: None,
+ provenance: Provenance::stated(),
+ }]
+}
+
+#[tokio::test]
+async fn overlapping_reprocessing_keeps_one_live_copy_of_each_chunk() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let (org, ws, kb, doc) = (
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ );
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'chunk-race')")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'chunk-race')")
+ .bind(ws)
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'chunk-race')",
+ )
+ .bind(kb)
+ .bind(ws)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO documents (id, kb_id, filename, sha256, status) VALUES ($1, $2, 'report.txt', $1::text, 'pending')").bind(doc).bind(kb).execute(&pool).await?;
+
+ let result = async {
+ let mut gate = pool.begin().await?;
+ sqlx::query("SELECT id FROM documents WHERE id = $1 FOR UPDATE")
+ .bind(doc).execute(&mut *gate).await?;
+ // Separate one-connection pools let us observe both writers at a real DB
+ // lock, rather than assuming that a sleep has produced the interleaving.
+ let mut tasks = Vec::new();
+ let mut pids = Vec::new();
+ for _ in 0..2 {
+ let writer = PgPoolOptions::new().max_connections(1).connect(&url).await?;
+ pids.push(sqlx::query_scalar::<_, i32>("SELECT pg_backend_pid()").fetch_one(&writer).await?);
+ tasks.push(tokio::spawn(async move {
+ utopia_store::documents::replace_chunks(&writer, kb, doc, &pieces()).await
+ }));
+ }
+ let waiting = tokio::time::timeout(Duration::from_secs(10), async {
+ loop {
+ let count: i64 = sqlx::query_scalar("SELECT count(*) FROM pg_stat_activity WHERE pid = ANY($1) AND wait_event_type = 'Lock'")
+ .bind(&pids).fetch_one(&pool).await?;
+ if count == 2 { break Ok::<_, sqlx::Error>(()); }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ }).await;
+ // Before the fix both have already read an empty chunk set and are
+ // blocked by the insert's FK check. With the fix they wait before reading.
+ gate.commit().await?;
+ let mut returned = Vec::new();
+ for task in tasks { returned.push(task.await??); }
+ waiting??;
+ let live: i64 = sqlx::query_scalar("SELECT count(*) FROM chunks WHERE document_id = $1 AND superseded_at IS NULL")
+ .bind(doc).fetch_one(&pool).await?;
+ Ok::<_, anyhow::Error>((live, returned))
+ }.await;
+ sqlx::query("DELETE FROM organizations WHERE id = $1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ let (live, returned) = result?;
+ assert_eq!(
+ live, 1,
+ "overlapping processing must not duplicate the document's live text"
+ );
+ assert_eq!(
+ returned[0], returned[1],
+ "both runs must return the adopted chunk id for indexing"
+ );
+ Ok(())
+}