Skip to content

World's first open-source enterprise world model.

Keep superseded document processing from overwriting the current revision (#1217)

Daniel0bae9c419ef8
diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rs
index 48503451..c012352e 100644
--- a/crates/utopia-server/src/pipeline.rs
+++ b/crates/utopia-server/src/pipeline.rs
@@ -6,6 +6,7 @@ use crate::state::AppState;
use futures_util::{stream, StreamExt};
use utopia_core::models::{LlmSettings, Proposer};
use utopia_llm::LlmClient;
+use utopia_store::documents::{self, ProcessingInput};
use uuid::Uuid;
/// 这份文档的来源要不要抽取。没有来源的文档(直接上传、记忆片段)照旧抽。
@@ -47,14 +48,19 @@ const EMBED_BATCH: usize = 16;
const EMBED_JOBS: usize = 4;
pub async fn process_document(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
- match run(state, document_id).await {
+ let input = documents::processing_input(&state.pool, document_id).await?;
+ let doc = &input.document;
+ match run(state, &input).await {
Ok(()) => Ok(()),
// 在等读字的服务读完:文档照旧是 parsing,任务过一会儿再来问
- Err(e) if utopia_core::is_deferred(&e).is_some() => Err(e),
+ Err(e) if utopia_core::is_deferred(&e).is_some() => {
+ if documents::processing_is_current(&state.pool, &input).await? {
+ Err(e)
+ } else {
+ Ok(())
+ }
+ }
Err(e) => {
- let doc = utopia_store::documents::get(&state.pool, document_id)
- .await
- .ok();
// 字要靠模型读、而那种模型没配(0040):**降级**。文件留着,文档停在 failed
// 并记下缺哪一种,报一条库级告警;配上模型会自己重新处理。重试没用——
// 模型不会在两分钟里自己配上,所以挂 Terminal。
@@ -66,33 +72,40 @@ pub async fn process_document(state: &AppState, document_id: Uuid) -> anyhow::Re
e.downcast_ref::<utopia_ingest::NoSpeakers>()
.map(|_| utopia_ingest::Reader::Transcribe)
});
+ if !documents::set_failed_if_current(
+ &state.pool,
+ &input,
+ &e.to_string(),
+ waiting.map(|reader| reader.as_str()),
+ )
+ .await?
+ {
+ // 过期模型结果可丢弃,数据库故障仍要让队列看见。
+ if e.is::<sqlx::Error>()
+ || matches!(
+ e.downcast_ref::<utopia_core::AppError>(),
+ Some(utopia_core::AppError::Db(_))
+ )
+ {
+ return Err(e);
+ }
+ tracing::info!(%document_id, "discarding a superseded document failure");
+ return Ok(());
+ }
if let Some(reader) = waiting {
- let _ = utopia_store::documents::set_needs_reader(
- &state.pool,
+ crate::alerting::observe_document_needs_reader(
+ state,
+ doc.kb_id,
document_id,
- reader.as_str(),
+ &doc.filename,
+ reader,
&e.to_string(),
)
.await;
- if let Some(doc) = &doc {
- crate::alerting::observe_document_needs_reader(
- state,
- doc.kb_id,
- document_id,
- &doc.filename,
- reader,
- &e.to_string(),
- )
- .await;
- state.emit_document(doc.kb_id, document_id);
- }
- return Err(e.context(utopia_core::Terminal));
- }
- let _ =
- utopia_store::documents::set_failed(&state.pool, document_id, &e.to_string()).await;
- if let Some(doc) = &doc {
state.emit_document(doc.kb_id, document_id);
+ return Err(e.context(utopia_core::Terminal));
}
+ state.emit_document(doc.kb_id, document_id);
// 读不了的格式(视频、二进制、空文件)换多少次也一样
if e.downcast_ref::<utopia_ingest::Unreadable>().is_some() {
return Err(e.context(utopia_core::Terminal));
@@ -102,8 +115,9 @@ pub async fn process_document(state: &AppState, document_id: Uuid) -> anyhow::Re
}
}
-async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
- let doc = utopia_store::documents::get(&state.pool, document_id).await?;
+async fn run(state: &AppState, input: &ProcessingInput) -> anyhow::Result<()> {
+ let doc = &input.document;
+ let document_id = doc.id;
// 排队之后被删了(#268):墓碑不重建分块、不回索引;清过的连原文都没了
if doc.deleted_at.is_some() {
tracing::info!(document = %document_id, "skipping a deleted document");
@@ -111,7 +125,9 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
}
// 1. 解析(CPU 密集,放 blocking 线程)
- utopia_store::documents::set_status(&state.pool, document_id, "parsing").await?;
+ if !documents::set_status_if_current(&state.pool, input, "parsing").await? {
+ return Ok(());
+ }
state.emit_document(doc.kb_id, document_id);
let bytes = state.blob.get(&doc.sha256).await?;
let filename = doc.filename.clone();
@@ -157,13 +173,13 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
let reading = match (needs.reader, settings.as_ref()) {
(utopia_ingest::Reader::Ocr, Some(s)) => {
match crate::readers::Ocr::from_settings(s) {
- Some(ocr) => ocr.read(state, &doc, bytes).await?,
+ Some(ocr) => ocr.read(state, doc, bytes).await?,
None => return Err(e),
}
}
(utopia_ingest::Reader::Transcribe, Some(s)) => {
match crate::readers::Transcriber::from_settings(s) {
- Some(t) => t.read(&doc, bytes).await?,
+ Some(t) => t.read(doc, bytes).await?,
None => return Err(e),
}
}
@@ -180,14 +196,8 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
}
};
let text_len = text.chars().count() as i32;
- let Some(chunk_pairs) = utopia_store::documents::replace_chunks_if_current(
- &state.pool,
- doc.kb_id,
- document_id,
- &pieces,
- &doc.sha256,
- )
- .await?
+ let Some(chunk_pairs) =
+ documents::replace_chunks_if_current(&state.pool, input, &pieces).await?
else {
// 读取期间源文档可能已更新或删除,丢弃过期结果,
// 不再改写新任务的索引、状态和抽取队列。
@@ -208,7 +218,9 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
}
// 3. 全文索引(Tantivy)
- utopia_store::documents::set_status(&state.pool, document_id, "indexing").await?;
+ if !documents::set_status_if_current(&state.pool, input, "indexing").await? {
+ return Ok(());
+ }
state.emit_document(doc.kb_id, document_id);
let search = state.search.clone();
let kb = doc.kb_id.to_string();
@@ -217,36 +229,39 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
// 4. embedding(工作区配置了 embedding 模型才做;没配也算 ready,先享受 BM25 搜索)
if let Some((settings, client)) = embedder(settings.as_ref()) {
- utopia_store::documents::set_status(&state.pool, document_id, "embedding").await?;
+ if !documents::set_status_if_current(&state.pool, input, "embedding").await? {
+ return Ok(());
+ }
state.emit_document(doc.kb_id, document_id);
- embed_pending(state, settings, &client, document_id).await?;
+ embed_pending(state, settings, &client, input).await?;
}
- utopia_store::documents::set_ready(&state.pool, document_id, text_len, chunk_count).await?;
-
// 来源说了不抽取的,到这里为止:可搜、可问,不进图。
//
// schema 文档就是这一类(0035 决定 7)——它是给问数检索表结构的语料,进抽取的
// 结果是抽取器把每个列名当成一个实体(宽表语料上四十个概念实体里二十八个是
// 列名,#553)。状态记成 `skipped` 而不是留在 `none`:`none` 在 Library 里读作
// 「还没排到」,而它永远不会排到
- if !source_extracts(state, doc.source_id).await? {
- utopia_store::documents::set_graph_status(&state.pool, document_id, "skipped").await?;
- state.emit_document(doc.kb_id, document_id);
- tracing::info!(%document_id, chunks = chunk_count, "文档处理完成,来源不抽取");
- return Ok(());
- }
-
// 两段式:索引就绪后,若配置了对话模型则排队图谱抽取(不阻塞可搜可问)。
// 推送来的陈述不问模型(0054),没配也排
- if pushed_statements || settings.as_ref().is_some_and(|s| s.chat_ready()) {
- utopia_store::documents::set_graph_status(&state.pool, document_id, "queued").await?;
- utopia_store::jobs::enqueue(
- &state.pool,
- "extract_document",
- serde_json::json!({ "document_id": document_id }),
- )
- .await?;
+ let graph_status = if !source_extracts(state, doc.source_id).await? {
+ Some("skipped")
+ } else if pushed_statements || settings.as_ref().is_some_and(|s| s.chat_ready()) {
+ Some("queued")
+ } else {
+ None
+ };
+ if !documents::finish_processing_if_current(
+ &state.pool,
+ input,
+ text_len,
+ chunk_count,
+ graph_status,
+ )
+ .await?
+ {
+ tracing::info!(%document_id, "discarding a superseded document completion");
+ return Ok(());
}
state.emit_document(doc.kb_id, document_id);
@@ -363,10 +378,9 @@ async fn embed_pending(
state: &AppState,
settings: &LlmSettings,
client: &LlmClient,
- document_id: Uuid,
+ input: &ProcessingInput,
) -> anyhow::Result<usize> {
- let pending =
- utopia_store::documents::chunks_pending_embedding(&state.pool, document_id).await?;
+ let pending = documents::chunks_pending_embedding(&state.pool, input.document.id).await?;
// 批次先切成拥有的数据:借来的切片捕进并发驱动的 future 过不了 Send 边界
//(ontology_index 里同一句话)
let batches: Vec<Vec<(Uuid, String)>> =
@@ -385,7 +399,9 @@ async fn embed_pending(
}
let items: Vec<(Uuid, Vec<f32>)> =
batch.iter().map(|(id, _)| *id).zip(embeddings).collect();
- utopia_store::documents::set_embeddings(&state.pool, &items).await?;
+ if !documents::set_embeddings_if_current(&state.pool, input, &items).await? {
+ return Ok(0);
+ }
Ok::<usize, anyhow::Error>(items.len())
})
.buffer_unordered(EMBED_JOBS);
diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rs
index 4db5fa41..c88e92d7 100644
--- a/crates/utopia-server/src/pipeline_tests.rs
+++ b/crates/utopia-server/src/pipeline_tests.rs
@@ -22,6 +22,7 @@
//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
use std::fmt::Write as _;
+use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use utopia_core::models::Proposer;
@@ -231,7 +232,8 @@ impl Fx {
.await?
.expect("settings were written by the fixture");
let client = crate::llm_util::embed_client(&settings).expect("embed model is configured");
- super::embed_pending(&self.state, &settings, &client, doc).await
+ let input = utopia_store::documents::processing_input(&self.pool, doc).await?;
+ super::embed_pending(&self.state, &settings, &client, &input).await
}
/// (正文, 向量) 按 seq;向量为空就是没嵌
@@ -353,16 +355,278 @@ async fn a_failed_batch_does_not_strand_the_document() -> anyhow::Result<()> {
return Ok(());
};
let doc = f.document_to_process(20).await?;
- assert!(super::process_document(&f.state, doc).await.is_err());
+ let original = utopia_store::documents::get(&f.pool, doc).await?;
+ let error = super::process_document(&f.state, doc)
+ .await
+ .expect_err("the current revision really failed");
+ assert!(utopia_core::is_deferred(&error).is_none());
+ assert!(!utopia_core::is_terminal(&error));
+ let error_message = error.to_string();
+ assert!(error_message.contains("model is down"));
let row = utopia_store::documents::get(&f.pool, doc).await?;
assert_eq!(row.status, "failed", "not left sitting in embedding");
- assert!(
- row.error.as_deref().is_some_and(|e| !e.is_empty()),
- "the reason is on the document"
+ assert_eq!(row.error.as_deref(), Some(error_message.as_str()));
+ assert_eq!(
+ (row.text_len, row.chunk_count),
+ (original.text_len, original.chunk_count)
);
f.cleanup().await
}
+/// #1215:第一批嵌入必须等测试放行;新版请求立即完成,不靠延时猜并发顺序。
+async fn pause_embedding(
+ f: &Fx,
+ doc: Uuid,
+ fail_old: bool,
+) -> anyhow::Result<(
+ Arc<tokio::sync::Notify>,
+ tokio::task::JoinHandle<anyhow::Result<()>>,
+ tokio::task::JoinHandle<()>,
+)> {
+ let entered = Arc::new(tokio::sync::Notify::new());
+ let resume = Arc::new(tokio::sync::Notify::new());
+ let first_request = Arc::new(AtomicBool::new(true));
+ let app = axum::Router::new().route(
+ "/embeddings",
+ axum::routing::post({
+ let entered = entered.clone();
+ let resume = resume.clone();
+ move |axum::Json(body): axum::Json<serde_json::Value>| {
+ let entered = entered.clone();
+ let resume = resume.clone();
+ let first_request = first_request.clone();
+ async move {
+ let is_old = first_request.swap(false, Ordering::SeqCst);
+ if is_old {
+ entered.notify_one();
+ resume.notified().await;
+ if fail_old {
+ return (
+ axum::http::StatusCode::INTERNAL_SERVER_ERROR,
+ axum::Json(serde_json::json!({"error": "old embedding failed"})),
+ );
+ }
+ }
+ let data: Vec<_> = body["input"]
+ .as_array()
+ .expect("embedding input is an array")
+ .iter()
+ .map(|input| {
+ let mut vector = vector_of(input.as_str().expect("input is text"));
+ // 同 SHA 的新版本可能复用 chunk ID,旧向量仍必须丢弃。
+ if is_old {
+ vector[0] += 10_000.0;
+ }
+ serde_json::json!({"embedding": vector})
+ })
+ .collect();
+ (
+ axum::http::StatusCode::OK,
+ axum::Json(serde_json::json!({"data": data})),
+ )
+ }
+ }
+ }),
+ );
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
+ let base = format!("http://{}", listener.local_addr()?);
+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ utopia_store::settings::upsert(
+ &f.pool,
+ f.ws,
+ Some(&base),
+ None,
+ Some("fake-chat"),
+ Some(&base),
+ None,
+ Some("fake-embed"),
+ Some(4),
+ )
+ .await?;
+ let state = f.state.clone();
+ let processing = tokio::spawn(async move { super::process_document(&state, doc).await });
+ tokio::time::timeout(Duration::from_secs(10), entered.notified()).await?;
+ Ok((resume, processing, server))
+}
+
+async fn replace_document_text(f: &Fx, doc: Uuid, text: &str) -> anyhow::Result<()> {
+ use sha2::{Digest, Sha256};
+ let sha: String = Sha256::digest(text.as_bytes())
+ .iter()
+ .map(|byte| format!("{byte:02x}"))
+ .collect();
+ f.state.blob.put(&sha, text.as_bytes()).await?;
+ utopia_store::documents::replace_content_and_enqueue_processing(
+ &f.pool,
+ doc,
+ "long.md",
+ "text/markdown",
+ text.len() as i64,
+ &sha,
+ None,
+ )
+ .await?;
+ Ok(())
+}
+
+async fn processing_snapshot(f: &Fx, doc: Uuid) -> anyhow::Result<serde_json::Value> {
+ Ok(sqlx::query_scalar(
+ "SELECT jsonb_build_object(
+ 'document', (SELECT to_jsonb(d) FROM documents d WHERE d.id = $1),
+ 'chunks', (SELECT coalesce(jsonb_agg(to_jsonb(c) ORDER BY c.seq, c.id), '[]')
+ FROM chunks c WHERE c.document_id = $1 AND c.superseded_at IS NULL),
+ 'jobs', (SELECT coalesce(jsonb_agg(to_jsonb(j) ORDER BY j.id), '[]')
+ FROM jobs j WHERE j.payload->>'document_id' = $1::text),
+ 'alerts', (SELECT coalesce(jsonb_agg(to_jsonb(a) ORDER BY a.id), '[]')
+ FROM alerts a WHERE a.kb_id = $2))",
+ )
+ .bind(doc)
+ .bind(f.kb)
+ .fetch_one(&f.pool)
+ .await?)
+}
+
+async fn late_embedding_preserves_newer_revision(
+ fail_old: bool,
+ restore_original: bool,
+) -> anyhow::Result<()> {
+ let Some(f) = fixture(FakeEmbed::new(Duration::ZERO)).await? else {
+ return Ok(());
+ };
+ let doc = f.document_to_process(4).await?;
+ let original = utopia_store::documents::get(&f.pool, doc).await?;
+ let versions = utopia_store::documents::versions(&f.pool, doc).await?;
+ let original_version = versions.last().expect("the document has a version").version;
+ let original_text = String::from_utf8(f.state.blob.get(&original.sha256).await?)?;
+ let (resume, old, server) = pause_embedding(&f, doc, fail_old).await?;
+ let old_chunks: Vec<Uuid> = sqlx::query_scalar(
+ "SELECT id FROM chunks WHERE document_id = $1 AND superseded_at IS NULL ORDER BY seq",
+ )
+ .bind(doc)
+ .fetch_all(&f.pool)
+ .await?;
+ assert!(old_chunks.len() > 1 && old_chunks.len() <= super::EMBED_BATCH);
+
+ let new_text = "The NEW budget is 200.";
+ replace_document_text(&f, doc, new_text).await?;
+ if restore_original {
+ // B 尚未解析,恢复 A 时会认领旧任务正在嵌入的同一批 chunk ID。
+ replace_document_text(&f, doc, &original_text).await?;
+ }
+ super::process_document(&f.state, doc).await?;
+ let newer = utopia_store::documents::get(&f.pool, doc).await?;
+ assert_eq!(newer.status, "ready");
+ assert_eq!(newer.error, None);
+ if restore_original {
+ assert_eq!(newer.sha256, original.sha256);
+ let versions = utopia_store::documents::versions(&f.pool, doc).await?;
+ assert!(versions.last().expect("the document has a version").version > original_version);
+ let current_chunks: Vec<Uuid> = sqlx::query_scalar(
+ "SELECT id FROM chunks
+ WHERE document_id = $1 AND superseded_at IS NULL ORDER BY seq",
+ )
+ .bind(doc)
+ .fetch_all(&f.pool)
+ .await?;
+ assert_eq!(
+ current_chunks, old_chunks,
+ "restoring A reuses the chunks held by the old request"
+ );
+ } else {
+ assert_ne!(newer.chunk_count, old_chunks.len() as i32);
+ }
+
+ // 新版抽取也已完成,旧任务不能再把它改成 queued 或另排一次抽取。
+ utopia_store::documents::set_graph_status(&f.pool, doc, "done").await?;
+ sqlx::query("UPDATE jobs SET status = 'done' WHERE payload->>'document_id' = $1")
+ .bind(doc.to_string())
+ .execute(&f.pool)
+ .await?;
+ let completed = processing_snapshot(&f, doc).await?;
+ resume.notify_one();
+ let old_result = tokio::time::timeout(Duration::from_secs(10), old).await??;
+ server.abort();
+ let current = processing_snapshot(&f, doc).await?;
+ sqlx::query("DELETE FROM jobs WHERE payload->>'document_id' = $1")
+ .bind(doc.to_string())
+ .execute(&f.pool)
+ .await?;
+ f.cleanup().await?;
+ old_result?;
+ assert_eq!(
+ current, completed,
+ "a late embedding must leave the newer revision intact"
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn a_late_successful_embedding_preserves_a_newer_revision() -> anyhow::Result<()> {
+ late_embedding_preserves_newer_revision(false, false).await
+}
+
+#[tokio::test]
+async fn a_late_failed_embedding_preserves_a_newer_revision() -> anyhow::Result<()> {
+ late_embedding_preserves_newer_revision(true, false).await
+}
+
+#[tokio::test]
+async fn a_late_embedding_cannot_accept_a_restored_sha_as_its_revision() -> anyhow::Result<()> {
+ late_embedding_preserves_newer_revision(false, true).await
+}
+
+#[tokio::test]
+async fn a_late_embedding_preserves_a_deleted_document() -> anyhow::Result<()> {
+ for fail_old in [false, true] {
+ let Some(f) = fixture(FakeEmbed::new(Duration::ZERO)).await? else {
+ return Ok(());
+ };
+ let doc = f.document_to_process(4).await?;
+ let (resume, old, server) = pause_embedding(&f, doc, fail_old).await?;
+ utopia_store::documents::delete(&f.pool, f.kb, doc, None).await?;
+ let deleted = processing_snapshot(&f, doc).await?;
+ assert!(deleted["document"]["deleted_at"].is_string());
+ assert_eq!(deleted["chunks"], serde_json::json!([]));
+ resume.notify_one();
+ let result = tokio::time::timeout(Duration::from_secs(10), old).await??;
+ server.abort();
+ let current = processing_snapshot(&f, doc).await?;
+ f.cleanup().await?;
+ result?;
+ assert_eq!(
+ current, deleted,
+ "deleted documents reject old success and failure"
+ );
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn an_embedding_failure_does_not_hide_a_database_write_error() -> anyhow::Result<()> {
+ let Some(mut f) = fixture(FakeEmbed::new(Duration::ZERO)).await? else {
+ return Ok(());
+ };
+ let url = utopia_store::test_db::url().expect("the fixture has a configured database");
+ let doc = f.document_to_process(4).await?;
+ let (resume, processing, server) = pause_embedding(&f, doc, true).await?;
+ f.pool.close().await;
+ resume.notify_one();
+ let result = tokio::time::timeout(Duration::from_secs(10), processing).await??;
+ server.abort();
+ // 重新连接只为拆夹具,处理器持有的旧连接池仍然关闭。
+ f.pool = sqlx::PgPool::connect(&url).await?;
+ f.cleanup().await?;
+ let error = result.expect_err("failure status could not be persisted");
+ assert!(
+ matches!(
+ error.downcast_ref::<utopia_core::AppError>(),
+ Some(utopia_core::AppError::Db(sqlx::Error::PoolClosed))
+ ),
+ "the database error must reach the caller: {error:#}"
+ );
+ Ok(())
+}
+
/// #611:一个 NUL 从前让整篇文档失败——坏的只是几个字节,丢的是整篇
#[tokio::test]
async fn a_nul_byte_does_not_fail_the_whole_document() -> anyhow::Result<()> {
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index 7072a99d..3822fbad 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -619,6 +619,57 @@ pub async fn get(pool: &PgPool, id: Uuid) -> AppResult<Document> {
.ok_or(AppError::NotFound)
}
+/// 一次处理的输入;版本号区分内容 A → B → A,不能只比较 SHA。
+pub struct ProcessingInput {
+ pub document: Document,
+ pub version: i32,
+}
+
+pub async fn processing_input(pool: &PgPool, id: Uuid) -> AppResult<ProcessingInput> {
+ let mut tx = pool.begin().await?;
+ let document = sqlx::query_as("SELECT * FROM documents WHERE id = $1 FOR NO KEY UPDATE")
+ .bind(id)
+ .fetch_optional(&mut *tx)
+ .await?
+ .ok_or(AppError::NotFound)?;
+ let version = processing_version_tx(&mut tx, id).await?;
+ tx.commit().await?;
+ Ok(ProcessingInput { document, version })
+}
+
+async fn processing_version_tx(tx: &mut Transaction<'_, Postgres>, id: Uuid) -> AppResult<i32> {
+ Ok(sqlx::query_scalar(
+ "SELECT coalesce(max(version), 0) FROM document_versions WHERE document_id = $1",
+ )
+ .bind(id)
+ .fetch_one(&mut **tx)
+ .await?)
+}
+
+/// 锁后再读版本:等待内容更新的事务提交后,要看到它刚追加的版本。
+async fn lock_processing_input_tx(
+ tx: &mut Transaction<'_, Postgres>,
+ input: &ProcessingInput,
+) -> AppResult<bool> {
+ let current: Option<(String, bool)> = sqlx::query_as(
+ "SELECT sha256, deleted_at IS NOT NULL FROM documents WHERE id = $1 FOR NO KEY UPDATE",
+ )
+ .bind(input.document.id)
+ .fetch_optional(&mut **tx)
+ .await?;
+ if !matches!(current, Some((ref sha, false)) if sha == &input.document.sha256) {
+ return Ok(false);
+ }
+ Ok(processing_version_tx(tx, input.document.id).await? == input.version)
+}
+
+pub async fn processing_is_current(pool: &PgPool, input: &ProcessingInput) -> AppResult<bool> {
+ let mut tx = pool.begin().await?;
+ let current = lock_processing_input_tx(&mut tx, input).await?;
+ tx.commit().await?;
+ Ok(current)
+}
+
/// A recorded original, with only the facts the ingestion ledger guarantees.
#[derive(Debug, Clone, Serialize, sqlx::FromRow)]
pub struct DocumentVersion {
@@ -771,6 +822,12 @@ pub async fn record_version(
sha256: &str,
size_bytes: i64,
) -> AppResult<()> {
+ let mut tx = pool.begin().await?;
+ // 版本追加也要和处理回写互斥,外键的 KEY SHARE 锁不会挡住它。
+ sqlx::query("SELECT id FROM documents WHERE id = $1 FOR NO KEY UPDATE")
+ .bind(document_id)
+ .execute(&mut *tx)
+ .await?;
sqlx::query(
"INSERT INTO document_versions (id, document_id, version, sha256, size_bytes, doc_time)
VALUES ($1, $2,
@@ -781,8 +838,9 @@ pub async fn record_version(
.bind(document_id)
.bind(sha256)
.bind(size_bytes)
- .execute(pool)
+ .execute(&mut *tx)
.await?;
+ tx.commit().await?;
Ok(())
}
@@ -858,32 +916,51 @@ pub async fn set_status(pool: &PgPool, id: Uuid, status: &str) -> AppResult<()>
Ok(())
}
-pub async fn set_failed(pool: &PgPool, id: Uuid, error: &str) -> AppResult<()> {
+pub async fn set_status_if_current(
+ pool: &PgPool,
+ input: &ProcessingInput,
+ status: &str,
+) -> AppResult<bool> {
+ let mut tx = pool.begin().await?;
+ if !lock_processing_input_tx(&mut tx, input).await? {
+ return Ok(false);
+ }
sqlx::query(
- "UPDATE documents SET status = 'failed', error = $2, reader_task = NULL, updated_at = now()
+ "UPDATE documents SET status = $2, error = NULL, reader_needed = NULL, updated_at = now()
WHERE id = $1",
)
- .bind(id)
- .bind(error)
- .execute(pool)
+ .bind(input.document.id)
+ .bind(status)
+ .execute(&mut *tx)
.await?;
- Ok(())
+ tx.commit().await?;
+ Ok(true)
}
-/// 这份文件要 `reader` 那一种模型才读得出字,而那种模型没配(0040):停在 failed,
-/// 记下缺的是哪一种,配上之后按它重新排队
-pub async fn set_needs_reader(pool: &PgPool, id: Uuid, reader: &str, error: &str) -> AppResult<()> {
+/// 当前输入失败才记下原因;缺读取模型时仍按原语义等待配上后重排。
+pub async fn set_failed_if_current(
+ pool: &PgPool,
+ input: &ProcessingInput,
+ error: &str,
+ reader: Option<&str>,
+) -> AppResult<bool> {
+ let mut tx = pool.begin().await?;
+ if !lock_processing_input_tx(&mut tx, input).await? {
+ return Ok(false);
+ }
sqlx::query(
- "UPDATE documents SET status = 'failed', error = $3, reader_needed = $2, reader_task = NULL,
+ "UPDATE documents SET status = 'failed', error = $2,
+ reader_needed = coalesce($3, reader_needed), reader_task = NULL,
updated_at = now()
WHERE id = $1",
)
- .bind(id)
- .bind(reader)
+ .bind(input.document.id)
.bind(error)
- .execute(pool)
+ .bind(reader)
+ .execute(&mut *tx)
.await?;
- Ok(())
+ tx.commit().await?;
+ Ok(true)
}
/// 配上了 `reader` 那一种模型:这个工作区里等着它的文档(活着的)重新排进处理队列。
@@ -971,6 +1048,44 @@ pub async fn set_ready(pool: &PgPool, id: Uuid, text_len: i32, chunk_count: i32)
Ok(())
}
+/// 就绪、抽取状态和入队一起提交,过期任务不能留下新版的后续工作。
+pub async fn finish_processing_if_current(
+ pool: &PgPool,
+ input: &ProcessingInput,
+ text_len: i32,
+ chunk_count: i32,
+ graph_status: Option<&str>,
+) -> AppResult<bool> {
+ let mut tx = pool.begin().await?;
+ if !lock_processing_input_tx(&mut tx, input).await? {
+ return Ok(false);
+ }
+ sqlx::query(
+ "UPDATE documents SET status = 'ready', error = NULL, reader_needed = NULL, reader_task = NULL,
+ text_len = $2, chunk_count = $3,
+ graph_status = coalesce($4, graph_status),
+ graph_error = CASE WHEN $4 IS NULL THEN graph_error ELSE NULL END,
+ updated_at = now() WHERE id = $1",
+ )
+ .bind(input.document.id)
+ .bind(text_len)
+ .bind(chunk_count)
+ .bind(graph_status)
+ .execute(&mut *tx)
+ .await?;
+ if graph_status == Some("queued") {
+ crate::jobs::enqueue_with_max_attempts_tx(
+ &mut tx,
+ "extract_document",
+ serde_json::json!({ "document_id": input.document.id }),
+ 3,
+ )
+ .await?;
+ }
+ tx.commit().await?;
+ Ok(true)
+}
+
/// 一次删除的产出,给审计与界面提示用。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DeletionReport {
@@ -1407,12 +1522,17 @@ pub async fn replace_chunks(
/// 返回 None 表示输入已过期,调用方应结束本次处理。
pub async fn replace_chunks_if_current(
pool: &PgPool,
- kb_id: Uuid,
- document_id: Uuid,
+ input: &ProcessingInput,
pieces: &[ChunkPiece],
- sha256: &str,
) -> AppResult<Option<Vec<(String, String)>>> {
- replace_chunks_for_snapshot(pool, kb_id, document_id, pieces, Some(sha256)).await
+ replace_chunks_for_snapshot(
+ pool,
+ input.document.kb_id,
+ input.document.id,
+ pieces,
+ Some(input),
+ )
+ .await
}
async fn replace_chunks_for_snapshot(
@@ -1420,7 +1540,7 @@ async fn replace_chunks_for_snapshot(
kb_id: Uuid,
document_id: Uuid,
pieces: &[ChunkPiece],
- sha256: Option<&str>,
+ input: Option<&ProcessingInput>,
) -> AppResult<Option<Vec<(String, String)>>> {
let mut tx = pool.begin().await?;
// 两个任务可能同时处理同一文档。先锁父记录,即使还没有分块,
@@ -1430,16 +1550,15 @@ async fn replace_chunks_for_snapshot(
// 冲突,于是这个事务活着的时候,这份文档所有子表的插入都被挡住(chunks、
// document_versions、memory::append),而这个事务是每个分块一个来回——四千块的
// 文档要锁四秒。两者对另一个 `replace_chunks` 的互斥是一样的
- let current: Option<(String, bool)> = sqlx::query_as(
- "SELECT sha256, deleted_at IS NOT NULL FROM documents WHERE id = $1 FOR NO KEY UPDATE",
- )
- .bind(document_id)
- .fetch_optional(&mut *tx)
- .await?;
- if let Some(expected) = sha256 {
- if !matches!(current, Some((ref sha, false)) if sha == expected) {
+ if let Some(input) = input {
+ if !lock_processing_input_tx(&mut tx, input).await? {
return Ok(None);
}
+ } else {
+ sqlx::query("SELECT id FROM documents WHERE id = $1 FOR NO KEY UPDATE")
+ .bind(document_id)
+ .execute(&mut *tx)
+ .await?;
}
let (version,): (i32,) = sqlx::query_as(
"SELECT COALESCE(MAX(version), 1) FROM document_versions WHERE document_id = $1",
@@ -1705,7 +1824,29 @@ pub async fn chunks_pending_embedding(
}
pub async fn set_embeddings(pool: &PgPool, items: &[(Uuid, Vec<f32>)]) -> AppResult<()> {
+ set_embeddings_for_input(pool, items, None).await?;
+ Ok(())
+}
+
+pub async fn set_embeddings_if_current(
+ pool: &PgPool,
+ input: &ProcessingInput,
+ items: &[(Uuid, Vec<f32>)],
+) -> AppResult<bool> {
+ set_embeddings_for_input(pool, items, Some(input)).await
+}
+
+async fn set_embeddings_for_input(
+ pool: &PgPool,
+ items: &[(Uuid, Vec<f32>)],
+ input: Option<&ProcessingInput>,
+) -> AppResult<bool> {
let mut tx = pool.begin().await?;
+ if let Some(input) = input {
+ if !lock_processing_input_tx(&mut tx, input).await? {
+ return Ok(false);
+ }
+ }
for (id, emb) in items {
sqlx::query("UPDATE chunks SET embedding = $2 WHERE id = $1")
.bind(id)
@@ -1719,7 +1860,7 @@ pub async fn set_embeddings(pool: &PgPool, items: &[(Uuid, Vec<f32>)]) -> AppRes
crate::vector_index::request(pool, crate::vector_index::Target::Chunks, first.len())
.await?;
}
- Ok(())
+ Ok(true)
}
/// 向量近邻检索(余弦距离)。维度写成字面量、两侧 cast、`relaxed_order`——三条