Skip to content

World's first open-source enterprise world model.

Keep uploaded originals readable by committing their version and processing job together (#1204)

Virtue03955c4e5cfee89
diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs
index 09ca883a..3d700e9d 100644
--- a/crates/utopia-cli/src/main.rs
+++ b/crates/utopia-cli/src/main.rs
@@ -82,7 +82,7 @@ struct ManifestDataDir {
/// not a side effect of a code change.
// 是迁移文件的**个数**,不是最大的编号(守卫 `schema_version_policy_compares_against_current`
// 按个数比):编号有空缺时两者不同——0071 由一个开放 PR 占着,0072 先落,个数是 71
-const CURRENT_SCHEMA_VERSION: u32 = 99;
+const CURRENT_SCHEMA_VERSION: u32 = 100;
fn main() -> anyhow::Result<()> {
dotenvy::dotenv().ok();
diff --git a/crates/utopia-server/src/api/documents_routes.rs b/crates/utopia-server/src/api/documents_routes.rs
index 2eb689f8..3159c691 100644
--- a/crates/utopia-server/src/api/documents_routes.rs
+++ b/crates/utopia-server/src/api/documents_routes.rs
@@ -305,15 +305,7 @@ pub async fn upload(
)
.await
{
- Ok(doc) => {
- utopia_store::jobs::enqueue(
- &state.pool,
- "process_document",
- json!({ "document_id": doc.id }),
- )
- .await?;
- created.push(doc);
- }
+ Ok(doc) => created.push(doc),
Err(AppError::Conflict(_)) => {
skipped.push(json!({ "filename": filename, "reason": "duplicate content" }));
}
diff --git a/crates/utopia-server/src/api/documents_routes_tests.rs b/crates/utopia-server/src/api/documents_routes_tests.rs
index 623cb83c..c5292605 100644
--- a/crates/utopia-server/src/api/documents_routes_tests.rs
+++ b/crates/utopia-server/src/api/documents_routes_tests.rs
@@ -597,6 +597,34 @@ async fn each_upload_keeps_its_own_date_before_processing_and_extraction() -> an
(doc.doc_time, doc.doc_time_source.as_str(), doc.source_id),
(date, "content", Some(f.folder))
);
+ let versions = documents::versions(&independent, doc.id).await?;
+ assert_eq!(versions.len(), 1);
+ assert_eq!(versions[0].version, 1);
+ assert_eq!(versions[0].sha256, doc.sha256);
+ assert_eq!(versions[0].size_bytes, doc.size_bytes);
+ let version_date: Option<DateTime<Utc>> = sqlx::query_scalar(
+ "SELECT doc_time FROM document_versions WHERE document_id=$1 AND version=1",
+ )
+ .bind(doc.id)
+ .fetch_one(&independent)
+ .await?;
+ assert_eq!(version_date, date);
+ let original = files
+ .iter()
+ .find(|(name, _)| *name == doc.filename)
+ .unwrap()
+ .1;
+ for suffix in ["", "?version=1"] {
+ let (status, headers, bytes) = f
+ .get_raw(
+ &format!("/api/v1/documents/{}/content{suffix}", doc.id),
+ Some(&f.token),
+ )
+ .await?;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(headers["content-type"], "text/plain");
+ assert_eq!(bytes, original.as_bytes());
+ }
// Claim only this test's job, using an independent connection as a worker would.
let id: i64 = sqlx::query_scalar(
"UPDATE jobs SET status='running', locked_at=now() WHERE kind='process_document'
@@ -641,6 +669,90 @@ async fn each_upload_keeps_its_own_date_before_processing_and_extraction() -> an
f.cleanup().await
}
+#[tokio::test]
+async fn reupload_restores_one_version_and_requeues_processing() -> anyhow::Result<()> {
+ let Some(f) = Fixture::new().await? else {
+ return Ok(());
+ };
+ let files = [("undated.txt", "An original without a date")];
+ let (_, body) = f.upload(f.kb, "", &files).await?;
+ let original = f.created_docs(&body).await?.remove(0);
+ assert_eq!(original.doc_time_source, "none");
+ documents::delete(&f.pool, f.kb, original.id, None).await?;
+ let (status, body) = f.upload(f.kb, "", &files).await?;
+ assert_eq!(status, StatusCode::OK);
+ let restored = f.created_docs(&body).await?.remove(0);
+ assert_eq!(restored.id, original.id);
+ assert_eq!(restored.doc_time_source, "none");
+ assert_eq!(documents::versions(&f.pool, restored.id).await?.len(), 1);
+ let jobs: i64 = sqlx::query_scalar(
+ "SELECT count(*) FROM jobs WHERE kind='process_document' AND payload->>'document_id'=$1",
+ )
+ .bind(restored.id.to_string())
+ .fetch_one(&f.pool)
+ .await?;
+ assert_eq!(jobs, 2);
+ let (status, _, bytes) = f
+ .get_raw(
+ &format!("/api/v1/documents/{}/content", restored.id),
+ Some(&f.token),
+ )
+ .await?;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(bytes, files[0].1.as_bytes());
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn failed_upload_queue_rolls_back_creation_and_restoration() -> anyhow::Result<()> {
+ let Some(f) = Fixture::new().await? else {
+ return Ok(());
+ };
+ let original = [("original.txt", "Retained original")];
+ let (_, body) = f.upload(f.kb, "", &original).await?;
+ let doc = f.created_docs(&body).await?.remove(0);
+ documents::delete(&f.pool, f.kb, doc.id, None).await?;
+ let guard = format!("fail_upload_queue_{}", f.kb.simple());
+ // Reject only this fixture's jobs, after the document and version were written.
+ sqlx::raw_sql(&format!(
+ "CREATE FUNCTION {guard}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN
+ IF NEW.kind='process_document' AND EXISTS
+ (SELECT 1 FROM documents WHERE id::text=NEW.payload->>'document_id' AND kb_id='{}')
+ THEN RAISE EXCEPTION 'fixture rejects upload queue'; END IF;
+ RETURN NEW; END $$;
+ CREATE TRIGGER {guard} BEFORE INSERT ON jobs FOR EACH ROW EXECUTE FUNCTION {guard}();",
+ f.kb
+ ))
+ .execute(&f.pool)
+ .await?;
+ let result = async {
+ for files in [[("new.txt", "A new original")], original] {
+ let (status, _) = f.upload(f.kb, "", &files).await?;
+ assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
+ }
+ let state: (i64, i64, i64) = sqlx::query_as(
+ "SELECT (SELECT count(*) FROM documents WHERE kb_id=$1),
+ (SELECT count(*) FROM document_versions v JOIN documents d ON d.id=v.document_id WHERE d.kb_id=$1),
+ (SELECT count(*) FROM jobs WHERE payload->>'document_id'=$2)",
+ )
+ .bind(f.kb)
+ .bind(doc.id.to_string())
+ .fetch_one(&f.pool)
+ .await?;
+ assert_eq!(state, (1, 1, 1));
+ assert!(documents::get(&f.pool, doc.id).await?.deleted_at.is_some());
+ Ok::<(), anyhow::Error>(())
+ }
+ .await;
+ sqlx::raw_sql(&format!(
+ "DROP TRIGGER {guard} ON jobs; DROP FUNCTION {guard}();"
+ ))
+ .execute(&f.pool)
+ .await?;
+ f.cleanup().await?;
+ result
+}
+
/// `#610`:文件名里有日期时,日期来自文件名而不是正文——这一档要把
/// `content_time` 走文件名优先的路径整条接到数据库层(不只是单测)。
///
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index 6980e9b3..acbce652 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -46,19 +46,42 @@ pub async fn create_from_upload(
source_id: Option<Uuid>,
content_time: Option<DateTime<Utc>>,
) -> AppResult<Document> {
- create_with_time_source(
- pool,
- kb_id,
- filename,
- mime,
- size_bytes,
- sha256,
- source_id,
- content_time,
- "content",
- None,
+ let mut tx = pool.begin().await?;
+ let deleted: Option<Uuid> = sqlx::query_scalar(
+ "SELECT id FROM documents
+ WHERE kb_id = $1 AND sha256 = $2 AND deleted_at IS NOT NULL AND purged_at IS NULL",
)
- .await
+ .bind(kb_id)
+ .bind(sha256)
+ .fetch_optional(&mut *tx)
+ .await?;
+ let document = if let Some(id) = deleted {
+ let document = restore_tx(&mut tx, kb_id, id).await?;
+ crate::jobs::enqueue_with_max_attempts_tx(
+ &mut tx,
+ "process_document",
+ serde_json::json!({ "document_id": id }),
+ 3,
+ )
+ .await?;
+ document
+ } else {
+ create_with_version_and_processing_on(
+ &mut tx,
+ kb_id,
+ filename,
+ mime,
+ size_bytes,
+ sha256,
+ source_id,
+ content_time,
+ "content",
+ None,
+ )
+ .await?
+ };
+ tx.commit().await?;
+ Ok(document)
}
#[allow(clippy::too_many_arguments)]
@@ -130,6 +153,38 @@ pub async fn create_with_version_and_processing(
external_key: Option<&str>,
) -> AppResult<Document> {
let mut tx = pool.begin().await?;
+ let document = create_with_version_and_processing_on(
+ &mut tx,
+ kb_id,
+ filename,
+ mime,
+ size_bytes,
+ sha256,
+ source_id,
+ doc_time,
+ "source",
+ external_key,
+ )
+ .await?;
+ tx.commit().await?;
+ Ok(document)
+}
+
+// Content reads require the current digest in the version ledger. Upload used to
+// commit only the document, so a successful upload could never serve its original.
+#[allow(clippy::too_many_arguments)]
+async fn create_with_version_and_processing_on(
+ tx: &mut Transaction<'_, Postgres>,
+ kb_id: Uuid,
+ filename: &str,
+ mime: &str,
+ size_bytes: i64,
+ sha256: &str,
+ source_id: Option<Uuid>,
+ doc_time: Option<DateTime<Utc>>,
+ time_source: &str,
+ external_key: Option<&str>,
+) -> AppResult<Document> {
let document: Document = sqlx::query_as(
"INSERT INTO documents (id, kb_id, filename, mime, size_bytes, sha256, source_id,
doc_time, doc_time_source, external_key)
@@ -144,9 +199,13 @@ pub async fn create_with_version_and_processing(
.bind(source_id)
.bind(doc_time)
// 没有日期就是没有日期:同步的时刻是记录时间,不是文档的日期(0045 决定 3,#714)
- .bind(if doc_time.is_some() { "source" } else { "none" })
+ .bind(if doc_time.is_some() {
+ time_source
+ } else {
+ "none"
+ })
.bind(external_key)
- .fetch_one(&mut *tx)
+ .fetch_one(&mut **tx)
.await
.map_err(|e| match &e {
sqlx::Error::Database(db) if db.is_unique_violation() => AppError::Conflict(format!(
@@ -162,16 +221,15 @@ pub async fn create_with_version_and_processing(
.bind(document.id)
.bind(sha256)
.bind(size_bytes)
- .execute(&mut *tx)
+ .execute(&mut **tx)
.await?;
crate::jobs::enqueue_with_max_attempts_tx(
- &mut tx,
+ tx,
"process_document",
serde_json::json!({ "document_id": document.id }),
3,
)
.await?;
- tx.commit().await?;
Ok(document)
}
diff --git a/crates/utopia-store/tests/store/an_upload_has_no_date.rs b/crates/utopia-store/tests/store/an_upload_has_no_date.rs
index 475b1a51..7ec077ba 100644
--- a/crates/utopia-store/tests/store/an_upload_has_no_date.rs
+++ b/crates/utopia-store/tests/store/an_upload_has_no_date.rs
@@ -14,6 +14,76 @@ use uuid::Uuid;
const ORG: &str = "upload-has-no-date-test";
+#[tokio::test]
+async fn version_repair_records_only_unversioned_retained_originals() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ utopia_store::db::migrate(&pool).await?;
+ let mut tx = pool.begin().await?;
+ // Temp copies retain the real migrated column types and checks. The repair
+ // deliberately scans all documents; it must not touch concurrent fixtures.
+ sqlx::raw_sql(
+ "CREATE TEMP TABLE documents (LIKE public.documents INCLUDING ALL) ON COMMIT DROP;
+ CREATE TEMP TABLE document_versions (LIKE public.document_versions INCLUDING ALL) ON COMMIT DROP;",
+ )
+ .execute(&mut *tx)
+ .await?;
+ let (current, recorded, purged, kb, version_id) = (
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ );
+ let written = Utc.with_ymd_and_hms(2024, 2, 29, 0, 0, 0).unwrap();
+ sqlx::query(
+ "INSERT INTO documents(id,kb_id,filename,sha256,size_bytes,doc_time,doc_time_source,purged_at)
+ VALUES ($1,$4,'current.txt','current',7,$5,'content',NULL),
+ ($2,$4,'recorded.txt','current-2',9,NULL,'none',NULL),
+ ($3,$4,'purged.txt','purged',6,NULL,'none',now())",
+ )
+ .bind(current).bind(recorded).bind(purged).bind(kb).bind(written)
+ .execute(&mut *tx).await?;
+ sqlx::query(
+ "INSERT INTO document_versions(id,document_id,version,sha256,size_bytes,ingested_at)
+ VALUES ($1,$2,3,'historical',4,'2020-01-01T00:00:00Z')",
+ )
+ .bind(version_id)
+ .bind(recorded)
+ .execute(&mut *tx)
+ .await?;
+ let migration =
+ include_str!("../../../../migrations/0111_an_uploaded_original_has_a_version.sql");
+ for _ in 0..2 {
+ sqlx::raw_sql(migration).execute(&mut *tx).await?;
+ }
+ let current_version: (i32, String, i64, Option<chrono::DateTime<Utc>>, bool) = sqlx::query_as(
+ "SELECT version,sha256,size_bytes,doc_time,ingested_at>=transaction_timestamp()
+ FROM document_versions WHERE document_id=$1",
+ )
+ .bind(current)
+ .fetch_one(&mut *tx)
+ .await?;
+ assert_eq!(
+ current_version,
+ (1, "current".into(), 7, Some(written), true)
+ );
+ let versions: Vec<(Uuid, Uuid, i32, String)> = sqlx::query_as(
+ "SELECT id,document_id,version,sha256 FROM document_versions WHERE document_id<>$1",
+ )
+ .bind(current)
+ .fetch_all(&mut *tx)
+ .await?;
+ assert_eq!(
+ versions,
+ vec![(version_id, recorded, 3, "historical".into())]
+ );
+ tx.rollback().await?;
+ Ok(())
+}
+
async fn seed(pool: &PgPool) -> anyhow::Result<(Uuid, Uuid)> {
let (org, ws, kb, src) = (
Uuid::now_v7(),
diff --git a/migrations/0111_an_uploaded_original_has_a_version.sql b/migrations/0111_an_uploaded_original_has_a_version.sql
new file mode 100644
index 00000000..f5ccffed
--- /dev/null
+++ b/migrations/0111_an_uploaded_original_has_a_version.sql
@@ -0,0 +1,8 @@
+-- Upload previously inserted only documents. Record the retained current content,
+-- at repair time: no missing historical revisions or ingestion times are invented.
+-- Purge deliberately removes versions; an existing ledger is left untouched.
+INSERT INTO document_versions (id, document_id, version, sha256, size_bytes, doc_time)
+SELECT gen_random_uuid(), d.id, 1, d.sha256, d.size_bytes, d.doc_time
+FROM documents d
+WHERE d.purged_at IS NULL
+ AND NOT EXISTS (SELECT 1 FROM document_versions v WHERE v.document_id = d.id);