World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Keep uploaded originals readable by committing their version and processing job together (#1204)
Virtue03955c4e5cfee89
diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rsindex 09ca883a..3d700e9d 100644--- a/crates/utopia-cli/src/main.rs+++ b/crates/utopia-cli/src/main.rs@@ -82,7 +82,7 @@ struct ManifestDataDir {/// not a side effect of a code change.// 是迁移文件的**个数**,不是最大的编号(守卫 `schema_version_policy_compares_against_current`// 按个数比):编号有空缺时两者不同——0071 由一个开放 PR 占着,0072 先落,个数是 71-const CURRENT_SCHEMA_VERSION: u32 = 99;+const CURRENT_SCHEMA_VERSION: u32 = 100;fn main() -> anyhow::Result<()> {dotenvy::dotenv().ok();diff --git a/crates/utopia-server/src/api/documents_routes.rs b/crates/utopia-server/src/api/documents_routes.rsindex 2eb689f8..3159c691 100644--- a/crates/utopia-server/src/api/documents_routes.rs+++ b/crates/utopia-server/src/api/documents_routes.rs@@ -305,15 +305,7 @@ pub async fn upload().await{- Ok(doc) => {- utopia_store::jobs::enqueue(- &state.pool,- "process_document",- json!({ "document_id": doc.id }),- )- .await?;- created.push(doc);- }+ Ok(doc) => created.push(doc),Err(AppError::Conflict(_)) => {skipped.push(json!({ "filename": filename, "reason": "duplicate content" }));}diff --git a/crates/utopia-server/src/api/documents_routes_tests.rs b/crates/utopia-server/src/api/documents_routes_tests.rsindex 623cb83c..c5292605 100644--- a/crates/utopia-server/src/api/documents_routes_tests.rs+++ b/crates/utopia-server/src/api/documents_routes_tests.rs@@ -597,6 +597,34 @@ async fn each_upload_keeps_its_own_date_before_processing_and_extraction() -> an(doc.doc_time, doc.doc_time_source.as_str(), doc.source_id),(date, "content", Some(f.folder)));+ let versions = documents::versions(&independent, doc.id).await?;+ assert_eq!(versions.len(), 1);+ assert_eq!(versions[0].version, 1);+ assert_eq!(versions[0].sha256, doc.sha256);+ assert_eq!(versions[0].size_bytes, doc.size_bytes);+ let version_date: Option<DateTime<Utc>> = sqlx::query_scalar(+ "SELECT doc_time FROM document_versions WHERE document_id=$1 AND version=1",+ )+ .bind(doc.id)+ .fetch_one(&independent)+ .await?;+ assert_eq!(version_date, date);+ let original = files+ .iter()+ .find(|(name, _)| *name == doc.filename)+ .unwrap()+ .1;+ for suffix in ["", "?version=1"] {+ let (status, headers, bytes) = f+ .get_raw(+ &format!("/api/v1/documents/{}/content{suffix}", doc.id),+ Some(&f.token),+ )+ .await?;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(headers["content-type"], "text/plain");+ assert_eq!(bytes, original.as_bytes());+ }// Claim only this test's job, using an independent connection as a worker would.let id: i64 = sqlx::query_scalar("UPDATE jobs SET status='running', locked_at=now() WHERE kind='process_document'@@ -641,6 +669,90 @@ async fn each_upload_keeps_its_own_date_before_processing_and_extraction() -> anf.cleanup().await}+#[tokio::test]+async fn reupload_restores_one_version_and_requeues_processing() -> anyhow::Result<()> {+ let Some(f) = Fixture::new().await? else {+ return Ok(());+ };+ let files = [("undated.txt", "An original without a date")];+ let (_, body) = f.upload(f.kb, "", &files).await?;+ let original = f.created_docs(&body).await?.remove(0);+ assert_eq!(original.doc_time_source, "none");+ documents::delete(&f.pool, f.kb, original.id, None).await?;+ let (status, body) = f.upload(f.kb, "", &files).await?;+ assert_eq!(status, StatusCode::OK);+ let restored = f.created_docs(&body).await?.remove(0);+ assert_eq!(restored.id, original.id);+ assert_eq!(restored.doc_time_source, "none");+ assert_eq!(documents::versions(&f.pool, restored.id).await?.len(), 1);+ let jobs: i64 = sqlx::query_scalar(+ "SELECT count(*) FROM jobs WHERE kind='process_document' AND payload->>'document_id'=$1",+ )+ .bind(restored.id.to_string())+ .fetch_one(&f.pool)+ .await?;+ assert_eq!(jobs, 2);+ let (status, _, bytes) = f+ .get_raw(+ &format!("/api/v1/documents/{}/content", restored.id),+ Some(&f.token),+ )+ .await?;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(bytes, files[0].1.as_bytes());+ f.cleanup().await+}++#[tokio::test]+async fn failed_upload_queue_rolls_back_creation_and_restoration() -> anyhow::Result<()> {+ let Some(f) = Fixture::new().await? else {+ return Ok(());+ };+ let original = [("original.txt", "Retained original")];+ let (_, body) = f.upload(f.kb, "", &original).await?;+ let doc = f.created_docs(&body).await?.remove(0);+ documents::delete(&f.pool, f.kb, doc.id, None).await?;+ let guard = format!("fail_upload_queue_{}", f.kb.simple());+ // Reject only this fixture's jobs, after the document and version were written.+ sqlx::raw_sql(&format!(+ "CREATE FUNCTION {guard}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN+ IF NEW.kind='process_document' AND EXISTS+ (SELECT 1 FROM documents WHERE id::text=NEW.payload->>'document_id' AND kb_id='{}')+ THEN RAISE EXCEPTION 'fixture rejects upload queue'; END IF;+ RETURN NEW; END $$;+ CREATE TRIGGER {guard} BEFORE INSERT ON jobs FOR EACH ROW EXECUTE FUNCTION {guard}();",+ f.kb+ ))+ .execute(&f.pool)+ .await?;+ let result = async {+ for files in [[("new.txt", "A new original")], original] {+ let (status, _) = f.upload(f.kb, "", &files).await?;+ assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);+ }+ let state: (i64, i64, i64) = sqlx::query_as(+ "SELECT (SELECT count(*) FROM documents WHERE kb_id=$1),+ (SELECT count(*) FROM document_versions v JOIN documents d ON d.id=v.document_id WHERE d.kb_id=$1),+ (SELECT count(*) FROM jobs WHERE payload->>'document_id'=$2)",+ )+ .bind(f.kb)+ .bind(doc.id.to_string())+ .fetch_one(&f.pool)+ .await?;+ assert_eq!(state, (1, 1, 1));+ assert!(documents::get(&f.pool, doc.id).await?.deleted_at.is_some());+ Ok::<(), anyhow::Error>(())+ }+ .await;+ sqlx::raw_sql(&format!(+ "DROP TRIGGER {guard} ON jobs; DROP FUNCTION {guard}();"+ ))+ .execute(&f.pool)+ .await?;+ f.cleanup().await?;+ result+}+/// `#610`:文件名里有日期时,日期来自文件名而不是正文——这一档要把/// `content_time` 走文件名优先的路径整条接到数据库层(不只是单测)。///diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rsindex 6980e9b3..acbce652 100644--- a/crates/utopia-store/src/documents.rs+++ b/crates/utopia-store/src/documents.rs@@ -46,19 +46,42 @@ pub async fn create_from_upload(source_id: Option<Uuid>,content_time: Option<DateTime<Utc>>,) -> AppResult<Document> {- create_with_time_source(- pool,- kb_id,- filename,- mime,- size_bytes,- sha256,- source_id,- content_time,- "content",- None,+ let mut tx = pool.begin().await?;+ let deleted: Option<Uuid> = sqlx::query_scalar(+ "SELECT id FROM documents+ WHERE kb_id = $1 AND sha256 = $2 AND deleted_at IS NOT NULL AND purged_at IS NULL",)- .await+ .bind(kb_id)+ .bind(sha256)+ .fetch_optional(&mut *tx)+ .await?;+ let document = if let Some(id) = deleted {+ let document = restore_tx(&mut tx, kb_id, id).await?;+ crate::jobs::enqueue_with_max_attempts_tx(+ &mut tx,+ "process_document",+ serde_json::json!({ "document_id": id }),+ 3,+ )+ .await?;+ document+ } else {+ create_with_version_and_processing_on(+ &mut tx,+ kb_id,+ filename,+ mime,+ size_bytes,+ sha256,+ source_id,+ content_time,+ "content",+ None,+ )+ .await?+ };+ tx.commit().await?;+ Ok(document)}#[allow(clippy::too_many_arguments)]@@ -130,6 +153,38 @@ pub async fn create_with_version_and_processing(external_key: Option<&str>,) -> AppResult<Document> {let mut tx = pool.begin().await?;+ let document = create_with_version_and_processing_on(+ &mut tx,+ kb_id,+ filename,+ mime,+ size_bytes,+ sha256,+ source_id,+ doc_time,+ "source",+ external_key,+ )+ .await?;+ tx.commit().await?;+ Ok(document)+}++// Content reads require the current digest in the version ledger. Upload used to+// commit only the document, so a successful upload could never serve its original.+#[allow(clippy::too_many_arguments)]+async fn create_with_version_and_processing_on(+ tx: &mut Transaction<'_, Postgres>,+ kb_id: Uuid,+ filename: &str,+ mime: &str,+ size_bytes: i64,+ sha256: &str,+ source_id: Option<Uuid>,+ doc_time: Option<DateTime<Utc>>,+ time_source: &str,+ external_key: Option<&str>,+) -> AppResult<Document> {let document: Document = sqlx::query_as("INSERT INTO documents (id, kb_id, filename, mime, size_bytes, sha256, source_id,doc_time, doc_time_source, external_key)@@ -144,9 +199,13 @@ pub async fn create_with_version_and_processing(.bind(source_id).bind(doc_time)// 没有日期就是没有日期:同步的时刻是记录时间,不是文档的日期(0045 决定 3,#714)- .bind(if doc_time.is_some() { "source" } else { "none" })+ .bind(if doc_time.is_some() {+ time_source+ } else {+ "none"+ }).bind(external_key)- .fetch_one(&mut *tx)+ .fetch_one(&mut **tx).await.map_err(|e| match &e {sqlx::Error::Database(db) if db.is_unique_violation() => AppError::Conflict(format!(@@ -162,16 +221,15 @@ pub async fn create_with_version_and_processing(.bind(document.id).bind(sha256).bind(size_bytes)- .execute(&mut *tx)+ .execute(&mut **tx).await?;crate::jobs::enqueue_with_max_attempts_tx(- &mut tx,+ tx,"process_document",serde_json::json!({ "document_id": document.id }),3,).await?;- tx.commit().await?;Ok(document)}diff --git a/crates/utopia-store/tests/store/an_upload_has_no_date.rs b/crates/utopia-store/tests/store/an_upload_has_no_date.rsindex 475b1a51..7ec077ba 100644--- a/crates/utopia-store/tests/store/an_upload_has_no_date.rs+++ b/crates/utopia-store/tests/store/an_upload_has_no_date.rs@@ -14,6 +14,76 @@ use uuid::Uuid;const ORG: &str = "upload-has-no-date-test";+#[tokio::test]+async fn version_repair_records_only_unversioned_retained_originals() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ utopia_store::db::migrate(&pool).await?;+ let mut tx = pool.begin().await?;+ // Temp copies retain the real migrated column types and checks. The repair+ // deliberately scans all documents; it must not touch concurrent fixtures.+ sqlx::raw_sql(+ "CREATE TEMP TABLE documents (LIKE public.documents INCLUDING ALL) ON COMMIT DROP;+ CREATE TEMP TABLE document_versions (LIKE public.document_versions INCLUDING ALL) ON COMMIT DROP;",+ )+ .execute(&mut *tx)+ .await?;+ let (current, recorded, purged, kb, version_id) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ let written = Utc.with_ymd_and_hms(2024, 2, 29, 0, 0, 0).unwrap();+ sqlx::query(+ "INSERT INTO documents(id,kb_id,filename,sha256,size_bytes,doc_time,doc_time_source,purged_at)+ VALUES ($1,$4,'current.txt','current',7,$5,'content',NULL),+ ($2,$4,'recorded.txt','current-2',9,NULL,'none',NULL),+ ($3,$4,'purged.txt','purged',6,NULL,'none',now())",+ )+ .bind(current).bind(recorded).bind(purged).bind(kb).bind(written)+ .execute(&mut *tx).await?;+ sqlx::query(+ "INSERT INTO document_versions(id,document_id,version,sha256,size_bytes,ingested_at)+ VALUES ($1,$2,3,'historical',4,'2020-01-01T00:00:00Z')",+ )+ .bind(version_id)+ .bind(recorded)+ .execute(&mut *tx)+ .await?;+ let migration =+ include_str!("../../../../migrations/0111_an_uploaded_original_has_a_version.sql");+ for _ in 0..2 {+ sqlx::raw_sql(migration).execute(&mut *tx).await?;+ }+ let current_version: (i32, String, i64, Option<chrono::DateTime<Utc>>, bool) = sqlx::query_as(+ "SELECT version,sha256,size_bytes,doc_time,ingested_at>=transaction_timestamp()+ FROM document_versions WHERE document_id=$1",+ )+ .bind(current)+ .fetch_one(&mut *tx)+ .await?;+ assert_eq!(+ current_version,+ (1, "current".into(), 7, Some(written), true)+ );+ let versions: Vec<(Uuid, Uuid, i32, String)> = sqlx::query_as(+ "SELECT id,document_id,version,sha256 FROM document_versions WHERE document_id<>$1",+ )+ .bind(current)+ .fetch_all(&mut *tx)+ .await?;+ assert_eq!(+ versions,+ vec![(version_id, recorded, 3, "historical".into())]+ );+ tx.rollback().await?;+ Ok(())+}+async fn seed(pool: &PgPool) -> anyhow::Result<(Uuid, Uuid)> {let (org, ws, kb, src) = (Uuid::now_v7(),diff --git a/migrations/0111_an_uploaded_original_has_a_version.sql b/migrations/0111_an_uploaded_original_has_a_version.sqlnew file mode 100644index 00000000..f5ccffed--- /dev/null+++ b/migrations/0111_an_uploaded_original_has_a_version.sql@@ -0,0 +1,8 @@+-- Upload previously inserted only documents. Record the retained current content,+-- at repair time: no missing historical revisions or ingestion times are invented.+-- Purge deliberately removes versions; an existing ledger is left untouched.+INSERT INTO document_versions (id, document_id, version, sha256, size_bytes, doc_time)+SELECT gen_random_uuid(), d.id, 1, d.sha256, d.size_bytes, d.doc_time+FROM documents d+WHERE d.purged_at IS NULL+ AND NOT EXISTS (SELECT 1 FROM document_versions v WHERE v.document_id = d.id);