Skip to content

World's first open-source enterprise world model.

A purge judges its blobs in one question (#537)

WaylandYangf3a384714ccc
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index 7d0500c1..f822f635 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -358,16 +358,6 @@ pub async fn upsert_source_document_tx(
Ok(document)
}
-pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<Document>> {
- let rows = sqlx::query_as(
- "SELECT * FROM documents WHERE kb_id = $1 AND deleted_at IS NULL ORDER BY created_at DESC",
- )
- .bind(kb_id)
- .fetch_all(pool)
- .await?;
- Ok(rows)
-}
-
/// 文库一页,**带筛选与统计**。
///
/// 从前是 `SELECT * FROM documents WHERE kb_id = $1` 不带上限、前端客户端分页。
@@ -970,20 +960,23 @@ pub async fn purge(pool: &PgPool, kb_id: Uuid, id: Uuid) -> AppResult<PurgeRepor
.bind(id)
.execute(&mut *tx)
.await?;
- // 原文按内容寻址,别的文档(任何库)可能共用同一份:只交出没人再引用的
- let mut blobs = Vec::new();
- for s in shas {
- let (referenced,): (bool,) = sqlx::query_as(
- "SELECT EXISTS (SELECT 1 FROM documents WHERE sha256 = $1 AND purged_at IS NULL)
- OR EXISTS (SELECT 1 FROM document_versions WHERE sha256 = $1)",
- )
- .bind(&s)
- .fetch_one(&mut *tx)
- .await?;
- if !referenced {
- blobs.push(s);
- }
- }
+ // 原文按内容寻址,别的文档(任何库)可能共用同一份:只交出没人再引用的。
+ //
+ // **一句话判完所有指纹,而且必须站在这个位置。** 判断看的是版本行删掉之后的
+ // 状态:这篇文档的旧版本不再算引用,它独占的原文才收得回来。把这一问挪到
+ // 删除之前,每份原文都显得有人引用、永远交不出去,而且不报任何错(#516)。
+ // 从前每个指纹各问一次,一篇重解析过三十次的文档就是三十个往返,全在已握着
+ // 来源锁与文档行锁的事务里;现在是一个往返,两问各走 0043 的 sha 索引。
+ // 名单顺序照 shas:现行指纹排在历史之后,与从前一样。
+ let blobs: Vec<String> = sqlx::query_scalar(
+ "SELECT s FROM unnest($1::text[]) WITH ORDINALITY AS u(s, n)
+ WHERE NOT EXISTS (SELECT 1 FROM documents WHERE sha256 = s AND purged_at IS NULL)
+ AND NOT EXISTS (SELECT 1 FROM document_versions WHERE sha256 = s)
+ ORDER BY n",
+ )
+ .bind(&shas)
+ .fetch_all(&mut *tx)
+ .await?;
tx.commit().await?;
Ok(PurgeReport { blobs, chunks })
}
diff --git a/crates/utopia-store/tests/a_deletion_is_an_event.rs b/crates/utopia-store/tests/a_deletion_is_an_event.rs
index 6daea47c..63d908bd 100644
--- a/crates/utopia-store/tests/a_deletion_is_an_event.rs
+++ b/crates/utopia-store/tests/a_deletion_is_an_event.rs
@@ -176,8 +176,9 @@ async fn chunk_live(pool: &PgPool, id: Uuid) -> anyhow::Result<(bool, String)> {
}
async fn listed(pool: &PgPool, f: &Fx, id: Uuid) -> anyhow::Result<bool> {
- Ok(documents::list(pool, f.kb)
+ Ok(documents::page(pool, f.kb, None, None, None, false, 200, 0)
.await?
+ .docs
.iter()
.any(|d| d.id == id))
}
diff --git a/crates/utopia-store/tests/a_purge_judges_its_blobs_once.rs b/crates/utopia-store/tests/a_purge_judges_its_blobs_once.rs
new file mode 100644
index 00000000..c1890c3f
--- /dev/null
+++ b/crates/utopia-store/tests/a_purge_judges_its_blobs_once.rs
@@ -0,0 +1,265 @@
+//! #516:purge 一句话判完所有指纹,答案与逐个问一模一样。
+//!
+//! 原文按内容寻址、跨库共用,一份指纹只有在**没有任何地方**再引用时才交给调用方去删文件。
+//! 判错的两个方向不对称:多留是漏磁盘,多交是把别人还在用的原文删掉。从前每个指纹
+//! 单独问一次库,问多少次锁就多握多久;现在一句 unnest 问完,这里守的是答案一个不移:
+//!
+//! 1. 另一篇文档还拿着的指纹不交。
+//! 2. 另一个库的文档拿着也不交——那一问故意不带 kb_id。
+//! 3. 只有旧版本拿过、如今谁都不拿的指纹交出去:判断发生在版本行删掉之后。
+//! 4. 已 purge 的文档不算引用:它的行还在,指纹已经不归它了。
+//! 5. 同一个指纹在多个版本里出现,名单上只有一次。
+//! 6. 一句话的答案与逐个问的答案相等,这是整个改法的正确性论证。
+//!
+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
+
+use sqlx::PgPool;
+use utopia_store::documents;
+use uuid::Uuid;
+
+struct Fx {
+ pool: PgPool,
+ org: Uuid,
+ ws: Uuid,
+ kb: Uuid,
+ tag: String,
+}
+
+async fn seed() -> anyhow::Result<Option<Fx>> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(None);
+ };
+ let pool = PgPool::connect(&url).await?;
+ let (org, ws) = (Uuid::now_v7(), Uuid::now_v7());
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'purge-blobs-test')")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'purge-blobs-test')")
+ .bind(ws)
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ let kb = base(&pool, ws).await?;
+ // 指纹带本次运行的随机后缀,别和库里残留的撞上
+ let tag = Uuid::now_v7().simple().to_string();
+ Ok(Some(Fx {
+ pool,
+ org,
+ ws,
+ kb,
+ tag,
+ }))
+}
+
+async fn base(pool: &PgPool, ws: Uuid) -> anyhow::Result<Uuid> {
+ let kb = Uuid::now_v7();
+ sqlx::query(
+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'purge-blobs-test')",
+ )
+ .bind(kb)
+ .bind(ws)
+ .execute(pool)
+ .await?;
+ Ok(kb)
+}
+
+impl Fx {
+ fn sha(&self, name: &str) -> String {
+ format!("sha-{name}-{}", self.tag)
+ }
+
+ async fn document(&self, kb: Uuid, name: &str, sha: &str) -> anyhow::Result<Uuid> {
+ let id = Uuid::now_v7();
+ sqlx::query(
+ "INSERT INTO documents (id, kb_id, filename, sha256, status)
+ VALUES ($1, $2, $3, $4, 'ready')",
+ )
+ .bind(id)
+ .bind(kb)
+ .bind(name)
+ .bind(sha)
+ .execute(&self.pool)
+ .await?;
+ Ok(id)
+ }
+
+ async fn version(&self, doc: Uuid, version: i32, sha: &str) -> anyhow::Result<()> {
+ sqlx::query(
+ "INSERT INTO document_versions (id, document_id, version, sha256) VALUES ($1, $2, $3, $4)",
+ )
+ .bind(Uuid::now_v7())
+ .bind(doc)
+ .bind(version)
+ .bind(sha)
+ .execute(&self.pool)
+ .await?;
+ Ok(())
+ }
+
+ async fn purge(&self, kb: Uuid, doc: Uuid) -> anyhow::Result<Vec<String>> {
+ documents::delete(&self.pool, kb, doc, None).await?;
+ Ok(documents::purge(&self.pool, kb, doc).await?.blobs)
+ }
+
+ /// 从前的判法,逐个指纹问一次;第 6 条拿它当真值
+ async fn judged_one_by_one(&self, shas: &[String]) -> anyhow::Result<Vec<String>> {
+ let mut out = Vec::new();
+ for s in shas {
+ let (referenced,): (bool,) = sqlx::query_as(
+ "SELECT EXISTS (SELECT 1 FROM documents WHERE sha256 = $1 AND purged_at IS NULL)
+ OR EXISTS (SELECT 1 FROM document_versions WHERE sha256 = $1)",
+ )
+ .bind(s)
+ .fetch_one(&self.pool)
+ .await?;
+ if !referenced {
+ out.push(s.clone());
+ }
+ }
+ Ok(out)
+ }
+
+ async fn cleanup(self) -> anyhow::Result<()> {
+ sqlx::query("DELETE FROM organizations WHERE id = $1")
+ .bind(self.org)
+ .execute(&self.pool)
+ .await?;
+ Ok(())
+ }
+}
+
+#[tokio::test]
+async fn a_blob_another_document_still_holds_is_never_collected() -> anyhow::Result<()> {
+ let Some(f) = seed().await? else {
+ return Ok(());
+ };
+ // 同一个库里两篇活文档不能同 sha(documents_kb_sha_idx),所以同库的另一个
+ // 持有者只能是别人的历史版本
+ let shared = f.sha("shared");
+ let a = f.document(f.kb, "a.md", &shared).await?;
+ let b = f.document(f.kb, "b.md", &f.sha("b-now")).await?;
+ f.version(b, 1, &shared).await?;
+ assert!(
+ f.purge(f.kb, a).await?.is_empty(),
+ "b still holds the blob in a version"
+ );
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn a_blob_shared_across_knowledge_bases_is_never_collected() -> anyhow::Result<()> {
+ let Some(f) = seed().await? else {
+ return Ok(());
+ };
+ let shared = f.sha("shared");
+ let other_kb = base(&f.pool, f.ws).await?;
+ let a = f.document(f.kb, "a.md", &shared).await?;
+ let _elsewhere = f.document(other_kb, "e.md", &shared).await?;
+ assert!(
+ f.purge(f.kb, a).await?.is_empty(),
+ "the holder in another base counts; the check carries no kb_id on purpose"
+ );
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn a_blob_only_an_old_version_held_is_collected() -> anyhow::Result<()> {
+ let Some(f) = seed().await? else {
+ return Ok(());
+ };
+ let (current, old) = (f.sha("current"), f.sha("old"));
+ let a = f.document(f.kb, "a.md", &current).await?;
+ f.version(a, 1, &old).await?;
+ let mut blobs = f.purge(f.kb, a).await?;
+ blobs.sort();
+ let mut expected = vec![current, old];
+ expected.sort();
+ assert_eq!(
+ blobs, expected,
+ "the old version's blob is judged after the version row is gone"
+ );
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn a_purged_document_no_longer_holds_its_blob() -> anyhow::Result<()> {
+ let Some(f) = seed().await? else {
+ return Ok(());
+ };
+ // 两篇活文档同 sha 只能在两个库里(documents_kb_sha_idx)
+ let shared = f.sha("shared");
+ let other_kb = base(&f.pool, f.ws).await?;
+ let x = f.document(f.kb, "x.md", &shared).await?;
+ let y = f.document(other_kb, "y.md", &shared).await?;
+ assert!(f.purge(f.kb, x).await?.is_empty(), "y still holds it");
+ assert_eq!(
+ f.purge(other_kb, y).await?,
+ vec![shared],
+ "x's row is still there but purged, so it is not a holder"
+ );
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn every_sha_is_judged_exactly_once() -> anyhow::Result<()> {
+ let Some(f) = seed().await? else {
+ return Ok(());
+ };
+ let repeated = f.sha("repeated");
+ let a = f.document(f.kb, "a.md", &repeated).await?;
+ f.version(a, 1, &repeated).await?;
+ f.version(a, 2, &repeated).await?;
+ assert_eq!(
+ f.purge(f.kb, a).await?,
+ vec![repeated],
+ "one blob, one line, however many versions carried it"
+ );
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn the_batched_answer_equals_the_one_by_one_answer() -> anyhow::Result<()> {
+ let Some(f) = seed().await? else {
+ return Ok(());
+ };
+ let other_kb = base(&f.pool, f.ws).await?;
+ // 甲拿着五个指纹:现行一个、历史四个;其中两个别处还有人拿,一个已被 purge 的文档拿过
+ let (own, shared_doc, shared_far, held_by_purged, old_only) = (
+ f.sha("own"),
+ f.sha("shared-doc"),
+ f.sha("shared-far"),
+ f.sha("held-by-purged"),
+ f.sha("old-only"),
+ );
+ let a = f.document(f.kb, "a.md", &own).await?;
+ f.version(a, 1, &shared_doc).await?;
+ f.version(a, 2, &shared_far).await?;
+ f.version(a, 3, &held_by_purged).await?;
+ f.version(a, 4, &old_only).await?;
+ let _b = f.document(f.kb, "b.md", &shared_doc).await?;
+ let _far = f.document(other_kb, "far.md", &shared_far).await?;
+ let p = f.document(f.kb, "p.md", &held_by_purged).await?;
+ assert!(
+ f.purge(f.kb, p).await?.is_empty(),
+ "a still holds it in a version"
+ );
+
+ let shas: Vec<String> = sqlx::query_scalar(
+ "SELECT DISTINCT sha256 FROM document_versions WHERE document_id = $1
+ UNION SELECT sha256 FROM documents WHERE id = $1",
+ )
+ .bind(a)
+ .fetch_all(&f.pool)
+ .await?;
+ let mut batched = f.purge(f.kb, a).await?;
+ // 提交之后的库状态与事务内判断时相同:版本行已删、文档已标 purged
+ let mut one_by_one = f.judged_one_by_one(&shas).await?;
+ batched.sort();
+ one_by_one.sort();
+ assert_eq!(batched, one_by_one);
+ let mut expected = vec![own, held_by_purged, old_only];
+ expected.sort();
+ assert_eq!(batched, expected);
+ f.cleanup().await
+}
diff --git a/migrations/0043_a_blob_is_looked_up_by_its_hash.sql b/migrations/0043_a_blob_is_looked_up_by_its_hash.sql
new file mode 100644
index 00000000..59931974
--- /dev/null
+++ b/migrations/0043_a_blob_is_looked_up_by_its_hash.sql
@@ -0,0 +1,11 @@
+-- #516:purge 交出原文之前要问「还有谁引用这个 sha」,问的是两张表,两张表
+-- 都没有按 sha 查的索引。
+--
+-- documents 上那个 (kb_id, sha256) 的唯一索引服务不了这一问:原文按内容寻址、
+-- 跨库共用,问的时候故意不带 kb_id,前导列对不上就是整表扫。document_versions
+-- 只有 (document_id, version),按 sha 问一样是整表扫。而这两问跑在 purge 的事务
+-- 里,此时来源锁和文档行锁都已握着,扫多久别的写入就等多久。
+--
+-- CI 的 migrations job 会把全部迁移重放两遍,所以 IF NOT EXISTS。
+CREATE INDEX IF NOT EXISTS documents_sha_live_idx ON documents (sha256) WHERE purged_at IS NULL;
+CREATE INDEX IF NOT EXISTS document_versions_sha_idx ON document_versions (sha256);