Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update vault pruner to always select item in a explicit manifest (#2159)
Leendert de Borstfab9684a5877
diff --git a/core/rust/src/vault_pruner/mod.rs b/core/rust/src/vault_pruner/mod.rsindex e02bea9d6..7ab861400 100644--- a/core/rust/src/vault_pruner/mod.rs+++ b/core/rust/src/vault_pruner/mod.rs@@ -17,7 +17,7 @@ use crate::vault_model::names::{DELETED_AT_COL, FILE_DATA_COL, ID_COL, IS_DELETED_COL, ITEMS_TABLE, ITEM_ID_COL, KIND_COL, LOGOS_TABLE,LOGO_ID_COL, LOGO_KIND_FAVICON, UPDATED_AT_COL,};-use crate::vault_model::{BLOB_COLUMNS, SYNCABLE_TABLES, TRASH_RETENTION_DEFAULT_DAYS};+use crate::vault_model::{BLOB_COLUMNS, MANIFEST_ID_COL, SYNCABLE_TABLES, TRASH_RETENTION_DEFAULT_DAYS};/// Input for the prune operation.#[derive(Debug, Clone, Serialize, Deserialize)]@@ -85,20 +85,20 @@ pub struct PruneTableQuery {pub fn get_prune_table_queries() -> Vec<PruneTableQuery> {let mut queries = vec![PruneTableQuery {name: ITEMS_TABLE.to_string(),- query: format!("SELECT {}, {}, {}, {} FROM {}", ID_COL, IS_DELETED_COL, DELETED_AT_COL, LOGO_ID_COL, ITEMS_TABLE),+ query: format!("SELECT {}, {}, {}, {}, {} FROM {}", MANIFEST_ID_COL, ID_COL, IS_DELETED_COL, DELETED_AT_COL, LOGO_ID_COL, ITEMS_TABLE),}];// One query per registered item-child table (TableConfig::item_child), so a table added to the// registry is read by the pruner automatically.for child in item_child_tables() {let query = match blob_column_for(child.name) {- Some(blob_col) => format!("SELECT {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", ID_COL, ITEM_ID_COL, IS_DELETED_COL, blob_col, blob_col, child.name),- None => format!("SELECT {}, {} FROM {}", child.item_ref_column(), IS_DELETED_COL, child.name),+ Some(blob_col) => format!("SELECT {}, {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", MANIFEST_ID_COL, ID_COL, ITEM_ID_COL, IS_DELETED_COL, blob_col, blob_col, child.name),+ None => format!("SELECT {}, {}, {} FROM {}", MANIFEST_ID_COL, child.item_ref_column(), IS_DELETED_COL, child.name),};queries.push(PruneTableQuery { name: child.name.to_string(), query });}queries.push(PruneTableQuery {name: LOGOS_TABLE.to_string(),- query: format!("SELECT {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", ID_COL, KIND_COL, IS_DELETED_COL, FILE_DATA_COL, FILE_DATA_COL, LOGOS_TABLE),+ query: format!("SELECT {}, {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", MANIFEST_ID_COL, ID_COL, KIND_COL, IS_DELETED_COL, FILE_DATA_COL, FILE_DATA_COL, LOGOS_TABLE),});queries}@@ -123,6 +123,14 @@ fn records_of<'a>(tables: &'a [CodecTableData], name: &str) -> Option<&'a [Codectables.iter().find(|t| t.name == name).map(|t| t.records.as_slice())}+/*+ * The manifest a row lives in. The same id can appear in multiple manifests, so every statement+ * addresses its rows by `(ManifestId, Id)`.+ */+fn manifest_of(record: &CodecRecord) -> &str {+ str_col(record, MANIFEST_ID_COL).unwrap_or("")+}+/// Main entry point: prune expired items from trash.////// Finds all Items with DeletedAt older than `retention_days` and generates SQL statements that mark@@ -143,13 +151,13 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {};let expired = expired_item_ids(items, cutoff_date);- for item_id in &expired {+ for (manifest_id, item_id) in &expired {statements.push(SqlStatement {- sql: format!("UPDATE {} SET {} = 1, {} = ? WHERE {} = ?", ITEMS_TABLE, IS_DELETED_COL, UPDATED_AT_COL, ID_COL),- params: vec![serde_json::json!(now_str), serde_json::json!(item_id)],+ sql: format!("UPDATE {} SET {} = 1, {} = ? WHERE {} = ? AND {} = ?", ITEMS_TABLE, IS_DELETED_COL, UPDATED_AT_COL, ID_COL, MANIFEST_ID_COL),+ params: vec![serde_json::json!(now_str), serde_json::json!(item_id), serde_json::json!(manifest_id)],});stats.items_pruned += 1;- tombstone_item_children(&input.tables, item_id, &now_str, &mut statements, &mut stats);+ tombstone_item_children(&input.tables, manifest_id, item_id, &now_str, &mut statements, &mut stats);}sweep_orphan_favicons(&input.tables, items, &expired, &now_str, &mut statements, &mut stats);@@ -158,15 +166,15 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {Ok(PruneOutput { success: true, statements, stats })}-/// Pass 1: the ids of live items whose trash date (`DeletedAt`) lies before the cutoff.-fn expired_item_ids(items: &[CodecRecord], cutoff_date: DateTime<Utc>) -> Vec<String> {+/// Pass 1: the `(ManifestId, Id)` of live items whose trash date (`DeletedAt`) lies before the cutoff.+fn expired_item_ids(items: &[CodecRecord], cutoff_date: DateTime<Utc>) -> Vec<(String, String)> {let mut expired = Vec::new();for item in items.iter().filter(|item| !is_deleted(item)) {let Some(deleted_at) = str_col(item, DELETED_AT_COL) else { continue };let Some(deleted_date) = crate::timestamp::parse_vault_datetime(deleted_at) else { continue };if deleted_date < cutoff_date {if let Some(id) = str_col(item, ID_COL) {- expired.push(id.to_string());+ expired.push((manifest_of(item).to_string(), id.to_string()));}}}@@ -179,21 +187,21 @@ fn expired_item_ids(items: &[CodecRecord], cutoff_date: DateTime<Utc>) -> Vec<St* extracted blob column has its bytes dropped in the same statement, leaving the column non-null* while reclaiming the storage on the next save.*/-fn tombstone_item_children(tables: &[CodecTableData], item_id: &str, now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {+fn tombstone_item_children(tables: &[CodecTableData], manifest_id: &str, item_id: &str, now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {for child in item_child_tables() {let Some(records) = records_of(tables, child.name) else { continue };let match_col = child.item_ref_column();- let related_count = count_related_records(records, match_col, item_id);+ let related_count = count_related_records(records, manifest_id, match_col, item_id);if related_count == 0 {continue;}let blob_clear = blob_clear_fragment(child.name);statements.push(SqlStatement {sql: format!(- "UPDATE {} SET {} = 1{}, {} = ? WHERE {} = ? AND {} = 0",- child.name, IS_DELETED_COL, blob_clear, UPDATED_AT_COL, match_col, IS_DELETED_COL+ "UPDATE {} SET {} = 1{}, {} = ? WHERE {} = ? AND {} = ? AND {} = 0",+ child.name, IS_DELETED_COL, blob_clear, UPDATED_AT_COL, match_col, MANIFEST_ID_COL, IS_DELETED_COL),- params: vec![serde_json::json!(now_str), serde_json::json!(item_id)],+ params: vec![serde_json::json!(now_str), serde_json::json!(item_id), serde_json::json!(manifest_id)],});*stats.child_rows_pruned.entry(child.name.to_string()).or_default() += related_count;}@@ -213,19 +221,19 @@ fn tombstone_item_children(tables: &[CodecTableData], item_id: &str, now_str: &sfn sweep_orphan_favicons(tables: &[CodecTableData],items: &[CodecRecord],- expired: &[String],+ expired: &[(String, String)],now_str: &str,statements: &mut Vec<SqlStatement>,stats: &mut PruneStats,) {let Some(logos) = records_of(tables, LOGOS_TABLE) else { return };let blob_clear = blob_clear_fragment(LOGOS_TABLE);- let expired_set: HashSet<&str> = expired.iter().map(String::as_str).collect();+ let expired_set: HashSet<(&str, &str)> = expired.iter().map(|(manifest_id, id)| (manifest_id.as_str(), id.as_str())).collect();- let referenced_logo_ids: HashSet<&str> = items.iter()+ let referenced_logos: HashSet<(&str, &str)> = items.iter().filter(|item| !is_deleted(item))- .filter(|item| !expired_set.contains(str_col(item, ID_COL).unwrap_or("")))- .filter_map(|item| str_col(item, LOGO_ID_COL))+ .filter(|item| !expired_set.contains(&(manifest_of(item), str_col(item, ID_COL).unwrap_or(""))))+ .filter_map(|item| str_col(item, LOGO_ID_COL).map(|logo_id| (manifest_of(item), logo_id))).collect();for logo in logos.iter().filter(|logo| !is_deleted(logo)) {@@ -233,12 +241,12 @@ fn sweep_orphan_favicons(continue;}let Some(logo_id) = str_col(logo, ID_COL) else { continue };- if referenced_logo_ids.contains(logo_id) {+ if referenced_logos.contains(&(manifest_of(logo), logo_id)) {continue;}statements.push(SqlStatement {- sql: format!("UPDATE {} SET {} = 1{}, {} = ? WHERE {} = ?", LOGOS_TABLE, IS_DELETED_COL, blob_clear, UPDATED_AT_COL, ID_COL),- params: vec![serde_json::json!(now_str), serde_json::json!(logo_id)],+ sql: format!("UPDATE {} SET {} = 1{}, {} = ? WHERE {} = ? AND {} = ?", LOGOS_TABLE, IS_DELETED_COL, blob_clear, UPDATED_AT_COL, ID_COL, MANIFEST_ID_COL),+ params: vec![serde_json::json!(now_str), serde_json::json!(logo_id), serde_json::json!(manifest_of(logo))],});stats.logos_pruned += 1;}@@ -258,17 +266,17 @@ fn clear_tombstoned_blobs(tables: &[CodecTableData], now_str: &str, statements:for row in stale {let Some(id) = str_col(row, ID_COL) else { continue };statements.push(SqlStatement {- sql: format!("UPDATE {} SET {} = NULL, {} = ? WHERE {} = ?", table, blob_col, UPDATED_AT_COL, ID_COL),- params: vec![serde_json::json!(now_str), serde_json::json!(id)],+ sql: format!("UPDATE {} SET {} = NULL, {} = ? WHERE {} = ? AND {} = ?", table, blob_col, UPDATED_AT_COL, ID_COL, MANIFEST_ID_COL),+ params: vec![serde_json::json!(now_str), serde_json::json!(id), serde_json::json!(manifest_of(row))],});*stats.blobs_cleared.entry(table.to_string()).or_default() += 1;}}}-/// Count related records that match a foreign key value and are not already deleted.-fn count_related_records(records: &[CodecRecord], fk_column: &str, fk_value: &str) -> u32 {- records.iter().filter(|r| str_col(r, fk_column) == Some(fk_value) && !is_deleted(r)).count() as u32+/// Count the live records of one manifest that match a foreign key value.+fn count_related_records(records: &[CodecRecord], manifest_id: &str, fk_column: &str, fk_value: &str) -> u32 {+ records.iter().filter(|r| manifest_of(r) == manifest_id && str_col(r, fk_column) == Some(fk_value) && !is_deleted(r)).count() as u32}#[cfg(test)]diff --git a/core/rust/src/vault_pruner/tests.rs b/core/rust/src/vault_pruner/tests.rsindex d9e393af5..122ad0211 100644--- a/core/rust/src/vault_pruner/tests.rs+++ b/core/rust/src/vault_pruner/tests.rs@@ -520,8 +520,8 @@ fn test_sweeper_clears_blob_on_already_tombstoned_attachment() {let stmt = output.statements.iter().find(|s| s.sql.starts_with("UPDATE Attachments SET Blob = NULL")).expect("expected the sweeper UPDATE");- // params: [updated_at, attachment_id]- assert_eq!(stmt.params.len(), 2);+ // params: [updated_at, attachment_id, manifest_id]+ assert_eq!(stmt.params.len(), 3);assert_eq!(stmt.params[1], serde_json::json!("att-old"));}@@ -680,3 +680,59 @@ fn test_prune_cascades_to_field_histories_and_item_tags() {assert!(sql.iter().any(|s| s.starts_with("UPDATE FieldHistories SET IsDeleted = 1")), "field history rows must die with their item: {:?}", sql);assert!(sql.iter().any(|s| s.starts_with("UPDATE ItemTags SET IsDeleted = 1")), "item tag rows must die with their item: {:?}", sql);}++fn in_manifest(mut record: CodecRecord, manifest_id: &str) -> CodecRecord {+ record.insert("ManifestId".to_string(), serde_json::json!(manifest_id));+ record+}++#[test]+fn a_prune_never_reaches_a_same_id_row_in_another_manifest() {+ let old_date = days_ago_iso(60);+ let input = PruneInput {+ tables: vec![+ CodecTableData {+ name: "Items".to_string(),+ records: vec![+ in_manifest(make_item_record("item-1", Some(&old_date), false), "m-personal"),+ in_manifest(make_item_record("item-1", None, false), "m-shared"),+ ],+ },+ CodecTableData {+ name: "FieldValues".to_string(),+ records: vec![+ in_manifest(make_field_value_record("fv-1", "item-1", false), "m-personal"),+ in_manifest(make_field_value_record("fv-1", "item-1", false), "m-shared"),+ ],+ },+ CodecTableData {+ name: "Attachments".to_string(),+ records: vec![+ in_manifest(make_attachment_record("att-1", "item-2", true, serde_json::json!("aGVsbG8=")), "m-personal"),+ in_manifest(make_attachment_record("att-1", "item-2", false, serde_json::json!("aGVsbG8=")), "m-shared"),+ ],+ },+ ],+ retention_days: 30,+ current_time: now_iso(),+ };++ let output = prune_vault(input).unwrap();++ assert_eq!(output.stats.items_pruned, 1);+ assert_eq!(count(&output.stats.child_rows_pruned, FIELD_VALUES_TABLE), 1);+ assert_eq!(count(&output.stats.blobs_cleared, ATTACHMENTS_TABLE), 1);+ assert_eq!(output.statements.len(), 3);+ for stmt in &output.statements {+ assert!(stmt.sql.ends_with("AND ManifestId = ?") || stmt.sql.contains("AND ManifestId = ? AND"), "statement must name its manifest: {}", stmt.sql);+ assert!(stmt.params.contains(&serde_json::json!("m-personal")), "statement must target the personal manifest: {:?}", stmt.params);+ assert!(!stmt.params.contains(&serde_json::json!("m-shared")), "statement must not reach the shared manifest: {:?}", stmt.params);+ }+}++#[test]+fn every_prune_query_reads_the_manifest_id() {+ for table_query in get_prune_table_queries() {+ assert!(table_query.query.starts_with("SELECT ManifestId, "), "{} must be read with its manifest: {}", table_query.name, table_query.query);+ }+}