Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update vault pruner to always select item in a explicit manifest (#2159)

Leendert de Borstfab9684a5877
diff --git a/core/rust/src/vault_pruner/mod.rs b/core/rust/src/vault_pruner/mod.rs
index e02bea9d6..7ab861400 100644
--- a/core/rust/src/vault_pruner/mod.rs
+++ b/core/rust/src/vault_pruner/mod.rs
@@ -17,7 +17,7 @@ use crate::vault_model::names::{
DELETED_AT_COL, FILE_DATA_COL, ID_COL, IS_DELETED_COL, ITEMS_TABLE, ITEM_ID_COL, KIND_COL, LOGOS_TABLE,
LOGO_ID_COL, LOGO_KIND_FAVICON, UPDATED_AT_COL,
};
-use crate::vault_model::{BLOB_COLUMNS, SYNCABLE_TABLES, TRASH_RETENTION_DEFAULT_DAYS};
+use crate::vault_model::{BLOB_COLUMNS, MANIFEST_ID_COL, SYNCABLE_TABLES, TRASH_RETENTION_DEFAULT_DAYS};
/// Input for the prune operation.
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -85,20 +85,20 @@ pub struct PruneTableQuery {
pub fn get_prune_table_queries() -> Vec<PruneTableQuery> {
let mut queries = vec![PruneTableQuery {
name: ITEMS_TABLE.to_string(),
- query: format!("SELECT {}, {}, {}, {} FROM {}", ID_COL, IS_DELETED_COL, DELETED_AT_COL, LOGO_ID_COL, ITEMS_TABLE),
+ query: format!("SELECT {}, {}, {}, {}, {} FROM {}", MANIFEST_ID_COL, ID_COL, IS_DELETED_COL, DELETED_AT_COL, LOGO_ID_COL, ITEMS_TABLE),
}];
// One query per registered item-child table (TableConfig::item_child), so a table added to the
// registry is read by the pruner automatically.
for child in item_child_tables() {
let query = match blob_column_for(child.name) {
- Some(blob_col) => format!("SELECT {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", ID_COL, ITEM_ID_COL, IS_DELETED_COL, blob_col, blob_col, child.name),
- None => format!("SELECT {}, {} FROM {}", child.item_ref_column(), IS_DELETED_COL, child.name),
+ Some(blob_col) => format!("SELECT {}, {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", MANIFEST_ID_COL, ID_COL, ITEM_ID_COL, IS_DELETED_COL, blob_col, blob_col, child.name),
+ None => format!("SELECT {}, {}, {} FROM {}", MANIFEST_ID_COL, child.item_ref_column(), IS_DELETED_COL, child.name),
};
queries.push(PruneTableQuery { name: child.name.to_string(), query });
}
queries.push(PruneTableQuery {
name: LOGOS_TABLE.to_string(),
- query: format!("SELECT {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", ID_COL, KIND_COL, IS_DELETED_COL, FILE_DATA_COL, FILE_DATA_COL, LOGOS_TABLE),
+ query: format!("SELECT {}, {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", MANIFEST_ID_COL, ID_COL, KIND_COL, IS_DELETED_COL, FILE_DATA_COL, FILE_DATA_COL, LOGOS_TABLE),
});
queries
}
@@ -123,6 +123,14 @@ fn records_of<'a>(tables: &'a [CodecTableData], name: &str) -> Option<&'a [Codec
tables.iter().find(|t| t.name == name).map(|t| t.records.as_slice())
}
+/*
+ * The manifest a row lives in. The same id can appear in multiple manifests, so every statement
+ * addresses its rows by `(ManifestId, Id)`.
+ */
+fn manifest_of(record: &CodecRecord) -> &str {
+ str_col(record, MANIFEST_ID_COL).unwrap_or("")
+}
+
/// Main entry point: prune expired items from trash.
///
/// Finds all Items with DeletedAt older than `retention_days` and generates SQL statements that mark
@@ -143,13 +151,13 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
};
let expired = expired_item_ids(items, cutoff_date);
- for item_id in &expired {
+ for (manifest_id, item_id) in &expired {
statements.push(SqlStatement {
- sql: format!("UPDATE {} SET {} = 1, {} = ? WHERE {} = ?", ITEMS_TABLE, IS_DELETED_COL, UPDATED_AT_COL, ID_COL),
- params: vec![serde_json::json!(now_str), serde_json::json!(item_id)],
+ sql: format!("UPDATE {} SET {} = 1, {} = ? WHERE {} = ? AND {} = ?", ITEMS_TABLE, IS_DELETED_COL, UPDATED_AT_COL, ID_COL, MANIFEST_ID_COL),
+ params: vec![serde_json::json!(now_str), serde_json::json!(item_id), serde_json::json!(manifest_id)],
});
stats.items_pruned += 1;
- tombstone_item_children(&input.tables, item_id, &now_str, &mut statements, &mut stats);
+ tombstone_item_children(&input.tables, manifest_id, item_id, &now_str, &mut statements, &mut stats);
}
sweep_orphan_favicons(&input.tables, items, &expired, &now_str, &mut statements, &mut stats);
@@ -158,15 +166,15 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
Ok(PruneOutput { success: true, statements, stats })
}
-/// Pass 1: the ids of live items whose trash date (`DeletedAt`) lies before the cutoff.
-fn expired_item_ids(items: &[CodecRecord], cutoff_date: DateTime<Utc>) -> Vec<String> {
+/// Pass 1: the `(ManifestId, Id)` of live items whose trash date (`DeletedAt`) lies before the cutoff.
+fn expired_item_ids(items: &[CodecRecord], cutoff_date: DateTime<Utc>) -> Vec<(String, String)> {
let mut expired = Vec::new();
for item in items.iter().filter(|item| !is_deleted(item)) {
let Some(deleted_at) = str_col(item, DELETED_AT_COL) else { continue };
let Some(deleted_date) = crate::timestamp::parse_vault_datetime(deleted_at) else { continue };
if deleted_date < cutoff_date {
if let Some(id) = str_col(item, ID_COL) {
- expired.push(id.to_string());
+ expired.push((manifest_of(item).to_string(), id.to_string()));
}
}
}
@@ -179,21 +187,21 @@ fn expired_item_ids(items: &[CodecRecord], cutoff_date: DateTime<Utc>) -> Vec<St
* extracted blob column has its bytes dropped in the same statement, leaving the column non-null
* while reclaiming the storage on the next save.
*/
-fn tombstone_item_children(tables: &[CodecTableData], item_id: &str, now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {
+fn tombstone_item_children(tables: &[CodecTableData], manifest_id: &str, item_id: &str, now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {
for child in item_child_tables() {
let Some(records) = records_of(tables, child.name) else { continue };
let match_col = child.item_ref_column();
- let related_count = count_related_records(records, match_col, item_id);
+ let related_count = count_related_records(records, manifest_id, match_col, item_id);
if related_count == 0 {
continue;
}
let blob_clear = blob_clear_fragment(child.name);
statements.push(SqlStatement {
sql: format!(
- "UPDATE {} SET {} = 1{}, {} = ? WHERE {} = ? AND {} = 0",
- child.name, IS_DELETED_COL, blob_clear, UPDATED_AT_COL, match_col, IS_DELETED_COL
+ "UPDATE {} SET {} = 1{}, {} = ? WHERE {} = ? AND {} = ? AND {} = 0",
+ child.name, IS_DELETED_COL, blob_clear, UPDATED_AT_COL, match_col, MANIFEST_ID_COL, IS_DELETED_COL
),
- params: vec![serde_json::json!(now_str), serde_json::json!(item_id)],
+ params: vec![serde_json::json!(now_str), serde_json::json!(item_id), serde_json::json!(manifest_id)],
});
*stats.child_rows_pruned.entry(child.name.to_string()).or_default() += related_count;
}
@@ -213,19 +221,19 @@ fn tombstone_item_children(tables: &[CodecTableData], item_id: &str, now_str: &s
fn sweep_orphan_favicons(
tables: &[CodecTableData],
items: &[CodecRecord],
- expired: &[String],
+ expired: &[(String, String)],
now_str: &str,
statements: &mut Vec<SqlStatement>,
stats: &mut PruneStats,
) {
let Some(logos) = records_of(tables, LOGOS_TABLE) else { return };
let blob_clear = blob_clear_fragment(LOGOS_TABLE);
- let expired_set: HashSet<&str> = expired.iter().map(String::as_str).collect();
+ let expired_set: HashSet<(&str, &str)> = expired.iter().map(|(manifest_id, id)| (manifest_id.as_str(), id.as_str())).collect();
- let referenced_logo_ids: HashSet<&str> = items.iter()
+ let referenced_logos: HashSet<(&str, &str)> = items.iter()
.filter(|item| !is_deleted(item))
- .filter(|item| !expired_set.contains(str_col(item, ID_COL).unwrap_or("")))
- .filter_map(|item| str_col(item, LOGO_ID_COL))
+ .filter(|item| !expired_set.contains(&(manifest_of(item), str_col(item, ID_COL).unwrap_or(""))))
+ .filter_map(|item| str_col(item, LOGO_ID_COL).map(|logo_id| (manifest_of(item), logo_id)))
.collect();
for logo in logos.iter().filter(|logo| !is_deleted(logo)) {
@@ -233,12 +241,12 @@ fn sweep_orphan_favicons(
continue;
}
let Some(logo_id) = str_col(logo, ID_COL) else { continue };
- if referenced_logo_ids.contains(logo_id) {
+ if referenced_logos.contains(&(manifest_of(logo), logo_id)) {
continue;
}
statements.push(SqlStatement {
- sql: format!("UPDATE {} SET {} = 1{}, {} = ? WHERE {} = ?", LOGOS_TABLE, IS_DELETED_COL, blob_clear, UPDATED_AT_COL, ID_COL),
- params: vec![serde_json::json!(now_str), serde_json::json!(logo_id)],
+ sql: format!("UPDATE {} SET {} = 1{}, {} = ? WHERE {} = ? AND {} = ?", LOGOS_TABLE, IS_DELETED_COL, blob_clear, UPDATED_AT_COL, ID_COL, MANIFEST_ID_COL),
+ params: vec![serde_json::json!(now_str), serde_json::json!(logo_id), serde_json::json!(manifest_of(logo))],
});
stats.logos_pruned += 1;
}
@@ -258,17 +266,17 @@ fn clear_tombstoned_blobs(tables: &[CodecTableData], now_str: &str, statements:
for row in stale {
let Some(id) = str_col(row, ID_COL) else { continue };
statements.push(SqlStatement {
- sql: format!("UPDATE {} SET {} = NULL, {} = ? WHERE {} = ?", table, blob_col, UPDATED_AT_COL, ID_COL),
- params: vec![serde_json::json!(now_str), serde_json::json!(id)],
+ sql: format!("UPDATE {} SET {} = NULL, {} = ? WHERE {} = ? AND {} = ?", table, blob_col, UPDATED_AT_COL, ID_COL, MANIFEST_ID_COL),
+ params: vec![serde_json::json!(now_str), serde_json::json!(id), serde_json::json!(manifest_of(row))],
});
*stats.blobs_cleared.entry(table.to_string()).or_default() += 1;
}
}
}
-/// Count related records that match a foreign key value and are not already deleted.
-fn count_related_records(records: &[CodecRecord], fk_column: &str, fk_value: &str) -> u32 {
- records.iter().filter(|r| str_col(r, fk_column) == Some(fk_value) && !is_deleted(r)).count() as u32
+/// Count the live records of one manifest that match a foreign key value.
+fn count_related_records(records: &[CodecRecord], manifest_id: &str, fk_column: &str, fk_value: &str) -> u32 {
+ records.iter().filter(|r| manifest_of(r) == manifest_id && str_col(r, fk_column) == Some(fk_value) && !is_deleted(r)).count() as u32
}
#[cfg(test)]
diff --git a/core/rust/src/vault_pruner/tests.rs b/core/rust/src/vault_pruner/tests.rs
index d9e393af5..122ad0211 100644
--- a/core/rust/src/vault_pruner/tests.rs
+++ b/core/rust/src/vault_pruner/tests.rs
@@ -520,8 +520,8 @@ fn test_sweeper_clears_blob_on_already_tombstoned_attachment() {
let stmt = output.statements.iter()
.find(|s| s.sql.starts_with("UPDATE Attachments SET Blob = NULL"))
.expect("expected the sweeper UPDATE");
- // params: [updated_at, attachment_id]
- assert_eq!(stmt.params.len(), 2);
+ // params: [updated_at, attachment_id, manifest_id]
+ assert_eq!(stmt.params.len(), 3);
assert_eq!(stmt.params[1], serde_json::json!("att-old"));
}
@@ -680,3 +680,59 @@ fn test_prune_cascades_to_field_histories_and_item_tags() {
assert!(sql.iter().any(|s| s.starts_with("UPDATE FieldHistories SET IsDeleted = 1")), "field history rows must die with their item: {:?}", sql);
assert!(sql.iter().any(|s| s.starts_with("UPDATE ItemTags SET IsDeleted = 1")), "item tag rows must die with their item: {:?}", sql);
}
+
+fn in_manifest(mut record: CodecRecord, manifest_id: &str) -> CodecRecord {
+ record.insert("ManifestId".to_string(), serde_json::json!(manifest_id));
+ record
+}
+
+#[test]
+fn a_prune_never_reaches_a_same_id_row_in_another_manifest() {
+ let old_date = days_ago_iso(60);
+ let input = PruneInput {
+ tables: vec![
+ CodecTableData {
+ name: "Items".to_string(),
+ records: vec![
+ in_manifest(make_item_record("item-1", Some(&old_date), false), "m-personal"),
+ in_manifest(make_item_record("item-1", None, false), "m-shared"),
+ ],
+ },
+ CodecTableData {
+ name: "FieldValues".to_string(),
+ records: vec![
+ in_manifest(make_field_value_record("fv-1", "item-1", false), "m-personal"),
+ in_manifest(make_field_value_record("fv-1", "item-1", false), "m-shared"),
+ ],
+ },
+ CodecTableData {
+ name: "Attachments".to_string(),
+ records: vec![
+ in_manifest(make_attachment_record("att-1", "item-2", true, serde_json::json!("aGVsbG8=")), "m-personal"),
+ in_manifest(make_attachment_record("att-1", "item-2", false, serde_json::json!("aGVsbG8=")), "m-shared"),
+ ],
+ },
+ ],
+ retention_days: 30,
+ current_time: now_iso(),
+ };
+
+ let output = prune_vault(input).unwrap();
+
+ assert_eq!(output.stats.items_pruned, 1);
+ assert_eq!(count(&output.stats.child_rows_pruned, FIELD_VALUES_TABLE), 1);
+ assert_eq!(count(&output.stats.blobs_cleared, ATTACHMENTS_TABLE), 1);
+ assert_eq!(output.statements.len(), 3);
+ for stmt in &output.statements {
+ assert!(stmt.sql.ends_with("AND ManifestId = ?") || stmt.sql.contains("AND ManifestId = ? AND"), "statement must name its manifest: {}", stmt.sql);
+ assert!(stmt.params.contains(&serde_json::json!("m-personal")), "statement must target the personal manifest: {:?}", stmt.params);
+ assert!(!stmt.params.contains(&serde_json::json!("m-shared")), "statement must not reach the shared manifest: {:?}", stmt.params);
+ }
+}
+
+#[test]
+fn every_prune_query_reads_the_manifest_id() {
+ for table_query in get_prune_table_queries() {
+ assert!(table_query.query.starts_with("SELECT ManifestId, "), "{} must be read with its manifest: {}", table_query.name, table_query.query);
+ }
+}