Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update Rust vault_pruner efficiency and to not trigger unnecessary (#2268)
Leendert de Borste40b924f510a
diff --git a/apps/browser-extension/src/utils/VaultMergeService.ts b/apps/browser-extension/src/utils/VaultMergeService.tsindex ff5ecb53b..eb37a20fa 100644--- a/apps/browser-extension/src/utils/VaultMergeService.ts+++ b/apps/browser-extension/src/utils/VaultMergeService.ts@@ -268,13 +268,24 @@ export class VaultMergeService {const db = this.loadDatabase(SQL, vaultBase64);try {- // Tables needed for pruning- const tableNames = ['Items', 'FieldValues', 'Attachments', 'TotpCodes', 'Passkeys', 'Logos'];+ /*+ * Read only the columns the Rust pruner inspects. Blob columns are reduced+ * to a 1-byte presence marker to avoid serializing megabytes of binary data+ * to JSON on every sync.+ */+ const tableQueries: Record<string, string> = {+ Items: 'SELECT Id, IsDeleted, DeletedAt, LogoId FROM Items',+ FieldValues: 'SELECT ItemId, IsDeleted FROM FieldValues',+ Attachments: 'SELECT Id, ItemId, IsDeleted, substr(Blob, 1, 1) AS Blob FROM Attachments',+ TotpCodes: 'SELECT ItemId, IsDeleted FROM TotpCodes',+ Passkeys: 'SELECT ItemId, IsDeleted FROM Passkeys',+ Logos: 'SELECT Id, IsDeleted, substr(FileData, 1, 1) AS FileData FROM Logos',+ };// Read tables as JSON- const tables: TableData[] = tableNames.map(name => ({+ const tables: TableData[] = Object.entries(tableQueries).map(([name, query]) => ({name,- records: this.readTableAsJson(db, name),+ records: this.readQueryAsJson(db, query),}));// Call Rust WASM prune@@ -363,8 +374,18 @@ export class VaultMergeService {* @returns Array of records as JSON objects*/private readTableAsJson(db: Database, tableName: string): JsonRecord[] {+ return this.readQueryAsJson(db, `SELECT * FROM ${tableName}`);+ }++ /**+ * Read all rows of a query as JSON objects.+ * @param db - The database to query+ * @param query - The SELECT query to run+ * @returns Array of records as JSON objects+ */+ private readQueryAsJson(db: Database, query: string): JsonRecord[] {const records: JsonRecord[] = [];- const stmt = db.prepare(`SELECT * FROM ${tableName}`);+ const stmt = db.prepare(query);while (stmt.step()) {const obj = stmt.getAsObject();diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/rustcore/VaultMergeService.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/rustcore/VaultMergeService.ktindex 33e9e328e..25c63a5de 100644--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/rustcore/VaultMergeService.kt+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/rustcore/VaultMergeService.kt@@ -220,15 +220,23 @@ object VaultMergeService {val db = SQLiteDatabase.openDatabase(dbFile.absolutePath, null, SQLiteDatabase.OPEN_READWRITE)try {- // Read tables needed for pruning- val pruneTableNames = listOf("Items", "FieldValues", "Attachments", "TotpCodes", "Passkeys", "Logos")+ // Read only the columns the Rust pruner inspects. Blob columns are reduced+ // to a 1-byte presence marker to avoid serializing large binary data to JSON.+ val pruneTableQueries = listOf(+ "Items" to "SELECT Id, IsDeleted, DeletedAt, LogoId FROM Items",+ "FieldValues" to "SELECT ItemId, IsDeleted FROM FieldValues",+ "Attachments" to "SELECT Id, ItemId, IsDeleted, substr(Blob, 1, 1) AS Blob FROM Attachments",+ "TotpCodes" to "SELECT ItemId, IsDeleted FROM TotpCodes",+ "Passkeys" to "SELECT ItemId, IsDeleted FROM Passkeys",+ "Logos" to "SELECT Id, IsDeleted, substr(FileData, 1, 1) AS FileData FROM Logos",+ )val tables = JSONArray()- for (tableName in pruneTableNames) {+ for ((tableName, query) in pruneTableQueries) {tables.put(JSONObject().apply {put("name", tableName)- put("records", readTable(db, tableName))+ put("records", readQuery(db, tableName, query))},)}@@ -275,6 +283,10 @@ object VaultMergeService {}private fun readTable(db: SQLiteDatabase, tableName: String): JSONArray {+ return readQuery(db, tableName, "SELECT * FROM $tableName")+ }++ private fun readQuery(db: SQLiteDatabase, tableName: String, query: String): JSONArray {val records = JSONArray()// Check if table exists@@ -287,7 +299,7 @@ object VaultMergeService {if (!tableExists) return records// Read all records- val cursor = db.rawQuery("SELECT * FROM $tableName", null)+ val cursor = db.rawQuery(query, null)try {while (cursor.moveToNext()) {val record = JSONObject()diff --git a/apps/mobile-app/ios/VaultStoreKit/Services/VaultMergeService.swift b/apps/mobile-app/ios/VaultStoreKit/Services/VaultMergeService.swiftindex f84a03e08..08cd5caa1 100644--- a/apps/mobile-app/ios/VaultStoreKit/Services/VaultMergeService.swift+++ b/apps/mobile-app/ios/VaultStoreKit/Services/VaultMergeService.swift@@ -154,14 +154,22 @@ public class VaultMergeService {let db = try openDatabase(from: decrypted)defer { sqlite3_close(db) }- // Read tables needed for pruning- let pruneTableNames = ["Items", "FieldValues", "Attachments", "TotpCodes", "Passkeys", "Logos"]+ // Read only the columns the Rust pruner inspects. Blob columns are reduced+ // to a 1-byte presence marker to avoid serializing large binary data to JSON.+ let pruneTableQueries: [(name: String, sql: String)] = [+ ("Items", "SELECT Id, IsDeleted, DeletedAt, LogoId FROM Items"),+ ("FieldValues", "SELECT ItemId, IsDeleted FROM FieldValues"),+ ("Attachments", "SELECT Id, ItemId, IsDeleted, substr(Blob, 1, 1) AS Blob FROM Attachments"),+ ("TotpCodes", "SELECT ItemId, IsDeleted FROM TotpCodes"),+ ("Passkeys", "SELECT ItemId, IsDeleted FROM Passkeys"),+ ("Logos", "SELECT Id, IsDeleted, substr(FileData, 1, 1) AS FileData FROM Logos")+ ]var tables: [[String: Any]] = []- for tableName in pruneTableNames {+ for entry in pruneTableQueries {tables.append([- "name": tableName,- "records": try readTable(from: db, tableName: tableName)+ "name": entry.name,+ "records": try readQuery(from: db, tableName: entry.name, sql: entry.sql)])}@@ -263,6 +271,10 @@ public class VaultMergeService {}private func readTable(from db: OpaquePointer, tableName: String) throws -> [[String: Any]] {+ return try readQuery(from: db, tableName: tableName, sql: "SELECT * FROM \(tableName)")+ }++ private func readQuery(from db: OpaquePointer, tableName: String, sql: String) throws -> [[String: Any]] {var records: [[String: Any]] = []// Check if table exists@@ -287,7 +299,6 @@ public class VaultMergeService {}// Read all records- let sql = "SELECT * FROM \(tableName)"var stmt: OpaquePointer?let selectPrepareResult = sqlite3_prepare_v2(db, sql, -1, &stmt, nil)guard selectPrepareResult == SQLITE_OK else {diff --git a/apps/server/AliasVault.Client/Services/Database/DbService.cs b/apps/server/AliasVault.Client/Services/Database/DbService.csindex 0026d91be..539c6299b 100644--- a/apps/server/AliasVault.Client/Services/Database/DbService.cs+++ b/apps/server/AliasVault.Client/Services/Database/DbService.cs@@ -977,8 +977,7 @@ public sealed class DbService : IDisposabletry{// Read table data for prune operation- var tableNames = new[] { "Items", "FieldValues", "Attachments", "TotpCodes", "Passkeys", "Logos" };- var tables = await ReadTablesAsJsonAsync(_sqlConnection!, tableNames);+ var tables = await ReadPruneTablesAsJsonAsync(_sqlConnection!);var pruneInput = new JsInterop.RustCore.PruneInput{@@ -1055,6 +1054,67 @@ public sealed class DbService : IDisposablereturn encryptionKey;}+ /// <summary>+ /// Reads only the columns the Rust pruner inspects. Blob columns are reduced to a+ /// 1-byte presence marker to avoid serializing large binary data to JSON.+ /// </summary>+ /// <param name="connection">The SQLite connection to read from.</param>+ /// <returns>List of TableData objects containing the trimmed table records.</returns>+ private async Task<List<TableData>> ReadPruneTablesAsJsonAsync(SqliteConnection connection)+ {+ var tableQueries = new (string Name, string Query)[]+ {+ ("Items", "SELECT Id, IsDeleted, DeletedAt, LogoId FROM Items"),+ ("FieldValues", "SELECT ItemId, IsDeleted FROM FieldValues"),+ ("Attachments", "SELECT Id, ItemId, IsDeleted, substr(Blob, 1, 1) AS Blob FROM Attachments"),+ ("TotpCodes", "SELECT ItemId, IsDeleted FROM TotpCodes"),+ ("Passkeys", "SELECT ItemId, IsDeleted FROM Passkeys"),+ ("Logos", "SELECT Id, IsDeleted, substr(FileData, 1, 1) AS FileData FROM Logos"),+ };++ var tables = new List<TableData>();++ foreach (var (tableName, query) in tableQueries)+ {+ var tableData = new TableData { Name = tableName };++ // Check if table exists in the database.+ await using var checkCommand = connection.CreateCommand();+ checkCommand.CommandText = "SELECT name FROM sqlite_master WHERE type='table' AND name=@tableName";+ checkCommand.Parameters.AddWithValue("@tableName", tableName);+ var exists = await checkCommand.ExecuteScalarAsync();++ if (exists == null)+ {+ // Table doesn't exist, add empty table data.+ tables.Add(tableData);+ continue;+ }++ await using var selectCommand = connection.CreateCommand();+ selectCommand.CommandText = query;+ await using var reader = await selectCommand.ExecuteReaderAsync();++ while (await reader.ReadAsync())+ {+ var record = new Dictionary<string, object?>();+ for (var i = 0; i < reader.FieldCount; i++)+ {+ var value = reader.GetValue(i);++ // Convert DBNull to null for proper JSON serialization.+ record[reader.GetName(i)] = value == DBNull.Value ? null : value;+ }++ tableData.Records.Add(record);+ }++ tables.Add(tableData);+ }++ return tables;+ }+/// <summary>/// Reads all specified tables from a SQLite connection as JSON data for the Rust merge./// </summary>diff --git a/apps/server/AliasVault.Client/wwwroot/css/tailwind.css b/apps/server/AliasVault.Client/wwwroot/css/tailwind.cssindex 35590b5a2..c5fc0d9a9 100644--- a/apps/server/AliasVault.Client/wwwroot/css/tailwind.css+++ b/apps/server/AliasVault.Client/wwwroot/css/tailwind.css@@ -2465,6 +2465,10 @@ video {font-weight: 600;}+.font-light {+ font-weight: 300;+}+.uppercase {text-transform: uppercase;}diff --git a/core/rust/src/vault_pruner/mod.rs b/core/rust/src/vault_pruner/mod.rsindex 53d125d1f..c2cc764be 100644--- a/core/rust/src/vault_pruner/mod.rs+++ b/core/rust/src/vault_pruner/mod.rs@@ -348,22 +348,22 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {/// True if the attachment's Blob field is present and non-empty.fn attachment_has_blob_bytes(attachment: &Record) -> bool {- match attachment.get("Blob") {- None => false,- Some(serde_json::Value::Null) => false,- Some(serde_json::Value::String(s)) => !s.is_empty(),- Some(serde_json::Value::Array(a)) => !a.is_empty(),- Some(_) => true,- }+ value_has_bytes(attachment.get("Blob"))}/// True if the logo's FileData field is present and non-empty.fn logo_has_file_data_bytes(logo: &Record) -> bool {- match logo.get("FileData") {+ value_has_bytes(logo.get("FileData"))+}++/// True if a JSON value represents non-empty blob bytes.+fn value_has_bytes(value: Option<&serde_json::Value>) -> bool {+ match value {None => false,Some(serde_json::Value::Null) => false,Some(serde_json::Value::String(s)) => !s.is_empty(),Some(serde_json::Value::Array(a)) => !a.is_empty(),+ Some(serde_json::Value::Object(o)) => !o.is_empty(),Some(_) => true,}}@@ -972,6 +972,60 @@ mod tests {assert!(output.statements.is_empty());}+ #[test]+ fn test_sweeper_skips_empty_uint8array_object_blob() {+ // An already-cleared blob must not generate a clear statement on every prune.+ let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let input = PruneInput {+ tables: vec![+ TableData {+ name: "Items".to_string(),+ records: vec![],+ },+ TableData {+ name: "Attachments".to_string(),+ records: vec![make_attachment_record("att-empty-object", "item-1", true, serde_json::json!({}))],+ },+ TableData {+ name: "Logos".to_string(),+ records: vec![make_logo_record_with_blob("logo-empty-object", true, serde_json::json!({}))],+ },+ ],+ retention_days: 30,+ current_time: now_str,+ };++ let output = prune_vault(input).unwrap();++ assert_eq!(output.stats.attachment_blobs_cleared, 0);+ assert_eq!(output.stats.logo_blobs_cleared, 0);+ assert!(output.statements.is_empty());+ }++ #[test]+ fn test_sweeper_clears_nonempty_uint8array_object_blob() {+ // Non-empty blobs are serialized as {"0":104,...}.+ let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let input = PruneInput {+ tables: vec![+ TableData {+ name: "Items".to_string(),+ records: vec![],+ },+ TableData {+ name: "Attachments".to_string(),+ records: vec![make_attachment_record("att-object", "item-1", true, serde_json::json!({"0": 104, "1": 105}))],+ },+ ],+ retention_days: 30,+ current_time: now_str,+ };++ let output = prune_vault(input).unwrap();++ assert_eq!(output.stats.attachment_blobs_cleared, 1);+ }+#[test]fn test_sweeper_skips_active_attachment_with_blob() {let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();