Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update Rust vault_pruner efficiency and to not trigger unnecessary (#2268)

Leendert de Borste40b924f510a
diff --git a/apps/browser-extension/src/utils/VaultMergeService.ts b/apps/browser-extension/src/utils/VaultMergeService.ts
index ff5ecb53b..eb37a20fa 100644
--- a/apps/browser-extension/src/utils/VaultMergeService.ts
+++ b/apps/browser-extension/src/utils/VaultMergeService.ts
@@ -268,13 +268,24 @@ export class VaultMergeService {
const db = this.loadDatabase(SQL, vaultBase64);
try {
- // Tables needed for pruning
- const tableNames = ['Items', 'FieldValues', 'Attachments', 'TotpCodes', 'Passkeys', 'Logos'];
+ /*
+ * Read only the columns the Rust pruner inspects. Blob columns are reduced
+ * to a 1-byte presence marker to avoid serializing megabytes of binary data
+ * to JSON on every sync.
+ */
+ const tableQueries: Record<string, string> = {
+ Items: 'SELECT Id, IsDeleted, DeletedAt, LogoId FROM Items',
+ FieldValues: 'SELECT ItemId, IsDeleted FROM FieldValues',
+ Attachments: 'SELECT Id, ItemId, IsDeleted, substr(Blob, 1, 1) AS Blob FROM Attachments',
+ TotpCodes: 'SELECT ItemId, IsDeleted FROM TotpCodes',
+ Passkeys: 'SELECT ItemId, IsDeleted FROM Passkeys',
+ Logos: 'SELECT Id, IsDeleted, substr(FileData, 1, 1) AS FileData FROM Logos',
+ };
// Read tables as JSON
- const tables: TableData[] = tableNames.map(name => ({
+ const tables: TableData[] = Object.entries(tableQueries).map(([name, query]) => ({
name,
- records: this.readTableAsJson(db, name),
+ records: this.readQueryAsJson(db, query),
}));
// Call Rust WASM prune
@@ -363,8 +374,18 @@ export class VaultMergeService {
* @returns Array of records as JSON objects
*/
private readTableAsJson(db: Database, tableName: string): JsonRecord[] {
+ return this.readQueryAsJson(db, `SELECT * FROM ${tableName}`);
+ }
+
+ /**
+ * Read all rows of a query as JSON objects.
+ * @param db - The database to query
+ * @param query - The SELECT query to run
+ * @returns Array of records as JSON objects
+ */
+ private readQueryAsJson(db: Database, query: string): JsonRecord[] {
const records: JsonRecord[] = [];
- const stmt = db.prepare(`SELECT * FROM ${tableName}`);
+ const stmt = db.prepare(query);
while (stmt.step()) {
const obj = stmt.getAsObject();
diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/rustcore/VaultMergeService.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/rustcore/VaultMergeService.kt
index 33e9e328e..25c63a5de 100644
--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/rustcore/VaultMergeService.kt
+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/rustcore/VaultMergeService.kt
@@ -220,15 +220,23 @@ object VaultMergeService {
val db = SQLiteDatabase.openDatabase(dbFile.absolutePath, null, SQLiteDatabase.OPEN_READWRITE)
try {
- // Read tables needed for pruning
- val pruneTableNames = listOf("Items", "FieldValues", "Attachments", "TotpCodes", "Passkeys", "Logos")
+ // Read only the columns the Rust pruner inspects. Blob columns are reduced
+ // to a 1-byte presence marker to avoid serializing large binary data to JSON.
+ val pruneTableQueries = listOf(
+ "Items" to "SELECT Id, IsDeleted, DeletedAt, LogoId FROM Items",
+ "FieldValues" to "SELECT ItemId, IsDeleted FROM FieldValues",
+ "Attachments" to "SELECT Id, ItemId, IsDeleted, substr(Blob, 1, 1) AS Blob FROM Attachments",
+ "TotpCodes" to "SELECT ItemId, IsDeleted FROM TotpCodes",
+ "Passkeys" to "SELECT ItemId, IsDeleted FROM Passkeys",
+ "Logos" to "SELECT Id, IsDeleted, substr(FileData, 1, 1) AS FileData FROM Logos",
+ )
val tables = JSONArray()
- for (tableName in pruneTableNames) {
+ for ((tableName, query) in pruneTableQueries) {
tables.put(
JSONObject().apply {
put("name", tableName)
- put("records", readTable(db, tableName))
+ put("records", readQuery(db, tableName, query))
},
)
}
@@ -275,6 +283,10 @@ object VaultMergeService {
}
private fun readTable(db: SQLiteDatabase, tableName: String): JSONArray {
+ return readQuery(db, tableName, "SELECT * FROM $tableName")
+ }
+
+ private fun readQuery(db: SQLiteDatabase, tableName: String, query: String): JSONArray {
val records = JSONArray()
// Check if table exists
@@ -287,7 +299,7 @@ object VaultMergeService {
if (!tableExists) return records
// Read all records
- val cursor = db.rawQuery("SELECT * FROM $tableName", null)
+ val cursor = db.rawQuery(query, null)
try {
while (cursor.moveToNext()) {
val record = JSONObject()
diff --git a/apps/mobile-app/ios/VaultStoreKit/Services/VaultMergeService.swift b/apps/mobile-app/ios/VaultStoreKit/Services/VaultMergeService.swift
index f84a03e08..08cd5caa1 100644
--- a/apps/mobile-app/ios/VaultStoreKit/Services/VaultMergeService.swift
+++ b/apps/mobile-app/ios/VaultStoreKit/Services/VaultMergeService.swift
@@ -154,14 +154,22 @@ public class VaultMergeService {
let db = try openDatabase(from: decrypted)
defer { sqlite3_close(db) }
- // Read tables needed for pruning
- let pruneTableNames = ["Items", "FieldValues", "Attachments", "TotpCodes", "Passkeys", "Logos"]
+ // Read only the columns the Rust pruner inspects. Blob columns are reduced
+ // to a 1-byte presence marker to avoid serializing large binary data to JSON.
+ let pruneTableQueries: [(name: String, sql: String)] = [
+ ("Items", "SELECT Id, IsDeleted, DeletedAt, LogoId FROM Items"),
+ ("FieldValues", "SELECT ItemId, IsDeleted FROM FieldValues"),
+ ("Attachments", "SELECT Id, ItemId, IsDeleted, substr(Blob, 1, 1) AS Blob FROM Attachments"),
+ ("TotpCodes", "SELECT ItemId, IsDeleted FROM TotpCodes"),
+ ("Passkeys", "SELECT ItemId, IsDeleted FROM Passkeys"),
+ ("Logos", "SELECT Id, IsDeleted, substr(FileData, 1, 1) AS FileData FROM Logos")
+ ]
var tables: [[String: Any]] = []
- for tableName in pruneTableNames {
+ for entry in pruneTableQueries {
tables.append([
- "name": tableName,
- "records": try readTable(from: db, tableName: tableName)
+ "name": entry.name,
+ "records": try readQuery(from: db, tableName: entry.name, sql: entry.sql)
])
}
@@ -263,6 +271,10 @@ public class VaultMergeService {
}
private func readTable(from db: OpaquePointer, tableName: String) throws -> [[String: Any]] {
+ return try readQuery(from: db, tableName: tableName, sql: "SELECT * FROM \(tableName)")
+ }
+
+ private func readQuery(from db: OpaquePointer, tableName: String, sql: String) throws -> [[String: Any]] {
var records: [[String: Any]] = []
// Check if table exists
@@ -287,7 +299,6 @@ public class VaultMergeService {
}
// Read all records
- let sql = "SELECT * FROM \(tableName)"
var stmt: OpaquePointer?
let selectPrepareResult = sqlite3_prepare_v2(db, sql, -1, &stmt, nil)
guard selectPrepareResult == SQLITE_OK else {
diff --git a/apps/server/AliasVault.Client/Services/Database/DbService.cs b/apps/server/AliasVault.Client/Services/Database/DbService.cs
index 0026d91be..539c6299b 100644
--- a/apps/server/AliasVault.Client/Services/Database/DbService.cs
+++ b/apps/server/AliasVault.Client/Services/Database/DbService.cs
@@ -977,8 +977,7 @@ public sealed class DbService : IDisposable
try
{
// Read table data for prune operation
- var tableNames = new[] { "Items", "FieldValues", "Attachments", "TotpCodes", "Passkeys", "Logos" };
- var tables = await ReadTablesAsJsonAsync(_sqlConnection!, tableNames);
+ var tables = await ReadPruneTablesAsJsonAsync(_sqlConnection!);
var pruneInput = new JsInterop.RustCore.PruneInput
{
@@ -1055,6 +1054,67 @@ public sealed class DbService : IDisposable
return encryptionKey;
}
+ /// <summary>
+ /// Reads only the columns the Rust pruner inspects. Blob columns are reduced to a
+ /// 1-byte presence marker to avoid serializing large binary data to JSON.
+ /// </summary>
+ /// <param name="connection">The SQLite connection to read from.</param>
+ /// <returns>List of TableData objects containing the trimmed table records.</returns>
+ private async Task<List<TableData>> ReadPruneTablesAsJsonAsync(SqliteConnection connection)
+ {
+ var tableQueries = new (string Name, string Query)[]
+ {
+ ("Items", "SELECT Id, IsDeleted, DeletedAt, LogoId FROM Items"),
+ ("FieldValues", "SELECT ItemId, IsDeleted FROM FieldValues"),
+ ("Attachments", "SELECT Id, ItemId, IsDeleted, substr(Blob, 1, 1) AS Blob FROM Attachments"),
+ ("TotpCodes", "SELECT ItemId, IsDeleted FROM TotpCodes"),
+ ("Passkeys", "SELECT ItemId, IsDeleted FROM Passkeys"),
+ ("Logos", "SELECT Id, IsDeleted, substr(FileData, 1, 1) AS FileData FROM Logos"),
+ };
+
+ var tables = new List<TableData>();
+
+ foreach (var (tableName, query) in tableQueries)
+ {
+ var tableData = new TableData { Name = tableName };
+
+ // Check if table exists in the database.
+ await using var checkCommand = connection.CreateCommand();
+ checkCommand.CommandText = "SELECT name FROM sqlite_master WHERE type='table' AND name=@tableName";
+ checkCommand.Parameters.AddWithValue("@tableName", tableName);
+ var exists = await checkCommand.ExecuteScalarAsync();
+
+ if (exists == null)
+ {
+ // Table doesn't exist, add empty table data.
+ tables.Add(tableData);
+ continue;
+ }
+
+ await using var selectCommand = connection.CreateCommand();
+ selectCommand.CommandText = query;
+ await using var reader = await selectCommand.ExecuteReaderAsync();
+
+ while (await reader.ReadAsync())
+ {
+ var record = new Dictionary<string, object?>();
+ for (var i = 0; i < reader.FieldCount; i++)
+ {
+ var value = reader.GetValue(i);
+
+ // Convert DBNull to null for proper JSON serialization.
+ record[reader.GetName(i)] = value == DBNull.Value ? null : value;
+ }
+
+ tableData.Records.Add(record);
+ }
+
+ tables.Add(tableData);
+ }
+
+ return tables;
+ }
+
/// <summary>
/// Reads all specified tables from a SQLite connection as JSON data for the Rust merge.
/// </summary>
diff --git a/apps/server/AliasVault.Client/wwwroot/css/tailwind.css b/apps/server/AliasVault.Client/wwwroot/css/tailwind.css
index 35590b5a2..c5fc0d9a9 100644
--- a/apps/server/AliasVault.Client/wwwroot/css/tailwind.css
+++ b/apps/server/AliasVault.Client/wwwroot/css/tailwind.css
@@ -2465,6 +2465,10 @@ video {
font-weight: 600;
}
+.font-light {
+ font-weight: 300;
+}
+
.uppercase {
text-transform: uppercase;
}
diff --git a/core/rust/src/vault_pruner/mod.rs b/core/rust/src/vault_pruner/mod.rs
index 53d125d1f..c2cc764be 100644
--- a/core/rust/src/vault_pruner/mod.rs
+++ b/core/rust/src/vault_pruner/mod.rs
@@ -348,22 +348,22 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
/// True if the attachment's Blob field is present and non-empty.
fn attachment_has_blob_bytes(attachment: &Record) -> bool {
- match attachment.get("Blob") {
- None => false,
- Some(serde_json::Value::Null) => false,
- Some(serde_json::Value::String(s)) => !s.is_empty(),
- Some(serde_json::Value::Array(a)) => !a.is_empty(),
- Some(_) => true,
- }
+ value_has_bytes(attachment.get("Blob"))
}
/// True if the logo's FileData field is present and non-empty.
fn logo_has_file_data_bytes(logo: &Record) -> bool {
- match logo.get("FileData") {
+ value_has_bytes(logo.get("FileData"))
+}
+
+/// True if a JSON value represents non-empty blob bytes.
+fn value_has_bytes(value: Option<&serde_json::Value>) -> bool {
+ match value {
None => false,
Some(serde_json::Value::Null) => false,
Some(serde_json::Value::String(s)) => !s.is_empty(),
Some(serde_json::Value::Array(a)) => !a.is_empty(),
+ Some(serde_json::Value::Object(o)) => !o.is_empty(),
Some(_) => true,
}
}
@@ -972,6 +972,60 @@ mod tests {
assert!(output.statements.is_empty());
}
+ #[test]
+ fn test_sweeper_skips_empty_uint8array_object_blob() {
+ // An already-cleared blob must not generate a clear statement on every prune.
+ let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
+ let input = PruneInput {
+ tables: vec![
+ TableData {
+ name: "Items".to_string(),
+ records: vec![],
+ },
+ TableData {
+ name: "Attachments".to_string(),
+ records: vec![make_attachment_record("att-empty-object", "item-1", true, serde_json::json!({}))],
+ },
+ TableData {
+ name: "Logos".to_string(),
+ records: vec![make_logo_record_with_blob("logo-empty-object", true, serde_json::json!({}))],
+ },
+ ],
+ retention_days: 30,
+ current_time: now_str,
+ };
+
+ let output = prune_vault(input).unwrap();
+
+ assert_eq!(output.stats.attachment_blobs_cleared, 0);
+ assert_eq!(output.stats.logo_blobs_cleared, 0);
+ assert!(output.statements.is_empty());
+ }
+
+ #[test]
+ fn test_sweeper_clears_nonempty_uint8array_object_blob() {
+ // Non-empty blobs are serialized as {"0":104,...}.
+ let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
+ let input = PruneInput {
+ tables: vec![
+ TableData {
+ name: "Items".to_string(),
+ records: vec![],
+ },
+ TableData {
+ name: "Attachments".to_string(),
+ records: vec![make_attachment_record("att-object", "item-1", true, serde_json::json!({"0": 104, "1": 105}))],
+ },
+ ],
+ retention_days: 30,
+ current_time: now_str,
+ };
+
+ let output = prune_vault(input).unwrap();
+
+ assert_eq!(output.stats.attachment_blobs_cleared, 1);
+ }
+
#[test]
fn test_sweeper_skips_active_attachment_with_blob() {
let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();