Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update vault bucket push logic (#2159)
Leendert de Borstdcb7c63509b9
diff --git a/core/rust/src/vault_sync/db.rs b/core/rust/src/vault_sync/db.rsindex 0b65f01ec..feebaf808 100644--- a/core/rust/src/vault_sync/db.rs+++ b/core/rust/src/vault_sync/db.rs@@ -231,8 +231,8 @@ const GET_ACCOUNT_KEY_BY_PUBLIC_KEY: &str = "SELECT x.PublicKey, x.PrivateKey, xconst DEMOTE_KEYS_FOR_MANIFEST: &str = "UPDATE EncryptionKeys SET IsPrimary = 0, UpdatedAt = ? WHERE ManifestId = ? AND IsPrimary = 1";const INSERT_KEY_FOR_MANIFEST: &str = "INSERT INTO EncryptionKeys (Id, ManifestId, PublicKey, PrivateKey, IsPrimary, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, 1, ?, ?, 0)";const INSERT_FOLDER: &str = "INSERT INTO Folders (Id, Name, ParentFolderId, ManifestId, Weight, IsDeleted, CreatedAt, UpdatedAt) VALUES (?, ?, ?, ?, 0, 0, ?, ?)";-const RESTAMP_SUBTREE_FOLDERS: &str = "UPDATE Folders SET ManifestId = ? WHERE Id IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";-const RESTAMP_SUBTREE_ITEMS: &str = "UPDATE Items SET ManifestId = ?, UpdatedAt = ? WHERE ManifestId <> ? AND FolderId IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";+const RESTAMP_SUBTREE_ITEMS: &str = "UPDATE Items SET ManifestId = ?, UpdatedAt = ? WHERE ManifestId = ? AND FolderId IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? AND ManifestId = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id WHERE f.ManifestId = ?) SELECT Id FROM subtree)";+const RESTAMP_SUBTREE_FOLDERS: &str = "UPDATE Folders SET ManifestId = ? WHERE ManifestId = ? AND Id IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? AND ManifestId = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id WHERE f.ManifestId = ?) SELECT Id FROM subtree)";const FIND_ITEMS_WITH_FOREIGN_LOGO: &str = "SELECT i.Id, i.ManifestId, origin.Kind, origin.Source FROM Items i INNER JOIN Logos origin ON origin.Id = i.LogoId LEFT JOIN Logos own ON own.Id = i.LogoId AND own.ManifestId = i.ManifestId WHERE i.LogoId IS NOT NULL AND own.Id IS NULL AND i.IsDeleted = 0";const GET_LOGO_ID_FOR_KEY: &str = "SELECT Id FROM Logos WHERE ManifestId = ? AND Kind = ? AND Source = ? AND IsDeleted = 0 LIMIT 1";const GET_BEST_LOGO_FOR_KEY: &str = "SELECT FileData, MimeType, Name FROM Logos WHERE Kind = ? AND Source = ? AND IsDeleted = 0 ORDER BY (FileData IS NOT NULL AND LENGTH(FileData) > 0) DESC, UpdatedAt DESC LIMIT 1";@@ -284,20 +284,21 @@ pub(crate) async fn has_rendered_manifest_folder(host: &Host, manifest_id: &str)Ok(!query(host, Db::Local, RENDERED_MANIFEST_FOLDER, vec![json!(manifest_id)]).await?.is_empty())}+/// The statements that create the folder a shared manifest is rendered as and pull everything under it into the manifest.+pub(crate) fn render_manifest_folder_statements(manifest_id: &str, name: &str, active_manifest_id: &str, now: &str) -> Vec<SqlStatement> {+ let folder_id = id_key(manifest_id);+ vec![+ SqlStatement { sql: INSERT_FOLDER.to_string(), params: vec![json!(folder_id), json!(name), Value::Null, json!(active_manifest_id), json!(now), json!(now)] },+ // Items first: their walk reads the folder stamps the last statement changes.+ SqlStatement { sql: RESTAMP_SUBTREE_ITEMS.to_string(), params: vec![json!(manifest_id), json!(now), json!(active_manifest_id), json!(folder_id), json!(active_manifest_id), json!(active_manifest_id)] },+ SqlStatement { sql: RESTAMP_SUBTREE_FOLDERS.to_string(), params: vec![json!(manifest_id), json!(active_manifest_id), json!(folder_id), json!(active_manifest_id), json!(active_manifest_id)] },+ ]+}+/// Create the folder a shared manifest is rendered as and pull everything under it into the manifest.pub(crate) async fn render_manifest_folder(host: &Host, manifest_id: &str, name: &str, active_manifest_id: &str) -> SyncResult<()> {- let folder_id = id_key(manifest_id);let now = now_vault_datetime();- exec(- host,- Db::Local,- vec![- SqlStatement { sql: INSERT_FOLDER.to_string(), params: vec![json!(folder_id), json!(name), Value::Null, json!(active_manifest_id), json!(now), json!(now)] },- SqlStatement { sql: RESTAMP_SUBTREE_FOLDERS.to_string(), params: vec![json!(manifest_id), json!(folder_id)] },- SqlStatement { sql: RESTAMP_SUBTREE_ITEMS.to_string(), params: vec![json!(manifest_id), json!(now), json!(manifest_id), json!(folder_id)] },- ],- )- .await?;+ exec(host, Db::Local, render_manifest_folder_statements(manifest_id, name, active_manifest_id, &now)).await?;reconcile_item_logo_scopes(host, &now).await}diff --git a/core/rust/src/vault_sync/push.rs b/core/rust/src/vault_sync/push.rsindex 5d5b5f571..4c48a5cd7 100644--- a/core/rust/src/vault_sync/push.rs+++ b/core/rust/src/vault_sync/push.rs@@ -705,7 +705,9 @@ async fn upload_blobs(ctx: &Ctx, blobs: &UploadBlobs, hashes: &[String], overwri* The bucket-only write.*/-/// Single-data-bucket upload through the unified write, rebasing onto the server's revision and retrying once.+/*+ * Single-data-bucket upload through the unified write. A stale revision reports `Outdated` returns early.+ */async fn push_data_bucket_only(ctx: &Ctx, bucket: &DataBucket, vek: &str) -> SyncResult<(PushStatus, i64)> {http::with_outdated_server_guard(push_data_bucket_only_internal(ctx, bucket, vek).await)}@@ -721,32 +723,23 @@ async fn push_data_bucket_only_internal(ctx: &Ctx, bucket: &DataBucket, vek: &st}let sealed = seal(ctx, &label, &plaintext, vek).await?;- let post = |current_revision: i64| {- let payload = VaultWriteRequest {- username: ctx.request.username.clone(),- manifests: Vec::new(),- buckets: vec![BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: sealed.ciphertext.clone(), ciphertext_hash: sealed.hash.clone(), current_revision }],- new_blobs: Vec::new(),- email_routing: None,- account_keys: None,- };- async move { http::post::<_, VaultWriteResponse>(&ctx.host, http::VAULT_ENDPOINT, &payload, true).await }+ let current_revision = baselines.bucket_revision(bucket);+ let payload = VaultWriteRequest {+ username: ctx.request.username.clone(),+ manifests: Vec::new(),+ buckets: vec![BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: sealed.ciphertext, ciphertext_hash: sealed.hash, current_revision }],+ new_blobs: Vec::new(),+ email_routing: None,+ account_keys: None,};- let reported = |response: &VaultWriteResponse| response.bucket_revisions.iter().find(|b| ids_equal(&b.manifest_id, &bucket.manifest_id) && b.category == bucket.category).map(|b| b.revision);-- let mut current_revision = baselines.bucket_revision(bucket);- let mut response = post(current_revision).await?;- if response.status != 0 {- let server_revision = reported(&response).unwrap_or(current_revision);- ctx.warn(format!("[V2Push] {} outdated (server at revision {}, we assumed {}); rebasing and retrying once.", label, server_revision, current_revision)).await;- current_revision = server_revision;- response = post(current_revision).await?;- }+ let response: VaultWriteResponse = http::post(&ctx.host, http::VAULT_ENDPOINT, &payload, true).await?;+ let reported = response.bucket_revisions.iter().find(|b| ids_equal(&b.manifest_id, &bucket.manifest_id) && b.category == bucket.category).map(|b| b.revision);if response.status != 0 {- return Ok((PushStatus::Outdated, reported(&response).unwrap_or(current_revision)));+ ctx.warn(format!("[V2Push] {} outdated (server at revision {}, we assumed {}); pulling and merging before the next attempt.", label, reported.unwrap_or(current_revision), current_revision)).await;+ return Ok((PushStatus::Outdated, reported.unwrap_or(current_revision)));}- let new_revision = reported(&response).unwrap_or(current_revision + 1);+ let new_revision = reported.unwrap_or(current_revision + 1);let advanced = BucketRevision { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), revision: new_revision };commit_push_baselines(ctx, baselines, &[], &[advanced], HashMap::from([(fingerprint_key, fingerprint)])).await?;Ok((PushStatus::Ok, new_revision))diff --git a/core/rust/src/vault_sync/tests/mod.rs b/core/rust/src/vault_sync/tests/mod.rsindex 985785aea..fcb880276 100644--- a/core/rust/src/vault_sync/tests/mod.rs+++ b/core/rust/src/vault_sync/tests/mod.rs@@ -1,6 +1,7 @@//! Engine tests: the whole sync driven through the command loop against a real SQLite host.mod item_move;+mod render_manifest_folder;mod test_host;use std::collections::HashMap;@@ -304,6 +305,78 @@ fn outdated_push_merges_the_server_change_and_retries() {assert_eq!(merged["tables"]["Items"].as_array().unwrap().len(), 3);}+fn insert_item_stats(conn: &rusqlite::Connection, item_id: &str, use_count: i64) {+ let now = crate::timestamp::now_vault_datetime();+ conn.execute(+ "INSERT INTO ItemStats (ManifestId, Id, UseCount, AutofillCount, CopyCount, PasskeyAuthCount, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 0, 0, 0, ?, ?, 0)",+ rusqlite::params![PERSONAL_MANIFEST_ID, item_id, use_count, now, now],+ )+ .unwrap();+}++#[test]+fn outdated_bucket_only_push_merges_the_server_bucket_instead_of_overwriting_it() {+ const ITEM_A: &str = "aaaaaaaa-0000-4000-8000-000000000001";+ const ITEM_B: &str = "aaaaaaaa-0000-4000-8000-000000000002";+ let vek = crypto::generate_key_base64();+ let salt = vault_codec::generate_manifest_salt();+ let mut host = TestHost::new(&vek);+ let server_db = test_host::open_schema_db(&host.schema_sql);+ insert_item(&server_db, ITEM_A, "Item A", PERSONAL_MANIFEST_ID);+ insert_item(&server_db, ITEM_B, "Item B", PERSONAL_MANIFEST_ID);+ let (status7, vault7) = snapshot_of(&server_db, &vek, 7, &salt);+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));+ host.respond("GET", "Status", status7.clone());+ host.respond("GET", "Vault", vault7);+ host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());++ // Another device writes the stats of item B after this client's status check, while this one used item A.+ insert_item_stats(&server_db, ITEM_B, 5);+ let (status8, vault8) = snapshot_of(&server_db, &vek, 8, &salt);+ insert_item_stats(&host.local, ITEM_A, 1);+ host.store_local_as_blob();+ host.mutation_sequence = 1;+ host.is_dirty = true;++ let status_calls = std::rc::Rc::new(std::cell::Cell::new(0u32));+ host.responders.clear();+ host.respond_with(Box::new(move |method, path, _| {+ if method != "GET" || path != "Status" {+ return None;+ }+ status_calls.set(status_calls.get() + 1);+ Some((200, if status_calls.get() == 1 { status7.clone() } else { status8.clone() }))+ }));+ host.respond("GET", "Vault", vault8);+ host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));+ host.respond_with(Box::new(|method, path, body| {+ if method != "POST" || path != "Vault" {+ return None;+ }+ let current = body.and_then(|b| b["buckets"][0]["currentRevision"].as_i64()).unwrap_or(-1);+ let revision = if current == 8 { 9 } else { 8 };+ Some((200, json!({ "status": if current == 8 { 0 } else { 2 }, "manifestRevisions": [], "bucketRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "category": "Stats", "revision": revision }], "missingBlobHashes": [] })))+ }));++ let mut bucket_request: Value = serde_json::from_str(&request("fullSync", &vek, true, 1)).unwrap();+ bucket_request["dirtyScopes"] = json!(["Stats"]);+ let result = host.drive(&SyncSession::new(&bucket_request.to_string()).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ let posts: Vec<_> = host.requests_to("Vault").into_iter().filter(|r| r.method == "POST").collect();+ let accepted = posts.last().unwrap().body.as_ref().unwrap();+ assert_eq!(accepted["buckets"][0]["currentRevision"], 8);+ let bucket: Value = {+ let plain = crypto::symmetric_decrypt_bytes(&crate::encoding::base64_decode(accepted["buckets"][0]["blob"].as_str().unwrap()).unwrap(), &vek).unwrap();+ serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap()+ };+ assert_eq!(bucket["tables"]["ItemStats"].as_array().unwrap().len(), 2, "the other device's stats must survive: {}", bucket);+ let tail: Vec<String> = host.requests.iter().rev().take(4).rev().map(|r| format!("{} {}", r.method, r.path)).collect();+ assert_eq!(tail, vec!["POST Vault", "GET Status", "GET Vault", "POST Vault"], "the refused write must be followed by a pull before the next attempt");+ assert_eq!(posts.len(), 2);+ assert_eq!(posts[0].body.as_ref().unwrap()["buckets"][0]["currentRevision"], 7);+}+#[test]fn unreachable_server_with_a_local_vault_goes_offline() {let vek = crypto::generate_key_base64();