Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update vault bucket push logic (#2159)

Leendert de Borstdcb7c63509b9
diff --git a/core/rust/src/vault_sync/db.rs b/core/rust/src/vault_sync/db.rs
index 0b65f01ec..feebaf808 100644
--- a/core/rust/src/vault_sync/db.rs
+++ b/core/rust/src/vault_sync/db.rs
@@ -231,8 +231,8 @@ const GET_ACCOUNT_KEY_BY_PUBLIC_KEY: &str = "SELECT x.PublicKey, x.PrivateKey, x
const DEMOTE_KEYS_FOR_MANIFEST: &str = "UPDATE EncryptionKeys SET IsPrimary = 0, UpdatedAt = ? WHERE ManifestId = ? AND IsPrimary = 1";
const INSERT_KEY_FOR_MANIFEST: &str = "INSERT INTO EncryptionKeys (Id, ManifestId, PublicKey, PrivateKey, IsPrimary, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, 1, ?, ?, 0)";
const INSERT_FOLDER: &str = "INSERT INTO Folders (Id, Name, ParentFolderId, ManifestId, Weight, IsDeleted, CreatedAt, UpdatedAt) VALUES (?, ?, ?, ?, 0, 0, ?, ?)";
-const RESTAMP_SUBTREE_FOLDERS: &str = "UPDATE Folders SET ManifestId = ? WHERE Id IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";
-const RESTAMP_SUBTREE_ITEMS: &str = "UPDATE Items SET ManifestId = ?, UpdatedAt = ? WHERE ManifestId <> ? AND FolderId IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";
+const RESTAMP_SUBTREE_ITEMS: &str = "UPDATE Items SET ManifestId = ?, UpdatedAt = ? WHERE ManifestId = ? AND FolderId IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? AND ManifestId = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id WHERE f.ManifestId = ?) SELECT Id FROM subtree)";
+const RESTAMP_SUBTREE_FOLDERS: &str = "UPDATE Folders SET ManifestId = ? WHERE ManifestId = ? AND Id IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? AND ManifestId = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id WHERE f.ManifestId = ?) SELECT Id FROM subtree)";
const FIND_ITEMS_WITH_FOREIGN_LOGO: &str = "SELECT i.Id, i.ManifestId, origin.Kind, origin.Source FROM Items i INNER JOIN Logos origin ON origin.Id = i.LogoId LEFT JOIN Logos own ON own.Id = i.LogoId AND own.ManifestId = i.ManifestId WHERE i.LogoId IS NOT NULL AND own.Id IS NULL AND i.IsDeleted = 0";
const GET_LOGO_ID_FOR_KEY: &str = "SELECT Id FROM Logos WHERE ManifestId = ? AND Kind = ? AND Source = ? AND IsDeleted = 0 LIMIT 1";
const GET_BEST_LOGO_FOR_KEY: &str = "SELECT FileData, MimeType, Name FROM Logos WHERE Kind = ? AND Source = ? AND IsDeleted = 0 ORDER BY (FileData IS NOT NULL AND LENGTH(FileData) > 0) DESC, UpdatedAt DESC LIMIT 1";
@@ -284,20 +284,21 @@ pub(crate) async fn has_rendered_manifest_folder(host: &Host, manifest_id: &str)
Ok(!query(host, Db::Local, RENDERED_MANIFEST_FOLDER, vec![json!(manifest_id)]).await?.is_empty())
}
+/// The statements that create the folder a shared manifest is rendered as and pull everything under it into the manifest.
+pub(crate) fn render_manifest_folder_statements(manifest_id: &str, name: &str, active_manifest_id: &str, now: &str) -> Vec<SqlStatement> {
+ let folder_id = id_key(manifest_id);
+ vec![
+ SqlStatement { sql: INSERT_FOLDER.to_string(), params: vec![json!(folder_id), json!(name), Value::Null, json!(active_manifest_id), json!(now), json!(now)] },
+ // Items first: their walk reads the folder stamps the last statement changes.
+ SqlStatement { sql: RESTAMP_SUBTREE_ITEMS.to_string(), params: vec![json!(manifest_id), json!(now), json!(active_manifest_id), json!(folder_id), json!(active_manifest_id), json!(active_manifest_id)] },
+ SqlStatement { sql: RESTAMP_SUBTREE_FOLDERS.to_string(), params: vec![json!(manifest_id), json!(active_manifest_id), json!(folder_id), json!(active_manifest_id), json!(active_manifest_id)] },
+ ]
+}
+
/// Create the folder a shared manifest is rendered as and pull everything under it into the manifest.
pub(crate) async fn render_manifest_folder(host: &Host, manifest_id: &str, name: &str, active_manifest_id: &str) -> SyncResult<()> {
- let folder_id = id_key(manifest_id);
let now = now_vault_datetime();
- exec(
- host,
- Db::Local,
- vec![
- SqlStatement { sql: INSERT_FOLDER.to_string(), params: vec![json!(folder_id), json!(name), Value::Null, json!(active_manifest_id), json!(now), json!(now)] },
- SqlStatement { sql: RESTAMP_SUBTREE_FOLDERS.to_string(), params: vec![json!(manifest_id), json!(folder_id)] },
- SqlStatement { sql: RESTAMP_SUBTREE_ITEMS.to_string(), params: vec![json!(manifest_id), json!(now), json!(manifest_id), json!(folder_id)] },
- ],
- )
- .await?;
+ exec(host, Db::Local, render_manifest_folder_statements(manifest_id, name, active_manifest_id, &now)).await?;
reconcile_item_logo_scopes(host, &now).await
}
diff --git a/core/rust/src/vault_sync/push.rs b/core/rust/src/vault_sync/push.rs
index 5d5b5f571..4c48a5cd7 100644
--- a/core/rust/src/vault_sync/push.rs
+++ b/core/rust/src/vault_sync/push.rs
@@ -705,7 +705,9 @@ async fn upload_blobs(ctx: &Ctx, blobs: &UploadBlobs, hashes: &[String], overwri
* The bucket-only write.
*/
-/// Single-data-bucket upload through the unified write, rebasing onto the server's revision and retrying once.
+/*
+ * Single-data-bucket upload through the unified write. A stale revision reports `Outdated` returns early.
+ */
async fn push_data_bucket_only(ctx: &Ctx, bucket: &DataBucket, vek: &str) -> SyncResult<(PushStatus, i64)> {
http::with_outdated_server_guard(push_data_bucket_only_internal(ctx, bucket, vek).await)
}
@@ -721,32 +723,23 @@ async fn push_data_bucket_only_internal(ctx: &Ctx, bucket: &DataBucket, vek: &st
}
let sealed = seal(ctx, &label, &plaintext, vek).await?;
- let post = |current_revision: i64| {
- let payload = VaultWriteRequest {
- username: ctx.request.username.clone(),
- manifests: Vec::new(),
- buckets: vec![BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: sealed.ciphertext.clone(), ciphertext_hash: sealed.hash.clone(), current_revision }],
- new_blobs: Vec::new(),
- email_routing: None,
- account_keys: None,
- };
- async move { http::post::<_, VaultWriteResponse>(&ctx.host, http::VAULT_ENDPOINT, &payload, true).await }
+ let current_revision = baselines.bucket_revision(bucket);
+ let payload = VaultWriteRequest {
+ username: ctx.request.username.clone(),
+ manifests: Vec::new(),
+ buckets: vec![BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: sealed.ciphertext, ciphertext_hash: sealed.hash, current_revision }],
+ new_blobs: Vec::new(),
+ email_routing: None,
+ account_keys: None,
};
- let reported = |response: &VaultWriteResponse| response.bucket_revisions.iter().find(|b| ids_equal(&b.manifest_id, &bucket.manifest_id) && b.category == bucket.category).map(|b| b.revision);
-
- let mut current_revision = baselines.bucket_revision(bucket);
- let mut response = post(current_revision).await?;
- if response.status != 0 {
- let server_revision = reported(&response).unwrap_or(current_revision);
- ctx.warn(format!("[V2Push] {} outdated (server at revision {}, we assumed {}); rebasing and retrying once.", label, server_revision, current_revision)).await;
- current_revision = server_revision;
- response = post(current_revision).await?;
- }
+ let response: VaultWriteResponse = http::post(&ctx.host, http::VAULT_ENDPOINT, &payload, true).await?;
+ let reported = response.bucket_revisions.iter().find(|b| ids_equal(&b.manifest_id, &bucket.manifest_id) && b.category == bucket.category).map(|b| b.revision);
if response.status != 0 {
- return Ok((PushStatus::Outdated, reported(&response).unwrap_or(current_revision)));
+ ctx.warn(format!("[V2Push] {} outdated (server at revision {}, we assumed {}); pulling and merging before the next attempt.", label, reported.unwrap_or(current_revision), current_revision)).await;
+ return Ok((PushStatus::Outdated, reported.unwrap_or(current_revision)));
}
- let new_revision = reported(&response).unwrap_or(current_revision + 1);
+ let new_revision = reported.unwrap_or(current_revision + 1);
let advanced = BucketRevision { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), revision: new_revision };
commit_push_baselines(ctx, baselines, &[], &[advanced], HashMap::from([(fingerprint_key, fingerprint)])).await?;
Ok((PushStatus::Ok, new_revision))
diff --git a/core/rust/src/vault_sync/tests/mod.rs b/core/rust/src/vault_sync/tests/mod.rs
index 985785aea..fcb880276 100644
--- a/core/rust/src/vault_sync/tests/mod.rs
+++ b/core/rust/src/vault_sync/tests/mod.rs
@@ -1,6 +1,7 @@
//! Engine tests: the whole sync driven through the command loop against a real SQLite host.
mod item_move;
+mod render_manifest_folder;
mod test_host;
use std::collections::HashMap;
@@ -304,6 +305,78 @@ fn outdated_push_merges_the_server_change_and_retries() {
assert_eq!(merged["tables"]["Items"].as_array().unwrap().len(), 3);
}
+fn insert_item_stats(conn: &rusqlite::Connection, item_id: &str, use_count: i64) {
+ let now = crate::timestamp::now_vault_datetime();
+ conn.execute(
+ "INSERT INTO ItemStats (ManifestId, Id, UseCount, AutofillCount, CopyCount, PasskeyAuthCount, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 0, 0, 0, ?, ?, 0)",
+ rusqlite::params![PERSONAL_MANIFEST_ID, item_id, use_count, now, now],
+ )
+ .unwrap();
+}
+
+#[test]
+fn outdated_bucket_only_push_merges_the_server_bucket_instead_of_overwriting_it() {
+ const ITEM_A: &str = "aaaaaaaa-0000-4000-8000-000000000001";
+ const ITEM_B: &str = "aaaaaaaa-0000-4000-8000-000000000002";
+ let vek = crypto::generate_key_base64();
+ let salt = vault_codec::generate_manifest_salt();
+ let mut host = TestHost::new(&vek);
+ let server_db = test_host::open_schema_db(&host.schema_sql);
+ insert_item(&server_db, ITEM_A, "Item A", PERSONAL_MANIFEST_ID);
+ insert_item(&server_db, ITEM_B, "Item B", PERSONAL_MANIFEST_ID);
+ let (status7, vault7) = snapshot_of(&server_db, &vek, 7, &salt);
+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
+ host.respond("GET", "Status", status7.clone());
+ host.respond("GET", "Vault", vault7);
+ host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
+
+ // Another device writes the stats of item B after this client's status check, while this one used item A.
+ insert_item_stats(&server_db, ITEM_B, 5);
+ let (status8, vault8) = snapshot_of(&server_db, &vek, 8, &salt);
+ insert_item_stats(&host.local, ITEM_A, 1);
+ host.store_local_as_blob();
+ host.mutation_sequence = 1;
+ host.is_dirty = true;
+
+ let status_calls = std::rc::Rc::new(std::cell::Cell::new(0u32));
+ host.responders.clear();
+ host.respond_with(Box::new(move |method, path, _| {
+ if method != "GET" || path != "Status" {
+ return None;
+ }
+ status_calls.set(status_calls.get() + 1);
+ Some((200, if status_calls.get() == 1 { status7.clone() } else { status8.clone() }))
+ }));
+ host.respond("GET", "Vault", vault8);
+ host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
+ host.respond_with(Box::new(|method, path, body| {
+ if method != "POST" || path != "Vault" {
+ return None;
+ }
+ let current = body.and_then(|b| b["buckets"][0]["currentRevision"].as_i64()).unwrap_or(-1);
+ let revision = if current == 8 { 9 } else { 8 };
+ Some((200, json!({ "status": if current == 8 { 0 } else { 2 }, "manifestRevisions": [], "bucketRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "category": "Stats", "revision": revision }], "missingBlobHashes": [] })))
+ }));
+
+ let mut bucket_request: Value = serde_json::from_str(&request("fullSync", &vek, true, 1)).unwrap();
+ bucket_request["dirtyScopes"] = json!(["Stats"]);
+ let result = host.drive(&SyncSession::new(&bucket_request.to_string()).unwrap());
+
+ assert_eq!(result["success"], true, "{}", result);
+ let posts: Vec<_> = host.requests_to("Vault").into_iter().filter(|r| r.method == "POST").collect();
+ let accepted = posts.last().unwrap().body.as_ref().unwrap();
+ assert_eq!(accepted["buckets"][0]["currentRevision"], 8);
+ let bucket: Value = {
+ let plain = crypto::symmetric_decrypt_bytes(&crate::encoding::base64_decode(accepted["buckets"][0]["blob"].as_str().unwrap()).unwrap(), &vek).unwrap();
+ serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap()
+ };
+ assert_eq!(bucket["tables"]["ItemStats"].as_array().unwrap().len(), 2, "the other device's stats must survive: {}", bucket);
+ let tail: Vec<String> = host.requests.iter().rev().take(4).rev().map(|r| format!("{} {}", r.method, r.path)).collect();
+ assert_eq!(tail, vec!["POST Vault", "GET Status", "GET Vault", "POST Vault"], "the refused write must be followed by a pull before the next attempt");
+ assert_eq!(posts.len(), 2);
+ assert_eq!(posts[0].body.as_ref().unwrap()["buckets"][0]["currentRevision"], 7);
+}
+
#[test]
fn unreachable_server_with_a_local_vault_goes_offline() {
let vek = crypto::generate_key_base64();