Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Add separate legacy sqlite-blob to manifest-v1 migration call (#2159)
Leendert de Borst322444f30e20
diff --git a/core/rust/src/vault_sync/engine.rs b/core/rust/src/vault_sync/engine.rsindex 357a70a65..53bcc07fe 100644--- a/core/rust/src/vault_sync/engine.rs+++ b/core/rust/src/vault_sync/engine.rs@@ -276,7 +276,7 @@ async fn enter_offline_mode(ctx: &mut Ctx) -> SyncResult<FullSyncResult> {}/// Where the local vault's schema stands against the current one.-async fn schema_state(ctx: &mut Ctx) -> SyncResult<SchemaState> {+pub(crate) async fn schema_state(ctx: &mut Ctx) -> SyncResult<SchemaState> {let schema = ctx.schema().await?;db::schema_state(&ctx.host, &schema.migration_id).await}@@ -500,43 +500,14 @@ async fn migration_status(ctx: &mut Ctx) -> MigrationStatusResult {MigrationStatusResult { kind, session: SessionOutcome::default() }}-/// Upgrade the local vault to the current storage model and push it.+/// Bring the local vault onto the current storage model and push it: a schema rebuild for a migrated account, the+/// whole sqlite-blob move (`legacy::migrate_sqlite_blob`) for an account without a key hierarchy yet.async fn migrate_manifest(ctx: &mut Ctx) -> MigrateManifestResult {let migrated: SyncResult<bool> = async {- let encryption_key = ctx.encryption_key()?;- let schema = schema_state(ctx).await?;- if schema == SchemaState::LegacyChain {- return Err(SyncError::LegacyUpgradePending);- }- if !keys::has_local_vault_key(&ctx.host).await? && !keys::adopt_hierarchy_created_elsewhere(ctx).await? {- return Err(SyncError::KeyOutOfSync);- }- let needs_schema_migration = schema == SchemaState::Stale;- let needs_vault_key = !keys::has_local_vault_key(&ctx.host).await?;- if !needs_schema_migration && !needs_vault_key {- return Ok(true);- }- if needs_schema_migration {- let migrated_bytes = legacy::migrate_vault_to_current_schema(ctx).await?;- let stored = ctx.store_vault(&state::encrypt_vault_blob(&migrated_bytes, &encryption_key)?, true, None, None).await?;- ctx.mutation_sequence = stored.mutation_sequence;- ctx.is_dirty = true;- ctx.vault_changed = true;- }- match push::upload_vault(ctx, None, false, needs_vault_key).await {- Ok(upload) if upload.status == PushStatus::Ok => {- state::mark_clean(&ctx.host, upload.mutation_seq_at_start).await?;- Ok(true)- }- Ok(upload) => {- ctx.warn(format!("[ManifestMigration] Migration push did not succeed ({:?}), vault stays dirty for the next sync.", upload.status)).await;- Ok(false)- }- Err(error) => {- ctx.warn(format!("[ManifestMigration] Migration push failed, vault stays dirty for the next sync: {}", error)).await;- Ok(false)- }+ if !keys::has_local_vault_key(&ctx.host).await? {+ return legacy::migrate_sqlite_blob(ctx).await;}+ migrate_schema(ctx).await}.await;match migrated {@@ -548,6 +519,61 @@ async fn migrate_manifest(ctx: &mut Ctx) -> MigrateManifestResult {}}+/// The permanent migration: rebuild a stale local schema onto the current one and push it. Returns whether the+/// push reached the server; a vault already on the current schema counts as pushed.+pub(crate) async fn migrate_schema(ctx: &mut Ctx) -> SyncResult<bool> {+ match schema_state(ctx).await? {+ SchemaState::Current => Ok(true),+ SchemaState::LegacyChain => Err(SyncError::LegacyUpgradePending),+ SchemaState::Stale => {+ rebuild_local_schema(ctx, None).await?;+ push_migrated_vault(ctx, false).await+ }+ }+}++/// Rebuild the local vault onto the current schema (a local round-trip through the codec, no server involved) and+/// store it as a pending change. `adopt_unstamped_into` stamps rows that carry no manifest yet, which only a+/// sqlite-blob vault has.+pub(crate) async fn rebuild_local_schema(ctx: &mut Ctx, adopt_unstamped_into: Option<String>) -> SyncResult<()> {+ ctx.log("[ManifestMigration] Migrating local vault onto the current schema (local round-trip, no server involved)...").await;+ let set = push::canonicalize_vault(ctx, adopt_unstamped_into).await?;+ let mut blob_map = HashMap::new();+ for entry in &set.canonicalized.manifests {+ for (hash, blob) in &entry.blobs {+ blob_map.insert(hash.clone(), crate::encoding::base64_decode(&blob.bytes_base64)?);+ }+ }+ let manifests: Vec<_> = set.canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect();+ let bytes = pull::materialize_to_sqlite(ctx, &manifests, &set.canonicalized.data_buckets, &blob_map).await?;+ ctx.log(format!("[ManifestMigration] Migration complete: {} blobs re-embedded, {} bytes.", blob_map.len(), bytes.len())).await;++ let stored = ctx.store_vault(&state::encrypt_vault_blob(&bytes, &ctx.encryption_key()?)?, true, None, None).await?;+ ctx.mutation_sequence = stored.mutation_sequence;+ ctx.is_dirty = true;+ ctx.vault_changed = true;+ Ok(())+}++/// Push the migrated vault and clear the dirty flag. A push the server refuses, or that fails, is not a migration+/// failure: the vault stays dirty and the next sync carries it. Returns whether the push reached the server.+pub(crate) async fn push_migrated_vault(ctx: &mut Ctx, create_vault_key: bool) -> SyncResult<bool> {+ match push::upload_vault(ctx, None, false, create_vault_key).await {+ Ok(upload) if upload.status == PushStatus::Ok => {+ state::mark_clean(&ctx.host, upload.mutation_seq_at_start).await?;+ Ok(true)+ }+ Ok(upload) => {+ ctx.warn(format!("[ManifestMigration] Migration push did not succeed ({:?}), vault stays dirty for the next sync.", upload.status)).await;+ Ok(false)+ }+ Err(error) => {+ ctx.warn(format!("[ManifestMigration] Migration push failed, vault stays dirty for the next sync: {}", error)).await;+ Ok(false)+ }+ }+}+/// The login-time key resolution; see `keys::resolve_vault_key`.async fn resolve_vault_key(ctx: &mut Ctx) -> ResolveVaultKeyResult {match keys::resolve_vault_key(ctx).await {diff --git a/core/rust/src/vault_sync/legacy.rs b/core/rust/src/vault_sync/legacy.rsindex 2428bc925..d7cb2b084 100644--- a/core/rust/src/vault_sync/legacy.rs+++ b/core/rust/src/vault_sync/legacy.rs@@ -3,12 +3,13 @@use std::collections::HashMap;+use super::db::SchemaState;use super::errors::{SyncError, SyncResult};-use super::pull::{email_routing_of, materialize_to_sqlite, PulledVault};-use super::push::{canonicalize_vault, resolve_personal_manifest_id};+use super::pull::{self, email_routing_of, PulledVault};+use super::push::resolve_personal_manifest_id;use super::state::{self, Ctx};use super::types::GetResponse;-use crate::vault_codec::Manifest;+use super::{engine, keys};/// The `storageFormat` a manifest-v1 snapshot declares; 0 (or absent) is a sqlite blob.const STORAGE_FORMAT_MANIFEST: i32 = 1;@@ -32,6 +33,51 @@ pub(crate) async fn open_legacy_snapshot(ctx: &Ctx, snapshot: &GetResponse) -> SOk(PulledVault { encrypted_vault: snapshot.legacy_vault_blob.clone().unwrap_or_default(), revision, email_routing: email_routing_of(snapshot), manifest_revisions, bucket_revisions: HashMap::new() })}+/// The one-way move of a sqlite-blob account onto the manifest storage format: the local vault is rebuilt onto the+/// current schema with its unstamped rows adopted into the personal manifest, and the push that carries it mints the+/// account key hierarchy. Returns whether that push reached the server.+pub(crate) async fn migrate_sqlite_blob(ctx: &mut Ctx) -> SyncResult<bool> {+ if engine::schema_state(ctx).await? == SchemaState::LegacyChain {+ return Err(SyncError::LegacyUpgradePending);+ }+ // Another device may have created the hierarchy since this one logged in. Adopting it swaps the session key to the VEK, which the baseline pull below needs.+ if !keys::adopt_hierarchy_created_elsewhere(ctx).await? {+ return Err(SyncError::KeyOutOfSync);+ }+ record_server_baseline_if_missing(ctx).await?;+ if keys::has_local_vault_key(&ctx.host).await? {+ // The account turned out to be migrated already (adopted above, or pulled with the baseline); a schema rebuild is all that can remain.+ return engine::migrate_schema(ctx).await;+ }+ if engine::schema_state(ctx).await? == SchemaState::LegacyChain {+ // The baseline pull adopted a server vault that is still on the chain.+ return Err(SyncError::LegacyUpgradePending);+ }+ let personal = resolve_personal_manifest_id(ctx).await?;+ engine::rebuild_local_schema(ctx, Some(personal)).await?;+ engine::push_migrated_vault(ctx, true).await+}++/// A session that logged in through a client predating the manifest storage format never pulled through this engine,+/// so it holds neither the personal manifest id nor the revision baseline, and the migration push needs both (the+/// server refuses a write whose revision it does not know). Calling this method before a push fixes this.+pub(crate) async fn record_server_baseline_if_missing(ctx: &mut Ctx) -> SyncResult<()> {+ if state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await?.is_some() {+ return Ok(());+ }+ ctx.log("[ManifestMigration] The session predates the manifest storage format and never pulled; fetching the server vault for the personal manifest id and the revision baseline.").await;+ let pulled = pull::pull(ctx).await?;+ if ctx.is_dirty && ctx.has_local_vault().await? {+ ctx.warn("[ManifestMigration] The local vault has pending changes; keeping it and recording only the server's revisions so the migration push carries them.").await;+ return pull::commit_revisions(ctx, &pulled.manifest_revisions, &pulled.bucket_revisions).await;+ }+ if !ctx.store_vault(&pulled.encrypted_vault, false, Some(ctx.mutation_sequence), Some(pulled.revision)).await?.success {+ return Err(SyncError::Other("a mutation raced the baseline pull; run the migration again".to_string()));+ }+ ctx.vault_changed = true;+ pull::commit_revisions(ctx, &pulled.manifest_revisions, &pulled.bucket_revisions).await+}+/// The frozen sqlite-blob upgrade chain: (revision, data version). It ends at 2.0.0, the first schema compatible/// with manifest-v1; later schema changes ship through the full schema only, which every materialization uses directly.const LEGACY_VAULT_VERSIONS: &[(u32, &str)] = &[@@ -84,24 +130,6 @@ pub(crate) fn stamp_predates_manifest_schema(migration_id: &str) -> SyncResult<bOk(revision < latest_legacy_revision())}-/// Migrate the local vault onto the current schema, entirely locally.-pub(crate) async fn migrate_vault_to_current_schema(ctx: &mut Ctx) -> SyncResult<Vec<u8>> {- ctx.log("[ManifestMigration] Migrating local vault onto the current schema (local round-trip, no server involved)...").await;- // A sqlite-blob vault's rows carry no ManifestId yet, so this one canonicalize adopts them.- let personal = resolve_personal_manifest_id(ctx).await?;- let set = canonicalize_vault(ctx, Some(personal)).await?;- let mut blob_map = HashMap::new();- for entry in &set.canonicalized.manifests {- for (hash, blob) in &entry.blobs {- blob_map.insert(hash.clone(), crate::encoding::base64_decode(&blob.bytes_base64)?);- }- }- let manifests: Vec<Manifest> = set.canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect();- let bytes = materialize_to_sqlite(ctx, &manifests, &set.canonicalized.data_buckets, &blob_map).await?;- ctx.log(format!("[ManifestMigration] Migration complete: {} blobs re-embedded, {} bytes.", blob_map.len(), bytes.len())).await;- Ok(bytes)-}-#[cfg(test)]mod tests {use super::*;diff --git a/core/rust/src/vault_sync/tests/mod.rs b/core/rust/src/vault_sync/tests/mod.rsnew file mode 100644index 000000000..fd942f72e--- /dev/null+++ b/core/rust/src/vault_sync/tests/mod.rs@@ -0,0 +1,669 @@+//! Engine tests: the whole sync driven through the command loop against a real SQLite host.++mod test_host;++use std::collections::HashMap;++use serde_json::{json, Value};++use self::test_host::{query, TestHost};+use crate::crypto;+use crate::vault_codec::{self, CanonicalizeInput, CodecTableData, ManifestSpec};+use crate::vault_sync::session::SyncSession;+use crate::vault_sync::state;+use crate::vault_sync::types::Command;++const PERSONAL_MANIFEST_ID: &str = "11111111-1111-4111-8111-111111111111";+const USERNAME: &str = "tester";++fn request(operation: &str, key: &str, dirty: bool, mutation_sequence: u64) -> String {+ json!({+ "operation": operation,+ "username": USERNAME,+ "encryptionKey": key,+ "isDirty": dirty,+ "mutationSequence": mutation_sequence,+ "dirtyScopes": if dirty { vec!["Main"] } else { vec![] },+ "privateEmailDomains": ["private.io"],+ "minServerVersion": "0.12.0-dev",+ })+ .to_string()+}++fn insert_item(conn: &rusqlite::Connection, id: &str, name: &str, manifest_id: &str) {+ let now = crate::vault_sync::db::now();+ conn.execute(+ "INSERT INTO Items (Id, ManifestId, Name, ItemType, FolderId, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'Login', NULL, ?, ?, 0)",+ rusqlite::params![id, manifest_id, name, now, now],+ )+ .unwrap();+}++/// Read every user table of a connection into the codec's input shape.+fn read_tables(conn: &rusqlite::Connection) -> Vec<CodecTableData> {+ let names: Vec<String> = query(conn, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name", &[]).unwrap().iter().map(|r| r["name"].as_str().unwrap().to_string()).collect();+ names+ .into_iter()+ .map(|name| CodecTableData { records: query(conn, &format!("SELECT * FROM \"{}\"", name), &[]).unwrap().into_iter().map(|row| row.into_iter().collect()).collect(), name })+ .collect()+}++/// A server-side snapshot of the given database as a manifest-v1 personal manifest encrypted under `vek`.+fn snapshot_of(conn: &rusqlite::Connection, vek: &str, revision: i64, salt: &str) -> (Value, Value) {+ let canonicalized = vault_codec::canonicalize_from_sqlite(CanonicalizeInput {+ tables: read_tables(conn),+ canonicalized_at: "2026-09-11T00:00:00.000Z".to_string(),+ manifests: vec![ManifestSpec { manifest_id: PERSONAL_MANIFEST_ID.to_string(), manifest_salt: salt.to_string(), name: None }],+ adopt_unstamped_into: None,+ })+ .unwrap();+ let manifest_json = serde_json::to_string(&canonicalized.manifests[0].manifest).unwrap();+ let blob = crypto::symmetric_encrypt_bytes(&vault_codec::pack_payload(&manifest_json).unwrap(), vek).unwrap();+ let ciphertext_hash = vault_codec::compute_ciphertext_hash(&blob);+ let mut buckets = Vec::new();+ for bucket in &canonicalized.data_buckets {+ let bucket_json = serde_json::to_string(bucket).unwrap();+ let bucket_blob = crypto::symmetric_encrypt_bytes(&vault_codec::pack_payload(&bucket_json).unwrap(), vek).unwrap();+ buckets.push(json!({ "manifestId": bucket.manifest_id, "category": bucket.category, "blob": bucket_blob, "ciphertextHash": vault_codec::compute_ciphertext_hash(&bucket_blob), "revision": revision }));+ }+ let status = json!({+ "clientVersionSupported": true,+ "serverVersion": "0.31.0",+ "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": revision }],+ "bucketRevisions": canonicalized.data_buckets.iter().map(|b| json!({ "manifestId": b.manifest_id, "category": b.category, "revision": revision })).collect::<Vec<_>>(),+ "personalManifestId": PERSONAL_MANIFEST_ID,+ "srpSalt": "salt",+ "capabilities": { "sharing": "on" },+ });+ let vault = json!({+ "status": 0,+ "storageFormat": 1,+ "personalManifestId": PERSONAL_MANIFEST_ID,+ "manifests": [{ "manifestId": PERSONAL_MANIFEST_ID, "blob": blob, "ciphertextHash": ciphertext_hash, "revision": revision, "blobReferences": [], "canAdminister": true, "keyType": "accountkey" }],+ "buckets": buckets,+ "emailRouting": { "privateEmailDomainList": ["private.io"], "publicEmailDomainList": [], "hiddenPrivateEmailDomainList": [], "emailAddressList": [] },+ });+ (status, vault)+}++fn item_names(conn: &rusqlite::Connection) -> Vec<String> {+ let mut names: Vec<String> = query(conn, "SELECT Name FROM Items WHERE IsDeleted = 0", &[]).unwrap().iter().map(|r| r["Name"].as_str().unwrap().to_string()).collect();+ names.sort();+ names+}++#[test]+fn fresh_client_pulls_and_materializes_the_server_vault() {+ let vek = crypto::generate_key_base64();+ let mut host = TestHost::new(&vek);+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));++ // The server holds a vault with one item; the client has an empty database and no baselines.+ let server_db = test_host::open_schema_db(&host.schema_sql);+ insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);+ let (status, vault) = snapshot_of(&server_db, &vek, 7, &vault_codec::generate_manifest_salt());+ host.respond("GET", "v2/Status", status);+ host.respond("GET", "v2/Vault", vault);++ let session = SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap();+ let result = host.drive(&session);++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["hasNewVault"], true);+ assert_eq!(result["vaultChanged"], true);+ assert_eq!(result["pulledRevision"], 7);+ assert_eq!(result["serverVersion"], "0.31.0");+ assert_eq!(result["manifestMigrationRequired"], false);+ assert_eq!(item_names(&host.local), vec!["Server item"]);+ assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 7);+ assert_eq!(host.state[state::VAULT_PERSONAL_MANIFEST_ID], PERSONAL_MANIFEST_ID);+ assert!(host.state[state::VAULT_CONTENT_FINGERPRINTS].as_object().unwrap().contains_key(&format!("manifest:{}", PERSONAL_MANIFEST_ID)));+ assert!(host.requests_to("v2/Vault").len() == 1);+}++#[test]+fn clean_client_in_sync_does_nothing() {+ let vek = crypto::generate_key_base64();+ let mut host = TestHost::new(&vek);+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));+ host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));+ host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));+ host.store_local_as_blob();+ host.respond("GET", "v2/Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 3 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));++ let session = SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap();+ let result = host.drive(&session);++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["hasNewVault"], false);+ assert!(host.requests_to("v2/Vault").is_empty());+ assert!(host.store_calls.is_empty());+}++#[test]+fn dirty_client_pushes_only_what_changed() {+ let vek = crypto::generate_key_base64();+ let salt = vault_codec::generate_manifest_salt();+ let mut host = TestHost::new(&vek);++ // Pull once to establish baselines, then mutate locally and push.+ let server_db = test_host::open_schema_db(&host.schema_sql);+ insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);+ let (status, vault) = snapshot_of(&server_db, &vek, 7, &salt);+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));+ host.respond("GET", "v2/Status", status.clone());+ host.respond("GET", "v2/Vault", vault);+ host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());++ insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000002", "Local item", PERSONAL_MANIFEST_ID);+ host.store_local_as_blob();+ host.mutation_sequence = 1;+ host.is_dirty = true;+ host.respond("POST", "v2/Vault/blobs/missing", json!({ "missing": [] }));+ host.respond("POST", "v2/Vault", json!({ "status": 0, "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 8 }], "bucketRevisions": [], "missingBlobHashes": [] }));++ let result = host.drive(&SyncSession::new(&request("fullSync", &vek, true, 1)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ let writes = host.requests_to("v2/Vault");+ let posts: Vec<_> = writes.iter().filter(|r| r.method == "POST").collect();+ assert_eq!(posts.len(), 1, "one write expected");+ let body = posts[0].body.as_ref().unwrap();+ assert_eq!(body["username"], USERNAME);+ assert_eq!(body["manifests"].as_array().unwrap().len(), 1, "only the personal manifest changed");+ assert_eq!(body["manifests"][0]["currentRevision"], 7);+ assert_eq!(body["manifests"][0]["credentialsCount"], 2);+ assert_eq!(body["buckets"].as_array().unwrap().len(), 0, "unchanged buckets stay out of the write");+ assert_eq!(body["emailRouting"]["coveredManifestIds"][0], PERSONAL_MANIFEST_ID);+ assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 8);+ assert_eq!(host.mark_clean_calls, vec![1]);+ assert!(!host.is_dirty);++ // The written manifest decrypts with the VEK and carries both items.+ let blob = body["manifests"][0]["manifestBlob"].as_str().unwrap();+ let plain = crypto::symmetric_decrypt_bytes(&base64::Engine::decode(&base64::engine::general_purpose::STANDARD, blob).unwrap(), &vek).unwrap();+ let manifest: Value = serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap();+ assert_eq!(manifest["tables"]["Items"].as_array().unwrap().len(), 2);+}++#[test]+fn no_op_mutation_clears_the_dirty_flag_without_a_write() {+ let vek = crypto::generate_key_base64();+ let mut host = TestHost::new(&vek);+ let server_db = test_host::open_schema_db(&host.schema_sql);+ insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);+ let (status, vault) = snapshot_of(&server_db, &vek, 7, &vault_codec::generate_manifest_salt());+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));+ host.respond("GET", "v2/Status", status);+ host.respond("GET", "v2/Vault", vault);+ host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());++ // Marked dirty, nothing actually changed.+ host.mutation_sequence = 1;+ host.is_dirty = true;+ let result = host.drive(&SyncSession::new(&request("fullSync", &vek, true, 1)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert!(host.requests.iter().all(|r| r.method != "POST"), "no write for a no-op mutation");+ assert_eq!(host.mark_clean_calls, vec![1]);+ assert!(!host.is_dirty);+}++#[test]+fn outdated_push_merges_the_server_change_and_retries() {+ let vek = crypto::generate_key_base64();+ let salt = vault_codec::generate_manifest_salt();+ let mut host = TestHost::new(&vek);+ let server_db = test_host::open_schema_db(&host.schema_sql);+ insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);+ let (status, vault) = snapshot_of(&server_db, &vek, 7, &salt);+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));+ host.respond("GET", "v2/Status", status);+ host.respond("GET", "v2/Vault", vault);+ host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());++ // Another client adds an item on the server (revision 8) while this one adds a different item locally.+ insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000003", "Other device item", PERSONAL_MANIFEST_ID);+ let (status8, vault8) = snapshot_of(&server_db, &vek, 8, &salt);+ insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000002", "Local item", PERSONAL_MANIFEST_ID);+ host.store_local_as_blob();+ host.mutation_sequence = 1;+ host.is_dirty = true;++ host.responders.clear();+ host.respond("GET", "v2/Status", status8);+ host.respond("GET", "v2/Vault", vault8);+ host.respond("POST", "v2/Vault/blobs/missing", json!({ "missing": [] }));+ host.respond_with(Box::new(|method, path, body| {+ if method != "POST" || path != "v2/Vault" {+ return None;+ }+ let current = body.and_then(|b| b["manifests"][0]["currentRevision"].as_i64()).unwrap_or(-1);+ Some(if current == 8 {+ (200, json!({ "status": 0, "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 9 }], "bucketRevisions": [], "missingBlobHashes": [] }))+ } else {+ (200, json!({ "status": 2, "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 8 }], "bucketRevisions": [], "missingBlobHashes": [] }))+ })+ }));++ let result = host.drive(&SyncSession::new(&request("fullSync", &vek, true, 1)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["hasNewVault"], true);+ assert_eq!(item_names(&host.local), vec!["Local item", "Other device item", "Server item"]);+ assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 9);+ let posts: Vec<_> = host.requests_to("v2/Vault").into_iter().filter(|r| r.method == "POST").collect();+ assert_eq!(posts.last().unwrap().body.as_ref().unwrap()["manifests"][0]["currentRevision"], 8);+ let merged: Value = {+ let blob = posts.last().unwrap().body.as_ref().unwrap()["manifests"][0]["manifestBlob"].as_str().unwrap();+ let plain = crypto::symmetric_decrypt_bytes(&base64::Engine::decode(&base64::engine::general_purpose::STANDARD, blob).unwrap(), &vek).unwrap();+ serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap()+ };+ assert_eq!(merged["tables"]["Items"].as_array().unwrap().len(), 3);+}++#[test]+fn unreachable_server_with_a_local_vault_goes_offline() {+ let vek = crypto::generate_key_base64();+ let mut host = TestHost::new(&vek);+ host.store_local_as_blob();++ let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["wasOffline"], true);+ assert_eq!(result["isOfflineMode"], true);+}++#[test]+fn expired_session_requires_logout() {+ let vek = crypto::generate_key_base64();+ let mut host = TestHost::new(&vek);+ host.respond_with(Box::new(|_, path, _| if path == "v2/Status" { Some((401, json!({}))) } else { None }));++ let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());++ assert_eq!(result["success"], false);+ assert_eq!(result["requiresLogout"], true);+ assert_eq!(result["errorKey"], "sessionExpired");+}++#[test]+fn password_changed_elsewhere_requires_logout() {+ let vek = crypto::generate_key_base64();+ let mut host = TestHost::new(&vek);+ host.state.insert(state::ENCRYPTION_KEY_DERIVATION_PARAMS.to_string(), json!({ "salt": "old-salt", "encryptionType": "Argon2Id", "encryptionSettings": "{}" }));+ host.respond("GET", "v2/Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "new-salt" }));++ let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());++ assert_eq!(result["requiresLogout"], true);+ assert_eq!(result["errorKey"], "passwordChanged");+}++#[test]+fn legacy_account_without_vault_key_reports_the_manifest_migration() {+ let kek = crypto::generate_key_base64();+ let mut host = TestHost::new(&kek);+ host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));+ host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));+ host.store_local_as_blob();+ host.respond("GET", "v2/Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 3 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));++ let result = host.drive(&SyncSession::new(&request("fullSync", &kek, false, 0)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["manifestMigrationRequired"], true);++ let status = host.drive(&SyncSession::new(&request("migrationStatus", &kek, false, 0)).unwrap());+ assert_eq!(status["kind"], "storage-format-upgrade");+}++#[test]+fn manifest_migration_generates_the_key_hierarchy_and_pushes() {+ let kek = crypto::generate_key_base64();+ let salt = vault_codec::generate_manifest_salt();+ let mut host = TestHost::new(&kek);+ insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000001", "Old item", PERSONAL_MANIFEST_ID);+ host.store_local_as_blob();+ host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));+ host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));+ host.state.insert(state::VAULT_MANIFEST_SALT.to_string(), json!(salt));+ host.respond_with(Box::new(|method, path, _| if method == "GET" && path == "v2/VaultKey/Password" { Some((200, json!({ "vaultKey": null }))) } else { None }));+ host.respond("POST", "v2/Vault/blobs/missing", json!({ "missing": [] }));+ host.respond("POST", "v2/Vault", json!({ "status": 0, "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "bucketRevisions": [], "missingBlobHashes": [] }));++ let result = host.drive(&SyncSession::new(&request("migrateManifest", &kek, false, 0)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["pushed"], true);+ let new_key = result["sessionUpdates"]["encryptionKey"].as_str().expect("the session adopts the new VEK");+ assert_ne!(new_key, kek);+ let posts: Vec<_> = host.requests_to("v2/Vault").into_iter().filter(|r| r.method == "POST").collect();+ let body = posts[0].body.as_ref().unwrap();+ assert!(body["accountKeys"]["encryptedAccountKey"].is_string(), "the migration push carries the key hierarchy");+ let (vek, _) = crypto::resolve_vault_encryption_key(body["accountKeys"]["encryptedAccountKey"].as_str().unwrap(), body["accountKeys"]["encryptedVek"].as_str().unwrap(), &kek).unwrap();+ assert_eq!(*vek, new_key);+ assert!(host.state.contains_key(state::ENCRYPTED_ACCOUNT_KEY));+ assert!(result["sessionUpdates"]["accountPrivateKey"].is_string());+}++/// The latest EF migration stamp of a database.+fn latest_migration_id(conn: &rusqlite::Connection) -> String {+ query(conn, "SELECT MigrationId FROM __EFMigrationsHistory ORDER BY MigrationId DESC LIMIT 1", &[]).unwrap()[0]["MigrationId"].as_str().unwrap().to_string()+}++#[test]+fn schema_rebuild_of_a_stale_vault_pushes_without_touching_the_key_hierarchy() {+ let vek = crypto::generate_key_base64();+ let mut host = TestHost::new(&vek);+ let current_stamp = latest_migration_id(&host.local);+ insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000001", "Kept item", PERSONAL_MANIFEST_ID);+ // A vault written by an older client: same tables, older schema stamp (still past the frozen sqlite-blob chain).+ host.local.execute_batch("DELETE FROM __EFMigrationsHistory; INSERT INTO __EFMigrationsHistory (MigrationId, ProductVersion) VALUES ('20250101000000_2.0.0-Stale', '9.0.0');").unwrap();+ host.store_local_as_blob();+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));+ host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));+ host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));+ host.state.insert(state::VAULT_MANIFEST_SALT.to_string(), json!(vault_codec::generate_manifest_salt()));+ host.respond("POST", "v2/Vault/blobs/missing", json!({ "missing": [] }));+ host.respond("POST", "v2/Vault", json!({ "status": 0, "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "bucketRevisions": [], "missingBlobHashes": [] }));++ let status = host.drive(&SyncSession::new(&request("migrationStatus", &vek, false, 0)).unwrap());+ assert_eq!(status["kind"], "schema-rebuild");++ let result = host.drive(&SyncSession::new(&request("migrateManifest", &vek, false, 0)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["pushed"], true);+ assert_eq!(latest_migration_id(&host.local), current_stamp, "the local vault is rebuilt onto the current schema");+ assert_eq!(item_names(&host.local), vec!["Kept item"]);+ assert!(host.requests_to("v2/VaultKey/Password").is_empty(), "a migrated account is not probed for a key hierarchy");+ let posts: Vec<_> = host.requests_to("v2/Vault").into_iter().filter(|r| r.method == "POST").collect();+ assert!(posts[0].body.as_ref().unwrap()["accountKeys"].is_null(), "no key hierarchy is minted");+ assert!(result["sessionUpdates"]["encryptionKey"].is_null(), "the session key stays the VEK");+}++/// A legacy sqlite-blob snapshot of the given database, as the server serves an account that has not migrated.+fn legacy_snapshot_of(conn: &rusqlite::Connection, kek: &str, revision: i64) -> Value {+ let bytes = conn.serialize(rusqlite::DatabaseName::Main).unwrap().to_vec();+ json!({+ "status": 0,+ "storageFormat": 0,+ "legacyVaultBlob": crypto::symmetric_encrypt_bytes(&bytes, kek).unwrap(),+ "legacyRevision": revision,+ "personalManifestId": PERSONAL_MANIFEST_ID,+ "emailRouting": { "privateEmailDomainList": ["private.io"], "publicEmailDomainList": [], "hiddenPrivateEmailDomainList": [], "emailAddressList": [] },+ })+}++/// A host as a client predating the manifest storage format leaves it: a sqlite blob under the KEK, no personal+/// manifest id and no revision baseline, against a server that still holds the account as a legacy vault.+fn pre_format_session_host(kek: &str, local_item: &str, server_item: &str) -> TestHost {+ let mut host = TestHost::new(kek);+ insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000001", local_item, PERSONAL_MANIFEST_ID);+ host.store_local_as_blob();+ let server_db = test_host::open_schema_db(&host.schema_sql);+ insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000002", server_item, PERSONAL_MANIFEST_ID);+ host.respond("GET", "v2/Vault", legacy_snapshot_of(&server_db, kek, 3));+ host.respond_with(Box::new(|method, path, _| if method == "GET" && path == "v2/VaultKey/Password" { Some((200, json!({ "vaultKey": null }))) } else { None }));+ host.respond("POST", "v2/Vault/blobs/missing", json!({ "missing": [] }));+ host.respond("POST", "v2/Vault", json!({ "status": 0, "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "bucketRevisions": [], "missingBlobHashes": [] }));+ host+}++#[test]+fn manifest_migration_of_a_pre_format_session_pulls_the_server_vault_first() {+ let kek = crypto::generate_key_base64();+ let mut host = pre_format_session_host(&kek, "Local item", "Server item");++ let result = host.drive(&SyncSession::new(&request("migrateManifest", &kek, false, 0)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["pushed"], true);+ assert_eq!(host.requests_to("v2/Vault").iter().filter(|r| r.method == "GET").count(), 1, "the baseline is pulled once");+ assert_eq!(host.state[state::VAULT_PERSONAL_MANIFEST_ID], PERSONAL_MANIFEST_ID);+ assert_eq!(item_names(&host.local), vec!["Server item"], "a clean local vault is replaced by the server's, like a login does");+ let posts: Vec<_> = host.requests_to("v2/Vault").into_iter().filter(|r| r.method == "POST").collect();+ let body = posts[0].body.as_ref().unwrap();+ assert_eq!(body["manifests"][0]["currentRevision"], 3, "the migration push names the revision the server holds");+ assert!(body["accountKeys"]["encryptedAccountKey"].is_string());+ assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 4);+}++#[test]+fn manifest_migration_of_a_dirty_pre_format_session_keeps_the_local_vault() {+ let kek = crypto::generate_key_base64();+ let mut host = pre_format_session_host(&kek, "Local item", "Server item");+ host.is_dirty = true;++ let result = host.drive(&SyncSession::new(&request("migrateManifest", &kek, true, 1)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["pushed"], true);+ assert_eq!(host.state[state::VAULT_PERSONAL_MANIFEST_ID], PERSONAL_MANIFEST_ID);+ assert_eq!(item_names(&host.local), vec!["Local item"], "pending local changes are not thrown away");+ let posts: Vec<_> = host.requests_to("v2/Vault").into_iter().filter(|r| r.method == "POST").collect();+ let body = posts[0].body.as_ref().unwrap();+ assert_eq!(body["manifests"][0]["currentRevision"], 3, "the baseline still comes from the server");+ let manifest_json = crypto::symmetric_decrypt_bytes(&crate::encoding::base64_decode(body["manifests"][0]["manifestBlob"].as_str().unwrap()).unwrap(), result["sessionUpdates"]["encryptionKey"].as_str().unwrap()).unwrap();+ assert!(vault_codec::unpack_payload(&manifest_json).unwrap().contains("Local item"), "the push carries the local changes");+}++#[test]+fn session_reports_when_a_response_is_missing() {+ let session = SyncSession::new(&request("fullSync", "key", false, 0)).unwrap();+ let first: Value = serde_json::from_str(&session.next_command().unwrap()).unwrap();+ assert_eq!(first["kind"], "log", "the engine announces the sync before touching the host");+ assert!(session.next_command().is_err(), "the session refuses to advance without a response");+ session.resume("{}").unwrap();+ let second: Value = serde_json::from_str(&session.next_command().unwrap()).unwrap();+ assert_eq!(second["kind"], "http");+ assert_eq!(second["path"], "v2/Status");+}++#[test]+fn state_helpers_key_buckets_and_fingerprints_like_the_client() {+ assert_eq!(state::bucket_revision_key("m", "Settings"), "m:Settings");+ assert_eq!(state::fingerprint_bucket_key("m", "Stats"), "bucket:m:Stats");+ assert_eq!(state::fingerprint_manifest_key("m"), "manifest:m");+ let _unused: HashMap<String, String> = HashMap::new();+}++#[test]+fn status_check_reports_newer_server_state_without_touching_the_vault() {+ let vek = crypto::generate_key_base64();+ let mut host = TestHost::new(&vek);+ host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));+ host.respond("GET", "v2/Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));++ let result = host.drive(&SyncSession::new(&request("statusCheck", &vek, true, 2)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["hasNewerVault"], true);+ assert_eq!(result["hasDirtyChanges"], true);+ assert_eq!(result["isOffline"], false);+ assert!(host.store_calls.is_empty());+ assert!(host.requests.iter().all(|r| r.path == "v2/Status"));+}++/// The `GET v2/VaultKey/Password` answer for a hierarchy the server holds.+fn vault_key_body(hierarchy: &crypto::AccountKeyHierarchy) -> Value {+ let blobs = &hierarchy.account_keys;+ json!({ "vaultKey": { "type": "password", "encryptedAccountKey": blobs.encrypted_account_key, "encryptedVek": blobs.encrypted_vek, "accountPublicKey": blobs.account_public_key, "encryptedAccountPrivateKey": blobs.encrypted_account_private_key, "salt": "salt", "encryptionType": "Argon2Id", "encryptionSettings": "{}" } })+}++/// The cross-device race: this device logged in while the account was legacy (no cached chain, KEK session), and+/// another device created the hierarchy since. The pull adopts it, and the stored vault reaches the host+/// together with the VEK it is now encrypted under.+#[test]+fn a_hierarchy_created_on_another_device_is_adopted_on_the_next_pull() {+ let kek = crypto::generate_key_base64();+ let hierarchy = crypto::create_account_key_hierarchy(&kek).unwrap();+ let vek = hierarchy.vault_encryption_key.clone();+ let mut host = TestHost::new(&kek);++ let server_db = test_host::open_schema_db(&host.schema_sql);+ insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);+ let (status, vault) = snapshot_of(&server_db, &vek, 7, &vault_codec::generate_manifest_salt());+ host.respond("GET", "v2/Status", status);+ host.respond("GET", "v2/VaultKey/Password", vault_key_body(&hierarchy));+ host.respond("GET", "v2/Vault", vault);++ let result = host.drive(&SyncSession::new(&request("fullSync", &kek, false, 0)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["sessionUpdates"]["encryptionKey"], vek);+ assert_eq!(result["sessionUpdates"]["accountPrivateKey"], hierarchy.account_private_key);+ assert_eq!(host.vault_key, vek, "the store carried the VEK, so the host adopted it before opening the blob");+ assert!(matches!(&host.store_calls[0], Command::VaultStore { encryption_key: Some(key), .. } if *key == vek));+ assert_eq!(item_names(&host.local), vec!["Server item"]);+ assert_eq!(host.state[state::ENCRYPTED_ACCOUNT_KEY], hierarchy.account_keys.encrypted_account_key);+ assert_eq!(host.state[state::ACCOUNT_PUBLIC_KEY], hierarchy.account_keys.account_public_key);+}++/// Login: the host hands the password-derived key to `resolveVaultKey`; the server's chain opens with it, is+/// cached for offline unlock, and the VEK comes back as the key to store.+#[test]+fn resolve_vault_key_opens_the_chain_from_the_server() {+ let kek = crypto::generate_key_base64();+ let hierarchy = crypto::create_account_key_hierarchy(&kek).unwrap();+ let mut host = TestHost::new(&kek);+ host.respond("GET", "v2/VaultKey/Password", vault_key_body(&hierarchy));++ let result = host.drive(&SyncSession::new(&request("resolveVaultKey", &kek, false, 0)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["hasVaultKey"], true);+ assert_eq!(result["encryptionKey"], hierarchy.vault_encryption_key);+ assert_eq!(result["sessionUpdates"]["encryptionKey"], hierarchy.vault_encryption_key);+ assert_eq!(result["sessionUpdates"]["accountPrivateKey"], hierarchy.account_private_key);+ assert_eq!(host.state[state::ENCRYPTED_ACCOUNT_KEY], hierarchy.account_keys.encrypted_account_key);+ assert!(host.store_calls.is_empty(), "resolving a key never touches the stored vault");+}++/// A legacy account has no chain: the password-derived key is the vault key and any stale cached chain is dropped.+#[test]+fn resolve_vault_key_keeps_the_kek_for_a_legacy_account() {+ let kek = crypto::generate_key_base64();+ let mut host = TestHost::new(&kek);+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("stale"));+ host.respond("GET", "v2/VaultKey/Password", json!({ "vaultKey": null }));++ let result = host.drive(&SyncSession::new(&request("resolveVaultKey", &kek, false, 0)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["hasVaultKey"], false);+ assert_eq!(result["encryptionKey"], kek);+ assert!(result["sessionUpdates"].get("encryptionKey").is_none());+ assert!(!host.state.contains_key(state::ENCRYPTED_ACCOUNT_KEY));+}++/// Offline, the cached chain opens with the password-derived key (a re-login after a forced logout).+#[test]+fn resolve_vault_key_opens_the_cached_chain_when_the_server_is_unreachable() {+ let kek = crypto::generate_key_base64();+ let hierarchy = crypto::create_account_key_hierarchy(&kek).unwrap();+ let mut host = TestHost::new(&kek);+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!(hierarchy.account_keys.encrypted_account_key));+ host.state.insert(state::ENCRYPTED_VEK.to_string(), json!(hierarchy.account_keys.encrypted_vek));++ let result = host.drive(&SyncSession::new(&request("resolveVaultKey", &kek, false, 0)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(result["hasVaultKey"], true);+ assert_eq!(result["encryptionKey"], hierarchy.vault_encryption_key);+}++/// A key that does not open the chain is reported as a decrypt failure, never silently kept.+#[test]+fn resolve_vault_key_refuses_a_key_that_does_not_open_the_chain() {+ let hierarchy = crypto::create_account_key_hierarchy(&crypto::generate_key_base64()).unwrap();+ let wrong_kek = crypto::generate_key_base64();+ let mut host = TestHost::new(&wrong_kek);+ host.respond("GET", "v2/VaultKey/Password", vault_key_body(&hierarchy));++ let result = host.drive(&SyncSession::new(&request("resolveVaultKey", &wrong_kek, false, 0)).unwrap());++ assert_eq!(result["success"], false);+ assert_eq!(result["errorCode"], "E-203");+ assert!(result.get("encryptionKey").is_none());+}++/// The sync trusts the key it is given: a KEK handed to a device that caches the chain is not upgraded any more,+/// so the run fails to open the vault instead of silently swapping keys. Resolving the key is the host's job.+#[test]+fn a_kek_session_on_a_migrated_device_is_not_upgraded_by_the_sync() {+ let kek = crypto::generate_key_base64();+ let hierarchy = crypto::create_account_key_hierarchy(&kek).unwrap();+ let vek = hierarchy.vault_encryption_key.clone();+ let mut host = TestHost::new(&vek);+ host.store_local_as_blob();+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!(hierarchy.account_keys.encrypted_account_key));+ host.state.insert(state::ENCRYPTED_VEK.to_string(), json!(hierarchy.account_keys.encrypted_vek));+ host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));+ host.respond("GET", "v2/Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));++ let result = host.drive(&SyncSession::new(&request("fullSync", &kek, false, 0)).unwrap());++ assert!(result["sessionUpdates"].get("encryptionKey").is_none());+ assert!(host.requests_to("v2/VaultKey/Password").is_empty(), "a device with a cached chain is never probed");+}++/// The VEK itself opens nothing in the chain and stands as the session key.+#[test]+fn a_vek_session_key_is_left_alone() {+ let kek = crypto::generate_key_base64();+ let hierarchy = crypto::create_account_key_hierarchy(&kek).unwrap();+ let vek = hierarchy.vault_encryption_key.clone();+ let mut host = TestHost::new(&vek);+ host.store_local_as_blob();+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!(hierarchy.account_keys.encrypted_account_key));+ host.state.insert(state::ENCRYPTED_VEK.to_string(), json!(hierarchy.account_keys.encrypted_vek));+ host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));+ host.respond("GET", "v2/Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 3 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));++ let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert!(result["sessionUpdates"].get("encryptionKey").is_none());+ assert!(host.store_calls.is_empty());+}++/// A legacy account that is clean and in sync is not probed for a vault key: the probe belongs to a pull or a push.+#[test]+fn clean_legacy_account_in_sync_is_not_probed_for_a_vault_key() {+ let kek = crypto::generate_key_base64();+ let mut host = TestHost::new(&kek);+ host.store_local_as_blob();+ host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));+ host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));+ host.respond("GET", "v2/Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 3 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));++ let result = host.drive(&SyncSession::new(&request("fullSync", &kek, false, 0)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert!(host.requests_to("v2/VaultKey/Password").is_empty());+}++/// A dirty legacy account probes the server for a vault key once per run, not once per step.+#[test]+fn dirty_legacy_account_probes_for_a_vault_key_once_per_run() {+ let kek = crypto::generate_key_base64();+ let salt = vault_codec::generate_manifest_salt();+ let mut host = TestHost::new(&kek);+ insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000001", "Local item", PERSONAL_MANIFEST_ID);+ host.store_local_as_blob();+ host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));+ host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));+ host.state.insert(state::VAULT_MANIFEST_SALT.to_string(), json!(salt));+ host.respond("GET", "v2/Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 3 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));+ host.respond("GET", "v2/VaultKey/Password", json!({ "vaultKey": null }));+ host.respond("POST", "v2/Vault/blobs/missing", json!({ "missing": [] }));+ host.respond("POST", "v2/Vault", json!({ "status": 0, "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "bucketRevisions": [], "missingBlobHashes": [] }));++ let result = host.drive(&SyncSession::new(&request("fullSync", &kek, true, 1)).unwrap());++ assert_eq!(result["success"], true, "{}", result);+ assert_eq!(host.requests_to("v2/VaultKey/Password").len(), 1);+}diff --git a/core/rust/src/vault_sync/tests/test_host.rs b/core/rust/src/vault_sync/tests/test_host.rsnew file mode 100644index 000000000..15625f3c0--- /dev/null+++ b/core/rust/src/vault_sync/tests/test_host.rs@@ -0,0 +1,263 @@+//! A reference host for the engine tests: real SQLite (rusqlite), in-memory state, scripted HTTP.++use std::collections::HashMap;++use base64::engine::general_purpose::STANDARD as BASE64;+use base64::Engine;+use rusqlite::types::{Value as SqlValue, ValueRef};+use rusqlite::{params_from_iter, Connection, DatabaseName};+use serde_json::{json, Map, Value};++use crate::crypto;+use crate::vault_sync::types::{Command, Db, SqlStatement};+use crate::vault_sync::session::SyncSession;++/// One recorded HTTP request.+#[derive(Debug, Clone)]+pub struct RecordedRequest {+ pub method: String,+ pub path: String,+ pub body: Option<Value>,+}++/// A scripted HTTP answer: status and JSON body for a (method, path) match.+pub type Responder = Box<dyn Fn(&str, &str, Option<&Value>) -> Option<(u16, Value)>>;++pub struct TestHost {+ pub local: Connection,+ pub staging: Option<Connection>,+ pub state: HashMap<String, Value>,+ pub vault_blob: Option<String>,+ pub vault_key: String,+ pub mutation_sequence: u64,+ pub is_dirty: bool,+ pub mark_clean_calls: Vec<u64>,+ pub store_calls: Vec<Command>,+ pub requests: Vec<RecordedRequest>,+ pub responders: Vec<Responder>,+ pub logs: Vec<String>,+ pub schema_sql: String,+}++/// The current client schema, read from the TypeScript source of truth.+pub fn complete_schema_sql() -> String {+ let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../vault/src/sql/SqlConstants.ts");+ let source = std::fs::read_to_string(path).expect("core/vault/src/sql/SqlConstants.ts");+ let start = source.find("COMPLETE_SCHEMA_SQL = `").expect("COMPLETE_SCHEMA_SQL") + "COMPLETE_SCHEMA_SQL = `".len();+ let end = source[start..].find("`;").expect("end of template literal") + start;+ source[start..end].replace('\u{feff}', "")+}++/// Open a database from its file bytes (through a scratch file, since rusqlite's deserialize wants sqlite-owned memory).+pub fn open_from_bytes(bytes: &[u8]) -> Connection {+ static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+ let unique = COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ let path = std::env::temp_dir().join(format!("aliasvault-sync-test-{}-{}.sqlite", std::process::id(), unique));+ std::fs::write(&path, bytes).unwrap();+ let file = Connection::open(&path).unwrap();+ let mut conn = Connection::open_in_memory().unwrap();+ {+ let backup = rusqlite::backup::Backup::new(&file, &mut conn).unwrap();+ backup.run_to_completion(5, std::time::Duration::from_millis(1), None).unwrap();+ }+ drop(file);+ let _ = std::fs::remove_file(&path);+ conn+}++pub fn open_schema_db(schema_sql: &str) -> Connection {+ let conn = Connection::open_in_memory().unwrap();+ conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();+ conn.execute_batch(schema_sql).unwrap();+ conn+}++impl TestHost {+ pub fn new(vault_key: &str) -> Self {+ let schema_sql = complete_schema_sql();+ Self {+ local: open_schema_db(&schema_sql),+ staging: None,+ state: HashMap::new(),+ vault_blob: None,+ vault_key: vault_key.to_string(),+ mutation_sequence: 0,+ is_dirty: false,+ mark_clean_calls: Vec::new(),+ store_calls: Vec::new(),+ requests: Vec::new(),+ responders: Vec::new(),+ logs: Vec::new(),+ schema_sql,+ }+ }++ /// Answer requests whose method and path match.+ pub fn respond(&mut self, method: &'static str, path: &'static str, body: Value) {+ self.responders.push(Box::new(move |m, p, _| if m == method && p == path { Some((200, body.clone())) } else { None }));+ }++ pub fn respond_with(&mut self, responder: Responder) {+ self.responders.push(responder);+ }++ /// Encrypt the local database as the at-rest blob, the way the app stores it.+ pub fn store_local_as_blob(&mut self) {+ let bytes = self.local.serialize(DatabaseName::Main).unwrap().to_vec();+ self.vault_blob = Some(crypto::symmetric_encrypt_bytes(&bytes, &self.vault_key).unwrap());+ }++ /// Drive a session to completion and return the `done` result.+ pub fn drive(&mut self, session: &SyncSession) -> Value {+ loop {+ let command: Command = serde_json::from_str(&session.next_command().unwrap()).unwrap();+ let response = match command {+ Command::Done { result } => return result,+ Command::Http { method, path, body, .. } => {+ let body_json = body.as_deref().map(|b| serde_json::from_str(b).unwrap_or(Value::String(b.to_string())));+ let method = serde_json::to_value(method).unwrap().as_str().expect("http method serializes as text").to_string();+ self.requests.push(RecordedRequest { method: method.clone(), path: path.clone(), body: body_json.clone() });+ match self.responders.iter().find_map(|r| r(&method, &path, body_json.as_ref())) {+ Some((status, body)) => json!({ "status": status, "body": body.to_string() }),+ None => json!({ "status": 0, "transportError": format!("no scripted response for {} {}", method, path) }),+ }+ }+ Command::StateGet { key } => json!({ "value": self.state.get(&key).cloned().unwrap_or(Value::Null) }),+ Command::StateSet { key, value } => {+ self.state.insert(key, value);+ json!({})+ }+ Command::StateRemove { key } => {+ self.state.remove(&key);+ json!({})+ }+ Command::DbOpen { bytes, .. } => {+ self.staging = Some(match bytes {+ None => open_schema_db(&self.schema_sql),+ Some(b64) => {+ let conn = open_from_bytes(&BASE64.decode(b64).unwrap());+ conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();+ conn+ }+ });+ json!({})+ }+ Command::DbQuery { db, .. } | Command::DbExec { db, .. } | Command::DbExport { db } if db == Db::Local && self.vault_blob.is_none() => {+ // Like the app hosts: without a stored vault there is no local database to open.+ json!({ "error": "Vault not available" })+ }+ Command::DbQuery { db, sql, params } => match query(self.db(db), &sql, ¶ms) {+ Ok(rows) => json!({ "rows": rows }),+ Err(error) => json!({ "error": format!("{} ({})", error, sql) }),+ },+ Command::DbExec { db, statements } => match exec(self.db(db), &statements) {+ Ok(()) => json!({}),+ Err(error) => json!({ "error": error }),+ },+ Command::DbExport { db } => json!({ "bytes": BASE64.encode(self.db(db).serialize(DatabaseName::Main).unwrap().to_vec()) }),+ Command::VaultStore { encrypted_blob, mark_dirty, encryption_key, expected_mutation_seq, revision } => {+ self.store_calls.push(Command::VaultStore { encrypted_blob: String::new(), mark_dirty, encryption_key: encryption_key.clone(), expected_mutation_seq, revision });+ if let Some(key) = encryption_key {+ self.vault_key = key;+ }+ if let Some(expected) = expected_mutation_seq {+ if expected != self.mutation_sequence {+ json!({ "success": false, "mutationSequence": self.mutation_sequence })+ } else {+ self.adopt_blob(&encrypted_blob);+ json!({ "success": true, "mutationSequence": self.mutation_sequence })+ }+ } else {+ if mark_dirty {+ self.mutation_sequence += 1;+ self.is_dirty = true;+ }+ self.adopt_blob(&encrypted_blob);+ json!({ "success": true, "mutationSequence": self.mutation_sequence })+ }+ }+ Command::VaultLoad => json!({ "encryptedBlob": self.vault_blob.clone() }),+ Command::MarkClean { mutation_seq_at_start } => {+ self.mark_clean_calls.push(mutation_seq_at_start);+ let cleared = mutation_seq_at_start == self.mutation_sequence;+ if cleared {+ self.is_dirty = false;+ }+ json!({ "cleared": cleared })+ }+ Command::Log { level, message } => {+ self.logs.push(format!("[{:?}] {}", level, message));+ json!({})+ }+ };+ session.resume(&response.to_string()).unwrap();+ }+ }++ /// Store a blob and reload `local` from it, as the contract requires.+ fn adopt_blob(&mut self, encrypted_blob: &str) {+ self.vault_blob = Some(encrypted_blob.to_string());+ let bytes = crypto::symmetric_decrypt_bytes(&BASE64.decode(encrypted_blob).unwrap(), &self.vault_key).expect("stored blob decrypts with the vault key");+ self.local = open_from_bytes(&bytes);+ }++ fn db(&mut self, db: Db) -> &Connection {+ match db {+ Db::Local => &self.local,+ Db::Staging => self.staging.as_ref().expect("staging database opened"),+ }+ }++ pub fn requests_to(&self, path: &str) -> Vec<&RecordedRequest> {+ self.requests.iter().filter(|r| r.path == path).collect()+ }+}++fn to_sql(value: &Value) -> SqlValue {+ match value {+ Value::Null => SqlValue::Null,+ Value::Bool(b) => SqlValue::Integer(*b as i64),+ Value::Number(n) => n.as_i64().map(SqlValue::Integer).unwrap_or_else(|| SqlValue::Real(n.as_f64().unwrap_or(0.0))),+ Value::String(s) => SqlValue::Text(s.clone()),+ Value::Object(o) if o.contains_key("__b64") => SqlValue::Blob(BASE64.decode(o["__b64"].as_str().unwrap_or("")).unwrap_or_default()),+ other => SqlValue::Text(other.to_string()),+ }+}++fn from_sql(value: ValueRef<'_>) -> Value {+ match value {+ ValueRef::Null => Value::Null,+ ValueRef::Integer(i) => json!(i),+ ValueRef::Real(f) => json!(f),+ ValueRef::Text(t) => Value::String(String::from_utf8_lossy(t).to_string()),+ ValueRef::Blob(b) => json!({ "__b64": BASE64.encode(b) }),+ }+}++pub fn query(conn: &Connection, sql: &str, params: &[Value]) -> Result<Vec<Map<String, Value>>, String> {+ let mut statement = conn.prepare(sql).map_err(|e| e.to_string())?;+ let columns: Vec<String> = statement.column_names().iter().map(|c| c.to_string()).collect();+ let values: Vec<SqlValue> = params.iter().map(to_sql).collect();+ let mut rows = statement.query(params_from_iter(values.iter())).map_err(|e| e.to_string())?;+ let mut out = Vec::new();+ while let Some(row) = rows.next().map_err(|e| e.to_string())? {+ let mut object = Map::new();+ for (index, column) in columns.iter().enumerate() {+ object.insert(column.clone(), from_sql(row.get_ref(index).map_err(|e| e.to_string())?));+ }+ out.push(object);+ }+ Ok(out)+}++pub fn exec(conn: &Connection, statements: &[SqlStatement]) -> Result<(), String> {+ conn.execute_batch("BEGIN").map_err(|e| e.to_string())?;+ for statement in statements {+ let values: Vec<SqlValue> = statement.params.iter().map(to_sql).collect();+ if let Err(error) = conn.execute(&statement.sql, params_from_iter(values.iter())) {+ let _ = conn.execute_batch("ROLLBACK");+ return Err(format!("{} ({})", error, statement.sql));+ }+ }+ conn.execute_batch("COMMIT").map_err(|e| e.to_string())+}