Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Refactor Rust core and remove overlapping tests

Leendert de Borstfe71d775c425
diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/nativevaultmanager/RustCoreDispatcher.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/nativevaultmanager/RustCoreDispatcher.kt
index a0077fe2a..8cb168465 100644
--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/nativevaultmanager/RustCoreDispatcher.kt
+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/nativevaultmanager/RustCoreDispatcher.kt
@@ -25,7 +25,6 @@ import uniffi.aliasvault_core.isRelatedOriginAllowed
import uniffi.aliasvault_core.isRpIdAllowedForHost
import uniffi.aliasvault_core.openAccountKeyChain
import uniffi.aliasvault_core.parseEmailSource
-import uniffi.aliasvault_core.pruneVaultJson
import uniffi.aliasvault_core.reencryptAccountKeyJson
import uniffi.aliasvault_core.selectFaviconTarget
import uniffi.aliasvault_core.srpDerivePrivateKey
@@ -101,7 +100,6 @@ object RustCoreDispatcher {
"srpVerifySession" -> srpVerifySession(args.string(0), args.string(1), args.string(2), args.string(3)).toString()
"getSyncableTableNames" -> json(getSyncableTableNames())
- "pruneVaultJson" -> pruneVaultJson(args.string(0))
"vaultCodecCanonicalizeFromSqlite" -> vaultCodecCanonicalizeFromSqlite(args.string(0))
"vaultCodecGenerateManifestSalt" -> json(vaultCodecGenerateManifestSalt())
diff --git a/apps/mobile-app/ios/NativeVaultManager/VaultManager.swift b/apps/mobile-app/ios/NativeVaultManager/VaultManager.swift
index 9b864a51f..50458405f 100644
--- a/apps/mobile-app/ios/NativeVaultManager/VaultManager.swift
+++ b/apps/mobile-app/ios/NativeVaultManager/VaultManager.swift
@@ -1619,7 +1619,6 @@ private enum RustCoreDispatcher {
return try json(try RustCoreFramework.srpVerifySession(clientPublic: try args.string(0), clientProof: try args.string(1), sessionKey: try args.string(2), serverProof: try args.string(3)))
case "getSyncableTableNames": return try json(RustCoreFramework.getSyncableTableNames())
- case "pruneVaultJson": return try RustCoreFramework.pruneVaultJson(inputJson: try args.string(0))
case "vaultCodecCanonicalizeFromSqlite": return try RustCoreFramework.vaultCodecCanonicalizeFromSqlite(inputJson: try args.string(0))
case "vaultCodecGenerateManifestSalt": return try json(RustCoreFramework.vaultCodecGenerateManifestSalt())
diff --git a/core/models/scripts/lib/vault-table-registry-emit.cjs b/core/models/scripts/lib/vault-table-registry-emit.cjs
index b900ee246..8f5a40841 100644
--- a/core/models/scripts/lib/vault-table-registry-emit.cjs
+++ b/core/models/scripts/lib/vault-table-registry-emit.cjs
@@ -168,7 +168,7 @@ pub static SINGLE_VALUE_FIELD_KEYS: &[&str] = ${rustStrSlice(registry.SINGLE_VAL
/// stamped with that manifest's id (\`ManifestId\`).
pub const ENCRYPTION_KEYS_TABLE: &str = "${registry.ENCRYPTION_KEYS_TABLE}";
-/// The scope column every stamped table carries: the id of the manifest that owns the row.
+/// The column every stamped table carries: the id of the manifest that owns the row.
pub const MANIFEST_ID_COL: &str = "${registry.VAULT_MANIFEST_ID_COLUMN}";
/// Local bookkeeping table materialize writes into the vault DB: one row per manifest this
@@ -185,7 +185,7 @@ pub const OVERFLOW_TABLE: &str = "${registry.CODEC_OVERFLOW_TABLE}";
pub const OVERFLOW_ROW_ID: &str = "${registry.CODEC_OVERFLOW_ROW_ID}";
/// All zero GUID used for default values which indicate unstamped rows.
-pub const UNSTAMPED_SCOPE_SENTINEL: &str = "${registry.UNSTAMPED_SCOPE_SENTINEL}";
+pub const UNSTAMPED_MANIFEST_ID: &str = "${registry.UNSTAMPED_SCOPE_SENTINEL}";
/// Rows referenced from inside manifest content: \`(target_table, [(referencing_table, column)])\`.
/// On a manifest split the referenced rows are reference-copied into the destination manifest.
diff --git a/core/rust/Cargo.lock b/core/rust/Cargo.lock
index 5fc66afba..fb276a50a 100644
--- a/core/rust/Cargo.lock
+++ b/core/rust/Cargo.lock
@@ -52,7 +52,6 @@ dependencies = [
"base64",
"chrono",
"console_error_panic_hook",
- "digest",
"ed25519-dalek",
"flate2",
"getrandom 0.2.17",
diff --git a/core/rust/Cargo.toml b/core/rust/Cargo.toml
index ea3945c6e..0aa98b509 100644
--- a/core/rust/Cargo.toml
+++ b/core/rust/Cargo.toml
@@ -24,7 +24,7 @@ default = []
uniffi = ["dep:uniffi"]
# Feature for UniFFI CLI bindgen tool (only for generating bindings, not for library)
uniffi-cli = ["uniffi/cli"]
-# Feature for WASM builds (browser extension)
+# Feature for WASM builds (web app and browser extension)
wasm = ["dep:wasm-bindgen", "dep:serde-wasm-bindgen", "dep:console_error_panic_hook", "dep:js-sys"]
[dependencies]
@@ -32,7 +32,7 @@ wasm = ["dep:wasm-bindgen", "dep:serde-wasm-bindgen", "dep:console_error_panic_h
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
-# Date/time handling for LWW comparison
+# Date/time handling: vault timestamps and last-write-wins comparison
chrono = { version = "0.4", default-features = false, features = ["serde", "now", "wasmbind"] }
# Error handling
@@ -43,7 +43,6 @@ srp = "0.6"
sha2 = "0.10"
rand = { version = "0.9", features = ["std", "std_rng"] }
num-bigint = "0.4"
-digest = "0.10"
subtle = "2.5"
getrandom = { version = "0.3", features = ["wasm_js"] }
@@ -64,10 +63,10 @@ rand_core06 = { package = "rand_core", version = "0.6", features = ["getrandom"]
# Wipes key material from memory on drop
zeroize = { version = "1.8", features = ["derive"] }
-# gzip compression for the vault_codec packaged format
+# gzip for the vault_codec packaged format and stored email sources
flate2 = { version = "1.0", default-features = false, features = ["rust_backend"] }
-# base64 for vault_codec blob byte transport + ciphertext hashing
+# base64 for keys, ciphertext and blob bytes in JSON transport
base64 = "0.22"
# UniFFI for Swift/Kotlin bindings (optional)
@@ -84,6 +83,8 @@ mail-parser = { version = "0.11.6", features = ["full_encoding"] }
# SQLite client which can be used by all host apps for uniform access
rusqlite = { version = "0.40", features = ["bundled", "serialize"] }
+
+# Ed25519 account signing keys
ed25519-dalek = { version = "2", default-features = false, features = ["std", "zeroize"] }
# rsa 0.9 pulls rand 0.8 / getrandom 0.2, which only builds for wasm32-unknown-unknown with its "js" feature.
diff --git a/core/rust/README.md b/core/rust/README.md
index dddcc0fe3..d53af79ee 100644
--- a/core/rust/README.md
+++ b/core/rust/README.md
@@ -19,15 +19,14 @@ content-addressed blobs, and materialize them back.
### vault_merge
Last-Write-Wins (LWW) merge of a local vault onto the server's, one manifest at a time, rows out.
-### vault_sharing
-Which manifests a push writes and which the account can still open, for multi-manifest (shared) vaults.
-
### vault_pruner
Permanently deletes items in trash older than retention period (default: 30 days).
### vault_sync
The sync engine every client drives: a sans-IO command loop that asks the host for HTTP, state and
-SQLite access and runs the status check, pull, merge, push and the storage-format migration.
+SQLite access and runs the status check, pull, merge, push and the storage-format migration. Its tests
+run the whole engine against an in-process model of the v2 vault API (`tests/fake_server.rs`) with a
+real SQLite host per device (`tests/test_host.rs`).
### sqlite_host
An in-memory SQLite database (`SqliteMemoryDatabase`) that hosts every client's vault, so no platform
diff --git a/core/rust/src/bindings/uniffi_api.rs b/core/rust/src/bindings/uniffi_api.rs
index 6f211ea95..8d5e3b5b1 100644
--- a/core/rust/src/bindings/uniffi_api.rs
+++ b/core/rust/src/bindings/uniffi_api.rs
@@ -1,7 +1,6 @@
//! UniFFI API module for Swift and Kotlin bindings.
//!
//! This module exposes the core vault operations via UniFFI for mobile platforms.
-//! All functions use JSON strings for input/output to simplify cross-language marshalling.
use crate::crypto::argon2::Argon2Error;
use crate::crypto::{KeyChainError, SrpInputError};
@@ -10,18 +9,15 @@ use crate::common::error::{json_call, VaultError};
use crate::sqlite_host::{MemoryDatabase, SqlResult, SqlValue};
use crate::vault_codec::{self, CanonicalizeInput};
+// Vault sync.
+
/// Get the list of table names that take part in a vault sync.
#[uniffi::export]
pub fn get_syncable_table_names() -> Vec<String> {
crate::vault_model::SYNCABLE_TABLE_NAMES.iter().map(|s| s.to_string()).collect()
}
-/// Prune expired items from trash (items with DeletedAt older than retention_days, default 30).
-/// Input: `PruneInput` JSON. Output: `PruneOutput` JSON.
-#[uniffi::export]
-pub fn prune_vault_json(input_json: String) -> Result<String, VaultError> {
- json_call(&input_json, crate::vault_pruner::prune_vault)
-}
+// Credential matcher.
/// Filter credentials for autofill by the current URL/app and page title.
/// Input: `CredentialMatcherInput` JSON. Output: `CredentialMatcherOutput` JSON.
@@ -30,16 +26,13 @@ pub fn filter_credentials_json(input_json: String) -> Result<String, VaultError>
crate::credential_matcher::filter_credentials_json(&input_json)
}
-/// Extract domain from a URL.
-/// Strips the www. prefix if present.
-/// Example: "https://www.example.com/path" -> "example.com"
+/// The domain of a URL or partial domain: no protocol, `www.` prefix, path, query or fragment.
#[uniffi::export]
pub fn extract_domain(url: String) -> String {
crate::credential_matcher::extract_domain(&url)
}
-/// Extract root domain from a domain.
-/// Example: "www.example.com" -> "example.com"
+/// The root domain of a domain: `sub.example.co.uk` gives `example.co.uk`.
#[uniffi::export]
pub fn extract_root_domain(domain: String) -> String {
crate::credential_matcher::extract_root_domain(&domain)
@@ -57,9 +50,7 @@ pub fn is_related_origin_allowed(caller_origin: String, origins: Vec<String>) ->
crate::credential_matcher::is_related_origin_allowed(&caller_origin, &origins)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Favicon Functions
-// ═══════════════════════════════════════════════════════════════════════════════
+// Favicon.
/// Pick the favicon target for an item from its URLs, in the order the item lists them.
///
@@ -77,9 +68,7 @@ pub fn favicon_source_key(url: String) -> String {
crate::favicon::favicon_source_key(&url)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Password Generator Functions
-// ═══════════════════════════════════════════════════════════════════════════════
+// Password generator.
/// Generate a password or passphrase from `PasswordSettings` JSON; `Type` selects "basic" or "diceware" and an
/// optional 64-character hex `Seed` makes the output deterministic for UI previews.
@@ -94,9 +83,7 @@ pub fn get_diceware_languages() -> Vec<String> {
crate::password_generator::available_languages()
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Identity Generator Functions
-// ═══════════════════════════════════════════════════════════════════════════════
+// Identity generator.
/// Generate a random identity from `IdentityRequest` JSON (`language`, `gender`, `ageRange`, `birthdateOptions`);
/// returns `Identity` JSON with camelCase fields.
@@ -131,6 +118,14 @@ pub fn get_identity_languages() -> Vec<String> {
crate::identity_generator::available_languages()
}
+/// Get the list of age range option values ("random" plus 5-year ranges).
+#[uniffi::export]
+pub fn get_identity_age_ranges() -> Vec<String> {
+ crate::identity_generator::available_age_ranges()
+}
+
+// Email parser.
+
/// Parse a raw RFC 822 email source into its html/plain bodies and attachment metadata, returned as
/// a JSON string (`{htmlBody, textBody, attachments: [{filename, mimeType, size, detached, partIndex}]}`). Input that
/// starts with the gzip magic bytes (0x1f 0x8b) is gunzipped, so the decrypted
@@ -152,15 +147,7 @@ pub fn extract_email_attachment(source: Vec<u8>, index: u32, detached_body: Opti
crate::email_parser::extract_email_attachment(&source, index as usize, detached_body.as_deref())
}
-/// Get the list of age range option values ("random" plus 5-year ranges).
-#[uniffi::export]
-pub fn get_identity_age_ranges() -> Vec<String> {
- crate::identity_generator::available_age_ranges()
-}
-
-// ═══════════════════════════════════════════════════════════════════════════════
-// Vault Codec Functions (manifest-v1 storage format), JSON-string in/out.
-// ═══════════════════════════════════════════════════════════════════════════════
+// Vault codec (manifest-v1 storage format).
/// Canonicalize normalized tables into manifest + metadata + blob map.
/// Input: `CanonicalizeInput` JSON. Output: `CanonicalizedVault` JSON.
@@ -201,12 +188,9 @@ pub fn vault_codec_unpack_payload(plain_bytes: Vec<u8>) -> Result<String, VaultE
crate::vault_codec::unpack_payload(&plain_bytes)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Argon2id Key Derivation Functions
-// ═══════════════════════════════════════════════════════════════════════════════
+// Argon2id key derivation and the account key chain.
-/// Derive a 32-byte key from a password and salt (UTF-8 bytes) with Argon2id under the `EncryptionSettings`
-/// JSON, or the defaults for an empty string.
+/// Derive a 32-byte key from a password and salt (UTF-8 bytes) with Argon2id under the `EncryptionSettings` JSON (required).
#[uniffi::export]
pub fn argon2_derive_key(password: String, salt: String, encryption_settings: String) -> Result<Vec<u8>, Argon2Error> {
crate::crypto::argon2::argon2_derive_key_from_settings(&password, &salt, &encryption_settings)
@@ -272,9 +256,7 @@ pub fn open_account_key_chain(stored_key: Vec<u8>, encrypted_account_key: String
})
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// SRP (Secure Remote Password) Functions
-// ═══════════════════════════════════════════════════════════════════════════════
+// SRP (Secure Remote Password).
/// A random 32-byte SRP salt as an uppercase hex string.
#[uniffi::export]
@@ -312,9 +294,7 @@ pub fn srp_verify_session(client_public: String, client_proof: String, session_k
crate::crypto::srp::srp_verify_session(&client_public, &client_proof, &session_key, &server_proof)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Crypto
-// ═══════════════════════════════════════════════════════════════════════════════
+// RSA-OAEP.
/// RSA-OAEP-256 decrypt base64 ciphertext with a JWK private key.
#[uniffi::export]
@@ -328,9 +308,7 @@ pub fn mobile_login_encrypt_account_key(account_key: Vec<u8>, public_key_jwk: St
crate::crypto::encrypt_with_public_key_and_label(&account_key, &public_key_jwk, crate::crypto::aad::MOBILE_LOGIN_ACCOUNT_KEY)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Vault sync engine
-// ═══════════════════════════════════════════════════════════════════════════════
+// Vault sync engine and SQLite host.
/// One engine operation. The host loops on `next_command` / `resume` until the command is `done`; see the
/// `vault_sync` module docs for the command and response shapes.
@@ -358,7 +336,7 @@ impl VaultSyncSession {
}
}
-/// An in-memory SQLite database that can be used by host applications to be have uniform access to the database.
+/// An in-memory SQLite database that can be used by host applications to have uniform access to the database.
#[derive(uniffi::Object)]
pub struct SqliteMemoryDatabase {
inner: MemoryDatabase,
@@ -411,48 +389,3 @@ impl SqliteMemoryDatabase {
}
}
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_get_syncable_table_names() {
- let names = get_syncable_table_names();
- assert!(names.contains(&"Items".to_string()));
- assert!(names.contains(&"FieldValues".to_string()));
- assert!(names.contains(&"Settings".to_string()));
- assert!(names.contains(&"ItemStats".to_string()));
- assert!(names.contains(&"EncryptionKeys".to_string()));
- assert!(!names.contains(&crate::vault_model::OVERFLOW_TABLE.to_string()), "the overflow carrier is not synced as a table of its own; it rides inside the manifest");
- assert_eq!(names.len(), 14);
- }
-
- #[test]
- fn test_prune_vault_json() {
- let input = r#"{
- "tables": [{"name": "Items", "records": []}],
- "retentionDays": 30,
- "currentTime": "2024-01-15T10:30:00.000Z"
- }"#;
-
- let result = prune_vault_json(input.to_string());
- assert!(result.is_ok());
-
- let output: serde_json::Value = serde_json::from_str(&result.unwrap()).unwrap();
- assert_eq!(output["success"], true);
- }
-
- #[test]
- fn test_extract_domain() {
- // extract_domain strips www. prefix from domains
- assert_eq!(extract_domain("https://www.example.com/path".to_string()), "example.com");
- assert_eq!(extract_domain("http://github.com".to_string()), "github.com");
- assert_eq!(extract_domain("https://subdomain.example.com".to_string()), "subdomain.example.com");
- }
-
- #[test]
- fn test_extract_root_domain() {
- assert_eq!(extract_root_domain("www.example.com".to_string()), "example.com");
- assert_eq!(extract_root_domain("github.com".to_string()), "github.com");
- }
-}
diff --git a/core/rust/src/bindings/wasm.rs b/core/rust/src/bindings/wasm.rs
index 5b64120a0..53faeee7e 100644
--- a/core/rust/src/bindings/wasm.rs
+++ b/core/rust/src/bindings/wasm.rs
@@ -1,4 +1,4 @@
-//! WASM bindings for web apps
+//! WASM bindings for the web app and the browser extension.
use serde::Serialize;
use wasm_bindgen::prelude::*;
@@ -23,9 +23,7 @@ fn to_js<T: Serialize>(value: &T) -> Result<JsValue, JsValue> {
value.serialize(&serde_wasm_bindgen::Serializer::new().serialize_maps_as_objects(true).serialize_missing_as_null(true)).map_err(js_err)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Vault Sync WASM Bindings
-// ═══════════════════════════════════════════════════════════════════════════════
+// Vault sync.
/// Get the list of table names that take part in a vault sync.
#[wasm_bindgen(js_name = getSyncableTableNames)]
@@ -33,9 +31,7 @@ pub fn get_syncable_table_names_js() -> Vec<String> {
crate::vault_model::SYNCABLE_TABLE_NAMES.iter().map(|s| s.to_string()).collect()
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Vault Codec WASM Bindings (manifest-v1 storage format)
-// ═══════════════════════════════════════════════════════════════════════════════
+// Vault codec (manifest-v1 storage format).
/// The sha256 (lowercase hex) of an uploaded logo's bytes: the `Source` of a `custom` logo row, and
/// what `vaultCodecLogoIdFor` then derives the row id from.
@@ -44,7 +40,7 @@ pub fn vault_codec_logo_content_hash_js(bytes: Vec<u8>) -> String {
vault_codec::logo_content_hash(&bytes)
}
-/// The `Logos.Id` to use for the logo `(kind, source)` inside the manifest with id `manifestId`
+/// The `Logos.Id` to use for the logo `(kind, source)` inside the manifest with id `manifestId`.
/// `kind` is 'favicon' (source = domain), 'builtin' (source = catalog key) or 'custom' (source = image content hash).
#[wasm_bindgen(js_name = vaultCodecLogoIdFor)]
pub fn vault_codec_logo_id_for_js(manifest_id: String, kind: String, source: String) -> String {
@@ -66,7 +62,7 @@ pub fn vault_codec_generate_manifest_salt_js() -> String {
vault_codec::generate_manifest_salt()
}
-/// Pack a payload JSON string into gzip(envelope{contentHash, payload}). Encryption is done by platform.
+/// Pack a payload JSON string into gzip(envelope{contentHash, payload}). The caller encrypts the result.
#[wasm_bindgen(js_name = vaultCodecPackPayload)]
pub fn vault_codec_pack_payload_js(payload_json: &str) -> Result<Vec<u8>, JsValue> {
vault_codec::pack_payload(payload_json).map_err(js_err)
@@ -78,9 +74,7 @@ pub fn vault_codec_unpack_payload_js(plain_bytes: &[u8]) -> Result<String, JsVal
vault_codec::unpack_payload(plain_bytes).map_err(js_err)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Credential Matcher WASM Bindings
-// ═══════════════════════════════════════════════════════════════════════════════
+// Credential matcher.
/// Filter credentials for autofill. Input: `CredentialMatcherInput`. Output: `CredentialMatcherOutput`.
#[wasm_bindgen(js_name = filterCredentials)]
@@ -89,7 +83,7 @@ pub fn filter_credentials_js(input: JsValue) -> Result<JsValue, JsValue> {
let output: CredentialMatcherOutput = filter_credentials(input);
- serde_wasm_bindgen::to_value(&output).map_err(js_err)
+ to_js(&output)
}
/// The domain of a URL or partial domain: no protocol, `www.` prefix, path, query or fragment.
@@ -116,22 +110,18 @@ pub fn is_related_origin_allowed_js(caller_origin: &str, origins: Vec<String>) -
crate::credential_matcher::is_related_origin_allowed(caller_origin, &origins)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Favicon WASM Bindings
-// ═══════════════════════════════════════════════════════════════════════════════
+// Favicon.
/// The favicon target for an item's URLs (in item order): the URL to fetch and the `Logos.Source` key, or null.
#[wasm_bindgen(js_name = selectFaviconTarget)]
pub fn select_favicon_target_js(urls: Vec<String>) -> Result<JsValue, JsValue> {
match crate::favicon::select_favicon_target(&urls) {
- Some(target) => serde_wasm_bindgen::to_value(&target).map_err(js_err),
+ Some(target) => to_js(&target),
None => Ok(JsValue::NULL),
}
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Password Generator WASM Bindings
-// ═══════════════════════════════════════════════════════════════════════════════
+// Password generator.
/// Generate a password or passphrase from `PasswordSettings` JSON; `Type` selects "basic" or "diceware".
#[wasm_bindgen(js_name = generatePassword)]
@@ -145,9 +135,7 @@ pub fn get_diceware_languages_js() -> Vec<String> {
available_languages()
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Identity Generator WASM Bindings
-// ═══════════════════════════════════════════════════════════════════════════════
+// Identity generator.
/// Generate a random identity from `IdentityRequest` JSON (`language`, `gender`, `ageRange`, `birthdateOptions`);
/// returns `Identity` JSON.
@@ -182,9 +170,13 @@ pub fn get_identity_languages_js() -> Vec<String> {
crate::identity_generator::available_languages()
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Email Parser WASM Bindings
-// ═══════════════════════════════════════════════════════════════════════════════
+/// Get the list of age range option values ("random" plus 5-year ranges).
+#[wasm_bindgen(js_name = getIdentityAgeRanges)]
+pub fn get_identity_age_ranges_js() -> Vec<String> {
+ crate::identity_generator::available_age_ranges()
+}
+
+// Email parser.
/// Parse a raw RFC 822 email source into its html/plain bodies and attachment metadata.
/// Input that starts with the gzip magic bytes (0x1f 0x8b) is gunzipped, so the
@@ -208,18 +200,9 @@ pub fn extract_email_attachment_js(source: &[u8], index: usize, detached_body: O
crate::email_parser::extract_email_attachment(source, index, detached_body.as_deref()).map_err(js_err)
}
-/// Get the list of age range option values ("random" plus 5-year ranges).
-#[wasm_bindgen(js_name = getIdentityAgeRanges)]
-pub fn get_identity_age_ranges_js() -> Vec<String> {
- crate::identity_generator::available_age_ranges()
-}
+// Argon2id key derivation and the account key chain.
-// ═══════════════════════════════════════════════════════════════════════════════
-// Argon2id Key Derivation WASM Bindings
-// ═══════════════════════════════════════════════════════════════════════════════
-
-/// Derive a 32-byte key from a password and salt (UTF-8 bytes) with Argon2id under the `EncryptionSettings`
-/// JSON, or the defaults for an empty string.
+/// Derive a 32-byte key from a password and salt (UTF-8 bytes) with Argon2id under the `EncryptionSettings` JSON (required).
#[wasm_bindgen(js_name = argon2DeriveKey)]
pub fn argon2_derive_key_js(password: &str, salt: &str, encryption_settings: &str) -> Result<Vec<u8>, JsValue> {
crate::crypto::argon2::argon2_derive_key_from_settings(password, salt, encryption_settings).map_err(js_err)
@@ -279,9 +262,7 @@ pub fn open_account_key_chain_js(stored_key: &str, encrypted_account_key: &str,
to_js(&result)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// SRP (Secure Remote Password) WASM Bindings
-// ═══════════════════════════════════════════════════════════════════════════════
+// SRP (Secure Remote Password).
/// A random 32-byte SRP salt as an uppercase hex string.
#[wasm_bindgen(js_name = srpGenerateSalt)]
@@ -305,14 +286,14 @@ pub fn srp_derive_verifier_js(private_key: &str) -> Result<String, JsValue> {
#[wasm_bindgen(js_name = srpGenerateEphemeral)]
pub fn srp_generate_ephemeral_js() -> Result<JsValue, JsValue> {
let ephemeral = crate::crypto::srp::srp_generate_ephemeral();
- serde_wasm_bindgen::to_value(&ephemeral).map_err(js_err)
+ to_js(&ephemeral)
}
/// The client session as `{ proof, key }` (uppercase hex) from the server's public ephemeral; hex inputs.
#[wasm_bindgen(js_name = srpDeriveSession)]
pub fn srp_derive_session_js(client_secret: &str, server_public: &str, salt: &str, identity: &str, private_key: &str) -> Result<JsValue, JsValue> {
let session = crate::crypto::srp::srp_derive_session(client_secret, server_public, salt, identity, private_key).map_err(js_err)?;
- serde_wasm_bindgen::to_value(&session).map_err(js_err)
+ to_js(&session)
}
/// Whether the server's proof (M2) matches, which confirms it derived the same session key; hex inputs.
@@ -321,9 +302,7 @@ pub fn srp_verify_session_js(client_public: &str, client_proof: &str, session_ke
crate::crypto::srp::srp_verify_session(client_public, client_proof, session_key, server_proof).map_err(js_err)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Vault sync engine WASM Bindings
-// ═══════════════════════════════════════════════════════════════════════════════
+// Vault sync engine and SQLite host.
/// One engine operation. The host loops on `nextCommand` / `resume` until the command is `done`; see the
/// `vault_sync` module docs for the command and response shapes.
@@ -352,10 +331,7 @@ impl VaultSyncSessionJs {
}
}
-// ============================================================================
-// SQLite host WASM Bindings
-// ============================================================================
-
+/// SQLite host bindings: an in-memory database the JS hosts read and write through.
mod sqlite_js {
use js_sys::{Array, Object, Reflect, Uint8Array};
use wasm_bindgen::prelude::*;
@@ -363,7 +339,10 @@ mod sqlite_js {
use super::js_err;
use crate::sqlite_host::{MemoryDatabase, SqlResult, SqlValue};
- /// An in-memory SQLite database that can be used by host applications to be have uniform access to the database.
+ /// JavaScript's `Number.MAX_SAFE_INTEGER` (2^53 - 1).
+ const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_991.0;
+
+ /// An in-memory SQLite database that can be used by host applications to have uniform access to the database.
#[wasm_bindgen(js_name = SqliteMemoryDatabase)]
pub struct SqliteMemoryDatabaseJs {
inner: Option<MemoryDatabase>,
@@ -428,7 +407,7 @@ mod sqlite_js {
Ok(SqlValue::Integer(flag as i64))
} else if let Some(number) = value.as_f64() {
// Whole numbers within the safe integer range bind as INTEGER.
- if number.fract() == 0.0 && number.abs() <= 9_007_199_254_740_992.0 { Ok(SqlValue::Integer(number as i64)) } else { Ok(SqlValue::Real(number)) }
+ if number.fract() == 0.0 && number.abs() <= MAX_SAFE_INTEGER { Ok(SqlValue::Integer(number as i64)) } else { Ok(SqlValue::Real(number)) }
} else if value.is_instance_of::<Uint8Array>() {
Ok(SqlValue::Blob(Uint8Array::new(value).to_vec()))
} else {
diff --git a/core/rust/src/common/encoding.rs b/core/rust/src/common/encoding.rs
index b48db1d53..96b416e2e 100644
--- a/core/rust/src/common/encoding.rs
+++ b/core/rust/src/common/encoding.rs
@@ -27,7 +27,7 @@ fn hex_encode(bytes: &[u8], alphabet: &[u8; 16]) -> String {
/// Decode a hex string of either case. `None` on odd length or a non-hex character.
pub(crate) fn hex_decode(hex: &str) -> Option<Vec<u8>> {
let bytes = hex.as_bytes();
- if bytes.len() % 2 != 0 {
+ if !bytes.len().is_multiple_of(2) {
return None;
}
bytes.chunks_exact(2).map(|pair| Some((nibble(pair[0])? << 4) | nibble(pair[1])?)).collect()
diff --git a/core/rust/src/common/gzip.rs b/core/rust/src/common/gzip.rs
new file mode 100644
index 000000000..cad4ac8ab
--- /dev/null
+++ b/core/rust/src/common/gzip.rs
@@ -0,0 +1,18 @@
+//! Capped gunzip shared by the vault payload codec and the email parser.
+
+use std::io::Read;
+
+use crate::common::error::{VaultError, VaultResult};
+
+/// gzip magic bytes (RFC 1952) that every gzip stream starts with.
+pub(crate) const GZIP_MAGIC: [u8; 2] = [0x1f, 0x8b];
+
+/// Gunzip `bytes`, refusing output above `limit` bytes.
+pub(crate) fn gunzip_capped(bytes: &[u8], limit: u64) -> VaultResult<Vec<u8>> {
+ let mut out = Vec::new();
+ flate2::read::GzDecoder::new(bytes).take(limit + 1).read_to_end(&mut out).map_err(|e| VaultError::General(format!("gunzip failed: {}", e)))?;
+ if out.len() as u64 > limit {
+ return Err(VaultError::General(format!("decompressed data exceeds the {} byte limit", limit)));
+ }
+ Ok(out)
+}
diff --git a/core/rust/src/common/mod.rs b/core/rust/src/common/mod.rs
index fe04303e1..67e5427e4 100644
--- a/core/rust/src/common/mod.rs
+++ b/core/rust/src/common/mod.rs
@@ -1,6 +1,7 @@
-//! Crate-wide helpers the feature modules share: errors, byte encodings, randomness and timestamps.
+//! Crate-wide helpers the feature modules share: errors, byte encodings, gzip, randomness and timestamps.
pub(crate) mod encoding;
pub mod error;
+pub(crate) mod gzip;
pub(crate) mod rng;
pub mod timestamp;
diff --git a/core/rust/src/credential_matcher/domain.rs b/core/rust/src/credential_matcher/domain.rs
index 14a6664b8..2485cf080 100644
--- a/core/rust/src/credential_matcher/domain.rs
+++ b/core/rust/src/credential_matcher/domain.rs
@@ -18,9 +18,8 @@ static COMMON_TLDS: &[&str] = &[
"au", "nz", "jp", "cn", "in", "kr", "tw", "hk", "sg", "my", "th", "id", "ph", "vn",
"za", "eg", "ng", "ke", "ug", "tz", "ma",
"ru", "ua", "by", "kz", "il", "tr", "sa", "ae", "qa", "kw",
- // New gTLDs (common ones)
- "app", "dev", "io", "ai", "tech", "shop", "store", "online", "site", "website",
- "blog", "news", "media", "tv", "video", "music", "pro", "info", "biz", "name",
+ // New gTLDs that lead real package names; words like "app" or "shop" are left out, they lead hostnames far more often.
+ "dev", "io", "ai", "tv",
];
/// [`COMMON_TLDS`] as a set for constant-time lookup.
@@ -108,12 +107,7 @@ pub fn extract_domain_with_port(url: &str) -> DomainWithPort {
return DomainWithPort::default();
}
- // The host is what follows any userinfo: "https://[email protected]" opens example.com.
- let host_and_port = authority.rsplit_once('@').map_or(authority, |(_, host)| host);
- let (host, port) = split_host_port(host_and_port);
-
- // A fully qualified name ("bank.com.") is the same host as without its trailing dot.
- let host = host.strip_suffix('.').unwrap_or(host);
+ let (host, port) = authority_host_port(authority);
let domain = host.strip_prefix("www.").unwrap_or(host);
if is_ip_literal(domain) {
@@ -134,6 +128,14 @@ pub fn extract_domain_with_port(url: &str) -> DomainWithPort {
DomainWithPort { domain: domain.to_string(), port }
}
+/// The host and numeric port of a URL authority, without userinfo and without the trailing dot of a fully qualified name.
+pub(crate) fn authority_host_port(authority: &str) -> (&str, Option<String>) {
+ // The host is what follows any userinfo: "https://[email protected]" opens example.com.
+ let host_and_port = authority.rsplit_once('@').map_or(authority, |(_, host)| host);
+ let (host, port) = split_host_port(host_and_port);
+ (host.strip_suffix('.').unwrap_or(host), port)
+}
+
/// Split an authority without userinfo into its host and numeric port; a bracketed IPv6 host keeps its brackets.
fn split_host_port(authority: &str) -> (&str, Option<String>) {
let (host, port) = match authority.strip_prefix('[').and_then(|rest| rest.find(']')) {
@@ -152,7 +154,7 @@ fn split_host_port(authority: &str) -> (&str, Option<String>) {
/// Extract domain from URL, handling both full URLs and partial domains.
/// Returns empty string if not a valid URL/domain.
-/// Note: This strips port numbers. Use extract_domain_with_port() to preserve port info.
+/// This strips port numbers; use extract_domain_with_port() to preserve port info.
pub fn extract_domain(url: &str) -> String {
extract_domain_with_port(url).domain
}
@@ -165,12 +167,7 @@ fn is_ip_literal(host: &str) -> bool {
bare.parse::<std::net::IpAddr>().is_ok()
}
-/// Extract the root (registrable) domain from a domain string, per the Public Suffix List.
-/// E.g., "sub.example.com" -> "example.com"
-/// E.g., "sub.example.co.uk" -> "example.co.uk"
-/// E.g., "preview.myproject.vercel.app" -> "myproject.vercel.app"
-/// A domain that is a public suffix itself ("github.io", "localhost") and IP address literals are returned
-/// unchanged: "192.168.1.5" must not become "1.5".
+/// The registrable domain of a domain per the Public Suffix List, or the input itself for a public suffix or IP literal.
pub fn extract_root_domain(domain: &str) -> String {
if is_ip_literal(domain) {
return domain.to_string();
@@ -179,12 +176,7 @@ pub fn extract_root_domain(domain: &str) -> String {
public_suffix::registrable_domain(domain).unwrap_or(domain).to_string()
}
-/// Check if two domains match: the same domain, or the same root domain (so subdomains match their parent).
-/// Note: Both parameters should be pre-extracted domains (without protocol, www, path, etc.)
-///
-/// Two hosts under a shared public suffix ("victim.vercel.app" and "attacker.vercel.app") have different root
-/// domains and never match, and neither do a public suffix and a host under it ("vercel.app" and
-/// "victim.vercel.app"): the suffix has no registrable domain to share.
+/// Whether two extracted domains are the same domain or share a registrable domain.
pub fn domains_match(domain1: &str, domain2: &str) -> bool {
if domain1.is_empty() || domain2.is_empty() {
return false;
@@ -200,6 +192,10 @@ pub fn domains_match(domain1: &str, domain2: &str) -> bool {
return true;
}
+ /*
+ * Anti-phishing: only a shared registrable domain matches. Two hosts under a public suffix ("victim.vercel.app",
+ * "attacker.vercel.app") never match, and neither do a suffix and a host under it (it has no registrable domain).
+ */
match (public_suffix::registrable_domain(domain1), public_suffix::registrable_domain(domain2)) {
(Some(root1), Some(root2)) => root1 == root2,
_ => false,
@@ -254,14 +250,12 @@ pub fn is_related_origin_allowed(caller_origin: &str, origins: &[String]) -> boo
/// The host of a serialized web origin ("https://login.example.com:8443" gives "login.example.com").
fn origin_host(origin: &str) -> Option<&str> {
- let (scheme, authority) = origin.split_once("://")?;
- if !is_web_scheme(scheme) || authority.is_empty() || authority.contains(['/', '?', '#', '@']) {
+ let (scheme, authority, rest) = split_url(origin);
+ let scheme = scheme.filter(|scheme| is_web_scheme(scheme))?;
+ if !origin[scheme.len() + 1..].starts_with("//") || !rest.is_empty() || authority.contains('@') {
return None;
}
- let host = match authority.rsplit_once(':') {
- Some((host, port)) if !host.ends_with(':') && port.chars().all(|c| c.is_ascii_digit()) => host,
- _ => authority,
- };
+ let (host, _) = split_host_port(authority);
(!host.is_empty()).then_some(host)
}
@@ -280,7 +274,7 @@ mod tests {
use super::*;
#[test]
- fn test_is_app_package_name() {
+ fn detects_app_package_names() {
assert!(is_app_package_name("com.coolblue.app"));
assert!(is_app_package_name("nl.marktplaats.android"));
assert!(is_app_package_name("org.example.app"));
@@ -289,10 +283,12 @@ mod tests {
assert!(!is_app_package_name("example.com"));
assert!(!is_app_package_name("coolblue.nl"));
assert!(!is_app_package_name("nodot"));
+ assert!(is_app_package_name("tv.twitch.android.app"));
+ assert!(!is_app_package_name("app.example.com"), "a scheme-less hostname under a common subdomain word");
}
#[test]
- fn test_extract_domain() {
+ fn extracts_domain() {
assert_eq!(extract_domain("https://www.example.com/path"), "example.com");
assert_eq!(extract_domain("http://example.com"), "example.com");
assert_eq!(extract_domain("example.com"), "example.com");
@@ -327,7 +323,7 @@ mod tests {
}
#[test]
- fn test_extract_domain_single_word_hostname_with_port() {
+ fn extract_domain_single_word_hostname_with_port() {
// Single-word hostnames with port (common for self-hosted services)
assert_eq!(extract_domain("http://localhost:8080"), "localhost");
assert_eq!(extract_domain("http://localhost:81"), "localhost");
@@ -365,7 +361,7 @@ mod tests {
}
#[test]
- fn test_extract_domain_with_port() {
+ fn extract_domain_strips_port() {
// Port numbers should be stripped from extract_domain
assert_eq!(extract_domain("https://example.com:8080"), "example.com");
assert_eq!(extract_domain("https://example.com:8080/path"), "example.com");
@@ -376,7 +372,7 @@ mod tests {
}
#[test]
- fn test_extract_domain_with_port_struct() {
+ fn extract_domain_with_port_struct() {
// Test the DomainWithPort struct
let result = extract_domain_with_port("https://example.com:8080/path");
assert_eq!(result.domain, "example.com");
@@ -403,7 +399,7 @@ mod tests {
}
#[test]
- fn test_extract_domain_normalizes_host() {
+ fn extract_domain_normalizes_host() {
// Userinfo is dropped: the host is what follows the last "@".
assert_eq!(extract_domain("https://[email protected]/"), "evil.com");
assert_eq!(extract_domain("https://user:[email protected]:8443/"), "example.com");
@@ -431,7 +427,7 @@ mod tests {
}
#[test]
- fn test_extract_root_domain() {
+ fn extracts_root_domain() {
assert_eq!(extract_root_domain("sub.example.com"), "example.com");
assert_eq!(extract_root_domain("example.com"), "example.com");
assert_eq!(extract_root_domain("sub.example.co.uk"), "example.co.uk");
@@ -457,7 +453,7 @@ mod tests {
}
#[test]
- fn test_extract_root_domain_ip_literals_unchanged() {
+ fn extract_root_domain_keeps_ip_literals() {
assert_eq!(extract_root_domain("192.168.1.5"), "192.168.1.5");
assert_eq!(extract_root_domain("10.0.0.1"), "10.0.0.1");
assert_eq!(extract_root_domain("::1"), "::1");
@@ -466,7 +462,7 @@ mod tests {
}
#[test]
- fn test_is_related_origin_allowed() {
+ fn related_origins_allowlist() {
let origins = |list: &[&str]| list.iter().map(|o| o.to_string()).collect::<Vec<_>>();
let x = origins(&["https://twitter.com", "https://x.com", "https://mobile.twitter.com:8443"]);
@@ -495,7 +491,7 @@ mod tests {
}
#[test]
- fn test_is_rp_id_allowed_for_host() {
+ fn rp_id_allowed_for_host_or_parent() {
// The host itself and parent domains below the public suffix.
assert!(is_rp_id_allowed_for_host("example.com", "login.example.com"));
assert!(is_rp_id_allowed_for_host("Example.COM.", "login.example.com"));
@@ -516,12 +512,11 @@ mod tests {
}
#[test]
- fn test_ip_literals_require_exact_match() {
+ fn ip_literals_require_exact_match() {
// Same IP matches
assert!(domains_match("192.168.1.5", "192.168.1.5"));
// Distinct IPs sharing trailing octets must not match
- // (previously both reduced to root "1.5" and matched)
assert!(!domains_match("192.168.1.5", "10.0.1.5"));
// An IP must not be treated as a "subdomain" of a suffix of itself
@@ -539,7 +534,7 @@ mod tests {
}
#[test]
- fn test_domains_match() {
+ fn matches_domains() {
// Exact match
assert!(domains_match("example.com", "example.com"));
@@ -563,7 +558,7 @@ mod tests {
}
#[test]
- fn test_domains_match_shared_hosting_tenants() {
+ fn domains_match_shared_hosting_tenants() {
// Sibling tenants on a shared suffix never match; the same tenant matches across its own subdomains.
assert!(!domains_match("myproject.vercel.app", "evil-phish.vercel.app"));
assert!(domains_match("preview.myproject.vercel.app", "myproject.vercel.app"));
diff --git a/core/rust/src/credential_matcher/mod.rs b/core/rust/src/credential_matcher/mod.rs
index 2c0098536..d83bcecd0 100644
--- a/core/rust/src/credential_matcher/mod.rs
+++ b/core/rust/src/credential_matcher/mod.rs
@@ -3,12 +3,14 @@
//! This implementation follows the unified filtering algorithm specification
//! for cross-platform consistency with browser extensions, iOS, and Android.
//!
-//! Algorithm structure (priority order with early returns):
-//! 1. Priority 1: app package name exact match (for mobile apps)
-//! 2. Priority 2: URL domain matching (exact, subdomain, root domain)
-//! 3. Priority 3b: root domain word matching against item names (only credentials without URLs)
-//! 4. Priority 3: page title fallback (only credentials without URLs, anti-phishing)
-//! 5. Priority 4: text matching of the search string against item names (only credentials without URLs)
+//! Stages in execution order with early returns, each with the `matched_priority` it reports:
+//! 1. App package name exact match, for mobile apps (priority 1)
+//! 2. URL domain matching: exact, subdomain, root domain (priority 2)
+//! 3. Root domain words against item names, web pages in Default mode only (priority 3)
+//! 4. Page title words against item names, for input that is not a web page (priority 3)
+//! 5. Search string words against item names (priority 4)
+//!
+//! Stages 3 to 5 only consider credentials without URLs.
//!
//! Anti-phishing: a name match never returns a credential that has a URL, and a URL (any input with a
//! scheme) whose host cannot be extracted matches nothing instead of falling through to the name stages.
@@ -78,7 +80,7 @@ pub struct CredentialMatcherInput {
pub struct CredentialMatcherOutput {
/// IDs of matched credentials, in priority order, capped at `input.max_results` (10 by default).
pub matched_ids: Vec<String>,
- /// Which priority level matched (1-4, or 0 if no match)
+ /// Which priority level matched (1-4, or 0 if no match).
pub matched_priority: u8,
}
@@ -101,7 +103,7 @@ pub fn filter_credentials(input: CredentialMatcherInput) -> CredentialMatcherOut
}
if is_app_package_name(&current_url) {
- // Priority 1: app package name exact match (e.g. com.coolblue.app).
+ // Stage 1: app package name exact match (e.g. com.coolblue.app).
let ids = match_package_name(&credentials, &current_url, max_results);
if !ids.is_empty() {
return CredentialMatcherOutput::matched(1, ids);
@@ -110,17 +112,18 @@ pub fn filter_credentials(input: CredentialMatcherInput) -> CredentialMatcherOut
} else {
let current = extract_domain_with_port(&current_url);
if !current.domain.is_empty() {
- // Priority 2: URL domain matching (exact domain+port, exact domain, then subdomain/root domain).
+ // Stage 2: URL domain matching (exact domain+port, exact domain, then subdomain/root domain).
let allow_subdomain = matches!(matching_mode, AutofillMatchingMode::Default | AutofillMatchingMode::UrlSubdomain);
let ids = match_domains(&credentials, &current, allow_subdomain, ignore_port, max_results);
if !ids.is_empty() {
return CredentialMatcherOutput::matched(2, ids);
}
- // Priority 3b: words from only the root domain (no subdomains, no path/query) against item
- // names, e.g. outlook.office.com contributes "office" but not "outlook". Same anti-phishing
- // rule as Priority 3: only credentials with no URLs are eligible. This wildcard is what the
- // Default mode adds on top of URL matching, so the URL-only modes must not fall back to it.
+ /*
+ * Stage 3: words from only the root domain against item names, e.g. outlook.office.com contributes "office" but not
+ * "outlook". Only credentials with no URLs are eligible. This is what Default mode adds on top of URL matching, so the
+ * URL-only modes must not fall back to it.
+ */
if matching_mode == AutofillMatchingMode::Default {
let domain_words = extract_words(&extract_root_domain(&current.domain));
let ids = match_item_names(&credentials, &domain_words, max_results);
@@ -139,7 +142,7 @@ pub fn filter_credentials(input: CredentialMatcherInput) -> CredentialMatcherOut
}
}
- // Priority 3: page title against item names for input that is neither a URL nor a linked app package.
+ // Stage 4: page title against item names for input that is neither a URL nor a linked app package.
// Anti-phishing: only credentials with no URLs are eligible.
let title_words = extract_words(&page_title);
let ids = match_item_names(&credentials, &title_words, max_results);
@@ -147,7 +150,7 @@ pub fn filter_credentials(input: CredentialMatcherInput) -> CredentialMatcherOut
return CredentialMatcherOutput::matched(3, ids);
}
- // Priority 4: the search string itself against item names. Anti-phishing: only credentials with no URLs
+ // Stage 5: the search string itself against item names. Anti-phishing: only credentials with no URLs
// are eligible, so an app package like "com.paypal.rewards" never gets the PayPal website credential.
let search_words = extract_words(&current_url);
let ids = match_item_names(&credentials, &search_words, max_results);
@@ -238,6 +241,9 @@ fn match_item_names(credentials: &[Credential], words: &[String], max_results: u
.collect()
}
+/// Title words of this many characters or fewer are too generic to match on.
+const MAX_IGNORED_WORD_CHARS: usize = 3;
+
/// Extract meaningful words from text, removing punctuation and filtering stop words.
fn extract_words(text: &str) -> Vec<String> {
if text.is_empty() {
@@ -249,7 +255,7 @@ fn extract_words(text: &str) -> Vec<String> {
.map(|c| if c.is_alphanumeric() { c } else { ' ' })
.collect::<String>()
.split_whitespace()
- .filter(|word| word.len() > 3 && !STOP_WORD_SET.contains(*word))
+ .filter(|word| word.chars().count() > MAX_IGNORED_WORD_CHARS && !STOP_WORD_SET.contains(*word))
.map(String::from)
.collect()
}
diff --git a/core/rust/src/credential_matcher/stop_words.rs b/core/rust/src/credential_matcher/stop_words.rs
index f4bde6817..9809be0cc 100644
--- a/core/rust/src/credential_matcher/stop_words.rs
+++ b/core/rust/src/credential_matcher/stop_words.rs
@@ -1,6 +1,5 @@
//! Stop words for filtering page titles during credential matching.
//!
-//! These words are filtered out to prevent generic terms from causing false positives.
use std::collections::HashSet;
use std::sync::LazyLock;
@@ -23,27 +22,24 @@ static STOP_WORDS: &[&str] = &[
"overview", "index", "main", "start", "landing",
// Marketing/Promotional
- "free", "create", "new", "your", "special", "offer",
+ "free", "create", "your", "special", "offer",
"deal", "discount", "promotion", "newsletter",
// Common website sections
- "help", "support", "contact", "about", "faq", "terms",
+ "help", "support", "contact", "about", "terms",
"privacy", "cookie", "service", "services", "products",
"shop", "store", "cart", "checkout",
// Generic descriptors
- "online", "web", "digital", "mobile", "my", "personal",
+ "online", "digital", "mobile", "personal",
"private", "general", "default", "standard", "website",
// System/Technical
"system", "admin", "administrator", "platform",
- "gateway", "api", "interface", "console",
+ "gateway", "interface", "console",
// Time-related
- "today", "now", "current", "latest", "newest", "recent",
-
- // General
- "the", "and", "or", "but", "to", "up",
+ "today", "current", "latest", "newest", "recent",
// Dutch stop words
@@ -54,7 +50,7 @@ static STOP_WORDS: &[&str] = &[
// Navigation/Site sections
"portaal", "overzicht", "startpagina", "welkom", "pagina",
- "beveiligd", "lid", "gebruiker", "profiel", "instellingen",
+ "beveiligd", "gebruiker", "profiel", "instellingen",
"begin", "hoofdpagina",
// Marketing/Promotional
@@ -77,5 +73,5 @@ static STOP_WORDS: &[&str] = &[
"vandaag", "huidig", "nieuwste",
// General
- "je", "in", "op", "de", "van", "ons", "allemaal",
+ "allemaal",
];
diff --git a/core/rust/src/credential_matcher/tests.rs b/core/rust/src/credential_matcher/tests.rs
index 08274c30f..330843039 100644
--- a/core/rust/src/credential_matcher/tests.rs
+++ b/core/rust/src/credential_matcher/tests.rs
@@ -1,1116 +1,250 @@
-//! Tests for credential matcher - ported from CredentialMatcher.test.ts
+//! The autofill matcher: which credentials a URL, app package or page title offers, in priority order, and which
+//! it must never offer (anti-phishing).
use super::*;
-/// Helper function to create test credentials with standardized structure.
-fn create_test_credential(item_name: &str, item_url: &str, username: &str) -> Credential {
- Credential {
- id: uuid_v4(),
- item_name: Some(item_name.to_string()),
- item_urls: if item_url.is_empty() {
- vec![]
- } else {
- vec![item_url.to_string()]
- },
- username: if username.is_empty() {
- None
- } else {
- Some(username.to_string())
- },
- }
-}
-
-/// Helper function to create test credentials with multiple URLs.
-fn create_test_credential_multi_url(item_name: &str, item_urls: Vec<&str>, username: &str) -> Credential {
- Credential {
- id: uuid_v4(),
- item_name: Some(item_name.to_string()),
- item_urls: item_urls.into_iter().map(String::from).collect(),
- username: if username.is_empty() {
- None
- } else {
- Some(username.to_string())
- },
- }
-}
-
-/// Simple unique ID generator for tests using atomic counter
-fn uuid_v4() -> String {
+fn credential(name: &str, urls: &[&str], username: &str) -> Credential {
use std::sync::atomic::{AtomicU64, Ordering};
static COUNTER: AtomicU64 = AtomicU64::new(0);
- let id = COUNTER.fetch_add(1, Ordering::SeqCst);
- format!("test-id-{:016x}", id)
+ Credential { id: format!("test-id-{:016x}", COUNTER.fetch_add(1, Ordering::SeqCst)), item_name: Some(name.to_string()), item_urls: urls.iter().map(|u| u.to_string()).collect(), username: Some(username).filter(|u| !u.is_empty()).map(str::to_string) }
}
-/// Creates the shared test credential dataset used across all platforms.
-fn create_shared_test_credentials() -> Vec<Credential> {
- vec![
- create_test_credential("Gmail", "https://gmail.com", "[email protected]"),
- create_test_credential("Google", "https://google.com", "[email protected]"),
- create_test_credential("Coolblue", "https://www.coolblue.nl", "[email protected]"),
- create_test_credential("Amazon", "https://amazon.com", "[email protected]"),
- create_test_credential("Coolblue App", "com.coolblue.app", "[email protected]"),
- create_test_credential("Dumpert", "dumpert.nl", "[email protected]"),
- create_test_credential("GitHub", "github.com", "[email protected]"),
- create_test_credential("Stack Overflow", "https://stackoverflow.com", "[email protected]"),
- create_test_credential("Subdomain Example", "https://app.example.com", "[email protected]"),
- create_test_credential("Title Only newyorktimes", "", ""),
- create_test_credential("Bank Account", "https://secure-bank.com", "[email protected]"),
- create_test_credential("AliExpress", "https://aliexpress.com", "[email protected]"),
- create_test_credential("Reddit", "", "[email protected]"),
- ]
-}
-
-/// Helper to filter and return matched credentials by looking up IDs
-fn filter(credentials: Vec<Credential>, current_url: &str, page_title: &str) -> Vec<Credential> {
- let input = CredentialMatcherInput {
- credentials: credentials.clone(),
- current_url: current_url.to_string(),
- page_title: page_title.to_string(),
- matching_mode: AutofillMatchingMode::Default,
- ignore_port: false,
- max_results: None,
- };
- let output = filter_credentials(input);
-
- // Look up credentials by matched IDs
- output.matched_ids
- .iter()
- .filter_map(|id| credentials.iter().find(|c| c.id == *id).cloned())
- .collect()
+fn input(credentials: &[Credential], current_url: &str, page_title: &str) -> CredentialMatcherInput {
+ CredentialMatcherInput { credentials: credentials.to_vec(), current_url: current_url.to_string(), page_title: page_title.to_string(), matching_mode: AutofillMatchingMode::Default, ignore_port: false, max_results: None }
}
-/// [#1] - Exact URL match
-#[test]
-fn test_exact_url_match() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "www.coolblue.nl", "");
-
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("Coolblue"));
-}
-
-/// [#2] - Base URL with path match
-#[test]
-fn test_base_url_with_path_match() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "https://gmail.com/signin", "");
-
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("Gmail"));
-}
-
-/// [#3] - Root domain with subdomain match
-#[test]
-fn test_root_domain_with_subdomain_match() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "https://mail.google.com", "");
-
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("Google"));
+/// The names of the credentials the matcher offers for `current_url` and `page_title`, in order.
+fn names(credentials: &[Credential], current_url: &str, page_title: &str) -> Vec<String> {
+ names_for(credentials, input(credentials, current_url, page_title))
}
-/// [#4] - No matches for non-existent domain
-#[test]
-fn test_no_matches_for_nonexistent_domain() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "https://nonexistent.com", "");
-
- assert_eq!(matches.len(), 0);
-}
-
-/// [#5] - Partial URL stored matches full URL search
-#[test]
-fn test_partial_url_matches_full_url() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "https://www.dumpert.nl", "");
-
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("Dumpert"));
-}
-
-/// [#6] - Full URL stored matches partial URL search
-#[test]
-fn test_full_url_matches_partial_url() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "coolblue.nl", "");
-
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("Coolblue"));
-}
-
-/// [#7] - Protocol variations (http/https/none) match
-#[test]
-fn test_protocol_variations() {
- let credentials = create_shared_test_credentials();
-
- let https_matches = filter(credentials.clone(), "https://github.com", "");
- let http_matches = filter(credentials.clone(), "http://github.com", "");
- let no_protocol_matches = filter(credentials, "github.com", "");
-
- assert_eq!(https_matches.len(), 1);
- assert_eq!(http_matches.len(), 1);
- assert_eq!(no_protocol_matches.len(), 1);
- assert_eq!(https_matches[0].item_name.as_deref(), Some("GitHub"));
- assert_eq!(http_matches[0].item_name.as_deref(), Some("GitHub"));
- assert_eq!(no_protocol_matches[0].item_name.as_deref(), Some("GitHub"));
-}
-
-/// [#8] - WWW prefix variations match
-#[test]
-fn test_www_variations() {
- let credentials = create_shared_test_credentials();
-
- let with_www = filter(credentials.clone(), "https://www.dumpert.nl", "");
- let without_www = filter(credentials, "https://dumpert.nl", "");
-
- assert_eq!(with_www.len(), 1);
- assert_eq!(without_www.len(), 1);
- assert_eq!(with_www[0].item_name.as_deref(), Some("Dumpert"));
- assert_eq!(without_www[0].item_name.as_deref(), Some("Dumpert"));
+fn names_for(credentials: &[Credential], input: CredentialMatcherInput) -> Vec<String> {
+ filter_credentials(input).matched_ids.iter().map(|id| credentials.iter().find(|c| c.id == *id).unwrap().item_name.clone().unwrap()).collect()
}
-/// [#9] - Subdomain matching
-#[test]
-fn test_subdomain_matching() {
- let credentials = create_shared_test_credentials();
-
- let app_subdomain = filter(credentials.clone(), "https://app.example.com", "");
- let www_subdomain = filter(credentials.clone(), "https://www.example.com", "");
- let no_subdomain = filter(credentials, "https://example.com", "");
-
- assert_eq!(app_subdomain.len(), 1);
- assert_eq!(app_subdomain[0].item_name.as_deref(), Some("Subdomain Example"));
- assert_eq!(www_subdomain.len(), 1);
- assert_eq!(www_subdomain[0].item_name.as_deref(), Some("Subdomain Example"));
- assert_eq!(no_subdomain.len(), 1);
- assert_eq!(no_subdomain[0].item_name.as_deref(), Some("Subdomain Example"));
-}
-
-/// [#10] - Paths and query strings ignored
-#[test]
-fn test_paths_and_query_strings_ignored() {
- let credentials = create_shared_test_credentials();
-
- let with_path = filter(credentials.clone(), "https://github.com/user/repo", "");
- let with_query = filter(credentials.clone(), "https://stackoverflow.com/questions?tab=newest", "");
- let with_fragment = filter(credentials, "https://gmail.com#inbox", "");
-
- assert_eq!(with_path.len(), 1);
- assert_eq!(with_path[0].item_name.as_deref(), Some("GitHub"));
- assert_eq!(with_query.len(), 1);
- assert_eq!(with_query[0].item_name.as_deref(), Some("Stack Overflow"));
- assert_eq!(with_fragment.len(), 1);
- assert_eq!(with_fragment[0].item_name.as_deref(), Some("Gmail"));
-}
-
-/// [#11] - Complex URL variations
-#[test]
-fn test_complex_url_variations() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "https://www.coolblue.nl/product/12345?ref=google", "");
-
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("Coolblue"));
-}
-
-/// [#12] - Priority ordering
-#[test]
-fn test_priority_ordering() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "coolblue.nl", "");
-
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("Coolblue"));
-}
-
-/// [#13] - Title-only matching (only works when domain extraction fails)
-#[test]
-fn test_title_only_matching() {
- let credentials = create_shared_test_credentials();
- // Use a non-URL string to trigger domain extraction failure, forcing Priority 3
- let matches = filter(credentials, "invalid-url", "newyorktimes");
-
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("Title Only newyorktimes"));
-}
-
-/// [#14] - Domain name part matching
-#[test]
-fn test_domain_name_part_matching() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "https://coolblue.be", "");
-
- // coolblue.be should NOT match coolblue.nl
- assert_eq!(matches.len(), 0);
-}
-
-/// [#15] - Package name matching
-#[test]
-fn test_package_name_matching() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "com.coolblue.app", "");
-
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("Coolblue App"));
-}
-
-/// [#16] - Invalid URL handling
-#[test]
-fn test_invalid_url_handling() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "not a url", "");
-
- assert_eq!(matches.len(), 0);
-}
-
-/// [#17] - Anti-phishing protection
-#[test]
-fn test_anti_phishing_protection() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "https://secure-bankk.com", "");
-
- // Should NOT match secure-bank.com (different domain)
- assert_eq!(matches.len(), 0);
-}
-
-/// [#18] - Ensure only full words are matched
-#[test]
-fn test_full_word_matching_only() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "", "Express Yourself App | Description");
-
- // The string above should not match "AliExpress" service name
- assert_eq!(matches.len(), 0);
+/// Every matching mode's output for `current_url`.
+fn all_modes(credentials: &[Credential], current_url: &str) -> Vec<(AutofillMatchingMode, CredentialMatcherOutput)> {
+ [AutofillMatchingMode::Default, AutofillMatchingMode::UrlSubdomain, AutofillMatchingMode::UrlExact].into_iter().map(|mode| (mode, filter_credentials(CredentialMatcherInput { matching_mode: mode, ..input(credentials, current_url, "") }))).collect()
}
-/// [#19] - Ensure separators and punctuation are stripped for matching (only works when domain extraction fails)
-#[test]
-fn test_separators_and_punctuation_stripped() {
- let credentials = create_shared_test_credentials();
- // Use a non-URL string to trigger domain extraction failure, forcing Priority 3
- let matches = filter(credentials, "invalid-url", "Reddit, social media platform");
-
- // Should match "Reddit" even though it's followed by a comma and description
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("Reddit"));
+/// The dataset every platform's matcher is checked against.
+fn shared_credentials() -> Vec<Credential> {
+ vec![
+ credential("Gmail", &["https://gmail.com"], "[email protected]"),
+ credential("Google", &["https://google.com"], "[email protected]"),
+ credential("Coolblue", &["https://www.coolblue.nl"], "[email protected]"),
+ credential("Amazon", &["https://amazon.com"], "[email protected]"),
+ credential("Coolblue App", &["com.coolblue.app"], "[email protected]"),
+ credential("Dumpert", &["dumpert.nl"], "[email protected]"),
+ credential("GitHub", &["github.com"], "[email protected]"),
+ credential("Stack Overflow", &["https://stackoverflow.com"], "[email protected]"),
+ credential("Subdomain Example", &["https://app.example.com"], "[email protected]"),
+ credential("Title Only newyorktimes", &[], ""),
+ credential("Bank Account", &["https://secure-bank.com"], "[email protected]"),
+ credential("AliExpress", &["https://aliexpress.com"], "[email protected]"),
+ credential("Reddit", &[], "[email protected]"),
+ ]
}
-/// [#20] - Test reversed domain (app package name) doesn't match on TLD
#[test]
-fn test_reversed_domain_no_tld_match() {
- let credentials = vec![
- create_test_credential("Dumpert.nl", "", "[email protected]"),
- create_test_credential("Marktplaats.nl", "", "[email protected]"),
- ];
-
- let matches = filter(credentials, "nl.marktplaats.android", "");
-
- // Should only match Marktplaats, not Dumpert (even though both have "nl")
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("Marktplaats.nl"));
+fn urls_match_their_domain_whatever_the_protocol_www_path_or_query() {
+ let credentials = shared_credentials();
+ for (url, expected) in [
+ ("www.coolblue.nl", vec!["Coolblue"]),
+ ("coolblue.nl", vec!["Coolblue"]),
+ ("https://www.coolblue.nl/product/12345?ref=google", vec!["Coolblue"]),
+ ("https://gmail.com/signin", vec!["Gmail"]),
+ ("https://gmail.com#inbox", vec!["Gmail"]),
+ ("https://stackoverflow.com/questions?tab=newest", vec!["Stack Overflow"]),
+ ("https://github.com", vec!["GitHub"]),
+ ("http://github.com", vec!["GitHub"]),
+ ("github.com", vec!["GitHub"]),
+ ("https://github.com/user/repo", vec!["GitHub"]),
+ ("https://www.dumpert.nl", vec!["Dumpert"]),
+ ("https://dumpert.nl", vec!["Dumpert"]),
+ ("https://mail.google.com", vec!["Google"]),
+ ("https://app.example.com", vec!["Subdomain Example"]),
+ ("https://www.example.com", vec!["Subdomain Example"]),
+ ("https://example.com", vec!["Subdomain Example"]),
+ ("https://another-example.com", vec![]),
+ ("com.coolblue.app", vec!["Coolblue App"]),
+ ("https://nonexistent.com", vec![]),
+ ("https://coolblue.be", vec![]),
+ ("https://secure-bankk.com", vec![]),
+ ("not a url", vec![]),
+ ("", vec![]),
+ ] {
+ assert_eq!(names(&credentials, url, ""), expected, "{url}");
+ }
}
-/// [#21] - Test app package names are properly detected and handled
#[test]
-fn test_app_package_names_handling() {
- let credentials = vec![
- create_test_credential("Google App", "com.google.android.googlequicksearchbox", "[email protected]"),
- create_test_credential("Facebook", "com.facebook.katana", "[email protected]"),
- create_test_credential("WhatsApp", "com.whatsapp", "[email protected]"),
- create_test_credential("Generic Site", "example.com", "[email protected]"),
- ];
-
- // Test com.google.android package matches
- let google_matches = filter(credentials.clone(), "com.google.android.googlequicksearchbox", "");
- assert_eq!(google_matches.len(), 1);
- assert_eq!(google_matches[0].item_name.as_deref(), Some("Google App"));
+fn names_match_only_when_the_input_is_no_url_and_only_credentials_without_urls() {
+ let credentials = shared_credentials();
+ // A non-URL search string falls through to the page title; punctuation is stripped and only whole words count.
+ assert_eq!(names(&credentials, "invalid-url", "newyorktimes"), vec!["Title Only newyorktimes"]);
+ assert_eq!(names(&credentials, "invalid-url", "Reddit, social media platform"), vec!["Reddit"]);
+ assert!(names(&credentials, "", "Express Yourself App | Description").is_empty(), "'Express' is not the whole word 'AliExpress'");
- // Test com.facebook package matches
- let facebook_matches = filter(credentials.clone(), "com.facebook.katana", "");
- assert_eq!(facebook_matches.len(), 1);
- assert_eq!(facebook_matches[0].item_name.as_deref(), Some("Facebook"));
+ // A web page never falls through to the title: a URL match wins, and without one items with URLs are never offered by name.
+ let with_url = vec![credential("blabla service", &["https://blabla.asd.com:1234"], "[email protected]"), credential("asd", &[], "[email protected]"), credential("blabla", &[], "[email protected]")];
+ assert_eq!(names(&with_url, "https://blabla.asd.com:1234/login", "Welcome to blabla asd service"), vec!["blabla service"]);
- // Test that web domain doesn't match package name
- let web_matches = filter(credentials, "https://example.com", "");
- assert_eq!(web_matches.len(), 1);
- assert_eq!(web_matches[0].item_name.as_deref(), Some("Generic Site"));
+ // Words of the root domain (not its subdomains) do match items without URLs, so a note named after a site is offered.
+ let notes = vec![credential("Test Dumpert", &["https://other-site.com"], ""), credential("Dumpert Account", &[], ""), credential("Some Other Item", &[], "")];
+ assert_eq!(names(&notes, "https://www.dumpert.nl", ""), vec!["Dumpert Account"]);
}
-/// [#22] - Test multi-part TLDs like .com.au don't match incorrectly
#[test]
-fn test_multi_part_tlds() {
+fn app_packages_match_exactly_and_never_borrow_a_website_credential() {
let credentials = vec![
- create_test_credential("Example Site AU", "https://example.com.au", "[email protected]"),
- create_test_credential("BlaBla AU", "https://blabla.blabla.com.au", "[email protected]"),
- create_test_credential("Another AU", "https://another.com.au", "[email protected]"),
- create_test_credential("UK Site", "https://example.co.uk", "[email protected]"),
+ credential("Google App", &["com.google.android.googlequicksearchbox"], ""),
+ credential("Facebook", &["com.facebook.katana"], ""),
+ credential("Coolblue", &["https://www.coolblue.nl", "com.coolblue.app", "nl.coolblue.ios"], ""),
+ credential("Generic Site", &["example.com"], ""),
+ credential("Marktplaats.nl", &[], ""),
+ credential("Dumpert.nl", &[], ""),
];
-
- // Test that blabla.blabla.com.au doesn't match other .com.au sites
- let blabla_matches = filter(credentials.clone(), "https://blabla.blabla.com.au", "");
- assert_eq!(blabla_matches.len(), 1);
- assert_eq!(blabla_matches[0].item_name.as_deref(), Some("BlaBla AU"));
-
- // Test that example.com.au doesn't match blabla.blabla.com.au
- let example_matches = filter(credentials.clone(), "https://example.com.au", "");
- assert_eq!(example_matches.len(), 1);
- assert_eq!(example_matches[0].item_name.as_deref(), Some("Example Site AU"));
-
- // Test that .co.uk domains work correctly too
- let uk_matches = filter(credentials, "https://example.co.uk", "");
- assert_eq!(uk_matches.len(), 1);
- assert_eq!(uk_matches[0].item_name.as_deref(), Some("UK Site"));
+ assert_eq!(names(&credentials, "com.google.android.googlequicksearchbox", ""), vec!["Google App"]);
+ assert_eq!(names(&credentials, "com.facebook.katana", ""), vec!["Facebook"]);
+ assert_eq!(names(&credentials, "nl.coolblue.ios", ""), vec!["Coolblue"], "any of a credential's URLs may be the package");
+ assert_eq!(names(&credentials, "https://example.com", ""), vec!["Generic Site"]);
+ assert_eq!(names(&credentials, "nl.marktplaats.android", ""), vec!["Marktplaats.nl"], "a reversed domain matches on its name, not on the TLD it shares");
+
+ // An app package without a linked credential gets, at most, a credential without URLs: never the website's.
+ let paypal = vec![credential("PayPal", &["https://paypal.com"], "victim"), credential("PayPal Notes", &[], "")];
+ for package in ["com.fake.paypal", "com.paypal.rewards", "net.paypal.cashback"] {
+ for (mode, output) in all_modes(&paypal, package) {
+ assert_eq!(output.matched_ids, vec![paypal[1].id.clone()], "{package} in {mode:?} mode");
+ assert_eq!(output.matched_priority, 4);
+ }
+ }
+ assert!(names(&paypal[..1], "com.paypal.rewards", "PayPal Rewards").is_empty(), "without a credential lacking URLs there is nothing to offer");
}
-/// Verify that public suffix list is correctly implemented and subdomains of domains classified as shared hosts
-/// should not be treated as the same domain.
#[test]
-fn test_shared_hosting_tenants_do_not_collapse_into_one_root_domain() {
+fn root_domains_follow_the_public_suffix_list() {
+ // Multi-part suffixes, and shared hosts whose tenants must never collapse into one root domain.
let credentials = vec![
- create_test_credential("My Vercel App", "https://myproject.vercel.app/login", "[email protected]"),
- create_test_credential("My GitHub Pages", "https://victim.github.io", "[email protected]"),
- create_test_credential("My Shop", "https://victim-shop.myshopify.com/admin", "[email protected]"),
- create_test_credential("My Worker", "https://victim.workers.dev", "[email protected]"),
+ credential("Example Site AU", &["https://example.com.au"], ""),
+ credential("BlaBla AU", &["https://blabla.blabla.com.au"], ""),
+ credential("UK Site", &["https://example.co.uk"], ""),
+ credential("My Vercel App", &["https://myproject.vercel.app/login"], ""),
+ credential("My GitHub Pages", &["https://victim.github.io"], ""),
+ credential("My Shop", &["https://victim-shop.myshopify.com/admin"], ""),
+ credential("My Worker", &["https://victim.workers.dev"], ""),
];
-
- for phishing_url in [
- "https://evil-phish.vercel.app/",
- "https://attacker.github.io/login",
- "https://attacker-shop.myshopify.com/admin",
- "https://attacker.workers.dev",
- "https://vercel.app",
- "https://github.io",
- ] {
- for mode in [AutofillMatchingMode::Default, AutofillMatchingMode::UrlSubdomain, AutofillMatchingMode::UrlExact] {
- let input = CredentialMatcherInput {
- credentials: credentials.clone(),
- current_url: phishing_url.to_string(),
- page_title: String::new(),
- matching_mode: mode,
- ignore_port: false,
- max_results: None,
- };
- let output = filter_credentials(input);
+ assert_eq!(names(&credentials, "https://blabla.blabla.com.au", ""), vec!["BlaBla AU"]);
+ assert_eq!(names(&credentials, "https://example.com.au", ""), vec!["Example Site AU"]);
+ assert_eq!(names(&credentials, "https://example.co.uk", ""), vec!["UK Site"]);
+ assert_eq!(names(&credentials, "https://preview.myproject.vercel.app", ""), vec!["My Vercel App"], "a tenant's own subdomains still match");
+ assert_eq!(names(&credentials, "https://docs.victim.github.io", ""), vec!["My GitHub Pages"]);
+ for phishing_url in ["https://evil-phish.vercel.app/", "https://attacker.github.io/login", "https://attacker-shop.myshopify.com/admin", "https://attacker.workers.dev", "https://vercel.app", "https://github.io"] {
+ for (mode, output) in all_modes(&credentials, phishing_url) {
assert!(output.matched_ids.is_empty(), "{phishing_url} in {mode:?} mode must not match: {:?}", output.matched_ids);
}
}
-
- // The tenant itself, and its own subdomains, still match.
- let matches = filter(credentials.clone(), "https://myproject.vercel.app/dashboard", "");
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("My Vercel App"));
-
- let matches = filter(credentials.clone(), "https://preview.myproject.vercel.app", "");
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("My Vercel App"));
-
- let matches = filter(credentials, "https://docs.victim.github.io", "");
- assert_eq!(matches.len(), 1);
- assert_eq!(matches[0].item_name.as_deref(), Some("My GitHub Pages"));
-}
-
-/// Test JSON serialization/deserialization
-#[test]
-fn test_json_roundtrip() {
- let credentials = create_shared_test_credentials();
- let input = CredentialMatcherInput {
- credentials: credentials.clone(),
- current_url: "https://github.com".to_string(),
- page_title: String::new(),
- matching_mode: AutofillMatchingMode::Default,
- ignore_port: false,
- max_results: None,
- };
-
- let json = serde_json::to_string(&input).unwrap();
- assert!(json.contains("\"currentUrl\"") && json.contains("\"matchingMode\""), "input fields are camelCase on the wire");
- let output_json = filter_credentials_json(&json).unwrap();
- assert!(output_json.contains("\"matchedIds\""), "output fields are camelCase on the wire");
- let output: CredentialMatcherOutput = serde_json::from_str(&output_json).unwrap();
-
- assert_eq!(output.matched_ids.len(), 1);
- // Look up the credential by ID to verify it's GitHub
- let matched = credentials.iter().find(|c| c.id == output.matched_ids[0]).unwrap();
- assert_eq!(matched.item_name.as_deref(), Some("GitHub"));
-}
-
-/// Test empty URL returns empty results
-#[test]
-fn test_empty_url() {
- let credentials = create_shared_test_credentials();
- let matches = filter(credentials, "", "");
-
- assert_eq!(matches.len(), 0);
-}
-
-/// Test that the default cap (DEFAULT_MAX_RESULTS = 10) bounds the output.
-/// Twelve candidates > exactly 10 returned.
-#[test]
-fn test_default_max_results_caps_at_ten() {
- let credentials: Vec<Credential> = (1..=12)
- .map(|i| create_test_credential(&format!("Site {i}"), &format!("sub{i}.example.com"), ""))
- .collect();
-
- let matches = filter(credentials, "https://example.com", "");
-
- assert_eq!(matches.len(), 10, "Default cap should be DEFAULT_MAX_RESULTS (10)");
-}
-
-/// Test that an explicit `max_results` override is honoured.
-#[test]
-fn test_max_results_override() {
- let credentials: Vec<Credential> = (1..=8)
- .map(|i| create_test_credential(&format!("Site {i}"), &format!("sub{i}.example.com"), ""))
- .collect();
-
- let input = CredentialMatcherInput {
- credentials,
- current_url: "https://example.com".to_string(),
- page_title: String::new(),
- matching_mode: AutofillMatchingMode::Default,
- ignore_port: false,
- max_results: Some(2),
- };
- let output = filter_credentials(input);
-
- assert_eq!(output.matched_ids.len(), 2, "Explicit max_results=2 should cap output");
-}
-
-/// [#23] - E2E test scenario: credentials with URLs should only match their specific domains
-/// This mirrors the browser extension E2E test setup
-#[test]
-fn test_e2e_scenario_url_only_matching() {
- let credentials = vec![
- create_test_credential("Example Site", "https://example.com/login", "[email protected]"),
- create_test_credential("Example Subdomain", "https://test.example.com/auth", "[email protected]"),
- create_test_credential("Another Site", "https://another-example.com/signin", "[email protected]"),
- ];
-
- // Test 1: Unrelated domain should return NO matches
- let unrelated_matches = filter(credentials.clone(), "https://unrelated-domain.com/login", "E2E Test Form");
- assert_eq!(unrelated_matches.len(), 0, "Unrelated domain should not match any credentials");
-
- // Test 2: example.com should only match Example Site (and possibly subdomain due to root domain matching)
- let example_matches = filter(credentials.clone(), "https://example.com/login", "E2E Test Form");
- println!("example.com matches: {:?}", example_matches.iter().map(|c| c.item_name.as_deref()).collect::<Vec<_>>());
- assert!(example_matches.len() >= 1, "example.com should match at least one credential");
- assert!(example_matches.iter().any(|c| c.item_name.as_deref() == Some("Example Site")),
- "example.com should match Example Site");
- assert!(!example_matches.iter().any(|c| c.item_name.as_deref() == Some("Another Site")),
- "example.com should NOT match Another Site");
-
- // Test 3: another-example.com should only match Another Site
- let another_matches = filter(credentials.clone(), "https://another-example.com/signin", "E2E Test Form");
- assert_eq!(another_matches.len(), 1, "another-example.com should match exactly one credential");
- assert_eq!(another_matches[0].item_name.as_deref(), Some("Another Site"));
-
- // Test 4: test.example.com subdomain should match Example Subdomain
- let subdomain_matches = filter(credentials, "https://test.example.com/auth", "E2E Test Form");
- assert!(subdomain_matches.len() >= 1, "test.example.com should match at least one credential");
- assert!(subdomain_matches.iter().any(|c| c.item_name.as_deref() == Some("Example Subdomain")),
- "test.example.com should match Example Subdomain");
- assert!(!subdomain_matches.iter().any(|c| c.item_name.as_deref() == Some("Another Site")),
- "test.example.com should NOT match Another Site");
-}
-
-/// [#24] - Multi-URL support: credentials with multiple URLs should match any of them
-#[test]
-fn test_multi_url_matching() {
- let credentials = vec![
- create_test_credential_multi_url(
- "Vodafone",
- vec!["https://www.vodafone.com", "https://my.vodafone.de", "https://www.vodafone.nl"],
- ),
- create_test_credential("Other Site", "https://example.com", "[email protected]"),
- ];
-
- // Test 1: First URL should match
- let first_url_matches = filter(credentials.clone(), "https://www.vodafone.com/account", "");
- assert_eq!(first_url_matches.len(), 1);
- assert_eq!(first_url_matches[0].item_name.as_deref(), Some("Vodafone"));
-
- // Test 2: Second URL should match
- let second_url_matches = filter(credentials.clone(), "https://my.vodafone.de/login", "");
- assert_eq!(second_url_matches.len(), 1);
- assert_eq!(second_url_matches[0].item_name.as_deref(), Some("Vodafone"));
-
- // Test 3: Third URL should match (this was the original bug!)
- let third_url_matches = filter(credentials.clone(), "https://www.vodafone.nl/inloggen", "");
- assert_eq!(third_url_matches.len(), 1, "Third URL should match");
- assert_eq!(third_url_matches[0].item_name.as_deref(), Some("Vodafone"));
-
- // Test 4: Subdomain of any URL should match
- let subdomain_matches = filter(credentials.clone(), "https://portal.vodafone.nl", "");
- assert_eq!(subdomain_matches.len(), 1);
- assert_eq!(subdomain_matches[0].item_name.as_deref(), Some("Vodafone"));
-
- // Test 5: Unrelated domain should not match
- let unrelated_matches = filter(credentials, "https://vodafone.be", "");
- assert_eq!(unrelated_matches.len(), 0);
-}
-
-/// [#25] - Multi-URL with exact match priority: exact match on any URL beats subdomain match
-#[test]
-fn test_multi_url_exact_match_priority() {
- let credentials = vec![
- create_test_credential_multi_url(
- "Site with subdomain first",
- vec!["https://app.example.com", "https://example.org"],
- ),
- ];
-
- // Searching example.org should be exact match (priority 1), not subdomain match
- let input = CredentialMatcherInput {
- credentials: credentials.clone(),
- current_url: "https://example.org".to_string(),
- page_title: String::new(),
- matching_mode: AutofillMatchingMode::Default,
- ignore_port: false,
- max_results: None,
- };
- let output = filter_credentials(input);
-
- assert_eq!(output.matched_ids.len(), 1);
- assert_eq!(output.matched_priority, 2); // Priority 2 = URL domain matching
-}
-
-/// [#26] - Multi-URL app package matching
-#[test]
-fn test_multi_url_app_package_matching() {
- let credentials = vec![
- create_test_credential_multi_url(
- "Coolblue",
- vec!["https://www.coolblue.nl", "com.coolblue.app", "nl.coolblue.ios"],
- ),
- ];
-
- // Web URL should match
- let web_matches = filter(credentials.clone(), "https://coolblue.nl", "");
- assert_eq!(web_matches.len(), 1);
- assert_eq!(web_matches[0].item_name.as_deref(), Some("Coolblue"));
-
- // Android package should match
- let android_matches = filter(credentials.clone(), "com.coolblue.app", "");
- assert_eq!(android_matches.len(), 1);
- assert_eq!(android_matches[0].item_name.as_deref(), Some("Coolblue"));
-
- // iOS package should match
- let ios_matches = filter(credentials, "nl.coolblue.ios", "");
- assert_eq!(ios_matches.len(), 1);
- assert_eq!(ios_matches[0].item_name.as_deref(), Some("Coolblue"));
}
-/// [#27] - URLs with port numbers should match correctly
#[test]
-fn test_url_with_port_matching() {
+fn any_of_a_credentials_urls_matches_and_whitespace_around_them_is_ignored() {
let credentials = vec![
- create_test_credential("Local Dev", "https://dev.local:3000", "[email protected]"),
- create_test_credential("Staging Server", "https://staging.example.com:8080", "[email protected]"),
- create_test_credential("Production", "https://example.com", "[email protected]"),
+ credential("Vodafone", &["https://www.vodafone.com", "https://my.vodafone.de", "https://www.vodafone.nl"], ""),
+ credential("Amazon", &["https://amazon.fr/", " https://amazon.de", "amazon.it\t", " com.amazon.app \n"], ""),
+ credential("Other Site", &["https://example.com"], ""),
];
+ for (url, expected) in [("https://www.vodafone.com/account", "Vodafone"), ("https://my.vodafone.de/login", "Vodafone"), ("https://www.vodafone.nl/inloggen", "Vodafone"), ("https://portal.vodafone.nl", "Vodafone"), ("https://www.amazon.de/ap/signin", "Amazon"), ("https://amazon.it", "Amazon"), ("com.amazon.app", "Amazon")] {
+ assert_eq!(names(&credentials, url, ""), vec![expected], "{url}");
+ }
+ assert!(names(&credentials, "https://vodafone.be", "").is_empty());
- // URL with port should match credential with same URL+port
- let local_matches = filter(credentials.clone(), "https://dev.local:3000/login", "");
- assert_eq!(local_matches.len(), 1);
- assert_eq!(local_matches[0].item_name.as_deref(), Some("Local Dev"));
-
- // URL with different port should still match domain (subdomain matching)
- let staging_matches = filter(credentials.clone(), "https://staging.example.com:8080/dashboard", "");
- assert_eq!(staging_matches.len(), 1);
- assert_eq!(staging_matches[0].item_name.as_deref(), Some("Staging Server"));
-
- // Production URL should match production credential
- let prod_matches = filter(credentials, "https://example.com/app", "");
- assert_eq!(prod_matches.len(), 1);
- assert_eq!(prod_matches[0].item_name.as_deref(), Some("Production"));
-}
-
-/// [#28] - Exact match exclusivity: when exact URL match exists, subdomain matches should be excluded
-#[test]
-fn test_exact_match_excludes_subdomain_matches() {
- let credentials = vec![
- create_test_credential("Exact Match Site", "https://blabla.asd.com", "[email protected]"),
- create_test_credential("Root Domain", "https://asd.com", "[email protected]"),
- create_test_credential("Other Subdomain", "https://bloe.asd.com", "[email protected]"),
- ];
-
- // When visiting blabla.asd.com, only the exact match should be shown
- // NOT the root domain (asd.com) or other subdomains (bloe.asd.com)
- let matches = filter(credentials.clone(), "https://blabla.asd.com/page", "");
- assert_eq!(matches.len(), 1, "Should only return the exact match, not subdomain matches");
- assert_eq!(matches[0].item_name.as_deref(), Some("Exact Match Site"));
-
- // When visiting asd.com (root), only root domain credential should match
- let root_matches = filter(credentials.clone(), "https://asd.com/login", "");
- assert_eq!(root_matches.len(), 1, "Root domain should only match exact root credential");
- assert_eq!(root_matches[0].item_name.as_deref(), Some("Root Domain"));
-
- // When visiting bloe.asd.com, only that subdomain should match
- let bloe_matches = filter(credentials, "https://bloe.asd.com/app", "");
- assert_eq!(bloe_matches.len(), 1, "Subdomain should only match exact subdomain credential");
- assert_eq!(bloe_matches[0].item_name.as_deref(), Some("Other Subdomain"));
-}
-
-/// [#29] - Port-aware matching: exact domain+port beats exact domain beats subdomain
-#[test]
-fn test_port_aware_matching_priority() {
- // Scenario: Self-hosted user with multiple services on same domain but different ports
- let credentials = vec![
- create_test_credential("Service A (Port 8080)", "https://myserver.local:8080", "[email protected]"),
- create_test_credential("Service B (Port 9000)", "https://myserver.local:9000", "[email protected]"),
- create_test_credential("Service C (No Port)", "https://myserver.local", "[email protected]"),
- ];
-
- // When visiting myserver.local:8080, ONLY the port 8080 credential should match
- let port_8080_matches = filter(credentials.clone(), "https://myserver.local:8080/dashboard", "");
- assert_eq!(port_8080_matches.len(), 1, "Should only return exact domain+port match");
- assert_eq!(port_8080_matches[0].item_name.as_deref(), Some("Service A (Port 8080)"));
-
- // When visiting myserver.local:9000, ONLY the port 9000 credential should match
- let port_9000_matches = filter(credentials.clone(), "https://myserver.local:9000/api", "");
- assert_eq!(port_9000_matches.len(), 1, "Should only return exact domain+port match");
- assert_eq!(port_9000_matches[0].item_name.as_deref(), Some("Service B (Port 9000)"));
-
- // When visiting myserver.local (no port), ONLY the no-port credential should match
- let no_port_matches = filter(credentials.clone(), "https://myserver.local/home", "");
- assert_eq!(no_port_matches.len(), 1, "Should only return exact domain match (no port)");
- assert_eq!(no_port_matches[0].item_name.as_deref(), Some("Service C (No Port)"));
-
- // When visiting myserver.local:5000 (port not matching any credential),
- // all credentials with exact domain match (priority 2) should be returned (up to 3)
- // This is expected behavior - when no exact port match exists, we show all domain matches
- let diff_port_matches = filter(credentials, "https://myserver.local:5000/new", "");
- assert_eq!(diff_port_matches.len(), 3, "Should return all domain matches when no exact port match");
-}
-
-/// [#30] - Port numbers in URL should still allow exact matching with subdomain fallback
-#[test]
-fn test_port_url_exact_match_exclusivity() {
- let credentials = vec![
- create_test_credential("Exact Port Site", "https://blabla.asd.com:1234", "[email protected]"),
- create_test_credential("Root Domain", "https://asd.com", "[email protected]"),
- create_test_credential("Other Subdomain", "https://bloe.asd.com", "[email protected]"),
- ];
-
- // When visiting blabla.asd.com:1234, only the exact domain+port match should be shown
- let matches = filter(credentials.clone(), "https://blabla.asd.com:1234/page", "");
- assert_eq!(matches.len(), 1, "Should only return the exact domain+port match");
- assert_eq!(matches[0].item_name.as_deref(), Some("Exact Port Site"));
-
- // When visiting blabla.asd.com (no port), should match via domain-only (priority 2)
- // since we have a credential with same domain (but different port)
- let no_port_matches = filter(credentials, "https://blabla.asd.com/page", "");
- assert_eq!(no_port_matches.len(), 1, "Should match domain-only when port differs");
- assert_eq!(no_port_matches[0].item_name.as_deref(), Some("Exact Port Site"));
-}
-
-/// [#31] - When no exact match exists, subdomain matching should still work
-#[test]
-fn test_subdomain_matching_without_exact() {
- let credentials = vec![
- create_test_credential("Root Domain Only", "https://example.com", "[email protected]"),
- ];
-
- // Visiting a subdomain should match the root domain credential (subdomain matching)
- let subdomain_matches = filter(credentials.clone(), "https://app.example.com/login", "");
- assert_eq!(subdomain_matches.len(), 1, "Subdomain should match root domain when no exact match exists");
- assert_eq!(subdomain_matches[0].item_name.as_deref(), Some("Root Domain Only"));
-
- // Visiting another subdomain should also match
- let another_subdomain = filter(credentials, "https://api.example.com/v1", "");
- assert_eq!(another_subdomain.len(), 1);
- assert_eq!(another_subdomain[0].item_name.as_deref(), Some("Root Domain Only"));
-}
-
-/// [#32] - Multiple credentials with same domain but different ports
-#[test]
-fn test_multiple_same_domain_different_ports() {
- let credentials = vec![
- create_test_credential("Portainer", "https://server.home:9443", "admin@portainer"),
- create_test_credential("Nextcloud", "https://server.home:8443", "admin@nextcloud"),
- create_test_credential("Home Assistant", "https://server.home:8123", "admin@hass"),
- create_test_credential("Main Site", "https://server.home", "admin@main"),
- ];
-
- // Each port should only match its specific credential
- let portainer = filter(credentials.clone(), "https://server.home:9443", "");
- assert_eq!(portainer.len(), 1);
- assert_eq!(portainer[0].item_name.as_deref(), Some("Portainer"));
-
- let nextcloud = filter(credentials.clone(), "https://server.home:8443", "");
- assert_eq!(nextcloud.len(), 1);
- assert_eq!(nextcloud[0].item_name.as_deref(), Some("Nextcloud"));
-
- let hass = filter(credentials.clone(), "https://server.home:8123", "");
- assert_eq!(hass.len(), 1);
- assert_eq!(hass[0].item_name.as_deref(), Some("Home Assistant"));
-
- let main = filter(credentials, "https://server.home", "");
- assert_eq!(main.len(), 1);
- assert_eq!(main[0].item_name.as_deref(), Some("Main Site"));
-}
-
-/// [#33] - User's exact scenario: URL with port should ONLY match exact URL, not items named after domain
-#[test]
-fn test_user_scenario_url_with_port_vs_named_items() {
- // User has:
- // 1. An item with exact URL https://blabla.asd.com:1234
- // 2. Items named "asd.com" and "bloe.asd.com" (possibly with URLs to those domains)
- let credentials = vec![
- create_test_credential("blabla.asd.com service", "https://blabla.asd.com:1234", "[email protected]"),
- create_test_credential("asd.com", "https://asd.com", "[email protected]"),
- create_test_credential("bloe.asd.com", "https://bloe.asd.com", "[email protected]"),
- ];
-
- // When visiting https://blabla.asd.com:1234, ONLY the exact match should be returned
- let matches = filter(credentials.clone(), "https://blabla.asd.com:1234/some/path", "Some Page Title");
-
- // This should return ONLY 1 credential - the exact domain+port match
- assert_eq!(matches.len(), 1, "Should ONLY return the exact domain+port match, not subdomain matches");
- assert_eq!(matches[0].item_name.as_deref(), Some("blabla.asd.com service"));
-
- // Double-check: items with URLs should NOT be matched via title/name
- // because we already have a URL match (which takes priority)
-}
-
-/// [#34] - Items WITHOUT URLs should NOT match when URL match exists
-#[test]
-fn test_items_without_urls_not_matched_when_url_match_exists() {
- let credentials = vec![
- create_test_credential("blabla service", "https://blabla.asd.com:1234", "[email protected]"),
- // These items have NO URLs - they should NOT be matched via title when URL match exists
- create_test_credential("asd", "", "[email protected]"),
- create_test_credential("blabla", "", "[email protected]"),
- ];
-
- // When visiting blabla.asd.com:1234, only the URL match should be returned
- // The items named "asd" and "blabla" should NOT match even though page title might contain those words
- let matches = filter(credentials.clone(), "https://blabla.asd.com:1234/login", "Welcome to blabla asd service");
-
- assert_eq!(matches.len(), 1, "Should only return URL match, not title matches");
- assert_eq!(matches[0].item_name.as_deref(), Some("blabla service"));
-}
-
-/// [#35] - Items WITHOUT URLs should match via URL-derived word matching on item name
-#[test]
-fn test_items_without_urls_matched_by_url_domain_words() {
- let credentials = vec![
- create_test_credential("Test Dumpert", "", "[email protected]"),
- create_test_credential("Some Other Item", "", "[email protected]"),
- ];
-
- // When visiting dumpert.nl with no page title, the credential named "Test Dumpert"
- // should still match because "dumpert" from the URL matches "dumpert" in the item name
- let matches = filter(credentials, "https://www.dumpert.nl", "");
-
- assert_eq!(matches.len(), 1, "Should match credential by URL-derived word against item name");
- assert_eq!(matches[0].item_name.as_deref(), Some("Test Dumpert"));
-}
-
-/// [#36] - URL-derived word matching should NOT match credentials that have URLs defined
-#[test]
-fn test_url_word_matching_skips_credentials_with_urls() {
- let credentials = vec![
- // This credential HAS a URL (different domain) - should NOT match via name
- create_test_credential("Test Dumpert", "https://other-site.com", "[email protected]"),
- // This credential has NO URL - should match via name
- create_test_credential("Dumpert Account", "", "[email protected]"),
- ];
-
- let matches = filter(credentials, "https://www.dumpert.nl", "");
-
- assert_eq!(matches.len(), 1, "Should only match the credential without URLs");
- assert_eq!(matches[0].item_name.as_deref(), Some("Dumpert Account"));
-}
-
-/// Helper to filter with ignore_port flag (for Android-style matching)
-fn filter_ignore_port(credentials: Vec<Credential>, current_url: &str, page_title: &str) -> Vec<Credential> {
- let input = CredentialMatcherInput {
- credentials: credentials.clone(),
- current_url: current_url.to_string(),
- page_title: page_title.to_string(),
- matching_mode: AutofillMatchingMode::Default,
- ignore_port: true,
- max_results: None,
- };
- let output = filter_credentials(input);
-
- output.matched_ids
- .iter()
- .filter_map(|id| credentials.iter().find(|c| c.id == *id).cloned())
- .collect()
-}
-
-/// [#37] - ignore_port flag: Android scenario where port info is unavailable
-/// When Android sends URL without port, it should still match credentials with ports
-#[test]
-fn test_ignore_port_android_scenario() {
- let credentials = vec![
- create_test_credential("Service A (Port 8080)", "https://myserver.local:8080", "admin@a"),
- create_test_credential("Service B (Port 9000)", "https://myserver.local:9000", "admin@b"),
- create_test_credential("Service C (No Port)", "https://myserver.local", "admin@c"),
- ];
-
- // Without ignore_port: visiting myserver.local (no port) only matches the no-port credential
- let normal_matches = filter(credentials.clone(), "https://myserver.local/home", "");
- assert_eq!(normal_matches.len(), 1, "Normal matching should only return no-port credential");
- assert_eq!(normal_matches[0].item_name.as_deref(), Some("Service C (No Port)"));
-
- // With ignore_port: visiting myserver.local (no port) matches ALL credentials with same domain
- // This simulates Android's autofill where port info is stripped by the OS
- let android_matches = filter_ignore_port(credentials.clone(), "https://myserver.local/home", "");
- assert_eq!(android_matches.len(), 3, "ignore_port should match all domain credentials regardless of port");
-}
-
-/// [#38] - ignore_port flag: IP address with port scenario
-/// Common for self-hosted services accessed via IP:port
-#[test]
-fn test_ignore_port_ip_address_scenario() {
- let credentials = vec![
- create_test_credential("Home Assistant", "https://192.168.1.100:8123", "admin@ha"),
- create_test_credential("Portainer", "https://192.168.1.100:9443", "admin@portainer"),
- create_test_credential("Router Admin", "https://192.168.1.1", "admin@router"),
- ];
-
- // Android sends just the IP without port
- let matches = filter_ignore_port(credentials.clone(), "https://192.168.1.100", "");
-
- // Should match both services on 192.168.1.100, regardless of their stored ports
- assert_eq!(matches.len(), 2, "Should match all credentials for the IP address");
-
- // Verify it doesn't match the different IP
- let router_matches = filter_ignore_port(credentials.clone(), "https://192.168.1.1", "");
- assert_eq!(router_matches.len(), 1);
- assert_eq!(router_matches[0].item_name.as_deref(), Some("Router Admin"));
-}
-
-/// [#39] - ignore_port should not affect subdomain matching
-#[test]
-fn test_ignore_port_with_subdomain_matching() {
- let credentials = vec![
- create_test_credential("App Portal", "https://app.example.com:8080", "user@app"),
- create_test_credential("Main Site", "https://example.com", "user@main"),
- ];
-
- // With ignore_port, subdomain matching should still work
- let matches = filter_ignore_port(credentials.clone(), "https://api.example.com", "");
-
- // Should match both via subdomain/root domain matching
- assert_eq!(matches.len(), 2, "Subdomain matching should still work with ignore_port");
-}
-
-/// [#40] - ignore_port flag: IP address with multiple ports and one without port
-/// Tests the exact scenario: 4 credentials on same IP (3 with ports, 1 without)
-/// Without flag: visiting IP without port should only match the no-port credential
-/// With flag: visiting IP without port should match all 4 credentials
-#[test]
-fn test_ignore_port_ip_with_multiple_ports_and_no_port() {
- let credentials = vec![
- create_test_credential("Service on 5000", "https://192.168.1.10:5000", "admin@5000"),
- create_test_credential("Service on 6000", "https://192.168.1.10:6000", "admin@6000"),
- create_test_credential("Service on 7000", "https://192.168.1.10:7000", "admin@7000"),
- create_test_credential("Service no port", "https://192.168.1.10", "admin@noport"),
- ];
-
- // WITHOUT ignore_port flag (normal browser behavior):
- // Visiting 192.168.1.10 (no port) should ONLY match the credential without port
- // because we have an exact domain+port match (both have no port)
- let normal_matches = filter(credentials.clone(), "https://192.168.1.10", "");
- assert_eq!(normal_matches.len(), 1, "Without ignore_port, should only match the no-port credential");
- assert_eq!(normal_matches[0].item_name.as_deref(), Some("Service no port"));
-
- // WITH ignore_port flag (Android behavior where port is stripped):
- // Visiting 192.168.1.10 (no port) should match ALL 4 credentials
- // because we ignore port differences entirely
- let android_matches = filter_ignore_port(credentials.clone(), "https://192.168.1.10", "");
- assert_eq!(android_matches.len(), 4, "With ignore_port, should match all 4 credentials");
-
- // Sanity-check with a smaller set that all match
- let small_credentials = vec![
- create_test_credential("Service on 5000", "https://192.168.1.10:5000", "admin@5000"),
- create_test_credential("Service no port", "https://192.168.1.10", "admin@noport"),
- ];
-
- let small_normal = filter(small_credentials.clone(), "https://192.168.1.10", "");
- assert_eq!(small_normal.len(), 1, "Without ignore_port, only no-port matches");
- assert_eq!(small_normal[0].item_name.as_deref(), Some("Service no port"));
-
- let small_android = filter_ignore_port(small_credentials.clone(), "https://192.168.1.10", "");
- assert_eq!(small_android.len(), 2, "With ignore_port, both credentials match");
-}
-
-/// [#41] - Localhost matching with ports (development scenarios)
-/// Common scenario: developers running multiple services on localhost with different ports
-#[test]
-fn test_localhost_matching_with_ports() {
- let credentials = vec![
- create_test_credential("Local API", "http://localhost:3000", "dev@api"),
- create_test_credential("Local Frontend", "http://localhost:8080", "dev@frontend"),
- create_test_credential("Local Backend", "http://localhost:81", "dev@backend"),
- create_test_credential("Local No Port", "http://localhost", "dev@default"),
- ];
-
- // Visiting localhost:81 should only match the :81 credential
- let matches_81 = filter(credentials.clone(), "http://localhost:81", "");
- assert_eq!(matches_81.len(), 1, "Should only return exact localhost:81 match");
- assert_eq!(matches_81[0].item_name.as_deref(), Some("Local Backend"));
-
- // Visiting localhost:3000 should only match the :3000 credential
- let matches_3000 = filter(credentials.clone(), "http://localhost:3000", "");
- assert_eq!(matches_3000.len(), 1, "Should only return exact localhost:3000 match");
- assert_eq!(matches_3000[0].item_name.as_deref(), Some("Local API"));
-
- // Visiting localhost:8080 should only match the :8080 credential
- let matches_8080 = filter(credentials.clone(), "http://localhost:8080", "");
- assert_eq!(matches_8080.len(), 1, "Should only return exact localhost:8080 match");
- assert_eq!(matches_8080[0].item_name.as_deref(), Some("Local Frontend"));
-
- // Visiting localhost (no port) should only match the no-port credential
- let matches_no_port = filter(credentials.clone(), "http://localhost", "");
- assert_eq!(matches_no_port.len(), 1, "Should only return localhost without port match");
- assert_eq!(matches_no_port[0].item_name.as_deref(), Some("Local No Port"));
-
- // Visiting localhost with a port not in credentials should show all domain matches
- let matches_unknown_port = filter(credentials.clone(), "http://localhost:5000", "");
- assert_eq!(matches_unknown_port.len(), 4, "Should return all domain matches when no exact port match");
-}
-
-/// [#42] - Localhost matching - exact URL stored
-/// User's specific scenario: credential with exactly "http://localhost:81" should match
-#[test]
-fn test_localhost_exact_url_stored() {
- let credentials = vec![
- create_test_credential("My Local Service", "http://localhost:81", "user@local"),
- ];
-
- // Should match when visiting exactly http://localhost:81
- let matches = filter(credentials.clone(), "http://localhost:81", "");
- assert_eq!(matches.len(), 1, "Should match exact localhost:81 URL");
- assert_eq!(matches[0].item_name.as_deref(), Some("My Local Service"));
-
- // Should also match with path
- let matches_with_path = filter(credentials.clone(), "http://localhost:81/some/path", "");
- assert_eq!(matches_with_path.len(), 1, "Should match localhost:81 with path");
-
- // Should also match with query string
- let matches_with_query = filter(credentials.clone(), "http://localhost:81?debug=true", "");
- assert_eq!(matches_with_query.len(), 1, "Should match localhost:81 with query");
-}
-
-/// [#43] - Single-word hostname matching (homelab/self-hosted scenarios)
-/// Common scenario: self-hosted services with local DNS or /etc/hosts entries
-/// like "plex", "nas", "router", "homeassistant", etc.
-#[test]
-fn test_single_word_hostname_matching() {
- let credentials = vec![
- create_test_credential("Plex Media Server", "http://plex:32400", "admin@plex"),
- create_test_credential("Synology NAS", "https://nas:5001", "admin@nas"),
- create_test_credential("Pi-hole", "http://pihole/admin", "admin@pihole"),
- create_test_credential("Home Assistant", "http://homeassistant:8123", "admin@ha"),
- create_test_credential("Router Admin", "http://router", "admin@router"),
- ];
-
- // Plex with exact port
- let plex_matches = filter(credentials.clone(), "http://plex:32400", "");
- assert_eq!(plex_matches.len(), 1, "Should match plex:32400");
- assert_eq!(plex_matches[0].item_name.as_deref(), Some("Plex Media Server"));
-
- // Plex with path
- let plex_path = filter(credentials.clone(), "http://plex:32400/web/index.html", "");
- assert_eq!(plex_path.len(), 1, "Should match plex:32400 with path");
-
- // NAS with HTTPS
- let nas_matches = filter(credentials.clone(), "https://nas:5001", "");
- assert_eq!(nas_matches.len(), 1, "Should match nas:5001");
- assert_eq!(nas_matches[0].item_name.as_deref(), Some("Synology NAS"));
-
- // Pi-hole with path
- let pihole_matches = filter(credentials.clone(), "http://pihole/admin/index.php", "");
- assert_eq!(pihole_matches.len(), 1, "Should match pihole with path");
- assert_eq!(pihole_matches[0].item_name.as_deref(), Some("Pi-hole"));
-
- // Home Assistant
- let ha_matches = filter(credentials.clone(), "http://homeassistant:8123/dashboard", "");
- assert_eq!(ha_matches.len(), 1, "Should match homeassistant:8123");
- assert_eq!(ha_matches[0].item_name.as_deref(), Some("Home Assistant"));
-
- // Router without port
- let router_matches = filter(credentials.clone(), "http://router", "");
- assert_eq!(router_matches.len(), 1, "Should match router without port");
- assert_eq!(router_matches[0].item_name.as_deref(), Some("Router Admin"));
-
- // Router with different port should still match via domain
- let router_port = filter(credentials.clone(), "http://router:8080", "");
- assert_eq!(router_port.len(), 1, "Should match router with different port via domain match");
+ let output = filter_credentials(input(&[credential("Two sites", &["https://app.example.com", "https://example.org"], "")], "https://example.org", ""));
+ assert_eq!((output.matched_ids.len(), output.matched_priority), (1, 2), "an exact match on the second URL is a URL match");
}
-/// [#44] - Single-word hostnames require protocol for extraction
-/// Credentials stored WITH protocol should match current URLs with protocol.
-/// Note: Browser always sends URLs with protocol, so this mainly affects
-/// what URLs can be stored in credentials.
#[test]
-fn test_single_word_hostname_extraction_requires_protocol() {
- // Credential stored WITHOUT protocol - should NOT be matchable
- let credentials_no_protocol = vec![
- create_test_credential("Plex No Protocol", "plex:32400", "admin@plex"),
+fn the_best_domain_rank_wins_and_excludes_the_rest() {
+ // Exact domain and port beats exact domain beats subdomain or root domain; only the best rank is offered.
+ let services = vec![
+ credential("Service A (Port 8080)", &["https://myserver.local:8080"], ""),
+ credential("Service B (Port 9000)", &["https://myserver.local:9000"], ""),
+ credential("Service C (No Port)", &["https://myserver.local"], ""),
];
+ assert_eq!(names(&services, "https://myserver.local:8080/dashboard", ""), vec!["Service A (Port 8080)"]);
+ assert_eq!(names(&services, "https://myserver.local/home", ""), vec!["Service C (No Port)"]);
+ assert_eq!(names(&services, "https://myserver.local:5000/new", "").len(), 3, "no port matches, so every credential on the domain is offered");
- // Even with protocol in current URL, credential without protocol won't match
- // because the credential URL "plex:32400" can't be extracted as a domain
- let no_match = filter(credentials_no_protocol.clone(), "http://plex:32400", "");
- assert_eq!(no_match.len(), 0, "Credential without protocol should not be matchable");
+ let asd = vec![credential("Exact Match Site", &["https://blabla.asd.com:1234"], ""), credential("Root Domain", &["https://asd.com"], ""), credential("Other Subdomain", &["https://bloe.asd.com"], "")];
+ assert_eq!(names(&asd, "https://blabla.asd.com:1234/page", "Some Page Title"), vec!["Exact Match Site"]);
+ assert_eq!(names(&asd, "https://blabla.asd.com/page", ""), vec!["Exact Match Site"], "the same domain on another port still beats the root domain");
+ assert_eq!(names(&asd, "https://asd.com/login", ""), vec!["Root Domain"]);
+ assert_eq!(names(&asd, "https://bloe.asd.com/app", ""), vec!["Other Subdomain"]);
- // Credential stored WITH protocol - should match
- let credentials_with_protocol = vec![
- create_test_credential("Plex With Protocol", "http://plex:32400", "admin@plex"),
- ];
-
- let with_match = filter(credentials_with_protocol.clone(), "http://plex:32400", "");
- assert_eq!(with_match.len(), 1, "Credential with protocol should match");
+ let root_only = vec![credential("Root Domain Only", &["https://example.com"], "")];
+ assert_eq!(names(&root_only, "https://app.example.com/login", ""), vec!["Root Domain Only"], "without an exact match a subdomain matches the root domain credential");
}
-/// [#2299] - Whitespace around a stored URL or package name must not prevent a match
#[test]
-fn test_stored_urls_are_trimmed() {
+fn single_word_hosts_and_localhost_match_with_their_ports() {
let credentials = vec![
- create_test_credential_multi_url("Amazon", vec!["https://amazon.fr/", " https://amazon.de", "amazon.it\t", " com.amazon.app \n"], "[email protected]"),
+ credential("Plex Media Server", &["http://plex:32400"], ""),
+ credential("Pi-hole", &["http://pihole/admin"], ""),
+ credential("Router Admin", &["http://router"], ""),
+ credential("Local API", &["http://localhost:3000"], ""),
+ credential("Local Backend", &["http://localhost:81"], ""),
+ credential("Local No Port", &["http://localhost"], ""),
+ credential("Plex No Protocol", &["plex:32400"], ""),
];
-
- assert_eq!(filter(credentials.clone(), "https://www.amazon.de/ap/signin", "").len(), 1, "Leading space should be ignored");
- assert_eq!(filter(credentials.clone(), "https://amazon.it", "").len(), 1, "Trailing whitespace should be ignored");
- assert_eq!(filter(credentials, "com.amazon.app", "").len(), 1, "Package name should match when padded");
-}
-
-/// Runs the matcher in every matching mode and returns the matched ids per mode.
-fn filter_all_modes(credentials: &[Credential], current_url: &str) -> Vec<(AutofillMatchingMode, CredentialMatcherOutput)> {
- [AutofillMatchingMode::Default, AutofillMatchingMode::UrlSubdomain, AutofillMatchingMode::UrlExact]
- .into_iter()
- .map(|mode| {
- let input = CredentialMatcherInput {
- credentials: credentials.to_vec(),
- current_url: current_url.to_string(),
- page_title: String::new(),
- matching_mode: mode,
- ignore_port: false,
- max_results: None,
- };
- (mode, filter_credentials(input))
- })
- .collect()
+ for (url, expected) in [
+ ("http://plex:32400/web/index.html", vec!["Plex Media Server"]),
+ ("http://pihole/admin/index.php", vec!["Pi-hole"]),
+ ("http://router", vec!["Router Admin"]),
+ ("http://router:8080", vec!["Router Admin"]),
+ ("http://localhost:81?debug=true", vec!["Local Backend"]),
+ ("http://localhost:3000", vec!["Local API"]),
+ ("http://localhost", vec!["Local No Port"]),
+ ("http://localhost:5000", vec!["Local API", "Local Backend", "Local No Port"]),
+ ] {
+ assert_eq!(names(&credentials, url, ""), expected, "{url}");
+ }
+ assert!(names(&credentials[6..], "http://plex:32400", "").is_empty(), "a stored single-word host needs its protocol to be extractable");
}
-/// [#45] - A URL whose host fails validation must not fall through to name matching (phishing autofill)
#[test]
-fn test_unusual_hosts_never_fall_through_to_name_matching() {
- let credentials = vec![
- create_test_credential("PayPal", "https://paypal.com", "[email protected]"),
- create_test_credential("PayPal Notes", "", ""),
+fn ignoring_the_port_treats_every_port_of_a_domain_alike() {
+ // Android autofill does not know the port, so a bare host has to offer the credentials stored with one.
+ let ignore_port = |credentials: &[Credential], url: &str| names_for(credentials, CredentialMatcherInput { ignore_port: true, ..input(credentials, url, "") });
+ let ip = vec![
+ credential("Service on 5000", &["https://192.168.1.10:5000"], ""),
+ credential("Service on 6000", &["https://192.168.1.10:6000"], ""),
+ credential("Service no port", &["https://192.168.1.10"], ""),
+ credential("Router Admin", &["https://192.168.1.1"], ""),
];
-
+ assert_eq!(names(&ip, "https://192.168.1.10", ""), vec!["Service no port"]);
+ assert_eq!(ignore_port(&ip, "https://192.168.1.10"), vec!["Service on 5000", "Service on 6000", "Service no port"]);
+ assert_eq!(ignore_port(&ip, "https://192.168.1.1"), vec!["Router Admin"]);
+
+ let sub = vec![credential("App Portal", &["https://app.example.com:8080"], ""), credential("Main Site", &["https://example.com"], "")];
+ assert_eq!(ignore_port(&sub, "https://api.example.com").len(), 2, "subdomain matching still applies");
+}
+
+#[test]
+fn unusual_hosts_are_normalized_and_never_fall_through_to_name_matching() {
+ let credentials = vec![credential("PayPal", &["https://paypal.com"], "victim"), credential("PayPal Notes", &[], ""), credential("Intranet", &["https://my_host.corp.example"], ""), credential("Local IPv6", &["http://[::1]:8080"], ""), credential("Evil", &["https://evil.com"], "")];
+ for (url, expected) in [
+ ("https://paypal.com./login", vec!["PayPal"]),
+ ("https://www.paypal.com./", vec!["PayPal"]),
+ ("blob:https://www.paypal.com/0b2c3d4e-0000-4000-8000-000000000000", vec!["PayPal"]),
+ ("https://my_host.corp.example/login", vec!["Intranet"]),
+ ("http://[::1]:8080/login", vec!["Local IPv6"]),
+ ("http://[::2]:8080/login", vec![]),
+ // The host is what follows the userinfo: the page is evil.com, not paypal.com.
+ ("https://[email protected]/", vec!["Evil"]),
+ ("https://paypal.com:[email protected]/", vec!["Evil"]),
+ ] {
+ assert_eq!(names(&credentials, url, ""), expected, "{url}");
+ }
+ // URLs that match no credential never fall through to name matching, in no mode.
for phishing_url in [
"https://www_paypal.evil.com/",
"https://paypal.evil.com./",
@@ -1120,51 +254,26 @@ fn test_unusual_hosts_never_fall_through_to_name_matching() {
"filesystem:https://paypal.evil.com/temporary/paypal",
"data:text/html,paypal",
] {
- for (mode, output) in filter_all_modes(&credentials, phishing_url) {
+ for (mode, output) in all_modes(&credentials[..2], phishing_url) {
assert!(output.matched_ids.is_empty(), "{phishing_url} in {mode:?} mode must not match: {output:?}");
}
}
}
-/// [#46] - An app package without a linked credential never gets a website credential through its name
#[test]
-fn test_unlinked_app_package_only_matches_credentials_without_urls() {
- let paypal_web = create_test_credential("PayPal", "https://paypal.com", "[email protected]");
- let paypal_note = create_test_credential("PayPal Notes", "", "");
- let credentials = vec![paypal_web.clone(), paypal_note.clone()];
-
- for package in ["com.fake.paypal", "com.paypal.rewards", "net.paypal.cashback"] {
- let output = filter_all_modes(&credentials, package);
- for (mode, output) in output {
- assert_eq!(output.matched_ids, vec![paypal_note.id.clone()], "{package} in {mode:?} mode");
- assert_eq!(output.matched_priority, 4);
- }
- }
-
- // Without a credential lacking URLs there is nothing left to offer.
- let matches = filter(vec![paypal_web], "com.paypal.rewards", "PayPal Rewards");
- assert!(matches.is_empty());
+fn results_are_capped_at_ten_by_default_or_at_the_requested_count() {
+ let credentials: Vec<Credential> = (1..=12).map(|i| credential(&format!("Site {i}"), &[&format!("sub{i}.example.com")], "")).collect();
+ assert_eq!(names(&credentials, "https://example.com", "").len(), 10);
+ assert_eq!(filter_credentials(CredentialMatcherInput { max_results: Some(2), ..input(&credentials, "https://example.com", "") }).matched_ids.len(), 2);
}
-/// [#47] - Hosts are normalized before matching, so legitimate pages in those shapes still match their own credential
#[test]
-fn test_unusual_hosts_match_their_own_credential() {
- let credentials = vec![
- create_test_credential("PayPal", "https://paypal.com", "[email protected]"),
- create_test_credential("Intranet", "https://my_host.corp.example", "[email protected]"),
- create_test_credential("Local IPv6", "http://[::1]:8080", "admin"),
- create_test_credential("Evil", "https://evil.com", "attacker"),
- ];
- let name_of = |url: &str| filter(credentials.clone(), url, "").into_iter().map(|c| c.item_name.unwrap_or_default()).collect::<Vec<_>>();
-
- assert_eq!(name_of("https://paypal.com./login"), vec!["PayPal"]);
- assert_eq!(name_of("https://www.paypal.com./"), vec!["PayPal"]);
- assert_eq!(name_of("blob:https://www.paypal.com/0b2c3d4e-0000-4000-8000-000000000000"), vec!["PayPal"]);
- assert_eq!(name_of("https://my_host.corp.example/login"), vec!["Intranet"]);
- assert_eq!(name_of("http://[::1]:8080/login"), vec!["Local IPv6"]);
- assert!(name_of("http://[::2]:8080/login").is_empty());
-
- // The host is what follows the userinfo: the page is evil.com, not paypal.com.
- assert_eq!(name_of("https://[email protected]/"), vec!["Evil"]);
- assert_eq!(name_of("https://paypal.com:[email protected]/"), vec!["Evil"]);
+fn the_json_boundary_uses_camel_case() {
+ let credentials = shared_credentials();
+ let json = serde_json::to_string(&input(&credentials, "https://github.com", "")).unwrap();
+ assert!(json.contains("\"currentUrl\"") && json.contains("\"matchingMode\""));
+ let output_json = filter_credentials_json(&json).unwrap();
+ assert!(output_json.contains("\"matchedIds\""));
+ let output: CredentialMatcherOutput = serde_json::from_str(&output_json).unwrap();
+ assert_eq!(credentials.iter().find(|c| c.id == output.matched_ids[0]).unwrap().item_name.as_deref(), Some("GitHub"));
}
diff --git a/core/rust/src/crypto/aad.rs b/core/rust/src/crypto/aad.rs
index 71c24d574..1cb457062 100644
--- a/core/rust/src/crypto/aad.rs
+++ b/core/rust/src/crypto/aad.rs
@@ -52,7 +52,8 @@ pub fn grant(manifest_id: &str) -> Vec<u8> {
format!("{}/grant/{}", PREFIX, normalize(manifest_id)).into_bytes()
}
-fn normalize(id: &str) -> String {
+/// An id as it appears in a label: lowercased, as the server prints a GUID.
+pub(crate) fn normalize(id: &str) -> String {
id.to_ascii_lowercase()
}
diff --git a/core/rust/src/crypto/key_chain.rs b/core/rust/src/crypto/key_chain.rs
index 73c387e96..3f97e72a4 100644
--- a/core/rust/src/crypto/key_chain.rs
+++ b/core/rust/src/crypto/key_chain.rs
@@ -142,8 +142,7 @@ pub fn open_account_private_key(encrypted_account_private_key: &str, account_key
/// Decrypt the account signing private key (base64 seed) with the Account Key.
pub fn open_account_signing_private_key(encrypted_signing_private_key: &str, account_key_base64: &str) -> VaultResult<Zeroizing<String>> {
- let seed = Zeroizing::new(symmetric_decrypt_bytes_with_aad(&base64_decode(encrypted_signing_private_key)?, account_key_base64, aad::ACCOUNT_SIGNING_PRIVATE_KEY)?);
- Ok(Zeroizing::new(base64_encode(&seed[..])))
+ unwrap_key(encrypted_signing_private_key, account_key_base64, aad::ACCOUNT_SIGNING_PRIVATE_KEY)
}
/// Why a key chain did not open.
diff --git a/core/rust/src/crypto/signing.rs b/core/rust/src/crypto/signing.rs
index 28aa563c8..cdcaa5447 100644
--- a/core/rust/src/crypto/signing.rs
+++ b/core/rust/src/crypto/signing.rs
@@ -11,6 +11,7 @@ use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing};
use crate::common::encoding::{base64_decode, base64_encode};
use crate::common::error::{VaultError, VaultResult};
use crate::common::rng::fill_random;
+use crate::crypto::aad::normalize;
/// The label of a user's signature over their own account encryption public key (the RSA JWK).
pub const ACCOUNT_PUBLIC_KEY_LABEL: &str = "aliasvault/v1/sig/account-public-key";
@@ -64,19 +65,19 @@ pub fn account_public_key_message(account_public_key: &str) -> Vec<u8> {
/// The signed message of a grant: version `key_version` of the manifest's VEK, encrypted for `recipient_public_key` (JWK,
/// as the server stores it) and handed out by `signer_user_id`.
pub fn grant_message(manifest_id: &str, key_version: i64, signer_user_id: &str, recipient_public_key: &str, algorithm: &str, encrypted_vek: &str) -> Vec<u8> {
- let manifest_id = manifest_id.to_ascii_lowercase();
+ let manifest_id = normalize(manifest_id);
let key_version = key_version.to_string();
signed_message(GRANT_LABEL, &[manifest_id.as_bytes(), key_version.as_bytes(), signer_user_id.as_bytes(), recipient_public_key.as_bytes(), algorithm.as_bytes(), encrypted_vek.as_bytes()])
}
/// The signed message of the vault name an invitation carries, encrypted for `recipient_public_key`.
pub fn invitation_name_message(manifest_id: &str, signer_user_id: &str, recipient_public_key: &str, encrypted_name: &str) -> Vec<u8> {
- signed_message(INVITATION_NAME_LABEL, &[manifest_id.to_ascii_lowercase().as_bytes(), signer_user_id.as_bytes(), recipient_public_key.as_bytes(), encrypted_name.as_bytes()])
+ signed_message(INVITATION_NAME_LABEL, &[normalize(manifest_id).as_bytes(), signer_user_id.as_bytes(), recipient_public_key.as_bytes(), encrypted_name.as_bytes()])
}
/// The signed message of a delivery key publish, bound to the revision the write is based on so it cannot be replayed.
pub fn delivery_key_message(manifest_id: &str, public_key: &str, current_revision: i64) -> Vec<u8> {
- signed_message(DELIVERY_KEY_LABEL, &[manifest_id.to_ascii_lowercase().as_bytes(), public_key.as_bytes(), current_revision.to_string().as_bytes()])
+ signed_message(DELIVERY_KEY_LABEL, &[normalize(manifest_id).as_bytes(), public_key.as_bytes(), current_revision.to_string().as_bytes()])
}
/// Sign a message with a base64 private seed. Returns the base64 signature.
diff --git a/core/rust/src/crypto/srp.rs b/core/rust/src/crypto/srp.rs
index 8b287e9f8..597f00ea8 100644
--- a/core/rust/src/crypto/srp.rs
+++ b/core/rust/src/crypto/srp.rs
@@ -4,10 +4,10 @@
//! where the format must stay compatible with the `SecureRemotePassword` .NET package
//! used by the API server and the `secure-remote-password` library used in the JS test suite.
-use digest::Digest;
use num_bigint::BigUint;
use serde::{Deserialize, Serialize};
-use sha2::Sha256;
+use sha2::{Digest, Sha256};
+use std::fmt;
use srp::client::SrpClient;
use srp::groups::G_2048;
use srp::server::SrpServer;
@@ -22,7 +22,7 @@ use crate::common::rng::fill_random;
const N_BYTES: usize = 256;
/// SRP ephemeral key pair (public and secret values).
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "uniffi", derive(uniffi::Record))]
pub struct SrpEphemeral {
pub public: String,
@@ -30,13 +30,25 @@ pub struct SrpEphemeral {
}
/// SRP session containing proof and shared key.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "uniffi", derive(uniffi::Record))]
pub struct SrpSession {
pub proof: String,
pub key: String,
}
+impl fmt::Debug for SrpEphemeral {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("SrpEphemeral").field("public", &self.public).field("secret", &"<redacted>").finish()
+ }
+}
+
+impl fmt::Debug for SrpSession {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("SrpSession").field("proof", &self.proof).field("key", &"<redacted>").finish()
+ }
+}
+
/// SRP-related errors.
#[derive(Error, Debug, Clone)]
#[cfg_attr(feature = "uniffi", derive(uniffi::Error))]
@@ -50,10 +62,6 @@ pub enum SrpError {
AuthenticationFailed(String),
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Hex / Byte Helpers
-// ═══════════════════════════════════════════════════════════════════════════════
-
/// Decode a hex parameter (either case, optional 0x prefix).
fn hex_to_bytes(hex: &str) -> Result<Vec<u8>, SrpError> {
let hex = hex.trim();
@@ -87,10 +95,6 @@ fn to_padded_bytes(value: &BigUint) -> Vec<u8> {
pad_to_length(value.to_bytes_be(), N_BYTES)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Client Operations
-// ═══════════════════════════════════════════════════════════════════════════════
-
/// A random 32-byte SRP salt as an uppercase hex string.
pub fn srp_generate_salt() -> String {
bytes_to_hex(&generate_random_bytes(32))
@@ -98,20 +102,15 @@ pub fn srp_generate_salt() -> String {
/// The SRP private key `x = H(salt | H(identity | ":" | password_hash))` as uppercase hex, from hex inputs.
/// The full 32-byte digest is returned (not via BigUint, which would strip leading zero bytes).
-pub fn srp_derive_private_key(
- salt: &str,
- identity: &str,
- password_hash: &str,
-) -> Result<String, SrpError> {
+pub fn srp_derive_private_key(salt: &str, identity: &str, password_hash: &str) -> Result<String, SrpError> {
let identity = identity.to_lowercase();
let salt_bytes = hex_to_bytes(salt)?;
- let identity_hash =
- SrpClient::<Sha256>::compute_identity_hash(identity.as_bytes(), password_hash.as_bytes());
+ let identity_hash = SrpClient::<Sha256>::compute_identity_hash(identity.as_bytes(), password_hash.as_bytes());
let mut x_hasher = Sha256::new();
x_hasher.update(&salt_bytes);
- x_hasher.update(&identity_hash);
+ x_hasher.update(identity_hash);
Ok(bytes_to_hex(&x_hasher.finalize()))
}
@@ -133,20 +132,11 @@ pub fn srp_generate_ephemeral() -> SrpEphemeral {
let a = generate_random_bytes(64);
let a_pub = client.compute_public_ephemeral(&a);
- SrpEphemeral {
- public: bytes_to_hex(&pad_to_length(a_pub, N_BYTES)),
- secret: bytes_to_hex(&a),
- }
+ SrpEphemeral { public: bytes_to_hex(&pad_to_length(a_pub, N_BYTES)), secret: bytes_to_hex(&a) }
}
/// The client session (key `K` and proof `M1`) from the server's public ephemeral `B`; hex in, hex out.
-pub fn srp_derive_session(
- client_secret: &str,
- server_public: &str,
- salt: &str,
- identity: &str,
- private_key: &str,
-) -> Result<SrpSession, SrpError> {
+pub fn srp_derive_session(client_secret: &str, server_public: &str, salt: &str, identity: &str, private_key: &str) -> Result<SrpSession, SrpError> {
let identity = identity.to_lowercase();
let a = BigUint::from_bytes_be(&hex_to_bytes(client_secret)?);
let b_pub = BigUint::from_bytes_be(&hex_to_bytes(server_public)?);
@@ -155,9 +145,7 @@ pub fn srp_derive_session(
// Safeguard against malicious B (B mod N must not be 0)
if &b_pub % &G_2048.n == BigUint::default() {
- return Err(SrpError::InvalidParameter(
- "server public ephemeral is invalid".to_string(),
- ));
+ return Err(SrpError::InvalidParameter("server public ephemeral is invalid".to_string()));
}
let client = SrpClient::<Sha256>::new(&G_2048);
@@ -167,6 +155,9 @@ pub fn srp_derive_session(
let b_pub_bytes = to_padded_bytes(&b_pub);
let u = compute_u::<Sha256>(&a_pub_bytes, &b_pub_bytes);
+ if u == BigUint::default() {
+ return Err(SrpError::InvalidParameter("scrambling parameter u is zero".to_string()));
+ }
let k = compute_k::<Sha256>(&G_2048);
// S = (B - k*g^x)^(a + u*x) mod N
@@ -175,19 +166,11 @@ pub fn srp_derive_session(
let key = derive_session_key(&s);
let m1 = compute_m1(&a_pub_bytes, &b_pub_bytes, &salt_bytes, &identity, &key);
- Ok(SrpSession {
- proof: bytes_to_hex(&m1),
- key: bytes_to_hex(&key),
- })
+ Ok(SrpSession { proof: bytes_to_hex(&m1), key: bytes_to_hex(&key) })
}
/// Whether the server's proof `M2` matches, which confirms it derived the same session key.
-pub fn srp_verify_session(
- client_public: &str,
- client_proof: &str,
- session_key: &str,
- server_proof: &str,
-) -> Result<bool, SrpError> {
+pub fn srp_verify_session(client_public: &str, client_proof: &str, session_key: &str, server_proof: &str) -> Result<bool, SrpError> {
let a_pub_bytes = hex_to_bytes(client_public)?;
let m1_bytes = hex_to_bytes(client_proof)?;
let key_bytes = hex_to_bytes(session_key)?;
@@ -198,9 +181,7 @@ pub fn srp_verify_session(
Ok(expected_m2.ct_eq(&server_m2_bytes).unwrap_u8() == 1)
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Server Operations
-// ═══════════════════════════════════════════════════════════════════════════════
+// Server side of the handshake.
/// A server ephemeral pair: `B = k*v + g^b mod N` for a random 64-byte secret `b`, from the hex verifier.
pub fn srp_generate_ephemeral_server(verifier: &str) -> Result<SrpEphemeral, SrpError> {
@@ -211,10 +192,7 @@ pub fn srp_generate_ephemeral_server(verifier: &str) -> Result<SrpEphemeral, Srp
let b = generate_random_bytes(64);
let b_pub = server.compute_public_ephemeral(&b, &v_bytes);
- Ok(SrpEphemeral {
- public: bytes_to_hex(&pad_to_length(b_pub, N_BYTES)),
- secret: bytes_to_hex(&b),
- })
+ Ok(SrpEphemeral { public: bytes_to_hex(&pad_to_length(b_pub, N_BYTES)), secret: bytes_to_hex(&b) })
}
/// The server session (key `K` and proof `M2`) once the client's proof `M1` verifies; `None` when it does not.
@@ -235,9 +213,7 @@ pub fn srp_derive_session_server(
// Safeguard against malicious A (A mod N must not be 0)
if &a_pub % &G_2048.n == BigUint::default() {
- return Err(SrpError::InvalidParameter(
- "client public ephemeral is invalid".to_string(),
- ));
+ return Err(SrpError::InvalidParameter("client public ephemeral is invalid".to_string()));
}
let server = SrpServer::<Sha256>::new(&G_2048);
@@ -250,6 +226,9 @@ pub fn srp_derive_session_server(
let b_pub_bytes = to_padded_bytes(&b_pub);
let u = compute_u::<Sha256>(&a_pub_bytes, &b_pub_bytes);
+ if u == BigUint::default() {
+ return Err(SrpError::InvalidParameter("scrambling parameter u is zero".to_string()));
+ }
// S = (A * v^u)^b mod N
let s = server.compute_premaster_secret(&a_pub, &v, &u, &b);
@@ -263,38 +242,32 @@ pub fn srp_derive_session_server(
let m2 = compute_m2(&a_pub_bytes, &expected_m1, &key);
- Ok(Some(SrpSession {
- proof: bytes_to_hex(&m2),
- key: bytes_to_hex(&key),
- }))
+ Ok(Some(SrpSession { proof: bytes_to_hex(&m2), key: bytes_to_hex(&key) }))
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Wire-Format-Specific Primitives
-//
-// These deviate from the `srp` crate and must not be replaced with its versions:
-// the SecureRemotePassword (.NET/JS) format hashes the padded premaster secret
-// into K and uses the RFC 2945 M1, while the crate uses raw S and M1 = H(A|B|S).
-// ═══════════════════════════════════════════════════════════════════════════════
+/*
+ * Wire format primitives that deviate from the `srp` crate on purpose: the SecureRemotePassword (.NET/JS) format hashes
+ * the padded premaster secret into K and uses the RFC 2945 M1, while the crate uses raw S and M1 = H(A|B|S).
+ */
/// Derive the session key K = H(PAD(S)) from the premaster secret.
fn derive_session_key(s: &BigUint) -> Vec<u8> {
- Sha256::digest(&to_padded_bytes(s)).to_vec()
+ Sha256::digest(to_padded_bytes(s)).to_vec()
}
/// Compute M1 = H(H(N) XOR H(g) | H(I) | s | A | B | K)
///
/// Note: H(g) uses g without padding, unlike k = H(N, PAD(g))
fn compute_m1(a_pub: &[u8], b_pub: &[u8], salt: &[u8], identity: &str, key: &[u8]) -> Vec<u8> {
- let h_n = Sha256::digest(&G_2048.n.to_bytes_be());
- let h_g = Sha256::digest(&G_2048.g.to_bytes_be());
+ let h_n = Sha256::digest(G_2048.n.to_bytes_be());
+ let h_g = Sha256::digest(G_2048.g.to_bytes_be());
let h_n_xor_h_g: Vec<u8> = h_n.iter().zip(h_g.iter()).map(|(a, b)| a ^ b).collect();
let h_i = Sha256::digest(identity.as_bytes());
let mut hasher = Sha256::new();
hasher.update(&h_n_xor_h_g);
- hasher.update(&h_i);
+ hasher.update(h_i);
hasher.update(salt);
hasher.update(a_pub);
hasher.update(b_pub);
@@ -311,90 +284,13 @@ fn compute_m2(a_pub: &[u8], m1: &[u8], key: &[u8]) -> Vec<u8> {
hasher.finalize().to_vec()
}
-// ═══════════════════════════════════════════════════════════════════════════════
-// Tests
-// ═══════════════════════════════════════════════════════════════════════════════
-
#[cfg(test)]
mod tests {
use super::*;
- #[test]
- fn test_generate_salt() {
- let salt = srp_generate_salt();
- assert_eq!(salt.len(), 64); // 32 bytes = 64 hex chars
- assert!(hex_to_bytes(&salt).is_ok());
- }
-
- #[test]
- fn test_derive_private_key() {
- let salt = "0A0B0C0D0E0F10111213141516171819";
- let identity = "testuser";
- let password_hash = "AABBCCDD";
-
- let private_key = srp_derive_private_key(salt, identity, password_hash).unwrap();
-
- let expected = "ACD81DF26882B20336CF2A8CDE3CABA35BA359805FDFC4567EA7BD74E8302473";
-
- // Should be 32 bytes = 64 hex chars
- assert_eq!(private_key.len(), 64);
- assert!(hex_to_bytes(&private_key).is_ok());
-
- // Same inputs should produce same output
- let private_key2 = srp_derive_private_key(salt, identity, password_hash).unwrap();
- assert_eq!(private_key, private_key2);
-
- assert_eq!(private_key.to_uppercase(), expected);
- }
-
- #[test]
- fn test_derive_verifier() {
- let salt = "0A0B0C0D0E0F10111213141516171819";
- let identity = "testuser";
- let password_hash = "AABBCCDD";
-
- let private_key = srp_derive_private_key(salt, identity, password_hash).unwrap();
- let verifier = srp_derive_verifier(&private_key).unwrap();
-
- let expected = "378FAC69B16F469FB21294F7C74429CD288F47E331E8BA02FFD7C36F2914472A9F2A8C69FFEA434C9F78FCA7E7E41CBBF591FFA589460F023EF3A6F7F6B84366458893C52F8A3304E2247C50BDAE13F4463281B8CDCC519DD563A926C93D9A33E08C1DE2EFB6102BD4BFFE97D9DA9A20354393FA041C8C0459D9D11907E11B75DE4F74990CD0364BA3884C697CF548E31707162D033576B96756A9C8B622332AC9631F62D170445CF33A5EF7E1BE82EC949A5F1FD4AAF1767EE861C729E348FD4209F552BEA5A2F059C64985F4DD2495896AE33315F54329192715AB27EA32B0AF56AC8991C9F708260EF3B5D263FA55B6380CDD294F272FFD1DD86116F0C06C";
-
- // Should be 256 bytes = 512 hex chars (padded to 2048-bit group size)
- assert_eq!(verifier.len(), 512);
- assert!(hex_to_bytes(&verifier).is_ok());
-
- assert_eq!(verifier.to_uppercase(), expected);
- }
-
- #[test]
- fn test_generate_ephemeral() {
- let ephemeral = srp_generate_ephemeral();
-
- // Public should be 256 bytes = 512 hex chars
- assert_eq!(ephemeral.public.len(), 512);
- // Secret should be 64 bytes = 128 hex chars
- assert_eq!(ephemeral.secret.len(), 128);
-
- assert!(hex_to_bytes(&ephemeral.public).is_ok());
- assert!(hex_to_bytes(&ephemeral.secret).is_ok());
- }
-
- #[test]
- fn test_generate_ephemeral_server() {
- let salt = srp_generate_salt();
- let private_key = srp_derive_private_key(&salt, "testuser", "PASSWORDHASH").unwrap();
- let verifier = srp_derive_verifier(&private_key).unwrap();
-
- let ephemeral = srp_generate_ephemeral_server(&verifier).unwrap();
-
- // Public should be 256 bytes = 512 hex chars
- assert_eq!(ephemeral.public.len(), 512);
- // Secret should be 64 bytes = 128 hex chars
- assert_eq!(ephemeral.secret.len(), 128);
- }
-
/// Test with fixed values for deterministic verification.
#[test]
- fn test_fixed_values() {
+ fn fixed_values() {
let salt = "0A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F";
let identity = "testuser";
let password_hash = "AABBCCDDEEFF00112233445566778899AABBCCDDEEFF00112233445566778899";
@@ -411,7 +307,7 @@ mod tests {
/// Test session derivation with fixed ephemeral values.
#[test]
- fn test_session_fixed_values() {
+ fn session_fixed_values() {
let salt = "0A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F";
let identity = "testuser";
let password_hash = "AABBCCDDEEFF00112233445566778899AABBCCDDEEFF00112233445566778899";
@@ -424,8 +320,7 @@ mod tests {
let expected_session_key = "AD713F5D8F520B7B9413CDD9EF6D9B5FE37F23A9B62C5E2B90D2291F8C3A9E6F";
let expected_session_proof = "698D0DA7137A0FC4A55B49525C1312ADCD07788E8CD5FFF5BD195B3C17B6B3DF";
- let session =
- srp_derive_session(client_secret, server_public, salt, identity, &private_key).unwrap();
+ let session = srp_derive_session(client_secret, server_public, salt, identity, &private_key).unwrap();
assert_eq!(session.key.to_uppercase(), expected_session_key);
assert_eq!(session.proof.to_uppercase(), expected_session_proof);
@@ -433,7 +328,7 @@ mod tests {
/// Test with realistic 32-byte salt.
#[test]
- fn test_realistic_salt() {
+ fn realistic_salt() {
let salt = "7c9d6615bfeb06c552c7fbcbfbe7030035a09f058ed7cf7755ca6d3bfa56393c";
let username = "testuser";
let password_hash = "ABCD1234567890ABCD1234567890ABCD1234567890ABCD1234567890ABCD1234";
@@ -453,15 +348,14 @@ mod tests {
let expected_session_key = "7564C550D5BF148D17B33C251B71EA2E0CD96D70E207B58622D9FF78BEE609A4";
let expected_session_proof = "87BF2829F780EF88C1BFB63F39547DAA3CC787B40978C27CDC50FDEBFD324470";
- let session =
- srp_derive_session(client_secret, server_public, salt, username, &private_key).unwrap();
+ let session = srp_derive_session(client_secret, server_public, salt, username, &private_key).unwrap();
assert_eq!(session.key.to_uppercase(), expected_session_key);
assert_eq!(session.proof.to_uppercase(), expected_session_proof);
}
#[test]
- fn test_full_srp_flow() {
+ fn full_srp_flow() {
// 1. Registration: Generate salt and verifier
let salt = srp_generate_salt();
let identity = "[email protected]";
@@ -476,15 +370,11 @@ mod tests {
// 3. Server generates ephemeral and sends to client
let server_ephemeral = srp_generate_ephemeral_server(&verifier).unwrap();
+ // Salts are 32 bytes, public values are padded to the 2048-bit group as the server expects.
+ assert_eq!((salt.len(), client_ephemeral.public.len(), server_ephemeral.public.len(), verifier.len()), (64, 512, 512, 512));
+
// 4. Client derives session
- let client_session = srp_derive_session(
- &client_ephemeral.secret,
- &server_ephemeral.public,
- &salt,
- identity,
- &private_key,
- )
- .unwrap();
+ let client_session = srp_derive_session(&client_ephemeral.secret, &server_ephemeral.public, &salt, identity, &private_key).unwrap();
// 5. Server verifies client proof and derives session
let server_session = srp_derive_session_server(
@@ -497,40 +387,26 @@ mod tests {
)
.unwrap();
- // Server should successfully verify and return a session
let server_session = server_session.expect("server should accept valid client proof");
- // Both should have the same session key
assert_eq!(client_session.key, server_session.key);
// 6. Client verifies server proof (M2)
- let verified = srp_verify_session(
- &client_ephemeral.public,
- &client_session.proof,
- &client_session.key,
- &server_session.proof,
- )
- .unwrap();
+ let verified = srp_verify_session(&client_ephemeral.public, &client_session.proof, &client_session.key, &server_session.proof).unwrap();
assert!(verified);
// A tampered server proof should fail client-side verification
let mut tampered = server_session.proof.clone();
let flipped = if tampered.starts_with('0') { "1" } else { "0" };
tampered.replace_range(0..1, flipped);
- let verified_tampered = srp_verify_session(
- &client_ephemeral.public,
- &client_session.proof,
- &client_session.key,
- &tampered,
- )
- .unwrap();
+ let verified_tampered = srp_verify_session(&client_ephemeral.public, &client_session.proof, &client_session.key, &tampered).unwrap();
assert!(!verified_tampered);
}
/// Mixed-case identity must produce the same values as lowercase, matching
/// the C# implementation's ToLowerInvariant() normalization.
#[test]
- fn test_identity_lowercased() {
+ fn identity_lowercased() {
let salt = "0A0B0C0D0E0F10111213141516171819";
let password_hash = "AABBCCDD";
@@ -546,14 +422,7 @@ mod tests {
let client_ephemeral = srp_generate_ephemeral();
let server_ephemeral = srp_generate_ephemeral_server(&verifier).unwrap();
- let client_session = srp_derive_session(
- &client_ephemeral.secret,
- &server_ephemeral.public,
- &salt,
- "TESTUSER",
- &private_key,
- )
- .unwrap();
+ let client_session = srp_derive_session(&client_ephemeral.secret, &server_ephemeral.public, &salt, "TESTUSER", &private_key).unwrap();
let server_session = srp_derive_session_server(
&server_ephemeral.secret,
@@ -570,15 +439,14 @@ mod tests {
}
#[test]
- fn test_wrong_password_fails() {
+ fn wrong_password_fails() {
// Setup with correct credentials
let salt = srp_generate_salt();
let identity = "testuser";
let correct_password_hash = "CORRECT_PASSWORD_HASH_0123456789";
let wrong_password_hash = "WRONG_PASSWORD_HASH_0123456789AB";
- let correct_private_key =
- srp_derive_private_key(&salt, identity, correct_password_hash).unwrap();
+ let correct_private_key = srp_derive_private_key(&salt, identity, correct_password_hash).unwrap();
let verifier = srp_derive_verifier(&correct_private_key).unwrap();
// Client uses wrong password
@@ -588,14 +456,7 @@ mod tests {
let server_ephemeral = srp_generate_ephemeral_server(&verifier).unwrap();
// Client derives session with wrong password
- let client_session = srp_derive_session(
- &client_ephemeral.secret,
- &server_ephemeral.public,
- &salt,
- identity,
- &wrong_private_key,
- )
- .unwrap();
+ let client_session = srp_derive_session(&client_ephemeral.secret, &server_ephemeral.public, &salt, identity, &wrong_private_key).unwrap();
// Server should reject the client proof
let server_session = srp_derive_session_server(
@@ -608,12 +469,11 @@ mod tests {
)
.unwrap();
- // Server should return None (authentication failed)
assert!(server_session.is_none());
}
#[test]
- fn test_malicious_server_public_rejected() {
+ fn malicious_server_public_rejected() {
let salt = srp_generate_salt();
let identity = "testuser";
let private_key = srp_derive_private_key(&salt, identity, "AABBCCDD").unwrap();
@@ -622,19 +482,13 @@ mod tests {
// B = 0 and B = N are both ≡ 0 mod N and must be rejected
let n_hex = bytes_to_hex(&G_2048.n.to_bytes_be());
for bad_b in ["00", n_hex.as_str()] {
- let result = srp_derive_session(
- &client_ephemeral.secret,
- bad_b,
- &salt,
- identity,
- &private_key,
- );
+ let result = srp_derive_session(&client_ephemeral.secret, bad_b, &salt, identity, &private_key);
assert!(matches!(result, Err(SrpError::InvalidParameter(_))));
}
}
#[test]
- fn test_malicious_client_public_rejected() {
+ fn malicious_client_public_rejected() {
let salt = srp_generate_salt();
let identity = "testuser";
let private_key = srp_derive_private_key(&salt, identity, "AABBCCDD").unwrap();
@@ -643,40 +497,17 @@ mod tests {
let n_hex = bytes_to_hex(&G_2048.n.to_bytes_be());
for bad_a in ["00", n_hex.as_str()] {
- let result = srp_derive_session_server(
- &server_ephemeral.secret,
- bad_a,
- &salt,
- identity,
- &verifier,
- "AABBCCDD",
- );
+ let result = srp_derive_session_server(&server_ephemeral.secret, bad_a, &salt, identity, &verifier, "AABBCCDD");
assert!(matches!(result, Err(SrpError::InvalidParameter(_))));
}
}
+ /// Decoding itself is `common::encoding`'s; SRP adds the `0x` prefix, trimming and a typed error for empty input.
#[test]
- fn test_hex_conversion() {
- // Test round-trip
- let original = vec![0x00, 0x01, 0x0A, 0xFF, 0x10];
- let hex = bytes_to_hex(&original);
- assert_eq!(hex, "00010AFF10");
-
- let decoded = hex_to_bytes(&hex).unwrap();
- assert_eq!(decoded, original);
-
- // Lowercase and 0x-prefixed input
- assert_eq!(hex_to_bytes("00010aff10").unwrap(), original);
- assert_eq!(hex_to_bytes("0x00010AFF10").unwrap(), original);
- }
-
- #[test]
- fn test_hex_invalid_input() {
- assert!(matches!(hex_to_bytes(""), Err(SrpError::InvalidHex(_))));
- assert!(matches!(hex_to_bytes("ABC"), Err(SrpError::InvalidHex(_))));
- assert!(matches!(hex_to_bytes("GG"), Err(SrpError::InvalidHex(_))));
- // Multi-byte UTF-8 must return an error, not panic on a byte-slice boundary
- assert!(matches!(hex_to_bytes("0é9"), Err(SrpError::InvalidHex(_))));
- assert!(matches!(hex_to_bytes("éé"), Err(SrpError::InvalidHex(_))));
+ fn hex_input_accepts_a_prefix_and_refuses_empty_or_malformed_text() {
+ assert_eq!(hex_to_bytes(" 0x00010aFF10 ").unwrap(), vec![0x00, 0x01, 0x0A, 0xFF, 0x10]);
+ for bad in ["", "0x", "ABC", "GG", "0é9"] {
+ assert!(matches!(hex_to_bytes(bad), Err(SrpError::InvalidHex(_))), "{bad:?}");
+ }
}
}
diff --git a/core/rust/src/email_parser/mod.rs b/core/rust/src/email_parser/mod.rs
index db2d67114..76bc3938b 100644
--- a/core/rust/src/email_parser/mod.rs
+++ b/core/rust/src/email_parser/mod.rs
@@ -1,9 +1,9 @@
//! Email MIME parsing for source-only stored emails.
use mail_parser::{GetHeader, Message, MessageParser, MessagePart, MimeHeaders};
use serde::Serialize;
-use std::io::Read;
use crate::common::error::{VaultError, VaultResult};
+use crate::common::gzip::{gunzip_capped, GZIP_MAGIC};
/// Header the server stamps on an attachment whose body it detached from the source at ingest, carrying the
/// index that body is stored and requested under. Reading the index from the message itself keeps it
@@ -14,6 +14,9 @@ const DETACHED_PART_INDEX_HEADER: &str = "X-AliasVault-Part";
/// the attachment list without downloading the body.
const DETACHED_PART_LENGTH_HEADER: &str = "X-AliasVault-Detached-Length";
+/// Cap on a gunzipped email source or detached part, with headroom over the ~14 MB the SMTP service accepts.
+const MAX_DECOMPRESSED_EMAIL_BYTES: u64 = 64 * 1024 * 1024;
+
/// A single attachment of a parsed email message. Metadata only: fetch the bytes with
/// [`extract_email_attachment`] using this attachment's index in [`ParsedEmail::attachments`].
#[derive(Debug, Clone, Serialize)]
@@ -92,7 +95,7 @@ pub fn extract_email_attachment(source: &[u8], index: usize, detached_body: Opti
// Resolve what the attachment needs before splicing: the parsed message borrows `raw`, so nothing that
// borrows it may outlive this block.
- let body_range = {
+ let (body_range, detached_body) = {
let message = parse_message(&raw)?;
let part = attachment_at(&message, index)?;
@@ -104,11 +107,11 @@ pub fn extract_email_attachment(source: &[u8], index: usize, detached_body: Opti
index
)))
}
- (true, Some(_)) => part.offset_body as usize..part.offset_end as usize,
+ (true, Some(body)) => (part.offset_body as usize..part.offset_end as usize, body),
}
};
- let body = decode_detached_part(detached_body.unwrap_or_default())?;
+ let body = decode_detached_part(detached_body)?;
let mut spliced = Vec::with_capacity(raw.len() + body.len());
spliced.extend_from_slice(&raw[..body_range.start]);
spliced.extend_from_slice(&body);
@@ -169,29 +172,15 @@ enum MaybeDecompressed {
/// Gunzip the input when it starts with the gzip magic bytes (0x1f 0x8b), pass it through otherwise.
fn decompress_if_gzip(source: &[u8]) -> VaultResult<MaybeDecompressed> {
- if source.len() < 2 || source[0] != 0x1f || source[1] != 0x8b {
+ if !source.starts_with(&GZIP_MAGIC) {
return Ok(MaybeDecompressed::AsIs(source.to_vec()));
}
- let mut decoder = flate2::read::GzDecoder::new(source);
- let mut decompressed = Vec::new();
- decoder
- .read_to_end(&mut decompressed)
- .map_err(|e| VaultError::General(format!("Failed to gunzip email source: {}", e)))?;
- Ok(MaybeDecompressed::Decompressed(decompressed))
+ gunzip_capped(source, MAX_DECOMPRESSED_EMAIL_BYTES).map(MaybeDecompressed::Decompressed)
}
-/// Undo the double UTF-8 encoding that legacy (pre source-only format) emails were stored with.
-///
-/// The server (pre-0.31.0) used to persist the raw source as a string via MimeKit's `MimeMessage.ToString()`, which
-/// decodes the message bytes as Latin-1 (one char per byte), and then encrypted that string as UTF-8.
-/// Every non-ASCII byte was therefore encoded twice on the way in: `é` (`C3 A9`) was stored as `C3 83 C2 A9`.
-/// That went unnoticed while clients rendered the separately stored (correctly decoded) html/plain columns,
-/// but shows up as incorrect now that the bodies are parsed out of the source itself. Mapping each decoded
-/// character back to a single byte reverses it exactly.
-///
-/// Only input that is valid UTF-8 consisting solely of characters <= U+00FF can be such a round trip, so
-/// anything else - including an already correct raw source - is returned unchanged.
+/// Undo the Latin-1 then UTF-8 double encoding of legacy (pre-0.31.0) stored sources, where `é` (`C3 A9`) became `C3 83 C2 A9`.
+/// A no-op for anything that is not valid UTF-8 made only of characters up to U+00FF, such as an already correct source.
fn repair_legacy_double_encoding(raw: Vec<u8>) -> Vec<u8> {
let text = match std::str::from_utf8(&raw) {
Ok(text) => text,
diff --git a/core/rust/src/email_parser/tests.rs b/core/rust/src/email_parser/tests.rs
index 837c537f0..6738f54d5 100644
--- a/core/rust/src/email_parser/tests.rs
+++ b/core/rust/src/email_parser/tests.rs
@@ -230,3 +230,10 @@ fn json_output_uses_camel_case_fields() {
assert!(json.contains("\"mimeType\""));
assert!(!json.contains("\"contentBase64\""), "attachment bytes must not ride along in the parse result");
}
+
+#[test]
+fn gunzip_rejects_output_over_the_limit() {
+ let compressed = gzip(&[b'a'; 1025]);
+ assert!(gunzip_capped(&compressed, 1024).is_err());
+ assert_eq!(gunzip_capped(&compressed, 1025).unwrap().len(), 1025);
+}
diff --git a/core/rust/src/favicon/mod.rs b/core/rust/src/favicon/mod.rs
index fa8dffcb2..fc698b1ca 100644
--- a/core/rust/src/favicon/mod.rs
+++ b/core/rust/src/favicon/mod.rs
@@ -1,6 +1,6 @@
//! Favicon handling and source selection.
-use crate::credential_matcher::domain::{is_web_scheme, split_url};
+use crate::credential_matcher::domain::{authority_host_port, is_web_scheme, split_url};
use crate::credential_matcher::extract_domain_with_port;
/// The URL to fetch a favicon from, paired with the `Logos.Source` key it is stored under.
@@ -60,18 +60,16 @@ pub fn select_favicon_target(urls: &[String]) -> Option<FaviconTarget> {
/// The origin to fetch a favicon from: lowercased scheme and host (with port), https when scheme-less. Userinfo, path
/// and query are dropped so the server never sees more of the item URL than the site it has to fetch from.
fn canonical_fetch_url(url: &str) -> String {
- let (scheme, authority, _) = split_url(url);
- let scheme = match scheme {
- Some(scheme) if is_web_scheme(scheme) => scheme.to_ascii_lowercase(),
- _ => "https".to_string(),
- };
-
- let host = authority.rsplit_once('@').map_or(authority, |(_, host)| host);
- format!("{}://{}", scheme, host.to_ascii_lowercase())
+ let lowered = url.to_ascii_lowercase();
+ let (scheme, authority, _) = split_url(&lowered);
+ let scheme = scheme.filter(|scheme| is_web_scheme(scheme)).unwrap_or("https");
+ match authority_host_port(authority) {
+ (host, Some(port)) => format!("{scheme}://{host}:{port}"),
+ (host, None) => format!("{scheme}://{host}"),
+ }
}
/// Whether a scheme-less host is shaped like a public hostname, meaning its last label could be a TLD.
-/// TODO: look into replacing this with the full public suffix list here once that is available.
fn looks_like_public_host(host: &str) -> bool {
match host.rsplit_once('.') {
Some((_, tld)) => tld.len() >= 2 && tld.chars().all(|c| c.is_ascii_alphabetic()),
@@ -97,7 +95,7 @@ mod tests {
}
#[test]
- fn test_web_urls_key_on_host() {
+ fn web_urls_key_on_host() {
assert_eq!(favicon_source_key("https://example.com"), "example.com");
assert_eq!(favicon_source_key("http://example.com/login?a=1"), "example.com");
assert_eq!(favicon_source_key("www.example.com"), "example.com");
@@ -108,8 +106,8 @@ mod tests {
}
#[test]
- fn test_app_urls_have_no_favicon_source() {
- // Illegitimate URLs are skipped.
+ fn app_urls_have_no_favicon_source() {
+ // App URLs have no favicon.
assert_eq!(favicon_source_key("androidapp://com.example.android"), "");
assert_eq!(favicon_source_key("androidapp://com.example.app"), "");
assert_eq!(favicon_source_key("android://com.example.android.app"), "");
@@ -120,7 +118,7 @@ mod tests {
}
#[test]
- fn test_only_web_schemes_are_fetchable() {
+ fn only_web_schemes_are_fetchable() {
// A scheme other than http(s) names something that is not a website.
assert_eq!(favicon_source_key("ftp://files.example.com"), "");
assert_eq!(favicon_source_key("file:///Users/me/index.html"), "");
@@ -136,7 +134,7 @@ mod tests {
}
#[test]
- fn test_scheme_less_input_needs_a_plausible_tld() {
+ fn scheme_less_input_needs_a_plausible_tld() {
// Dotted text a user typed into a URL field is not a host to fetch from.
assert_eq!(favicon_source_key("backup.7z"), "");
assert_eq!(favicon_source_key("v1.0"), "");
@@ -155,7 +153,7 @@ mod tests {
}
#[test]
- fn test_unusable_input_has_no_favicon_source() {
+ fn unusable_input_has_no_favicon_source() {
assert_eq!(favicon_source_key(""), "");
assert_eq!(favicon_source_key(" "), "");
assert_eq!(favicon_source_key("not a url"), "");
@@ -164,7 +162,7 @@ mod tests {
}
#[test]
- fn test_non_default_port_is_part_of_the_key() {
+ fn non_default_port_is_part_of_the_key() {
assert_eq!(favicon_source_key("http://localhost:8080"), "localhost:8080");
assert_eq!(favicon_source_key("http://localhost:9090"), "localhost:9090");
assert_ne!(
@@ -176,7 +174,7 @@ mod tests {
}
#[test]
- fn test_default_port_collapses_onto_the_bare_host() {
+ fn default_port_collapses_onto_the_bare_host() {
assert_eq!(favicon_source_key("http://localhost:80"), "localhost");
assert_eq!(favicon_source_key("https://example.com:443"), "example.com");
assert_eq!(favicon_source_key("example.com:443"), "example.com");
@@ -190,8 +188,8 @@ mod tests {
}
#[test]
- fn test_select_skips_app_urls_and_keeps_item_order() {
- // The reported item shape: an app URL listed first, the website second.
+ fn select_skips_app_urls_and_keeps_item_order() {
+ // An app URL listed before the website.
let target = select_favicon_target(&urls(&[
"androidapp://com.example.app",
"https://example.com",
@@ -210,7 +208,7 @@ mod tests {
}
#[test]
- fn test_select_prefers_the_first_usable_url() {
+ fn select_prefers_the_first_usable_url() {
let target = select_favicon_target(&urls(&[
"com.example.app",
"https://first.example.com",
@@ -221,7 +219,7 @@ mod tests {
}
#[test]
- fn test_select_returns_none_when_no_url_is_fetchable() {
+ fn select_returns_none_when_no_url_is_fetchable() {
assert_eq!(select_favicon_target(&urls(&[])), None);
assert_eq!(
select_favicon_target(&urls(&[
@@ -235,7 +233,7 @@ mod tests {
}
#[test]
- fn test_casing_is_ignored() {
+ fn casing_is_ignored() {
let lower = select_favicon_target(&urls(&["example.com"])).unwrap();
for variant in ["Example.com", "example.com", "https://example.com"] {
let target = select_favicon_target(&urls(&[variant])).unwrap();
@@ -252,13 +250,13 @@ mod tests {
}
#[test]
- fn test_fetch_url_is_the_origin_only() {
+ fn fetch_url_is_the_origin_only() {
let target = select_favicon_target(&urls(&["https://user:[email protected]:8443/account/reset?token=abc#top"])).unwrap();
assert_eq!(target.url, "https://example.com:8443");
}
#[test]
- fn test_select_adds_missing_scheme_to_fetch_url() {
+ fn select_adds_missing_scheme_to_fetch_url() {
let target = select_favicon_target(&urls(&["www.example.com"])).unwrap();
assert_eq!(target.url, "https://www.example.com");
assert_eq!(target.source, "example.com");
diff --git a/core/rust/src/identity_generator/mod.rs b/core/rust/src/identity_generator/mod.rs
index fb7bacf74..1cdf3d900 100644
--- a/core/rust/src/identity_generator/mod.rs
+++ b/core/rust/src/identity_generator/mod.rs
@@ -1,14 +1,12 @@
//! Cross-platform random identity (alias persona) generation.
//!
-//! This module is the single source of truth for identity generation across AliasVault.
-//! It accepts a JSON-serialized [`IdentityRequest`] and returns the generated
-//! [`Identity`] as JSON, matching the JSON-in/JSON-out convention used by the other
-//! core modules.
+//! This module is the single source of truth for identity generation across AliasVault. It accepts a JSON-serialized
+//! [`IdentityRequest`] and returns the generated [`Identity`] as JSON, matching the JSON-in/JSON-out convention used by the
+//! other core modules.
//!
-//! An identity consists of a first name, last name, gender, birth date, email prefix
-//! and nickname (username). Names are drawn from embedded per-language dictionaries;
-//! for languages with decade-based dictionaries (de, it, ro) the first name is matched
-//! to names that were popular around the generated birth year.
+//! An identity consists of a first name, last name, gender, birth date, email prefix and nickname (username). Names are drawn
+//! from embedded per-language dictionaries; for languages with decade-based dictionaries (de, it, ro) the first name is
+//! matched to names that were popular around the generated birth year.
pub(crate) mod dictionaries;
mod username_email;
@@ -33,14 +31,10 @@ const DEFAULT_MIN_AGE: i32 = 21;
/// Default maximum age (in years) for generated birth dates.
const DEFAULT_MAX_AGE: i32 = 65;
-/// The age range option values offered by the identity generator settings UIs.
-/// The first entry means "no preference" (default 21-65 age behavior).
-const AGE_RANGES: &[&str] = &[
- "random", "21-25", "26-30", "31-35", "36-40", "41-45", "46-50", "51-55", "56-60", "61-65",
-];
+/// The age range option values offered by the settings UIs; "random" means the default 21-65 range.
+const AGE_RANGES: &[&str] = &["random", "21-25", "26-30", "31-35", "36-40", "41-45", "46-50", "51-55", "56-60", "61-65"];
-/// A generated identity. Serialized with camelCase field names to match the
-/// JSON contract shared by all clients.
+/// A generated identity, serialized with camelCase field names to match the JSON contract shared by all clients.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Identity {
@@ -68,8 +62,7 @@ pub struct BirthdateOptions {
pub year_deviation: i32,
}
-/// Request controlling identity generation. Every field is optional so callers can
-/// pass only what they need; unknown languages fall back to English.
+/// Request controlling identity generation. Every field is optional; unknown languages fall back to English.
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct IdentityRequest {
@@ -77,18 +70,15 @@ pub struct IdentityRequest {
pub language: String,
/// Gender preference: "male", "female" or "random" (default).
pub gender: Option<String>,
- /// Age range preference as stored in settings (e.g. "21-25" or "random").
- /// Ignored when `birthdate_options` is set.
+ /// Age range preference as stored in settings (e.g. "21-25" or "random"). Ignored when `birthdate_options` is set.
pub age_range: Option<String>,
/// Explicit birth year window. Takes precedence over `age_range`.
pub birthdate_options: Option<BirthdateOptions>,
- /// Optional 32-byte RNG seed as a 64-character hex string for deterministic output
- /// (used by tests). Normal generation is non-deterministic.
+ /// Optional 32-byte RNG seed as a 64-character hex string for deterministic output (used by tests).
pub seed: Option<String>,
}
-/// Name and birth date input for username/email prefix generation, e.g. an identity
-/// previously generated by [`generate_identity`] or persona fields typed by the user.
+/// Name and birth date input for username/email prefix generation, from a generated identity or persona fields typed by the user.
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct NameInput {
@@ -104,8 +94,7 @@ pub struct NameInput {
/// Generate a random identity from a JSON-serialized [`IdentityRequest`].
///
-/// Returns the generated [`Identity`] as a JSON string, or a [`VaultError`] if the
-/// request JSON is invalid.
+/// Returns the generated [`Identity`] as a JSON string, or a [`VaultError`] if the request JSON is invalid.
pub fn generate_identity(request_json: &str) -> Result<String, VaultError> {
json_call(request_json, |request: IdentityRequest| Ok(generate_from_request(&request)))
}
@@ -123,9 +112,7 @@ fn generate_from_request(request: &IdentityRequest) -> Identity {
};
// Generate the birth date first: decade-based languages select first names by birth year.
- let birthdate_options = request
- .birthdate_options
- .or_else(|| age_range_to_birthdate_options(request.age_range.as_deref().unwrap_or("")));
+ let birthdate_options = request.birthdate_options.or_else(|| age_range_to_birthdate_options(request.age_range.as_deref().unwrap_or("")));
let birth_date = random_birth_date(&mut rng, birthdate_options, Utc::now().date_naive());
let first_names = select_firstnames_for_birth_year(dictionary, birth_date.year(), is_male);
@@ -136,8 +123,7 @@ fn generate_from_request(request: &IdentityRequest) -> Identity {
let birth_year = Some(birth_date.year());
// The email prefix and nickname are generated independently (separate random rolls).
- let email_prefix =
- username_email::generate_email_prefix(&mut rng, &first_name, &last_name, birth_year);
+ let email_prefix = username_email::generate_email_prefix(&mut rng, &first_name, &last_name, birth_year);
let nick_name = username_email::generate_username(&mut rng, &first_name, &last_name, birth_year);
Identity {
@@ -154,28 +140,17 @@ fn generate_from_request(request: &IdentityRequest) -> Identity {
pub fn generate_username(input_json: &str) -> Result<String, VaultError> {
let input: NameInput = serde_json::from_str(input_json)?;
let mut rng = make_rng(input.seed.as_deref());
- Ok(username_email::generate_username(
- &mut rng,
- &input.first_name,
- &input.last_name,
- parse_birth_year(&input.birth_date),
- ))
+ Ok(username_email::generate_username(&mut rng, &input.first_name, &input.last_name, parse_birth_year(&input.birth_date)))
}
/// Generate an email prefix from a JSON-serialized [`NameInput`].
pub fn generate_email_prefix(input_json: &str) -> Result<String, VaultError> {
let input: NameInput = serde_json::from_str(input_json)?;
let mut rng = make_rng(input.seed.as_deref());
- Ok(username_email::generate_email_prefix(
- &mut rng,
- &input.first_name,
- &input.last_name,
- parse_birth_year(&input.birth_date),
- ))
+ Ok(username_email::generate_email_prefix(&mut rng, &input.first_name, &input.last_name, parse_birth_year(&input.birth_date)))
}
-/// Generate a random alphanumeric email prefix that is not based on any identity,
-/// suitable for login-type credentials without persona fields.
+/// Generate a random alphanumeric email prefix that is not based on any identity, for credentials without persona fields.
pub fn generate_random_email_prefix(length: u32) -> String {
let mut rng = make_rng(None);
username_email::generate_random_string(&mut rng, length.clamp(1, 64) as usize)
@@ -183,21 +158,15 @@ pub fn generate_random_email_prefix(length: u32) -> String {
/// List the language codes of all bundled identity dictionaries.
pub fn available_languages() -> Vec<String> {
- dictionaries::available_codes()
- .into_iter()
- .map(|c| c.to_string())
- .collect()
+ dictionaries::available_codes().into_iter().map(|c| c.to_string()).collect()
}
-/// List the age range option values offered by the settings UIs
-/// ("random" plus 5-year ranges from 21-25 to 61-65).
+/// List the age range option values offered by the settings UIs ("random" plus 5-year ranges from 21-25 to 61-65).
pub fn available_age_ranges() -> Vec<String> {
AGE_RANGES.iter().map(|r| r.to_string()).collect()
}
-/// Convert an age range string (e.g. "21-25") to birth year options relative to the
-/// current year. Returns `None` for "random", empty or malformed input, which means
-/// the default age behavior (21-65) applies.
+/// Birth year options for an age range (e.g. "21-25") relative to the current year; `None` (default 21-65) for anything else.
fn age_range_to_birthdate_options(age_range: &str) -> Option<BirthdateOptions> {
age_range_to_birthdate_options_at(age_range, Utc::now().year())
}
@@ -215,54 +184,27 @@ fn age_range_to_birthdate_options_at(age_range: &str, current_year: i32) -> Opti
})
}
-/// Generate a random birth date. With options, the date is uniform within the year
-/// window; otherwise it is uniform between `today - 65 years` and `today - 21 years`.
-fn random_birth_date<R: RngCore + ?Sized>(
- rng: &mut R,
- options: Option<BirthdateOptions>,
- today: NaiveDate,
-) -> NaiveDate {
+/// A random birth date, uniform within the options' year window, or else between 65 and 21 years before `today`.
+fn random_birth_date<R: RngCore + ?Sized>(rng: &mut R, options: Option<BirthdateOptions>, today: NaiveDate) -> NaiveDate {
let (start, end) = match options {
- Some(BirthdateOptions {
- target_year,
- year_deviation,
- }) => {
+ Some(BirthdateOptions { target_year, year_deviation }) => {
// Sanity check for overflows in the calculation.
let deviation = year_deviation.saturating_abs();
- (
- first_day_of_year(target_year.saturating_sub(deviation)),
- last_day_of_year(target_year.saturating_add(deviation)),
- )
+ (first_day_of_year(target_year.saturating_sub(deviation)), last_day_of_year(target_year.saturating_add(deviation)))
}
None => (years_ago(today, DEFAULT_MAX_AGE), years_ago(today, DEFAULT_MIN_AGE)),
};
let days = (end - start).num_days().max(0) as usize + 1;
- start
- .checked_add_days(Days::new(unbiased_index(rng, days) as u64))
- .unwrap_or(start)
+ start.checked_add_days(Days::new(unbiased_index(rng, days) as u64)).unwrap_or(start)
}
-/// Select the first name pool for a birth year: the matching decade lists where
-/// available, all decade names when the year falls outside every range, otherwise
-/// the flat per-gender list.
-fn select_firstnames_for_birth_year(
- dictionary: &'static dictionaries::LanguageDictionary,
- birth_year: i32,
- is_male: bool,
-) -> Vec<&'static str> {
- let decades = if is_male {
- dictionary.firstnames_male_by_decade
- } else {
- dictionary.firstnames_female_by_decade
- };
+/// The first name pool for a birth year: matching decade lists, else all decade names, else the flat per-gender list.
+fn select_firstnames_for_birth_year(dictionary: &'static dictionaries::LanguageDictionary, birth_year: i32, is_male: bool) -> Vec<&'static str> {
+ let decades = if is_male { dictionary.firstnames_male_by_decade } else { dictionary.firstnames_female_by_decade };
if !decades.is_empty() {
- let matching: Vec<&'static str> = decades
- .iter()
- .filter(|d| birth_year >= d.start_year && birth_year <= d.end_year)
- .flat_map(|d| d.names())
- .collect();
+ let matching: Vec<&'static str> = decades.iter().filter(|d| (d.start_year..=d.end_year).contains(&birth_year)).flat_map(|d| d.names()).collect();
if !matching.is_empty() {
return matching;
}
@@ -274,15 +216,10 @@ fn select_firstnames_for_birth_year(
}
}
- if is_male {
- dictionary.firstnames_male()
- } else {
- dictionary.firstnames_female()
- }
+ if is_male { dictionary.firstnames_male() } else { dictionary.firstnames_female() }
}
-/// Parse the leading `yyyy` year from a date string like `1990-05-15` or a full
-/// ISO 8601 timestamp. Returns `None` when no valid year prefix is present.
+/// The leading `yyyy` year of a date string like `1990-05-15` or an ISO 8601 timestamp, or `None` when absent.
fn parse_birth_year(birth_date: &str) -> Option<i32> {
let digits: String = birth_date.trim().chars().take(4).collect();
if digits.len() == 4 && digits.chars().all(|c| c.is_ascii_digit()) {
diff --git a/core/rust/src/lib.rs b/core/rust/src/lib.rs
index 647343165..d2b3a39d4 100644
--- a/core/rust/src/lib.rs
+++ b/core/rust/src/lib.rs
@@ -3,9 +3,8 @@
//! Cross-platform core functionality for AliasVault, including:
//! - **vault_model**: the client vault datamodel registry, generated from `core/models`
//! - **vault_codec**: the manifest-v1 storage format, mapped to and from the local SQLite vault
-//! - **vault_merge**: Vault merge using Last-Write-Wins (LWW) strategy
-//! - **vault_sharing**: Sharing write logic for multi-manifest vaults
-//! - **vault_pruner**: Prunes expired items from trash (30-day retention)
+//! - **vault_merge**: the last-write-wins merge of a local vault onto the server's, one manifest at a time
+//! - **vault_pruner**: tombstones expired trash (30-day retention) and reclaims orphan favicons and blob bytes
//! - **vault_sync**: the vault sync engine that every client uses for syncing with the server
//! - **sqlite_host**: an in-memory SQLite database for hosts that cannot open one from bytes
//! - **credential_matcher**: Cross-platform credential filtering for autofill
@@ -21,7 +20,6 @@ pub mod common;
pub mod vault_model;
pub mod vault_merge;
pub mod vault_codec;
-pub mod vault_sharing;
pub mod vault_pruner;
pub mod credential_matcher;
pub mod email_parser;
diff --git a/core/rust/src/password_generator/basic.rs b/core/rust/src/password_generator/basic.rs
index e16fd63ab..7ef891894 100644
--- a/core/rust/src/password_generator/basic.rs
+++ b/core/rust/src/password_generator/basic.rs
@@ -1,8 +1,7 @@
//! Basic character-set password generator.
//!
-//! Builds a character set from the enabled options and constructs the password so that it
-//! is guaranteed to contain at least one character from every enabled set (as some websites
-//! require this).
+//! Builds a character set from the enabled options and constructs the password so that it is guaranteed to contain at least
+//! one character from every enabled set (as some websites require this).
use rand::RngCore;
@@ -21,20 +20,17 @@ pub fn generate<R: RngCore + ?Sized>(settings: &PasswordSettings, rng: &mut R) -
let chars = build_character_set(settings);
let length = settings.length as usize;
- // Reserve one mandatory character per enabled class so the result is guaranteed to
- // contain at least one of each.
+ // Reserve one mandatory character per enabled class so the result is guaranteed to contain at least one of each.
let mut mandatory = mandatory_characters(settings, rng);
- // If the requested length cannot fit every mandatory character (e.g. length 2 with four
- // enabled classes), shuffle and truncate so we never exceed the requested length. Which
- // classes "win" is then random rather than always favouring the same ones.
+ // If the requested length cannot fit every mandatory character (e.g. length 2 with four enabled classes), shuffle and
+ // truncate so we never exceed the requested length. Which classes "win" is then random rather than always the same ones.
if mandatory.len() > length {
shuffle(&mut mandatory, rng);
mandatory.truncate(length);
}
- // Fill the remaining positions from the full character set, then shuffle so the mandatory
- // characters are not clustered at the front.
+ // Fill the remaining positions from the full character set, then shuffle so the mandatory characters are not clustered at the front.
let mut password = mandatory;
while password.len() < length {
password.push(chars[unbiased_index(rng, chars.len())]);
@@ -66,31 +62,11 @@ fn build_character_set(settings: &PasswordSettings) -> Vec<char> {
chars.push_str(LOWERCASE_CHARS);
}
- let mut set: Vec<char> = chars.chars().collect();
-
- // Remove ambiguous characters if needed.
- if settings.use_non_ambiguous_chars {
- set = remove_ambiguous_characters(&set);
- }
-
- set
+ safe_character_set(&chars, settings)
}
-/// Remove ambiguous characters from a character set.
-fn remove_ambiguous_characters(chars: &[char]) -> Vec<char> {
- chars
- .iter()
- .copied()
- .filter(|c| !AMBIGUOUS_CHARS.contains(*c))
- .collect()
-}
-
-/// Collect one mandatory character per enabled class, so the constructed password is
-/// guaranteed to contain at least one character from each.
-fn mandatory_characters<R: RngCore + ?Sized>(
- settings: &PasswordSettings,
- rng: &mut R,
-) -> Vec<char> {
+/// Collect one mandatory character per enabled class, so the password contains at least one character from each.
+fn mandatory_characters<R: RngCore + ?Sized>(settings: &PasswordSettings, rng: &mut R) -> Vec<char> {
let mut mandatory = Vec::new();
if settings.use_lowercase {
push_one(&mut mandatory, LOWERCASE_CHARS, settings, rng);
@@ -108,12 +84,7 @@ fn mandatory_characters<R: RngCore + ?Sized>(
}
/// Pick one random character from the (ambiguity-filtered) class set and push it onto `out`.
-fn push_one<R: RngCore + ?Sized>(
- out: &mut Vec<char>,
- char_set: &str,
- settings: &PasswordSettings,
- rng: &mut R,
-) {
+fn push_one<R: RngCore + ?Sized>(out: &mut Vec<char>, char_set: &str, settings: &PasswordSettings, rng: &mut R) {
let safe = safe_character_set(char_set, settings);
if !safe.is_empty() {
out.push(safe[unbiased_index(rng, safe.len())]);
@@ -122,15 +93,10 @@ fn push_one<R: RngCore + ?Sized>(
/// Get a character set with ambiguous characters removed if the option is enabled.
fn safe_character_set(char_set: &str, settings: &PasswordSettings) -> Vec<char> {
- let chars: Vec<char> = char_set.chars().collect();
- if !settings.use_non_ambiguous_chars {
- return chars;
- }
- remove_ambiguous_characters(&chars)
+ char_set.chars().filter(|c| !settings.use_non_ambiguous_chars || !AMBIGUOUS_CHARS.contains(*c)).collect()
}
-/// Shuffle a slice in place with an unbiased Fisher–Yates shuffle, reusing [`unbiased_index`]
-/// so the result stays deterministic under a fixed seed.
+/// Shuffle a slice in place with an unbiased Fisher-Yates shuffle, reusing [`unbiased_index`] so a fixed seed stays deterministic.
fn shuffle<R: RngCore + ?Sized>(items: &mut [char], rng: &mut R) {
for i in (1..items.len()).rev() {
let j = unbiased_index(rng, i + 1);
diff --git a/core/rust/src/password_generator/diceware.rs b/core/rust/src/password_generator/diceware.rs
index cf56308c6..853a0fe09 100644
--- a/core/rust/src/password_generator/diceware.rs
+++ b/core/rust/src/password_generator/diceware.rs
@@ -1,9 +1,8 @@
//! Diceware passphrase generator.
//!
-//! Ported from the MIT-licensed `SpamOK.PasswordGenerator` Diceware algorithm. Picks
-//! `word_count` words uniformly at random from the selected language wordlist, applies
-//! capitalization, joins them with a separator, and optionally adds a random alphanumeric
-//! "salt" character.
+//! Ported from the MIT-licensed `SpamOK.PasswordGenerator` Diceware algorithm. Picks `word_count` words uniformly at random
+//! from the selected language wordlist, applies capitalization, joins them with a separator, and optionally adds a random
+//! alphanumeric "salt" character.
use rand::RngCore;
@@ -42,11 +41,7 @@ fn separator_char(separator: Separator) -> Option<char> {
}
/// Apply the configured capitalization to a single word.
-fn capitalize_word<R: RngCore + ?Sized>(
- word: &str,
- capitalization: Capitalization,
- rng: &mut R,
-) -> String {
+fn capitalize_word<R: RngCore + ?Sized>(word: &str, capitalization: Capitalization, rng: &mut R) -> String {
match capitalization {
Capitalization::None => word.to_string(),
Capitalization::Lowercase => word.to_lowercase(),
@@ -54,9 +49,7 @@ fn capitalize_word<R: RngCore + ?Sized>(
Capitalization::TitleCase => {
let mut chars = word.chars();
match chars.next() {
- Some(first) => {
- first.to_uppercase().collect::<String>() + &chars.as_str().to_lowercase()
- }
+ Some(first) => first.to_uppercase().collect::<String>() + &chars.as_str().to_lowercase(),
None => String::new(),
}
}
@@ -75,22 +68,21 @@ fn capitalize_word<R: RngCore + ?Sized>(
/// Add a random alphanumeric salt character to the passphrase based on the salt option.
fn add_salt<R: RngCore + ?Sized>(passphrase: String, salt: Salt, rng: &mut R) -> String {
- if matches!(salt, Salt::None) {
- return passphrase;
- }
-
- let salt_chars: Vec<char> = ALPHANUMERIC.chars().collect();
- let salt_char = salt_chars[unbiased_index(rng, salt_chars.len())];
-
match salt {
- Salt::Prefix => format!("{}{}", salt_char, passphrase),
- Salt::Suffix => format!("{}{}", passphrase, salt_char),
+ Salt::None => passphrase,
+ Salt::Prefix => format!("{}{}", random_alphanumeric(rng), passphrase),
+ Salt::Suffix => format!("{}{}", passphrase, random_alphanumeric(rng)),
Salt::Sprinkle => {
+ let salt_char = random_alphanumeric(rng);
let mut chars: Vec<char> = passphrase.chars().collect();
let index = unbiased_index(rng, chars.len() + 1);
chars.insert(index, salt_char);
chars.into_iter().collect()
}
- Salt::None => passphrase,
}
}
+
+/// Pick one random alphanumeric character.
+fn random_alphanumeric<R: RngCore + ?Sized>(rng: &mut R) -> char {
+ char::from(ALPHANUMERIC.as_bytes()[unbiased_index(rng, ALPHANUMERIC.len())])
+}
diff --git a/core/rust/src/password_generator/mod.rs b/core/rust/src/password_generator/mod.rs
index 206d34e58..a9fa40f61 100644
--- a/core/rust/src/password_generator/mod.rs
+++ b/core/rust/src/password_generator/mod.rs
@@ -1,9 +1,8 @@
//! Cross-platform password and passphrase (Diceware) generation.
//!
-//! This module is the single source of truth for password generation across AliasVault.
-//! It accepts a JSON-serialized [`PasswordSettings`] object and returns the generated
-//! password/passphrase as a string, matching the JSON-in/string-out convention used by
-//! the other core modules.
+//! This module is the single source of truth for password generation across AliasVault. It accepts a JSON-serialized
+//! [`PasswordSettings`] object and returns the generated password/passphrase as a string, matching the JSON-in/string-out
+//! convention used by the other core modules.
//!
//! Two generators are supported, selected by the `Type` field:
//! - **Basic** (`basic`): a configurable character-set password.
@@ -84,9 +83,8 @@ pub enum Salt {
/// Settings controlling password/passphrase generation.
///
-/// Field names use PascalCase to match the JSON blob persisted by the apps under the
-/// `PasswordGenerationSettings` settings key. Every field has a serde default so that
-/// older blobs (which lack the Diceware fields) deserialize cleanly.
+/// Field names use PascalCase to match the JSON blob persisted by the apps under the `PasswordGenerationSettings` settings key.
+/// Every field has a serde default so that older blobs (which lack the Diceware fields) deserialize cleanly.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "PascalCase")]
pub(crate) struct PasswordSettings {
@@ -94,7 +92,6 @@ pub(crate) struct PasswordSettings {
#[serde(rename = "Type", default)]
pub generator_type: GeneratorType,
- // --- Basic generator ---
/// Length of the generated password.
#[serde(default = "default_length")]
pub length: u32,
@@ -114,12 +111,11 @@ pub(crate) struct PasswordSettings {
#[serde(default)]
pub use_non_ambiguous_chars: bool,
- // --- Diceware generator ---
/// Number of words in the passphrase.
#[serde(default = "default_word_count")]
pub word_count: u32,
- /// Wordlist language code (free text, case-insensitive). Unknown codes fall back to
- /// English, so the TypeScript model and apps never need updating to add a language.
+ /// Wordlist language code (free text, case-insensitive). Unknown codes fall back to English, so the TypeScript model and
+ /// apps never need updating to add a language.
#[serde(default = "default_language")]
pub language: String,
/// Capitalization applied to each word.
@@ -132,12 +128,7 @@ pub(crate) struct PasswordSettings {
#[serde(default)]
pub salt: Salt,
- /// Optional 32-byte RNG seed as a 64-character hex string.
- ///
- /// When supplied, generation is deterministic: the same seed yields the same output,
- /// so the UI can re-apply formatting options (separator, capitalization, salt, word
- /// count) to the *same* underlying words for easy comparison. When absent, a fresh
- /// random seed is drawn from the OS CSPRNG, so output is non-deterministic by default.
+ /// Optional 32-byte RNG seed as 64 hex characters; the same seed gives the same output, absent means a fresh OS random seed.
#[serde(default)]
pub seed: Option<String>,
}
@@ -150,8 +141,9 @@ fn default_word_count() -> u32 {
defaults::DEFAULT_WORD_COUNT
}
-/// Minimum values for the password and passphrase length.
+/// Minimum password length.
const HARD_MIN_PASSWORD_LENGTH: u32 = 1;
+/// Minimum passphrase word count.
const HARD_MIN_WORD_COUNT: u32 = 1;
fn default_true() -> bool {
@@ -164,10 +156,7 @@ fn default_language() -> String {
/// List the language codes of all bundled Diceware wordlists (first is the default, English).
pub fn available_languages() -> Vec<String> {
- wordlists::available_codes()
- .into_iter()
- .map(|c| c.to_string())
- .collect()
+ wordlists::available_codes().into_iter().map(|c| c.to_string()).collect()
}
/// Generate a password or passphrase from a JSON-serialized [`PasswordSettings`].
@@ -182,14 +171,10 @@ pub fn generate_password(settings_json: &str) -> Result<String, VaultError> {
fn generate_from_settings(settings: &PasswordSettings) -> String {
let mut rng = make_rng(settings.seed.as_deref());
- // Limit the maximum values for the password and passphrase length.
+ // Clamp length and word count.
let mut settings = settings.clone();
- settings.length = settings
- .length
- .clamp(HARD_MIN_PASSWORD_LENGTH, defaults::MAX_PASSWORD_LENGTH);
- settings.word_count = settings
- .word_count
- .clamp(HARD_MIN_WORD_COUNT, defaults::MAX_WORD_COUNT);
+ settings.length = settings.length.clamp(HARD_MIN_PASSWORD_LENGTH, defaults::MAX_PASSWORD_LENGTH);
+ settings.word_count = settings.word_count.clamp(HARD_MIN_WORD_COUNT, defaults::MAX_WORD_COUNT);
match settings.generator_type {
GeneratorType::Basic => basic::generate(&settings, &mut rng),
diff --git a/core/rust/src/password_generator/tests.rs b/core/rust/src/password_generator/tests.rs
index 011d1558c..6293cc9aa 100644
--- a/core/rust/src/password_generator/tests.rs
+++ b/core/rust/src/password_generator/tests.rs
@@ -74,27 +74,13 @@ fn basic_short_password_keeps_every_class() {
}
#[test]
-fn basic_length_clamped_to_max() {
- // Out-of-range lengths are clamped to the supported maximum (256) rather than honoured.
- let json = r#"{"Type":"basic","Length":100000}"#;
- let pw = generate_password(json).unwrap();
- assert_eq!(pw.chars().count(), 256);
-}
-
-#[test]
-fn basic_zero_length_clamped_to_min() {
- // Length 0 must not yield an empty password; it is clamped up to the minimum of 1.
- let json = r#"{"Type":"basic","Length":0}"#;
- let pw = generate_password(json).unwrap();
- assert_eq!(pw.chars().count(), 1);
-}
-
-#[test]
-fn diceware_word_count_clamped_to_max() {
- // More than the supported 10 words is clamped down to 10.
- let json = r#"{"Type":"diceware","WordCount":50,"Separator":"Dash","Salt":"None"}"#;
- let pw = generate_password(json).unwrap();
- assert_eq!(pw.split('-').count(), 10);
+fn out_of_range_sizes_are_clamped() {
+ // Basic length to 1..=256 (0 must not yield an empty password), diceware to at most 10 words.
+ for (json, expected) in [(r#"{"Type":"basic","Length":100000}"#, 256), (r#"{"Type":"basic","Length":0}"#, 1)] {
+ assert_eq!(generate_password(json).unwrap().chars().count(), expected, "{json}");
+ }
+ let words = generate_password(r#"{"Type":"diceware","WordCount":50,"Separator":"Dash","Salt":"None"}"#).unwrap();
+ assert_eq!(words.split('-').count(), 10);
}
#[test]
diff --git a/core/rust/src/sqlite_host/mod.rs b/core/rust/src/sqlite_host/mod.rs
index 838757416..87591bd90 100644
--- a/core/rust/src/sqlite_host/mod.rs
+++ b/core/rust/src/sqlite_host/mod.rs
@@ -1,6 +1,6 @@
-//! An in-memory SQLite database that can be used by host applications to be have uniform access to the database.
+//! An in-memory SQLite database that host applications use for uniform access to the database.
//!
-//! The database lives in SQLite's own memory and are never persisted to the filesystem.
+//! The database lives in SQLite's own memory and is never persisted to the filesystem.
use std::sync::Mutex;
@@ -97,7 +97,7 @@ impl MemoryDatabase {
Ok(SqlResult { columns, rows: out })
}
- /// Run one statement with typed parameters and return the rows it changed.
+ /// Run one DML statement with typed parameters and return the rows it changed (stale for a SELECT, which changes nothing).
pub fn execute(&self, sql: &str, params: &[SqlValue]) -> VaultResult<u64> {
let conn = self.lock();
let mut statement = conn.prepare_cached(sql).map_err(sql_error_at(sql))?;
@@ -162,7 +162,7 @@ fn to_sql(value: &Value) -> VaultResult<RsValue> {
Ok(match value {
Value::Null => RsValue::Null,
Value::Bool(b) => RsValue::Integer(*b as i64),
- Value::Number(n) => n.as_i64().map(RsValue::Integer).unwrap_or_else(|| RsValue::Real(n.as_f64().unwrap_or(0.0))),
+ Value::Number(n) => n.as_i64().map_or_else(|| n.as_f64().map_or(RsValue::Null, RsValue::Real), RsValue::Integer),
Value::String(s) => RsValue::Text(s.clone()),
other => match inline_b64(other) {
Some(b64) => RsValue::Blob(base64_decode(b64).map_err(|_| VaultError::General("A BLOB parameter is not valid base64".to_string()))?),
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rs
index 256d4bc41..f6b738b64 100644
--- a/core/rust/src/vault_codec/canonicalize.rs
+++ b/core/rust/src/vault_codec/canonicalize.rs
@@ -1,70 +1,57 @@
-//! Canonicalize a SQLite source dataset into the canonical vault manifest-v1 persisted representation:
-//! normalized `CodecTableData[]` + salt > manifest + data buckets + content-addressed blob map.
+//! Canonicalize a SQLite source dataset into the manifest-v1 persisted representation: normalized tables plus a
+//! salt per manifest > one manifest per declared manifest, data buckets and content-addressed blob maps.
//!
-//! The input rows are already JSON-normalized by the platform read. Every SQLite byte column
-//! arrives as `{ "__b64": <base64> }`. This module applies the *format* rules:
-//! - each bucketed table is split out into a data bucket per manifest: rows route by their own `ManifestId`.
-//! - skip-tables are dropped;
-//! - the two blob columns (`Logos.FileData`, `Attachments.Blob`) have their bytes extracted into a
-//! content-addressed blob map (hash = `HMAC-SHA256(salt, bytes)`) and the cell replaced with
-//! `{ "__blobRef": hash, "__blobKind": kind }`;
+//! The input rows are already JSON-normalized by the platform read. Every SQLite byte column arrives as
+//! `{ "__b64": <base64> }`. This module applies the *format* rules:
+//! - skip-tables are dropped and the codec overflow row is folded back into the rows it came from;
+//! - every row routes to the manifest its `ManifestId` names (see [`sharing`](super::sharing));
+//! - each bucketed table is split out into a data bucket per manifest;
+//! - the blob columns (`Logos.FileData`, `Attachments.Blob`) have their bytes extracted into a content-addressed
+//! blob map (hash = `HMAC-SHA256(salt, bytes)`) and the cell replaced with `{ "__blobRef": hash, "__blobKind": kind }`;
//! - every other column (including non-blob `{ "__b64" }` inline bytes) is copied as-is.
use std::collections::HashMap;
use serde_json::{json, Value};
+use super::integrity;
use super::normalize::{normalize_id_spelling, normalize_row_shapes};
use super::hash::salted_blob_hash;
use super::row::{blob_ref, inline_b64, is_deleted, str_col};
-use super::scoped_assets::{normalize_logo_scope, reconcile_logo_references};
-use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedManifest, CanonicalizedVault, CodecOverflow, DataBucket, Manifest, ManifestSpec, CodecRecord};
-use super::sharing::{clone_referenced_rows, partition_by_manifest, prune_unreferenced_logos, referenced_tables};
-use super::types::{
- blob_spec_for, bucket_categories, bucket_category_for, is_bucketed_table, is_skip_table, is_unstamped_scope,
- manifest_scoped_tables, row_identity, SCHEMA_VERSION,
-};
-use crate::common::error::VaultResult;
-use crate::vault_model::names::LOGOS_TABLE;
+use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedManifest, CanonicalizedVault, CodecOverflow, DataBucket, Manifest, CodecRecord};
+use super::sharing::{partition_by_manifest, referenced_tables};
+use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_bucketed_table, is_skip_table, is_unstamped_manifest_id, manifest_scoped_tables, primary_key_columns_for, row_identity, SCHEMA_VERSION};
+use crate::common::error::{VaultError, VaultResult};
+use crate::vault_model::names::ID_COL;
use crate::vault_model::{ids_equal, MANIFEST_ID_COL, OVERFLOW_TABLE};
-/// Canonicalize normalized tables into the split resources: the manifest, one data bucket per declared
-/// category (see [`BUCKET_TABLES`](crate::vault_model::BUCKET_TABLES)), and the content-addressed blob map.
+type Tables = HashMap<String, Vec<CodecRecord>>;
+
+/// Canonicalize normalized tables into the split resources: one manifest per spec (in spec order), one data
+/// bucket per declared category per manifest, and the content-addressed blob map of each manifest.
pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<CanonicalizedVault> {
- let writing_spec = match input.manifests.first() {
- Some(spec) if !spec.manifest_id.is_empty() => spec.clone(),
- Some(_) => return Err(crate::common::error::VaultError::General("canonicalize requires a manifest id on every manifest".to_string())),
- None => return Err(crate::common::error::VaultError::General("canonicalize input declares no manifests".to_string())),
- };
- let writing_manifest_id = writing_spec.manifest_id.clone();
- let writing_manifest_salt = writing_spec.manifest_salt.clone();
+ if input.manifests.is_empty() {
+ return Err(VaultError::General("canonicalize input declares no manifests".to_string()));
+ }
+ if input.manifests.iter().any(|spec| spec.manifest_id.is_empty()) {
+ return Err(VaultError::General("canonicalize requires a manifest id on every manifest".to_string()));
+ }
- // Collect every non-skip table into a name > rows map (row order preserved per table). Blob
- // extraction and bucket-splitting happen below.
- let mut all_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();
+ // Collect every non-skip table into a name > rows map (row order preserved per table). The OVERFLOW_TABLE row
+ // carries a newer writer's tables/columns this client's schema could not hold (written by the last materialize);
+ // it is consumed here, folded back below, and never emitted into a manifest itself.
+ let mut all_tables = Tables::new();
let mut overflow = CodecOverflow::default();
for table in &input.tables {
- // The OVERFLOW_TABLE row carries a newer writer's tables/columns this client's schema
- // couldn't hold (written by the last materialize). Consume it here, re-merged below,
- // never emitted into the manifest itself.
if table.name == OVERFLOW_TABLE {
overflow = CodecOverflow::from_table_records(&table.records);
- continue;
+ } else if !is_skip_table(&table.name) {
+ all_tables.entry(table.name.clone()).or_default().extend(table.records.iter().cloned());
}
- if is_skip_table(&table.name) {
- continue;
- }
- all_tables.entry(table.name.clone()).or_default().extend(table.records.iter().cloned());
}
-
- // Re-merge the overflow so this push doesn't drop a newer writer's data. See `CodecOverflow`.
remerge_overflow_columns(&mut all_tables, &overflow);
- for (name, rows) in &overflow.tables {
- // Local rows win if the table somehow exists locally now (e.g. client upgraded since the pull).
- all_tables.entry(name.clone()).or_insert_with(|| rows.clone());
- }
+ fold_overflow_tables(&mut all_tables, Some(&overflow.tables));
- // Normalize all id columns.
normalize_id_spelling(&mut all_tables);
for bucket_tables in overflow.bucket_tables.values_mut() {
normalize_id_spelling(bucket_tables);
@@ -80,85 +67,28 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
}
let bucketed_names: Vec<String> = all_tables.keys().filter(|name| is_bucketed_table(name)).cloned().collect();
- let bucketed_rows: HashMap<String, Vec<CodecRecord>> = bucketed_names.into_iter().filter_map(|name| all_tables.remove_entry(&name)).collect();
-
- let snapshots: HashMap<String, Vec<CodecRecord>> = referenced_tables()
- .into_iter()
- .filter_map(|name| all_tables.get(name).map(|rows| (name.to_string(), rows.clone())))
- .collect();
- let no_rows: Vec<CodecRecord> = Vec::new();
- let all_logos = snapshots.get(LOGOS_TABLE).unwrap_or(&no_rows);
+ let bucketed_rows: Tables = bucketed_names.into_iter().filter_map(|name| all_tables.remove_entry(&name)).collect();
+ let snapshots: Tables = referenced_tables().into_iter().filter_map(|name| all_tables.get(name).map(|rows| (name.to_string(), rows.clone()))).collect();
+ let manifest_ids: Vec<String> = input.manifests.iter().map(|spec| spec.manifest_id.clone()).collect();
- let other_specs: Vec<ManifestSpec> = input.manifests.iter().skip(1).cloned().collect();
- let other_ids = other_specs.iter().map(|spec| spec.manifest_id.clone());
- let manifest_ids: Vec<String> = std::iter::once(writing_manifest_id.clone()).chain(other_ids).collect();
- let partitions = partition_by_manifest(&mut all_tables, &other_specs, &snapshots, &writing_manifest_id)?;
-
- // The writing manifest is finished exactly like every partition.
- reconcile_logo_references(&mut all_tables, &writing_manifest_id, all_logos);
- normalize_logo_scope(&mut all_tables, &writing_manifest_id);
- prune_unreferenced_logos(&mut all_tables);
- clone_referenced_rows(&mut all_tables, &writing_manifest_id, &snapshots, &writing_manifest_id);
- normalize_row_shapes(&mut all_tables);
-
- let mut blobs: HashMap<String, BlobEntry> = HashMap::new();
- let mut manifest_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();
- for (name, records) in all_tables {
- // Manifest table: extract any blob column into the content-addressed map.
- let out_rows = extract_table_blobs(&name, records, &writing_manifest_salt, &mut blobs)?;
- manifest_tables.insert(name, out_rows);
- }
-
- let data_buckets = build_data_buckets(bucketed_rows, &overflow, &manifest_ids);
-
- // Start with the manifest the caller wrote from, which the input lists first.
- let writing_extra = overflow.manifest_extra(&writing_manifest_id);
- let mut manifests: Vec<CanonicalizedManifest> = Vec::with_capacity(1 + partitions.len());
- manifests.push(CanonicalizedManifest {
- manifest: Manifest {
- schema_version: SCHEMA_VERSION,
- manifest_salt: writing_manifest_salt,
- canonicalized_at: input.canonicalized_at.clone(),
- manifest_id: writing_manifest_id,
- name: writing_spec.name.clone(),
- tables: manifest_tables,
- extra: writing_extra,
- },
- blobs,
- });
-
- // Each remaining partition becomes its own manifest, its blobs hashed with its own per-manifest salt.
- for mut partition in partitions {
+ let mut manifests = Vec::with_capacity(input.manifests.len());
+ for mut partition in partition_by_manifest(all_tables, &input.manifests, &snapshots)? {
+ integrity::repair(&mut partition.tables);
normalize_row_shapes(&mut partition.tables);
- let mut partition_blobs: HashMap<String, BlobEntry> = HashMap::new();
- let partition_tables: HashMap<String, Vec<CodecRecord>> = partition
- .tables
- .into_iter()
- .map(|(name, records)| {
- let out_rows = extract_table_blobs(&name, records, &partition.manifest_salt, &mut partition_blobs)?;
- Ok((name, out_rows))
- })
- .collect::<VaultResult<_>>()?;
- let partition_extra = overflow.manifest_extra(&partition.manifest_id);
+ let mut blobs: HashMap<String, BlobEntry> = HashMap::new();
+ let tables: Tables = partition.tables.into_iter().map(|(name, records)| Ok((name.clone(), extract_table_blobs(&name, records, &partition.spec.manifest_salt, &mut blobs)?))).collect::<VaultResult<_>>()?;
+ let extra = overflow.manifest_extra(&partition.spec.manifest_id);
manifests.push(CanonicalizedManifest {
- manifest: Manifest {
- schema_version: SCHEMA_VERSION,
- manifest_salt: partition.manifest_salt,
- canonicalized_at: input.canonicalized_at.clone(),
- manifest_id: partition.manifest_id,
- name: partition.name,
- tables: partition_tables,
- extra: partition_extra,
- },
- blobs: partition_blobs,
+ manifest: Manifest { schema_version: SCHEMA_VERSION, manifest_salt: partition.spec.manifest_salt, canonicalized_at: input.canonicalized_at.clone(), manifest_id: partition.spec.manifest_id, name: partition.spec.name, tables, extra },
+ blobs,
});
}
- Ok(CanonicalizedVault { manifests, data_buckets })
+ Ok(CanonicalizedVault { manifests, data_buckets: build_data_buckets(bucketed_rows, &overflow, &manifest_ids) })
}
/// Route every bucketed row into the bucket of the manifest that owns it: `(ManifestId, category)`.
-fn build_data_buckets(bucketed_rows: HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow, manifest_ids: &[String]) -> Vec<DataBucket> {
+fn build_data_buckets(bucketed_rows: Tables, overflow: &CodecOverflow, manifest_ids: &[String]) -> Vec<DataBucket> {
let mut data_buckets: Vec<DataBucket> = Vec::new();
for category in categories_present(overflow) {
let grouped = group_category_rows(category_tables(&category, &bucketed_rows, overflow), manifest_ids);
@@ -169,13 +99,13 @@ fn build_data_buckets(bucketed_rows: HashMap<String, Vec<CodecRecord>>, overflow
}
/// A data bucket carrying the unknown top-level keys the overflow last saw on it.
-fn bucket_with_extra(manifest_id: String, category: &str, tables: HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow) -> DataBucket {
+fn bucket_with_extra(manifest_id: String, category: &str, tables: Tables, overflow: &CodecOverflow) -> DataBucket {
let extra = overflow.bucket_extra(&manifest_id, category);
DataBucket { extra, ..DataBucket::new(manifest_id, category, tables) }
}
-/// Every bucket category to emit: the declared ones plus any a newer writer put in the overflow (a category
-/// this client's schema does not know yet still has to be carried forward). Sorted, so output stays stable.
+/// Every bucket category to emit: the declared ones plus any a newer writer put in the overflow (a category this
+/// client's schema does not know yet still has to be carried forward). Sorted, so output stays stable.
fn categories_present(overflow: &CodecOverflow) -> Vec<String> {
let mut categories: Vec<String> = bucket_categories().into_iter().map(str::to_string).collect();
let mut extra: Vec<String> = overflow.bucket_tables.keys().filter(|category| !categories.contains(category)).cloned().collect();
@@ -184,29 +114,19 @@ fn categories_present(overflow: &CodecOverflow) -> Vec<String> {
categories
}
-/// The tables read for one bucket category: the ones the local schema holds, plus whole tables a newer writer
-/// put in that bucket which this client's schema cannot hold (carried in the overflow). Local rows win.
-fn category_tables(category: &str, bucketed_rows: &HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow) -> HashMap<String, Vec<CodecRecord>> {
- let mut tables: HashMap<String, Vec<CodecRecord>> = bucketed_rows
- .iter()
- .filter(|(name, _)| bucket_category_for(name) == Some(category))
- .map(|(name, rows)| (name.clone(), rows.clone()))
- .collect();
- if let Some(ov_tables) = overflow.bucket_tables.get(category) {
- for (name, rows) in ov_tables {
- tables.entry(name.clone()).or_insert_with(|| rows.clone());
- }
- }
+/// The tables read for one bucket category: the ones the local schema holds, plus whole tables a newer writer put
+/// in that bucket which this client's schema cannot hold (carried in the overflow). Local rows win.
+fn category_tables(category: &str, bucketed_rows: &Tables, overflow: &CodecOverflow) -> Tables {
+ let mut tables: Tables = bucketed_rows.iter().filter(|(name, _)| bucket_category_for(name) == Some(category)).map(|(name, rows)| (name.clone(), rows.clone())).collect();
+ fold_overflow_tables(&mut tables, overflow.bucket_tables.get(category));
tables
}
-/// Split one bucket category's tables into the table set each manifest owns. This is the single grouping
-/// rule behind both write paths: the full push (via [`build_data_buckets`]) and the bucket-only push
-/// (via [`extract_buckets`]).
-fn group_category_rows(tables: HashMap<String, Vec<CodecRecord>>, manifest_ids: &[String]) -> HashMap<String, HashMap<String, Vec<CodecRecord>>> {
- let empty: HashMap<String, Vec<CodecRecord>> = tables.keys().map(|name| (name.clone(), Vec::new())).collect();
- let mut grouped: HashMap<String, HashMap<String, Vec<CodecRecord>>> = manifest_ids.iter().map(|id| (id.clone(), empty.clone())).collect();
-
+/// Split one bucket category's tables into the table set each manifest owns: the single grouping rule behind both
+/// the full push ([`build_data_buckets`]) and the bucket-only push ([`extract_buckets`]).
+fn group_category_rows(tables: Tables, manifest_ids: &[String]) -> HashMap<String, Tables> {
+ let empty: Tables = tables.keys().map(|name| (name.clone(), Vec::new())).collect();
+ let mut grouped: HashMap<String, Tables> = manifest_ids.iter().map(|id| (id.clone(), empty.clone())).collect();
for (name, rows) in tables {
for mut row in rows {
let Some(owner) = owning_manifest(&row, manifest_ids) else { continue };
@@ -215,25 +135,22 @@ fn group_category_rows(tables: HashMap<String, Vec<CodecRecord>>, manifest_ids:
grouped.entry(owner).or_default().entry(name.clone()).or_default().push(row);
}
}
-
grouped
}
-/// The manifest a bucketed row belongs to, spelled the way `manifest_ids` spells it, or `None` when the row
-/// names no manifest or one this vault does not carry.
+/// The manifest a bucketed row belongs to, spelled the way `manifest_ids` spells it, or `None` when the row names
+/// no manifest or one this vault does not carry.
fn owning_manifest(row: &CodecRecord, manifest_ids: &[String]) -> Option<String> {
let stamp = str_col(row, MANIFEST_ID_COL)?;
- if is_unstamped_scope(Some(stamp)) {
+ if is_unstamped_manifest_id(Some(stamp)) {
return None;
}
manifest_ids.iter().find(|id| ids_equal(id, stamp)).cloned()
}
-/// LEGACY: only the sqlite-blob migration stamps unstamped rows. Remove once every account has migrated to manifest-v1.
-///
-/// Stamp every unstamped row of a manifest-scoped table with `manifest_id`. A row that already names a
-/// manifest keeps it, so a vault that has been converted once pays nothing on later runs.
-fn stamp_unstamped_rows(tables: &mut HashMap<String, Vec<CodecRecord>>, manifest_id: &str) {
+/// LEGACY: stamp every unstamped row of a manifest-scoped table with `manifest_id`. Only the sqlite-blob migration
+/// stamps rows; remove once every account has migrated to manifest-v1.
+fn stamp_unstamped_rows(tables: &mut Tables, manifest_id: &str) {
for name in manifest_scoped_tables() {
let Some(rows) = tables.get_mut(name) else { continue };
for row in rows.iter_mut().filter(|row| is_unstamped(row)) {
@@ -243,18 +160,15 @@ fn stamp_unstamped_rows(tables: &mut HashMap<String, Vec<CodecRecord>>, manifest
}
/// Reject the whole push when any row names no manifest.
-fn reject_unstamped_rows(tables: &HashMap<String, Vec<CodecRecord>>) -> VaultResult<()> {
+fn reject_unstamped_rows(tables: &Tables) -> VaultResult<()> {
let mut names: Vec<&String> = tables.keys().collect();
names.sort();
for name in names {
let rows = &tables[name];
let unstamped = rows.iter().filter(|row| is_unstamped(row)).count();
if unstamped > 0 {
- let first = rows.iter().find(|row| is_unstamped(row)).and_then(|row| row.get("Id")).cloned().unwrap_or(Value::Null);
- return Err(crate::common::error::VaultError::General(format!(
- "the codec refuses to write {} row(s) of {} that name no manifest (first: Id {}); every row must carry the manifest it belongs to",
- unstamped, name, first
- )));
+ let first = rows.iter().find(|row| is_unstamped(row)).and_then(|row| row.get(ID_COL)).cloned().unwrap_or(Value::Null);
+ return Err(VaultError::General(format!("the codec refuses to write {} row(s) of {} that name no manifest (first: Id {}); every row must carry the manifest it belongs to", unstamped, name, first)));
}
}
Ok(())
@@ -262,10 +176,11 @@ fn reject_unstamped_rows(tables: &HashMap<String, Vec<CodecRecord>>) -> VaultRes
/// True when a row carries no usable `ManifestId`: absent, JSON null, a non-string, or the empty string.
fn is_unstamped(row: &CodecRecord) -> bool {
- is_unstamped_scope(str_col(row, MANIFEST_ID_COL))
+ is_unstamped_manifest_id(str_col(row, MANIFEST_ID_COL))
}
-/// Extract `table`'s blob column (if it owns one) into `blobs`, returning the rewritten rows.
+/// Extract `table`'s blob column (if it owns one) into `blobs`, returning the rewritten rows. A live row whose bytes
+/// are not loaded keeps the reference its local hash column remembers.
fn extract_table_blobs(table: &str, records: Vec<CodecRecord>, manifest_salt: &str, blobs: &mut HashMap<String, BlobEntry>) -> VaultResult<Vec<CodecRecord>> {
let Some(spec) = blob_spec_for(table) else { return Ok(records) };
let mut out_rows: Vec<CodecRecord> = Vec::with_capacity(records.len());
@@ -283,55 +198,31 @@ fn extract_table_blobs(table: &str, records: Vec<CodecRecord>, manifest_salt: &s
Ok(out_rows)
}
-/// Extract a blob column cell: if it holds non-empty `{ "__b64" }` bytes, hash + register them and
-/// return a blob-ref; otherwise return JSON null.
-fn extract_blob_cell(
- cell: Option<&serde_json::Value>,
- manifest_salt: &str,
- kind: &str,
- blobs: &mut HashMap<String, BlobEntry>,
-) -> VaultResult<serde_json::Value> {
- let b64 = match cell.and_then(inline_b64) {
- Some(s) => s,
- None => return Ok(serde_json::Value::Null),
- };
-
+/// Extract a blob column cell: non-empty `{ "__b64" }` bytes are hashed and registered and the cell becomes a
+/// blob reference; anything else becomes JSON null.
+fn extract_blob_cell(cell: Option<&Value>, manifest_salt: &str, kind: &str, blobs: &mut HashMap<String, BlobEntry>) -> VaultResult<Value> {
+ let Some(b64) = cell.and_then(inline_b64) else { return Ok(Value::Null) };
let bytes = match crate::common::encoding::base64_decode(b64) {
- Ok(b) if !b.is_empty() => b,
- _ => return Ok(serde_json::Value::Null),
+ Ok(bytes) if !bytes.is_empty() => bytes,
+ _ => return Ok(Value::Null),
};
-
let hash = salted_blob_hash(&bytes, manifest_salt)?;
- blobs.entry(hash.clone()).or_insert_with(|| BlobEntry {
- kind: kind.to_string(),
- bytes_base64: b64.to_string(),
- });
-
+ blobs.entry(hash.clone()).or_insert_with(|| BlobEntry { kind: kind.to_string(), bytes_base64: b64.to_string() });
Ok(blob_ref(&hash, kind))
}
-/// Re-attach overflow columns.
-fn remerge_overflow_columns(tables: &mut HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow) {
+/// Re-attach the columns the last materialize split off into the overflow, by row identity.
+fn remerge_overflow_columns(tables: &mut Tables, overflow: &CodecOverflow) {
for (table_name, by_identity) in &overflow.columns {
- let rows = match tables.get_mut(table_name) {
- Some(rows) => rows,
- None => continue,
- };
-
+ let Some(rows) = tables.get_mut(table_name) else { continue };
+ // A row that changed manifest since the overflow was written is found by its primary key alone, as long as that is unique.
let mut by_primary_key: HashMap<&str, Option<&CodecRecord>> = HashMap::new();
for (identity, extra_columns) in by_identity {
- let primary_key = primary_key_of(identity);
- by_primary_key.entry(primary_key).and_modify(|entry| *entry = None).or_insert(Some(extra_columns));
+ by_primary_key.entry(primary_key_of(table_name, identity)).and_modify(|entry| *entry = None).or_insert(Some(extra_columns));
}
-
for row in rows {
- let identity = match row_identity(table_name, row) {
- Some(v) => v,
- None => continue,
- };
- let extra_columns = by_identity
- .get(&identity)
- .or_else(|| by_primary_key.get(primary_key_of(&identity)).copied().flatten());
+ let Some(identity) = row_identity(table_name, row) else { continue };
+ let extra_columns = by_identity.get(&identity).or_else(|| by_primary_key.get(primary_key_of(table_name, &identity)).copied().flatten());
if let Some(extra_columns) = extra_columns {
for (column, value) in extra_columns {
row.entry(column.clone()).or_insert_with(|| value.clone());
@@ -341,36 +232,34 @@ fn remerge_overflow_columns(tables: &mut HashMap<String, Vec<CodecRecord>>, over
}
}
-/// The primary-key half of a row identity.
-fn primary_key_of(identity: &str) -> &str {
- identity.rsplit('\u{1f}').next().unwrap_or(identity)
+/// Add the whole tables a newer writer left in the overflow; a table that also exists locally keeps its local rows.
+fn fold_overflow_tables(tables: &mut Tables, overflow_tables: Option<&Tables>) {
+ for (name, rows) in overflow_tables.into_iter().flatten() {
+ tables.entry(name.clone()).or_insert_with(|| rows.clone());
+ }
+}
+
+/// The primary-key part of a row identity: its trailing part per primary-key column (`ItemTags` has two).
+fn primary_key_of<'a>(table_name: &str, identity: &'a str) -> &'a str {
+ let parts = primary_key_columns_for(table_name).len();
+ identity.rmatch_indices('\u{1f}').nth(parts - 1).map_or(identity, |(at, _)| &identity[at + 1..])
}
-/// Build `category`'s data buckets, one per manifest in `manifest_ids`, from the category's tables as the
-/// platform reads them out of its local vault (name > rows): the bucket-only push path, taken when a bucket
-/// changed but the manifest did not. Rows route by the manifest each one names, exactly as the full push routes them.
-///
-/// Include the [`OVERFLOW_TABLE`] row in `tables` (read it alongside the category's tables) so a newer
-/// writer's columns/tables re-merge and survive; it is consumed and never emitted into a bucket.
-pub fn extract_buckets(category: String, manifest_ids: Vec<String>, mut tables: HashMap<String, Vec<CodecRecord>>) -> VaultResult<Vec<DataBucket>> {
+/// Build `category`'s data buckets, one per manifest in `manifest_ids`, from the category's tables as the platform
+/// reads them out of its local vault (name > rows): the bucket-only push path. Rows route by the manifest each one
+/// names, exactly as the full push routes them. Include the [`OVERFLOW_TABLE`] row in `tables` so a newer writer's
+/// columns and tables re-merge and survive; it is consumed and never emitted into a bucket.
+pub fn extract_buckets(category: String, manifest_ids: Vec<String>, mut tables: Tables) -> VaultResult<Vec<DataBucket>> {
if manifest_ids.is_empty() {
- let message = "extract_buckets declares no manifests; a bucket write needs the manifests it may be addressed to";
- return Err(crate::common::error::VaultError::General(message.to_string()));
+ return Err(VaultError::General("extract_buckets declares no manifests; a bucket write needs the manifests it may be addressed to".to_string()));
}
-
let overflow = tables.remove(OVERFLOW_TABLE).map(|records| CodecOverflow::from_table_records(&records)).unwrap_or_default();
remerge_overflow_columns(&mut tables, &overflow);
- if let Some(ov_tables) = overflow.bucket_tables.get(&category) {
- for (name, rows) in ov_tables {
- tables.entry(name.clone()).or_insert_with(|| rows.clone());
- }
- }
+ fold_overflow_tables(&mut tables, overflow.bucket_tables.get(&category));
+ normalize_id_spelling(&mut tables);
reject_unstamped_rows(&tables)?;
- let mut buckets: Vec<DataBucket> = group_category_rows(tables, &manifest_ids)
- .into_iter()
- .map(|(manifest_id, tables)| bucket_with_extra(manifest_id, &category, tables, &overflow))
- .collect();
+ let mut buckets: Vec<DataBucket> = group_category_rows(tables, &manifest_ids).into_iter().map(|(manifest_id, tables)| bucket_with_extra(manifest_id, &category, tables, &overflow)).collect();
buckets.sort_by(|a, b| a.manifest_id.cmp(&b.manifest_id));
Ok(buckets)
}
diff --git a/core/rust/src/vault_codec/compress.rs b/core/rust/src/vault_codec/compress.rs
index bcf3a18a6..19b732079 100644
--- a/core/rust/src/vault_codec/compress.rs
+++ b/core/rust/src/vault_codec/compress.rs
@@ -1,12 +1,12 @@
//! gzip compress / gunzip decompress.
-use std::io::{Read, Write};
+use std::io::Write;
-use flate2::read::GzDecoder;
use flate2::write::GzEncoder;
use flate2::Compression;
use crate::common::error::{VaultError, VaultResult};
+use crate::common::gzip::{gunzip_capped, GZIP_MAGIC};
/// gzip the given bytes.
pub fn gzip(bytes: &[u8]) -> VaultResult<Vec<u8>> {
@@ -19,19 +19,20 @@ pub fn gzip(bytes: &[u8]) -> VaultResult<Vec<u8>> {
.map_err(|e| VaultError::General(format!("gzip finish failed: {}", e)))
}
-/// Decompress gzipped bytes into a UTF-8 string.
+/// Cap on a decompressed payload, so a gzip bomb in a shared manifest cannot exhaust memory. Derived from the
+/// server's default `MAX_UPLOAD_SIZE_MB` (100 MB per vault write, compressed): 5x that leaves room for highly
+/// compressible JSON while staying well below what a wasm32 client can hold.
+pub const MAX_DECOMPRESSED_PAYLOAD_BYTES: u64 = 512 * 1024 * 1024;
+
+/// Decompress gzipped bytes into a UTF-8 string, refusing output above [`MAX_DECOMPRESSED_PAYLOAD_BYTES`].
pub fn gunzip_to_string(bytes: &[u8]) -> VaultResult<String> {
- let mut decoder = GzDecoder::new(bytes);
- let mut out = String::new();
- decoder
- .read_to_string(&mut out)
- .map_err(|e| VaultError::General(format!("gunzip failed: {}", e)))?;
- Ok(out)
+ gunzip_to_string_capped(bytes, MAX_DECOMPRESSED_PAYLOAD_BYTES)
}
-/// gzip magic bytes (RFC 1952). A gzip stream always begins with these two bytes; UTF-8 JSON never
-/// does (it starts with `{`, `[`, or whitespace), so their presence unambiguously flags gzip.
-pub const GZIP_MAGIC: [u8; 2] = [0x1f, 0x8b];
+/// Decompress gzipped bytes into a UTF-8 string, refusing output above `limit` bytes.
+fn gunzip_to_string_capped(bytes: &[u8], limit: u64) -> VaultResult<String> {
+ String::from_utf8(gunzip_capped(bytes, limit)?).map_err(|e| VaultError::General(format!("gunzip failed: {}", e)))
+}
/// Decompress a packed payload to its envelope JSON string, supporting both gzipped and plain values:
/// gzip (leading `1f 8b`) is gunzipped; anything else is treated as raw, uncompressed UTF-8 JSON.
@@ -54,4 +55,11 @@ mod tests {
assert_eq!(&gz[0..2], &[0x1f, 0x8b]);
assert_eq!(gunzip_to_string(&gz).unwrap(), text);
}
+
+ #[test]
+ fn gunzip_rejects_output_over_the_limit() {
+ let gz = gzip(&[b'a'; 1025]).unwrap();
+ assert!(gunzip_to_string_capped(&gz, 1024).is_err());
+ assert_eq!(gunzip_to_string_capped(&gz, 1025).unwrap().len(), 1025);
+ }
}
diff --git a/core/rust/src/vault_codec/hash.rs b/core/rust/src/vault_codec/hash.rs
index 36ec8ae4a..8e7041a44 100644
--- a/core/rust/src/vault_codec/hash.rs
+++ b/core/rust/src/vault_codec/hash.rs
@@ -1,16 +1,9 @@
-//! Hashing + canonical JSON for the vault_codec format.
+//! Hashing and canonical JSON for the vault_codec format.
//!
-//! ## Why canonical JSON lives here
-//!
-//! Each encrypted blob embeds an integrity envelope `{ schemaVersion, contentHash, payload }` where
-//! `contentHash = sha256(canonical(payload))`. JSON serialization is NOT canonical across languages
-//! (key order, number formatting, whitespace, unicode escaping all differ), so a manifest written by
-//! one platform and re-hashed by another would fail integrity even when byte-identical. This module is
-//! the single canonical-serialization contract every binding reproduces.
-//!
-//! [`canonical_json`] is that contract: object keys sorted ascending, recursively; arrays kept in
-//! order; primitives as serde_json renders them, which matches `JSON.stringify` for the escaping and
-//! integer cases that occur in vault data, so a hash computed here agrees with one a client computed.
+//! Each encrypted payload embeds an integrity envelope `{ schemaVersion, contentHash, payload }` where
+//! `contentHash = sha256(canonical(payload))`. [`canonical_json`] is the canonical form: object keys sorted
+//! ascending, recursively; arrays kept in order; primitives as serde_json renders them. Every client hashes through
+//! this crate, so the form only has to stay stable against itself, and stored vaults depend on it staying so.
use hmac::{Hmac, Mac};
use sha2::{Digest, Sha256};
@@ -108,7 +101,8 @@ mod tests {
#[test]
fn derived_uuid_is_stable_and_well_formed() {
let id = derived_uuid("aliasvault:test");
- assert_eq!(id, derived_uuid("aliasvault:test"));
+ // Known-answer vector: derived row ids must match on every platform, so this value must never change.
+ assert_eq!(id, "eadc54c6-fb5f-89b0-93f0-18a3af670e1e");
assert_eq!(id.len(), 36);
assert_eq!(&id[14..15], "8");
assert!(matches!(&id[19..20], "8" | "9" | "a" | "b"));
diff --git a/core/rust/src/vault_codec/integrity.rs b/core/rust/src/vault_codec/integrity.rs
new file mode 100644
index 000000000..48079e187
--- /dev/null
+++ b/core/rust/src/vault_codec/integrity.rs
@@ -0,0 +1,181 @@
+//! Referential integrity of one manifest's table set: the references a row may carry and what to do when one
+//! dangles. Every foreign key is composite, `(ManifestId, Id)`, so a reference that resolves only in another
+//! manifest is as dangling as one that resolves nowhere.
+//!
+//! The same rule table serves both directions: `repair` is what canonicalize and combine apply, `violations` is
+//! what validation reports. Validation therefore cannot refuse anything a repair leaves behind.
+
+use std::collections::{HashMap, HashSet};
+
+use serde_json::Value;
+
+use super::manifest::CodecRecord;
+use super::row::{is_deleted, str_col};
+use super::types::is_bucketed_table;
+use crate::vault_model::names::{FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_VALUES_TABLE, FOLDERS_TABLE, FOLDER_ID_COL, ID_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, PARENT_FOLDER_ID_COL, TAGS_TABLE, TAG_ID_COL};
+use crate::vault_model::{id_key, MANIFEST_ID_COL, SYNCABLE_TABLES};
+
+type Tables = HashMap<String, Vec<CodecRecord>>;
+
+/// What becomes of a row whose reference dangles.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum OnDangling {
+ /// The column is nullable and the row still means something without it (an item outside any folder).
+ Null,
+ /// The reference is the row's reason to exist (a child row of an item, a tag link).
+ DropRow,
+}
+
+/// One reference rule: `table.column` must name a row of `target` in the same manifest.
+struct Reference {
+ table: String,
+ column: &'static str,
+ target: &'static str,
+ on_dangling: OnDangling,
+}
+
+impl Reference {
+ /// The stable identifier validation reports this rule under.
+ fn id(&self) -> String {
+ format!("{}-{}-fk-broken", self.table.to_lowercase(), self.column.trim_end_matches("Id").to_lowercase())
+ }
+}
+
+/// Every reference rule, parents before children so a repair never dangles what a later rule relies on.
+fn rules() -> Vec<Reference> {
+ let mut rules = vec![
+ Reference { table: FOLDERS_TABLE.to_string(), column: PARENT_FOLDER_ID_COL, target: FOLDERS_TABLE, on_dangling: OnDangling::Null },
+ Reference { table: ITEMS_TABLE.to_string(), column: FOLDER_ID_COL, target: FOLDERS_TABLE, on_dangling: OnDangling::Null },
+ ];
+ // Every registered table whose rows hang off an item through `ItemId`; the stats bucket keys on the item id itself and syncs apart.
+ for child in SYNCABLE_TABLES.iter().filter(|t| t.item_child && !is_bucketed_table(t.name) && t.item_ref_column() == ITEM_ID_COL) {
+ rules.push(Reference { table: child.name.to_string(), column: ITEM_ID_COL, target: ITEMS_TABLE, on_dangling: OnDangling::DropRow });
+ }
+ rules.push(Reference { table: ITEM_TAGS_TABLE.to_string(), column: TAG_ID_COL, target: TAGS_TABLE, on_dangling: OnDangling::DropRow });
+ // A value with no definition is how a system field is stored, so nulling keeps what the user typed.
+ rules.push(Reference { table: FIELD_VALUES_TABLE.to_string(), column: FIELD_DEFINITION_ID_COL, target: FIELD_DEFINITIONS_TABLE, on_dangling: OnDangling::Null });
+ rules.push(Reference { table: FIELD_HISTORIES_TABLE.to_string(), column: FIELD_DEFINITION_ID_COL, target: FIELD_DEFINITIONS_TABLE, on_dangling: OnDangling::Null });
+ rules
+}
+
+/// Repair every dangling reference in a manifest's table set, and drop the rows of tombstoned items: an item is
+/// deleted as a unit, so children a merge or an older client left under a tombstone leave with it. A rule whose
+/// target table is absent from the set is skipped: that is a partial table set, not a vault without that table.
+pub(crate) fn repair(tables: &mut Tables) {
+ for rule in rules() {
+ let Some(targets) = qualified_ids(tables, rule.target) else { continue };
+ let Some(rows) = tables.get_mut(&rule.table) else { continue };
+ match rule.on_dangling {
+ OnDangling::Null => {
+ for row in rows.iter_mut().filter(|row| dangles(row, rule.column, &targets)) {
+ row.insert(rule.column.to_string(), Value::Null);
+ }
+ }
+ OnDangling::DropRow => rows.retain(|row| !dangles(row, rule.column, &targets)),
+ }
+ }
+ drop_children_of_tombstoned_items(tables);
+}
+
+/// Every dangling reference in a manifest's table set, as `(rule id, description)`, at most one per rule.
+pub(crate) fn violations(tables: &Tables) -> Vec<(String, String)> {
+ let mut out = Vec::new();
+ for rule in rules() {
+ let Some(targets) = qualified_ids(tables, rule.target) else { continue };
+ let Some(rows) = tables.get(&rule.table) else { continue };
+ if let Some(row) = rows.iter().find(|row| dangles(row, rule.column, &targets)) {
+ out.push((rule.id(), format!("{} {} references missing {} {}", rule.table, str_col(row, ID_COL).unwrap_or(""), rule.target, str_col(row, rule.column).unwrap_or(""))));
+ }
+ }
+ out
+}
+
+/// Whether `row.column` names something that is not in `targets` (an absent or null column dangles nothing).
+fn dangles(row: &CodecRecord, column: &str, targets: &HashSet<(String, String)>) -> bool {
+ match str_col(row, column) {
+ None => false,
+ Some(reference) => !targets.contains(&qualified(row, reference)),
+ }
+}
+
+/// The `(ManifestId, Id)` identity of every row in `table`, `None` when the table is absent from the set.
+fn qualified_ids(tables: &Tables, table: &str) -> Option<HashSet<(String, String)>> {
+ Some(tables.get(table)?.iter().filter_map(|row| str_col(row, ID_COL).map(|id| qualified(row, id))).collect())
+}
+
+/// `id` as seen from `row`'s own manifest.
+fn qualified(row: &CodecRecord, id: &str) -> (String, String) {
+ (id_key(str_col(row, MANIFEST_ID_COL).unwrap_or_default()), id_key(id))
+}
+
+/// Remove every row hanging off a tombstoned item.
+fn drop_children_of_tombstoned_items(tables: &mut Tables) {
+ let deleted: HashSet<(String, String)> = tables
+ .get(ITEMS_TABLE)
+ .map(|items| items.iter().filter(|item| is_deleted(item)).filter_map(|item| str_col(item, ID_COL).map(|id| qualified(item, id))).collect())
+ .unwrap_or_default();
+ if deleted.is_empty() {
+ return;
+ }
+ for child in SYNCABLE_TABLES.iter().filter(|table| table.item_child) {
+ let Some(rows) = tables.get_mut(child.name) else { continue };
+ rows.retain(|row| str_col(row, child.item_ref_column()).is_none_or(|item| !deleted.contains(&qualified(row, item))));
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use super::super::test_support::{row, stamped, PERSONAL_MANIFEST_ID as M};
+ use serde_json::json;
+
+ fn tables(entries: &[(&str, Vec<CodecRecord>)]) -> Tables {
+ entries.iter().map(|(name, rows)| (name.to_string(), rows.clone())).collect()
+ }
+
+ #[test]
+ fn a_reference_that_resolves_only_in_another_manifest_dangles() {
+ let mut set = tables(&[
+ ("Folders", vec![row(&[("ManifestId", json!("m-other")), ("Id", json!("f-1"))])]),
+ ("Items", vec![stamped(M, &[("Id", json!("i-1")), ("FolderId", json!("f-1"))])]),
+ ]);
+ assert_eq!(violations(&set).len(), 1);
+ repair(&mut set);
+ assert_eq!(set["Items"][0]["FolderId"], Value::Null);
+ assert!(violations(&set).is_empty(), "validation cannot refuse what repair leaves behind");
+ }
+
+ #[test]
+ fn child_rows_are_dropped_and_nullable_references_nulled() {
+ let mut set = tables(&[
+ ("Folders", vec![stamped(M, &[("Id", json!("f-1")), ("ParentFolderId", json!("gone"))])]),
+ ("Items", vec![stamped(M, &[("Id", json!("i-1")), ("FolderId", json!("gone")), ("IsDeleted", json!(0))]), stamped(M, &[("Id", json!("i-dead")), ("IsDeleted", json!(1))])]),
+ ("Tags", vec![]),
+ ("FieldDefinitions", vec![]),
+ ("TotpCodes", vec![stamped(M, &[("Id", json!("t-1")), ("ItemId", json!("i-1"))]), stamped(M, &[("Id", json!("t-orphan")), ("ItemId", json!("nobody"))]), stamped(M, &[("Id", json!("t-dead")), ("ItemId", json!("i-dead"))])]),
+ ("ItemTags", vec![stamped(M, &[("Id", json!("it-1")), ("ItemId", json!("i-1")), ("TagId", json!("no-tag"))])]),
+ ("FieldValues", vec![stamped(M, &[("Id", json!("fv-1")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("no-def")), ("Value", json!("kept"))])]),
+ ]);
+ let ids: Vec<String> = violations(&set).into_iter().map(|(id, _)| id).collect();
+ assert_eq!(ids, vec!["folders-parentfolder-fk-broken", "items-folder-fk-broken", "totpcodes-item-fk-broken", "itemtags-tag-fk-broken", "fieldvalues-fielddefinition-fk-broken"]);
+
+ repair(&mut set);
+
+ assert_eq!(set["Folders"][0]["ParentFolderId"], Value::Null);
+ assert_eq!(set["Items"][0]["FolderId"], Value::Null);
+ assert_eq!(set["TotpCodes"].iter().map(|r| r["Id"].as_str().unwrap()).collect::<Vec<_>>(), vec!["t-1"], "the orphan and the tombstoned item's child are gone");
+ assert!(set["ItemTags"].is_empty());
+ assert_eq!(set["FieldValues"][0]["Value"], json!("kept"), "the value survives, only its definition link is cut");
+ assert_eq!(set["FieldValues"][0]["FieldDefinitionId"], Value::Null);
+ assert!(violations(&set).is_empty());
+ }
+
+ #[test]
+ fn a_rule_whose_target_table_is_absent_is_skipped() {
+ let mut set = tables(&[("Items", vec![stamped(M, &[("Id", json!("i-1")), ("FolderId", json!("f-1"))])]), ("TotpCodes", vec![stamped(M, &[("Id", json!("t-1")), ("ItemId", json!("i-1"))])])]);
+ let before = set.clone();
+ repair(&mut set);
+ assert_eq!(set, before, "a partial table set is not a vault with empty tables");
+ assert!(violations(&set).is_empty());
+ }
+}
diff --git a/core/rust/src/vault_codec/logos.rs b/core/rust/src/vault_codec/logos.rs
new file mode 100644
index 000000000..255b22866
--- /dev/null
+++ b/core/rust/src/vault_codec/logos.rs
@@ -0,0 +1,239 @@
+//! Logo identity: one `Logos` row per `(ManifestId, Kind, Source)` per *manifest*.
+//!
+//! One row shape covers every logo an item can have, and `Items.LogoId` is the single pointer to it:
+//! - `Kind = "favicon"`, `Source` = the domain the logo was fetched from (`github.com`);
+//! - `Kind = "builtin"`, `Source` = a key into the shared built-in catalog (`shopping`), no image bytes:
+//! every platform draws that logo itself;
+//! - `Kind = "custom"`, `Source` = the sha256 of the image the user uploaded.
+//!
+//! A logo belongs to the manifest that owns it: `ManifestId` is that manifest's id, and `Id` derives from
+//! `(manifest id, Kind, Source)` (see [`logo_id_for`]), so every writer derives the same id for the same logo in
+//! the same manifest and the uniqueness invariant is self-enforcing rather than repaired after the fact. A logo
+//! is never shared across manifests: an item that moves into another manifest gets a copy under that manifest's
+//! id, and a favicon nothing references any more is dropped, since the client can fetch it again.
+
+use std::collections::{HashMap, HashSet};
+
+use serde_json::{json, Value};
+
+use super::manifest::CodecRecord;
+use crate::common::timestamp::updated_at;
+use super::row::{has_bytes, is_deleted, logo_kind, normalize_logo_kind, str_col};
+use crate::vault_model::names::{FILE_DATA_COL, ID_COL, ITEMS_TABLE, KIND_COL, LOGOS_TABLE, LOGO_ID_COL, LOGO_KIND_CUSTOM, LOGO_KIND_FAVICON, SOURCE_COL, UPDATED_AT_COL};
+use crate::vault_model::{id_key, ids_equal, MANIFEST_ID_COL};
+
+type Tables = HashMap<String, Vec<CodecRecord>>;
+
+/// Domain-separation prefix for favicon ids. It predates the `Kind` column and is kept as-is so every favicon row
+/// that already exists keeps its id: changing it would re-derive the logo id of every item in every vault.
+const FAVICON_ID_NAMESPACE: &str = "aliasvault:logo:v1";
+
+/// The `Id` of the logo `(manifest id, kind, source)`: a UUIDv8 whose bytes come from
+/// `sha256(namespace | manifest id | source)`, with a namespace per kind so the three key spaces (a domain, a
+/// catalog key, a content hash) can never collide. `source` is matched case-insensitively.
+pub fn logo_id_for(manifest_id: &str, kind: &str, source: &str) -> String {
+ super::hash::derived_uuid(&format!("{}\n{}\n{}", namespace_for_kind(kind), id_key(manifest_id), source.to_lowercase()))
+}
+
+/// The derivation namespace for a kind. An unknown kind gets one derived from its own name, so a newer client
+/// can add a kind without this one having to know about it.
+fn namespace_for_kind(kind: &str) -> String {
+ match normalize_logo_kind(kind) {
+ k if k == LOGO_KIND_FAVICON => FAVICON_ID_NAMESPACE.to_string(),
+ k => format!("aliasvault:logo:{}:v1", k),
+ }
+}
+
+/// True when a row holds an image the user supplied themselves, which the client cannot produce again.
+fn is_custom_logo(row: &CodecRecord) -> bool {
+ logo_kind(row) == LOGO_KIND_CUSTOM
+}
+
+/// The `(kind, source)` natural key of a row, or `None` when it carries no `Source` to key on.
+fn natural_key(row: &CodecRecord) -> Option<(String, String)> {
+ let source = str_col(row, SOURCE_COL)?.to_lowercase();
+ Some((logo_kind(row), source))
+}
+
+/// Bring one manifest's logos into that manifest: clone in the logos its items reference from elsewhere, derive every
+/// row's id from `(manifest id, Kind, Source)` and collapse rows that then share one, repoint `Items.LogoId`, and drop
+/// the favicons nothing references. `all_logos` is the vault-wide snapshot the clones come from.
+pub(super) fn localize_logos(tables: &mut Tables, manifest_id: &str, all_logos: &[CodecRecord]) {
+ reconcile_logo_references(tables, manifest_id, all_logos);
+ let remap = rewrite_logo_rows(tables, manifest_id);
+ repoint_items(tables, &remap);
+ prune_unreferenced_logos(tables);
+}
+
+/// Stamp and re-derive one manifest's logos without cloning or pruning: the combine direction, where a manifest
+/// written before logos were per manifest (random ids, no stamps) has to land in its own uniqueness bucket.
+pub(super) fn restamp_logos(tables: &mut Tables, manifest_id: &str) {
+ let remap = rewrite_logo_rows(tables, manifest_id);
+ repoint_items(tables, &remap);
+}
+
+/// Rewrite `Logos` rows to `manifest_id` and return the `old Id -> new Id` map. Rows without a `Source` cannot be
+/// addressed by the natural key, so they keep their identity and are left alone.
+fn rewrite_logo_rows(tables: &mut Tables, manifest_id: &str) -> HashMap<String, String> {
+ let mut remap: HashMap<String, String> = HashMap::new();
+ let logos = match tables.get_mut(LOGOS_TABLE) {
+ Some(rows) if !rows.is_empty() => rows,
+ _ => return remap,
+ };
+
+ // (kind, source) -> index of the row that survives (deterministic, see `is_better_logo`).
+ let mut survivor_idx: HashMap<(String, String), usize> = HashMap::new();
+ for (idx, row) in logos.iter().enumerate() {
+ let Some(key) = natural_key(row) else { continue };
+ match survivor_idx.get(&key) {
+ Some(&cur) if !is_better_logo(row, &logos[cur]) => {}
+ _ => {
+ survivor_idx.insert(key, idx);
+ }
+ }
+ }
+
+ // Every row with a natural key maps onto that key's survivor id, and the survivor itself is rewritten in place.
+ let survivors: HashSet<usize> = survivor_idx.values().copied().collect();
+ for (idx, row) in logos.iter().enumerate() {
+ let (Some((kind, source)), Some(old_id)) = (natural_key(row), str_col(row, ID_COL)) else { continue };
+ let new_id = logo_id_for(manifest_id, &kind, &source);
+ if old_id != new_id || !survivors.contains(&idx) {
+ remap.insert(id_key(old_id), new_id);
+ }
+ }
+
+ let manifest_id_value = json!(manifest_id);
+ let mut kept: Vec<CodecRecord> = Vec::with_capacity(survivors.len());
+ for (idx, mut row) in std::mem::take(logos).into_iter().enumerate() {
+ let Some((kind, source)) = natural_key(&row) else {
+ kept.push(row);
+ continue;
+ };
+ if !survivors.contains(&idx) {
+ continue;
+ }
+ row.insert(ID_COL.to_string(), json!(logo_id_for(manifest_id, &kind, &source)));
+ row.insert(KIND_COL.to_string(), json!(kind));
+ row.insert(MANIFEST_ID_COL.to_string(), manifest_id_value.clone());
+ kept.push(row);
+ }
+ *logos = kept;
+
+ remap
+}
+
+/// Repair every `Items.LogoId` in this table set: follow `remap`, then null a reference that resolves to no logo
+/// present here (dangling, like the FK's `ON DELETE SET NULL`).
+fn repoint_items(tables: &mut Tables, remap: &HashMap<String, String>) {
+ let valid_ids = logo_ids(tables);
+ let Some(items) = tables.get_mut(ITEMS_TABLE) else { return };
+ for item in items.iter_mut() {
+ let Some(current) = str_col(item, LOGO_ID_COL).map(str::to_string) else { continue };
+ let resolved = remap.get(&id_key(&current)).cloned().unwrap_or(current);
+ let repaired = if valid_ids.contains(&id_key(&resolved)) { json!(resolved) } else { Value::Null };
+ item.insert(LOGO_ID_COL.to_string(), repaired);
+ }
+}
+
+/// Clone the logos this manifest's items reference but the manifest does not hold, under their manifest-local id,
+/// reusing a row the manifest already has for the same `(kind, source)`.
+fn reconcile_logo_references(tables: &mut Tables, manifest_id: &str, all_logos: &[CodecRecord]) {
+ if !tables.contains_key(ITEMS_TABLE) {
+ return;
+ }
+ let present = logo_ids(tables);
+ let mut missing: Vec<String> = tables[ITEMS_TABLE].iter().filter_map(|item| str_col(item, LOGO_ID_COL)).filter(|id| !present.contains(&id_key(id))).map(str::to_string).collect();
+ missing.sort();
+ missing.dedup();
+ if missing.is_empty() {
+ return;
+ }
+
+ let mut id_by_key: HashMap<(String, String), String> = tables.get(LOGOS_TABLE).map(|rows| rows.iter().filter_map(|r| Some((natural_key(r)?, str_col(r, ID_COL)?.to_string()))).collect()).unwrap_or_default();
+ let mut remap: HashMap<String, String> = HashMap::new();
+ let mut clones: Vec<CodecRecord> = Vec::new();
+ let manifest_id_value = json!(manifest_id);
+ for missing_id in missing {
+ // The referenced row as it exists in its original manifest, if it exists at all.
+ let Some(origin) = all_logos.iter().find(|row| str_col(row, ID_COL).is_some_and(|id| ids_equal(id, &missing_id))) else { continue };
+ let Some((kind, source)) = natural_key(origin) else { continue };
+
+ if let Some(existing_id) = id_by_key.get(&(kind.clone(), source.clone())).cloned() {
+ refill_empty_logo_row(tables, &existing_id, origin);
+ remap.insert(id_key(&missing_id), existing_id);
+ continue;
+ }
+
+ let local_id = logo_id_for(manifest_id, &kind, &source);
+ let mut clone = origin.clone();
+ clone.insert(ID_COL.to_string(), json!(local_id));
+ clone.insert(KIND_COL.to_string(), json!(kind.clone()));
+ clone.insert(MANIFEST_ID_COL.to_string(), manifest_id_value.clone());
+ clones.push(clone);
+ id_by_key.insert((kind, source), local_id.clone());
+ remap.insert(id_key(&missing_id), local_id);
+ }
+
+ if !clones.is_empty() {
+ tables.entry(LOGOS_TABLE.to_string()).or_default().extend(clones);
+ }
+ repoint_items(tables, &remap);
+}
+
+/// Refill this manifest's row for a natural key from the row an incoming item pointed at, when that row carries no
+/// image (or is tombstoned) and the incoming one does.
+fn refill_empty_logo_row(tables: &mut Tables, existing_id: &str, origin: &CodecRecord) {
+ let Some(logos) = tables.get_mut(LOGOS_TABLE) else { return };
+ let Some(existing) = logos.iter_mut().find(|r| str_col(r, ID_COL) == Some(existing_id)) else { return };
+ let upgrades = (!has_file_data(existing) && has_file_data(origin)) || (is_deleted(existing) && !is_deleted(origin));
+ if !upgrades {
+ return;
+ }
+
+ // Take the origin's content, keep this manifest's identity, and never move the row's clock backwards: the healed
+ // row has to win last-writer-wins against the empty one still sitting on other devices.
+ let (previous, previous_at) = (existing.get(UPDATED_AT_COL).cloned(), updated_at(existing));
+ for (column, value) in origin.iter() {
+ if column == ID_COL || column == MANIFEST_ID_COL {
+ continue;
+ }
+ existing.insert(column.clone(), value.clone());
+ }
+ if let Some(previous) = previous.filter(|_| previous_at > updated_at(existing)) {
+ existing.insert(UPDATED_AT_COL.to_string(), previous);
+ }
+}
+
+/// Drop the logos no item in this table set references, except the ones the user uploaded: a favicon or built-in
+/// logo can be produced again, an uploaded image cannot.
+fn prune_unreferenced_logos(tables: &mut Tables) {
+ let referenced: HashSet<String> = tables.get(ITEMS_TABLE).map(|rows| rows.iter().filter_map(|r| str_col(r, LOGO_ID_COL)).map(id_key).collect()).unwrap_or_default();
+ if let Some(logos) = tables.get_mut(LOGOS_TABLE) {
+ logos.retain(|row| is_custom_logo(row) || str_col(row, ID_COL).is_some_and(|id| referenced.contains(&id_key(id))));
+ }
+}
+
+/// Every `Logos.Id` present in this table set, lowercased.
+fn logo_ids(tables: &Tables) -> HashSet<String> {
+ tables.get(LOGOS_TABLE).map(|rows| rows.iter().filter_map(|r| str_col(r, ID_COL).map(id_key)).collect()).unwrap_or_default()
+}
+
+/// Which row's *content* survives a natural-key collision within one manifest (the id is derived either way): a live
+/// row beats a tombstoned one, a row with image bytes beats an empty one, and the highest `Id` breaks the tie.
+fn is_better_logo(candidate: &CodecRecord, incumbent: &CodecRecord) -> bool {
+ let (cand_live, inc_live) = (!is_deleted(candidate), !is_deleted(incumbent));
+ if cand_live != inc_live {
+ return cand_live;
+ }
+ let (cand_has_data, inc_has_data) = (has_file_data(candidate), has_file_data(incumbent));
+ if cand_has_data != inc_has_data {
+ return cand_has_data;
+ }
+ str_col(candidate, ID_COL).unwrap_or("") > str_col(incumbent, ID_COL).unwrap_or("")
+}
+
+/// True when `FileData` holds actual bytes; a tombstoned row's NULL must not beat a row with an image.
+fn has_file_data(row: &CodecRecord) -> bool {
+ has_bytes(row.get(FILE_DATA_COL))
+}
diff --git a/core/rust/src/vault_codec/manifest.rs b/core/rust/src/vault_codec/manifest.rs
index d6b6e8af1..1ba51ac43 100644
--- a/core/rust/src/vault_codec/manifest.rs
+++ b/core/rust/src/vault_codec/manifest.rs
@@ -110,19 +110,6 @@ pub struct CanonicalizedVault {
pub data_buckets: Vec<DataBucket>,
}
-#[cfg(test)]
-impl CanonicalizedVault {
- /// The first manifest, which is the one the caller wrote this vault from. Canonicalize refuses empty input, so it always exists.
- pub fn first(&self) -> &CanonicalizedManifest {
- self.manifests.first().expect("canonicalize rejects input declaring no manifests")
- }
-
- /// Every manifest after the first, in spec order.
- pub fn rest(&self) -> Vec<&CanonicalizedManifest> {
- self.manifests.iter().skip(1).collect()
- }
-}
-
/// A single table's rows for reassembly.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CodecTableData {
@@ -130,12 +117,8 @@ pub struct CodecTableData {
pub records: Vec<CodecRecord>,
}
-/// Data a newer writer put in the manifest that this client's local SQLite schema cannot hold.
-///
-/// Materialize splits it off (so inserts don't crash on unknown tables/columns) and canonicalize
-/// re-merges it (so this client's next push never drops it). The platform persists this value
-/// opaquely between pull and push; it is rebuilt from scratch on every pull, so it tracks the same
-/// staleness/LWW semantics as the rest of the row data.
+/// Data a newer writer put in the manifest that the local schema cannot hold: split off by materialize, re-merged by
+/// canonicalize, so this client's next push never drops it.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CodecOverflow {
@@ -146,7 +129,7 @@ pub struct CodecOverflow {
/// category so both full pushes and bucket-only pushes re-emit them into the right bucket.
#[serde(default)]
pub bucket_tables: HashMap<String, HashMap<String, Vec<CodecRecord>>>,
- /// Unknown columns split off rows of known tables: table > row primary-key value > {column: value}.
+ /// Unknown columns split off rows of known tables: table > row identity > {column: value}.
#[serde(default)]
pub columns: HashMap<String, HashMap<String, CodecRecord>>,
/// Unknown top-level manifest keys: lowercased manifest id > {key: value}.
@@ -209,11 +192,8 @@ impl Manifest {
}
}
-/// Materialized tables the platform inserts into a fresh schema DB. Blob columns carry
-/// `{ "__blobRef": hash }`; inline byte columns carry `{ "__b64": ... }`. Any overflow (see
-/// [`CodecOverflow`]) is already included in `tables` as the `OVERFLOW_TABLE` row, the platform
-/// inserts it like any other table and needs no separate persistence. The `overflow` field is a
-/// diagnostics copy of the same data (for logging), not something the platform must store.
+/// Materialized tables the platform inserts into a fresh schema DB, the overflow included as the `OVERFLOW_TABLE` row.
+/// `overflow` is a diagnostics copy of that row, not something the platform must store.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MaterializedTables {
diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rs
index 13fb8cfe2..66ed7c7e1 100644
--- a/core/rust/src/vault_codec/materialize.rs
+++ b/core/rust/src/vault_codec/materialize.rs
@@ -1,18 +1,8 @@
-//! Materialize the canonical persisted representation as a concrete SQLite projection.
+//! Materialize the canonical representation as SQLite tables; the inverse of `canonicalize`.
//!
-//! `materialize_as_sqlite` is the SQLite adapter of the materialize direction. SQLite is one possible
-//! projection of the canonical dataset, not an authoritative destination. Future targets would add
-//! sibling `materialize_as_*` entry points. The inverse direction lives in `canonicalize`.
-//!
-//! Forward compatibility: the caller supplies its local schema (`schema_columns`), and anything a
-//! newer writer put in the manifest that this schema cannot hold (whole unknown tables or unknown
-//! columns on known tables) is split into [`CodecOverflow`] instead of being emitted (which would
-//! crash the platform insert). The overflow is emitted as a regular table row (`OVERFLOW_TABLE`),
-//! so it lives inside the vault DB itself and `canonicalize_from_sqlite` re-merges it from the
-//! ordinary table read, this client's next push never drops the data, and no platform has to wire
-//! (or remember) a separate persistence channel. Unknown top-level keys of a manifest or bucket ride in
-//! the same overflow. A manifest or bucket written at a newer major format version is refused and requires
-//! updating the app to read it.
+//! Whatever the caller's schema (`schema_columns`) cannot hold is split into [`CodecOverflow`] and emitted as the
+//! `OVERFLOW_TABLE` row, so it lives in the vault DB and `canonicalize_from_sqlite` re-merges it on the next push.
+//! A manifest or bucket written at a newer major format version is refused.
use std::collections::{HashMap, HashSet};
@@ -25,13 +15,11 @@ use crate::common::error::{VaultError, VaultResult};
use crate::vault_model::names::ID_COL;
use crate::vault_model::{id_key, MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_TABLE};
-/// Materialize the vault's manifests into the table set the platform inserts. Every manifest arrives
-/// in one list, each carrying its own data buckets; they are combined into a single table set with
-/// per-manifest logo scoping and key-scope filtering.
+/// Materialize the vault's manifests into the table set the platform inserts: every manifest's rows combined into
+/// one table set, each row stamped with the manifest it arrived in, plus the data buckets folded back in.
pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<MaterializedTables> {
- let MaterializeInput { mut manifests, data_buckets, schema_columns } = input;
+ let MaterializeInput { manifests, data_buckets, schema_columns } = input;
- // Check for a non-empty schema.
if schema_columns.is_empty() {
return Err(VaultError::General("materialize input carries an empty schema_columns map".to_string()));
}
@@ -56,19 +44,12 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
}
let manifest_records = manifest_bookkeeping_records(&manifests);
-
- // The first manifest is the caller's own (see `MaterializeInput::manifests`).
- let base = manifests.remove(0);
- let others: Vec<Manifest> = manifests;
-
- let base_manifest_id = base.manifest_id.clone();
- let mut combined = super::sharing::combine_manifest_tables(base.tables, &base_manifest_id, others);
+ let mut combined = super::sharing::combine_manifest_tables(manifests);
// The wire omits derived row ids (single-value FieldValues, FieldHistories); derive them back so the
// SQLite projection has the primary keys it expects back, identical on every device.
super::normalize::derive_missing_ids(&mut combined);
- // Normalize all id columns.
super::normalize::normalize_id_spelling(&mut combined);
let mut tables: Vec<CodecTableData> = Vec::with_capacity(combined.len() + data_buckets.len());
@@ -160,8 +141,7 @@ enum SplitResult {
UnknownTable(Vec<CodecRecord>),
}
-/// Fit `records` to the caller's schema. Unknown columns are stashed in `column_overflow` keyed by
-/// the row's primary-key value.
+/// Fit `records` to the caller's schema. Unknown columns are stashed in `column_overflow` keyed by row identity.
fn split_for_schema(
table_name: &str,
records: Vec<CodecRecord>,
diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rs
index 7f7fcb227..8c34a3294 100644
--- a/core/rust/src/vault_codec/mod.rs
+++ b/core/rust/src/vault_codec/mod.rs
@@ -1,20 +1,17 @@
-//! Vault codec: logic for translating between the canonical manifest-v1 storage format (as persisted on the server)
-//! and local vault formats (e.g., SQLite, or others), including integrity envelope (canonical hash),
-//! gzip packing/unpacking, and structural validation.
-//!
-//! This module defines the *format* codec that maps between canonical artifacts (manifest, data buckets,
-//! content-addressed blobs) and platform-specific representations, without embedding knowledge of encryption
-//! or storage engine internals. Each platform interacts with its own storage and applies encryption/decryption
-//! outside of this codec.
+//! The manifest-v1 storage format: `canonicalize` maps the local SQLite tables to manifests, data buckets and
+//! content-addressed blobs, `materialize` maps them back. The codec knows the format rules (routing by manifest,
+//! `integrity`, `logos`, `normalize`), the payload envelope (`hash`, `compress`) and `validate`; encryption and
+//! storage stay with the caller.
mod compress;
mod canonicalize;
+pub(crate) mod integrity;
pub(crate) mod normalize;
mod hash;
+mod logos;
mod manifest;
mod materialize;
pub(crate) mod row;
-mod scoped_assets;
mod sharing;
#[cfg(test)]
pub(crate) mod test_support;
@@ -33,9 +30,9 @@ pub use manifest::{
Manifest, MaterializeInput, MaterializedTables, CodecRecord, CodecTableData, ManifestSpec,
};
pub use materialize::materialize_as_sqlite;
-pub use scoped_assets::logo_id_for;
+pub use logos::logo_id_for;
pub use sharing::extract_encryption_key_for_public_key;
-pub(crate) use types::ensure_readable_schema_version;
+pub(crate) use types::{blob_spec_for, ensure_readable_schema_version};
pub use types::{bucket_categories, identity_part, is_bucketed_table, is_readable_schema_version, is_skip_table, manifest_scoped_tables, tables_for_category};
pub use validate::{validate_data_bucket, validate_manifest, ValidationResult};
diff --git a/core/rust/src/vault_codec/normalize.rs b/core/rust/src/vault_codec/normalize.rs
index 3bf64fdd0..508c23751 100644
--- a/core/rust/src/vault_codec/normalize.rs
+++ b/core/rust/src/vault_codec/normalize.rs
@@ -1,17 +1,18 @@
-//! Normalization of row shapes for converting from materialized SQLite to the manifest format to save on filesize.
+//! The wire shape of rows: ids lowercased, derived ids stripped (FieldValues, FieldHistories, ItemTags) and
+//! multi-value fields renumbered, so every writer produces the same manifest for the same data.
use std::collections::{HashMap, HashSet};
use serde_json::json;
use super::manifest::CodecRecord;
-use super::row::{is_deleted, str_col, truthy};
+use super::row::{str_col, truthy};
use super::types::{is_guid, is_id_column};
use crate::common::timestamp::updated_at;
-use crate::vault_model::{id_key, MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS, SINGLE_VALUE_FIELD_KEYS, SYNCABLE_TABLES};
+use crate::vault_model::{id_key, MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS, SINGLE_VALUE_FIELD_KEYS};
use crate::vault_model::names::{
CHANGED_AT_COL, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_KEY_COL,
- FIELD_VALUES_TABLE, ID_COL, IS_MULTI_VALUE_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, TAG_ID_COL, VALUE_INDEX_COL,
+ FIELD_VALUES_TABLE, ID_COL, IS_MULTI_VALUE_COL, ITEM_ID_COL, ITEM_TAGS_TABLE, TAG_ID_COL, VALUE_INDEX_COL,
};
/// Domain-separation prefix for derived field value ids.
@@ -26,10 +27,8 @@ pub fn field_value_id_for(manifest_id: &str, item_id: &str, field_key: &str, fie
derived_field_id(FIELD_VALUE_ID_NAMESPACE, manifest_id, item_id, field_key, field_definition_id, &value_index.to_string())
}
-/// The `FieldHistories.Id` of the history row `(manifest, item, field, changed at)`: every row derives
-/// it, since `ChangedAt` (millisecond precision) is the natural discriminator: two devices changing the
-/// same field concurrently snapshot at different times and union. Two snapshots of one field in the very
-/// same millisecond collapse to one, which history can afford. `changed_at` is used as-is.
+/// The `FieldHistories.Id` of the history row `(manifest, item, field, changed at)`; two snapshots of one field in the
+/// same millisecond collapse to one.
pub fn field_history_id_for(manifest_id: &str, item_id: &str, field_key: &str, field_definition_id: &str, changed_at: &str) -> String {
derived_field_id(FIELD_HISTORY_ID_NAMESPACE, manifest_id, item_id, field_key, field_definition_id, changed_at)
}
@@ -64,11 +63,9 @@ pub(crate) fn normalize_row_id_spelling(row: &mut CodecRecord) {
}
}
-/// Normalize the shape of rows for converting from materialized SQLite to the manifest format to save on filesize.
+/// Normalize the rows of the tables whose wire shape differs from their SQLite shape. The caller repairs
+/// referential integrity first (see [`integrity`](super::integrity)); this only reshapes rows.
pub(crate) fn normalize_row_shapes(tables: &mut HashMap<String, Vec<CodecRecord>>) {
- super::sharing::prune_dangling_references(tables);
- // Drop rows of deleted items first (any leftover state after a potential delete/update scenario in LWW merge).
- drop_children_of_deleted_items(tables);
let multi_value_defs = multi_value_definition_ids(tables);
if let Some(rows) = tables.get_mut(FIELD_VALUES_TABLE) {
normalize_field_values(rows, &multi_value_defs);
@@ -81,26 +78,6 @@ pub(crate) fn normalize_row_shapes(tables: &mut HashMap<String, Vec<CodecRecord>
}
}
-/*
- * Check for deleted items and remove their child rows in case they are still present after a LWW merge (leftover state).
- */
-fn drop_children_of_deleted_items(tables: &mut HashMap<String, Vec<CodecRecord>>) {
- let deleted: HashSet<(String, String)> = tables
- .get(ITEMS_TABLE)
- .map(|items| items.iter().filter(|item| is_deleted(item)).filter_map(|item| Some((lower_str(item, MANIFEST_ID_COL)?, lower_str(item, ID_COL)?))).collect())
- .unwrap_or_default();
- if deleted.is_empty() {
- return;
- }
- for child in SYNCABLE_TABLES.iter().filter(|table| table.item_child) {
- let Some(rows) = tables.get_mut(child.name) else { continue };
- rows.retain(|row| match (lower_str(row, MANIFEST_ID_COL), lower_str(row, child.item_ref_column())) {
- (Some(manifest), Some(item)) => !deleted.contains(&(manifest, item)),
- _ => true,
- });
- }
-}
-
/// The materialize direction: a FieldValues or FieldHistories row without an `Id` is a row whose
/// derived id the wire omits; derive it from the row's own natural key so every device materializes
/// the same SQLite row.
@@ -132,9 +109,8 @@ fn multi_value_definition_ids(tables: &HashMap<String, Vec<CodecRecord>>) -> Has
enum FieldShape {
MultiValue,
SingleValue,
- /*
- * A system field this build's registry does not list, so it is unknown to this build.
- */
+ /// Nothing says: a system field this build's registry does not list, or a custom field whose definition is gone.
+ /// Its rows keep the ids they carry, so none of their values is lost.
Unknown,
}
@@ -144,8 +120,11 @@ fn field_shape(row: &CodecRecord, manifest: &str, multi_value_defs: &HashSet<(St
Some(key) if MULTI_VALUE_FIELD_KEYS.contains(&key.as_str()) => FieldShape::MultiValue,
Some(key) if SINGLE_VALUE_FIELD_KEYS.contains(&key.as_str()) => FieldShape::SingleValue,
Some(_) => FieldShape::Unknown,
- None if lower_str(row, FIELD_DEFINITION_ID_COL).is_some_and(|def| multi_value_defs.contains(&(manifest.to_string(), def))) => FieldShape::MultiValue,
- None => FieldShape::SingleValue,
+ None => match lower_str(row, FIELD_DEFINITION_ID_COL) {
+ Some(def) if multi_value_defs.contains(&(manifest.to_string(), def.clone())) => FieldShape::MultiValue,
+ Some(_) => FieldShape::SingleValue,
+ None => FieldShape::Unknown,
+ },
}
}
@@ -206,12 +185,8 @@ fn normalize_field_values(rows: &mut Vec<CodecRecord>, multi_value_defs: &HashSe
}
if !removed.is_empty() {
- let mut position = 0;
- rows.retain(|_| {
- let keep = !removed.contains(&position);
- position += 1;
- keep
- });
+ let keep: HashSet<usize> = (0..rows.len()).filter(|position| !removed.contains(position)).collect();
+ retain_positions(rows, &keep);
}
}
@@ -323,7 +298,8 @@ mod tests {
#[test]
fn derivation_is_stable_and_well_formed() {
let id = field_value_id_for("m-1", "item-1", "login.username", "", 0);
- assert_eq!(id, field_value_id_for("m-1", "item-1", "login.username", "", 0));
+ // Known-answer vector: every platform must derive the same row id, so this value must never change.
+ assert_eq!(id, "f36327d9-f225-86ab-a061-1ba3a668d89a");
assert_eq!(id.len(), 36);
assert_eq!(id.as_bytes()[14], b'8', "UUIDv8 version nibble");
}
@@ -378,7 +354,7 @@ mod tests {
field_value(Some("b"), "login.future_multi", "two", "2024-01-02T00:00:00Z"),
]);
assert_eq!(rows.len(), 2);
- assert!(rows.iter().all(|row| has_id(row)), "and the rows keep the ids they own");
+ assert!(rows.iter().all(has_id), "and the rows keep the ids they own");
}
#[test]
@@ -405,29 +381,11 @@ mod tests {
assert!(!has_id(&rows[0]), "a single-value row travels without its derived id");
}
- #[test]
- fn a_tombstoned_item_keeps_no_rows() {
- let item = |id: &str, deleted: i64| -> CodecRecord {
- [(MANIFEST_ID_COL.to_string(), json!("m-1")), (ID_COL.to_string(), json!(id)), ("IsDeleted".to_string(), json!(deleted))].into_iter().collect()
- };
- let child = |item_id: &str| -> CodecRecord {
- [(MANIFEST_ID_COL.to_string(), json!("m-1")), (ID_COL.to_string(), json!(format!("totp-{}", item_id))), (ITEM_ID_COL.to_string(), json!(item_id))].into_iter().collect()
- };
- let mut tables: HashMap<String, Vec<CodecRecord>> = [
- (ITEMS_TABLE.to_string(), vec![item("gone", 1), item("kept", 0)]),
- ("TotpCodes".to_string(), vec![child("gone"), child("kept")]),
- ]
- .into_iter()
- .collect();
- normalize_row_shapes(&mut tables);
- let left: Vec<&str> = tables["TotpCodes"].iter().map(|row| row[ITEM_ID_COL].as_str().unwrap()).collect();
- assert_eq!(left, vec!["kept"], "a pruned or permanently deleted item's secrets do not travel on");
- }
-
#[test]
fn history_derivation_is_stable_and_disjoint_from_field_values() {
let id = field_history_id_for("m-1", "item-1", "login.password", "", "2026-01-01 10:00:00.000");
- assert_eq!(id, field_history_id_for("m-1", "item-1", "login.password", "", "2026-01-01 10:00:00.000"));
+ // Known-answer vector: every platform must derive the same row id, so this value must never change.
+ assert_eq!(id, "7431819b-eefd-834b-b4de-9e1f16df1a22");
assert_eq!(id.len(), 36);
// A different millisecond is a different row; the two namespaces never collide.
assert_ne!(id, field_history_id_for("m-1", "item-1", "login.password", "", "2026-01-01 10:00:00.001"));
diff --git a/core/rust/src/vault_codec/scoped_assets.rs b/core/rust/src/vault_codec/scoped_assets.rs
deleted file mode 100644
index 5d6209f3e..000000000
--- a/core/rust/src/vault_codec/scoped_assets.rs
+++ /dev/null
@@ -1,267 +0,0 @@
-//! Logo identity: one `Logos` row per `(ManifestId, Kind, Source)` per *manifest*.
-//!
-//! One row shape covers every logo an item can have, and `Items.LogoId` is the single pointer to it:
-//! - `Kind = "favicon"`, `Source` = the domain the logo was fetched from (`github.com`);
-//! - `Kind = "builtin"`, `Source` = a key into the shared built-in catalog (`shopping`), no image bytes:
-//! every platform draws that logo itself;
-//! - `Kind = "custom"`, `Source` = the sha256 of the image the user uploaded.
-//!
-//! A logo is scoped to the manifest that owns it:
-//! - `ManifestId` is that manifest's id, personal or shared alike (no NULL convention);
-//! - `Id` is derived from `(manifest id, Kind, Source)` (see [`logo_id_for`]), so every writer derives
-//! the same id for the same logo in the same manifest and the uniqueness invariant is
-//! self-enforcing rather than repaired after the fact.
-//!
-//! Deriving a custom logo's id from its content hash is also what makes an uploaded image reusable:
-//! picking it again resolves to the row that already holds those bytes instead of storing a second copy.
-
-use std::collections::{HashMap, HashSet};
-
-use serde_json::{json, Value};
-
-use super::manifest::CodecRecord;
-use super::row::{has_bytes, is_deleted, logo_kind, normalize_logo_kind, str_col};
-use crate::vault_model::names::{FILE_DATA_COL, ID_COL, ITEMS_TABLE, KIND_COL, LOGOS_TABLE, LOGO_ID_COL, LOGO_KIND_CUSTOM, LOGO_KIND_FAVICON, SOURCE_COL, UPDATED_AT_COL};
-use crate::vault_model::{id_key, ids_equal, MANIFEST_ID_COL};
-
-/// Domain-separation prefix for favicon ids. It predates the `Kind` column and is kept as-is so
-/// every favicon row that already exists keeps its id: changing it would re-derive the logo id of
-/// every item in every vault on the next push.
-const FAVICON_ID_NAMESPACE: &str = "aliasvault:logo:v1";
-
-/// The `Id` of the logo `(manifest id, kind, source)`: a UUIDv8 (RFC 9562 custom-format) whose bytes
-/// come from `sha256(namespace | manifest id | source)`, with a namespace per kind so the three key
-/// spaces (a domain, a catalog key, a content hash) can never collide.
-///
-/// Deriving the id from the row's natural key is what removes cross-writer identity conflicts: two
-/// devices (or two members of one shared manifest) that fetch the same domain, or upload the very same
-/// image, produce the same row, which then merges by ordinary LWW.
-///
-/// `manifest_id` is the owning manifest's id, the personal manifest's own id for personal logos (every
-/// vault derives distinct ids for the same domain, so two vaults materialized side by side can never
-/// collide). `source` is matched case-insensitively (callers already normalize to a lowercase
-/// hostname or lowercase hex digest; this makes it robust anyway).
-pub fn logo_id_for(manifest_id: &str, kind: &str, source: &str) -> String {
- super::hash::derived_uuid(&format!("{}\n{}\n{}", namespace_for_kind(kind), id_key(manifest_id), source.to_lowercase()))
-}
-
-/// The derivation namespace for a kind. Unknown kinds get one derived from their own name, so a newer
-/// client can add a kind without this one having to know about it: ids stay stable and distinct either
-/// way, and the rows simply round-trip until this client learns to render them.
-fn namespace_for_kind(kind: &str) -> String {
- match normalize_logo_kind(kind) {
- k if k == LOGO_KIND_FAVICON => FAVICON_ID_NAMESPACE.to_string(),
- k => format!("aliasvault:logo:{}:v1", k),
- }
-}
-
-/// True when a row holds an image the user supplied themselves ([`LOGO_KIND_CUSTOM`]) rather than one the
-/// client can produce again on its own: a favicon it can refetch from the domain, a built-in logo it
-/// draws from the catalog. Only that first group is worth keeping around once nothing references it.
-pub(super) fn is_custom_logo(row: &CodecRecord) -> bool {
- logo_kind(row) == LOGO_KIND_CUSTOM
-}
-
-/// The `(kind, source)` natural key of a row, or `None` when it carries no `Source` to key on.
-fn natural_key(row: &CodecRecord) -> Option<(String, String)> {
- let source = str_col(row, SOURCE_COL)?.to_lowercase();
- Some((logo_kind(row), source))
-}
-
-/// Normalize one table set's logo rows to `scope` (the owning manifest's id): stamp `ManifestId`,
-/// re-derive `Id` from `(scope, Kind, Source)`, collapse rows that now share a natural key (keeping
-/// the better row, see [`is_better_logo`]), and repoint every `Items.LogoId` at the surviving row.
-///
-/// It runs once per manifest, each with its own id, so a personal logo is never merged with a shared
-/// one. It also migrates legacy rows (random GUIDs, folder-id or NULL scopes) and heals a writer that
-/// stamped the wrong scope.
-pub(super) fn normalize_logo_scope(tables: &mut HashMap<String, Vec<CodecRecord>>, scope: &str) {
- let remap = rewrite_logo_rows(tables, scope);
- repoint_items(tables, &remap);
-}
-
-/// Rewrite `Logos` rows to `scope` and return the `old Id -> new Id` map. Rows without a `Source`
-/// cannot be addressed by the natural key, so they keep their identity and are left alone.
-fn rewrite_logo_rows(tables: &mut HashMap<String, Vec<CodecRecord>>, scope: &str) -> HashMap<String, String> {
- let mut remap: HashMap<String, String> = HashMap::new();
- let logos = match tables.get_mut(LOGOS_TABLE) {
- Some(rows) if !rows.is_empty() => rows,
- _ => return remap,
- };
-
- // (kind, source) -> index of the row that survives (deterministic, see `is_better_logo`).
- let mut survivor_idx: HashMap<(String, String), usize> = HashMap::new();
- for (idx, row) in logos.iter().enumerate() {
- let Some(key) = natural_key(row) else { continue };
- match survivor_idx.get(&key) {
- Some(&cur) if !is_better_logo(row, &logos[cur]) => {}
- _ => {
- survivor_idx.insert(key, idx);
- }
- }
- }
-
- // Every row with a natural key maps onto that key's survivor id, and the survivor itself is
- // rewritten in place. A row whose id already equals the derived one maps to itself.
- let survivors: HashSet<usize> = survivor_idx.values().copied().collect();
- for (idx, row) in logos.iter().enumerate() {
- let (Some((kind, source)), Some(old_id)) = (natural_key(row), str_col(row, ID_COL)) else { continue };
- let new_id = logo_id_for(scope, &kind, &source);
- if old_id != new_id || !survivors.contains(&idx) {
- remap.insert(id_key(old_id), new_id);
- }
- }
-
- let scope_value = json!(scope);
- let mut kept: Vec<CodecRecord> = Vec::with_capacity(survivors.len());
- for (idx, mut row) in std::mem::take(logos).into_iter().enumerate() {
- // A row with no Source keeps its identity: it has no natural key to derive one from.
- let Some((kind, source)) = natural_key(&row) else {
- kept.push(row);
- continue;
- };
- if !survivors.contains(&idx) {
- continue;
- }
- row.insert(ID_COL.to_string(), json!(logo_id_for(scope, &kind, &source)));
- row.insert(KIND_COL.to_string(), json!(kind));
- row.insert(MANIFEST_ID_COL.to_string(), scope_value.clone());
- kept.push(row);
- }
- *logos = kept;
-
- remap
-}
-
-/// Repair every `Items.LogoId` in this table set: follow `remap`, then null a reference that resolves
-/// to no logo present here (dangling -> the FK's `ON DELETE SET NULL`). Callers run
-/// [`reconcile_logo_references`] first, so by this point a reference that crossed a scope boundary has
-/// already been pulled into this scope and only dead references are left to `NULL`.
-fn repoint_items(tables: &mut HashMap<String, Vec<CodecRecord>>, remap: &HashMap<String, String>) {
- let valid_ids = logo_ids(tables);
- let Some(items) = tables.get_mut(ITEMS_TABLE) else { return };
- for item in items.iter_mut() {
- let Some(current) = str_col(item, LOGO_ID_COL).map(str::to_string) else { continue };
- let resolved = remap.get(&id_key(&current)).cloned().unwrap_or(current);
- let repaired = if valid_ids.contains(&id_key(&resolved)) { json!(resolved) } else { Value::Null };
- item.insert(LOGO_ID_COL.to_string(), repaired);
- }
-}
-
-/// Repair `Items.LogoId` references that point outside this table set's scope, by cloning the
-/// referenced logo into this scope under its scope-local id.
-pub(super) fn reconcile_logo_references(tables: &mut HashMap<String, Vec<CodecRecord>>, scope: &str, all_logos: &[CodecRecord]) {
- if !tables.contains_key(ITEMS_TABLE) {
- return;
- }
- let present = logo_ids(tables);
-
- // Logo ids referenced from this scope's items that aren't satisfied by a row in this scope.
- let missing: Vec<String> = {
- let mut ids: Vec<String> = tables[ITEMS_TABLE]
- .iter()
- .filter_map(|item| str_col(item, LOGO_ID_COL))
- .filter(|id| !present.contains(&id_key(id)))
- .map(str::to_string)
- .collect();
- ids.sort();
- ids.dedup();
- ids
- };
- if missing.is_empty() {
- return;
- }
-
- // (kind, source) -> id of the row already in this scope, so an incoming item reuses it rather than
- // cloning a second row onto the same natural key.
- let mut id_by_key: HashMap<(String, String), String> = tables
- .get(LOGOS_TABLE)
- .map(|rows| rows.iter().filter_map(|r| Some((natural_key(r)?, str_col(r, ID_COL)?.to_string()))).collect())
- .unwrap_or_default();
-
- let mut remap: HashMap<String, String> = HashMap::new();
- let mut clones: Vec<CodecRecord> = Vec::new();
- let scope_value = json!(scope);
- for missing_id in missing {
- // The referenced row as it exists in its original scope, if it exists at all.
- let Some(origin) = all_logos.iter().find(|row| str_col(row, ID_COL).is_some_and(|id| ids_equal(id, &missing_id))) else { continue };
- let Some((kind, source)) = natural_key(origin) else { continue };
-
- if let Some(existing_id) = id_by_key.get(&(kind.clone(), source.clone())).cloned() {
- refill_empty_scope_row(tables, &existing_id, origin);
- remap.insert(id_key(&missing_id), existing_id);
- continue;
- }
-
- let scoped_id = logo_id_for(scope, &kind, &source);
- let mut clone = origin.clone();
- clone.insert(ID_COL.to_string(), json!(scoped_id));
- clone.insert(KIND_COL.to_string(), json!(kind.clone()));
- clone.insert(MANIFEST_ID_COL.to_string(), scope_value.clone());
- clones.push(clone);
- id_by_key.insert((kind, source), scoped_id.clone());
- remap.insert(id_key(&missing_id), scoped_id);
- }
-
- if !clones.is_empty() {
- tables.entry(LOGOS_TABLE.to_string()).or_default().extend(clones);
- }
- repoint_items(tables, &remap);
-}
-
-/// Refill this scope's row for a natural key from the row an incoming item pointed at, when the
-/// scope's row carries no image (or is tombstoned) and the incoming one does.
-fn refill_empty_scope_row(tables: &mut HashMap<String, Vec<CodecRecord>>, existing_id: &str, origin: &CodecRecord) {
- let Some(logos) = tables.get_mut(LOGOS_TABLE) else { return };
- let Some(existing) = logos.iter_mut().find(|r| str_col(r, ID_COL) == Some(existing_id)) else { return };
- let upgrades = (!has_file_data(existing) && has_file_data(origin)) || (is_deleted(existing) && !is_deleted(origin));
- if !upgrades {
- return;
- }
-
- // Take the origin's content, keep this scope's identity, and never move the row's clock backwards:
- // the healed row has to win last-writer-wins against the empty one still sitting on other devices.
- let updated_at = existing.get(UPDATED_AT_COL).cloned();
- for (column, value) in origin.iter() {
- if column == ID_COL || column == MANIFEST_ID_COL {
- continue;
- }
- existing.insert(column.clone(), value.clone());
- }
- if let (Some(previous), Some(incoming)) = (updated_at, existing.get(UPDATED_AT_COL)) {
- if previous.as_str() > incoming.as_str() {
- existing.insert(UPDATED_AT_COL.to_string(), previous);
- }
- }
-}
-
-/// Every `Logos.Id` present in this table set, lowercased: GUIDs are case-insensitive and should be stored/compared as such.
-fn logo_ids(tables: &HashMap<String, Vec<CodecRecord>>) -> HashSet<String> {
- tables
- .get(LOGOS_TABLE)
- .map(|rows| rows.iter().filter_map(|r| str_col(r, ID_COL).map(id_key)).collect())
- .unwrap_or_default()
-}
-
-/// Total order used to choose which row's *content* survives a natural-key collision within one scope
-/// (the id is derived either way): a live row beats a tombstoned one, a row with image bytes beats an
-/// empty one, and the lexicographically-highest (newest) `Id` breaks the remaining ties.
-fn is_better_logo(candidate: &CodecRecord, incumbent: &CodecRecord) -> bool {
- let cand_live = !is_deleted(candidate);
- let inc_live = !is_deleted(incumbent);
- if cand_live != inc_live {
- return cand_live;
- }
- let cand_has_data = has_file_data(candidate);
- let inc_has_data = has_file_data(incumbent);
- if cand_has_data != inc_has_data {
- return cand_has_data;
- }
- str_col(candidate, ID_COL).unwrap_or("") > str_col(incumbent, ID_COL).unwrap_or("")
-}
-
-/// True when `FileData` holds actual bytes; a tombstoned row's NULL must not beat a row with an image.
-fn has_file_data(row: &CodecRecord) -> bool {
- has_bytes(row.get(FILE_DATA_COL))
-}
-
diff --git a/core/rust/src/vault_codec/sharing.rs b/core/rust/src/vault_codec/sharing.rs
index a50d22a89..b14c9cedd 100644
--- a/core/rust/src/vault_codec/sharing.rs
+++ b/core/rust/src/vault_codec/sharing.rs
@@ -1,91 +1,67 @@
//! Split/combine logic for multi-manifest vaults.
//!
-//! A manifest is a namespace: an id, a display name, its own VEK and blob salt, and the rows carrying
-//! its id in their `ManifestId` column. No manifest is privileged by itself, and there is no fallback scope: every
-//! row routes by its own stamp, canonicalize rejects an unstamped one, and a row stamped for a manifest this
-//! vault no longer carries is dropped, since there is no namespace left to write it to. How a manifest
-//! is presented (a folder in the tree or as a separate vault) is each client's runtime choice.
+//! A manifest is a namespace: an id, a display name, its own VEK and blob salt, and the rows carrying its id in
+//! their `ManifestId` column. No manifest is privileged and there is no fallback manifest: every row routes by its own
+//! stamp, canonicalize rejects an unstamped one, and a row stamped for a manifest this vault no longer carries is
+//! dropped, since there is no namespace left to write it to.
//!
-//! **partition** (canonicalize side) routes every row of the unified table set to its owning manifest:
-//! - `Folders`, `Items`, `Tags`, `FieldDefinitions`, `Logos` and `EncryptionKeys` route by stamp and
-//! are restamped with the owning manifest's id. A folder whose parent lives in another namespace
-//! has its `ParentFolderId` nulled.
-//! - Any other registered table row that has a parent `ItemId` follows it, so the parent manifest location is leading.
-//! - Tables this build's registry does not know (a newer writer's, carried through the codec
-//! overflow) route by stamp too.
-//! - Bucketed tables (Settings) leave the routing set beforehand: one bucket type per manifest.
+//! **partition** (canonicalize side) routes every row of the unified table set to the manifest its stamp names,
+//! with two refinements: a row hanging off an item follows its item (the parent's location is leading), and a
+//! folder whose parent lives in another manifest has its `ParentFolderId` nulled. Each manifest is then made
+//! self-contained: it gets its own copy of the content its rows reference (see [`clone_referenced_rows`]) and its
+//! own logos (see [`logos`](super::logos)).
//!
-//! Each manifest is then made self-contained: it copies in the content its rows reference (see
-//! below) and drops what nothing references any more (see [`prune_unreferenced_logos`]).
+//! **combine** (materialize side) stamps every manifest's rows with the manifest they arrived in, drops the
+//! `EncryptionKeys` rows it may not publish plus the bucketed, personal and bookkeeping tables no manifest may
+//! carry, and repairs the references a namespace boundary leaves dangling (see [`integrity`](super::integrity)).
//!
-//! **combine** (materialize side) stamps every manifest's rows with the manifest they arrived in,
-//! recomputing the ids that derive from the scope, drops the `EncryptionKeys` rows it may not publish
-//! plus the bucketed, personal and bookkeeping tables no manifest may carry, folds in its data buckets,
-//! then repairs the references a namespace boundary leaves dangling.
-//!
-//! **Referenced content** is copied, never moved. A foreign key is composite, so a row in another
-//! namespace is as absent as a deleted one, and the manifest referencing it gets its own stamped copy:
-//! two members editing "the same" tag or logo then never overwrite each other. A row with an opaque id
-//! keeps it ([`clone_referenced_rows`]); a `Logos` id derives from the scope, so the copy recomputes it
-//! (see [`scoped_assets`](super::scoped_assets)).
+//! **Referenced content** is copied, never moved. A foreign key is composite, so a row in another namespace is as
+//! absent as a deleted one, and the manifest referencing it gets its own stamped copy: two members editing "the
+//! same" tag or logo then never overwrite each other.
use std::collections::{HashMap, HashSet};
use serde_json::{json, Value};
+use super::integrity;
+use super::logos::{localize_logos, restamp_logos};
use super::manifest::{CodecRecord, Manifest, ManifestSpec};
use super::row::{is_deleted, rows_of, str_col};
-use super::scoped_assets::{is_custom_logo, normalize_logo_scope, reconcile_logo_references};
-use super::types::{is_bucketed_table, is_local_only_table, is_manifest_scoped, is_personal_table, manifest_scoped_tables, row_identity};
+use super::types::{is_bucketed_table, is_local_only_table, is_manifest_scoped, is_personal_table, row_identity};
use crate::common::error::{VaultError, VaultResult};
-use crate::vault_model::names::{
- ENCRYPTION_KEYS_TABLE, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_VALUES_TABLE, FOLDERS_TABLE,
- FOLDER_ID_COL, ID_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, LOGOS_TABLE, LOGO_ID_COL, PARENT_FOLDER_ID_COL, PUBLIC_KEY_COL,
- TAGS_TABLE, TAG_ID_COL,
-};
+use crate::vault_model::names::{ENCRYPTION_KEYS_TABLE, FOLDERS_TABLE, ID_COL, ITEMS_TABLE, ITEM_ID_COL, LOGOS_TABLE, PARENT_FOLDER_ID_COL, PUBLIC_KEY_COL};
use crate::vault_model::{ids_equal, MANIFEST_ID_COL, REFERENCED_TABLES};
-/// The tables a caller must snapshot before routing, so the referenced content can be copied out of
-/// them afterwards: every [`REFERENCED_TABLES`] target plus `Logos`.
+type Tables = HashMap<String, Vec<CodecRecord>>;
+
+/// The tables a caller must snapshot before routing, so the referenced content can be copied out of them
+/// afterwards: every [`REFERENCED_TABLES`] target plus `Logos`.
pub(super) fn referenced_tables() -> Vec<&'static str> {
let mut out: Vec<&'static str> = vec![LOGOS_TABLE];
out.extend(REFERENCED_TABLES.iter().map(|(target, _)| *target));
out
}
-/// Where one row lands when [`partition_by_manifest`] routes it.
-#[derive(Clone, Copy, PartialEq, Eq)]
-enum Route {
- /// The manifest being written from: the rows stamped for it.
- Base,
- /// One of the other manifests in this push, by spec index.
- Partition(usize),
- /// A manifest this vault no longer carries: no namespace is left for the row, so it is dropped.
- Gone,
+/// One manifest's rows after partitioning.
+pub(super) struct ManifestPartition {
+ pub spec: ManifestSpec,
+ pub tables: Tables,
}
-/// Where each item was routed, so the rows hanging off it can be sent the same way.
-///
-/// A child row names its item through the composite foreign key `(ManifestId, ItemId)` and is resolved
-/// by it. Keying on the bare `ItemId` would be a leak: two manifests may hold an item with the same id,
-/// and the first one found would then pull in every manifest's rows for that id, sending a personal
-/// password, TOTP secret or attachment into a manifest shared with other people.
-///
-/// `by_id` is the fallback for the one case the composite key cannot answer: a client that moved an
-/// item between manifests without re-stamping its children. It applies only while exactly one item
-/// carries that id.
+/// Where each item was routed, so the rows hanging off it follow it. Children resolve by `(ManifestId, ItemId)`, never by
+/// bare `ItemId`: two manifests may hold the same item id, and that would leak personal rows into a shared manifest.
+/// `by_id` covers an item moved without re-stamping its children, and only while exactly one item carries that id.
#[derive(Default)]
struct ItemRoutes {
- by_identity: HashMap<(String, String), Route>,
- by_id: HashMap<String, Option<Route>>,
+ by_identity: HashMap<(String, String), Option<usize>>,
+ by_id: HashMap<String, Option<Option<usize>>>,
}
impl ItemRoutes {
- /// Record where one item row was routed, under both its full identity and its bare id.
- fn record(&mut self, row: &CodecRecord, destination: Route) {
+ fn record(&mut self, row: &CodecRecord, destination: Option<usize>) {
let Some(id) = str_col(row, ID_COL) else { return };
- let scope = str_col(row, MANIFEST_ID_COL).unwrap_or_default().to_string();
- self.by_identity.insert((scope, id.to_string()), destination);
+ let manifest_id = str_col(row, MANIFEST_ID_COL).unwrap_or_default().to_string();
+ self.by_identity.insert((manifest_id, id.to_string()), destination);
// Two items with the same id agree only while they route to the same place.
self.by_id
.entry(id.to_string())
@@ -98,275 +74,99 @@ impl ItemRoutes {
}
/// Where `row`'s item went, or `None` when it names no item this vault holds.
- fn route_for(&self, row: &CodecRecord) -> Option<Route> {
+ fn route_for(&self, row: &CodecRecord) -> Option<Option<usize>> {
let item_id = str_col(row, ITEM_ID_COL)?;
- let scope = str_col(row, MANIFEST_ID_COL).unwrap_or_default();
- if let Some(destination) = self.by_identity.get(&(scope.to_string(), item_id.to_string())) {
+ let manifest_id = str_col(row, MANIFEST_ID_COL).unwrap_or_default();
+ if let Some(destination) = self.by_identity.get(&(manifest_id.to_string(), item_id.to_string())) {
return Some(*destination);
}
self.by_id.get(item_id).copied().flatten()
}
}
-/// One manifest's partition produced by [`partition_by_manifest`].
-pub(super) struct ManifestPartition {
- pub manifest_id: String,
- pub name: Option<String>,
- pub manifest_salt: String,
- pub tables: HashMap<String, Vec<CodecRecord>>,
-}
-
-/// Route every row of `tables` to the manifest that owns it, moving the rows of `specs` out into one
-/// partition each (in spec order) and leaving the writing manifest's rows behind in `tables`.
-pub(super) fn partition_by_manifest(
- tables: &mut HashMap<String, Vec<CodecRecord>>,
- specs: &[ManifestSpec],
- snapshots: &HashMap<String, Vec<CodecRecord>>,
- writing_manifest_id: &str,
-) -> VaultResult<Vec<ManifestPartition>> {
- let mut seen_spec_manifests: HashSet<&str> = HashSet::new();
+/// Route every row of `tables` to the manifest that owns it, one partition per spec in spec order. A row stamped
+/// for a manifest not in `specs` is dropped. `snapshots` holds the referenced tables as they were before routing.
+pub(super) fn partition_by_manifest(mut tables: Tables, specs: &[ManifestSpec], snapshots: &Tables) -> VaultResult<Vec<ManifestPartition>> {
+ let mut seen: HashSet<&str> = HashSet::new();
for spec in specs {
- if !seen_spec_manifests.insert(spec.manifest_id.as_str()) || spec.manifest_id == writing_manifest_id {
+ if !seen.insert(spec.manifest_id.as_str()) {
return Err(VaultError::General(format!("duplicate manifest id in manifest spec {}", spec.manifest_id)));
}
}
- // Manifest id -> spec index: the routing key every stamped table uses.
- let manifest_to_spec: HashMap<&str, usize> = specs.iter().enumerate().map(|(idx, spec)| (spec.manifest_id.as_str(), idx)).collect();
-
// Where a row goes, read straight off its `ManifestId` stamp.
- let route = |row: &CodecRecord| -> Route {
- let Some(scope) = str_col(row, MANIFEST_ID_COL) else { return Route::Gone };
- if ids_equal(scope, writing_manifest_id) {
- return Route::Base;
- }
- manifest_to_spec
- .iter()
- .find(|(id, _)| ids_equal(id, scope))
- .map(|(_, index)| Route::Partition(*index))
- .unwrap_or(Route::Gone)
+ let route = |row: &CodecRecord| -> Option<usize> {
+ let manifest_id = str_col(row, MANIFEST_ID_COL)?;
+ specs.iter().position(|spec| ids_equal(&spec.manifest_id, manifest_id))
};
- /*
- * The `(ManifestId, Id)` of every folder, so rule 1 can tell whether a folder's parent lives in the
- * same namespace. Both halves, because `Folders.ParentFolderId` resolves through the composite key:
- * the same id in another manifest is another folder, not this one's parent.
- */
- let folder_identities: HashSet<(String, String)> = rows_of(tables, FOLDERS_TABLE)
- .iter()
- .filter_map(|row| Some((str_col(row, MANIFEST_ID_COL)?.to_string(), str_col(row, ID_COL)?.to_string())))
- .collect();
+ // The `(ManifestId, Id)` of every folder, so a folder's parent link can be checked within its own namespace.
+ let folder_identities: HashSet<(String, String)> = rows_of(&tables, FOLDERS_TABLE).iter().filter_map(|row| Some((str_col(row, MANIFEST_ID_COL)?.to_string(), str_col(row, ID_COL)?.to_string()))).collect();
- let mut partitions: Vec<ManifestPartition> = specs
- .iter()
- .map(|spec| ManifestPartition {
- manifest_id: spec.manifest_id.clone(),
- name: spec.name.clone(),
- manifest_salt: spec.manifest_salt.clone(),
- tables: HashMap::new(),
- })
- .collect();
+ let mut partitions: Vec<ManifestPartition> = specs.iter().map(|spec| ManifestPartition { spec: spec.clone(), tables: Tables::new() }).collect();
+ let mut item_routes = ItemRoutes::default();
- // 1. Folders, routed by stamp and restamped with their owning manifest's id.
- let mut item_routes: ItemRoutes = ItemRoutes::default();
- if let Some(folder_rows) = tables.remove(FOLDERS_TABLE) {
- let mut base_rows: Vec<CodecRecord> = Vec::with_capacity(folder_rows.len());
- for mut row in folder_rows {
- match route(&row) {
- Route::Partition(spec_idx) => {
- /*
- * A folder whose parent belongs to another manifest heads this one, so the link is
- * cut. The parent is looked up under this folder's own scope, the namespace its
- * foreign key resolves in.
- */
- let scope = str_col(&row, MANIFEST_ID_COL).unwrap_or_default().to_string();
- let parent_in_other_manifest = str_col(&row, PARENT_FOLDER_ID_COL).map(|parent| !folder_identities.contains(&(scope.clone(), parent.to_string()))).unwrap_or(false);
- if parent_in_other_manifest {
- row.insert(PARENT_FOLDER_ID_COL.to_string(), Value::Null);
- }
- row.insert(MANIFEST_ID_COL.to_string(), json!(specs[spec_idx].manifest_id));
- partitions[spec_idx].tables.entry(FOLDERS_TABLE.to_string()).or_default().push(row);
- }
- Route::Base => {
- row.insert(MANIFEST_ID_COL.to_string(), json!(writing_manifest_id));
- base_rows.push(row);
- }
- Route::Gone => {}
- }
- }
- tables.insert(FOLDERS_TABLE.to_string(), base_rows);
- }
-
- // 2. Items, routed like folders; remember where each one went for rule 4.
- if let Some(item_rows) = tables.remove(ITEMS_TABLE) {
- let mut base_rows: Vec<CodecRecord> = Vec::with_capacity(item_rows.len());
- for mut row in item_rows {
- let destination = route(&row);
- item_routes.record(&row, destination);
- match destination {
- Route::Partition(spec_idx) => {
- row.insert(MANIFEST_ID_COL.to_string(), json!(specs[spec_idx].manifest_id));
- partitions[spec_idx].tables.entry(ITEMS_TABLE.to_string()).or_default().push(row);
- }
- Route::Base => {
- row.insert(MANIFEST_ID_COL.to_string(), json!(writing_manifest_id));
- base_rows.push(row);
- }
- // The item goes with its manifest, and rule 4 sends the rows hanging off it the same way.
- Route::Gone => {}
- }
- }
- tables.insert(ITEMS_TABLE.to_string(), base_rows);
- }
-
- /*
- * 3. Logos are scoped, not copied: a row moves to the partition its ManifestId names. The stamp
- * value itself is rewritten to the owning manifest below.
- */
- if let Some(logo_rows) = tables.remove(LOGOS_TABLE) {
- let mut base_rows: Vec<CodecRecord> = Vec::with_capacity(logo_rows.len());
- for row in logo_rows {
- match route(&row) {
- Route::Partition(spec_idx) => partitions[spec_idx].tables.entry(LOGOS_TABLE.to_string()).or_default().push(row),
- Route::Base => base_rows.push(row),
- Route::Gone => {}
- }
- }
- tables.insert(LOGOS_TABLE.to_string(), base_rows);
- }
-
- /*
- * 3b. EncryptionKeys routes by stamp like every other table; it is called out only because a
- * `Route::Gone` row here must be dropped. Re-homing a dead manifest's keypair would resurrect it
- * on a future re-share (handing pre-share mail to the new members) and would let a tampered local
- * DB pass a fabricated key row off as the writing manifest's own. Mail encrypted to a dropped key
- * becomes unreadable, which is the intended outcome: the key is readable only by whoever currently
- * holds the manifest.
- */
- if let Some(rows) = tables.remove(ENCRYPTION_KEYS_TABLE) {
- let mut base_rows: Vec<CodecRecord> = Vec::with_capacity(rows.len());
- for mut row in rows {
- match route(&row) {
- Route::Base => {
- row.insert(MANIFEST_ID_COL.to_string(), json!(writing_manifest_id));
- base_rows.push(row);
- }
- Route::Partition(spec_idx) => partitions[spec_idx].tables.entry(ENCRYPTION_KEYS_TABLE.to_string()).or_default().push(row),
- Route::Gone => {}
- }
- }
- tables.insert(ENCRYPTION_KEYS_TABLE.to_string(), base_rows);
- }
-
- /*
- * 4. Any remaining *registered* table row carrying an ItemId follows its item, restamped to agree
- * with it: its foreign key is `(ManifestId, ItemId)`, both halves, so following the same id in
- * another manifest would carry a password, TOTP secret or attachment out of the namespace it
- * belongs to (see [`ItemRoutes`]). A table the registry does not know is not assumed to follow
- * items and routes by its rows' own stamps in rule 5 instead.
- */
- let item_scoped_names: Vec<String> = tables
- .iter()
- .filter(|(name, rows)| is_manifest_scoped(name) && !is_bucketed_table(name) && !is_personal_table(name) && rows.iter().any(|r| r.contains_key(ITEM_ID_COL)))
- .map(|(name, _)| name.clone())
- .collect();
- for name in item_scoped_names.clone() {
- let rows = tables.remove(&name).unwrap_or_default();
- let mut base_rows: Vec<CodecRecord> = Vec::with_capacity(rows.len());
+ // Items first, so the rows hanging off them can follow; every other table routes by its rows' own stamps.
+ let item_child_tables: Vec<String> = tables.iter().filter(|(name, rows)| is_manifest_scoped(name) && !is_bucketed_table(name) && !is_personal_table(name) && *name != ITEMS_TABLE && rows.iter().any(|r| r.contains_key(ITEM_ID_COL))).map(|(name, _)| name.clone()).collect();
+ let mut order: Vec<String> = vec![ITEMS_TABLE.to_string()];
+ order.extend(tables.keys().filter(|name| *name != ITEMS_TABLE).cloned());
+ for name in order {
+ let Some(rows) = tables.remove(&name) else { continue };
for mut row in rows {
- // Where this row's item went, or, when it names no item at all, its own stamp.
- match item_routes.route_for(&row).unwrap_or_else(|| route(&row)) {
- Route::Partition(spec_idx) => {
- row.insert(MANIFEST_ID_COL.to_string(), json!(specs[spec_idx].manifest_id));
- partitions[spec_idx].tables.entry(name.clone()).or_default().push(row);
+ let destination = match name.as_str() {
+ ITEMS_TABLE => {
+ let destination = route(&row);
+ item_routes.record(&row, destination);
+ destination
}
- Route::Base => {
- row.insert(MANIFEST_ID_COL.to_string(), json!(writing_manifest_id));
- base_rows.push(row);
+ // A folder whose parent belongs to another manifest heads this one, so the link is cut.
+ FOLDERS_TABLE => {
+ let manifest_id = str_col(&row, MANIFEST_ID_COL).unwrap_or_default().to_string();
+ if str_col(&row, PARENT_FOLDER_ID_COL).is_some_and(|parent| !folder_identities.contains(&(manifest_id, parent.to_string()))) {
+ row.insert(PARENT_FOLDER_ID_COL.to_string(), Value::Null);
+ }
+ route(&row)
}
- // The item left with a manifest this vault no longer carries; its rows go with it.
- Route::Gone => {}
- }
+ _ if item_child_tables.contains(&name) => item_routes.route_for(&row).unwrap_or_else(|| route(&row)),
+ _ => route(&row),
+ };
+ let Some(index) = destination else { continue };
+ row.insert(MANIFEST_ID_COL.to_string(), json!(specs[index].manifest_id));
+ partitions[index].tables.entry(name.clone()).or_default().push(row);
}
- tables.insert(name, base_rows);
- }
-
- /*
- * 5. Every table left routes by its rows' own stamps: the registered independent-row tables (`Tags`,
- * `FieldDefinitions`, which exist without any item) and any table this build's registry does not
- * know. Rule 6 then gives every manifest a copy of the referenced rows its own rows point at.
- */
- let handled: HashSet<&str> = [FOLDERS_TABLE, ITEMS_TABLE, LOGOS_TABLE, ENCRYPTION_KEYS_TABLE]
- .into_iter()
- .chain(item_scoped_names.iter().map(String::as_str))
- .collect();
- let remaining_names: Vec<String> = tables.keys().filter(|name| !handled.contains(name.as_str())).cloned().collect();
- for name in remaining_names {
- let rows = tables.remove(&name).unwrap_or_default();
- let mut base_rows: Vec<CodecRecord> = Vec::with_capacity(rows.len());
- for mut row in rows {
- match route(&row) {
- Route::Partition(spec_idx) => {
- row.insert(MANIFEST_ID_COL.to_string(), json!(specs[spec_idx].manifest_id));
- partitions[spec_idx].tables.entry(name.clone()).or_default().push(row);
- }
- Route::Base => {
- row.insert(MANIFEST_ID_COL.to_string(), json!(writing_manifest_id));
- base_rows.push(row);
- }
- Route::Gone => {}
- }
+ // A table the vault carries stays declared in every partition, so an emptied table is written as empty.
+ for partition in &mut partitions {
+ partition.tables.entry(name.clone()).or_default();
}
- tables.insert(name, base_rows);
}
- /*
- * 6. Each manifest ends up self-contained: every row it references but does not hold is copied out
- * of the vault-wide snapshot and stamped for this manifest, keeping its id where the id is opaque
- * and recomputing it where it derives from the scope.
- */
- let no_logos: Vec<CodecRecord> = Vec::new();
- let all_logos = snapshots.get(LOGOS_TABLE).unwrap_or(&no_logos);
+ // Each manifest ends up self-contained: the logos and referenced rows its own rows point at, under its own id.
+ let all_logos = rows_of(snapshots, LOGOS_TABLE);
+ let writing_manifest_id = specs.first().map(|spec| spec.manifest_id.as_str()).unwrap_or_default();
for partition in partitions.iter_mut() {
- let scope = partition.manifest_id.clone();
- reconcile_logo_references(&mut partition.tables, &scope, all_logos);
- normalize_logo_scope(&mut partition.tables, &scope);
- prune_unreferenced_logos(&mut partition.tables);
- clone_referenced_rows(&mut partition.tables, &scope, snapshots, writing_manifest_id);
+ let manifest_id = partition.spec.manifest_id.clone();
+ localize_logos(&mut partition.tables, &manifest_id, all_logos);
+ clone_referenced_rows(&mut partition.tables, &manifest_id, snapshots, writing_manifest_id);
}
-
Ok(partitions)
}
-/// Clone any row this manifest references but does not hold.
-pub(super) fn clone_referenced_rows(tables: &mut HashMap<String, Vec<CodecRecord>>, scope: &str, snapshots: &HashMap<String, Vec<CodecRecord>>, writing_manifest_id: &str) {
+/// Clone any row this manifest references but does not hold, keeping its id and stamping it for `manifest_id`. When
+/// the vault holds the row in several manifests, the copy comes from `preferred_manifest_id` when it has one.
+fn clone_referenced_rows(tables: &mut Tables, manifest_id: &str, snapshots: &Tables, preferred_manifest_id: &str) {
for (target, referencing) in REFERENCED_TABLES {
let Some(source_rows) = snapshots.get(*target) else { continue };
-
- let present: HashSet<String> = tables
- .get(*target)
- .map(|rows| rows.iter().filter_map(|r| str_col(r, ID_COL)).map(str::to_string).collect())
- .unwrap_or_default();
-
- let mut missing: Vec<String> = referencing
- .iter()
- .flat_map(|(ref_table, ref_column)| rows_of(tables, ref_table).iter().filter_map(move |row| str_col(row, ref_column)))
- .filter(|id| !present.contains(*id))
- .map(str::to_string)
- .collect();
+ let present: HashSet<String> = rows_of(tables, target).iter().filter_map(|r| str_col(r, ID_COL)).map(str::to_string).collect();
+ let mut missing: Vec<String> = referencing.iter().flat_map(|(ref_table, ref_column)| rows_of(tables, ref_table).iter().filter_map(move |row| str_col(row, ref_column))).filter(|id| !present.contains(*id)).map(str::to_string).collect();
missing.sort();
missing.dedup();
- if missing.is_empty() {
- continue;
- }
- let scope_value = json!(scope);
let clones: Vec<CodecRecord> = missing
.iter()
- .filter_map(|id| pick_source_row(source_rows, id, writing_manifest_id))
+ .filter_map(|id| pick_source_row(source_rows, id, preferred_manifest_id))
.map(|row| {
let mut clone = row.clone();
- clone.insert(MANIFEST_ID_COL.to_string(), scope_value.clone());
+ clone.insert(MANIFEST_ID_COL.to_string(), json!(manifest_id));
clone
})
.collect();
@@ -376,254 +176,70 @@ pub(super) fn clone_referenced_rows(tables: &mut HashMap<String, Vec<CodecRecord
}
}
-/// The snapshot row to clone for `id`.
-fn pick_source_row<'a>(rows: &'a [CodecRecord], id: &str, writing_manifest_id: &str) -> Option<&'a CodecRecord> {
+/// The snapshot row to clone for `id`: the one in `preferred_manifest_id`, else the one with the lowest manifest id.
+fn pick_source_row<'a>(rows: &'a [CodecRecord], id: &str, preferred_manifest_id: &str) -> Option<&'a CodecRecord> {
let candidates: Vec<&CodecRecord> = rows.iter().filter(|row| str_col(row, ID_COL) == Some(id)).collect();
- if let Some(own) = candidates.iter().find(|row| str_col(row, MANIFEST_ID_COL).map(|scope| ids_equal(scope, writing_manifest_id)).unwrap_or(false)) {
+ if let Some(own) = candidates.iter().find(|row| str_col(row, MANIFEST_ID_COL).is_some_and(|stamp| ids_equal(stamp, preferred_manifest_id))) {
return Some(own);
}
candidates.into_iter().min_by_key(|row| str_col(row, MANIFEST_ID_COL).unwrap_or_default().to_ascii_lowercase())
}
-/// Reproducible content like `Logos` can be refetched, so we drop rows no item in this table set references.
-pub(super) fn prune_unreferenced_logos(tables: &mut HashMap<String, Vec<CodecRecord>>) {
- let referenced: HashSet<String> = tables
- .get(ITEMS_TABLE)
- .map(|rows| rows.iter().filter_map(|r| str_col(r, LOGO_ID_COL)).map(str::to_string).collect())
- .unwrap_or_default();
- if let Some(logos) = tables.get_mut(LOGOS_TABLE) {
- logos.retain(|row| is_custom_logo(row) || str_col(row, ID_COL).map(|id| referenced.contains(id)).unwrap_or(false));
- }
-}
-
-/// Combine the caller's own manifest tables with every other manifest's tables into one unified set.
-/// Every manifest is held to the same rules: its rows are stamped with its own id, it may publish only its own key material, and it may carry no bucketed or bookkeeping tables.
-pub(super) fn combine_manifest_tables(
- mut tables: HashMap<String, Vec<CodecRecord>>,
- base_manifest_id: &str,
- other_manifests: Vec<Manifest>,
-) -> HashMap<String, Vec<CodecRecord>> {
- tables.retain(|name, _| !is_bucketed_table(name));
- claim_manifest_scope(&mut tables, base_manifest_id);
-
- /*
- * First-manifest-wins registry: table -> set of row identities already present. Every row is stamped
- * by `claim_manifest_scope` above, tables this build does not know included, and the identity folds
- * the stamp in, so this can never fire across manifests; it only dedupes a truly duplicated row
- * inside one manifest.
- */
+/// Combine every manifest's tables into one unified set. Every manifest is held to the same rules: its rows are
+/// stamped with its own id, it may publish only its own key material, and it may carry no bucketed, personal or
+/// bookkeeping table. The references a namespace boundary leaves dangling are repaired at the end.
+pub(super) fn combine_manifest_tables(manifests: Vec<Manifest>) -> Tables {
+ let mut combined = Tables::new();
+ // First-manifest-wins per row identity. The identity folds the stamp in, so this can never fire across manifests;
+ // it only dedupes a truly duplicated row inside one manifest.
let mut seen: HashMap<String, HashSet<String>> = HashMap::new();
- for (name, rows) in &tables {
- let keys = seen.entry(name.clone()).or_default();
- for row in rows {
- if let Some(identity) = row_identity(name, row) {
- keys.insert(identity);
- }
- }
- }
-
- for mut manifest in other_manifests {
- // The manifest's own id is the scope every row inside must claim.
+ for mut manifest in manifests {
let manifest_id = manifest.manifest_id.clone();
- claim_manifest_scope(&mut manifest.tables, &manifest_id);
-
+ stamp_manifest_rows(&mut manifest.tables, &manifest_id);
for (name, rows) in manifest.tables {
- /*
- * Bucketed tables sync as their own resource beside the manifest, so a manifest
- * carrying them is malformed however it was produced; a personal-only table has no business
- * in a manifest that is not this vault's own; and local bookkeeping and platform
- * skip-tables never travel inside one. Dropping them here also means a manifest authored by
- * another user cannot inject rows into this vault's personal scope.
- */
+ // Bucketed tables sync beside the manifest, a personal table may not leave the user's own vault, and local
+ // bookkeeping never travels inside one. Dropping them here also keeps a manifest authored by another user
+ // from injecting rows into this vault's personal manifest.
if is_local_only_table(&name) || is_bucketed_table(&name) || is_personal_table(&name) {
continue;
}
-
let keys = seen.entry(name.clone()).or_default();
- let target = tables.entry(name.clone()).or_default();
+ let target = combined.entry(name.clone()).or_default();
for row in rows {
- match row_identity(&name, &row) {
- Some(identity) => {
- if keys.insert(identity) {
- target.push(row);
- }
- }
- None => target.push(row),
+ if row_identity(&name, &row).is_none_or(|identity| keys.insert(identity)) {
+ target.push(row);
}
}
}
}
-
- /*
- * Repair what a namespace boundary can leave dangling, so the combined set satisfies the schema's
- * foreign keys: a folder's parent or an item's folder may live in a manifest this user has no access
- * to, and a manifest may carry a child row naming an item it does not hold. Each lookup is per
- * manifest, since with a composite foreign key, resolving in *some* manifest is not resolving.
- */
- null_dangling_parent_folders(&mut tables);
- null_dangling_item_folders(&mut tables);
- null_dangling_field_definitions(&mut tables);
- drop_orphan_item_children(&mut tables);
- drop_item_tags_without_a_tag(&mut tables);
-
- tables
-}
-
-/// Null or drop rows whose reference names no row in this manifest.
-pub(super) fn prune_dangling_references(tables: &mut HashMap<String, Vec<CodecRecord>>) {
- if tables.contains_key(FOLDERS_TABLE) {
- null_dangling_item_folders(tables);
- }
- drop_orphan_item_children(tables);
- drop_item_tags_without_a_tag(tables);
+ integrity::repair(&mut combined);
+ combined
}
-/// Make every row of a single manifest's table set claim `manifest_id` as its scope, every table alike,
-/// registered or not. Stamping a row with the manifest it arrived in is what lets the next push route
-/// it home by its own stamp, instead of trusting whatever scope its author wrote into it (which would
-/// let one manifest inject rows into another's namespace through a table the registry cannot police).
-///
-/// Two tables are held out of the blanket stamp. `Logos` are stamped by [`normalize_logo_scope`], which
-/// re-derives their ids and repoints `Items.LogoId` in the same pass. `EncryptionKeys` are not
-/// stamped at all: there the stamp is a *claim* that [`retain_own_encryption_keys`] checks, and stamping
-/// first would make every claim true.
-fn claim_manifest_scope(tables: &mut HashMap<String, Vec<CodecRecord>>, manifest_id: &str) {
- normalize_logo_scope(tables, manifest_id);
+/// Stamp every row of one manifest's tables with `manifest_id`, so the next push routes it by the manifest it arrived in.
+/// `EncryptionKeys` are never stamped: their stamp is a claim [`retain_own_encryption_keys`] checks, and stamping would make it true.
+fn stamp_manifest_rows(tables: &mut Tables, manifest_id: &str) {
+ restamp_logos(tables, manifest_id);
retain_own_encryption_keys(tables, manifest_id);
-
- let names: Vec<String> = tables.keys().filter(|name| *name != LOGOS_TABLE && *name != ENCRYPTION_KEYS_TABLE).cloned().collect();
- for table in names {
- stamp_table(tables, &table, manifest_id);
- }
-}
-
-/// Keep only the `EncryptionKeys` rows a manifest may legitimately publish: the ones stamped with its
-/// own id. Applied to every manifest alike, so each namespace is protected from the others.
-///
-/// Unstamped rows are dropped here rather than stamped. Stamping them is a *local* migration concern handled
-/// on the canonicalize side, so an unstamped row in a manifest coming back through combine is not a
-/// legacy row: it is a manifest asking to have key material moved into a scope it never proved it
-/// owns.
-fn retain_own_encryption_keys(tables: &mut HashMap<String, Vec<CodecRecord>>, manifest_id: &str) {
- if let Some(rows) = tables.get_mut(ENCRYPTION_KEYS_TABLE) {
- rows.retain(|row| str_col(row, MANIFEST_ID_COL).is_some_and(|scope| ids_equal(scope, manifest_id)));
- }
-}
-
-/// Stamp every row of `table` with `manifest_id` in its `ManifestId` column.
-fn stamp_table(tables: &mut HashMap<String, Vec<CodecRecord>>, table: &str, manifest_id: &str) {
- if let Some(rows) = tables.get_mut(table) {
+ for (_, rows) in tables.iter_mut().filter(|(name, _)| *name != LOGOS_TABLE && *name != ENCRYPTION_KEYS_TABLE) {
for row in rows {
row.insert(MANIFEST_ID_COL.to_string(), json!(manifest_id));
}
}
}
-/// Null every `Folders.ParentFolderId` that doesn't resolve to a folder *in the same manifest*.
-fn null_dangling_parent_folders(tables: &mut HashMap<String, Vec<CodecRecord>>) {
- null_dangling_reference(tables, FOLDERS_TABLE, FOLDERS_TABLE, PARENT_FOLDER_ID_COL);
-}
-
-/// Null every `Items.FolderId` that doesn't resolve to a folder *in the same manifest*.
-fn null_dangling_item_folders(tables: &mut HashMap<String, Vec<CodecRecord>>) {
- null_dangling_reference(tables, ITEMS_TABLE, FOLDERS_TABLE, FOLDER_ID_COL);
-}
-
-/// Null every `FieldDefinitionId` that names no definition in the referencing row's own manifest.
-///
-/// The column is nullable (a value with no definition is how a *system* field is stored), so nulling
-/// keeps the user's value and loses only the metadata this vault does not hold anyway. Dropping the row
-/// instead would throw away what the user actually typed.
-fn null_dangling_field_definitions(tables: &mut HashMap<String, Vec<CodecRecord>>) {
- for table in [FIELD_VALUES_TABLE, FIELD_HISTORIES_TABLE] {
- null_dangling_reference(tables, table, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL);
- }
-}
-
-/// Drop every `ItemTags` row whose `(ManifestId, TagId)` names no tag in the combined set.
-///
-/// Unlike a field definition, the tag reference is the entire content of the row and cannot be nulled.
-/// Skipped when the combined set carries no `Tags` table at all, which means a partial table set rather
-/// than a vault with no tags.
-fn drop_item_tags_without_a_tag(tables: &mut HashMap<String, Vec<CodecRecord>>) {
- if !tables.contains_key(TAGS_TABLE) {
- return;
- }
- let tag_ids = scoped_ids(tables, TAGS_TABLE);
- if let Some(rows) = tables.get_mut(ITEM_TAGS_TABLE) {
- rows.retain(|row| match str_col(row, TAG_ID_COL) {
- None => true,
- Some(tag_id) => tag_ids.contains(&(str_col(row, MANIFEST_ID_COL).unwrap_or_default().to_string(), tag_id.to_string())),
- });
- }
-}
-
-/// Null every `table.column` reference that names no row of `target` inside the referencing row's own
-/// manifest. Both foreign keys this serves have a composite `(ManifestId, Id)` parent, so a reference
-/// that resolves only in *another* namespace is as dangling as one that resolves nowhere.
-fn null_dangling_reference(tables: &mut HashMap<String, Vec<CodecRecord>>, table: &str, target: &str, column: &str) {
- let target_ids = scoped_ids(tables, target);
- if let Some(rows) = tables.get_mut(table) {
- for row in rows {
- let scope = str_col(row, MANIFEST_ID_COL).unwrap_or_default().to_string();
- if let Some(reference) = str_col(row, column) {
- if !target_ids.contains(&(scope, reference.to_string())) {
- row.insert(column.to_string(), Value::Null);
- }
- }
- }
- }
-}
-
-/// Drop every item-scoped row whose `(ManifestId, ItemId)` names no item in the combined set.
-///
-/// A manifest is self-contained, so a leftover child is one whose item this vault does not hold: a
-/// manifest built against a stale item set, or one hand-crafted to attach rows to an item in a
-/// namespace it has no access to. Either way the composite foreign key rejects it, so it is dropped
-/// here rather than failing the whole insert.
-///
-/// Skipped when the combined set carries no `Items` table at all: that is a caller passing a partial
-/// table set, and pruning against an absent parent would delete every child row.
-fn drop_orphan_item_children(tables: &mut HashMap<String, Vec<CodecRecord>>) {
- if !tables.contains_key(ITEMS_TABLE) {
- return;
- }
- let item_ids = scoped_ids(tables, ITEMS_TABLE);
- for table in manifest_scoped_tables() {
- let Some(rows) = tables.get_mut(table) else { continue };
- rows.retain(|row| match str_col(row, ITEM_ID_COL) {
- None => true,
- Some(item_id) => {
- let scope = str_col(row, MANIFEST_ID_COL).unwrap_or_default().to_string();
- item_ids.contains(&(scope, item_id.to_string()))
- }
- });
+/// Keep only the `EncryptionKeys` rows a manifest may legitimately publish: the ones stamped with its own id. An
+/// unstamped row is dropped rather than stamped: it is a manifest asking to have key material moved into a manifest it
+/// never proved it owns.
+fn retain_own_encryption_keys(tables: &mut Tables, manifest_id: &str) {
+ if let Some(rows) = tables.get_mut(ENCRYPTION_KEYS_TABLE) {
+ rows.retain(|row| str_col(row, MANIFEST_ID_COL).is_some_and(|stamp| ids_equal(stamp, manifest_id)));
}
}
-/// The `(ManifestId, Id)` identity of every row in `table`.
-fn scoped_ids(tables: &HashMap<String, Vec<CodecRecord>>, table: &str) -> HashSet<(String, String)> {
- tables
- .get(table)
- .map(|rows| {
- rows.iter()
- .filter_map(|row| str_col(row, ID_COL).map(|id| (str_col(row, MANIFEST_ID_COL).unwrap_or_default().to_string(), id.to_string())))
- .collect()
- })
- .unwrap_or_default()
-}
-
-/// The encryption-key row whose `PublicKey` matches `public_key`, from a decrypted manifest. The row
-/// must be stamped with the manifest's own id, so a caller that skips the combine step still cannot be
-/// tricked into using another manifest's key material. Deleted rows are skipped; returns `None` when no
-/// live row carries that key.
+/// The encryption-key row whose `PublicKey` matches `public_key`, from a decrypted manifest. The row must be
+/// stamped with the manifest's own id, so a caller that skips the combine step still cannot be tricked into using
+/// another manifest's key material. Deleted rows are skipped.
pub fn extract_encryption_key_for_public_key(manifest: &Manifest, public_key: &str) -> Option<CodecRecord> {
- let scope = Some(manifest.manifest_id.as_str());
- manifest
- .tables
- .get(ENCRYPTION_KEYS_TABLE)?
- .iter()
- .find(|row| str_col(row, PUBLIC_KEY_COL) == Some(public_key) && str_col(row, MANIFEST_ID_COL) == scope && !is_deleted(row))
- .cloned()
+ manifest.tables.get(ENCRYPTION_KEYS_TABLE)?.iter().find(|row| str_col(row, PUBLIC_KEY_COL) == Some(public_key) && str_col(row, MANIFEST_ID_COL).is_some_and(|stamp| ids_equal(stamp, &manifest.manifest_id)) && !is_deleted(row)).cloned()
}
-
diff --git a/core/rust/src/vault_codec/sharing_tests.rs b/core/rust/src/vault_codec/sharing_tests.rs
index a8c92fdec..cf1829039 100644
--- a/core/rust/src/vault_codec/sharing_tests.rs
+++ b/core/rust/src/vault_codec/sharing_tests.rs
@@ -1,55 +1,33 @@
-//! Unit tests for the shared-manifest split (canonicalize) and combine (materialize) logic.
-//!
-//! These cover the core contract of vault sharing: `canonicalize_from_sqlite` with shared-manifest
-//! specs pulls each anchor folder's subtree into its own manifest, and `materialize_as_sqlite` with
-//! shared manifests combines everything back into one unified table set, for both the owner (whose
-//! local vault holds every row) and a recipient (whose personal manifest knows nothing of the share).
+//! The shared-manifest split (canonicalize) and combine (materialize): every row routes to the manifest its stamp
+//! names, each manifest is self-contained, and nothing crosses a manifest boundary that should not.
use super::*;
-use super::test_support::{b64, materialize_input, materialize_manifests, row, stamp_unstamped, table};
+use super::row::rows_of;
+use super::test_support::{b64, materialize_input, materialized_map, row, stamped, table, PERSONAL_MANIFEST_ID as PERSONAL, PERSONAL_SALT as SALT_PERSONAL, SHARED_MANIFEST_ID as SHARED};
use super::types::{is_bucketed_table, is_personal_table, manifest_scoped_tables, SCHEMA_VERSION};
-use crate::vault_model::names::LOGO_KIND_FAVICON;
use crate::vault_model::OVERFLOW_TABLE;
use serde_json::json;
use std::collections::{HashMap, HashSet};
-const SALT_PERSONAL: &str = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff";
const SALT_SHARED: &str = "ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766554433221100";
-/// The personal manifest id every test canonicalizes against. Personal rows are stamped with it, there is
-/// no NULL-scope convention anywhere in the format.
-const PERSONAL_M: &str = "m-personal";
-
-/// A spec for `folder_id`, whose manifest id is derived as `m-<folder_id>` so every test can predict
-/// the scope stamp of a partition's rows.
-fn spec(folder_id: &str) -> ManifestSpec {
- ManifestSpec {
- manifest_id: format!("m-{folder_id}"),
- manifest_salt: SALT_SHARED.to_string(),
- name: Some(format!("Share {folder_id}")),
- }
+fn spec(manifest_id: &str) -> ManifestSpec {
+ let (salt, name) = if manifest_id == PERSONAL { (SALT_PERSONAL, None) } else { (SALT_SHARED, Some(format!("Share {manifest_id}"))) };
+ ManifestSpec { manifest_id: manifest_id.to_string(), manifest_salt: salt.to_string(), name }
}
-/// A canonicalize input whose rows are stamped the way a real client writes them.
-fn input_with_shares(tables: Vec<CodecTableData>, shared_manifests: Vec<ManifestSpec>) -> CanonicalizeInput {
- raw_input_with_shares(stamp_unstamped(tables, PERSONAL_M), shared_manifests)
+/// A canonicalize input for the personal manifest plus `shared` manifests; rows carry their stamps already.
+fn input(tables: Vec<CodecTableData>, shared: &[&str]) -> CanonicalizeInput {
+ CanonicalizeInput { tables, canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(), manifests: std::iter::once(PERSONAL).chain(shared.iter().copied()).map(spec).collect(), stamp_unstamped_into: None }
}
-/// [`input_with_shares`] without the stamping pass, for the tests that hand canonicalize rows a client
-/// failed to stamp.
-fn raw_input_with_shares(tables: Vec<CodecTableData>, shared_manifests: Vec<ManifestSpec>) -> CanonicalizeInput {
- let personal = ManifestSpec { manifest_id: PERSONAL_M.to_string(), manifest_salt: SALT_PERSONAL.to_string(), name: None };
- CanonicalizeInput {
- tables,
- canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(),
- manifests: std::iter::once(personal).chain(shared_manifests).collect(),
- stamp_unstamped_into: None,
- }
+fn canonicalize(tables: Vec<CodecTableData>, shared: &[&str]) -> CanonicalizedVault {
+ canonicalize_from_sqlite(input(tables, shared)).unwrap()
}
-/// The rows of `name` in a manifest (empty slice when the table is absent).
-fn rows<'a>(m: &'a Manifest, name: &str) -> &'a [CodecRecord] {
- m.tables.get(name).map(Vec::as_slice).unwrap_or(&[])
+/// The manifest with `id` in a canonicalize result.
+fn manifest<'a>(out: &'a CanonicalizedVault, id: &str) -> &'a CanonicalizedManifest {
+ out.manifests.iter().find(|m| m.manifest.manifest_id == id).unwrap_or_else(|| panic!("no manifest {id}"))
}
fn ids(records: &[CodecRecord]) -> Vec<&str> {
@@ -58,834 +36,508 @@ fn ids(records: &[CodecRecord]) -> Vec<&str> {
out
}
-/// FieldValues rows are identified by their `Value` marker.
+/// FieldValues rows are identified by their `Value`.
fn values(records: &[CodecRecord]) -> Vec<&str> {
let mut out: Vec<&str> = records.iter().filter_map(|r| r.get("Value").and_then(|v| v.as_str())).collect();
out.sort();
out
}
-/// ItemTags rows are identified by their natural key.
-fn tag_links(records: &[CodecRecord]) -> Vec<(String, String)> {
- let mut out: Vec<(String, String)> = records
- .iter()
- .filter_map(|r| Some((r.get("ItemId")?.as_str()?.to_string(), r.get("TagId")?.as_str()?.to_string())))
- .collect();
- out.sort();
- out
+fn find<'a>(records: &'a [CodecRecord], id: &str) -> &'a CodecRecord {
+ records.iter().find(|r| r["Id"] == json!(id)).unwrap_or_else(|| panic!("no row {id}"))
+}
+
+fn find_mut<'a>(tables: &'a mut [CodecTableData], name: &str, id: &str) -> &'a mut CodecRecord {
+ tables.iter_mut().find(|t| t.name == name).unwrap().records.iter_mut().find(|r| r["Id"] == json!(id)).unwrap()
+}
+
+fn find_mut_in<'a>(m: &'a mut Manifest, table: &str, id: &str) -> &'a mut CodecRecord {
+ m.tables.get_mut(table).unwrap().iter_mut().find(|r| r["Id"] == json!(id)).unwrap()
}
-/// The scoped logo id the codec derives for `(manifest id, source)`, what tests assert against, since
-/// a logo's identity is a function of its manifest and domain rather than whatever id the writer generated.
-fn logo_id(scope: &str, source: &str) -> String {
- scoped_assets::logo_id_for(scope, LOGO_KIND_FAVICON, source)
+fn logo_id(manifest_id: &str, source: &str) -> String {
+ logo_id_for(manifest_id, "favicon", source)
}
-/// The `Logos` row for `source` in a manifest, resolved through its manifest-id-derived id.
+/// The `Logos` row for `source` in a manifest, resolved through its derived id.
fn logo_row<'a>(m: &'a Manifest, source: &str) -> &'a CodecRecord {
- let id = logo_id(&m.manifest_id, source);
- rows(m, "Logos")
- .iter()
- .find(|r| r.get("Id").and_then(|v| v.as_str()) == Some(id.as_str()))
- .unwrap_or_else(|| panic!("no {source} logo in manifest {:?}", m.manifest_id))
+ find(rows_of(&m.tables, "Logos"), &logo_id(&m.manifest_id, source))
}
-/// The `Source` values of a manifest's logos, sorted.
fn logo_sources(m: &Manifest) -> Vec<&str> {
- let mut out: Vec<&str> = rows(m, "Logos").iter().filter_map(|r| r.get("Source").and_then(|v| v.as_str())).collect();
+ let mut out: Vec<&str> = rows_of(&m.tables, "Logos").iter().filter_map(|r| r.get("Source").and_then(|v| v.as_str())).collect();
out.sort();
out
}
-/// Stamp a folder's whole subtree (its folders and the items in them) into `manifest_id`. Mirrors
-/// exactly what the client's `FolderRepository.restampSubtree` writes when a folder starts being
-/// shared, so fixtures carry the membership the codec now routes on.
-///
-/// Logos are not stamped here, just as the client does not stamp them: a logo is a
-/// per-manifest asset the codec reconciles itself (cloning a copy into every manifest whose items
-/// reference it), so moving items between manifests never has to move logo rows.
-fn stamp_subtree(mut tables: Vec<CodecTableData>, folder_id: &str, manifest_id: &str) -> Vec<CodecTableData> {
- // Walk the folder tree down from `folder_id`.
- let mut subtree: HashSet<String> = HashSet::new();
- subtree.insert(folder_id.to_string());
- loop {
- let mut grew = false;
- for t in tables.iter().filter(|t| t.name == "Folders") {
- for row in &t.records {
- let (Some(id), Some(parent)) = (row.get("Id").and_then(|v| v.as_str()), row.get("ParentFolderId").and_then(|v| v.as_str())) else { continue };
- if subtree.contains(parent) && subtree.insert(id.to_string()) {
- grew = true;
- }
- }
- }
- if !grew {
- break;
- }
- }
-
- // Folders in the subtree join the manifest; every other folder keeps the stamp it already had.
- for t in tables.iter_mut().filter(|t| t.name == "Folders") {
- for row in t.records.iter_mut() {
- let id = row.get("Id").and_then(|v| v.as_str()).unwrap_or("").to_string();
- if subtree.contains(&id) {
- row.insert("ManifestId".to_string(), json!(manifest_id));
- } else if !row.contains_key("ManifestId") {
- row.insert("ManifestId".to_string(), json!(PERSONAL_M));
- }
- }
- }
+fn manifests_of(out: &CanonicalizedVault) -> Vec<Manifest> {
+ out.manifests.iter().map(|m| m.manifest.clone()).collect()
+}
- /*
- * Every item takes the stamp of the folder it sits in, which is what the client's INSERT/UPDATE
- * statements write from the folder reference the caller names. Deriving it here rather than
- * carrying it forward is what makes an item moved OUT of a shared folder leave that manifest.
- */
- let folder_scope: HashMap<String, String> = tables
- .iter()
- .filter(|t| t.name == "Folders")
- .flat_map(|t| t.records.iter())
- .filter_map(|row| {
- let id = row.get("Id").and_then(|v| v.as_str())?;
- let scope = row.get("ManifestId").and_then(|v| v.as_str()).unwrap_or(PERSONAL_M);
- Some((id.to_string(), scope.to_string()))
- })
- .collect();
- for t in tables.iter_mut().filter(|t| t.name == "Items") {
- for row in t.records.iter_mut() {
- let scope = row
- .get("FolderId")
- .and_then(|v| v.as_str())
- .and_then(|f| folder_scope.get(f).cloned())
- .unwrap_or_else(|| PERSONAL_M.to_string());
- row.insert("ManifestId".to_string(), json!(scope));
+/// Every string a manifest carries, so a test can ask whether a secret leaked into it, base64 blobs included.
+fn mentions(m: &Manifest, needle: &str) -> bool {
+ fn walk(value: &serde_json::Value, out: &mut Vec<String>) {
+ match value {
+ serde_json::Value::String(s) => out.push(s.clone()),
+ serde_json::Value::Array(items) => items.iter().for_each(|v| walk(v, out)),
+ serde_json::Value::Object(map) => map.values().for_each(|v| walk(v, out)),
+ _ => {}
}
}
-
- tables
+ let mut strings = Vec::new();
+ m.tables.values().flatten().flat_map(|row| row.values()).for_each(|v| walk(v, &mut strings));
+ let encoded = b64(needle.as_bytes());
+ strings.iter().any(|s| s == needle || s == &encoded)
}
-/// A representative owner vault: a personal folder+item, and a shared manifest ("m-f-shared") carrying
-/// the "f-shared" subtree, two items (one in a subfolder), and their child rows.
+/// A representative owner vault, stamped as the client stamps it: a personal folder and item, a shared manifest
+/// holding the "f-shared" folder tree with two items and their child rows, and the content both reference.
fn owner_tables() -> Vec<CodecTableData> {
- stamp_subtree(owner_tables_unstamped(), "f-shared", "m-f-shared")
-}
-
-/// The same vault with no `ManifestId` stamps at all, as a 2.0.0 client schema vault holds it.
-fn owner_tables_unstamped() -> Vec<CodecTableData> {
vec![
table("Folders", vec![
- row(&[("Id", json!("f-personal")), ("Name", json!("Personal")), ("ParentFolderId", serde_json::Value::Null)]),
- row(&[("Id", json!("f-shared")), ("Name", json!("Family")), ("ParentFolderId", serde_json::Value::Null)]),
- row(&[("Id", json!("f-sub")), ("Name", json!("Streaming")), ("ParentFolderId", json!("f-shared"))]),
+ stamped(PERSONAL, &[("Id", json!("f-personal")), ("Name", json!("Personal")), ("ParentFolderId", serde_json::Value::Null)]),
+ stamped(SHARED, &[("Id", json!("f-shared")), ("Name", json!("Family")), ("ParentFolderId", serde_json::Value::Null)]),
+ stamped(SHARED, &[("Id", json!("f-sub")), ("Name", json!("Streaming")), ("ParentFolderId", json!("f-shared"))]),
]),
table("Items", vec![
- row(&[("Id", json!("i-personal")), ("FolderId", json!("f-personal")), ("LogoId", json!("logo-both"))]),
- row(&[("Id", json!("i-shared")), ("FolderId", json!("f-shared")), ("LogoId", json!("logo-both"))]),
- row(&[("Id", json!("i-sub")), ("FolderId", json!("f-sub")), ("LogoId", json!("logo-shared-only"))]),
- row(&[("Id", json!("i-nofolder")), ("FolderId", serde_json::Value::Null), ("LogoId", serde_json::Value::Null)]),
+ stamped(PERSONAL, &[("Id", json!("i-personal")), ("FolderId", json!("f-personal")), ("LogoId", json!("logo-both"))]),
+ stamped(SHARED, &[("Id", json!("i-shared")), ("FolderId", json!("f-shared")), ("LogoId", json!("logo-both"))]),
+ stamped(SHARED, &[("Id", json!("i-sub")), ("FolderId", json!("f-sub")), ("LogoId", json!("logo-shared-only"))]),
+ stamped(PERSONAL, &[("Id", json!("i-nofolder")), ("FolderId", serde_json::Value::Null), ("LogoId", serde_json::Value::Null)]),
]),
table("FieldValues", vec![
- row(&[("Id", json!("fv-personal")), ("ItemId", json!("i-personal")), ("FieldDefinitionId", json!("fd-1")), ("FieldKey", json!("username")), ("Value", json!("me"))]),
- row(&[("Id", json!("fv-shared")), ("ItemId", json!("i-shared")), ("FieldDefinitionId", json!("fd-1")), ("FieldKey", json!("username")), ("Value", json!("family"))]),
- row(&[("Id", json!("fv-sub")), ("ItemId", json!("i-sub")), ("FieldDefinitionId", serde_json::Value::Null), ("FieldKey", json!("password")), ("Value", json!("hunter2"))]),
- ]),
- table("TotpCodes", vec![
- row(&[("Id", json!("totp-shared")), ("ItemId", json!("i-shared")), ("SecretKey", json!({ "__b64": b64(&[9, 9, 9]) }))]),
+ stamped(PERSONAL, &[("Id", json!("fv-personal")), ("ItemId", json!("i-personal")), ("FieldDefinitionId", json!("fd-1")), ("FieldKey", json!("username")), ("Value", json!("me"))]),
+ stamped(SHARED, &[("Id", json!("fv-shared")), ("ItemId", json!("i-shared")), ("FieldDefinitionId", json!("fd-1")), ("FieldKey", json!("username")), ("Value", json!("family"))]),
+ stamped(SHARED, &[("Id", json!("fv-sub")), ("ItemId", json!("i-sub")), ("FieldDefinitionId", serde_json::Value::Null), ("FieldKey", json!("password")), ("Value", json!("hunter2"))]),
]),
+ table("TotpCodes", vec![stamped(SHARED, &[("Id", json!("totp-shared")), ("ItemId", json!("i-shared")), ("SecretKey", json!({ "__b64": b64(&[9, 9, 9]) }))])]),
table("Attachments", vec![
- row(&[("Id", json!("att-shared")), ("ItemId", json!("i-sub")), ("Blob", json!({ "__b64": b64(&[1, 2, 3, 4]) }))]),
- row(&[("Id", json!("att-personal")), ("ItemId", json!("i-personal")), ("Blob", json!({ "__b64": b64(&[5, 6]) }))]),
+ stamped(SHARED, &[("Id", json!("att-shared")), ("ItemId", json!("i-sub")), ("Blob", json!({ "__b64": b64(&[1, 2, 3, 4]) }))]),
+ stamped(PERSONAL, &[("Id", json!("att-personal")), ("ItemId", json!("i-personal")), ("Blob", json!({ "__b64": b64(&[5, 6]) }))]),
]),
table("Tags", vec![
- row(&[("Id", json!("tag-both")), ("Name", json!("work"))]),
- row(&[("Id", json!("tag-shared-only")), ("Name", json!("family"))]),
- row(&[("Id", json!("tag-unused")), ("Name", json!("todo"))]),
+ stamped(PERSONAL, &[("Id", json!("tag-both")), ("Name", json!("work"))]),
+ stamped(PERSONAL, &[("Id", json!("tag-shared-only")), ("Name", json!("family"))]),
+ stamped(PERSONAL, &[("Id", json!("tag-unused")), ("Name", json!("todo"))]),
]),
table("ItemTags", vec![
- row(&[("Id", json!("it-1")), ("ItemId", json!("i-personal")), ("TagId", json!("tag-both"))]),
- row(&[("Id", json!("it-2")), ("ItemId", json!("i-shared")), ("TagId", json!("tag-both"))]),
- row(&[("Id", json!("it-3")), ("ItemId", json!("i-sub")), ("TagId", json!("tag-shared-only"))]),
- ]),
- table("FieldDefinitions", vec![
- row(&[("Id", json!("fd-1")), ("Key", json!("username"))]),
- row(&[("Id", json!("fd-unused")), ("Key", json!("custom"))]),
+ stamped(PERSONAL, &[("ItemId", json!("i-personal")), ("TagId", json!("tag-both"))]),
+ stamped(SHARED, &[("ItemId", json!("i-shared")), ("TagId", json!("tag-both"))]),
+ stamped(SHARED, &[("ItemId", json!("i-sub")), ("TagId", json!("tag-shared-only"))]),
]),
+ table("FieldDefinitions", vec![stamped(PERSONAL, &[("Id", json!("fd-1")), ("Key", json!("username"))]), stamped(PERSONAL, &[("Id", json!("fd-unused")), ("Key", json!("custom"))])]),
table("Logos", vec![
- row(&[("Id", json!("logo-both")), ("Source", json!("github.com")), ("FileData", json!({ "__b64": b64(&[0xAA, 0xBB]) }))]),
- row(&[("Id", json!("logo-shared-only")), ("Source", json!("netflix.com")), ("FileData", json!({ "__b64": b64(&[0xCC, 0xDD]) }))]),
- ]),
- table("EncryptionKeys", vec![
- row(&[("Id", json!("ek-1")), ("PublicKey", json!("pub")), ("PrivateKey", json!("priv")), ("IsPrimary", json!(1)), ("IsDeleted", json!(0))]),
- ]),
- table("Settings", vec![
- row(&[("Key", json!("theme")), ("Value", json!("dark"))]),
+ stamped(PERSONAL, &[("Id", json!("logo-both")), ("Source", json!("github.com")), ("FileData", json!({ "__b64": b64(&[0xAA, 0xBB]) }))]),
+ stamped(PERSONAL, &[("Id", json!("logo-shared-only")), ("Source", json!("netflix.com")), ("FileData", json!({ "__b64": b64(&[0xCC, 0xDD]) }))]),
]),
+ table("EncryptionKeys", vec![stamped(PERSONAL, &[("Id", json!("ek-1")), ("PublicKey", json!("pub")), ("PrivateKey", json!("priv")), ("IsPrimary", json!(1)), ("IsDeleted", json!(0))])]),
+ table("Settings", vec![stamped(PERSONAL, &[("Key", json!("theme")), ("Value", json!("dark"))])]),
]
}
fn canonicalize_owner() -> CanonicalizedVault {
- canonicalize_from_sqlite(input_with_shares(owner_tables(), vec![spec("f-shared")])).unwrap()
-}
-
-// Canonicalize: splitting
-
-#[test]
-fn split_moves_folder_subtree_and_items_into_shared_manifest() {
- let out = canonicalize_owner();
- assert_eq!(out.rest().len(), 1);
- let shared = &out.rest()[0].manifest;
-
- assert_eq!(out.rest()[0].manifest.name.as_deref(), Some("Share f-shared"));
- assert_eq!(shared.manifest_id, "m-f-shared");
- assert_eq!(shared.name.as_deref(), Some("Share f-shared"), "the manifest carries its display name, not a folder anchor");
- assert_eq!(out.first().manifest.manifest_id, PERSONAL_M);
- assert_eq!(shared.manifest_salt, SALT_SHARED);
-
- // Folder subtree (anchor + subfolder) moved; personal folder stayed.
- assert_eq!(ids(rows(shared, "Folders")), vec!["f-shared", "f-sub"]);
- assert_eq!(ids(rows(&out.first().manifest, "Folders")), vec!["f-personal"]);
-
- // Items in the subtree moved (including nested subfolder items); others stayed.
- assert_eq!(ids(rows(shared, "Items")), vec!["i-shared", "i-sub"]);
- assert_eq!(ids(rows(&out.first().manifest, "Items")), vec!["i-nofolder", "i-personal"]);
+ canonicalize(owner_tables(), &[SHARED])
}
-#[test]
-fn split_normalizes_shared_anchor_parent_to_null() {
- // Owner had nested the anchor folder under a personal folder: the shared manifest must not leak
- // (or depend on) that personal placement.
+/// `owner_tables()` with `change` applied to the named table's rows.
+fn owner_tables_with(name: &str, change: impl FnOnce(&mut Vec<CodecRecord>)) -> Vec<CodecTableData> {
let mut tables = owner_tables();
- tables[0].records[1].insert("ParentFolderId".to_string(), json!("f-personal"));
- // Moving an item across a folder boundary moves it across a manifest boundary: the client
- // re-stamps it on the move, so the fixture does too.
- let tables = stamp_subtree(tables, "f-shared", "m-f-shared");
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- let shared_anchor = rows(&out.rest()[0].manifest, "Folders").iter().find(|r| r["Id"] == json!("f-shared")).unwrap().clone();
- assert_eq!(shared_anchor["ParentFolderId"], serde_json::Value::Null);
- // The subfolder keeps its parent (it points inside the shared manifest).
- let sub = rows(&out.rest()[0].manifest, "Folders").iter().find(|r| r["Id"] == json!("f-sub")).unwrap().clone();
- assert_eq!(sub["ParentFolderId"], json!("f-shared"));
+ change(&mut tables.iter_mut().find(|t| t.name == name).unwrap().records);
+ tables
}
-#[test]
-fn split_routes_item_scoped_tables_generically() {
- // FieldValues / TotpCodes / Attachments / ItemTags all follow their item with zero per-table wiring.
- let out = canonicalize_owner();
- let shared = &out.rest()[0].manifest;
-
- assert_eq!(values(rows(shared, "FieldValues")), vec!["family", "hunter2"]);
- assert_eq!(values(rows(&out.first().manifest, "FieldValues")), vec!["me"]);
-
- assert_eq!(ids(rows(shared, "TotpCodes")), vec!["totp-shared"]);
- assert!(rows(&out.first().manifest, "TotpCodes").is_empty());
-
- assert_eq!(ids(rows(shared, "Attachments")), vec!["att-shared"]);
- assert_eq!(ids(rows(&out.first().manifest, "Attachments")), vec!["att-personal"]);
-
- assert_eq!(tag_links(rows(shared, "ItemTags")), vec![("i-shared".into(), "tag-both".into()), ("i-sub".into(), "tag-shared-only".into())]);
- assert_eq!(tag_links(rows(&out.first().manifest, "ItemTags")), vec![("i-personal".into(), "tag-both".into())]);
+/// Move `item_id` into `folder_id`, which lives in `manifest_id`: the client restamps the item and its children.
+fn move_item(tables: &mut [CodecTableData], item_id: &str, folder_id: &str, manifest_id: &str) {
+ for table in tables.iter_mut() {
+ for row in table.records.iter_mut() {
+ let is_item = table.name == "Items" && row["Id"] == json!(item_id);
+ let is_child = row.get("ItemId") == Some(&json!(item_id));
+ if is_item {
+ row.insert("FolderId".to_string(), json!(folder_id));
+ }
+ if is_item || is_child {
+ row.insert("ManifestId".to_string(), json!(manifest_id));
+ }
+ }
+ }
}
+/*
+ * Canonicalize: routing by stamp.
+ */
+
#[test]
-fn split_gives_each_manifest_its_own_copy_of_the_tags_it_uses() {
+fn every_row_routes_to_the_manifest_its_stamp_names() {
let out = canonicalize_owner();
- let shared = &out.rest()[0].manifest;
-
- /*
- * A tag is manifest-scoped like everything else, so the shared manifest gets its own stamped copy
- * of the two its items carry, the composite `(ManifestId, TagId)` foreign key cannot reach into
- * another namespace. The copies are copies: the base keeps every tag it holds, including the one
- * only shared items use now and the one nothing uses at all. An unused tag is still the user's.
- */
- assert_eq!(ids(rows(shared, "Tags")), vec!["tag-both", "tag-shared-only"]);
- assert_eq!(ids(rows(&out.first().manifest, "Tags")), vec!["tag-both", "tag-shared-only", "tag-unused"]);
- for row in rows(shared, "Tags") {
- assert_eq!(row["ManifestId"], json!("m-f-shared"), "the copy claims the manifest it lives in");
- }
+ let personal = &manifest(&out, PERSONAL).manifest;
+ let shared = &manifest(&out, SHARED).manifest;
- // Field definitions follow the same rule (referenced via FieldValues.FieldDefinitionId).
- assert_eq!(ids(rows(shared, "FieldDefinitions")), vec!["fd-1"]);
- assert_eq!(ids(rows(&out.first().manifest, "FieldDefinitions")), vec!["fd-1", "fd-unused"]);
+ assert_eq!(shared.name.as_deref(), Some(format!("Share {SHARED}").as_str()), "the manifest carries its display name");
+ assert_eq!(shared.manifest_salt, SALT_SHARED);
+ assert_eq!(ids(rows_of(&shared.tables, "Folders")), vec!["f-shared", "f-sub"]);
+ assert_eq!(ids(rows_of(&personal.tables, "Folders")), vec!["f-personal"]);
+ assert_eq!(ids(rows_of(&shared.tables, "Items")), vec!["i-shared", "i-sub"]);
+ assert_eq!(ids(rows_of(&personal.tables, "Items")), vec!["i-nofolder", "i-personal"]);
+
+ // Every table of rows hanging off an item follows it, with zero per-table wiring, and claims the manifest it joined.
+ assert_eq!(values(rows_of(&shared.tables, "FieldValues")), vec!["family", "hunter2"]);
+ assert_eq!(values(rows_of(&personal.tables, "FieldValues")), vec!["me"]);
+ assert_eq!(ids(rows_of(&shared.tables, "TotpCodes")), vec!["totp-shared"]);
+ assert_eq!(ids(rows_of(&shared.tables, "Attachments")), vec!["att-shared"]);
+ assert_eq!(ids(rows_of(&personal.tables, "Attachments")), vec!["att-personal"]);
+ assert_eq!(rows_of(&shared.tables, "ItemTags").len(), 2);
+ assert_eq!(rows_of(&personal.tables, "ItemTags").len(), 1);
+
+ // The blanket invariant: a row inside a manifest names that manifest, in every table and bucket.
+ for canonicalized in &out.manifests {
+ for (name, rows) in &canonicalized.manifest.tables {
+ for row in rows {
+ assert_eq!(row["ManifestId"], json!(canonicalized.manifest.manifest_id), "{name} row {:?} sits in manifest {} but names another", row.get("Id"), canonicalized.manifest.manifest_id);
+ }
+ }
+ }
+ for bucket in &out.data_buckets {
+ assert!(bucket.tables.values().flatten().all(|row| row["ManifestId"] == json!(bucket.manifest_id)), "bucket {} of {} carries a row naming another manifest", bucket.category, bucket.manifest_id);
+ }
+ assert!(validate_manifest(personal).ok && validate_manifest(shared).ok);
}
#[test]
-fn split_clones_a_tag_into_the_manifest_an_item_moved_into() {
- /*
- * An item moves into the shared folder carrying a tag that only ever existed in the user's own
- * manifest. Its ItemTags row follows the item, and the tag it points at has to come with it: the
- * foreign key is `(ManifestId, TagId)`, so the copy sitting in another namespace cannot satisfy it.
- */
- let mut tables = owner_tables_unstamped();
- tables.iter_mut().find(|t| t.name == "Items").unwrap().records.iter_mut()
- .find(|r| r["Id"] == json!("i-personal")).unwrap()
- .insert("FolderId".to_string(), json!("f-shared"));
- let tables = stamp_subtree(tables, "f-shared", "m-f-shared");
-
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- let shared = &out.rest()[0].manifest;
-
- // "tag-both" travelled with the item; the base manifest keeps its own copy regardless.
- assert!(ids(rows(shared, "Tags")).contains(&"tag-both"));
- assert!(ids(rows(&out.first().manifest, "Tags")).contains(&"tag-both"));
- let cloned = rows(shared, "Tags").iter().find(|r| r["Id"] == json!("tag-both")).unwrap();
- assert_eq!(cloned["ManifestId"], json!("m-f-shared"), "the copy claims the manifest it moved into");
- assert_eq!(cloned["Name"], json!("work"), "content copied as-is, id kept");
+fn a_folder_parent_link_only_counts_within_the_folders_own_manifest() {
+ // The shared tree's root was nested under a personal folder: the link leaves with the manifest. The subfolder's
+ // link stays, also when a personal folder carries the id of the subfolder or of its parent.
+ let tables = owner_tables_with("Folders", |folders| {
+ folders.iter_mut().find(|r| r["Id"] == json!("f-shared")).unwrap().insert("ParentFolderId".to_string(), json!("f-personal"));
+ folders.push(stamped(PERSONAL, &[("Id", json!("f-sub")), ("Name", json!("Mine, same id")), ("ParentFolderId", serde_json::Value::Null)]));
+ folders.push(stamped(PERSONAL, &[("Id", json!("f-shared")), ("Name", json!("Mine, parent id")), ("ParentFolderId", serde_json::Value::Null)]));
+ });
+ let out = canonicalize(tables, &[SHARED]);
+ let shared = &manifest(&out, SHARED).manifest;
+ assert_eq!(find(rows_of(&shared.tables, "Folders"), "f-shared")["ParentFolderId"], serde_json::Value::Null, "a parent in another manifest is cut");
+ assert_eq!(find(rows_of(&shared.tables, "Folders"), "f-sub")["ParentFolderId"], json!("f-shared"), "a parent in the same manifest stands");
}
#[test]
-fn split_refuses_rows_that_name_no_manifest() {
- /*
- * Membership is the stamp, so a row that names no manifest is a client that could not say where its
- * data belongs. There is no fallback scope to demote it into: the push is refused, naming the table
- * and row, rather than homing it somewhere plausible and hoping.
- */
- let mut tables = stamp_unstamped(owner_tables(), PERSONAL_M);
- for row in tables.iter_mut().find(|t| t.name == "Items").unwrap().records.iter_mut() {
- row.insert("ManifestId".to_string(), json!(""));
+fn each_manifest_gets_its_own_copy_of_the_tags_and_definitions_its_rows_reference() {
+ // The composite `(ManifestId, Id)` foreign key cannot reach into another namespace, so a referenced row is
+ // copied, never moved: the base keeps every tag, including the unused one, which is still the user's.
+ let out = canonicalize_owner();
+ let shared = &manifest(&out, SHARED).manifest;
+ assert_eq!(ids(rows_of(&shared.tables, "Tags")), vec!["tag-both", "tag-shared-only"]);
+ assert_eq!(ids(rows_of(&manifest(&out, PERSONAL).manifest.tables, "Tags")), vec!["tag-both", "tag-shared-only", "tag-unused"]);
+ assert_eq!(find(rows_of(&shared.tables, "Tags"), "tag-both")["Name"], json!("work"), "content copied as-is, id kept");
+ assert_eq!(ids(rows_of(&shared.tables, "FieldDefinitions")), vec!["fd-1"]);
+ assert_eq!(ids(rows_of(&manifest(&out, PERSONAL).manifest.tables, "FieldDefinitions")), vec!["fd-1", "fd-unused"]);
+
+ // An item moving into the share brings the tag it carries along.
+ let mut tables = owner_tables();
+ move_item(&mut tables, "i-personal", "f-shared", SHARED);
+ let out = canonicalize(tables, &[SHARED]);
+ assert!(ids(rows_of(&manifest(&out, SHARED).manifest.tables, "Tags")).contains(&"tag-both"));
+}
+
+#[test]
+fn canonicalize_refuses_input_it_cannot_route() {
+ // No fallback manifest: a row that names no manifest is refused, naming the table, in a registered table, an
+ // unregistered one, and under the all-zero sentinel the schema stamps when it cannot derive a manifest.
+ let unstamped = [
+ ("Items", owner_tables_with("Items", |items| items.iter_mut().for_each(|r| drop(r.insert("ManifestId".to_string(), json!("")))))),
+ ("Tags", owner_tables_with("Tags", |tags| tags.iter_mut().for_each(|r| drop(r.insert("ManifestId".to_string(), json!("00000000-0000-0000-0000-000000000000")))))),
+ ("Widgets", {
+ let mut t = owner_tables();
+ t.push(table("Widgets", vec![row(&[("Id", json!("w-1"))])]));
+ t
+ }),
+ ];
+ for (offending, tables) in unstamped {
+ let err = canonicalize_from_sqlite(input(tables, &[SHARED])).unwrap_err().to_string();
+ assert!(err.contains(offending) && err.contains("name no manifest"), "{err}");
}
- let err = canonicalize_from_sqlite(raw_input_with_shares(tables, vec![spec("f-shared")])).unwrap_err().to_string();
- assert!(err.contains("Items"), "the error names the offending table: {err}");
- assert!(err.contains("name no manifest"), "the error says what is wrong: {err}");
+ let err = canonicalize_from_sqlite(input(owner_tables(), &[SHARED, SHARED])).unwrap_err().to_string();
+ assert!(err.contains("duplicate"), "{err}");
+ let mut without_id = input(owner_tables(), &[]);
+ without_id.manifests[0].manifest_id = String::new();
+ assert!(canonicalize_from_sqlite(without_id).is_err());
+ assert!(canonicalize_from_sqlite(CanonicalizeInput { manifests: vec![], ..input(vec![], &[]) }).is_err());
}
#[test]
-fn split_reads_the_all_zero_guid_as_naming_no_manifest() {
- /*
- * The schema stamps `00000000-0000-0000-0000-000000000000` into rows whose manifest it could not
- * derive (SQLite will not add a NOT NULL column without a default). Read as a real id it would route
- * to a manifest nobody carries and be dropped as revoked; it has to mean "unstamped" instead.
- */
- let mut tables = stamp_unstamped(owner_tables(), PERSONAL_M);
- for row in tables.iter_mut().find(|t| t.name == "Tags").unwrap().records.iter_mut() {
- row.insert("ManifestId".to_string(), json!("00000000-0000-0000-0000-000000000000"));
+fn the_sqlite_blob_migration_stamps_the_rows_an_old_vault_left_unstamped() {
+ // LEGACY. A vault whose schema predates the `ManifestId` column carries no stamp at all; the migration names
+ // the manifest those rows join, rows that already name one keep it, and nothing is dropped.
+ let mut tables = owner_tables();
+ for table in tables.iter_mut().filter(|t| t.name != "Items" && t.name != "Folders") {
+ table.records.iter_mut().for_each(|r| drop(r.remove("ManifestId")));
}
+ let mut legacy = input(tables, &[SHARED]);
+ legacy.stamp_unstamped_into = Some(PERSONAL.to_string());
- let err = canonicalize_from_sqlite(raw_input_with_shares(tables.clone(), vec![spec("f-shared")])).unwrap_err().to_string();
- assert!(err.contains("Tags"), "the sentinel is refused like any other unstamped row: {err}");
-
- let mut input = raw_input_with_shares(tables, vec![spec("f-shared")]);
- input.stamp_unstamped_into = Some(PERSONAL_M.to_string());
- let out = canonicalize_from_sqlite(input).unwrap();
- assert_eq!(ids(rows(&out.first().manifest, "Tags")), vec!["tag-both", "tag-shared-only", "tag-unused"], "and stamped, never dropped");
-}
-
-#[test]
-fn split_stamps_unstamped_rows_only_when_the_client_asks() {
- // The sqlite-blob conversion is the one caller allowed to hand over unstamped rows, and it has to
- // say so: `stampUnstampedInto` names the manifest they join. Rows that already name one keep it.
- let tables = stamp_subtree(owner_tables_unstamped(), "f-shared", "m-f-shared");
- let mut input = raw_input_with_shares(tables, vec![spec("f-shared")]);
- input.stamp_unstamped_into = Some(PERSONAL_M.to_string());
-
- let out = canonicalize_from_sqlite(input).unwrap();
- let base = &out.first().manifest;
- assert_eq!(ids(rows(base, "Items")), vec!["i-nofolder", "i-personal"], "no row was dropped");
- assert_eq!(ids(rows(base, "Tags")), vec!["tag-both", "tag-shared-only", "tag-unused"]);
- for row in rows(base, "Tags") {
- assert_eq!(row["ManifestId"], json!(PERSONAL_M), "stamped rows carry a real manifest id on the way out");
- }
- assert_eq!(ids(rows(&out.rest()[0].manifest, "Items")), vec!["i-shared", "i-sub"], "explicit stamps still route");
-}
+ let out = canonicalize_from_sqlite(legacy).unwrap();
+ let personal = &manifest(&out, PERSONAL).manifest;
+ assert_eq!(ids(rows_of(&personal.tables, "Tags")), vec!["tag-both", "tag-shared-only", "tag-unused"]);
+ assert!(rows_of(&personal.tables, "Tags").iter().all(|r| r["ManifestId"] == json!(PERSONAL)), "stamped rows carry a real manifest id on the way out");
+ assert_eq!(ids(rows_of(&manifest(&out, SHARED).manifest.tables, "Items")), vec!["i-shared", "i-sub"], "explicit stamps still route");
+ assert_eq!(values(rows_of(&manifest(&out, SHARED).manifest.tables, "FieldValues")), vec!["family", "hunter2"], "unstamped children followed their item");
-#[test]
-fn sqlite_blob_migration_stamps_every_row_of_a_vault_that_has_no_manifest_id_column() {
- /*
- * The sqlite-blob > manifest-v1 migration, end to end: a vault whose schema stops short of the
- * `ManifestId` column (so no row carries the key at all) is canonicalized and materialized straight
- * back out, with no share in sight. Nothing stamps those rows beforehand, the column they would be
- * stamped in does not exist yet, so canonicalize stamping them with the manifest being written from
- * is the conversion. Every row must come out carrying a real manifest id: the materialized schema
- * declares the column NOT NULL, so a single unstamped row fails the whole migration.
- */
- let mut input = raw_input_with_shares(owner_tables_unstamped(), vec![]);
- input.stamp_unstamped_into = Some(PERSONAL_M.to_string());
- let out = canonicalize_from_sqlite(input).unwrap();
- assert_eq!(out.rest().len(), 0, "no shares in this push");
-
- let base = &out.first().manifest;
- assert_eq!(ids(rows(base, "Items")), vec!["i-nofolder", "i-personal", "i-shared", "i-sub"], "no row was dropped");
- for name in manifest_scoped_tables() {
- for record in rows(base, name) {
- assert_eq!(record["ManifestId"], json!(PERSONAL_M), "{name} row {:?} left canonicalize unstamped", record.get("Id"));
- }
+ let re = materialize_as_sqlite(materialize_input(manifests_of(&out), out.data_buckets.clone())).unwrap();
+ for table in re.tables.iter().filter(|t| manifest_scoped_tables().contains(&t.name.as_str())) {
+ assert!(table.records.iter().all(|r| r["ManifestId"].is_string()), "{} would violate the NOT NULL stamp", table.name);
}
- // And the same holds of what actually gets written back into the SQLite database.
- let re = materialize_as_sqlite(materialize_input(base.clone(), vec![], out.data_buckets.clone())).unwrap();
+ // A vault with no stamp on any row and no share yet: every row, nothing dropped, comes out stamped the personal manifest.
+ let mut tables = owner_tables();
+ tables.iter_mut().flat_map(|t| t.records.iter_mut()).for_each(|r| drop(r.remove("ManifestId")));
+ let mut legacy = input(tables, &[]);
+ legacy.stamp_unstamped_into = Some(PERSONAL.to_string());
+ let out = canonicalize_from_sqlite(legacy).unwrap();
+ assert_eq!(ids(rows_of(&out.manifests[0].manifest.tables, "Items")), vec!["i-nofolder", "i-personal", "i-shared", "i-sub"]);
+ let re = materialize_as_sqlite(materialize_input(manifests_of(&out), out.data_buckets.clone())).unwrap();
for table in re.tables.iter().filter(|t| manifest_scoped_tables().contains(&t.name.as_str())) {
- for record in &table.records {
- assert_eq!(record["ManifestId"], json!(PERSONAL_M), "{} row {:?} would violate the NOT NULL stamp", table.name, record.get("Id"));
- }
+ assert!(table.records.iter().all(|r| r["ManifestId"] == json!(PERSONAL)), "{} row left unstamped or stamped elsewhere", table.name);
}
}
#[test]
-fn split_scopes_logos_per_manifest_instead_of_copying_them() {
- // Logos are not reference-copied: each manifest gets its own row per domain, identified by its
- // own scope. The owner's legacy rows carry no scope.
- let out = canonicalize_owner();
- let shared = &out.rest()[0].manifest;
-
- // The shared manifest holds exactly the domains its items use, under folder-scoped ids.
- assert_eq!(logo_sources(shared), vec!["github.com", "netflix.com"]);
- let mut expected_ids = vec![logo_id("m-f-shared", "github.com"), logo_id("m-f-shared", "netflix.com")];
- expected_ids.sort();
- assert_eq!(ids(rows(shared, "Logos")), expected_ids.iter().map(String::as_str).collect::<Vec<&str>>());
- for row in rows(shared, "Logos") {
- assert_eq!(row["ManifestId"], json!("m-f-shared"));
- }
-
- // The personal manifest keeps its own row under a personal-scoped id, but only for a domain it still uses:
- // netflix.com left with the item that referenced it, and a favicon is refetchable.
- assert_eq!(logo_sources(&out.first().manifest), vec!["github.com"]);
- assert_eq!(logo_row(&out.first().manifest, "github.com")["ManifestId"], json!(PERSONAL_M));
-
- // Same domain, two manifests, two distinct rows: nothing converged.
- assert_ne!(logo_row(&out.first().manifest, "github.com")["Id"], logo_row(shared, "github.com")["Id"]);
-
- // Items point at the row in their own manifest.
- let personal = rows(&out.first().manifest, "Items").iter().find(|r| r["Id"] == json!("i-personal")).unwrap();
- assert_eq!(personal["LogoId"], logo_row(&out.first().manifest, "github.com")["Id"]);
- let shared_item = rows(shared, "Items").iter().find(|r| r["Id"] == json!("i-shared")).unwrap();
- assert_eq!(shared_item["LogoId"], logo_row(shared, "github.com")["Id"]);
+fn rows_of_a_manifest_this_push_does_not_declare_are_dropped_not_rehomed() {
+ // The vault still holds rows stamped for the shared manifest, but this push declares only the personal one:
+ // whoever wrote it no longer has that manifest. Those rows have no namespace to go to, and an item left behind
+ // must not take the child rows of a live item that carries the same id down with it.
+ let tables = owner_tables_with("Items", |items| items.push(stamped("m-revoked", &[("Id", json!("i-personal")), ("FolderId", serde_json::Value::Null)])));
+ let out = canonicalize(tables, &[]);
+ assert_eq!(out.manifests.len(), 1);
+ let personal = &out.manifests[0].manifest;
+ assert_eq!(ids(rows_of(&personal.tables, "Items")), vec!["i-nofolder", "i-personal"]);
+ assert_eq!(ids(rows_of(&personal.tables, "Folders")), vec!["f-personal"]);
+ assert_eq!(values(rows_of(&personal.tables, "FieldValues")), vec!["me"], "the dropped items' rows went with them, the live item kept its own");
+ assert!(rows_of(&personal.tables, "TotpCodes").is_empty());
+ assert!(!mentions(personal, "hunter2"), "a dropped item's secrets are not re-homed");
}
#[test]
-fn manifests_drop_logos_no_item_references() {
- // An item leaves the shared manifest: its logo must not linger there (every member downloads it),
- // and the copy it now needs lands in the manifest it moved to.
+fn several_shared_manifests_split_apart_and_an_empty_one_is_still_emitted() {
+ // Two shares at once each get exactly their own subtree. A declared manifest with no rows left (its folder was
+ // deleted) is still emitted, empty buckets included, so the emptying reaches the server.
+ let mut tables = owner_tables();
+ move_item(&mut tables, "i-sub", "f-sub", "m-other");
+ find_mut(&mut tables, "Folders", "f-sub").insert("ManifestId".to_string(), json!("m-other"));
+ let out = canonicalize(tables, &[SHARED, "m-other", "m-empty"]);
+ assert_eq!(out.manifests.len(), 4);
+ assert_eq!(ids(rows_of(&manifest(&out, SHARED).manifest.tables, "Items")), vec!["i-shared"]);
+ assert_eq!(ids(rows_of(&manifest(&out, "m-other").manifest.tables, "Items")), vec!["i-sub"]);
+ assert_eq!(values(rows_of(&manifest(&out, "m-other").manifest.tables, "FieldValues")), vec!["hunter2"]);
+ assert_eq!(ids(rows_of(&manifest(&out, PERSONAL).manifest.tables, "Items")), vec!["i-nofolder", "i-personal"]);
+ assert!(manifest(&out, "m-empty").manifest.tables.values().all(Vec::is_empty));
+ let empty_settings = out.data_buckets.iter().find(|b| b.manifest_id == "m-empty" && b.category == "settings").unwrap();
+ assert!(empty_settings.tables["Settings"].is_empty(), "the emptied table is declared, so the delete reaches the server");
+}
+
+#[test]
+fn unregistered_tables_route_by_their_stamps_like_every_other() {
+ // A newer writer's table carried through the codec overflow: the writing manifest is not special, it gets
+ // exactly its own rows, a shared manifest gets its own, and a row for a manifest this vault lost is dropped.
let mut tables = owner_tables();
- let items = tables.iter_mut().find(|t| t.name == "Items").unwrap();
- items.records.iter_mut().find(|r| r["Id"] == json!("i-sub")).unwrap().insert("FolderId".to_string(), json!("f-personal"));
-
- // Moving an item across a folder boundary moves it across a manifest boundary: the client
- // re-stamps it on the move, so the fixture does too.
- let tables = stamp_subtree(tables, "f-shared", "m-f-shared");
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- assert_eq!(logo_sources(&out.rest()[0].manifest), vec!["github.com"], "netflix logo left with its item");
- assert_eq!(logo_sources(&out.first().manifest), vec!["github.com", "netflix.com"]);
+ tables.push(table("Widgets", vec![stamped(PERSONAL, &[("Id", json!("w-personal"))]), stamped(SHARED, &[("Id", json!("w-shared"))]), stamped("m-revoked", &[("Id", json!("w-gone"))])]));
+ let out = canonicalize(tables, &[SHARED]);
+ assert_eq!(ids(rows_of(&manifest(&out, PERSONAL).manifest.tables, "Widgets")), vec!["w-personal"]);
+ assert_eq!(ids(rows_of(&manifest(&out, SHARED).manifest.tables, "Widgets")), vec!["w-shared"]);
}
#[test]
-fn orphan_favicons_are_pruned_everywhere_and_uploads_survive_everywhere() {
- /*
- * A favicon nothing references is refetchable, so it goes, in every manifest alike. An image the
- * user uploaded is not reproducible, so it stays, also in every manifest alike: each one carries
- * its own library, and which of them a client offers to pick from is a runtime choice, not
- * something the format decides by anointing one manifest.
- */
+fn overflow_columns_regraft_onto_a_row_that_changed_manifest() {
+ // A newer writer added a column to an item; it rode in the overflow under the identity the row had before the
+ // item moved into the share. The re-graft follows the row by its primary key.
+ let overflow = CodecOverflow { columns: [("Items".to_string(), [(format!("{}\u{1f}i-shared", PERSONAL), row(&[("FutureCol", json!("keep-me"))]))].into_iter().collect())].into_iter().collect(), ..Default::default() };
let mut tables = owner_tables();
- let logos = tables.iter_mut().find(|t| t.name == "Logos").unwrap();
- logos.records.push(row(&[
- ("Id", json!("upload-personal")), ("Kind", json!("custom")), ("Source", json!("aa11")),
- ("ManifestId", json!(PERSONAL_M)), ("FileData", json!({ "__b64": b64(&[0x55]) })),
- ]));
- logos.records.push(row(&[
- ("Id", json!("upload-shared")), ("Kind", json!("custom")), ("Source", json!("bb22")),
- ("ManifestId", json!("m-f-shared")), ("FileData", json!({ "__b64": b64(&[0x66]) })),
- ]));
-
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
-
- // netflix.com left with the item that used it; each manifest kept the upload stamped for it.
- assert_eq!(logo_sources(&out.first().manifest), vec!["aa11", "github.com"]);
- assert_eq!(logo_sources(&out.rest()[0].manifest), vec!["bb22", "github.com", "netflix.com"]);
+ tables.push(table(OVERFLOW_TABLE, overflow.to_table_records()));
+ let out = canonicalize(tables, &[SHARED]);
+ assert_eq!(find(rows_of(&manifest(&out, SHARED).manifest.tables, "Items"), "i-shared")["FutureCol"], json!("keep-me"));
+ assert!(rows_of(&manifest(&out, PERSONAL).manifest.tables, "Items").iter().all(|r| !r.contains_key("FutureCol")));
}
-#[test]
-fn logo_ids_are_derived_from_scope_and_source() {
- // The identity contract every platform reproduces: same (scope, domain) -> same row, and the
- // derivation is case-insensitive on the domain.
- assert_eq!(logo_id(PERSONAL_M, "github.com"), logo_id(PERSONAL_M, "GitHub.com"));
- assert_ne!(logo_id(PERSONAL_M, "github.com"), logo_id("m-f-shared", "github.com"));
- assert_ne!(logo_id("m-f-a", "github.com"), logo_id("m-f-b", "github.com"));
- // UUID-shaped so it round-trips through Guid/UUID types on every platform.
- let id = logo_id(PERSONAL_M, "github.com");
- assert_eq!(id.len(), 36);
- assert_eq!(id.chars().filter(|c| *c == '-').count(), 4);
-}
+/*
+ * Canonicalize: logos and blobs.
+ */
#[test]
-fn split_hashes_blobs_with_per_manifest_salts() {
+fn logos_are_scoped_per_manifest_and_hashed_with_its_salt() {
let out = canonicalize_owner();
- let shared = &out.rest()[0];
-
- // Personal blob map: personal attachment + the one logo a personal item still references.
- let personal_hashes: Vec<&String> = out.first().blobs.keys().collect();
- assert_eq!(personal_hashes.len(), 2, "personal: att-personal + github logo, got {:?}", out.first().blobs.values().map(|b| &b.kind).collect::<Vec<_>>());
-
- // Shared blob map: shared attachment + both logo copies, hashed with the SHARED salt.
- assert_eq!(shared.blobs.len(), 3);
+ let personal = manifest(&out, PERSONAL);
+ let shared = manifest(&out, SHARED);
+
+ // Each manifest holds exactly the domains its items use, under ids derived from its own id; the same domain in
+ // two manifests is two rows, and items point at the row in their own manifest.
+ assert_eq!(logo_sources(&shared.manifest), vec!["github.com", "netflix.com"]);
+ assert_eq!(logo_sources(&personal.manifest), vec!["github.com"], "netflix.com left with the item that referenced it, and a favicon is refetchable");
+ assert_ne!(logo_row(&personal.manifest, "github.com")["Id"], logo_row(&shared.manifest, "github.com")["Id"]);
+ assert_eq!(find(rows_of(&personal.manifest.tables, "Items"), "i-personal")["LogoId"], logo_row(&personal.manifest, "github.com")["Id"]);
+ assert_eq!(find(rows_of(&shared.manifest.tables, "Items"), "i-shared")["LogoId"], logo_row(&shared.manifest, "github.com")["Id"]);
+
+ // Blob maps: every manifest hashes its blobs with its own salt, and the refs inside it point at its own map.
+ assert_eq!(personal.blobs.len(), 2, "att-personal + github logo");
+ assert_eq!(shared.blobs.len(), 3, "att-shared + both logos");
let logo_bytes = [0xAAu8, 0xBB];
- let expected_personal_hash = hash::salted_blob_hash(&logo_bytes, SALT_PERSONAL).unwrap();
- let expected_shared_hash = hash::salted_blob_hash(&logo_bytes, SALT_SHARED).unwrap();
- assert!(out.first().blobs.contains_key(&expected_personal_hash), "github favicon hashed with the personal salt in personal manifest");
- assert!(shared.blobs.contains_key(&expected_shared_hash), "github favicon hashed with shared salt in shared manifest");
- assert_ne!(expected_personal_hash, expected_shared_hash);
-
- // The blob refs inside each manifest point at their own map's hashes.
- assert_eq!(logo_row(&out.first().manifest, "github.com")["FileData"]["__blobRef"], json!(expected_personal_hash));
- assert_eq!(logo_row(&shared.manifest, "github.com")["FileData"]["__blobRef"], json!(expected_shared_hash));
+ let (personal_hash, shared_hash) = (hash::salted_blob_hash(&logo_bytes, SALT_PERSONAL).unwrap(), hash::salted_blob_hash(&logo_bytes, SALT_SHARED).unwrap());
+ assert_ne!(personal_hash, shared_hash);
+ assert_eq!(logo_row(&personal.manifest, "github.com")["FileData"]["__blobRef"], json!(personal_hash));
+ assert_eq!(logo_row(&shared.manifest, "github.com")["FileData"]["__blobRef"], json!(shared_hash));
+ assert!(personal.blobs.contains_key(&personal_hash) && shared.blobs.contains_key(&shared_hash));
}
#[test]
-fn split_keeps_bucketed_and_foreign_key_tables_out_of_shared_manifests() {
- let out = canonicalize_owner();
- let shared = &out.rest()[0].manifest;
- assert!(!shared.tables.contains_key("EncryptionKeys"), "personal (personal-stamped) key material must never enter a shared manifest");
- assert!(!shared.tables.contains_key("Settings"), "a bucketed table belongs in its manifest's data bucket, never in a manifest");
- assert!(!out.first().manifest.tables.contains_key("Settings"), "…and that holds for the personal manifest too");
- // The personal keypair set rides inside the personal manifest itself; Settings rides in its bucket.
- assert_eq!(ids(rows(&out.first().manifest, "EncryptionKeys")), vec!["ek-1"], "personal keys live in the personal manifest");
- assert!(!out.data_buckets.iter().any(|b| b.category == "EncryptionKeys"), "EncryptionKeys is no longer a bucket category");
-
- /*
- * Every manifest gets its own Settings bucket, holding exactly its own rows, the shared one empty
- * here, since the single Settings row belongs to the personal manifest. Leaving a manifest's bucket
- * out would read as "unchanged" rather than "emptied", so its last row could never be deleted.
- */
- let settings: Vec<&DataBucket> = out.data_buckets.iter().filter(|b| b.category == "settings").collect();
- assert_eq!(settings.iter().map(|b| b.manifest_id.as_str()).collect::<Vec<&str>>(), vec!["m-f-shared", PERSONAL_M]);
- assert_eq!(settings[0].tables["Settings"].len(), 0, "the shared manifest holds no settings of its own, and says so");
- assert_eq!(settings[1].tables["Settings"].len(), 1);
+fn orphan_favicons_are_pruned_and_uploads_kept_in_every_manifest_alike() {
+ // An item left the share: its favicon must not linger there (every member downloads it) and lands where the item
+ // went. An image the user uploaded is not reproducible, so it stays, in every manifest alike.
+ let mut tables = owner_tables_with("Logos", |logos| {
+ logos.push(stamped(PERSONAL, &[("Id", json!("upload-personal")), ("Kind", json!("custom")), ("Source", json!("aa11")), ("FileData", json!({ "__b64": b64(&[0x55]) }))]));
+ logos.push(stamped(SHARED, &[("Id", json!("upload-shared")), ("Kind", json!("custom")), ("Source", json!("bb22")), ("FileData", json!({ "__b64": b64(&[0x66]) }))]));
+ });
+ move_item(&mut tables, "i-sub", "f-personal", PERSONAL);
+ let out = canonicalize(tables, &[SHARED]);
+ assert_eq!(logo_sources(&manifest(&out, SHARED).manifest), vec!["bb22", "github.com"], "netflix logo left with its item, the upload stayed");
+ assert_eq!(logo_sources(&manifest(&out, PERSONAL).manifest), vec!["aa11", "github.com", "netflix.com"]);
}
#[test]
-fn split_routes_bucket_rows_to_the_manifest_that_owns_them() {
- /*
- * The whole point of addressing a bucket by manifest: a settings row stamped for a shared manifest
- * lands in THAT manifest's bucket, not the writer's. Nothing in the vault does this yet (only the
- * personal manifest carries settings today), but the routing is the row's stamp, not a hardcoded
- * assumption about which manifest may own a bucket.
- */
- let mut tables = owner_tables();
- let settings = tables.iter_mut().find(|t| t.name == "Settings").expect("settings fixture");
- settings.records.push(row(&[("Key", json!("sort")), ("Value", json!("name")), ("ManifestId", json!("m-f-shared"))]));
- settings.records.push(row(&[("Key", json!("stale")), ("Value", json!("x")), ("ManifestId", json!("m-gone"))]));
-
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
-
- let bucket_of = |manifest_id: &str| -> Option<&DataBucket> {
- out.data_buckets.iter().find(|b| b.category == "settings" && b.manifest_id == manifest_id)
- };
- let keys = |b: &DataBucket| -> Vec<String> {
- let mut out: Vec<String> = b.tables["Settings"].iter().filter_map(|r| r.get("Key").and_then(|v| v.as_str()).map(str::to_string)).collect();
- out.sort();
- out
- };
-
- assert_eq!(keys(bucket_of(PERSONAL_M).expect("personal bucket")), vec!["theme"]);
- assert_eq!(keys(bucket_of("m-f-shared").expect("shared bucket")), vec!["sort"]);
- assert!(bucket_of("m-gone").is_none(), "a row stamped for a manifest this vault no longer carries is dropped, like every other row");
-
- // Each bucket is self-consistent: its rows claim its own manifest, so validation passes.
- for bucket in out.data_buckets.iter() {
- let result = validate_data_bucket(bucket);
- assert!(result.ok, "bucket {} invalid: {:?}", bucket.manifest_id, result.failed_rules);
+fn a_logo_follows_its_item_into_a_share_reusing_the_rows_the_share_has() {
+ // i-personal (github.com, like i-shared), i-nofolder (now pointing at an upload) and a built-in item move into
+ // the share: the share gets one github row both items point at, and a per-manifest copy of every other kind.
+ let content_hash = "a".repeat(64);
+ let mut tables = owner_tables_with("Logos", |logos| {
+ logos.push(stamped(PERSONAL, &[("Id", json!("personal-custom")), ("Kind", json!("custom")), ("Source", json!(content_hash.clone())), ("FileData", json!({ "__b64": b64(&[0xaa, 0xbb, 0xcc]) }))]));
+ logos.push(stamped(PERSONAL, &[("Id", json!("builtin-row")), ("Kind", json!("builtin")), ("Source", json!("shopping")), ("FileData", serde_json::Value::Null)]));
+ });
+ tables.iter_mut().find(|t| t.name == "Items").unwrap().records.push(stamped(PERSONAL, &[("Id", json!("i-builtin")), ("FolderId", serde_json::Value::Null), ("LogoId", json!("builtin-row"))]));
+ find_mut(&mut tables, "Items", "i-nofolder").insert("LogoId".to_string(), json!("personal-custom"));
+ for item in ["i-personal", "i-nofolder", "i-builtin"] {
+ move_item(&mut tables, item, "f-shared", SHARED);
}
-}
-
-#[test]
-fn combine_stamps_bucket_rows_with_the_manifest_that_delivered_them() {
- /*
- * The mirror rule of "the shipping manifest is the membership": a bucket arrives under one manifest,
- * so its rows claim that manifest. A bucket whose rows name someone else cannot move them there.
- */
- let out = canonicalize_owner();
- let mut evil = out.data_buckets.iter().find(|b| b.category == "settings" && b.manifest_id == PERSONAL_M).expect("personal settings bucket").clone();
- evil.manifest_id = "m-f-shared".to_string();
-
- let re = materialize_as_sqlite(materialize_input(out.first().manifest.clone(), out.rest().iter().map(|s| s.manifest.clone()).collect(), vec![evil])).unwrap();
- let map = materialized_map(&re);
- assert_eq!(map["Settings"].len(), 1);
- assert_eq!(map["Settings"][0]["ManifestId"], json!("m-f-shared"), "the delivering manifest decides the scope, not the row's own claim");
-}
-#[test]
-fn combine_keeps_two_manifests_settings_side_by_side() {
- // Two manifests holding the same setting key are two rows: (ManifestId, Key) keeps them apart.
- let out = canonicalize_owner();
- let personal_bucket = out.data_buckets.iter().find(|b| b.category == "settings" && b.manifest_id == PERSONAL_M).expect("personal settings bucket").clone();
- let mut shared_bucket = personal_bucket.clone();
- shared_bucket.manifest_id = "m-f-shared".to_string();
-
- let re = materialize_as_sqlite(materialize_input(
- out.first().manifest.clone(),
- out.rest().iter().map(|s| s.manifest.clone()).collect(),
- vec![personal_bucket, shared_bucket],
- ))
- .unwrap();
- let map = materialized_map(&re);
- let mut scopes: Vec<&str> = map["Settings"].iter().filter_map(|r| r["ManifestId"].as_str()).collect();
- scopes.sort();
- assert_eq!(scopes, vec!["m-f-shared", PERSONAL_M], "one 'theme' row per manifest, neither shadowing the other");
- assert!(map["Settings"].iter().all(|r| r["Key"] == json!("theme")));
-}
+ let out = canonicalize(tables, &[SHARED]);
+ let shared = manifest(&out, SHARED);
+ assert_eq!(logo_sources(&shared.manifest), vec![content_hash.as_str(), "github.com", "netflix.com", "shopping"], "no duplicate github row");
+ let github = logo_row(&shared.manifest, "github.com");
+ assert_eq!(find(rows_of(&shared.manifest.tables, "Items"), "i-personal")["LogoId"], github["Id"]);
+ assert_eq!(find(rows_of(&shared.manifest.tables, "Items"), "i-shared")["LogoId"], github["Id"]);
+ assert!(github["FileData"]["__blobRef"].is_string(), "and it carries real bytes");
+
+ let custom = find(rows_of(&shared.manifest.tables, "Logos"), &logo_id_for(SHARED, "custom", &content_hash));
+ assert_eq!((custom["Kind"].as_str(), custom["ManifestId"].as_str()), (Some("custom"), Some(SHARED)));
+ assert!(custom["FileData"]["__blobRef"].is_string(), "the upload travels with its bytes, so members can render it");
+ assert_eq!(find(rows_of(&shared.manifest.tables, "Items"), "i-nofolder")["LogoId"], custom["Id"]);
+ let builtin = find(rows_of(&shared.manifest.tables, "Logos"), &logo_id_for(SHARED, "builtin", "shopping"));
+ assert_eq!(builtin["Source"], json!("shopping"), "a bytesless built-in logo still belongs in the manifest; the catalog key identifies it");
+ assert_eq!(find(rows_of(&shared.manifest.tables, "Items"), "i-builtin")["LogoId"], builtin["Id"]);
+}
+
+#[test]
+fn logo_ids_derive_from_scope_kind_and_source() {
+ // Same (manifest id, kind, domain) -> same row on every platform; the domain is case-insensitive; kinds key apart so a
+ // domain and a catalog key can never collide; a row without a Kind is a favicon and keeps its legacy derivation.
+ assert_eq!(logo_id(PERSONAL, "github.com"), logo_id(&PERSONAL.to_uppercase(), "GitHub.com"));
+ assert_ne!(logo_id(PERSONAL, "github.com"), logo_id(SHARED, "github.com"));
+ assert_ne!(logo_id_for(PERSONAL, "favicon", "shopping"), logo_id_for(PERSONAL, "builtin", "shopping"));
+ assert_eq!(logo_id(PERSONAL, "github.com").len(), 36);
+ // Known-answer vectors: every platform must derive the same logo ids, so these values must never change.
+ assert_eq!(logo_id_for("11111111-aaaa-4111-8111-111111111111", "favicon", "github.com"), "500424ce-570f-8807-a476-a64ccf8084e7");
+ assert_eq!(logo_id_for("11111111-aaaa-4111-8111-111111111111", "builtin", "shopping"), "19f3ef9c-5a8b-87e7-bbea-08b60c2dfd66");
+
+ let out = canonicalize(
+ vec![
+ table("Items", vec![stamped(PERSONAL, &[("Id", json!("i-fav")), ("LogoId", json!("legacy-fav"))]), stamped(PERSONAL, &[("Id", json!("i-builtin")), ("LogoId", json!("legacy-builtin"))])]),
+ table("Logos", vec![
+ stamped(PERSONAL, &[("Id", json!("legacy-fav")), ("Source", json!("shopping")), ("FileData", json!({ "__b64": b64(&[1, 2]) }))]),
+ stamped(PERSONAL, &[("Id", json!("legacy-builtin")), ("Kind", json!("builtin")), ("Source", json!("shopping")), ("FileData", serde_json::Value::Null)]),
+ ]),
+ ],
+ &[],
+ );
+ let logos = rows_of(&out.manifests[0].manifest.tables, "Logos");
+ assert_eq!(logos.len(), 2, "both kinds survive: {logos:?}");
+ let kindless = find(logos, &logo_id(PERSONAL, "shopping"));
+ assert_eq!(kindless["Kind"], json!("favicon"), "a kindless row derives as a favicon and is stamped with the kind it had");
+ assert_eq!(find(rows_of(&out.manifests[0].manifest.tables, "Items"), "i-builtin")["LogoId"], json!(logo_id_for(PERSONAL, "builtin", "shopping")));
+}
+
+/*
+ * Buckets.
+ */
+
+#[test]
+fn every_manifest_gets_its_own_buckets_and_rows_route_into_them_by_stamp() {
+ // A bucketed table never enters a manifest; every declared manifest gets a bucket per category, empty or not
+ // (leaving one out would read as "unchanged" rather than "emptied"), and a row names the bucket it lands in.
+ let tables = owner_tables_with("Settings", |settings| {
+ settings.push(stamped(SHARED, &[("Key", json!("sort")), ("Value", json!("name"))]));
+ settings.push(stamped("m-gone", &[("Key", json!("stale")), ("Value", json!("x"))]));
+ });
+ let out = canonicalize(tables, &[SHARED]);
+ assert!(out.manifests.iter().all(|m| !m.manifest.tables.contains_key("Settings")));
+ let keys = |manifest_id: &str| -> Vec<String> {
+ let bucket = out.data_buckets.iter().find(|b| b.category == "settings" && b.manifest_id == manifest_id).unwrap_or_else(|| panic!("no settings bucket for {manifest_id}"));
+ assert!(validate_data_bucket(bucket).ok);
+ let mut keys: Vec<String> = bucket.tables["Settings"].iter().map(|r| r["Key"].as_str().unwrap().to_string()).collect();
+ keys.sort();
+ keys
+ };
+ assert_eq!(keys(PERSONAL), vec!["theme"]);
+ assert_eq!(keys(SHARED), vec!["sort"]);
+ assert!(!out.data_buckets.iter().any(|b| b.manifest_id == "m-gone"), "a row stamped for a manifest this vault no longer carries is dropped");
+ assert_eq!(out.data_buckets.iter().filter(|b| b.manifest_id == SHARED && b.category == "stats").count(), 1, "an empty bucket is still declared");
-#[test]
-fn extract_buckets_splits_a_category_the_way_canonicalize_does() {
- /*
- * The bucket-only push path routes rows by the manifest each one names, exactly like the full push:
- * the caller hands over the whole category and gets one bucket per manifest it can write. Every
- * manifest asked for gets a bucket even when it holds nothing (otherwise deleting its last row
- * would never reach the server), a row naming a manifest this vault does not carry is dropped with
- * that manifest, and a row that names no manifest has no scope the codec may invent, it refuses.
- */
+ // The bucket-only push path splits a category the same way.
let settings = |rows: Vec<CodecRecord>| -> HashMap<String, Vec<CodecRecord>> { [("Settings".to_string(), rows)].into_iter().collect() };
-
let buckets = extract_buckets(
"settings".to_string(),
- vec![PERSONAL_M.to_string(), "m-f-shared".to_string()],
+ vec![PERSONAL.to_string(), SHARED.to_string()],
settings(vec![
- row(&[("Key", json!("sort")), ("Value", json!("name")), ("ManifestId", json!("M-F-SHARED"))]),
- row(&[("Key", json!("theme")), ("Value", json!("dark")), ("ManifestId", json!(PERSONAL_M))]),
- row(&[("Key", json!("locale")), ("Value", json!("nl")), ("ManifestId", json!("m-gone"))]),
+ stamped(&SHARED.to_uppercase(), &[("Key", json!("sort")), ("Value", json!("name"))]),
+ stamped(PERSONAL, &[("Key", json!("theme")), ("Value", json!("dark"))]),
+ stamped("m-gone", &[("Key", json!("locale")), ("Value", json!("nl"))]),
]),
)
.unwrap();
-
- let ids: Vec<&str> = buckets.iter().map(|b| b.manifest_id.as_str()).collect();
- assert_eq!(ids, vec!["m-f-shared", PERSONAL_M], "one bucket per manifest asked for, in a stable order");
- assert!(buckets.iter().all(|b| validate_data_bucket(b).ok));
-
- let shared = buckets.iter().find(|b| b.manifest_id == "m-f-shared").unwrap();
- assert_eq!(shared.tables["Settings"].len(), 1, "the shared manifest gets exactly its own row");
- assert_eq!(shared.tables["Settings"][0]["Key"], json!("sort"));
- assert_eq!(shared.tables["Settings"][0]["ManifestId"], json!("m-f-shared"), "a casing difference is normalized to the declared id");
-
- let personal = buckets.iter().find(|b| b.manifest_id == PERSONAL_M).unwrap();
- assert_eq!(personal.tables["Settings"].len(), 1, "the row stamped for a manifest this vault no longer carries is dropped, not re-homed");
- assert_eq!(personal.tables["Settings"][0]["Key"], json!("theme"));
-
- // A manifest holding nothing still gets its bucket, with the category's tables declared empty.
- let empty = extract_buckets("settings".to_string(), vec![PERSONAL_M.to_string()], settings(vec![])).unwrap();
- assert_eq!(empty.len(), 1);
- assert_eq!(empty[0].tables["Settings"].len(), 0, "an emptied table is declared, so the delete reaches the server");
-
- let unstamped = extract_buckets("settings".to_string(), vec![PERSONAL_M.to_string()], settings(vec![row(&[("Key", json!("k")), ("Value", json!("v"))])]))
- .unwrap_err()
- .to_string();
- assert!(unstamped.contains("name no manifest"), "the error says what is wrong: {unstamped}");
-}
-
-#[test]
-fn split_routes_unregistered_tables_by_their_own_stamps() {
- /*
- * A table this build's registry does not know, a newer writer's table carried through the codec
- * overflow, routes by its rows' own stamps like every registered table. The writing (personal)
- * manifest is not special: it gets exactly its own rows, a shared manifest gets exactly its own,
- * and a row stamped for a manifest this vault no longer carries is dropped with it.
- */
- let mut tables = owner_tables();
- tables.push(table("Widgets", vec![
- row(&[("Id", json!("w-personal")), ("ManifestId", json!(PERSONAL_M))]),
- row(&[("Id", json!("w-shared")), ("ManifestId", json!("m-f-shared"))]),
- row(&[("Id", json!("w-gone")), ("ManifestId", json!("m-revoked"))]),
- ]));
-
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- assert_eq!(ids(rows(&out.first().manifest, "Widgets")), vec!["w-personal"], "the writing manifest keeps only rows stamped for it");
- assert_eq!(ids(rows(&out.rest()[0].manifest, "Widgets")), vec!["w-shared"], "a shared manifest's rows come home to it");
-}
-
-#[test]
-fn split_refuses_unregistered_rows_that_name_no_manifest() {
- // The no-fallback-scope rule holds for tables the registry does not know too: an unstamped row of
- // one is refused, never demoted into the writing manifest.
- let mut tables = owner_tables();
- tables.push(table("Widgets", vec![row(&[("Id", json!("w-1"))])]));
-
- let err = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap_err().to_string();
- assert!(err.contains("Widgets"), "the error names the offending table: {err}");
- assert!(err.contains("name no manifest"), "the error says what is wrong: {err}");
-}
-
-#[test]
-fn validate_rejects_an_unaddressed_or_cross_stamped_bucket() {
- let out = canonicalize_owner();
- let bucket = out.data_buckets.iter().find(|b| b.category == "settings" && b.manifest_id == PERSONAL_M).expect("personal settings bucket");
-
- let mut unaddressed = bucket.clone();
- unaddressed.manifest_id = String::new();
- assert!(validate_data_bucket(&unaddressed).failed_rules.iter().any(|r| r == "dataBucket-manifestId-missing"));
-
- let mut cross_stamped = bucket.clone();
- cross_stamped.manifest_id = "m-f-shared".to_string();
- assert!(validate_data_bucket(&cross_stamped).failed_rules.iter().any(|r| r == "dataBucket-scope-mismatch"));
-}
-
-#[test]
-fn split_supports_multiple_disjoint_shared_manifests() {
- let tables = vec![
- table("Folders", vec![
- row(&[("Id", json!("f-a")), ("ParentFolderId", serde_json::Value::Null)]),
- row(&[("Id", json!("f-b")), ("ParentFolderId", serde_json::Value::Null)]),
- ]),
- table("Items", vec![
- row(&[("Id", json!("i-a")), ("FolderId", json!("f-a"))]),
- row(&[("Id", json!("i-b")), ("FolderId", json!("f-b"))]),
- ]),
- ];
- let mut spec_b = spec("f-b");
- spec_b.manifest_salt = SALT_PERSONAL.to_string();
- let tables = stamp_subtree(stamp_subtree(tables, "f-a", "m-f-a"), "f-b", "m-f-b");
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-a"), spec_b])).unwrap();
- assert_eq!(out.rest().len(), 2);
- assert_eq!(out.rest()[0].manifest.manifest_id, "m-f-a");
- assert_eq!(out.rest()[1].manifest.manifest_id, "m-f-b");
- assert_eq!(ids(rows(&out.rest()[0].manifest, "Items")), vec!["i-a"]);
- assert_eq!(ids(rows(&out.rest()[1].manifest, "Items")), vec!["i-b"]);
- assert!(rows(&out.first().manifest, "Items").is_empty());
-}
-
-#[test]
-fn nested_manifests_each_keep_their_own_subtree() {
- // Nesting is no longer a special case: a folder inside another manifest's folder simply carries a
- // different stamp, and each manifest gets exactly the rows stamped for it. The inner manifest's
- // folder loses its ParentFolderId, since that parent lives in a namespace it cannot see.
- let tables = vec![table("Folders", vec![
- row(&[("Id", json!("f-outer")), ("ParentFolderId", serde_json::Value::Null)]),
- row(&[("Id", json!("f-inner")), ("ParentFolderId", json!("f-outer"))]),
- ])];
- let tables = stamp_subtree(stamp_subtree(tables, "f-outer", "m-f-outer"), "f-inner", "m-f-inner");
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-outer"), spec("f-inner")])).unwrap();
- assert_eq!(ids(rows(&out.rest()[0].manifest, "Folders")), vec!["f-outer"]);
- assert_eq!(ids(rows(&out.rest()[1].manifest, "Folders")), vec!["f-inner"]);
- assert_eq!(rows(&out.rest()[1].manifest, "Folders")[0]["ParentFolderId"], serde_json::Value::Null);
-}
-
-#[test]
-fn split_rejects_duplicate_shared_manifest_specs() {
- let err = canonicalize_from_sqlite(input_with_shares(owner_tables(), vec![spec("f-shared"), spec("f-shared")])).unwrap_err();
- assert!(err.to_string().contains("duplicate"), "unexpected error: {err}");
-}
-
-#[test]
-fn split_of_deleted_folder_yields_empty_shared_manifest() {
- // The anchor folder was deleted locally: its manifest empties out (and would be pushed as such).
- let tables = vec![
- table("Folders", vec![row(&[("Id", json!("f-personal")), ("ParentFolderId", serde_json::Value::Null)])]),
- table("Items", vec![row(&[("Id", json!("i-1")), ("FolderId", json!("f-personal"))])]),
- ];
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-gone")])).unwrap();
- assert_eq!(out.rest().len(), 1);
- assert!(rows(&out.rest()[0].manifest, "Folders").is_empty());
- assert!(rows(&out.rest()[0].manifest, "Items").is_empty());
- assert_eq!(ids(rows(&out.first().manifest, "Items")), vec!["i-1"]);
+ assert_eq!(buckets.iter().map(|b| b.manifest_id.as_str()).collect::<Vec<_>>(), vec![PERSONAL, SHARED], "one bucket per manifest asked for, in a stable order");
+ let shared = buckets.iter().find(|b| b.manifest_id == SHARED).unwrap();
+ assert_eq!(shared.tables["Settings"][0]["ManifestId"], json!(SHARED), "a casing difference is normalized to the declared id");
+ assert_eq!(buckets[0].tables["Settings"].len(), 1, "the row for a lost manifest is dropped, not re-homed");
+ let empty = extract_buckets("settings".to_string(), vec![PERSONAL.to_string()], settings(vec![])).unwrap();
+ assert!(empty.len() == 1 && empty[0].tables["Settings"].is_empty(), "a manifest holding nothing still declares the emptied table");
+ let unstamped = extract_buckets("settings".to_string(), vec![PERSONAL.to_string()], settings(vec![row(&[("Key", json!("k"))])])).unwrap_err().to_string();
+ assert!(unstamped.contains("name no manifest"), "{unstamped}");
}
#[test]
-fn split_includes_tombstoned_rows_in_subtree() {
- // Deleted (tombstoned) subfolders/items still sync through the shared manifest so other members
- // learn about the deletion via LWW merge.
- let tables = vec![
- table("Folders", vec![
- row(&[("Id", json!("f-shared")), ("ParentFolderId", serde_json::Value::Null), ("IsDeleted", json!(0))]),
- row(&[("Id", json!("f-sub")), ("ParentFolderId", json!("f-shared")), ("IsDeleted", json!(1))]),
- ]),
- table("Items", vec![row(&[("Id", json!("i-del")), ("FolderId", json!("f-sub")), ("IsDeleted", json!(1))])]),
- ];
- let out = canonicalize_from_sqlite(input_with_shares(stamp_subtree(tables, "f-shared", "m-f-shared"), vec![spec("f-shared")])).unwrap();
- let shared = &out.rest()[0].manifest;
- assert_eq!(ids(rows(shared, "Folders")), vec!["f-shared", "f-sub"]);
- assert_eq!(ids(rows(shared, "Items")), vec!["i-del"]);
-}
-
-#[test]
-fn split_with_a_single_spec_drops_rows_of_manifests_it_does_not_carry() {
- /*
- * The vault still holds rows stamped for "m-f-shared", but this push declares only one manifest:
- * whoever wrote it no longer has that one. Those rows have no namespace to go to and are left out
- * rather than re-homed into the manifest that happens to be first.
- */
- let out = canonicalize_from_sqlite(input_with_shares(owner_tables(), vec![])).unwrap();
- assert!(out.rest().is_empty());
- assert_eq!(ids(rows(&out.first().manifest, "Items")), vec!["i-nofolder", "i-personal"]);
- assert_eq!(ids(rows(&out.first().manifest, "Folders")), vec!["f-personal"]);
- // The rows hanging off a dropped item go with it rather than dangling.
- assert_eq!(values(rows(&out.first().manifest, "FieldValues")), vec!["me"]);
- assert!(rows(&out.first().manifest, "TotpCodes").is_empty());
-
- // The serialized output carries the one manifest, with nothing flagging it and no folder anchor.
- let json = serde_json::to_value(&out).unwrap();
- assert_eq!(json["manifests"].as_array().unwrap().len(), 1);
- assert!(json["manifests"][0].get("isPersonal").is_none());
- assert!(json["manifests"][0]["manifest"].get("anchorFolderId").is_none());
- assert_eq!(json["manifests"][0]["manifest"]["manifestId"], json!(PERSONAL_M));
-}
-
-#[test]
-fn shared_manifests_validate_clean_after_split() {
+fn a_bucket_claims_the_manifest_that_delivered_it() {
+ // The mirror rule of "the shipping manifest is the membership": a bucket arrives under one manifest, so its rows
+ // claim that manifest, and two manifests holding the same setting key are two rows.
let out = canonicalize_owner();
- let personal_result = validate_manifest(&out.first().manifest);
- assert!(personal_result.ok, "personal manifest invalid: {:?}", personal_result.failed_rules);
- let shared_result = validate_manifest(&out.rest()[0].manifest);
- assert!(shared_result.ok, "shared manifest invalid: {:?}", shared_result.failed_rules);
-}
+ let personal_bucket = out.data_buckets.iter().find(|b| b.category == "settings" && b.manifest_id == PERSONAL).unwrap().clone();
+ let mut relabelled = personal_bucket.clone();
+ relabelled.manifest_id = SHARED.to_string();
-#[test]
-fn validate_rejects_shared_manifest_carrying_out_of_manifest_tables() {
- // A NULL-scope (personal) key row in a shared manifest trips the key-scope rule…
- let mut manifest = canonicalize_owner().rest()[0].manifest.clone();
- manifest.tables.insert("EncryptionKeys".to_string(), vec![row(&[("Id", json!("ek-evil")), ("PrivateKey", json!("x"))])]);
- let result = validate_manifest(&manifest);
- assert!(!result.ok);
- assert!(result.failed_rules.iter().any(|r| r == "encryption-keys-scope-mismatch"));
-
- /*
- * …and a bucketed table (Settings) trips the bucketed-table rule, not the personal-table one: it is
- * rejected because it belongs in the manifest's own data bucket, not because settings could never
- * belong to a shared manifest at all.
- */
- let mut manifest = canonicalize_owner().rest()[0].manifest.clone();
- manifest.tables.insert("Settings".to_string(), vec![row(&[("Key", json!("theme")), ("Value", json!("evil"))])]);
- let result = validate_manifest(&manifest);
- assert!(!result.ok);
- assert!(result.failed_rules.iter().any(|r| r == "manifest-carries-bucketed-table"));
- assert!(!result.failed_rules.iter().any(|r| r == "manifest-carries-personal-table"));
-}
+ let re = materialize_as_sqlite(materialize_input(manifests_of(&out), vec![personal_bucket, relabelled])).unwrap();
+ let map = materialized_map(&re);
+ let mut manifest_ids: Vec<&str> = map["Settings"].iter().filter_map(|r| r["ManifestId"].as_str()).collect();
+ manifest_ids.sort();
+ assert_eq!(manifest_ids, vec![PERSONAL, SHARED], "the delivering manifest decides the manifest id, not the row's own claim");
-#[test]
-fn split_regrafts_overflow_columns_onto_shared_rows() {
- // A newer writer added a column to a shared item; this client's schema couldn't hold it, so it
- // rode in the overflow carrier. On push it must re-graft and travel with the row into the
- // shared manifest, not the personal one.
- // Recorded under the identity the row had when the overflow was written, before the folder was
- // shared, so under the personal scope. The re-graft has to survive the row changing manifest.
- let overflow = CodecOverflow {
- columns: [(
- "Items".to_string(),
- [(format!("{}\u{1f}i-shared", PERSONAL_M), row(&[("FutureCol", json!("keep-me"))]))].into_iter().collect(),
- )]
- .into_iter()
- .collect(),
- ..Default::default()
- };
- let mut tables = owner_tables();
- tables.push(table(OVERFLOW_TABLE, overflow.to_table_records()));
- // Moving an item across a folder boundary moves it across a manifest boundary: the client
- // re-stamps it on the move, so the fixture does too.
- let tables = stamp_subtree(tables, "f-shared", "m-f-shared");
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- let shared_item = rows(&out.rest()[0].manifest, "Items").iter().find(|r| r["Id"] == json!("i-shared")).unwrap().clone();
- assert_eq!(shared_item["FutureCol"], json!("keep-me"));
- assert!(rows(&out.first().manifest, "Items").iter().all(|r| !r.contains_key("FutureCol")));
+ let mut unaddressed = out.data_buckets[0].clone();
+ unaddressed.manifest_id = String::new();
+ assert!(validate_data_bucket(&unaddressed).failed_rules.contains(&"dataBucket-manifestId-missing".to_string()));
+ let mut cross_stamped = out.data_buckets.iter().find(|b| b.manifest_id == PERSONAL).unwrap().clone();
+ cross_stamped.manifest_id = SHARED.to_string();
+ assert!(validate_data_bucket(&cross_stamped).failed_rules.contains(&"dataBucket-manifest-mismatch".to_string()));
}
-// Materialize: combining
-
-/// Materialized tables as a name > rows map for easy assertions.
-fn materialized_map(m: &MaterializedTables) -> HashMap<String, Vec<CodecRecord>> {
- m.tables.iter().map(|t| (t.name.clone(), t.records.clone())).collect()
-}
+/*
+ * Materialize: combining.
+ */
#[test]
-fn owner_split_then_combine_roundtrips_to_original_tables() {
- // The core round-trip contract: canonicalize with shares, then materialize personal+shared
- // semantically identical to the original unified vault.
+fn an_owners_split_combines_back_to_the_original_vault_and_stays_stable() {
let out = canonicalize_owner();
- let re = materialize_as_sqlite(materialize_input(out.first().manifest.clone(), out.rest().iter().map(|s| s.manifest.clone()).collect(), out.data_buckets.clone())).unwrap();
+ let re = materialize_as_sqlite(materialize_input(manifests_of(&out), out.data_buckets.clone())).unwrap();
let map = materialized_map(&re);
assert_eq!(ids(&map["Items"]), vec!["i-nofolder", "i-personal", "i-shared", "i-sub"]);
@@ -893,208 +545,124 @@ fn owner_split_then_combine_roundtrips_to_original_tables() {
assert_eq!(values(&map["FieldValues"]), vec!["family", "hunter2", "me"]);
assert_eq!(ids(&map["Attachments"]), vec!["att-personal", "att-shared"]);
assert_eq!(ids(&map["TotpCodes"]), vec!["totp-shared"]);
- assert_eq!(tag_links(&map["ItemTags"]), vec![("i-personal".into(), "tag-both".into()), ("i-shared".into(), "tag-both".into()), ("i-sub".into(), "tag-shared-only".into())]);
- /*
- * Tags and field definitions are manifest-scoped like the rest: the two the shared items carry
- * exist once in each manifest that uses them, side by side under `(ManifestId, Id)`. The base keeps
- * its own copies, including "tag-unused", which nothing references at all.
- */
+ assert_eq!(map["ItemTags"].len(), 3);
+ // Tags and definitions exist once in each manifest that uses them, side by side under `(ManifestId, Id)`.
assert_eq!(ids(&map["Tags"]), vec!["tag-both", "tag-both", "tag-shared-only", "tag-shared-only", "tag-unused"]);
- let tag_scopes: HashSet<(&str, &str)> = map["Tags"]
- .iter()
- .map(|r| (r["ManifestId"].as_str().unwrap(), r["Id"].as_str().unwrap()))
- .collect();
- assert_eq!(tag_scopes.len(), 5, "each (scope, id) appears once");
assert_eq!(ids(&map["FieldDefinitions"]), vec!["fd-1", "fd-1", "fd-unused"]);
- /*
- * Logos do NOT collapse back to one row per domain: the personal manifest's github row and the shared
- * manifest's own github row are separate rows, which is the point of scoping them. They coexist
- * under UNIQUE(ManifestId, Kind, Source), and every scope is internally unique. Three rows, not
- * four: netflix.com exists only in the shared scope, since the personal manifest holds no item using it.
- */
+ // Logos do not collapse back to one row per domain: three rows, netflix.com only in the shared manifest.
assert_eq!(map["Logos"].len(), 3);
- let scoped: HashSet<(Option<&str>, &str)> = map["Logos"]
- .iter()
- .map(|r| (r.get("ManifestId").and_then(|v| v.as_str()), r["Source"].as_str().unwrap()))
- .collect();
- assert_eq!(scoped.len(), 3, "each (scope, source) appears once");
assert_eq!(map["EncryptionKeys"].len(), 1);
assert_eq!(map["Settings"].len(), 1);
-}
-
-#[test]
-fn owner_roundtrip_is_stable_across_a_second_split() {
- // Split -> combine -> split again must reproduce the same manifests (no oscillation).
- let first = canonicalize_owner();
- let re = materialize_as_sqlite(materialize_input(first.first().manifest.clone(), first.rest().iter().map(|s| s.manifest.clone()).collect(), first.data_buckets.clone())).unwrap();
-
- // Simulate the platform read-back: blob refs stay as-is (the platform rebinds bytes, and a
- // re-canonicalize would read real bytes; for stability we compare table row id sets).
- let second = canonicalize_from_sqlite(input_with_shares(
- re.tables.iter().map(|t| table(&t.name, t.records.clone())).collect(),
- vec![spec("f-shared")],
- ))
- .unwrap();
+ // The bookkeeping rows record every manifest alike, in input order, without a claim about which one is whose.
+ assert_eq!(map["Manifests"].iter().map(|r| r["Id"].as_str().unwrap()).collect::<Vec<_>>(), vec![PERSONAL, SHARED]);
+ assert!(map["Manifests"].iter().all(|r| !r.contains_key("IsPersonal")));
+ // Split -> combine -> split again reproduces the same manifests (no oscillation).
+ let second = canonicalize(re.tables.iter().map(|t| table(&t.name, t.records.clone())).collect(), &[SHARED]);
for name in ["Folders", "Items", "FieldValues", "Attachments", "TotpCodes", "ItemTags", "Tags", "FieldDefinitions", "Logos"] {
- assert_eq!(ids(rows(&second.first().manifest, name)), ids(rows(&first.first().manifest, name)), "personal {name} drifted");
- assert_eq!(
- ids(rows(&second.rest()[0].manifest, name)),
- ids(rows(&first.rest()[0].manifest, name)),
- "shared {name} drifted"
- );
+ for id in [PERSONAL, SHARED] {
+ assert_eq!(ids(rows_of(&manifest(&second, id).manifest.tables, name)), ids(rows_of(&manifest(&out, id).manifest.tables, name)), "{id} {name} drifted");
+ }
}
}
/// A recipient's own vault (no knowledge of the share) canonicalized to a personal manifest.
fn recipient_personal_manifest() -> (Manifest, Vec<DataBucket>) {
- let out = canonicalize_from_sqlite(input_with_shares(
+ let out = canonicalize(
vec![
- table("Folders", vec![row(&[("Id", json!("f-mine")), ("ParentFolderId", serde_json::Value::Null)])]),
- table("Items", vec![row(&[("Id", json!("i-mine")), ("FolderId", json!("f-mine")), ("LogoId", json!("logo-mine"))])]),
- table("Logos", vec![row(&[("Id", json!("logo-mine")), ("Source", json!("netflix.com")), ("FileData", json!({ "__b64": b64(&[0x11]) }))])]),
- table("EncryptionKeys", vec![row(&[("Id", json!("ek-r")), ("PublicKey", json!("pub-r")), ("PrivateKey", json!("priv-r")), ("IsPrimary", json!(1))])]),
+ table("Folders", vec![stamped(PERSONAL, &[("Id", json!("f-mine")), ("ParentFolderId", serde_json::Value::Null)])]),
+ table("Items", vec![stamped(PERSONAL, &[("Id", json!("i-mine")), ("FolderId", json!("f-mine")), ("LogoId", json!("logo-mine"))])]),
+ table("Logos", vec![stamped(PERSONAL, &[("Id", json!("logo-mine")), ("Source", json!("netflix.com")), ("FileData", json!({ "__b64": b64(&[0x11]) }))])]),
+ table("EncryptionKeys", vec![stamped(PERSONAL, &[("Id", json!("ek-r")), ("PublicKey", json!("pub-r")), ("PrivateKey", json!("priv-r")), ("IsPrimary", json!(1))])]),
],
- vec![],
- ))
- .unwrap();
- (out.first().manifest.clone(), out.data_buckets.clone())
+ &[],
+ );
+ (out.manifests[0].manifest.clone(), out.data_buckets.clone())
}
-#[test]
-fn recipient_combine_materializes_shared_manifest_into_their_vault() {
- let owner = canonicalize_owner();
- let (recipient_manifest, buckets) = recipient_personal_manifest();
- let re = materialize_as_sqlite(materialize_input(recipient_manifest, vec![owner.rest()[0].manifest.clone()], buckets)).unwrap();
- let map = materialized_map(&re);
-
- // Recipient sees their own rows plus the shared subtree.
- assert_eq!(ids(&map["Folders"]), vec!["f-mine", "f-shared", "f-sub"]);
- assert_eq!(ids(&map["Items"]), vec!["i-mine", "i-shared", "i-sub"]);
- assert_eq!(values(&map["FieldValues"]), vec!["family", "hunter2"]);
- /*
- * The owner and the recipient both hold a netflix.com logo, legitimately different images (fetched
- * at different times, or one of them uploaded by hand). They must BOTH survive, in their own
- * scopes: the recipient keeps their personal logo and the shared manifest keeps the owner's. This is
- * the case the old cross-manifest Source dedup collapsed, silently repointing the recipient's own
- * item at a row authored inside someone else's manifest.
- */
- let netflix: Vec<Option<&str>> = map["Logos"]
- .iter()
- .filter(|r| r["Source"] == json!("netflix.com"))
- .map(|r| r.get("ManifestId").and_then(|v| v.as_str()))
- .collect();
- assert_eq!(netflix.len(), 2, "one netflix logo per scope");
- assert!(netflix.contains(&Some(PERSONAL_M)), "recipient's personal logo survives");
- assert!(netflix.contains(&Some("m-f-shared")), "shared manifest's logo survives");
-
- // The recipient's own item still points at their own row, untouched by the share.
- let mine = map["Items"].iter().find(|r| r["Id"] == json!("i-mine")).unwrap();
- assert_eq!(mine["LogoId"], json!(logo_id(PERSONAL_M, "netflix.com")));
-
- // ...and every reference resolves to a logo that is actually present.
- let logo_ids: HashSet<&str> = map["Logos"].iter().filter_map(|r| r.get("Id").and_then(|v| v.as_str())).collect();
- for item in &map["Items"] {
- if let Some(logo_id) = item.get("LogoId").and_then(|v| v.as_str()) {
- assert!(logo_ids.contains(logo_id), "Item {:?} points at missing logo {logo_id}", item.get("Id"));
+/// Rebind every `{ "__blobRef": hash }` cell to inline bytes, as the platform inserts them into SQLite.
+fn rebind_blobs(tables: &mut [CodecTableData], blobs: &HashMap<String, BlobEntry>) {
+ for record in tables.iter_mut().flat_map(|t| t.records.iter_mut()) {
+ let refs: Vec<(String, String)> = record.iter().filter_map(|(col, value)| Some((col.clone(), value.get("__blobRef")?.as_str()?.to_string()))).collect();
+ for (col, hash) in refs {
+ record.insert(col, json!({ "__b64": blobs.get(&hash).map(|b| b.bytes_base64.clone()).unwrap_or_default() }));
}
}
- // Only the recipient's own key material is present.
- assert_eq!(ids(&map["EncryptionKeys"]), vec!["ek-r"]);
}
-#[test]
-fn recipient_roundtrip_reproduces_shared_manifest_without_leaking_into_personal() {
- // Recipient pulls (combine), then pushes (split): the shared manifest must contain exactly the
- // shared rows, and none of them may leak into the recipient's personal manifest.
- let owner = canonicalize_owner();
+/// The member's SQLite after pulling the owner's share, blob bytes bound.
+fn member_tables_after_pull(owner: &CanonicalizedVault) -> Vec<CodecTableData> {
let (recipient_manifest, buckets) = recipient_personal_manifest();
- let re = materialize_as_sqlite(materialize_input(recipient_manifest, vec![owner.rest()[0].manifest.clone()], buckets)).unwrap();
-
- let pushed = canonicalize_from_sqlite(input_with_shares(
- re.tables.iter().map(|t| table(&t.name, t.records.clone())).collect(),
- vec![spec("f-shared")],
- ))
- .unwrap();
-
- // Shared manifest reproduced.
- let shared = &pushed.rest()[0].manifest;
- assert_eq!(ids(rows(shared, "Items")), vec!["i-shared", "i-sub"]);
- assert_eq!(ids(rows(shared, "Folders")), vec!["f-shared", "f-sub"]);
- assert_eq!(values(rows(shared, "FieldValues")), vec!["family", "hunter2"]);
- assert_eq!(ids(rows(shared, "Tags")), vec!["tag-both", "tag-shared-only"]);
-
- // Recipient's personal manifest holds only their own rows, no shared items/folders leaked in.
- assert_eq!(ids(rows(&pushed.first().manifest, "Items")), vec!["i-mine"]);
- assert_eq!(ids(rows(&pushed.first().manifest, "Folders")), vec!["f-mine"]);
- assert!(rows(&pushed.first().manifest, "FieldValues").is_empty());
- // Owner's tags are referenced only by shared items, so they stay out of the recipient's personal manifest.
- assert!(rows(&pushed.first().manifest, "Tags").is_empty());
-
- /*
- * The shared manifest the recipient pushes back carries the same logo rows they pulled, their own
- * netflix.com logo never bleeds into it. This is the property that keeps a member's routine push
- * from rewriting a folder they merely participate in (burning a revision, and racing the owner's
- * concurrent write on the all-or-nothing gate). Ids and scopes only: this round trip never rebinds
- * blob bytes, so FileData reads as null on the way back out.
- */
- assert_eq!(ids(rows(shared, "Logos")), ids(rows(&owner.rest()[0].manifest, "Logos")));
- assert_eq!(logo_sources(shared), logo_sources(&owner.rest()[0].manifest));
- assert_eq!(logo_row(shared, "netflix.com")["ManifestId"], json!("m-f-shared"));
-
- // And the recipient's personal manifest keeps its own row for that same domain.
- assert_eq!(logo_row(&pushed.first().manifest, "netflix.com")["ManifestId"], json!(PERSONAL_M));
- assert_ne!(logo_row(&pushed.first().manifest, "netflix.com")["Id"], logo_row(shared, "netflix.com")["Id"]);
+ let re = materialize_as_sqlite(materialize_input(vec![recipient_manifest, manifest(owner, SHARED).manifest.clone()], buckets)).unwrap();
+ let mut tables: Vec<CodecTableData> = re.tables.iter().map(|t| table(&t.name, t.records.clone())).collect();
+ let mut all_blobs = manifest(owner, PERSONAL).blobs.clone();
+ all_blobs.extend(manifest(owner, SHARED).blobs.clone());
+ rebind_blobs(&mut tables, &all_blobs);
+ tables
}
#[test]
-fn item_moved_into_shared_manifest_brings_its_logo_along() {
- // A personal item is dragged into the shared manifest. Its logo lives in the personal scope, so the
- // partition must clone it under the folder's scope, otherwise the reference would dangle and the
- // logo would be nulled away on the way out.
- let mut tables = owner_tables();
- let items = tables.iter_mut().find(|t| t.name == "Items").unwrap();
- items.records.iter_mut().find(|r| r["Id"] == json!("i-personal")).unwrap().insert("FolderId".to_string(), json!("f-shared"));
-
- // Moving an item across a folder boundary moves it across a manifest boundary: the client
- // re-stamps it on the move, so the fixture does too.
- let tables = stamp_subtree(tables, "f-shared", "m-f-shared");
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- let shared = &out.rest()[0].manifest;
-
- let moved = rows(shared, "Items").iter().find(|r| r["Id"] == json!("i-personal")).unwrap();
- assert_eq!(moved["LogoId"], json!(logo_id("m-f-shared", "github.com")), "logo followed the item into the folder");
- assert!(logo_row(shared, "github.com")["FileData"]["__blobRef"].is_string(), "and it carries real bytes");
-}
+fn a_recipient_combines_the_share_into_their_vault_and_pushes_it_back_unchanged() {
+ let owner = canonicalize_owner();
+ let (recipient_manifest, buckets) = recipient_personal_manifest();
+ let re = materialize_as_sqlite(materialize_input(vec![recipient_manifest, manifest(&owner, SHARED).manifest.clone()], buckets)).unwrap();
+ let map = materialized_map(&re);
-#[test]
-fn item_moved_into_shared_manifest_reuses_its_existing_logo() {
- // Same move, but the folder already shows a logo for that domain. The item reuses it rather than
- // dragging a second row onto the same (scope, domain), the folder's members keep seeing one logo.
- let mut tables = owner_tables();
- let items = tables.iter_mut().find(|t| t.name == "Items").unwrap();
- // i-personal (github.com) joins the shared manifest, where i-shared already uses github.com.
- items.records.iter_mut().find(|r| r["Id"] == json!("i-personal")).unwrap().insert("FolderId".to_string(), json!("f-shared"));
-
- // Moving an item across a folder boundary moves it across a manifest boundary: the client
- // re-stamps it on the move, so the fixture does too.
- let tables = stamp_subtree(tables, "f-shared", "m-f-shared");
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- let shared = &out.rest()[0].manifest;
-
- assert_eq!(logo_sources(shared), vec!["github.com", "netflix.com"], "no duplicate github row");
- let shared_github = logo_row(shared, "github.com")["Id"].clone();
- for item_id in ["i-personal", "i-shared"] {
- let item = rows(shared, "Items").iter().find(|r| r["Id"] == json!(item_id)).unwrap();
- assert_eq!(item["LogoId"], shared_github);
+ // Their own rows plus the shared tree; the owner's and the recipient's netflix.com logos both survive, in their own manifests.
+ assert_eq!(ids(&map["Folders"]), vec!["f-mine", "f-shared", "f-sub"]);
+ assert_eq!(ids(&map["Items"]), vec!["i-mine", "i-shared", "i-sub"]);
+ assert_eq!(values(&map["FieldValues"]), vec!["family", "hunter2"]);
+ let netflix_manifests: HashSet<&str> = map["Logos"].iter().filter(|r| r["Source"] == json!("netflix.com")).map(|r| r["ManifestId"].as_str().unwrap()).collect();
+ assert_eq!(netflix_manifests, HashSet::from([PERSONAL, SHARED]));
+ assert_eq!(find(&map["Items"], "i-mine")["LogoId"], json!(logo_id(PERSONAL, "netflix.com")), "the recipient's item still points at their own row");
+ let logo_ids: HashSet<&str> = map["Logos"].iter().map(|r| r["Id"].as_str().unwrap()).collect();
+ assert!(map["Items"].iter().filter_map(|i| i["LogoId"].as_str()).all(|id| logo_ids.contains(id)), "every reference resolves");
+ assert_eq!(ids(&map["EncryptionKeys"]), vec!["ek-r"], "only the recipient's own key material is present");
+
+ // Pushing back: the shared manifest is reproduced, nothing leaks into the personal one, and the member's own
+ // netflix row never bleeds into the share (which would rewrite a manifest they merely participate in).
+ let pushed = canonicalize(re.tables.iter().map(|t| table(&t.name, t.records.clone())).collect(), &[SHARED]);
+ let shared = &manifest(&pushed, SHARED).manifest;
+ let personal = &manifest(&pushed, PERSONAL).manifest;
+ assert_eq!(ids(rows_of(&shared.tables, "Items")), vec!["i-shared", "i-sub"]);
+ assert_eq!(ids(rows_of(&shared.tables, "Tags")), vec!["tag-both", "tag-shared-only"]);
+ assert_eq!(ids(rows_of(&personal.tables, "Items")), vec!["i-mine"]);
+ assert!(rows_of(&personal.tables, "FieldValues").is_empty() && rows_of(&personal.tables, "Tags").is_empty());
+ assert_eq!(ids(rows_of(&shared.tables, "Logos")), ids(rows_of(&manifest(&owner, SHARED).manifest.tables, "Logos")));
+ assert_ne!(logo_row(personal, "netflix.com")["Id"], logo_row(shared, "netflix.com")["Id"]);
+}
+
+#[test]
+fn a_members_push_keeps_the_shared_logo_and_its_bytes() {
+ // The member edits a shared item. FaviconService only ever looks up personal-manifest rows, so the item ends up
+ // pointing at a fresh personal row: the push folds it back onto the share's own row, bytes included, also when the
+ // member's copy of the shared row is empty because its blob never arrived.
+ let owner = canonicalize_owner();
+ for shared_row_empty in [false, true] {
+ let mut tables = member_tables_after_pull(&owner);
+ let personal_id = logo_id(PERSONAL, "github.com");
+ if shared_row_empty {
+ find_mut(&mut tables, "Logos", &logo_id(SHARED, "github.com")).insert("FileData".to_string(), serde_json::Value::Null);
+ }
+ tables.iter_mut().find(|t| t.name == "Logos").unwrap().records.push(stamped(PERSONAL, &[("Id", json!(personal_id)), ("Source", json!("github.com")), ("FileData", json!({ "__b64": b64(&[0x77, 0x88]) }))]));
+ find_mut(&mut tables, "Items", "i-shared").insert("LogoId".to_string(), json!(personal_id));
+
+ let pushed = canonicalize(tables, &[SHARED]);
+ let shared = manifest(&pushed, SHARED);
+ let logo = logo_row(&shared.manifest, "github.com");
+ let blob_ref = logo["FileData"]["__blobRef"].as_str().unwrap_or_else(|| panic!("shared row empty {shared_row_empty}: the pushed folder logo must still carry bytes, got {:?}", logo["FileData"]));
+ assert!(shared.blobs.contains_key(blob_ref), "and registers them so the write keeps a live blob reference");
+ if !shared_row_empty {
+ assert_eq!(shared.blobs[blob_ref].bytes_base64, b64(&[0xAA, 0xBB]), "the folder keeps the logo its members agreed on");
+ }
+ assert_eq!(find(rows_of(&shared.manifest.tables, "Items"), "i-shared")["LogoId"], logo["Id"], "the item was folded back onto the folder's row");
}
}
#[test]
-fn combine_scopes_a_legacy_shared_manifest_before_it_can_collide() {
- // A shared manifest whose ROWS predate manifest-id scoping: its logos were generated at random and
- // carry no scope stamp. Materializing it next to the recipient's own rows must not collide on
- // UNIQUE(ManifestId, Kind, Source), combine normalizes every row to the manifest's own id.
+fn combine_scopes_a_manifest_written_before_scoping_into_its_own_namespace() {
+ // Rows with random logo ids and no stamps: combine stamps them with the manifest they arrived in and re-derives
+ // their ids, so they land in their own uniqueness bucket next to the recipient's rows.
let (recipient_manifest, buckets) = recipient_personal_manifest();
let legacy_shared = Manifest {
schema_version: SCHEMA_VERSION,
@@ -1109,1125 +677,306 @@ fn combine_scopes_a_legacy_shared_manifest_before_it_can_collide() {
]),
extra: HashMap::new(),
};
-
- let re = materialize_as_sqlite(materialize_input(recipient_manifest, vec![legacy_shared], buckets)).unwrap();
+ let re = materialize_as_sqlite(materialize_input(vec![recipient_manifest, legacy_shared], buckets)).unwrap();
let map = materialized_map(&re);
-
- let scoped: Vec<(Option<&str>, &str)> = map["Logos"]
- .iter()
- .map(|r| (r.get("ManifestId").and_then(|v| v.as_str()), r["Source"].as_str().unwrap()))
- .collect();
- assert_eq!(scoped.len(), 2);
- assert!(scoped.contains(&(Some(PERSONAL_M), "netflix.com")), "recipient's own row keeps the personal scope");
- assert!(scoped.contains(&(Some("m-legacy"), "netflix.com")), "legacy shared row was pulled into its manifest's scope");
-
- // The legacy manifest's item follows its row into the new scope.
- let legacy_item = map["Items"].iter().find(|r| r["Id"] == json!("i-legacy")).unwrap();
- assert_eq!(legacy_item["LogoId"], json!(logo_id("m-legacy", "netflix.com")));
+ let stamped: HashSet<(&str, &str)> = map["Logos"].iter().map(|r| (r["ManifestId"].as_str().unwrap(), r["Source"].as_str().unwrap())).collect();
+ assert_eq!(stamped, HashSet::from([(PERSONAL, "netflix.com"), ("m-legacy", "netflix.com")]));
+ assert_eq!(find(&map["Items"], "i-legacy")["LogoId"], json!(logo_id("m-legacy", "netflix.com")));
}
#[test]
-fn combine_keeps_both_rows_when_two_manifests_share_a_primary_key() {
- // Ids are client-generated, so a shared manifest may legitimately carry a row whose Id equals one of
- // the reader's own. Because rows are keyed by (ManifestId, Id), the primary key the local schema
- // declares, both survive in their own namespace and neither shadows the other.
- let mut personal = canonicalize_from_sqlite(input_with_shares(
- vec![table("Items", vec![row(&[("Id", json!("i-dup")), ("Name", json!("personal-version")), ("FolderId", serde_json::Value::Null)])])],
- vec![],
- ))
- .unwrap()
- .first()
- .manifest
- .clone();
- personal.tables.insert("Folders".to_string(), vec![]);
-
- let mut shared = personal.clone();
- shared.manifest_id = "m-x".to_string();
- shared.name = Some("Renamed".to_string());
- shared.tables.insert("Items".to_string(), vec![row(&[("Id", json!("i-dup")), ("Name", json!("shared-version")), ("FolderId", serde_json::Value::Null)])]);
-
- let re = materialize_as_sqlite(materialize_input(personal, vec![shared], vec![])).unwrap();
- let map = materialized_map(&re);
-
- let by_scope: Vec<(&str, &str)> = map["Items"]
- .iter()
- .map(|r| (r["ManifestId"].as_str().unwrap(), r["Name"].as_str().unwrap()))
- .collect();
- assert_eq!(by_scope.len(), 2, "one row per manifest, not one row total");
- assert!(by_scope.contains(&(PERSONAL_M, "personal-version")));
- assert!(by_scope.contains(&("m-x", "shared-version")));
+fn combine_overflows_what_the_schema_lacks_on_a_shared_manifest_too() {
+ // An older reader of a newer writer's share: an unknown column or table on its rows is kept in the overflow.
+ let (recipient, buckets) = recipient_personal_manifest();
+ let mut shared = manifest(&canonicalize_owner(), SHARED).manifest.clone();
+ find_mut_in(&mut shared, "Items", "i-shared").insert("FutureCol".to_string(), json!("v"));
+ shared.tables.insert("Widgets".to_string(), vec![row(&[("Id", json!("w-1"))])]);
+ let mut input = materialize_input(vec![recipient, shared], buckets);
+ input.schema_columns.get_mut("Items").unwrap().retain(|c| c != "FutureCol");
+ input.schema_columns.remove("Widgets");
+ let re = materialize_as_sqlite(input).unwrap();
+ assert!(materialized_map(&re)["Items"].iter().all(|r| !r.contains_key("FutureCol")));
+ assert_eq!(re.overflow.columns["Items"][&format!("{SHARED}\u{1f}i-shared")]["FutureCol"], json!("v"));
+ assert_eq!(re.overflow.tables["Widgets"][0]["ManifestId"], json!(SHARED));
}
#[test]
-fn combine_keeps_each_manifests_own_copy_of_a_shared_tag() {
- /*
- * Two manifests carrying the same tag id are two rows, not one: `Tags` is keyed by
- * `(ManifestId, Id)` like everything else, so neither copy can shadow the other and a member
- * renaming their copy never overwrites anyone else's. Presenting them as one tag (by name, say)
- * is the client's call.
- */
- let mut base = canonicalize_from_sqlite(input_with_shares(
- vec![table("Tags", vec![row(&[("Id", json!("t-1")), ("Name", json!("base-copy"))])])],
- vec![],
- ))
- .unwrap()
- .first()
- .manifest
- .clone();
- base.tables.insert("Folders".to_string(), vec![]);
-
- let mut shared = base.clone();
+fn combine_keeps_same_id_rows_of_two_manifests_apart() {
+ // Ids are client-generated, so a shared manifest may carry a row whose Id equals one of the reader's own. Rows
+ // are keyed by `(ManifestId, Id)`, so both survive in their own namespace and neither shadows the other.
+ let mut personal = canonicalize(vec![table("Items", vec![stamped(PERSONAL, &[("Id", json!("i-dup")), ("Name", json!("personal-version"))])]), table("Tags", vec![stamped(PERSONAL, &[("Id", json!("t-1")), ("Name", json!("base-copy"))])])], &[]).manifests.remove(0).manifest;
+ personal.tables.insert("Folders".to_string(), vec![]);
+ let mut shared = personal.clone();
shared.manifest_id = "m-x".to_string();
+ shared.tables.insert("Items".to_string(), vec![row(&[("Id", json!("i-dup")), ("Name", json!("shared-version"))])]);
shared.tables.insert("Tags".to_string(), vec![row(&[("Id", json!("t-1")), ("Name", json!("shared-copy"))])]);
- let re = materialize_as_sqlite(materialize_input(base, vec![shared], vec![])).unwrap();
- let map = materialized_map(&re);
- assert_eq!(map["Tags"].len(), 2, "one row per manifest");
- let by_scope: HashSet<(&str, &str)> = map["Tags"]
- .iter()
- .map(|r| (r["ManifestId"].as_str().unwrap(), r["Name"].as_str().unwrap()))
- .collect();
- assert!(by_scope.contains(&(PERSONAL_M, "base-copy")));
- assert!(by_scope.contains(&("m-x", "shared-copy")));
-}
-
-#[test]
-fn split_stamps_item_scoped_rows_with_the_manifest_their_item_joined() {
- // A child row's foreign key is (ManifestId, ItemId), so it has to carry the same stamp as the item
- // it followed, otherwise it points at a row in a namespace it isn't in.
- let tables = stamp_subtree(owner_tables(), "f-shared", "m-f-shared");
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
-
- for name in ["FieldValues", "TotpCodes", "Attachments", "ItemTags"] {
- let shared_rows = rows(&out.rest()[0].manifest, name);
- assert!(!shared_rows.is_empty(), "{name} should have rows in the shared manifest");
- assert!(shared_rows.iter().all(|r| r["ManifestId"] == json!("m-f-shared")), "{name} rows carry the shared manifest's stamp");
- assert!(rows(&out.first().manifest, name).iter().all(|r| r["ManifestId"] == json!(PERSONAL_M)), "{name} rows left behind carry the personal manifest's");
- }
+ let map = materialized_map(&materialize_as_sqlite(materialize_input(vec![personal, shared], vec![])).unwrap());
+ let by_manifest = |table: &str| -> HashSet<(String, String)> { map[table].iter().map(|r| (r["ManifestId"].as_str().unwrap().to_string(), r["Name"].as_str().unwrap().to_string())).collect() };
+ assert_eq!(by_manifest("Items"), HashSet::from([(PERSONAL.to_string(), "personal-version".to_string()), ("m-x".to_string(), "shared-version".to_string())]));
+ assert_eq!(by_manifest("Tags"), HashSet::from([(PERSONAL.to_string(), "base-copy".to_string()), ("m-x".to_string(), "shared-copy".to_string())]));
}
#[test]
-fn combine_drops_child_rows_naming_an_item_the_manifest_does_not_hold() {
- // A manifest may only describe its own items. One carrying a FieldValue for an item in the
- // reader's personal scope is either stale or hand-crafted to graft a field onto it; the composite
- // foreign key rejects it either way, so combine drops it rather than failing the whole insert.
- let mut personal = canonicalize_from_sqlite(input_with_shares(
+fn combine_never_lets_a_manifest_reach_into_another_namespace() {
+ // A manifest may only describe its own rows. One carrying a FieldValue for an item in the reader's personal
+ // manifest, rows stamped for a manifest it cannot write, or key material, settings and bookkeeping tables it has
+ // no business carrying is either stale or hand-crafted; none of it crosses over.
+ let mut personal = canonicalize(
vec![
- table("Items", vec![row(&[("Id", json!("i-mine")), ("Name", json!("mine")), ("FolderId", serde_json::Value::Null)])]),
- table("FieldValues", vec![row(&[("Id", json!("fv-mine")), ("ItemId", json!("i-mine")), ("FieldKey", json!("username")), ("Value", json!("me"))])]),
+ table("Items", vec![stamped(PERSONAL, &[("Id", json!("i-mine")), ("Name", json!("mine"))])]),
+ table("FieldValues", vec![stamped(PERSONAL, &[("Id", json!("fv-mine")), ("ItemId", json!("i-mine")), ("FieldKey", json!("username")), ("Value", json!("me"))])]),
+ table("EncryptionKeys", vec![stamped(PERSONAL, &[("Id", json!("ek-r")), ("PublicKey", json!("pub-r")), ("PrivateKey", json!("priv-r")), ("IsPrimary", json!(1))])]),
],
- vec![],
- ))
- .unwrap()
- .first()
- .manifest
- .clone();
+ &[],
+ )
+ .manifests
+ .remove(0)
+ .manifest;
personal.tables.insert("Folders".to_string(), vec![]);
- let mut hostile = personal.clone();
- hostile.manifest_id = "m-x".to_string();
- hostile.tables.insert("Items".to_string(), vec![]);
- hostile.tables.insert(
- "FieldValues".to_string(),
- vec![row(&[("Id", json!("fv-injected")), ("ItemId", json!("i-mine")), ("FieldKey", json!("password")), ("Value", json!("stolen"))])],
- );
+ let mut victim = personal.clone();
+ victim.manifest_id = "m-victim".to_string();
+ victim.tables.insert("Items".to_string(), vec![row(&[("Id", json!("i-victim")), ("Name", json!("theirs"))])]);
+ victim.tables.insert("FieldValues".to_string(), vec![]);
+ victim.tables.remove("EncryptionKeys");
- let re = materialize_as_sqlite(materialize_input(personal, vec![hostile], vec![])).unwrap();
- let map = materialized_map(&re);
+ let mut hostile = victim.clone();
+ hostile.manifest_id = "m-hostile".to_string();
+ hostile.tables.insert("Items".to_string(), vec![stamped("m-victim", &[("Id", json!("i-planted")), ("Name", json!("planted"))]), stamped("m-victim", &[("Id", json!("i-victim")), ("Name", json!("overwritten"))])]);
+ hostile.tables.insert("FieldValues".to_string(), vec![row(&[("Id", json!("fv-injected")), ("ItemId", json!("i-mine")), ("FieldKey", json!("password")), ("Value", json!("stolen"))])]);
+ hostile.tables.insert("Tags".to_string(), vec![stamped(PERSONAL, &[("Id", json!("t-planted")), ("Name", json!("planted"))])]);
+ hostile.tables.insert("EncryptionKeys".to_string(), vec![row(&[("Id", json!("ek-evil")), ("PrivateKey", json!("stolen"))])]);
+ hostile.tables.insert("Settings".to_string(), vec![row(&[("Key", json!("theme")), ("Value", json!("evil"))])]);
+ hostile.tables.insert(OVERFLOW_TABLE.to_string(), vec![row(&[("Id", json!("x")), ("Data", json!("{}"))])]);
+
+ let map = materialized_map(&materialize_as_sqlite(materialize_input(vec![personal, victim, hostile], vec![])).unwrap());
assert_eq!(values(&map["FieldValues"]), vec!["me"], "the foreign row never reaches the reader's item");
+ let in_victim: Vec<&str> = map["Items"].iter().filter(|r| r["ManifestId"] == json!("m-victim")).map(|r| r["Name"].as_str().unwrap()).collect();
+ assert_eq!(in_victim, vec!["theirs"], "the victim manifest holds only its own item, not overwritten");
+ assert!(["i-planted", "i-victim"].iter().all(|id| map["Items"].iter().any(|r| r["Id"] == json!(id) && r["ManifestId"] == json!("m-hostile"))), "rows stay in the manifest they arrived in");
+ assert!(map["Tags"].iter().all(|r| r["ManifestId"] == json!("m-hostile")));
+ assert_eq!(ids(&map["EncryptionKeys"]), vec!["ek-r"], "injected key row dropped");
+ assert!(!map.contains_key("Settings") && !map.contains_key(OVERFLOW_TABLE), "injected settings and overflow carrier dropped");
}
#[test]
-fn combine_restamps_rows_a_manifest_wrote_for_another_manifest() {
- // A member of one shared manifest writes rows stamped for a manifest it cannot write. The stamp an
- // author wrote is never trusted: every row claims the manifest it arrived in, so nothing crosses over.
- let mut personal = canonicalize_from_sqlite(input_with_shares(vec![table("Items", vec![row(&[("Id", json!("i-mine")), ("Name", json!("mine")), ("FolderId", serde_json::Value::Null)])])], vec![])).unwrap().first().manifest.clone();
- personal.tables.insert("Folders".to_string(), vec![]);
+fn combine_repairs_references_that_resolve_only_in_another_manifest() {
+ // Folder ids are per manifest: a folder with the right Id in the wrong namespace is not this item's folder, and
+ // a parent written before parent normalization may point outside every manifest this user holds.
+ let (mut recipient, buckets) = recipient_personal_manifest();
+ recipient.tables.get_mut("Items").unwrap()[0].insert("FolderId".to_string(), json!("f-shared"));
+ let mut shared = manifest(&canonicalize_owner(), SHARED).manifest.clone();
+ find_mut_in(&mut shared, "Folders", "f-shared").insert("ParentFolderId".to_string(), json!("f-owner-personal-folder"));
- let mut victim = personal.clone();
- victim.manifest_id = "m-victim".to_string();
- victim.tables.insert("Items".to_string(), vec![row(&[("Id", json!("i-victim")), ("Name", json!("theirs")), ("FolderId", serde_json::Value::Null)])]);
+ let map = materialized_map(&materialize_as_sqlite(materialize_input(vec![recipient, shared], buckets)).unwrap());
+ assert_eq!(find(&map["Items"], "i-mine")["FolderId"], serde_json::Value::Null);
+ assert_eq!(find(&map["Folders"], "f-shared")["ParentFolderId"], serde_json::Value::Null);
+ assert_eq!(find(&map["Folders"], "f-sub")["ParentFolderId"], json!("f-shared"), "intact parents are untouched");
+}
- let mut hostile = personal.clone();
- hostile.manifest_id = "m-hostile".to_string();
- hostile.tables.insert(
- "Items".to_string(),
- vec![
- row(&[("Id", json!("i-planted")), ("ManifestId", json!("m-victim")), ("Name", json!("planted")), ("FolderId", serde_json::Value::Null)]),
- row(&[("Id", json!("i-victim")), ("ManifestId", json!("m-victim")), ("Name", json!("overwritten")), ("FolderId", serde_json::Value::Null)]),
- ],
- );
- hostile.tables.insert("Tags".to_string(), vec![row(&[("Id", json!("t-planted")), ("ManifestId", json!(PERSONAL_M)), ("Name", json!("planted"))])]);
+/*
+ * Delivery keypairs (manifest-stamped `EncryptionKeys` rows). One table serves every manifest: a personal key must
+ * never travel INTO a shared manifest, and a shared manifest's key must never travel OUT of its own. Both are
+ * attacks a co-owner can attempt by writing rows into a manifest the victim materializes.
+ */
- let re = materialize_as_sqlite(materialize_input(personal, vec![victim, hostile], vec![])).unwrap();
- let map = materialized_map(&re);
+fn keypair(id: &str, manifest_id: &str, public_key: &str, is_primary: i32) -> CodecRecord {
+ stamped(manifest_id, &[("Id", json!(id)), ("PublicKey", json!(public_key)), ("PrivateKey", json!(format!("priv-{}", public_key))), ("IsPrimary", json!(is_primary))])
+}
- let in_victim: Vec<&CodecRecord> = map["Items"].iter().filter(|r| r["ManifestId"] == json!("m-victim")).collect();
- assert_eq!(in_victim.len(), 1, "the victim manifest holds only its own item");
- assert_eq!(in_victim[0]["Name"], json!("theirs"), "the victim's item is not overwritten");
- for id in ["i-planted", "i-victim"] {
- assert!(map["Items"].iter().any(|r| r["Id"] == json!(id) && r["ManifestId"] == json!("m-hostile")), "{id} stays in the manifest it arrived in");
- }
- assert!(map["Tags"].iter().all(|r| r["ManifestId"] == json!("m-hostile")), "a row stamped for the personal manifest stays out of it");
+fn owner_tables_with_keys(keys: Vec<CodecRecord>) -> Vec<CodecTableData> {
+ owner_tables_with("EncryptionKeys", |rows| rows.extend(keys))
}
#[test]
-fn combine_nulls_an_item_folder_that_only_resolves_in_another_manifest() {
- // Folder ids are per-manifest now: a folder with the right Id in the wrong namespace is not this
- // item's folder, and leaving the reference would fail the platform's foreign_key_check.
- let mut personal = canonicalize_from_sqlite(input_with_shares(
- vec![
- table("Items", vec![row(&[("Id", json!("i-orphan")), ("Name", json!("mine")), ("FolderId", json!("f-elsewhere"))])]),
- table("Folders", vec![]),
- ],
- vec![],
- ))
- .unwrap()
- .first()
- .manifest
- .clone();
- personal.tables.insert("Folders".to_string(), vec![]);
+fn a_keypair_travels_only_inside_the_manifest_its_stamp_names() {
+ // A key stamped for a manifest not in this push (revoked, deleted, fabricated) has nowhere to go: dropped, never
+ // demoted into the personal manifest, where it would resurrect on a re-share and escape revocation. The same
+ // holds in the unshare window, when nothing is shared any more.
+ let keys = vec![keypair("sfk-live", SHARED, "pub-folder", 1), keypair("sfk-orphan", "m-dead", "pub-orphan", 1)];
+ let out = canonicalize(owner_tables_with_keys(keys.clone()), &[SHARED]);
+ assert_eq!(ids(rows_of(&manifest(&out, SHARED).manifest.tables, "EncryptionKeys")), vec!["sfk-live"]);
+ assert_eq!(ids(rows_of(&manifest(&out, PERSONAL).manifest.tables, "EncryptionKeys")), vec!["ek-1"], "the personal manifest keeps exactly its personal keys");
- let mut shared = personal.clone();
- shared.manifest_id = "m-x".to_string();
- shared.tables.insert("Items".to_string(), vec![]);
- shared.tables.insert("Folders".to_string(), vec![row(&[("Id", json!("f-elsewhere")), ("Name", json!("Theirs")), ("ParentFolderId", serde_json::Value::Null)])]);
-
- let re = materialize_as_sqlite(materialize_input(personal, vec![shared], vec![])).unwrap();
- let map = materialized_map(&re);
- let item = map["Items"].iter().find(|r| r["Id"] == json!("i-orphan")).unwrap();
- assert_eq!(item["FolderId"], serde_json::Value::Null);
-}
-
-#[test]
-fn combine_strips_personal_and_bookkeeping_tables_from_shared_manifests() {
- // A malicious/buggy shared manifest tries to inject key material, settings, and an overflow
- // carrier into the recipient's vault: all must be dropped.
- let owner = canonicalize_owner();
- let mut evil = owner.rest()[0].manifest.clone();
- evil.tables.insert("EncryptionKeys".to_string(), vec![row(&[("Id", json!("ek-evil")), ("PrivateKey", json!("stolen"))])]);
- evil.tables.insert("Settings".to_string(), vec![row(&[("Key", json!("theme")), ("Value", json!("evil"))])]);
- evil.tables.insert(OVERFLOW_TABLE.to_string(), vec![row(&[("Id", json!("x")), ("Data", json!("{}"))])]);
-
- let (recipient_manifest, buckets) = recipient_personal_manifest();
- let re = materialize_as_sqlite(materialize_input(recipient_manifest, vec![evil], buckets)).unwrap();
- let map = materialized_map(&re);
- assert_eq!(ids(&map["EncryptionKeys"]), vec!["ek-r"], "injected key row dropped");
- assert!(!map.contains_key("Settings"), "injected settings dropped");
- assert!(!map.contains_key(OVERFLOW_TABLE), "injected overflow carrier dropped");
-}
-
-#[test]
-fn combine_nulls_dangling_parent_folder_references() {
- // A shared manifest written before parent normalization (or with a partial grant set) may carry
- // a ParentFolderId pointing outside every manifest this user holds: null it so FK checks pass.
- let mut shared = canonicalize_owner().rest()[0].manifest.clone();
- for folder in shared.tables.get_mut("Folders").unwrap() {
- if folder["Id"] == json!("f-shared") {
- folder.insert("ParentFolderId".to_string(), json!("f-owner-personal-folder"));
- }
- }
- let (recipient_manifest, buckets) = recipient_personal_manifest();
- let re = materialize_as_sqlite(materialize_input(recipient_manifest, vec![shared], buckets)).unwrap();
- let map = materialized_map(&re);
- let shared_anchor = map["Folders"].iter().find(|r| r["Id"] == json!("f-shared")).unwrap();
- assert_eq!(shared_anchor["ParentFolderId"], serde_json::Value::Null);
- // Intact parents are untouched.
- let sub = map["Folders"].iter().find(|r| r["Id"] == json!("f-sub")).unwrap();
- assert_eq!(sub["ParentFolderId"], json!("f-shared"));
-}
-
-#[test]
-fn combine_applies_schema_overflow_to_shared_rows_too() {
- // An old client materializing a share written by a newer client: unknown columns on shared rows
- // must land in overflow, not crash the insert.
- let mut shared = canonicalize_owner().rest()[0].manifest.clone();
- shared.tables.get_mut("Items").unwrap()[0].insert("FutureCol".to_string(), json!("v"));
-
- let schema: HashMap<String, Vec<String>> = [
- ("Items".to_string(), vec!["Id".to_string(), "FolderId".to_string(), "LogoId".to_string()]),
- ("Folders".to_string(), vec!["Id".to_string(), "Name".to_string(), "ParentFolderId".to_string()]),
- (OVERFLOW_TABLE.to_string(), vec!["Id".to_string(), "Data".to_string()]),
- ]
- .into_iter()
- .collect();
-
- let (recipient_manifest, buckets) = recipient_personal_manifest();
- let re = materialize_as_sqlite(MaterializeInput { manifests: vec![recipient_manifest, shared], data_buckets: buckets, schema_columns: schema }).unwrap();
- let map = materialized_map(&re);
- assert!(map["Items"].iter().all(|r| !r.contains_key("FutureCol")));
- let overflowed: Vec<&String> = re.overflow.columns.get("Items").map(|m| m.keys().collect()).unwrap_or_default();
- assert!(!overflowed.is_empty(), "unknown shared column stashed in overflow");
-}
-
-#[test]
-fn combine_of_a_single_manifest_carries_only_its_own_rows() {
- // Canonicalizing without listing the shared manifest leaves its rows out of the push entirely,
- // so the vault materialized back holds only what the one manifest carries.
- let out = canonicalize_from_sqlite(input_with_shares(owner_tables(), vec![])).unwrap();
- let re = materialize_as_sqlite(materialize_input(out.first().manifest.clone(), vec![], out.data_buckets.clone())).unwrap();
- let map = materialized_map(&re);
- assert_eq!(ids(&map["Items"]), vec!["i-nofolder", "i-personal"]);
-}
-
-// Primary encryption key extraction
-
-#[test]
-fn extract_encryption_key_for_public_key_picks_matching_row_over_primary() {
- // A rotated vault: the old (non-primary) key must still be selectable so a grant wrapped for it unwraps.
- let out = canonicalize_from_sqlite(input_with_shares(
- vec![table("EncryptionKeys", vec![
- row(&[("Id", json!("ek-old")), ("PublicKey", json!("pub-old")), ("PrivateKey", json!("priv-old")), ("IsPrimary", json!(0))]),
- row(&[("Id", json!("ek-cur")), ("PublicKey", json!("pub-cur")), ("PrivateKey", json!("priv-cur")), ("IsPrimary", json!(1))]),
- ])],
- vec![],
- ))
- .unwrap();
- let key = extract_encryption_key_for_public_key(&out.first().manifest, "pub-old").expect("old key present");
- assert_eq!(key["Id"], json!("ek-old"));
- assert_eq!(key["PrivateKey"], json!("priv-old"));
-
- let cur = extract_encryption_key_for_public_key(&out.first().manifest, "pub-cur").expect("current key present");
- assert_eq!(cur["Id"], json!("ek-cur"));
-
- // The manifest's live primary row, the one whose public half is published for delivery, is the current key.
- assert_eq!(ids(&primary_keys(&out.first().manifest)), vec!["ek-cur"]);
-}
-
-/// The live `IsPrimary` rows of a manifest's `EncryptionKeys` table stamped with the manifest's own id.
-fn primary_keys(m: &Manifest) -> Vec<CodecRecord> {
- rows(m, "EncryptionKeys")
- .iter()
- .filter(|r| r["IsPrimary"] == json!(1) && r["ManifestId"] == json!(m.manifest_id) && r.get("IsDeleted").is_none_or(|d| *d == json!(0)))
- .cloned()
- .collect()
-}
-
-#[test]
-fn extract_encryption_key_for_public_key_skips_deleted_and_returns_none_on_miss() {
- let out = canonicalize_from_sqlite(input_with_shares(
- vec![table("EncryptionKeys", vec![
- row(&[("Id", json!("ek-del")), ("PublicKey", json!("pub-del")), ("PrivateKey", json!("priv-del")), ("IsPrimary", json!(1)), ("IsDeleted", json!(1))]),
- ])],
- vec![],
- ))
- .unwrap();
- // A deleted row is never returned even when its public key matches.
- assert!(extract_encryption_key_for_public_key(&out.first().manifest, "pub-del").is_none());
- // An unknown public key yields nothing.
- assert!(extract_encryption_key_for_public_key(&out.first().manifest, "pub-unknown").is_none());
-}
-
-// Wire-format compatibility
-
-#[test]
-fn manifest_specs_deserialize_from_camel_case_json() {
- // Every manifest is described the same way on the wire (id, salt, optional name) with nothing
- // marking one of them as special. A manifest carries a name, never a folder anchor.
- let input_json = json!({
- "tables": [{ "name": "Items", "records": [] }],
- "canonicalizedAt": "2026-01-01T00:00:00.000Z",
- "manifests": [
- { "manifestId": PERSONAL_M, "manifestSalt": SALT_PERSONAL },
- { "manifestId": "m-1", "manifestSalt": SALT_SHARED, "name": "Team", "anchorFolderId": "f-1" }
- ]
- })
- .to_string();
- let out_json = crate::common::error::json_call(&input_json, |input: CanonicalizeInput| canonicalize_from_sqlite(input)).unwrap();
- let out: CanonicalizedVault = serde_json::from_str(&out_json).unwrap();
- assert_eq!(out.rest().len(), 1);
- assert_eq!(out.rest()[0].manifest.manifest_id, "m-1");
- assert_eq!(out.rest()[0].manifest.name.as_deref(), Some("Team"));
-
- // One entry per spec, in spec order, all shaped alike; the buckets sit beside them, not inside one.
- let value: serde_json::Value = serde_json::from_str(&out_json).unwrap();
- assert_eq!(value["manifests"][0]["manifest"]["manifestId"], json!(PERSONAL_M));
- assert_eq!(value["manifests"][1]["manifest"]["manifestId"], json!("m-1"));
- assert_eq!(value["manifests"][1]["manifest"]["name"], json!("Team"));
- assert!(value["manifests"].as_array().unwrap().iter().all(|m| m.get("isPersonal").is_none()), "no manifest is flagged");
- assert!(value["dataBuckets"].is_array(), "buckets belong to the vault");
- assert!(value["manifests"][0].get("dataBuckets").is_none());
- assert!(value["manifests"][1]["manifest"].get("anchorFolderId").is_none(), "folder anchoring is a client concern, never persisted");
-}
-
-/// A vault is made of manifests; with none there is nothing to combine into and no data-model version
-/// to report, so materialize refuses rather than emitting an empty vault.
-#[test]
-fn materialize_rejects_an_empty_manifest_list() {
- let empty = materialize_as_sqlite(materialize_manifests(vec![], vec![]));
- assert!(empty.unwrap_err().to_string().contains("no manifests"));
-}
-
-/// Nothing marks a manifest as the caller's own: the list order says it. The first entry is the base
-/// the others combine into, and the bookkeeping rows record every manifest alike: id and name, no
-/// claim about which one is whose. Nothing about the base becomes a vault-wide fact either: the base
-/// contributes no schema/version claim the other manifests are then held to.
-#[test]
-fn materialize_treats_the_first_manifest_as_the_base() {
- let owner = canonicalize_owner();
- let shared = owner.rest()[0].manifest.clone();
- let own = owner.first().manifest.clone();
-
- let re = materialize_as_sqlite(materialize_manifests(vec![own, shared], owner.data_buckets.clone())).unwrap();
-
- // Nothing version-shaped travels with the manifests; the platform's own schema decides that alone.
- let json = serde_json::to_string(&re).unwrap();
- assert!(!json.contains("migration"), "materialize reports no migration id: the schema the platform applies is the only one there is");
-
- let manifests = re.tables.iter().find(|t| t.name == "Manifests").unwrap();
- assert_eq!(manifests.records.len(), 2);
- assert_eq!(manifests.records[0]["Id"], json!(PERSONAL_M), "bookkeeping rows follow the input order");
- assert!(manifests.records.iter().all(|r| !r.contains_key("IsPersonal")), "no manifest is flagged as the vault's own");
-}
+ let out = canonicalize(owner_tables_with_keys(keys), &[]);
+ assert_eq!(ids(rows_of(&out.manifests[0].manifest.tables, "EncryptionKeys")), vec!["ek-1"], "stale shared keypairs dropped from the personal manifest");
-/// Rebind every `{ "__blobRef": hash }` cell to the inline bytes the platform inserts into SQLite,
-/// what a real pull does before the next read hands the rows back to canonicalize. Without this a
-/// round-trip test never exercises the blob path (an unresolved ref canonicalizes to NULL).
-fn rebind_blobs(tables: &mut [CodecTableData], blobs: &HashMap<String, BlobEntry>) {
- for table in tables.iter_mut() {
- for record in table.records.iter_mut() {
- let refs: Vec<(String, String)> = record
- .iter()
- .filter_map(|(col, value)| Some((col.clone(), value.get("__blobRef")?.as_str()?.to_string())))
- .collect();
- for (col, hash) in refs {
- let bytes = blobs.get(&hash).map(|b| b.bytes_base64.clone()).unwrap_or_default();
- record.insert(col, json!({ "__b64": bytes }));
- }
- }
- }
-}
-
-#[test]
-fn member_push_preserves_the_shared_logo_bytes() {
- // Full member round trip WITH blob bytes bound, as a real pull does: owner shares a folder, the
- // member materializes it, then pushes back an unrelated change. The shared manifest they push must
- // still carry the logo, row, scope, and bytes.
- let owner = canonicalize_owner();
- let owner_shared = &owner.rest()[0];
- let (recipient_manifest, buckets) = recipient_personal_manifest();
-
- let re = materialize_as_sqlite(materialize_input(recipient_manifest, vec![owner_shared.manifest.clone()], buckets)).unwrap();
-
- // The member's SQLite: rows with real bytes in the blob columns.
- let mut member_tables: Vec<CodecTableData> = re.tables.iter().map(|t| table(&t.name, t.records.clone())).collect();
- let mut all_blobs = owner.first().blobs.clone();
- all_blobs.extend(owner_shared.blobs.clone());
- rebind_blobs(&mut member_tables, &all_blobs);
-
- let pushed = canonicalize_from_sqlite(input_with_shares(member_tables, vec![spec("f-shared")])).unwrap();
- let pushed_shared = &pushed.rest()[0];
- let logo = logo_row(&pushed_shared.manifest, "github.com");
-
- let blob_ref = logo["FileData"]["__blobRef"].as_str().expect("member's push keeps the logo's bytes as a blob ref");
- let entry = pushed_shared.blobs.get(blob_ref).expect("and registers those bytes in the shared manifest's blob map");
- assert_eq!(entry.bytes_base64, b64(&[0xAA, 0xBB]), "the bytes are the owner's original logo");
-
- // The item still points at that row, so the logo survives the member's next pull.
- let item = rows(&pushed_shared.manifest, "Items").iter().find(|r| r["Id"] == json!("i-shared")).unwrap();
- assert_eq!(item["LogoId"], logo["Id"]);
-}
-
-/// The member's SQLite after pulling the owner's share, with blob bytes bound as the platform binds them.
-fn member_tables_after_pull(owner: &CanonicalizedVault) -> Vec<CodecTableData> {
- let (recipient_manifest, buckets) = recipient_personal_manifest();
- let re = materialize_as_sqlite(materialize_input(recipient_manifest, vec![owner.rest()[0].manifest.clone()], buckets)).unwrap();
- let mut tables: Vec<CodecTableData> = re.tables.iter().map(|t| table(&t.name, t.records.clone())).collect();
- let mut all_blobs = owner.first().blobs.clone();
- all_blobs.extend(owner.rest()[0].blobs.clone());
- rebind_blobs(&mut tables, &all_blobs);
- tables
-}
-
-#[test]
-fn member_edit_repointing_the_item_at_a_personal_logo_keeps_the_shared_logo() {
- // What the client does when a member edits a shared item: FaviconService only ever looks up (and
- // creates) personal-scope rows, so the item ends up pointing at a fresh personal row. The push must
- // fold that back onto the folder's own row instead of losing the folder's logo.
- let owner = canonicalize_owner();
- let mut tables = member_tables_after_pull(&owner);
-
- let personal_id = logo_id(PERSONAL_M, "github.com");
- let logos = tables.iter_mut().find(|t| t.name == "Logos").unwrap();
- logos.records.push(row(&[
- ("Id", json!(personal_id)),
- ("Source", json!("github.com")),
- ("ManifestId", json!(PERSONAL_M)),
- ("FileData", json!({ "__b64": b64(&[0x77, 0x88]) })),
- ]));
- let items = tables.iter_mut().find(|t| t.name == "Items").unwrap();
- items.records.iter_mut().find(|r| r["Id"] == json!("i-shared")).unwrap().insert("LogoId".to_string(), json!(personal_id));
-
- let pushed = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- let shared = &pushed.rest()[0];
- let logo = logo_row(&shared.manifest, "github.com");
- let blob_ref = logo["FileData"]["__blobRef"].as_str().expect("shared logo still carries bytes");
- assert_eq!(shared.blobs[blob_ref].bytes_base64, b64(&[0xAA, 0xBB]), "the folder keeps the logo its members agreed on");
-
- let item = rows(&shared.manifest, "Items").iter().find(|r| r["Id"] == json!("i-shared")).unwrap();
- assert_eq!(item["LogoId"], logo["Id"], "the item was folded back onto the folder's row");
-}
-
-#[test]
-fn member_whose_shared_logo_blob_never_arrived_does_not_wipe_the_folders_logo() {
- // The member's pull could not resolve the shared logo's bytes (blob missing/undecryptable), so their
- // local row sits there empty while their own personal row for that domain has real bytes. Their next
- // push must not publish the empty row as the folder's logo: that drops the last reference to the
- // owner's blob and every member loses the image.
- let owner = canonicalize_owner();
- let mut tables = member_tables_after_pull(&owner);
-
- let shared_logo_id = logo_id("m-f-shared", "github.com");
- let personal_id = logo_id(PERSONAL_M, "github.com");
- let logos = tables.iter_mut().find(|t| t.name == "Logos").unwrap();
- logos.records.iter_mut().find(|r| r["Id"] == json!(shared_logo_id)).unwrap().insert("FileData".to_string(), serde_json::Value::Null);
- logos.records.push(row(&[
- ("Id", json!(personal_id)),
- ("Source", json!("github.com")),
- ("ManifestId", json!(PERSONAL_M)),
- ("FileData", json!({ "__b64": b64(&[0x77, 0x88]) })),
- ]));
- let items = tables.iter_mut().find(|t| t.name == "Items").unwrap();
- items.records.iter_mut().find(|r| r["Id"] == json!("i-shared")).unwrap().insert("LogoId".to_string(), json!(personal_id));
-
- let pushed = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- let shared = &pushed.rest()[0];
- let logo = logo_row(&shared.manifest, "github.com");
- assert!(logo["FileData"]["__blobRef"].is_string(), "the pushed folder logo must still carry bytes, got {:?}", logo["FileData"]);
- assert!(!shared.blobs.is_empty(), "and register them so the write keeps a live blob reference");
-}
-
-// Logo kinds: one row shape, one `Items.LogoId` pointer, three key spaces.
-
-/// The scoped logo id for any `(scope, kind, source)`.
-fn logo_id_of_kind(scope: &str, kind: &str, source: &str) -> String {
- scoped_assets::logo_id_for(scope, kind, source)
-}
-
-#[test]
-fn logo_kinds_key_independently_so_a_domain_and_a_catalog_key_never_collide() {
- // Kind is part of the natural key: the same Source string in two kinds is two different logos,
- // and neither may swallow the other when a scope is normalized.
- let tables = vec![
- table("Items", vec![
- row(&[("Id", json!("i-fav")), ("FolderId", serde_json::Value::Null), ("LogoId", json!("legacy-fav"))]),
- row(&[("Id", json!("i-builtin")), ("FolderId", serde_json::Value::Null), ("LogoId", json!("legacy-builtin"))]),
- ]),
- table("Logos", vec![
- row(&[("Id", json!("legacy-fav")), ("Kind", json!("favicon")), ("Source", json!("shopping")), ("FileData", json!({ "__b64": b64(&[1, 2]) }))]),
- row(&[("Id", json!("legacy-builtin")), ("Kind", json!("builtin")), ("Source", json!("shopping")), ("FileData", serde_json::Value::Null)]),
- ]),
- ];
-
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![])).unwrap();
- let logos = rows(&out.first().manifest, "Logos");
- assert_eq!(logos.len(), 2, "both kinds survive: {:?}", logos);
-
- let fav_id = logo_id_of_kind(PERSONAL_M, "favicon", "shopping");
- let builtin_id = logo_id_of_kind(PERSONAL_M, "builtin", "shopping");
- assert_ne!(fav_id, builtin_id, "kinds must derive into different ids");
- assert_eq!(ids(logos), { let mut e = vec![fav_id.as_str(), builtin_id.as_str()]; e.sort(); e });
-
- let items = rows(&out.first().manifest, "Items");
- let item = |id: &str| items.iter().find(|r| r["Id"] == json!(id)).unwrap().clone();
- assert_eq!(item("i-fav")["LogoId"], json!(fav_id));
- assert_eq!(item("i-builtin")["LogoId"], json!(builtin_id));
-}
-
-#[test]
-fn icon_row_without_a_kind_is_a_favicon_and_keeps_its_legacy_id() {
- // Rows written before the Kind column exists must not be re-derived: their id has to stay exactly
- // what `logo_id_for` derives for a favicon, or every item in every older vault loses its logo once.
- let tables = vec![
- table("Items", vec![row(&[("Id", json!("i-1")), ("FolderId", serde_json::Value::Null), ("LogoId", json!("random-guid"))])]),
- table("Logos", vec![row(&[("Id", json!("random-guid")), ("Source", json!("github.com")), ("FileData", json!({ "__b64": b64(&[7]) }))])]),
- ];
-
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![])).unwrap();
- let logo = &rows(&out.first().manifest, "Logos")[0];
- assert_eq!(logo["Id"], json!(logo_id(PERSONAL_M, "github.com")), "a kindless row derives as a favicon");
- assert_eq!(logo["Kind"], json!("favicon"), "and is stamped with the kind it implicitly had");
-}
-
-#[test]
-fn a_custom_logo_follows_its_item_into_a_shared_manifest() {
- // An uploaded logo is scoped like a favicon: the item that uses it moves into the share, so the
- // image is cloned into that manifest under the folder-scoped id and the item repointed at it.
- let content_hash = "a".repeat(64);
- let mut tables = owner_tables();
- tables.iter_mut().find(|t| t.name == "Items").unwrap().records.iter_mut().find(|r| r["Id"] == json!("i-shared")).unwrap()
- .insert("LogoId".to_string(), json!("personal-custom"));
- tables.iter_mut().find(|t| t.name == "Logos").unwrap().records.push(row(&[
- ("Id", json!("personal-custom")),
- ("Kind", json!("custom")),
- ("Source", json!(content_hash.clone())),
- ("ManifestId", json!(PERSONAL_M)),
- ("FileData", json!({ "__b64": b64(&[0xaa, 0xbb, 0xcc]) })),
- ]));
-
- // Moving an item across a folder boundary moves it across a manifest boundary: the client
- // re-stamps it on the move, so the fixture does too.
- let tables = stamp_subtree(tables, "f-shared", "m-f-shared");
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- let shared = &out.rest()[0];
- let scoped_id = logo_id_of_kind("m-f-shared", "custom", &content_hash);
- let cloned = rows(&shared.manifest, "Logos").iter().find(|r| r["Id"] == json!(scoped_id))
- .expect("the uploaded logo is cloned into the folder's manifest");
- assert_eq!(cloned["Kind"], json!("custom"));
- assert_eq!(cloned["ManifestId"], json!("m-f-shared"));
- assert!(cloned["FileData"]["__blobRef"].is_string(), "with its bytes, so members can render it");
-
- let shared_item = rows(&shared.manifest, "Items").iter().find(|r| r["Id"] == json!("i-shared")).unwrap().clone();
- assert_eq!(shared_item["LogoId"], json!(scoped_id), "and the item points at the folder-scoped row");
+ // Pinned: the keypair table travels inside its manifest, so it is neither personal-only nor bucketed; Settings is bucketed and not personal.
+ assert!(!is_personal_table("EncryptionKeys") && !is_bucketed_table("EncryptionKeys"));
+ assert!(is_bucketed_table("Settings") && !is_personal_table("Settings"));
}
#[test]
-fn a_builtin_logo_survives_without_any_image_bytes() {
- // Built-in logos carry no FileData at all: every platform draws them from the shared catalog. The
- // row must still round-trip (and travel into a share) rather than being dropped as "empty".
- let mut tables = owner_tables();
- tables.iter_mut().find(|t| t.name == "Items").unwrap().records.iter_mut().find(|r| r["Id"] == json!("i-shared")).unwrap()
- .insert("LogoId".to_string(), json!("builtin-row"));
- tables.iter_mut().find(|t| t.name == "Logos").unwrap().records.push(row(&[
- ("Id", json!("builtin-row")),
- ("Kind", json!("builtin")),
- ("Source", json!("shopping")),
- ("ManifestId", json!(PERSONAL_M)),
- ("FileData", serde_json::Value::Null),
- ]));
-
- // Moving an item across a folder boundary moves it across a manifest boundary: the client
- // re-stamps it on the move, so the fixture does too.
- let tables = stamp_subtree(tables, "f-shared", "m-f-shared");
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- let shared = &out.rest()[0];
- let scoped_id = logo_id_of_kind("m-f-shared", "builtin", "shopping");
- let logo = rows(&shared.manifest, "Logos").iter().find(|r| r["Id"] == json!(scoped_id))
- .expect("a bytesless built-in logo still belongs in the manifest");
- assert_eq!(logo["Source"], json!("shopping"), "the catalog key is what identifies it");
-
- let shared_item = rows(&shared.manifest, "Items").iter().find(|r| r["Id"] == json!("i-shared")).unwrap().clone();
- assert_eq!(shared_item["LogoId"], json!(scoped_id));
-}
+fn combine_accepts_a_manifests_own_keypair_and_drops_every_other_claim() {
+ // A recipient materializes the share's keypair to decrypt mail to the share's aliases. A keypair claiming to be
+ // another manifest's (Mallory co-owns A and writes B's key into A), or a shared key carried by a personal manifest,
+ // is dropped rather than restamped: stamping it would merely turn the attack into displacing the real key.
+ let owner = canonicalize(owner_tables_with_keys(vec![keypair("sfk-1", SHARED, "pub-folder", 1)]), &[SHARED]);
+ let mut evil = manifest(&owner, SHARED).manifest.clone();
+ evil.tables.get_mut("EncryptionKeys").unwrap().push(keypair("sfk-evil", "m-other", "pub-mallory", 1));
+ let (mut recipient, buckets) = recipient_personal_manifest();
+ recipient.tables.get_mut("EncryptionKeys").unwrap().push(keypair("sfk-orphan", "f-whatever", "pub-x", 1));
-// Folder keypairs (manifest-stamped `EncryptionKeys` rows)
-//
-// One table serves every manifest: personal-stamped rows are the personal manifest's own delivery keys,
-// shared-manifest-stamped rows are that folder's delivery keypair. The guards point both ways, a
-// personal key must never travel INTO a shared manifest, and a folder key must never travel OUT of
-// its own. Both directions are attacks a folder co-owner can attempt by writing rows into a manifest
-// the victim materializes.
-
-/// A delivery keypair row stamped with `scope` (a manifest id, or a legacy folder-id stamp).
-fn folder_key(id: &str, scope: &str, public_key: &str, is_primary: i32) -> CodecRecord {
- row(&[
- ("Id", json!(id)),
- ("ManifestId", json!(scope)),
- ("PublicKey", json!(public_key)),
- ("PrivateKey", json!(format!("priv-{}", public_key))),
- ("IsPrimary", json!(is_primary)),
- ])
-}
-
-/// Owner tables plus extra `EncryptionKeys` rows (canonicalize merges same-named table chunks), so the
-/// keypair tests don't disturb the shared fixtures. The base fixture already carries the personal key "ek-1".
-fn owner_tables_with_folder_keys(keys: Vec<CodecRecord>) -> Vec<CodecTableData> {
- let mut tables = owner_tables();
- tables.push(table("EncryptionKeys", keys));
- tables
-}
-
-#[test]
-fn split_routes_folder_keypair_into_its_manifest_and_never_the_personal_one() {
- let out = canonicalize_from_sqlite(input_with_shares(
- owner_tables_with_folder_keys(vec![folder_key("sfk-1", "m-f-shared", "pub-folder", 1)]),
- vec![spec("f-shared")],
- ))
- .unwrap();
-
- assert_eq!(ids(rows(&out.rest()[0].manifest, "EncryptionKeys")), vec!["sfk-1"]);
- assert_eq!(ids(rows(&out.first().manifest, "EncryptionKeys")), vec!["ek-1"], "the personal manifest keeps exactly its personal keys");
- // The unstamped legacy personal row was stamped with the personal manifest's id.
- assert_eq!(rows(&out.first().manifest, "EncryptionKeys")[0]["ManifestId"], json!(PERSONAL_M));
-}
-
-#[test]
-fn split_drops_folder_keypair_whose_scope_is_not_shared() {
- // A key row stamped for a manifest that is not part of this push (revoked, deleted, or fabricated
- // locally) has nowhere to go: it must be dropped, not fall back into the personal manifest, a stale
- // copy demoted into the personal manifest would resurrect the old keypair on a future re-share, and would
- // leave a private delivery key in a namespace revocation cannot reach.
- let out = canonicalize_from_sqlite(input_with_shares(
- owner_tables_with_folder_keys(vec![
- folder_key("sfk-live", "m-f-shared", "pub-folder", 1),
- folder_key("sfk-orphan", "m-dead-manifest", "pub-orphan", 1),
- folder_key("sfk-orphan-legacy", "f-personal", "pub-orphan2", 1),
- ]),
- vec![spec("f-shared")],
- ))
- .unwrap();
-
- assert_eq!(ids(rows(&out.rest()[0].manifest, "EncryptionKeys")), vec!["sfk-live"]);
- assert_eq!(ids(rows(&out.first().manifest, "EncryptionKeys")), vec!["ek-1"], "orphaned folder keypair dropped, never demoted to the personal manifest");
-}
-
-#[test]
-fn split_drops_folder_keypair_even_when_nothing_is_shared() {
- // The unshare window: sharing was just turned off (no specs this push) but the local DB still
- // holds the folder's key rows. They must be dropped, not leak into the personal manifest.
- let out = canonicalize_from_sqlite(input_with_shares(
- owner_tables_with_folder_keys(vec![folder_key("sfk-stale", "m-f-shared", "pub-stale", 1)]),
- vec![],
- ))
- .unwrap();
- assert!(out.rest().is_empty());
- assert_eq!(ids(rows(&out.first().manifest, "EncryptionKeys")), vec!["ek-1"], "stale folder keypair dropped from the personal manifest");
-}
-
-#[test]
-fn combine_accepts_folder_keypair_from_the_manifest_that_owns_it() {
- // The whole point of folder-scoped rows: a recipient materializes the folder's keypair so they can
- // decrypt mail sent to the folder's shared aliases.
- let owner = canonicalize_from_sqlite(input_with_shares(
- owner_tables_with_folder_keys(vec![folder_key("sfk-1", "m-f-shared", "pub-folder", 1)]),
- vec![spec("f-shared")],
- ))
- .unwrap();
-
- let (recipient_manifest, buckets) = recipient_personal_manifest();
- let re = materialize_as_sqlite(materialize_input(recipient_manifest, vec![owner.rest()[0].manifest.clone()], buckets)).unwrap();
- let map = materialized_map(&re);
-
- // The folder's keypair lands next to the recipient's own personal key, distinguished by scope.
+ let map = materialized_map(&materialize_as_sqlite(materialize_input(vec![recipient, evil], buckets)).unwrap());
assert_eq!(ids(&map["EncryptionKeys"]), vec!["ek-r", "sfk-1"]);
- let folder_row = map["EncryptionKeys"].iter().find(|r| r["Id"] == json!("sfk-1")).unwrap();
- assert_eq!(folder_row["PrivateKey"], json!("priv-pub-folder"));
- assert_eq!(folder_row["ManifestId"], json!("m-f-shared"));
-}
-
-#[test]
-fn combine_drops_folder_keypair_scoped_to_another_folder() {
- /*
- * The injection attack the scope filter exists to resist. Mallory co-owns folder A and writes into
- * A's manifest a keypair claiming to be folder B's delivery key. If the recipient accepted it,
- * their client would publish Mallory's public key for B and all of B's mail would be readable by her.
- *
- * The row is DROPPED rather than re-scoped to A: stamping it with A's id would merely turn the
- * attack into "displace A's real keypair", which is no better.
- */
- let owner = canonicalize_from_sqlite(input_with_shares(
- owner_tables_with_folder_keys(vec![folder_key("sfk-legit", "m-f-shared", "pub-folder", 1)]),
- vec![spec("f-shared")],
- ))
- .unwrap();
-
- let mut evil = owner.rest()[0].manifest.clone();
- evil.tables.get_mut("EncryptionKeys").unwrap().push(folder_key("sfk-evil", "m-other-manifest", "pub-mallory", 1));
+ assert_eq!(find(&map["EncryptionKeys"], "sfk-1")["PrivateKey"], json!("priv-pub-folder"));
+ assert!(!map["EncryptionKeys"].iter().any(|r| r["PublicKey"] == json!("pub-mallory")), "the injected key survives under no manifest");
- let (recipient_manifest, buckets) = recipient_personal_manifest();
- let re = materialize_as_sqlite(materialize_input(recipient_manifest, vec![evil], buckets)).unwrap();
- let map = materialized_map(&re);
-
- assert_eq!(ids(&map["EncryptionKeys"]), vec!["ek-r", "sfk-legit"], "cross-folder key row dropped");
- assert!(
- !map["EncryptionKeys"].iter().any(|r| r["PublicKey"] == json!("pub-mallory")),
- "the injected key must not survive under any scope, including a re-stamped one"
- );
+ // And the owner's own round trip keeps both keypairs.
+ let re = materialize_as_sqlite(materialize_input(manifests_of(&owner), owner.data_buckets.clone())).unwrap();
+ assert_eq!(ids(&materialized_map(&re)["EncryptionKeys"]), vec!["ek-1", "sfk-1"]);
}
#[test]
-fn combine_drops_folder_keypair_carried_by_a_personal_manifest() {
- // Canonicalize never emits one, so a folder-scoped key row in a personal manifest means a tampered or
- // malformed manifest. Drop it rather than materialize key material with no owning folder.
- let (mut recipient_manifest, buckets) = recipient_personal_manifest();
- recipient_manifest.tables.get_mut("EncryptionKeys").unwrap().push(folder_key("sfk-orphan", "f-whatever", "pub-x", 1));
+fn a_keypair_resolves_by_public_key_within_its_manifest_only() {
+ // After a rotation the superseded row stays so older mail remains decryptable; only the live primary is the
+ // delivery key. A caller that skips combine must still not be handed another manifest's key material.
+ let out = canonicalize(owner_tables_with_keys(vec![keypair("sfk-old", SHARED, "pub-old", 0), keypair("sfk-cur", SHARED, "pub-cur", 1)]), &[SHARED]);
+ let mut shared = manifest(&out, SHARED).manifest.clone();
+ assert_eq!(extract_encryption_key_for_public_key(&shared, "pub-old").unwrap()["PrivateKey"], json!("priv-pub-old"), "a rotated key still resolves");
+ assert_eq!(extract_encryption_key_for_public_key(&shared, "pub-cur").unwrap()["Id"], json!("sfk-cur"));
+ assert!(extract_encryption_key_for_public_key(&shared, "pub-nonexistent").is_none());
- let re = materialize_as_sqlite(materialize_input(recipient_manifest, vec![], buckets)).unwrap();
- let map = materialized_map(&re);
- assert_eq!(ids(&map["EncryptionKeys"]), vec!["ek-r"], "personal-manifest-carried folder keypair dropped");
+ shared.tables.get_mut("EncryptionKeys").unwrap().push(keypair("sfk-foreign", "m-other", "pub-foreign", 1));
+ let mut deleted = keypair("sfk-dead", SHARED, "pub-dead", 0);
+ deleted.insert("IsDeleted".to_string(), json!(1));
+ shared.tables.get_mut("EncryptionKeys").unwrap().push(deleted);
+ assert!(extract_encryption_key_for_public_key(&shared, "pub-foreign").is_none(), "another manifest's row ignored");
+ assert!(extract_encryption_key_for_public_key(&shared, "pub-dead").is_none(), "deleted row ignored");
}
#[test]
-fn encryption_keys_is_scoped_rather_than_personal_or_bucketed() {
- /*
- * Pins the routing class itself: the keypair table travels inside the manifest its scope names, so
- * it must be neither personal-only (which would pin every row to the personal manifest) nor bucketed
- * (which would move it out of the manifest into a resource of its own).
- */
- assert!(!is_personal_table("EncryptionKeys"));
- assert!(!is_bucketed_table("EncryptionKeys"));
-
- /*
- * And pins the two axes apart. Bucketed says where a table syncs; personal says which manifests may
- * hold it. Settings is bucketed and NOT personal, its bucket belongs to a manifest, so a shared
- * manifest carrying its own settings is a routing question, never a rule violation.
- */
- assert!(is_bucketed_table("Settings"));
- assert!(!is_personal_table("Settings"));
-}
+fn validation_refuses_a_manifest_carrying_what_is_not_its_own() {
+ // Catching this at validate time means a tampered vault fails loudly on push instead of quietly losing rows at
+ // each recipient's combine step.
+ let out = canonicalize(owner_tables_with_keys(vec![keypair("sfk-1", SHARED, "pub-folder", 1)]), &[SHARED]);
+ assert!(validate_manifest(&manifest(&out, PERSONAL).manifest).ok && validate_manifest(&manifest(&out, SHARED).manifest).ok);
-#[test]
-fn folder_keypair_survives_a_full_split_combine_roundtrip() {
- let out = canonicalize_from_sqlite(input_with_shares(
- owner_tables_with_folder_keys(vec![folder_key("sfk-1", "m-f-shared", "pub-folder", 1)]),
- vec![spec("f-shared")],
- ))
- .unwrap();
- let re = materialize_as_sqlite(materialize_input(out.first().manifest.clone(), out.rest().iter().map(|s| s.manifest.clone()).collect(), out.data_buckets.clone())).unwrap();
- let map = materialized_map(&re);
- assert_eq!(ids(&map["EncryptionKeys"]), vec!["ek-1", "sfk-1"]);
-}
+ let mut personal = manifest(&out, PERSONAL).manifest.clone();
+ personal.tables.get_mut("EncryptionKeys").unwrap().push(keypair("sfk-orphan", SHARED, "pub-x", 1));
+ assert!(validate_manifest(&personal).failed_rules.contains(&"encryption-keys-manifest-mismatch".to_string()));
-#[test]
-fn active_folder_key_is_the_primary_row_and_rotated_keys_stay_resolvable() {
- // After a rotation the superseded row stays in the manifest so mail received before the rotation
- // remains decryptable, but only the live primary is offered as the folder's delivery key.
- let out = canonicalize_from_sqlite(input_with_shares(
- owner_tables_with_folder_keys(vec![
- folder_key("sfk-old", "m-f-shared", "pub-old", 0),
- folder_key("sfk-cur", "m-f-shared", "pub-cur", 1),
- ]),
- vec![spec("f-shared")],
- ))
- .unwrap();
- let shared = &out.rest()[0].manifest;
+ let mut shared = manifest(&out, SHARED).manifest.clone();
+ shared.tables.get_mut("EncryptionKeys").unwrap().push(keypair("sfk-evil", "m-other", "pub-mallory", 1));
+ assert!(validate_manifest(&shared).failed_rules.contains(&"encryption-keys-manifest-mismatch".to_string()));
- assert_eq!(ids(&primary_keys(shared)), vec!["sfk-cur"]);
- assert_eq!(
- extract_encryption_key_for_public_key(shared, "pub-old").expect("rotated key still resolvable")["PrivateKey"],
- json!("priv-pub-old")
- );
- assert!(extract_encryption_key_for_public_key(shared, "pub-nonexistent").is_none());
+ // A bucketed table trips the bucketed-table rule, not the personal-table one: it belongs in the manifest's own
+ // bucket, settings are not personal as such.
+ let mut shared = manifest(&out, SHARED).manifest.clone();
+ shared.tables.insert("Settings".to_string(), vec![row(&[("Key", json!("theme")), ("Value", json!("evil"))])]);
+ let result = validate_manifest(&shared);
+ assert!(result.failed_rules.contains(&"manifest-carries-bucketed-table".to_string()));
+ assert!(!result.failed_rules.contains(&"manifest-carries-personal-table".to_string()));
}
#[test]
-fn folder_key_lookup_ignores_deleted_and_out_of_scope_rows() {
- let mut manifest = canonicalize_from_sqlite(input_with_shares(
- owner_tables_with_folder_keys(vec![folder_key("sfk-cur", "m-f-shared", "pub-cur", 1)]),
- vec![spec("f-shared")],
- ))
- .unwrap()
- .rest()[0]
- .manifest
- .clone();
-
- // A caller that skips combine must still not be handed another folder's key material.
- manifest.tables.get_mut("EncryptionKeys").unwrap().push(folder_key("sfk-foreign", "m-other", "pub-foreign", 1));
- assert!(extract_encryption_key_for_public_key(&manifest, "pub-foreign").is_none(), "out-of-scope row ignored");
-
- let mut deleted = folder_key("sfk-dead", "m-f-shared", "pub-dead", 0);
- deleted.insert("IsDeleted".to_string(), json!(1));
- manifest.tables.get_mut("EncryptionKeys").unwrap().push(deleted);
- assert!(extract_encryption_key_for_public_key(&manifest, "pub-dead").is_none(), "deleted row ignored");
+fn no_key_material_crosses_a_manifest_boundary_in_a_full_roundtrip() {
+ // The delivery keypair decides who can read a manifest's alias mail, so it is the one row whose misrouting is
+ // unrecoverable: the personal private key inside a shared manifest hands every member the user's own mail.
+ let tables = owner_tables_with("EncryptionKeys", |keys| {
+ keys.clear();
+ keys.push(stamped(PERSONAL, &[("Id", json!("ek-personal")), ("PublicKey", json!("pub-personal")), ("PrivateKey", json!("priv_personal_only")), ("IsPrimary", json!(1)), ("IsDeleted", json!(0))]));
+ keys.push(stamped(SHARED, &[("Id", json!("ek-shared")), ("PublicKey", json!("pub-shared")), ("PrivateKey", json!("priv_shared_only")), ("IsPrimary", json!(1)), ("IsDeleted", json!(0))]));
+ });
+ let out = canonicalize(tables, &[SHARED]);
+ let personal = manifest(&out, PERSONAL).manifest.clone();
+ let shared = manifest(&out, SHARED).manifest.clone();
+ assert!(!mentions(&shared, "priv_personal_only"), "the personal private key must never be inside a manifest other people hold");
+ assert!(!mentions(&personal, "priv_shared_only"), "nor the share's key inside the personal manifest");
+
+ let map = materialized_map(&materialize_as_sqlite(materialize_input(vec![personal, shared], out.data_buckets.clone())).unwrap());
+ assert_eq!(map["EncryptionKeys"].len(), 2);
+ for key_row in &map["EncryptionKeys"] {
+ let expected = if key_row["PrivateKey"] == json!("priv_personal_only") { PERSONAL } else { SHARED };
+ assert_eq!(key_row["ManifestId"], json!(expected), "a key row materialized into the wrong manifest");
+ }
}
-#[test]
-fn validate_rejects_misplaced_folder_keypairs_before_upload() {
- // Catching this at validate time means a tampered vault fails loudly on push instead of quietly
- // losing rows at each recipient's combine step.
- let out = canonicalize_from_sqlite(input_with_shares(
- owner_tables_with_folder_keys(vec![folder_key("sfk-1", "m-f-shared", "pub-folder", 1)]),
- vec![spec("f-shared")],
- ))
- .unwrap();
+/*
+ * The same id in two manifests. A manifest is a namespace, so two of them may each hold a row with the same `Id`
+ * (a member moved a shared item into their own vault, an item moved out came back through another member's push).
+ * Every rule that resolves a reference therefore resolves it by `(ManifestId, Id)`; anything keyed on the bare id
+ * sends rows across a manifest boundary, which for a shared manifest means handing them to other people.
+ */
- let mut personal = out.first().manifest.clone();
- personal.tables.get_mut("EncryptionKeys").unwrap().push(folder_key("sfk-orphan", "m-f-shared", "pub-x", 1));
- let result = validate_manifest(&personal);
- assert!(!result.ok);
- assert!(result.failed_rules.contains(&"encryption-keys-scope-mismatch".to_string()));
-
- let mut shared = out.rest()[0].manifest.clone();
- shared.tables.get_mut("EncryptionKeys").unwrap().push(folder_key("sfk-evil", "m-other", "pub-mallory", 1));
- let result = validate_manifest(&shared);
- assert!(!result.ok);
- assert!(result.failed_rules.contains(&"encryption-keys-scope-mismatch".to_string()));
-
- // The well-formed pair still validates.
- assert!(validate_manifest(&out.rest()[0].manifest).ok);
- assert!(validate_manifest(&out.first().manifest).ok);
-}
-
-// Cross-manifest routing: the same id in two manifests
-//
-// A manifest is a namespace, so two of them may each hold a row with the same `Id`, and they do, for
-// ordinary reasons: a member moves a shared item into their own vault (the client re-stamps the row and
-// keeps its id), or an item that was moved out comes back through another member's push. Every rule
-// that resolves a reference therefore has to resolve it by `(ManifestId, Id)`, both halves. Anything
-// that keys on the bare id sends rows across a manifest boundary, which for a shared manifest means
-// handing them to other people.
-
-/// Two manifests each holding an item with the same id, each with its own child rows. `personal_secret`
-/// / `shared_secret` are what the assertions look for: the personal one must never appear in the share.
+/// Two manifests each holding item `i-dup` with its own child rows; `personal_secret` must never appear in the share.
fn tables_with_a_duplicated_item_id() -> Vec<CodecTableData> {
+ let child = |id: &str, column: &str, manifest_id: &str, value: serde_json::Value| stamped(manifest_id, &[("Id", json!(id)), ("ItemId", json!("i-dup")), (column, value)]);
+ let bytes = |text: &str| json!({ "__b64": b64(text.as_bytes()) });
vec![
- table("Folders", vec![
- row(&[("Id", json!("f-personal")), ("Name", json!("Personal")), ("ParentFolderId", serde_json::Value::Null), ("ManifestId", json!(PERSONAL_M))]),
- row(&[("Id", json!("f-shared")), ("Name", json!("Family")), ("ParentFolderId", serde_json::Value::Null), ("ManifestId", json!("m-f-shared"))]),
- ]),
- table("Items", vec![
- row(&[("Id", json!("i-dup")), ("Name", json!("mine")), ("FolderId", json!("f-personal")), ("ManifestId", json!(PERSONAL_M))]),
- row(&[("Id", json!("i-dup")), ("Name", json!("ours")), ("FolderId", json!("f-shared")), ("ManifestId", json!("m-f-shared"))]),
- ]),
+ table("Folders", vec![stamped(PERSONAL, &[("Id", json!("f-personal")), ("ParentFolderId", serde_json::Value::Null)]), stamped(SHARED, &[("Id", json!("f-shared")), ("ParentFolderId", serde_json::Value::Null)])]),
+ table("Items", vec![stamped(PERSONAL, &[("Id", json!("i-dup")), ("Name", json!("mine")), ("FolderId", json!("f-personal"))]), stamped(SHARED, &[("Id", json!("i-dup")), ("Name", json!("ours")), ("FolderId", json!("f-shared"))])]),
table("FieldValues", vec![
- row(&[("Id", json!("fv-personal")), ("ItemId", json!("i-dup")), ("FieldKey", json!("password")), ("Value", json!("personal_secret")), ("ManifestId", json!(PERSONAL_M))]),
- row(&[("Id", json!("fv-shared")), ("ItemId", json!("i-dup")), ("FieldKey", json!("password")), ("Value", json!("shared_secret")), ("ManifestId", json!("m-f-shared"))]),
- ]),
- table("TotpCodes", vec![
- row(&[("Id", json!("totp-personal")), ("ItemId", json!("i-dup")), ("SecretKey", json!({ "__b64": b64(b"personal_secret") })), ("ManifestId", json!(PERSONAL_M))]),
- row(&[("Id", json!("totp-shared")), ("ItemId", json!("i-dup")), ("SecretKey", json!({ "__b64": b64(b"shared_secret") })), ("ManifestId", json!("m-f-shared"))]),
- ]),
- table("Attachments", vec![
- row(&[("Id", json!("att-personal")), ("ItemId", json!("i-dup")), ("Blob", json!({ "__b64": b64(b"personal_secret") })), ("ManifestId", json!(PERSONAL_M))]),
- ]),
- table("Passkeys", vec![
- row(&[("Id", json!("pk-personal")), ("ItemId", json!("i-dup")), ("PrivateKey", json!("personal_secret")), ("ManifestId", json!(PERSONAL_M))]),
- ]),
- table("FieldHistories", vec![
- row(&[("Id", json!("fh-personal")), ("ItemId", json!("i-dup")), ("Value", json!("personal_secret")), ("ManifestId", json!(PERSONAL_M))]),
- ]),
- table("ItemTags", vec![
- row(&[("Id", json!("it-personal")), ("ItemId", json!("i-dup")), ("TagId", json!("tag-personal")), ("ManifestId", json!(PERSONAL_M))]),
- ]),
- table("Tags", vec![
- row(&[("Id", json!("tag-personal")), ("Name", json!("personal_secret")), ("ManifestId", json!(PERSONAL_M))]),
- ]),
+ stamped(PERSONAL, &[("Id", json!("fv-personal")), ("ItemId", json!("i-dup")), ("FieldKey", json!("password")), ("Value", json!("personal_secret"))]),
+ stamped(SHARED, &[("Id", json!("fv-shared")), ("ItemId", json!("i-dup")), ("FieldKey", json!("password")), ("Value", json!("shared_secret"))]),
+ ]),
+ table("TotpCodes", vec![child("totp-personal", "SecretKey", PERSONAL, bytes("personal_secret")), child("totp-shared", "SecretKey", SHARED, bytes("shared_secret"))]),
+ table("Attachments", vec![child("att-personal", "Blob", PERSONAL, bytes("personal_secret"))]),
+ table("Passkeys", vec![child("pk-personal", "PrivateKey", PERSONAL, json!("personal_secret"))]),
+ table("FieldHistories", vec![child("fh-personal", "Value", PERSONAL, json!("personal_secret"))]),
+ table("ItemTags", vec![stamped(PERSONAL, &[("ItemId", json!("i-dup")), ("TagId", json!("tag-personal"))])]),
+ table("Tags", vec![stamped(PERSONAL, &[("Id", json!("tag-personal")), ("Name", json!("personal_secret"))])]),
]
}
-/// Every string a manifest carries, at any depth, so a test can ask whether a secret leaked into it
-/// without having to know which column of which table it would have arrived in.
-fn all_strings(m: &Manifest) -> Vec<String> {
- fn walk(value: &serde_json::Value, out: &mut Vec<String>) {
- match value {
- serde_json::Value::String(s) => out.push(s.clone()),
- serde_json::Value::Array(items) => items.iter().for_each(|v| walk(v, out)),
- serde_json::Value::Object(map) => map.values().for_each(|v| walk(v, out)),
- _ => {}
- }
- }
- let mut out = Vec::new();
- for rows in m.tables.values() {
- for row in rows {
- for value in row.values() {
- walk(value, &mut out);
- }
- }
- }
- out
-}
-
-/// True when `needle` appears anywhere in the manifest, base64-encoded blobs included.
-fn manifest_mentions(m: &Manifest, needle: &str) -> bool {
- let encoded = b64(needle.as_bytes());
- all_strings(m).iter().any(|s| s == needle || s == &encoded)
-}
-
#[test]
-fn split_never_follows_a_bare_item_id_into_another_manifest() {
- /*
- * The leak this whole section exists for. Both manifests hold an item called `i-dup`; the personal
- * one's child rows carry a password, a TOTP secret, an attachment, a passkey and a field history.
- * Routing them by the bare `ItemId` would send every one of them into the manifest shared with
- * other people.
- */
- let out = canonicalize_from_sqlite(raw_input_with_shares(tables_with_a_duplicated_item_id(), vec![spec("f-shared")])).unwrap();
- let personal = &out.first().manifest;
- let shared = &out.rest()[0].manifest;
-
- assert!(!manifest_mentions(shared, "personal_secret"), "a personal item's child rows must never follow a same-id item into a shared manifest");
- assert!(manifest_mentions(personal, "personal_secret"), "and they must still be in the manifest they belong to");
-
- // Each manifest keeps exactly its own child rows, and the shared one keeps working.
- assert_eq!(values(rows(personal, "FieldValues")), vec!["personal_secret"]);
- assert_eq!(values(rows(shared, "FieldValues")), vec!["shared_secret"]);
- assert_eq!(ids(rows(personal, "TotpCodes")), vec!["totp-personal"]);
- assert_eq!(ids(rows(shared, "TotpCodes")), vec!["totp-shared"]);
+fn child_rows_never_follow_a_bare_item_id_into_another_manifest() {
+ let out = canonicalize(tables_with_a_duplicated_item_id(), &[SHARED]);
+ let personal = &manifest(&out, PERSONAL).manifest;
+ let shared = &manifest(&out, SHARED).manifest;
+ assert!(!mentions(shared, "personal_secret"), "a personal item's child rows must never follow a same-id item into a shared manifest");
+ assert!(!mentions(personal, "shared_secret"), "and the share's rows stay in the share");
+ assert!(mentions(personal, "personal_secret") && mentions(shared, "shared_secret"), "each keeps its own");
for name in ["Attachments", "Passkeys", "FieldHistories", "ItemTags"] {
- assert!(rows(shared, name).is_empty(), "{name} of the personal item stayed out of the share");
- assert_eq!(rows(personal, name).len(), 1, "{name} of the personal item stayed with it");
+ assert!(rows_of(&shared.tables, name).is_empty() && rows_of(&personal.tables, name).len() == 1, "{name} of the personal item stayed with it");
}
-}
-
-#[test]
-fn split_never_pulls_a_shared_items_child_into_the_personal_manifest() {
- // The same rule read the other way: the share's own rows must not be swept into the personal
- // manifest because a personal item happens to carry the same id.
- let out = canonicalize_from_sqlite(raw_input_with_shares(tables_with_a_duplicated_item_id(), vec![spec("f-shared")])).unwrap();
- assert!(!manifest_mentions(&out.first().manifest, "shared_secret"), "the share's rows stayed in the share");
- assert!(manifest_mentions(&out.rest()[0].manifest, "shared_secret"));
-}
-
-#[test]
-fn split_keeps_child_rows_when_a_manifest_this_vault_lost_shares_their_item_id() {
- /*
- * A row stamped for a manifest this vault no longer carries is dropped, but only its own rows. An
- * item left behind by a revoked share must not take the child rows of a live item that happens to
- * carry the same id down with it.
- */
- let tables = vec![
- table("Folders", vec![row(&[("Id", json!("f-personal")), ("Name", json!("Personal")), ("ParentFolderId", serde_json::Value::Null), ("ManifestId", json!(PERSONAL_M))])]),
- table("Items", vec![
- row(&[("Id", json!("i-dup")), ("Name", json!("mine")), ("FolderId", json!("f-personal")), ("ManifestId", json!(PERSONAL_M))]),
- row(&[("Id", json!("i-dup")), ("Name", json!("revoked")), ("FolderId", serde_json::Value::Null), ("ManifestId", json!("m-revoked"))]),
- ]),
- table("FieldValues", vec![
- row(&[("Id", json!("fv-personal")), ("ItemId", json!("i-dup")), ("FieldKey", json!("password")), ("Value", json!("personal_secret")), ("ManifestId", json!(PERSONAL_M))]),
- row(&[("Id", json!("fv-revoked")), ("ItemId", json!("i-dup")), ("FieldKey", json!("password")), ("Value", json!("gone")), ("ManifestId", json!("m-revoked"))]),
- ]),
- ];
- let out = canonicalize_from_sqlite(raw_input_with_shares(tables, vec![])).unwrap();
- let personal = &out.first().manifest;
- assert_eq!(ids(rows(personal, "Items")), vec!["i-dup"], "the revoked manifest's item is dropped, the live one kept");
- assert_eq!(values(rows(personal, "FieldValues")), vec!["personal_secret"], "the live item keeps its own child rows");
- assert!(!manifest_mentions(personal, "gone"), "the revoked manifest's rows are not moved into the personal one");
+ // End to end: the duplicate survives a split and combine as two independent items, each with its own secrets.
+ let map = materialized_map(&materialize_as_sqlite(materialize_input(manifests_of(&out), out.data_buckets.clone())).unwrap());
+ let field_values: HashSet<(&str, &str)> = map["FieldValues"].iter().map(|r| (r["ManifestId"].as_str().unwrap(), r["Value"].as_str().unwrap())).collect();
+ assert_eq!(field_values, HashSet::from([(PERSONAL, "personal_secret"), (SHARED, "shared_secret")]));
}
#[test]
-fn split_still_follows_an_item_whose_children_were_left_unstamped() {
- /*
- * The safety net the item-follow rule exists for, which the scope-aware lookup must not cost us: a
- * client that moved an item between manifests without re-stamping its children (no trigger, an
- * older platform) leaves them naming the manifest the item left. While only one item carries that
- * id there is no ambiguity, so the child follows it and is re-stamped to agree.
- */
- let tables = stamp_subtree(owner_tables_unstamped(), "f-shared", "m-f-shared");
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
-
- // `input_with_shares` stamps everything the fixture left unstamped as personal, child rows included.
- let shared = &out.rest()[0].manifest;
- assert_eq!(values(rows(shared, "FieldValues")), vec!["family", "hunter2"], "children followed their item across the boundary");
- assert!(rows(shared, "FieldValues").iter().all(|r| r["ManifestId"] == json!("m-f-shared")), "and were re-stamped to agree with it");
-}
+fn children_left_unstamped_follow_their_item_only_while_that_is_unambiguous() {
+ // The safety net for a client that moved an item between manifests without re-stamping its children (no
+ // trigger, an older platform): while only one item carries the id, the child follows it and is re-stamped. With
+ // two same-id items, a child naming neither routes by its own stamp rather than being guessed into a share.
+ let mut tables = owner_tables();
+ for table in tables.iter_mut().filter(|t| t.name == "FieldValues" || t.name == "TotpCodes") {
+ table.records.iter_mut().for_each(|r| drop(r.insert("ManifestId".to_string(), json!(PERSONAL))));
+ }
+ let out = canonicalize(tables, &[SHARED]);
+ let shared = &manifest(&out, SHARED).manifest;
+ assert_eq!(values(rows_of(&shared.tables, "FieldValues")), vec!["family", "hunter2"], "children followed their item across the boundary");
+ assert!(rows_of(&shared.tables, "FieldValues").iter().all(|r| r["ManifestId"] == json!(SHARED)), "and were re-stamped to agree with it");
-#[test]
-fn split_falls_back_to_a_rows_own_stamp_when_two_items_could_claim_it() {
- /*
- * The fallback above is only safe while it is unambiguous. With two same-id items in different
- * manifests, a child naming neither (its own manifest holds no such item) has no item to follow,
- * so it routes by its own stamp rather than being handed to whichever manifest happens to be found
- * first.
- */
let mut tables = tables_with_a_duplicated_item_id();
- // A third manifest's orphan child: its stamp names a manifest that is not in this push.
- tables.push(table("FieldValues", vec![
- row(&[("Id", json!("fv-orphan")), ("ItemId", json!("i-dup")), ("FieldKey", json!("password")), ("Value", json!("orphan_secret")), ("ManifestId", json!("m-revoked"))]),
- ]));
-
- let out = canonicalize_from_sqlite(raw_input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- assert!(!manifest_mentions(&out.rest()[0].manifest, "orphan_secret"), "an ambiguous child is never guessed into a shared manifest");
- assert!(!manifest_mentions(&out.first().manifest, "orphan_secret"), "its own stamp names a manifest this vault no longer carries, so it is dropped");
+ tables.push(table("FieldValues", vec![stamped("m-revoked", &[("Id", json!("fv-orphan")), ("ItemId", json!("i-dup")), ("FieldKey", json!("password")), ("Value", json!("orphan_secret"))])]));
+ let out = canonicalize(tables, &[SHARED]);
+ assert!(!mentions(&manifest(&out, SHARED).manifest, "orphan_secret"), "an ambiguous child is never guessed into a shared manifest");
+ assert!(!mentions(&manifest(&out, PERSONAL).manifest, "orphan_secret"), "its own stamp names a lost manifest, so it is dropped");
}
-#[test]
-fn split_keeps_a_parent_folder_link_that_resolves_in_the_folders_own_manifest() {
- /*
- * The same-id rule for folders. A personal folder carrying the id of the share's subfolder must not
- * make the share's own parent link look foreign, nulling it would silently flatten the shared tree.
- */
- let tables = vec![
- table("Folders", vec![
- row(&[("Id", json!("f-shared")), ("Name", json!("Family")), ("ParentFolderId", serde_json::Value::Null), ("ManifestId", json!("m-f-shared"))]),
- row(&[("Id", json!("f-sub")), ("Name", json!("Streaming")), ("ParentFolderId", json!("f-shared")), ("ManifestId", json!("m-f-shared"))]),
- // A personal folder that happens to carry the shared anchor's id.
- row(&[("Id", json!("f-shared")), ("Name", json!("Mine")), ("ParentFolderId", serde_json::Value::Null), ("ManifestId", json!(PERSONAL_M))]),
- ]),
- ];
-
- let out = canonicalize_from_sqlite(raw_input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- let shared = &out.rest()[0].manifest;
- let sub = rows(shared, "Folders").iter().find(|r| r["Id"] == json!("f-sub")).unwrap();
- assert_eq!(sub["ParentFolderId"], json!("f-shared"), "the parent resolves inside this manifest, so the link stands");
-}
+/*
+ * Wire format.
+ */
#[test]
-fn split_roundtrips_two_same_id_items_without_mixing_their_children() {
- // End to end: the duplicate survives a full split + combine as two independent items, each with
- // its own secrets, which is what `(ManifestId, Id)` promises everywhere else.
- let out = canonicalize_from_sqlite(raw_input_with_shares(tables_with_a_duplicated_item_id(), vec![spec("f-shared")])).unwrap();
- let re = materialize_as_sqlite(materialize_input(out.first().manifest.clone(), vec![out.rest()[0].manifest.clone()], out.data_buckets.clone())).unwrap();
- let map = materialized_map(&re);
-
- let field_values: Vec<(&str, &str)> = map["FieldValues"]
- .iter()
- .map(|r| (r["ManifestId"].as_str().unwrap(), r["Value"].as_str().unwrap()))
- .collect();
- assert!(field_values.contains(&(PERSONAL_M, "personal_secret")));
- assert!(field_values.contains(&("m-f-shared", "shared_secret")));
- assert_eq!(field_values.len(), 2, "neither row was dropped or duplicated into the other manifest");
-}
-
-// Cross-manifest routing: the blanket invariant
-
-#[test]
-fn every_row_of_a_canonicalized_manifest_carries_that_manifests_own_stamp() {
- /*
- * The invariant every routing rule has to preserve, asserted over a whole vault rather than
- * table by table: a manifest is a namespace, so a row inside one that names another is either a row
- * that leaked in or a reference that will resolve against the wrong namespace on the way back.
- * A new table added to the codec is covered by this the day it appears in a manifest.
- */
- let out = canonicalize_owner();
- for canonicalized in out.manifests.iter() {
- let manifest = &canonicalized.manifest;
- for (name, rows) in manifest.tables.iter() {
- for row in rows {
- assert_eq!(
- row.get("ManifestId").and_then(|v| v.as_str()),
- Some(manifest.manifest_id.as_str()),
- "{name} row {:?} sits in manifest {} but names another",
- row.get("Id"),
- manifest.manifest_id
- );
- }
- }
- }
- for bucket in out.data_buckets.iter() {
- for (name, rows) in bucket.tables.iter() {
- for row in rows {
- assert_eq!(
- row.get("ManifestId").and_then(|v| v.as_str()),
- Some(bucket.manifest_id.as_str()),
- "{name} row in the {} bucket of {} names another manifest",
- bucket.category,
- bucket.manifest_id
- );
- }
- }
- }
-}
-
-#[test]
-fn no_key_material_crosses_a_manifest_boundary_in_a_full_roundtrip() {
- /*
- * The delivery keypair decides who can read a manifest's alias mail, so it is the one row whose
- * misrouting is unrecoverable: publishing the personal private key inside a shared manifest hands
- * every member the user's own mail. Split and combine are both checked here, on a vault where each
- * manifest holds a keypair whose private half names the manifest it belongs to.
- */
- let mut tables = owner_tables();
- tables.retain(|t| t.name != "EncryptionKeys");
- tables.push(table("EncryptionKeys", vec![
- row(&[("Id", json!("ek-personal")), ("PublicKey", json!("pub-personal")), ("PrivateKey", json!("priv_personal_only")), ("IsPrimary", json!(1)), ("IsDeleted", json!(0)), ("ManifestId", json!(PERSONAL_M))]),
- row(&[("Id", json!("ek-shared")), ("PublicKey", json!("pub-shared")), ("PrivateKey", json!("priv_shared_only")), ("IsPrimary", json!(1)), ("IsDeleted", json!(0)), ("ManifestId", json!("m-f-shared"))]),
- ]));
-
- let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();
- let personal = out.first().manifest.clone();
- let shared = out.rest()[0].manifest.clone();
-
- assert!(!manifest_mentions(&shared, "priv_personal_only"), "the personal private key must never be inside a manifest other people hold");
- assert!(!manifest_mentions(&personal, "priv_shared_only"), "nor the share's key inside the personal manifest");
- assert!(validate_manifest(&shared).ok && validate_manifest(&personal).ok);
-
- // And the same after combine, where the rule is enforced against the manifest's own claim.
- let re = materialize_as_sqlite(materialize_input(personal, vec![shared], out.data_buckets.clone())).unwrap();
- let map = materialized_map(&re);
- for key_row in map["EncryptionKeys"].iter() {
- let expected = if key_row["PrivateKey"] == json!("priv_personal_only") { PERSONAL_M } else { "m-f-shared" };
- assert_eq!(key_row["ManifestId"], json!(expected), "a key row materialized into the wrong manifest");
- }
- assert_eq!(map["EncryptionKeys"].len(), 2);
+fn manifest_specs_deserialize_from_camel_case_json() {
+ // Every manifest is described the same way on the wire (id, salt, optional name) with nothing marking one as special.
+ let input_json = json!({
+ "tables": [{ "name": "Items", "records": [] }],
+ "canonicalizedAt": "2026-01-01T00:00:00.000Z",
+ "manifests": [{ "manifestId": PERSONAL, "manifestSalt": SALT_PERSONAL }, { "manifestId": "m-1", "manifestSalt": SALT_SHARED, "name": "Team", "anchorFolderId": "f-1" }]
+ })
+ .to_string();
+ let out_json = crate::common::error::json_call(&input_json, |input: CanonicalizeInput| canonicalize_from_sqlite(input)).unwrap();
+ let value: serde_json::Value = serde_json::from_str(&out_json).unwrap();
+ assert_eq!(value["manifests"][0]["manifest"]["manifestId"], json!(PERSONAL));
+ assert_eq!(value["manifests"][1]["manifest"]["manifestId"], json!("m-1"));
+ assert_eq!(value["manifests"][1]["manifest"]["name"], json!("Team"));
+ assert!(value["manifests"].as_array().unwrap().iter().all(|m| m.get("isPersonal").is_none() && m["manifest"].get("anchorFolderId").is_none()), "nothing flags a manifest as the vault's own, and folder anchoring is never persisted");
+ assert!(value["dataBuckets"].is_array() && value["manifests"][0].get("dataBuckets").is_none(), "buckets belong to the vault, beside the manifests");
}
-
diff --git a/core/rust/src/vault_codec/test_support.rs b/core/rust/src/vault_codec/test_support.rs
index 4df9a7c8f..4fab51dc6 100644
--- a/core/rust/src/vault_codec/test_support.rs
+++ b/core/rust/src/vault_codec/test_support.rs
@@ -1,13 +1,22 @@
-//! Fixtures shared by the codec, sharing and pruner tests.
+//! Fixtures shared by the codec and pruner tests.
use std::collections::{HashMap, HashSet};
use serde_json::json;
-use super::manifest::{CodecRecord, CodecTableData, DataBucket, Manifest, MaterializeInput};
+use super::manifest::{CodecRecord, CodecTableData, DataBucket, Manifest, MaterializeInput, MaterializedTables};
use super::types::manifest_scoped_tables;
use crate::vault_model::{MANIFESTS_TABLE, MANIFEST_ID_COL};
+/// The personal manifest id every test suite uses.
+pub(crate) const PERSONAL_MANIFEST_ID: &str = "11111111-aaaa-4111-8111-111111111111";
+
+/// The shared manifest id every test suite uses.
+pub(crate) const SHARED_MANIFEST_ID: &str = "22222222-bbbb-4222-8222-222222222222";
+
+/// The blob salt of the personal manifest every test suite uses.
+pub(crate) const PERSONAL_SALT: &str = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff";
+
/// Standard base64 of `bytes`, the inline byte spelling the tests hand the codec.
pub(crate) fn b64(bytes: &[u8]) -> String {
crate::common::encoding::base64_encode(bytes)
@@ -18,6 +27,17 @@ pub(crate) fn row(pairs: &[(&str, serde_json::Value)]) -> CodecRecord {
pairs.iter().map(|(k, v)| (k.to_string(), v.clone())).collect()
}
+/// A row stamped for `manifest_id`, as the client writes every manifest-scoped row.
+pub(crate) fn stamped(manifest_id: &str, pairs: &[(&str, serde_json::Value)]) -> CodecRecord {
+ restamp(row(pairs), manifest_id)
+}
+
+/// `record` with its `ManifestId` set to `manifest_id`.
+pub(crate) fn restamp(mut record: CodecRecord, manifest_id: &str) -> CodecRecord {
+ record.insert(MANIFEST_ID_COL.to_string(), json!(manifest_id));
+ record
+}
+
/// A table with the given rows.
pub(crate) fn table(name: &str, records: Vec<CodecRecord>) -> CodecTableData {
CodecTableData { name: name.to_string(), records }
@@ -38,8 +58,7 @@ pub(crate) fn stamp_unstamped(mut tables: Vec<CodecTableData>, manifest_id: &str
let scoped = manifest_scoped_tables();
for table in tables.iter_mut().filter(|t| scoped.contains(&t.name.as_str())) {
for record in table.records.iter_mut() {
- let unstamped = record.get(MANIFEST_ID_COL).and_then(|v| v.as_str()).is_none_or(str::is_empty);
- if unstamped {
+ if record.get(MANIFEST_ID_COL).and_then(|v| v.as_str()).is_none_or(str::is_empty) {
record.insert(MANIFEST_ID_COL.to_string(), json!(manifest_id));
}
}
@@ -47,8 +66,8 @@ pub(crate) fn stamp_unstamped(mut tables: Vec<CodecTableData>, manifest_id: &str
tables
}
-/// A schema map that knows every table and column the given manifests and buckets carry, so
-/// materialize fits all of it and splits nothing off into overflow.
+/// A schema map that knows every table and column the given manifests and buckets carry, so materialize fits all
+/// of it and splits nothing off into overflow.
pub(crate) fn fitting_schema<'a>(manifests: impl IntoIterator<Item = &'a Manifest>, buckets: &[DataBucket]) -> HashMap<String, Vec<String>> {
let mut schema: HashMap<String, HashSet<String>> = HashMap::new();
let mut absorb = |tables: &HashMap<String, Vec<CodecRecord>>| {
@@ -66,23 +85,18 @@ pub(crate) fn fitting_schema<'a>(manifests: impl IntoIterator<Item = &'a Manifes
for bucket in buckets {
absorb(&bucket.tables);
}
-
// Local bookkeeping materialize emits itself; it is never present in a manifest to be absorbed above.
schema.insert(MANIFESTS_TABLE.to_string(), ["Id", "Name"].iter().map(|c| c.to_string()).collect());
-
schema.into_iter().map(|(name, columns)| (name, columns.into_iter().collect())).collect()
}
-/// A `MaterializeInput` for a personal manifest plus the other manifests combined into it, with the
-/// buckets hung off it and a schema fitted to everything they carry (see [`fitting_schema`]).
-pub(crate) fn materialize_input(own: Manifest, others: Vec<Manifest>, data_buckets: Vec<DataBucket>) -> MaterializeInput {
- let schema = fitting_schema(std::iter::once(&own).chain(others.iter()), &data_buckets);
- MaterializeInput { manifests: std::iter::once(own).chain(others).collect(), data_buckets, schema_columns: schema }
+/// Materialized tables as a name > rows map.
+pub(crate) fn materialized_map(materialized: &MaterializedTables) -> HashMap<String, Vec<CodecRecord>> {
+ materialized.tables.iter().map(|t| (t.name.clone(), t.records.clone())).collect()
}
-/// A `MaterializeInput` from an explicit manifest list, for the tests that assert on inputs
-/// [`materialize_input`] cannot express (an empty manifest list, for one).
-pub(crate) fn materialize_manifests(manifests: Vec<Manifest>, data_buckets: Vec<DataBucket>) -> MaterializeInput {
+/// A `MaterializeInput` for the given manifests and buckets with a schema fitted to everything they carry.
+pub(crate) fn materialize_input(manifests: Vec<Manifest>, data_buckets: Vec<DataBucket>) -> MaterializeInput {
let schema = fitting_schema(manifests.iter(), &data_buckets);
MaterializeInput { manifests, data_buckets, schema_columns: schema }
}
diff --git a/core/rust/src/vault_codec/tests.rs b/core/rust/src/vault_codec/tests.rs
index 14825b705..dfa68ac77 100644
--- a/core/rust/src/vault_codec/tests.rs
+++ b/core/rust/src/vault_codec/tests.rs
@@ -1,917 +1,392 @@
-//! Unit tests for vault_codec, covering the round-trip contract.
+//! The codec round trip for one manifest: canonicalize, pack, unpack, materialize, and the forward-compatibility
+//! overflow. Multi-manifest routing lives in `sharing_tests`.
+
use super::*;
-use super::test_support::{b64, materialize_input, row, stamp_unstamped};
+use super::test_support::{b64, materialize_input, materialized_map, row, stamp_unstamped, stamped, table, PERSONAL_MANIFEST_ID as PERSONAL, PERSONAL_SALT as SALT};
use super::types::{bucket_category_for, SCHEMA_VERSION};
-use crate::vault_model::names::LOGO_KIND_FAVICON;
use crate::vault_model::{MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_ROW_ID, OVERFLOW_TABLE};
use serde_json::json;
use std::collections::HashMap;
-/// The rows of `table` inside the data bucket for `category` (empty if absent).
-fn bucket_rows<'a>(out: &'a CanonicalizedVault, category: &str, table: &str) -> &'a [CodecRecord] {
- out.data_buckets
- .iter()
- .find(|b| b.category == category)
- .and_then(|b| b.tables.get(table))
- .map(Vec::as_slice)
- .unwrap_or(&[])
+/// Canonicalize `tables` as the personal manifest, every row stamped the way a client writes it.
+fn canonicalize(tables: Vec<CodecTableData>) -> CanonicalizedVault {
+ canonicalize_from_sqlite(CanonicalizeInput {
+ tables: stamp_unstamped(tables, PERSONAL),
+ canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(),
+ manifests: vec![ManifestSpec { manifest_id: PERSONAL.to_string(), manifest_salt: SALT.to_string(), name: None }],
+ stamp_unstamped_into: None,
+ })
+ .unwrap()
}
-/// The personal manifest id every basic-input test canonicalizes against.
-const PERSONAL_MANIFEST: &str = "00000000-0000-0000-0000-00000000self";
+/// The one manifest of a single-manifest canonicalize.
+fn manifest_of(out: &CanonicalizedVault) -> &CanonicalizedManifest {
+ &out.manifests[0]
+}
-/// A canonicalize input whose rows are stamped the way a real client writes them: every row of a
-/// manifest-scoped table names a manifest.
-fn basic_input(tables: Vec<CodecTableData>) -> CanonicalizeInput {
- CanonicalizeInput {
- tables: stamp_unstamped(tables, PERSONAL_MANIFEST),
- canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(),
- manifests: vec![ManifestSpec {
- manifest_id: PERSONAL_MANIFEST.to_string(),
- manifest_salt: "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff".to_string(),
- name: None,
- }],
- stamp_unstamped_into: None,
- }
+/// Canonicalize then materialize with a schema that fits everything.
+fn round_trip(tables: Vec<CodecTableData>) -> (CanonicalizedVault, MaterializedTables) {
+ let out = canonicalize(tables);
+ let re = materialize_as_sqlite(materialize_input(vec![manifest_of(&out).manifest.clone()], out.data_buckets.clone())).unwrap();
+ (out, re)
}
-#[test]
-fn canonicalize_from_sqlite_splits_settings_into_data_bucket() {
- let input = basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] },
- CodecTableData { name: "Settings".to_string(), records: vec![row(&[("Key", json!("k")), ("Value", json!("v"))])] },
- ]);
- let out = canonicalize_from_sqlite(input).unwrap();
- assert!(out.first().manifest.tables.contains_key("Items"));
- assert!(!out.first().manifest.tables.contains_key("Settings"));
- assert_eq!(bucket_rows(&out, "settings", "Settings").len(), 1);
+/// The rows of `table` inside the data bucket for `category` (empty if absent).
+fn bucket_rows<'a>(out: &'a CanonicalizedVault, category: &str, table: &str) -> &'a [CodecRecord] {
+ out.data_buckets.iter().find(|b| b.category == category).and_then(|b| b.tables.get(table)).map(Vec::as_slice).unwrap_or(&[])
}
-#[test]
-fn bucket_layout_matches_bucket_tables_source_of_truth() {
- // The layout platforms consume must be derived purely from BUCKET_TABLES, one entry per distinct
- // category, each listing exactly that category's tables.
- let layout = bucket_layout();
- assert_eq!(layout.len(), bucket_categories().len());
- for entry in &layout {
- assert_eq!(entry.tables, tables_for_category(&entry.category));
- for table in &entry.tables {
- assert_eq!(bucket_category_for(table), Some(entry.category.as_str()));
- }
- }
- assert_eq!(layout.iter().map(|e| e.category.as_str()).collect::<Vec<_>>(), vec!["settings", "stats"]);
- assert_eq!(serde_json::to_string(&bucket_layout()).unwrap(), serde_json::to_string(&layout).unwrap());
+/// A client one release behind a writer: knows `Items` and `Settings` plus the overflow carrier, nothing else.
+fn narrow_client_schema() -> HashMap<String, Vec<String>> {
+ [
+ ("Items".to_string(), vec![MANIFEST_ID_COL.to_string(), "Id".to_string(), "Name".to_string()]),
+ ("Settings".to_string(), vec![MANIFEST_ID_COL.to_string(), "Key".to_string(), "Value".to_string()]),
+ (OVERFLOW_TABLE.to_string(), vec!["Id".to_string(), "Data".to_string()]),
+ ]
+ .into_iter()
+ .collect()
}
+fn overflow_table_of(re: &MaterializedTables) -> Option<&CodecTableData> {
+ re.tables.iter().find(|t| t.name == OVERFLOW_TABLE)
+}
+
+/*
+ * The registry the codec runs on.
+ */
+
#[test]
-fn every_bucketed_table_is_manifest_scoped_with_a_composite_identity() {
- // The invariant that makes per-manifest buckets work at all: a bucket is addressed by `(manifest_id, category)`, so a row that carries no manifest cannot be routed into one.
+fn the_bucket_layout_follows_the_registry() {
+ // A bucket is addressed by `(manifest_id, category)`, so every bucketed table must be registered, manifest-scoped
+ // and identified by `(ManifestId, primary key)`; the layout the platforms consume derives purely from the registry.
for (table, category) in crate::vault_model::BUCKET_TABLES {
- assert!(
- crate::vault_model::SYNCABLE_TABLES.iter().any(|t| t.name == *table),
- "bucketed table {} ({}) is not registered in SYNCABLE_TABLES, so it would never row-merge",
- table,
- category
- );
- assert!(
- super::types::is_manifest_scoped(table),
- "bucketed table {} ({}) must be manifest_scoped: its rows route to a bucket by their own ManifestId",
- table,
- category
- );
+ assert!(crate::vault_model::SYNCABLE_TABLES.iter().any(|t| t.name == *table), "bucketed table {table} ({category}) is not registered");
+ assert!(super::types::is_manifest_scoped(table), "bucketed table {table} must be manifest scoped");
let mut expected = vec![MANIFEST_ID_COL];
expected.extend_from_slice(super::types::primary_key_columns_for(table));
- assert_eq!(
- super::types::identity_columns_for(table),
- expected,
- "bucketed table {} must be addressed by (ManifestId, primary key)",
- table
- );
+ assert_eq!(super::types::identity_columns_for(table), expected, "{table} must be addressed by (ManifestId, primary key)");
+ }
+ let layout = bucket_layout();
+ assert_eq!(layout.iter().map(|e| e.category.as_str()).collect::<Vec<_>>(), vec!["settings", "stats"]);
+ for entry in &layout {
+ assert_eq!(entry.tables, tables_for_category(&entry.category));
+ assert!(entry.tables.iter().all(|t| bucket_category_for(t) == Some(entry.category.as_str())));
}
}
-#[test]
-fn canonicalize_from_sqlite_skips_internal_tables() {
- // Skip-tables must never enter the manifest.
- let input = basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![] },
- CodecTableData { name: "__EFMigrationsHistory".to_string(), records: vec![row(&[("MigrationId", json!("x"))])] },
- CodecTableData { name: "android_metadata".to_string(), records: vec![row(&[("locale", json!("en_US"))])] },
- CodecTableData { name: "sqlite_sequence".to_string(), records: vec![] },
- ]);
- let out = canonicalize_from_sqlite(input).unwrap();
- assert!(out.first().manifest.tables.contains_key("Items"));
- assert!(!out.first().manifest.tables.contains_key("__EFMigrationsHistory"));
- assert!(!out.first().manifest.tables.contains_key("android_metadata"));
- assert!(!out.first().manifest.tables.contains_key("sqlite_sequence"));
-}
+/*
+ * Canonicalize and materialize.
+ */
#[test]
-fn canonicalize_from_sqlite_extracts_blob_columns_and_hashes() {
+fn a_vault_round_trips_through_manifest_buckets_and_blobs() {
+ // Bucketed tables leave the manifest for their bucket, skip tables never enter it, blob columns become
+ // content-addressed references, other byte columns keep their inline bytes, and materialize puts it all back.
let favicon = vec![0xde, 0xad, 0xbe, 0xef];
- // The logo needs an item pointing at it: an unreferenced one is pruned (see `prune_unreferenced_logos`).
- let input = basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("LogoId", json!("l1"))])] },
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![row(&[("Id", json!("l1")), ("FileData", json!({ "__b64": b64(&favicon) }))])],
- },
+ let attachment = vec![0xaa, 0xbb, 0xcc];
+ let secret = vec![1u8, 2, 3];
+ let (out, re) = round_trip(vec![
+ table("Items", vec![row(&[("Id", json!("i1")), ("FolderId", serde_json::Value::Null), ("LogoId", json!("l1")), ("Secret", json!({ "__b64": b64(&secret) }))])]),
+ table("Logos", vec![row(&[("Id", json!("l1")), ("Source", json!("github.com")), ("FileData", json!({ "__b64": b64(&favicon) }))])]),
+ table("Attachments", vec![
+ row(&[("Id", json!("a1")), ("ItemId", json!("i1")), ("Blob", json!({ "__b64": b64(&attachment) }))]),
+ row(&[("Id", json!("a2")), ("ItemId", json!("i1")), ("Blob", serde_json::Value::Null)]),
+ row(&[("Id", json!("a3")), ("ItemId", json!("i1")), ("Blob", json!({ "__b64": "" }))]),
+ ]),
+ table("Settings", vec![row(&[("Key", json!("theme")), ("Value", json!("dark"))])]),
+ table("__EFMigrationsHistory", vec![row(&[("MigrationId", json!("x"))])]),
+ table("android_metadata", vec![row(&[("locale", json!("en_US"))])]),
]);
- let out = canonicalize_from_sqlite(input).unwrap();
- assert_eq!(out.first().blobs.len(), 1);
- let (hash, entry) = out.first().blobs.iter().next().unwrap();
- assert_eq!(entry.kind, "favicon");
- assert_eq!(hash, &hash::salted_blob_hash(&favicon, "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff").unwrap());
-
- let logos = &out.first().manifest.tables["Logos"][0];
- let cell = &logos["FileData"];
- assert_eq!(cell["__blobRef"], json!(hash));
- assert_eq!(cell["__blobKind"], json!("favicon"));
-}
+ let manifest = &manifest_of(&out).manifest;
-#[test]
-fn canonicalize_from_sqlite_nulls_empty_blob_cells() {
- let input = basic_input(vec![CodecTableData {
- name: "Attachments".to_string(),
- records: vec![
- row(&[("Id", json!("a1")), ("Blob", serde_json::Value::Null)]),
- row(&[("Id", json!("a2")), ("Blob", json!({ "__b64": "" }))]),
- ],
- }]);
- let out = canonicalize_from_sqlite(input).unwrap();
- assert_eq!(out.first().blobs.len(), 0);
- assert_eq!(out.first().manifest.tables["Attachments"][0]["Blob"], serde_json::Value::Null);
- assert_eq!(out.first().manifest.tables["Attachments"][1]["Blob"], serde_json::Value::Null);
-}
+ assert!(!manifest.tables.contains_key("Settings") && !manifest.tables.contains_key("__EFMigrationsHistory") && !manifest.tables.contains_key("android_metadata"));
+ assert_eq!(bucket_rows(&out, "settings", "Settings").len(), 1);
+ assert_eq!(manifest.manifest_id, PERSONAL);
+ assert_eq!(manifest.tables["Items"][0]["ManifestId"], json!(PERSONAL), "the stamp every row arrives with travels through");
+ assert_eq!(manifest.tables["Items"][0]["Secret"]["__b64"], json!(b64(&secret)), "a non-blob byte column keeps its inline bytes");
+
+ let blobs = &manifest_of(&out).blobs;
+ assert_eq!(blobs.len(), 2);
+ let logo_hash = hash::salted_blob_hash(&favicon, SALT).unwrap();
+ assert_eq!(blobs[&logo_hash].kind, "favicon");
+ assert_eq!(manifest.tables["Logos"][0]["FileData"], json!({ "__blobRef": logo_hash, "__blobKind": "favicon" }));
+ let attachments: HashMap<&str, &CodecRecord> = manifest.tables["Attachments"].iter().map(|r| (r["Id"].as_str().unwrap(), r)).collect();
+ assert!(attachments["a1"]["Blob"]["__blobRef"].is_string());
+ assert!(attachments["a2"]["Blob"].is_null() && attachments["a3"]["Blob"].is_null(), "an empty or absent blob cell is null");
+
+ let tables = materialized_map(&re);
+ for name in ["Items", "Logos", "Attachments", "Settings"] {
+ assert!(tables.contains_key(name), "missing table {name}");
+ }
+ assert!(!tables.contains_key("android_metadata"), "a bookkeeping table smuggled into a manifest is not re-emitted");
+ assert_eq!(tables["Items"][0]["Secret"]["__b64"], json!(b64(&secret)));
+ assert!(tables["Logos"][0]["FileData"].get("__blobRef").is_some(), "blob cells are refs the platform rebinds from the blob map");
+ assert!(re.overflow.is_empty() && overflow_table_of(&re).is_none(), "a fitting schema splits nothing off");
-#[test]
-fn inline_b64_columns_survive_roundtrip() {
- // Non-blob byte columns (e.g. a TOTP secret) keep their {__b64} marker as-is.
- let secret = vec![1u8, 2, 3, 4, 5];
- let input = basic_input(vec![CodecTableData {
- name: "Items".to_string(),
- records: vec![row(&[("Id", json!("i1")), ("Secret", json!({ "__b64": b64(&secret) }))])],
- }]);
- let out = canonicalize_from_sqlite(input).unwrap();
- let cell = &out.first().manifest.tables["Items"][0]["Secret"];
- assert_eq!(cell["__b64"], json!(b64(&secret)));
-
- let re = materialize_as_sqlite(materialize_input(out.first().manifest.clone(), vec![], out.data_buckets.clone())).unwrap();
- let items = re.tables.iter().find(|t| t.name == "Items").unwrap();
- assert_eq!(items.records[0]["Secret"]["__b64"], json!(b64(&secret)));
+ // The vault DB carries one Manifests row per materialized manifest, which canonicalize consumes as bookkeeping.
+ assert_eq!(tables[MANIFESTS_TABLE], vec![row(&[("Id", json!(PERSONAL)), ("Name", serde_json::Value::Null)])]);
+ assert!(!canonicalize(re.tables.clone()).manifests[0].manifest.tables.contains_key(MANIFESTS_TABLE));
}
#[test]
-fn materialize_as_sqlite_emits_settings_table() {
- // Settings reconstituted from the bucket.
- let input = basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] },
- CodecTableData { name: "Settings".to_string(), records: vec![row(&[("Key", json!("k"))])] },
- ]);
- let out = canonicalize_from_sqlite(input).unwrap();
- let re = materialize_as_sqlite(materialize_input(out.first().manifest.clone(), vec![], out.data_buckets.clone())).unwrap();
- assert!(re.tables.iter().any(|t| t.name == "Settings" && t.records.len() == 1));
- assert!(re.tables.iter().any(|t| t.name == "Items"));
-}
+fn packing_wraps_a_payload_in_a_verified_envelope() {
+ let payload = json!({ "schemaVersion": 1, "tables": { "Items": [] }, "manifestSalt": "abcd" });
+ let packed = pack_payload(&payload.to_string()).unwrap();
+ assert_eq!(&packed[0..2], &[0x1f, 0x8b], "packed output is gzip");
+ assert_eq!(serde_json::from_str::<serde_json::Value>(&unpack_payload(&packed).unwrap()).unwrap(), payload);
-#[test]
-fn materialize_as_sqlite_drops_skip_tables() {
- // A manifest carrying a platform bookkeeping table (android_metadata) must not be re-emitted.
- let mut manifest = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![] }]))
- .unwrap()
- .first()
- .manifest
- .clone();
- manifest
- .tables
- .insert("android_metadata".to_string(), vec![row(&[("locale", json!("en_US"))])]);
- let re = materialize_as_sqlite(materialize_input(manifest, vec![], vec![])).unwrap();
- assert!(!re.tables.iter().any(|t| t.name == "android_metadata"));
+ // Tampered: the original hash over a changed payload.
+ let envelope = json!({ "schemaVersion": 1, "contentHash": hash::content_hash(&json!({ "a": 1 })), "payload": { "a": 2 } });
+ assert!(unpack_payload(&super::compress::gzip(envelope.to_string().as_bytes()).unwrap()).is_err());
+
+ // A format version this build cannot read is reported as such, before the hash is checked.
+ let newer = json!({ "schemaVersion": SCHEMA_VERSION + 1, "contentHash": "x", "payload": {} });
+ assert!(matches!(unpack_versioned_payload(&super::compress::gzip(newer.to_string().as_bytes()).unwrap()).unwrap(), UnpackedPayload::NewerFormat(v) if v == SCHEMA_VERSION + 1));
}
#[test]
-fn full_roundtrip_with_blobs_is_semantically_equal() {
- let favicon = vec![0x01, 0x02, 0x03];
- let attachment = vec![0xaa, 0xbb, 0xcc, 0xdd, 0xee];
- let tables = vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("FolderId", serde_json::Value::Null), ("LogoId", json!("l1"))])] },
- CodecTableData { name: "Logos".to_string(), records: vec![row(&[("Id", json!("l1")), ("FileData", json!({ "__b64": b64(&favicon) }))])] },
- CodecTableData { name: "Attachments".to_string(), records: vec![row(&[("Id", json!("a1")), ("ItemId", json!("i1")), ("Blob", json!({ "__b64": b64(&attachment) }))])] },
- CodecTableData { name: "Settings".to_string(), records: vec![row(&[("Key", json!("theme")), ("Value", json!("dark"))])] },
- ];
- let out = canonicalize_from_sqlite(basic_input(tables)).unwrap();
- assert_eq!(out.first().blobs.len(), 2);
-
- let re = materialize_as_sqlite(materialize_input(out.first().manifest.clone(), vec![], out.data_buckets.clone())).unwrap();
- // Items/Logos/Attachments/Settings all present (skip tables aside).
- for name in ["Items", "Logos", "Attachments", "Settings"] {
- assert!(re.tables.iter().any(|t| t.name == name), "missing table {name}");
+fn salts_and_fingerprints() {
+ let salt = generate_manifest_salt();
+ assert_eq!(salt.len(), 64);
+ assert!(salt.chars().all(|c| c.is_ascii_hexdigit()));
+ for bad in ["", "not-hex", "abc"] {
+ assert!(hash::salted_blob_hash(&[1, 2, 3], bad).is_err(), "a blob hash without a usable salt would break the per-manifest separation");
}
- // Blob cells are refs the platform will rebind from the blob map.
- let logos = re.tables.iter().find(|t| t.name == "Logos").unwrap();
- assert!(logos.records[0]["FileData"].get("__blobRef").is_some());
-}
-#[test]
-fn pack_unpack_roundtrip_verifies_content_hash() {
- let payload = json!({ "schemaVersion": 1, "tables": { "Items": [] }, "manifestSalt": "abcd" });
- let payload_json = serde_json::to_string(&payload).unwrap();
- let packed = pack_payload(&payload_json).unwrap();
- // Packed output is gzip (magic bytes present).
- assert_eq!(&packed[0..2], &[0x1f, 0x8b]);
- let unpacked = unpack_payload(&packed).unwrap();
- let unpacked_val: serde_json::Value = serde_json::from_str(&unpacked).unwrap();
- assert_eq!(unpacked_val, payload);
+ // A content fingerprint ignores key order, whitespace and the volatile `canonicalizedAt`, and nothing else.
+ let a = compute_content_fingerprint(r#"{"canonicalizedAt":"2026-01-01T00:00:00.000Z","schemaVersion":1,"tables":{"Items":[{"Id":"x","Name":"n"}]}}"#);
+ let b = compute_content_fingerprint(r#"{ "tables": { "Items": [ { "Name": "n", "Id": "x" } ] }, "schemaVersion": 1, "canonicalizedAt": "2026-07-25T12:34:56.789Z" }"#);
+ assert_eq!(a, b);
+ assert_eq!(a.len(), 64);
+ assert_ne!(a, compute_content_fingerprint(r#"{"schemaVersion":1,"tables":{"Items":[{"Id":"y","Name":"n"}]}}"#));
}
#[test]
-fn unpack_payload_rejects_tampered_payload() {
- let payload = json!({ "a": 1 });
- let content_hash = hash::content_hash(&payload);
- // Tamper: keep the original hash but change the payload, then pack it.
- let envelope = json!({ "schemaVersion": 1, "contentHash": content_hash, "payload": { "a": 2 } });
- let packed = super::compress::gzip(serde_json::to_string(&envelope).unwrap().as_bytes()).unwrap();
- assert!(unpack_payload(&packed).is_err());
+fn unknown_top_level_manifest_fields_round_trip() {
+ let manifest_json = json!({ "schemaVersion": 1, "manifestSalt": SALT, "canonicalizedAt": "2026-01-01T00:00:00.000Z", "manifestId": "m-1", "tables": { "Items": [] }, "futureField": { "nested": true } });
+ let manifest: Manifest = serde_json::from_value(manifest_json).unwrap();
+ assert!(manifest.extra.contains_key("futureField"));
+ assert_eq!(serde_json::to_value(&manifest).unwrap()["futureField"], json!({ "nested": true }));
}
-#[test]
-fn validate_manifest_catches_broken_fk() {
- let mut manifest = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] },
- CodecTableData { name: "Folders".to_string(), records: vec![] },
- ]))
- .unwrap()
- .first()
- .manifest
- .clone();
- // Canonicalize nulls a dangling FolderId itself, so the broken reference is put in afterwards.
- manifest.tables.get_mut("Items").unwrap()[0].insert("FolderId".to_string(), json!("missing"));
- manifest.tables.entry("Folders".to_string()).or_default();
- let result = validate_manifest(&manifest);
- assert!(!result.ok);
- assert!(result.failed_rules.iter().any(|r| r == "item-folder-fk-broken"));
-}
+/*
+ * Validation and repair.
+ */
#[test]
-fn canonicalize_prunes_references_the_validator_would_refuse() {
- let manifest = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("FolderId", json!("missing"))]), row(&[("Id", json!("i2")), ("FolderId", json!(""))])] },
- CodecTableData { name: "Folders".to_string(), records: vec![] },
- CodecTableData { name: "Tags".to_string(), records: vec![] },
- CodecTableData { name: "ItemTags".to_string(), records: vec![row(&[("ItemId", json!("i1")), ("TagId", json!("gone"))])] },
- CodecTableData { name: "TotpCodes".to_string(), records: vec![row(&[("Id", json!("t1")), ("ItemId", json!("gone"))]), row(&[("Id", json!("t2")), ("ItemId", json!("i1"))])] },
- ]))
- .unwrap()
- .first()
- .manifest
- .clone();
+fn canonicalize_repairs_what_validation_would_refuse() {
+ // Dangling references are nulled or dropped on the way out, so a push is never refused for them; a manifest
+ // handed to validation with them still in place is refused, with a stable rule id per reference.
+ let out = canonicalize(vec![
+ table("Items", vec![row(&[("Id", json!("i1")), ("FolderId", json!("missing"))]), row(&[("Id", json!("i2")), ("FolderId", json!(""))])]),
+ table("Folders", vec![]),
+ table("Tags", vec![]),
+ table("ItemTags", vec![row(&[("ItemId", json!("i1")), ("TagId", json!("gone"))])]),
+ table("TotpCodes", vec![row(&[("Id", json!("t1")), ("ItemId", json!("gone"))]), row(&[("Id", json!("t2")), ("ItemId", json!("i1"))])]),
+ ]);
+ let mut manifest = manifest_of(&out).manifest.clone();
assert!(validate_manifest(&manifest).ok, "{:?}", validate_manifest(&manifest).failed_rules);
assert!(manifest.tables["Items"].iter().all(|item| item["FolderId"].is_null()));
assert!(manifest.tables["ItemTags"].is_empty());
assert_eq!(manifest.tables["TotpCodes"].iter().map(|r| r["Id"].as_str().unwrap()).collect::<Vec<_>>(), vec!["t2"]);
-}
-#[test]
-fn validate_manifest_ok_for_clean_vault() {
- let manifest = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("FolderId", json!("f1"))])] },
- CodecTableData { name: "Folders".to_string(), records: vec![row(&[("Id", json!("f1"))])] },
- ]))
- .unwrap()
- .first()
- .manifest
- .clone();
- assert!(validate_manifest(&manifest).ok);
+ manifest.tables.get_mut("Items").unwrap()[0].insert("FolderId".to_string(), json!("missing"));
+ let result = validate_manifest(&manifest);
+ assert!(!result.ok);
+ assert!(result.failed_rules.contains(&"items-folder-fk-broken".to_string()));
+ assert!(result.message.contains("missing Folders missing"), "{}", result.message);
}
#[test]
-fn generate_manifest_salt_is_64_hex_chars() {
- let salt = generate_manifest_salt();
- assert_eq!(salt.len(), 64);
- assert!(salt.chars().all(|c| c.is_ascii_hexdigit()));
-}
+fn validation_checks_the_header_and_uniqueness() {
+ let mut manifest = canonicalize(vec![table("Items", vec![row(&[("Id", json!("i1"))])])]).manifests.remove(0).manifest;
+ assert!(validate_manifest(&manifest).ok);
-#[test]
-fn forward_compat_unknown_manifest_fields_preserved() {
- let manifest_json = json!({
- "schemaVersion": 1,
- "manifestSalt": "00112233445566778899aabbccddeeff",
- "canonicalizedAt": "2026-01-01T00:00:00.000Z",
- "manifestId": "m-1",
- "tables": { "Items": [] },
- "futureField": { "nested": true }
- })
- .to_string();
- let manifest: Manifest = serde_json::from_str(&manifest_json).unwrap();
- assert!(manifest.extra.contains_key("futureField"));
- let reser = serde_json::to_value(&manifest).unwrap();
- assert_eq!(reser["futureField"], json!({ "nested": true }));
-}
+ manifest.tables.insert("Logos".to_string(), vec![row(&[("Id", json!("logo-a")), ("ManifestId", json!(PERSONAL)), ("Source", json!("github.com"))]), row(&[("Id", json!("logo-b")), ("ManifestId", json!(PERSONAL)), ("Source", json!("GitHub.com"))])]);
+ manifest.tables.get_mut("Items").unwrap().push(row(&[("Id", json!("i1")), ("ManifestId", json!(PERSONAL))]));
+ manifest.manifest_id = String::new();
+ let result = validate_manifest(&manifest);
+ for rule in ["manifestId-missing", "logo-sources-not-unique", "item-ids-not-unique"] {
+ assert!(result.failed_rules.contains(&rule.to_string()), "{rule} missing from {:?}", result.failed_rules);
+ }
-#[test]
-fn logo_id_ignores_manifest_id_casing() {
- let id = scoped_assets::logo_id_for("6bdd3e29-3add-4c3f-8d63-6ada1e74c8c6", LOGO_KIND_FAVICON, "github.com");
- assert_eq!(id, scoped_assets::logo_id_for("6BDD3E29-3ADD-4C3F-8D63-6ADA1E74C8C6", LOGO_KIND_FAVICON, "GitHub.com"));
+ manifest.tables.clear();
+ assert_eq!(validate_manifest(&manifest).failed_rules.last().map(String::as_str), Some("tables-missing"));
}
-#[test]
-fn salted_blob_hash_refuses_a_missing_or_malformed_salt() {
- assert!(hash::salted_blob_hash(&[1, 2, 3], "").is_err());
- assert!(hash::salted_blob_hash(&[1, 2, 3], "not-hex").is_err());
- assert!(hash::salted_blob_hash(&[1, 2, 3], "abc").is_err());
-}
+/*
+ * Logos.
+ */
#[test]
-fn canonicalize_rederives_legacy_logo_ids_and_collapses_duplicate_sources() {
- // Two clients generated distinct random Ids for the same domain; Items point at each. Canonicalize
- // re-derives the row's id against its scope, which is what stops the two from ever drifting apart
- // again and repoints both Items at it.
+fn canonicalize_derives_logo_ids_collapses_duplicate_sources_and_nulls_dead_references() {
+ // Two clients generated distinct random ids for the same domain; items point at each. Canonicalize re-derives the
+ // row's id against its manifest, keeps the row that carries bytes, repoints both items, and nulls a reference no
+ // logo satisfies (matching the FK's ON DELETE SET NULL).
let favicon = vec![0x01, 0x02, 0x03];
- let out = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![
- row(&[("Id", json!("logo-a")), ("Source", json!("github.com")), ("FileData", json!({ "__b64": b64(&favicon) }))]),
- row(&[("Id", json!("logo-b")), ("Source", json!("github.com")), ("FileData", serde_json::Value::Null)]),
- ],
- },
- CodecTableData {
- name: "Items".to_string(),
- records: vec![
- row(&[("Id", json!("i1")), ("LogoId", json!("logo-a"))]),
- row(&[("Id", json!("i2")), ("LogoId", json!("logo-b"))]),
- ],
- },
- ]))
- .unwrap();
-
- let expected_id = json!(scoped_assets::logo_id_for(PERSONAL_MANIFEST, LOGO_KIND_FAVICON, "github.com"));
- let logos = &out.first().manifest.tables["Logos"];
- assert_eq!(logos.len(), 1, "duplicate Source collapsed to one row");
- assert_eq!(logos[0]["Id"], expected_id, "id derived from (personal manifest id, source)");
- assert_eq!(logos[0]["ManifestId"], json!(PERSONAL_MANIFEST), "personal rows are stamped with the personal manifest's own id");
- // The row carrying favicon bytes is the one that survived, not the empty one.
- assert!(logos[0]["FileData"].get("__blobRef").is_some());
-
- let items = &out.first().manifest.tables["Items"];
- for item in items {
- assert_eq!(item["LogoId"], expected_id, "every Item repointed at the derived row");
- }
-
- // No orphan blob: exactly the survivor's favicon is registered.
- assert_eq!(out.first().blobs.len(), 1);
-}
-
-#[test]
-fn canonicalize_dedup_tiebreak_prefers_the_row_with_favicon_bytes() {
- // The derived id is the same either way, so the tiebreak decides which row's *content* survives:
- // the one that actually carries bytes wins regardless of row order.
- let favicon = vec![0x07];
- let out = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![
- row(&[("Id", json!("logo-z")), ("Source", json!("github.com")), ("MimeType", json!("empty"))]),
- row(&[("Id", json!("logo-a")), ("Source", json!("github.com")), ("MimeType", json!("image/png")), ("FileData", json!({ "__b64": b64(&favicon) }))]),
- ],
- },
- CodecTableData {
- name: "Items".to_string(),
- records: vec![row(&[("Id", json!("i1")), ("LogoId", json!("logo-z"))])],
- },
- ]))
- .unwrap();
-
- let logos = &out.first().manifest.tables["Logos"];
+ let out = canonicalize(vec![
+ table("Logos", vec![
+ row(&[("Id", json!("logo-z")), ("Source", json!("github.com")), ("MimeType", json!("empty")), ("FileData", serde_json::Value::Null)]),
+ row(&[("Id", json!("logo-a")), ("Source", json!("github.com")), ("MimeType", json!("image/png")), ("FileData", json!({ "__b64": b64(&favicon) }))]),
+ ]),
+ table("Items", vec![row(&[("Id", json!("i1")), ("LogoId", json!("logo-z"))]), row(&[("Id", json!("i2")), ("LogoId", json!("logo-a"))]), row(&[("Id", json!("i3")), ("LogoId", json!("ghost"))])]),
+ ]);
+ let manifest = &manifest_of(&out).manifest;
+ let expected_id = json!(logo_id_for(PERSONAL, "favicon", "github.com"));
+ let logos = &manifest.tables["Logos"];
assert_eq!(logos.len(), 1, "duplicate Source collapsed to one row");
- assert_eq!(logos[0]["MimeType"], json!("image/png"), "the row with bytes supplies the surviving content");
- assert_eq!(out.first().manifest.tables["Items"][0]["LogoId"], json!(scoped_assets::logo_id_for(PERSONAL_MANIFEST, LOGO_KIND_FAVICON, "github.com")));
-}
-
-#[test]
-fn canonicalize_nulls_dangling_logo_reference() {
- // An Item pointing at a logo Id that doesn't exist anywhere in the vault is nulled, matching
- // FK_Items_Logos_LogoId ON DELETE SET NULL, so materialize's foreign_key_check passes.
- let out = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![row(&[("Id", json!("logo-a")), ("Source", json!("github.com"))])],
- },
- CodecTableData {
- name: "Items".to_string(),
- records: vec![
- row(&[("Id", json!("i1")), ("LogoId", json!("logo-a"))]),
- row(&[("Id", json!("i2")), ("LogoId", json!("ghost"))]),
- ],
- },
- ]))
- .unwrap();
-
- let items = &out.first().manifest.tables["Items"];
- let i1 = items.iter().find(|r| r["Id"] == json!("i1")).unwrap();
- let i2 = items.iter().find(|r| r["Id"] == json!("i2")).unwrap();
- assert_eq!(i1["LogoId"], json!(scoped_assets::logo_id_for(PERSONAL_MANIFEST, LOGO_KIND_FAVICON, "github.com")), "valid reference follows the re-derive");
- assert_eq!(i2["LogoId"], serde_json::Value::Null, "dangling reference nulled");
-}
-
-#[test]
-fn canonicalize_logo_normalization_is_idempotent() {
- // Push stability: canonicalizing already-normalized rows must not change them, or every push would
- // rewrite the manifest and burn a revision for nothing.
- let favicon = vec![0x01, 0x02];
- let first = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![row(&[("Id", json!("legacy-id")), ("Source", json!("github.com")), ("FileData", json!({ "__b64": b64(&favicon) }))])],
- },
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("LogoId", json!("legacy-id"))])] },
- ]))
- .unwrap();
-
- // Feed the normalized rows back in (bytes restored, as the platform read would).
- let mut logo_row = first.first().manifest.tables["Logos"][0].clone();
+ assert_eq!(logos[0]["Id"], expected_id);
+ assert_eq!(logos[0]["ManifestId"], json!(PERSONAL));
+ assert_eq!(logos[0]["MimeType"], json!("image/png"), "the row with bytes supplies the surviving content regardless of row order");
+ assert_eq!(manifest_of(&out).blobs.len(), 1, "exactly the survivor's favicon is registered");
+ let items: HashMap<&str, &CodecRecord> = manifest.tables["Items"].iter().map(|r| (r["Id"].as_str().unwrap(), r)).collect();
+ assert_eq!(items["i1"]["LogoId"], expected_id);
+ assert_eq!(items["i2"]["LogoId"], expected_id);
+ assert_eq!(items["i3"]["LogoId"], serde_json::Value::Null);
+
+ // Push stability: canonicalizing already-normalized rows must not change them, or every push would rewrite the manifest.
+ let mut logo_row = logos[0].clone();
logo_row.insert("FileData".to_string(), json!({ "__b64": b64(&favicon) }));
- let second = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Logos".to_string(), records: vec![logo_row] },
- CodecTableData { name: "Items".to_string(), records: first.first().manifest.tables["Items"].clone() },
- ]))
- .unwrap();
-
- assert_eq!(second.first().manifest.tables["Logos"], first.first().manifest.tables["Logos"]);
- assert_eq!(second.first().manifest.tables["Items"], first.first().manifest.tables["Items"]);
-}
-
-#[test]
-fn validate_manifest_rejects_duplicate_logo_sources() {
- // Guard: if a duplicate-Source manifest ever reaches validation (dedup bypassed), reject it.
- let mut manifest = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![] },
- ]))
- .unwrap()
- .first()
- .manifest
- .clone();
- manifest.tables.insert(
- "Logos".to_string(),
- vec![
- row(&[("Id", json!("logo-a")), ("Source", json!("github.com"))]),
- row(&[("Id", json!("logo-b")), ("Source", json!("github.com"))]),
- ],
- );
- let result = validate_manifest(&manifest);
- assert!(!result.ok);
- assert!(result.failed_rules.iter().any(|r| r == "logo-sources-not-unique"));
-}
-
-/// A minimal local schema map for the overflow tests: a client that knows `Items` and `Settings` (each
-/// with the `ManifestId` every manifest-scoped table carries) plus the `CodecOverflows` carrier table,
-/// and nothing else. It stands in for a client one release behind a writer that has since added a
-/// column or a whole table.
-fn narrow_client_schema() -> std::collections::HashMap<String, Vec<String>> {
- [
- ("Items".to_string(), vec![MANIFEST_ID_COL.to_string(), "Id".to_string(), "Name".to_string()]),
- ("Settings".to_string(), vec![MANIFEST_ID_COL.to_string(), "Key".to_string(), "Value".to_string()]),
- (OVERFLOW_TABLE.to_string(), vec!["Id".to_string(), "Data".to_string()]),
- ]
- .into_iter()
- .collect()
-}
-
-/// The OVERFLOW_TABLE entry from a materialize result (what the platform inserts into the vault DB).
-fn overflow_table_of(re: &MaterializedTables) -> Option<&CodecTableData> {
- re.tables.iter().find(|t| t.name == OVERFLOW_TABLE)
-}
-
-#[test]
-fn materialize_splits_unknown_columns_into_overflow_table_and_canonicalize_remerges() {
- // A newer client wrote Items.AliasEnabled; this client's schema doesn't know it. The column must
- // not reach the Items insert (it would crash), must land in the emitted OVERFLOW_TABLE row, and
- // must reappear on the next canonicalize (which reads that row back like any table) so the push
- // doesn't drop it.
- let out = canonicalize_from_sqlite(basic_input(vec![CodecTableData {
- name: "Items".to_string(),
- records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub")), ("AliasEnabled", json!(true))])],
- }]))
- .unwrap();
-
- let re = materialize_as_sqlite(MaterializeInput { manifests: vec![out.first().manifest.clone()], data_buckets: out.data_buckets.clone(), schema_columns: narrow_client_schema() }).unwrap();
- let items = re.tables.iter().find(|t| t.name == "Items").unwrap();
- assert!(!items.records[0].contains_key("AliasEnabled"), "unknown column filtered out of the insert set");
- assert_eq!(items.records[0]["Name"], json!("GitHub"));
- // Keyed by the row's full (ManifestId, Id) identity so two manifests holding the same Id keep their own overflow.
- let identity = format!("{}\u{1f}{}", PERSONAL_MANIFEST, "i1");
- assert_eq!(re.overflow.columns["Items"][&identity]["AliasEnabled"], json!(true), "diagnostics copy populated");
-
- let overflow_table = overflow_table_of(&re).expect("overflow emitted as a regular table row");
- assert_eq!(overflow_table.records.len(), 1);
- assert_eq!(overflow_table.records[0]["Id"], json!(OVERFLOW_ROW_ID));
-
- // The old client edits Name locally, then pushes: canonicalize reads the overflow row back from
- // the DB (plain SELECT *) and re-attaches the column; the carrier table itself never reaches the manifest.
- let pushed = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub (renamed)"))])] },
- overflow_table.clone(),
- ]))
- .unwrap();
- let item = &pushed.first().manifest.tables["Items"][0];
- assert_eq!(item["Name"], json!("GitHub (renamed)"));
- assert_eq!(item["AliasEnabled"], json!(true), "newer writer's column survives the old client's push");
- assert!(!pushed.first().manifest.tables.contains_key(OVERFLOW_TABLE), "carrier table consumed, never emitted into the manifest");
+ let second = canonicalize(vec![table("Logos", vec![logo_row]), table("Items", manifest.tables["Items"].clone())]);
+ assert_eq!(second.manifests[0].manifest.tables["Logos"], manifest.tables["Logos"]);
+ assert_eq!(second.manifests[0].manifest.tables["Items"], manifest.tables["Items"]);
}
-#[test]
-fn overflow_of_locally_deleted_row_is_dropped_on_canonicalize() {
- // The row carrying the unknown column was deleted locally: its overflow must vanish with it.
- let overflow = CodecOverflow {
- columns: [("Items".to_string(), [("gone".to_string(), row(&[("AliasEnabled", json!(true))]))].into_iter().collect())].into_iter().collect(),
- ..Default::default()
- };
- let out = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("kept"))])] },
- CodecTableData { name: OVERFLOW_TABLE.to_string(), records: overflow.to_table_records() },
- ]))
- .unwrap();
- assert!(!out.first().manifest.tables["Items"][0].contains_key("AliasEnabled"));
-}
+/*
+ * Forward compatibility: the overflow a narrow schema carries between pull and push.
+ */
#[test]
-fn materialize_splits_unknown_tables_into_overflow_and_canonicalize_reemits() {
- // A newer client added a whole table (manifest-level) and a whole bucket table. Neither exists
- // in this client's schema; both must round-trip through the overflow row back to their original place.
- let mut out = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }])).unwrap();
+fn unknown_columns_and_tables_round_trip_through_the_overflow_row() {
+ // A newer client wrote a column and two tables this client's schema does not know. None of it may reach the
+ // insert set (it would crash), all of it lands in the emitted OVERFLOW_TABLE row, and all of it reappears on the
+ // next canonicalize (which reads that row back like any table), on both push paths.
+ let mut out = canonicalize(vec![table("Items", vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub")), ("AliasEnabled", json!(true))])])]);
out.manifests[0].manifest.tables.insert("NewTable".to_string(), vec![row(&[("Id", json!("n1")), ("Data", json!("x"))])]);
- let settings_bucket = out.data_buckets.iter_mut().find(|b| b.category == "settings").expect("settings bucket");
- settings_bucket.tables.insert("Preferences".to_string(), vec![row(&[("Key", json!("p1"))])]);
-
- let re = materialize_as_sqlite(MaterializeInput { manifests: vec![out.first().manifest.clone()], data_buckets: out.data_buckets.clone(), schema_columns: narrow_client_schema() }).unwrap();
- assert!(!re.tables.iter().any(|t| t.name == "NewTable" || t.name == "Preferences"), "unknown tables never reach the insert set");
- assert_eq!(re.overflow.tables["NewTable"].len(), 1);
+ out.data_buckets.iter_mut().find(|b| b.category == "settings").unwrap().tables.insert("Preferences".to_string(), vec![row(&[("Key", json!("p1"))])]);
+
+ let re = materialize_as_sqlite(MaterializeInput { manifests: vec![out.manifests[0].manifest.clone()], data_buckets: out.data_buckets.clone(), schema_columns: narrow_client_schema() }).unwrap();
+ let tables = materialized_map(&re);
+ assert!(!tables["Items"][0].contains_key("AliasEnabled") && !tables.contains_key("NewTable") && !tables.contains_key("Preferences"), "unknown data never reaches the insert set");
+ assert_eq!(tables["Items"][0]["Name"], json!("GitHub"));
+ // Overflow columns are keyed by the row's full (ManifestId, Id) identity so two manifests holding the same Id keep their own.
+ assert_eq!(re.overflow.columns["Items"][&format!("{}\u{1f}i1", PERSONAL)]["AliasEnabled"], json!(true));
+ assert_eq!(re.overflow.tables["NewTable"][0]["ManifestId"], json!(PERSONAL), "unknown tables are stamped with the manifest they arrived in");
assert_eq!(re.overflow.bucket_tables["settings"]["Preferences"].len(), 1);
- // Materialize stamps every row with the manifest it arrived in, unknown tables included. The stamp is what routes the row back to its own manifest on the next canonicalize.
- assert_eq!(re.overflow.tables["NewTable"][0]["ManifestId"], json!(PERSONAL_MANIFEST));
- assert_eq!(re.overflow.bucket_tables["settings"]["Preferences"][0]["ManifestId"], json!(PERSONAL_MANIFEST));
let overflow_table = overflow_table_of(&re).expect("overflow emitted as a regular table row").clone();
+ assert_eq!(overflow_table.records[0]["Id"], json!(OVERFLOW_ROW_ID));
- let pushed = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] },
- overflow_table.clone(),
- ]))
- .unwrap();
- assert_eq!(pushed.first().manifest.tables["NewTable"].len(), 1, "unknown manifest table re-emitted into the manifest its stamp names");
- assert_eq!(bucket_rows(&pushed, "settings", "Preferences").len(), 1, "unknown bucket table re-emitted into its category");
-
- // Bucket-only push path: extract_buckets consumes the overflow row read alongside the category's tables.
- let buckets = extract_buckets(
- "settings".to_string(),
- vec![PERSONAL_MANIFEST.to_string()],
- [
- ("Settings".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Key", json!("k")), ("Value", json!("v"))])]),
- (OVERFLOW_TABLE.to_string(), overflow_table.records),
- ]
- .into_iter()
- .collect(),
- )
- .unwrap();
- let bucket = buckets.into_iter().find(|b| b.manifest_id == PERSONAL_MANIFEST).expect("a bucket for the manifest that was asked for");
- assert_eq!(bucket.tables["Preferences"].len(), 1);
- assert_eq!(bucket.tables["Settings"].len(), 1);
- assert!(!bucket.tables.contains_key(OVERFLOW_TABLE), "carrier table consumed, never emitted into the bucket");
-}
+ // The old client renames the item and pushes.
+ let pushed = canonicalize(vec![table("Items", vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub (renamed)"))])]), overflow_table.clone()]);
+ let item = &pushed.manifests[0].manifest.tables["Items"][0];
+ assert_eq!((item["Name"].as_str(), item["AliasEnabled"].as_bool()), (Some("GitHub (renamed)"), Some(true)), "the newer writer's column survives the old client's push");
+ assert_eq!(pushed.manifests[0].manifest.tables["NewTable"].len(), 1);
+ assert_eq!(bucket_rows(&pushed, "settings", "Preferences").len(), 1);
+ assert!(!pushed.manifests[0].manifest.tables.contains_key(OVERFLOW_TABLE), "carrier table consumed, never emitted into the manifest");
-#[test]
-fn extract_buckets_remerges_overflow_columns() {
- // A newer client added a column to Settings; a bucket-only push from this client must keep it.
- // Overflow columns are keyed by the row's identity, which for Settings is (ManifestId, Key).
- let identity = format!("{}\u{1f}{}", PERSONAL_MANIFEST, "theme");
- let overflow = CodecOverflow {
- columns: [("Settings".to_string(), [(identity, row(&[("SyncScope", json!("device"))]))].into_iter().collect())].into_iter().collect(),
- ..Default::default()
- };
- let buckets = extract_buckets(
- "settings".to_string(),
- vec![PERSONAL_MANIFEST.to_string()],
- [
- (
- "Settings".to_string(),
- vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Key", json!("theme")), ("Value", json!("dark"))])],
- ),
- (OVERFLOW_TABLE.to_string(), overflow.to_table_records()),
- ]
- .into_iter()
- .collect(),
- )
- .unwrap();
- let bucket = &buckets[0];
- assert_eq!(bucket.tables["Settings"][0]["SyncScope"], json!("device"));
- assert_eq!(bucket.tables["Settings"][0]["Value"], json!("dark"));
+ // Bucket-only push: extract_buckets consumes the overflow row read alongside the category's tables.
+ let buckets = extract_buckets("settings".to_string(), vec![PERSONAL.to_string()], [("Settings".to_string(), vec![row(&[("ManifestId", json!(PERSONAL)), ("Key", json!("k")), ("Value", json!("v"))])]), (OVERFLOW_TABLE.to_string(), overflow_table.records)].into_iter().collect()).unwrap();
+ assert_eq!(buckets[0].tables["Preferences"].len(), 1);
+ assert_eq!(buckets[0].tables["Settings"].len(), 1);
+ assert!(!buckets[0].tables.contains_key(OVERFLOW_TABLE));
}
#[test]
-fn item_stats_route_into_the_stats_bucket_of_the_manifest_that_owns_the_item() {
- // Test stats bucket routing.
- let shared_manifest = "00000000-0000-0000-0000-0000000shared";
- let input = CanonicalizeInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![
- row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Id", json!("i-personal"))]),
- row(&[("ManifestId", json!(shared_manifest)), ("Id", json!("i-shared"))]),
- ],
- },
- CodecTableData {
- name: "ItemStats".to_string(),
- records: vec![
- row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Id", json!("i-personal")), ("UseCount", json!(3))]),
- row(&[("ManifestId", json!(shared_manifest)), ("Id", json!("i-shared")), ("UseCount", json!(7))]),
- ],
- },
- ],
- canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(),
- manifests: vec![
- ManifestSpec {
- manifest_id: PERSONAL_MANIFEST.to_string(),
- manifest_salt: "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff".to_string(),
- name: None,
- },
- ManifestSpec {
- manifest_id: shared_manifest.to_string(),
- manifest_salt: "ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766554433221100".to_string(),
- name: Some("Shared".to_string()),
- },
- ],
- stamp_unstamped_into: None,
- };
-
- let out = canonicalize_from_sqlite(input).unwrap();
-
- // Never in a manifest: stats are bucketed precisely so a use does not rewrite the content manifest.
- for manifest in &out.manifests {
- assert!(!manifest.manifest.tables.contains_key("ItemStats"), "ItemStats must never be serialized into a manifest");
- }
-
- let stats_bucket = |manifest_id: &str| -> &[CodecRecord] {
- out.data_buckets
- .iter()
- .find(|b| b.category == "stats" && b.manifest_id == manifest_id)
- .and_then(|b| b.tables.get("ItemStats"))
- .map(Vec::as_slice)
- .unwrap_or(&[])
- };
-
- let personal = stats_bucket(PERSONAL_MANIFEST);
- assert_eq!(personal.len(), 1);
- assert_eq!(personal[0]["UseCount"], json!(3));
-
- let shared = stats_bucket(shared_manifest);
- assert_eq!(shared.len(), 1);
- assert_eq!(shared[0]["UseCount"], json!(7));
-}
+fn overflow_columns_follow_their_row_and_die_with_it() {
+ // A column stashed for a settings row re-attaches by the row's (ManifestId, Key) identity on a bucket-only push;
+ // one stashed for a row that was deleted locally vanishes with the row.
+ let settings_overflow = CodecOverflow { columns: [("Settings".to_string(), [(format!("{}\u{1f}theme", PERSONAL), row(&[("SyncScope", json!("device"))]))].into_iter().collect())].into_iter().collect(), ..Default::default() };
+ let buckets = extract_buckets("settings".to_string(), vec![PERSONAL.to_string()], [("Settings".to_string(), vec![row(&[("ManifestId", json!(PERSONAL)), ("Key", json!("theme")), ("Value", json!("dark"))])]), (OVERFLOW_TABLE.to_string(), settings_overflow.to_table_records())].into_iter().collect()).unwrap();
+ assert_eq!(buckets[0].tables["Settings"][0]["SyncScope"], json!("device"));
-#[test]
-fn materialize_with_a_fitting_schema_splits_nothing_off() {
- // A schema that knows every column the manifest carries: rows pass through untouched and no
- // overflow row is emitted. This is the normal case, and the guard on the `fitting_schema` helper
- // the rest of these tests lean on.
- let out = canonicalize_from_sqlite(basic_input(vec![CodecTableData {
- name: "Items".to_string(),
- records: vec![row(&[("Id", json!("i1")), ("AliasEnabled", json!(true))])],
- }]))
- .unwrap();
- let re = materialize_as_sqlite(materialize_input(out.first().manifest.clone(), vec![], out.data_buckets.clone())).unwrap();
- let items = re.tables.iter().find(|t| t.name == "Items").unwrap();
- assert_eq!(items.records[0]["AliasEnabled"], json!(true));
- assert!(re.overflow.is_empty());
- assert!(overflow_table_of(&re).is_none(), "no overflow row emitted when there is nothing to carry");
+ let gone = CodecOverflow { columns: [("Items".to_string(), [("gone".to_string(), row(&[("AliasEnabled", json!(true))]))].into_iter().collect())].into_iter().collect(), ..Default::default() };
+ let out = canonicalize(vec![table("Items", vec![row(&[("Id", json!("kept"))])]), table(OVERFLOW_TABLE, gone.to_table_records())]);
+ assert!(!out.manifests[0].manifest.tables["Items"][0].contains_key("AliasEnabled"));
}
#[test]
-fn materialize_drops_overflow_table_smuggled_into_a_manifest() {
- // Defense: OVERFLOW_TABLE is local-only bookkeeping. A manifest that somehow carries one (corrupt
- // or malicious) must not pass through, it would collide with the row materialize emits itself.
- let mut out = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("AliasEnabled", json!(true))])] }])).unwrap();
+fn an_overflow_row_smuggled_into_a_manifest_is_dropped() {
+ // OVERFLOW_TABLE is local-only bookkeeping; a manifest carrying one would collide with the row materialize emits.
+ let mut out = canonicalize(vec![table("Items", vec![row(&[("Id", json!("i1")), ("AliasEnabled", json!(true))])])]);
out.manifests[0].manifest.tables.insert(OVERFLOW_TABLE.to_string(), vec![row(&[("Id", json!("smuggled")), ("Data", json!("{}"))])]);
-
- let re = materialize_as_sqlite(MaterializeInput { manifests: vec![out.first().manifest.clone()], data_buckets: out.data_buckets.clone(), schema_columns: narrow_client_schema() }).unwrap();
+ let re = materialize_as_sqlite(MaterializeInput { manifests: vec![out.manifests[0].manifest.clone()], data_buckets: out.data_buckets.clone(), schema_columns: narrow_client_schema() }).unwrap();
let overflow_table = overflow_table_of(&re).expect("legitimate overflow row still emitted");
assert_eq!(overflow_table.records.len(), 1);
- assert_eq!(overflow_table.records[0]["Id"], json!(OVERFLOW_ROW_ID), "smuggled row dropped, only the codec's own row remains");
-}
-
-#[test]
-fn content_fingerprint_ignores_key_order_and_whitespace() {
- let a = compute_content_fingerprint(r#"{"schemaVersion":1,"tables":{"Items":[{"Id":"x","Name":"n"}]}}"#);
- let b = compute_content_fingerprint(r#"{ "tables": { "Items": [ { "Name": "n", "Id": "x" } ] }, "schemaVersion": 1 }"#);
- assert_eq!(a, b);
- assert_eq!(a.len(), 64);
-}
-
-#[test]
-fn content_fingerprint_excludes_canonicalized_at() {
- let a = compute_content_fingerprint(r#"{"canonicalizedAt":"2026-01-01T00:00:00.000Z","tables":{}}"#);
- let b = compute_content_fingerprint(r#"{"canonicalizedAt":"2026-07-25T12:34:56.789Z","tables":{}}"#);
- let c = compute_content_fingerprint(r#"{"tables":{}}"#);
- assert_eq!(a, b);
- assert_eq!(a, c);
-}
-
-#[test]
-fn content_fingerprint_detects_content_change() {
- let a = compute_content_fingerprint(r#"{"tables":{"Items":[{"Id":"x"}]}}"#);
- let b = compute_content_fingerprint(r#"{"tables":{"Items":[{"Id":"y"}]}}"#);
- assert_ne!(a, b);
-}
-
-#[test]
-fn canonicalize_requires_manifest_id() {
- let mut input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![] }]);
- input.manifests[0].manifest_id = String::new();
- assert!(canonicalize_from_sqlite(input).is_err());
-}
-
-#[test]
-fn canonicalize_keeps_the_manifest_stamp_every_row_arrives_with() {
- // The client stamps at write time and the codec carries that stamp through unchanged: the manifest
- // names itself and every row of every manifest-scoped table still names the manifest it came in with.
- let out = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] },
- CodecTableData { name: "Folders".to_string(), records: vec![row(&[("Id", json!("f1"))])] },
- ]))
- .unwrap();
- assert_eq!(out.first().manifest.manifest_id, PERSONAL_MANIFEST);
- assert_eq!(out.first().manifest.tables["Items"][0]["ManifestId"], json!(PERSONAL_MANIFEST));
- assert_eq!(out.first().manifest.tables["Folders"][0]["ManifestId"], json!(PERSONAL_MANIFEST));
-}
-
-#[test]
-fn materialize_emits_manifests_bookkeeping_table() {
- // The vault DB carries one Manifests row per materialized manifest so app queries can name and group by them.
- let out = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }])).unwrap();
- let re = materialize_as_sqlite(materialize_input(out.first().manifest.clone(), vec![], out.data_buckets.clone())).unwrap();
- let manifests = re.tables.iter().find(|t| t.name == "Manifests").expect("Manifests bookkeeping table emitted");
- assert_eq!(manifests.records.len(), 1);
- assert_eq!(manifests.records[0]["Id"], json!(PERSONAL_MANIFEST));
- assert_eq!(manifests.records[0]["Name"], serde_json::Value::Null);
- assert!(!manifests.records[0].contains_key("IsPersonal"), "the vault records which manifests it holds, not which one is the client's");
-
- // Feeding the materialized tables back into canonicalize must consume the bookkeeping table (skip-table).
- let pushed = canonicalize_from_sqlite(basic_input(re.tables)).unwrap();
- assert!(!pushed.first().manifest.tables.contains_key("Manifests"));
-}
-
-#[test]
-fn validate_manifest_requires_manifest_id() {
- let mut manifest = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![] }])).unwrap().first().manifest.clone();
- assert!(validate_manifest(&manifest).ok);
- manifest.manifest_id = String::new();
- let result = validate_manifest(&manifest);
- assert!(!result.ok);
- assert!(result.failed_rules.iter().any(|r| r == "manifestId-missing"));
+ assert_eq!(overflow_table.records[0]["Id"], json!(OVERFLOW_ROW_ID), "only the codec's own row remains");
}
-// ---------------------------------------------------------------------------
-// Derived ids + wire shape (see normalize.rs)
-// ---------------------------------------------------------------------------
-
-/// The FieldValues rows of the first canonicalized manifest, keyed by their Value marker.
-fn field_values_by_value(out: &CanonicalizedVault) -> HashMap<String, CodecRecord> {
- out.first().manifest.tables["FieldValues"]
- .iter()
- .map(|r| (r["Value"].as_str().unwrap().to_string(), r.clone()))
- .collect()
-}
+/*
+ * Derived ids: single-value field values, histories and tag links travel without an id and get it back on materialize.
+ */
#[test]
fn canonicalize_strips_derived_ids_and_renumbers_multi_value_rows() {
- let out = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i-1"))])] },
- CodecTableData {
- name: "FieldDefinitions".to_string(),
- records: vec![row(&[("Id", json!("fd-multi")), ("IsMultiValue", json!(1))]), row(&[("Id", json!("fd-single")), ("IsMultiValue", json!(0))])],
- },
- CodecTableData {
- name: "FieldValues".to_string(),
- records: vec![
- row(&[("Id", json!("fv-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("Value", json!("me"))]),
- row(&[("Id", json!("u-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("Value", json!("https://a.example"))]),
- row(&[("Id", json!("u-2")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("Value", json!("https://b.example"))]),
- row(&[("Id", json!("c-1")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-multi")), ("Value", json!("multi-a"))]),
- row(&[("Id", json!("c-2")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-multi")), ("Value", json!("multi-b"))]),
- row(&[("Id", json!("c-3")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-single")), ("Value", json!("single-c"))]),
- ],
- },
- CodecTableData { name: "Tags".to_string(), records: vec![row(&[("Id", json!("t-1"))])] },
- CodecTableData { name: "ItemTags".to_string(), records: vec![row(&[("Id", json!("it-legacy")), ("ItemId", json!("i-1")), ("TagId", json!("t-1"))])] },
- ]))
- .unwrap();
-
- let fv = field_values_by_value(&out);
- assert!(fv["me"].get("Id").is_none(), "a single-value system field's id never reaches the wire");
- assert_eq!(fv["me"]["ValueIndex"], json!(0));
- assert!(fv["single-c"].get("Id").is_none(), "a single-value custom field's id never reaches the wire");
- assert_eq!(fv["https://a.example"]["Id"], json!("u-1"), "a multi-value system field's value owns its id");
- assert_eq!(fv["https://b.example"]["Id"], json!("u-2"));
- assert_eq!((fv["https://a.example"]["ValueIndex"].as_i64().unwrap(), fv["https://b.example"]["ValueIndex"].as_i64().unwrap()), (0, 1), "values are renumbered in read order");
- assert_eq!(fv["multi-a"]["Id"], json!("c-1"), "a multi-value custom field's value owns its id");
- assert_eq!(fv["multi-b"]["Id"], json!("c-2"));
-
- let item_tags = &out.first().manifest.tables["ItemTags"];
- assert_eq!(item_tags.len(), 1);
- assert!(item_tags[0].get("Id").is_none(), "ItemTags carries no id at all");
-}
-
-#[test]
-fn canonicalize_strips_field_history_ids() {
- let out = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i-1"))])] },
- CodecTableData {
- name: "FieldHistories".to_string(),
- records: vec![
- row(&[("Id", json!("fh-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-01-01 10:00:00.000")), ("ValueSnapshot", json!("old-pass"))]),
- row(&[("Id", json!("fh-2")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-02-01 10:00:00.000")), ("ValueSnapshot", json!("newer-pass"))]),
- ],
- },
- ]))
- .unwrap();
-
- let rows = &out.first().manifest.tables["FieldHistories"];
- assert_eq!(rows.len(), 2, "distinct ChangedAt values are distinct history rows");
- assert!(rows.iter().all(|r| r.get("Id").is_none()), "every history row derives its id; none reaches the wire");
-}
-
-#[test]
-fn canonicalize_collapses_duplicate_single_value_rows_to_the_newest() {
- // Two rows for one single-value field (two local writers racing before their next sync) must
- // converge instead of reaching the wire as colliding rows.
- let out = canonicalize_from_sqlite(basic_input(vec![
- CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i-1"))])] },
- CodecTableData {
- name: "FieldValues".to_string(),
- records: vec![
- row(&[("Id", json!("fv-old")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("Value", json!("stale")), ("UpdatedAt", json!("2024-01-01 00:00:00.000"))]),
- row(&[("Id", json!("fv-new")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("Value", json!("fresh")), ("UpdatedAt", json!("2024-06-01 00:00:00.000"))]),
- ],
- },
- ]))
- .unwrap();
-
- let rows = &out.first().manifest.tables["FieldValues"];
- assert_eq!(rows.len(), 1);
- assert_eq!(rows[0]["Value"], json!("fresh"));
-}
-
-#[test]
-fn materialize_derives_missing_field_value_ids() {
+ let out = canonicalize(vec![
+ table("Items", vec![row(&[("Id", json!("i-1"))])]),
+ table("FieldDefinitions", vec![row(&[("Id", json!("fd-multi")), ("IsMultiValue", json!(1))]), row(&[("Id", json!("fd-single")), ("IsMultiValue", json!(0))])]),
+ table("FieldValues", vec![
+ row(&[("Id", json!("fv-old")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("Value", json!("stale")), ("UpdatedAt", json!("2024-01-01 00:00:00.000"))]),
+ row(&[("Id", json!("fv-new")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("Value", json!("fresh")), ("UpdatedAt", json!("2024-06-01 00:00:00.000"))]),
+ row(&[("Id", json!("u-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("Value", json!("https://a.example"))]),
+ row(&[("Id", json!("u-2")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("Value", json!("https://b.example"))]),
+ row(&[("Id", json!("c-1")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-multi")), ("Value", json!("multi-a"))]),
+ row(&[("Id", json!("c-2")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-multi")), ("Value", json!("multi-b"))]),
+ row(&[("Id", json!("c-3")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-single")), ("Value", json!("single-c"))]),
+ ]),
+ table("FieldHistories", vec![
+ row(&[("Id", json!("fh-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-01-01 10:00:00.000")), ("ValueSnapshot", json!("old-pass"))]),
+ row(&[("Id", json!("fh-2")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-02-01 10:00:00.000")), ("ValueSnapshot", json!("newer-pass"))]),
+ ]),
+ table("Tags", vec![row(&[("Id", json!("t-1"))])]),
+ table("ItemTags", vec![row(&[("Id", json!("it-legacy")), ("ItemId", json!("i-1")), ("TagId", json!("t-1"))])]),
+ ]);
+ let manifest = &manifest_of(&out).manifest;
+ let fv: HashMap<&str, &CodecRecord> = manifest.tables["FieldValues"].iter().map(|r| (r["Value"].as_str().unwrap(), r)).collect();
+ assert!(!fv.contains_key("stale"), "two rows for one single-value field collapse to the newest");
+ assert!(fv["fresh"].get("Id").is_none() && fv["single-c"].get("Id").is_none(), "a single-value field's id never reaches the wire");
+ assert_eq!(fv["fresh"]["ValueIndex"], json!(0));
+ assert_eq!((fv["https://a.example"]["Id"].as_str(), fv["https://b.example"]["Id"].as_str()), (Some("u-1"), Some("u-2")), "a multi-value field's values own their ids");
+ assert_eq!((fv["https://a.example"]["ValueIndex"].as_i64(), fv["https://b.example"]["ValueIndex"].as_i64()), (Some(0), Some(1)), "renumbered in read order");
+ assert_eq!((fv["multi-a"]["Id"].as_str(), fv["multi-b"]["Id"].as_str()), (Some("c-1"), Some("c-2")));
+ assert_eq!(manifest.tables["FieldHistories"].len(), 2, "distinct ChangedAt values are distinct history rows");
+ assert!(manifest.tables["FieldHistories"].iter().all(|r| r.get("Id").is_none()), "every history row derives its id");
+ assert_eq!(manifest.tables["ItemTags"].len(), 1);
+ assert!(!manifest.tables["ItemTags"][0].contains_key("Id"), "ItemTags carries no id at all");
+}
+
+#[test]
+fn materialize_derives_the_missing_ids_back() {
let manifest = Manifest {
schema_version: SCHEMA_VERSION,
- manifest_salt: "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff".to_string(),
+ manifest_salt: SALT.to_string(),
canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(),
- manifest_id: PERSONAL_MANIFEST.to_string(),
+ manifest_id: PERSONAL.to_string(),
name: None,
tables: [
- ("Items".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Id", json!("i-1"))])]),
- (
- "FieldValues".to_string(),
- vec![
- // Single-value wire shape: no id, explicit position; the id is derived here.
- row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("ValueIndex", json!(0)), ("Value", json!("me"))]),
- // Multi-value: the owned id rides along untouched.
- row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Id", json!("u-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("ValueIndex", json!(0)), ("Value", json!("https://a.example"))]),
- ],
- ),
- ("Tags".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Id", json!("t-1"))])]),
- ("ItemTags".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("ItemId", json!("i-1")), ("TagId", json!("t-1"))])]),
- ("FieldHistories".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-01-01 10:00:00.000")), ("ValueSnapshot", json!("old-pass"))])]),
+ ("Items".to_string(), vec![stamped(PERSONAL, &[("Id", json!("i-1"))])]),
+ ("FieldValues".to_string(), vec![
+ stamped(PERSONAL, &[("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("ValueIndex", json!(0)), ("Value", json!("me"))]),
+ stamped(PERSONAL, &[("Id", json!("u-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("ValueIndex", json!(0)), ("Value", json!("https://a.example"))]),
+ ]),
+ ("Tags".to_string(), vec![stamped(PERSONAL, &[("Id", json!("t-1"))])]),
+ ("ItemTags".to_string(), vec![stamped(PERSONAL, &[("ItemId", json!("i-1")), ("TagId", json!("t-1"))])]),
+ ("FieldHistories".to_string(), vec![stamped(PERSONAL, &[("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-01-01 10:00:00.000")), ("ValueSnapshot", json!("old-pass"))])]),
]
.into_iter()
.collect(),
extra: HashMap::new(),
};
-
- let schema: HashMap<String, Vec<String>> = [
- ("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string()]),
- ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string()]),
- ("Tags".to_string(), vec!["ManifestId".to_string(), "Id".to_string()]),
- ("ItemTags".to_string(), vec!["ManifestId".to_string(), "ItemId".to_string(), "TagId".to_string()]),
- ("FieldHistories".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ChangedAt".to_string(), "ValueSnapshot".to_string()]),
- (MANIFESTS_TABLE.to_string(), vec!["Id".to_string(), "Name".to_string()]),
- ]
- .into_iter()
- .collect();
-
+ // The schema knows the id columns the wire omits, so the derived ids fit instead of landing in the overflow.
+ let mut schema = test_support::fitting_schema([&manifest], &[]);
+ for table in ["FieldValues", "FieldHistories"] {
+ schema.get_mut(table).unwrap().push("Id".to_string());
+ }
let out = materialize_as_sqlite(MaterializeInput { manifests: vec![manifest], data_buckets: vec![], schema_columns: schema }).unwrap();
- let tables: HashMap<&str, &Vec<CodecRecord>> = out.tables.iter().map(|t| (t.name.as_str(), &t.records)).collect();
-
+ let tables = materialized_map(&out);
let fv: HashMap<&str, &CodecRecord> = tables["FieldValues"].iter().map(|r| (r["Value"].as_str().unwrap(), r)).collect();
- let expected_username_id = super::normalize::field_value_id_for(PERSONAL_MANIFEST, "i-1", "login.username", "", 0);
- assert_eq!(fv["me"]["Id"], json!(expected_username_id), "materialize derives the id");
+ assert_eq!(fv["me"]["Id"], json!(super::normalize::field_value_id_for(PERSONAL, "i-1", "login.username", "", 0)));
assert_eq!(fv["https://a.example"]["Id"], json!("u-1"), "an owned multi-value id is kept as-is");
-
assert_eq!(tables["ItemTags"].len(), 1, "the id-less join row inserts as-is");
-
- let history = &tables["FieldHistories"][0];
- let expected_history_id = super::normalize::field_history_id_for(PERSONAL_MANIFEST, "i-1", "login.password", "", "2026-01-01 10:00:00.000");
- assert_eq!(history["Id"], json!(expected_history_id), "materialize derives the history id");
-
+ assert_eq!(tables["FieldHistories"][0]["Id"], json!(super::normalize::field_history_id_for(PERSONAL, "i-1", "login.password", "", "2026-01-01 10:00:00.000")));
assert!(out.overflow.is_empty());
}
diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rs
index 75a8ff08c..b1e266760 100644
--- a/core/rust/src/vault_codec/types.rs
+++ b/core/rust/src/vault_codec/types.rs
@@ -5,7 +5,7 @@
//! core/models/src/vault/VaultTableRegistry.ts; this module adds the codec-owned accessors on top.
use crate::vault_model::names::ID_COL;
-use crate::vault_model::{ids_equal, BlobColumn, BLOB_COLUMNS, BUCKET_TABLES, MANIFEST_ID_COL, OVERFLOW_TABLE, PERSONAL_TABLES, SKIP_TABLES, SYNCABLE_TABLES, UNSTAMPED_SCOPE_SENTINEL};
+use crate::vault_model::{ids_equal, BlobColumn, BLOB_COLUMNS, BUCKET_TABLES, MANIFEST_ID_COL, OVERFLOW_TABLE, PERSONAL_TABLES, SKIP_TABLES, SYNCABLE_TABLES, UNSTAMPED_MANIFEST_ID};
/// Manifest / data bucket format version; bump only for a change older clients cannot carry without the built-in overflow.
pub const SCHEMA_VERSION: u32 = 1;
@@ -47,10 +47,6 @@ pub(crate) fn is_guid(text: &str) -> bool {
})
}
-// ---------------------------------------------------------------------------
-// Accessor methods
-// ---------------------------------------------------------------------------
-
/// The blob column of a table, if it owns one.
pub fn blob_spec_for(table_name: &str) -> Option<&'static BlobColumn> {
BLOB_COLUMNS.iter().find(|spec| spec.table == table_name)
@@ -89,10 +85,10 @@ pub fn tables_for_category(category: &str) -> Vec<&'static str> {
}
/// True when a `ManifestId` value names no manifest.
-pub fn is_unstamped_scope(scope: Option<&str>) -> bool {
- match scope {
+pub fn is_unstamped_manifest_id(manifest_id: Option<&str>) -> bool {
+ match manifest_id {
None => true,
- Some(value) => value.is_empty() || ids_equal(value, UNSTAMPED_SCOPE_SENTINEL),
+ Some(value) => value.is_empty() || ids_equal(value, UNSTAMPED_MANIFEST_ID),
}
}
diff --git a/core/rust/src/vault_codec/validate.rs b/core/rust/src/vault_codec/validate.rs
index 44e14a0f1..74923f02f 100644
--- a/core/rust/src/vault_codec/validate.rs
+++ b/core/rust/src/vault_codec/validate.rs
@@ -1,18 +1,19 @@
-//! Validate structure for the manifest-v1 format.
-//!
-//! We refuse obviously-broken manifests before they are encrypted/uploaded.
+//! Structural validation of a manifest or data bucket before it is encrypted and uploaded.
+
+use std::collections::HashSet;
use serde::{Deserialize, Serialize};
+use super::integrity;
use super::manifest::{DataBucket, Manifest};
use super::row::{logo_kind, rows_of, str_col};
use super::types::{is_bucketed_table, is_manifest_scoped, is_personal_table};
-use crate::vault_model::names::{
- ENCRYPTION_KEYS_TABLE, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_VALUES_TABLE, FOLDERS_TABLE, FOLDER_ID_COL, ID_COL,
- ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, LOGOS_TABLE, SOURCE_COL, TAGS_TABLE, TAG_ID_COL,
-};
+use crate::vault_model::names::{ENCRYPTION_KEYS_TABLE, FOLDERS_TABLE, ID_COL, ITEMS_TABLE, LOGOS_TABLE, SOURCE_COL};
use crate::vault_model::MANIFEST_ID_COL;
+/// The shortest `manifestSalt` (hex characters) a manifest may carry.
+const MIN_MANIFEST_SALT_LEN: usize = 32;
+
/// Outcome of a structural validation run.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -24,199 +25,85 @@ pub struct ValidationResult {
pub message: String,
}
-/// Structurally validate a fresh manifest before it is encrypted/uploaded.
+impl ValidationResult {
+ fn from_failures(failed: Vec<(String, String)>) -> Self {
+ let (failed_rules, explain): (Vec<String>, Vec<String>) = failed.into_iter().unzip();
+ Self { ok: failed_rules.is_empty(), message: explain.into_iter().filter(|why| !why.is_empty()).collect::<Vec<_>>().join("; "), failed_rules }
+ }
+}
+
+/// Structurally validate a manifest: header, table classes, manifest stamps, referential integrity and uniqueness.
pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {
- let mut failed: Vec<String> = Vec::new();
- let mut explain: Vec<String> = Vec::new();
+ let mut failed: Vec<(String, String)> = Vec::new();
+ let mut fail = |rule: &str, why: String| failed.push((rule.to_string(), why));
if manifest.schema_version < 1 {
- failed.push("schemaVersion-missing-or-too-low".to_string());
+ fail("schemaVersion-missing-or-too-low", String::new());
}
- if manifest.manifest_salt.len() < 32 {
- failed.push("manifestSalt-missing-or-short".to_string());
+ if manifest.manifest_salt.len() < MIN_MANIFEST_SALT_LEN {
+ fail("manifestSalt-missing-or-short", String::new());
}
- // Every manifest carries its own id: rows inside are stamped with it.
if manifest.manifest_id.is_empty() {
- failed.push("manifestId-missing".to_string());
+ fail("manifestId-missing", String::new());
}
-
if manifest.tables.is_empty() {
- failed.push("tables-missing".to_string());
- return ValidationResult {
- ok: false,
- failed_rules: failed,
- message: "Manifest has no tables, refusing upload.".to_string(),
- };
+ fail("tables-missing", "Manifest has no tables, refusing upload.".to_string());
+ return ValidationResult::from_failures(failed);
}
- /*
- * Two independent rules, reported apart because they fail for different reasons: a bucketed table
- * syncs as its own resource (one bucket per manifest) so no manifest may carry it, and a
- * personal-only table may not leave the user's own vault at all.
- */
- for name in manifest.tables.keys().filter(|name| !manifest.tables[*name].is_empty()) {
- if is_bucketed_table(name) {
- failed.push("manifest-carries-bucketed-table".to_string());
- explain.push(format!("Manifest carries bucketed table {}, which belongs in its data bucket", name));
- break;
- }
+ // A bucketed table syncs as its own resource, so no manifest may carry it; a personal table may not leave the user's own vault.
+ let carried = |predicate: fn(&str) -> bool| manifest.tables.iter().find(|(name, rows)| !rows.is_empty() && predicate(name)).map(|(name, _)| name.clone());
+ if let Some(name) = carried(is_bucketed_table) {
+ fail("manifest-carries-bucketed-table", format!("Manifest carries bucketed table {}, which belongs in its data bucket", name));
}
- for name in manifest.tables.keys().filter(|name| !manifest.tables[*name].is_empty()) {
- if is_personal_table(name) {
- failed.push("manifest-carries-personal-table".to_string());
- explain.push(format!("Manifest carries personal table {}", name));
- break;
- }
+ if let Some(name) = carried(is_personal_table) {
+ fail("manifest-carries-personal-table", format!("Manifest carries personal table {}", name));
}
- /*
- * Every EncryptionKeys row must be stamped with the manifest's own id.
- */
- let expected_scope = Some(manifest.manifest_id.as_str());
- if rows_of(&manifest.tables, ENCRYPTION_KEYS_TABLE).iter().any(|r| str_col(r, MANIFEST_ID_COL) != expected_scope) {
- failed.push("encryption-keys-scope-mismatch".to_string());
- explain.push("EncryptionKeys carries rows stamped for another manifest".to_string());
+ // Every row claims the manifest's own id: key material by rule, content rows because canonicalize restamps them.
+ let expected_manifest_id = Some(manifest.manifest_id.as_str());
+ if rows_of(&manifest.tables, ENCRYPTION_KEYS_TABLE).iter().any(|r| str_col(r, MANIFEST_ID_COL) != expected_manifest_id) {
+ fail("encryption-keys-manifest-mismatch", "EncryptionKeys carries rows stamped for another manifest".to_string());
}
-
- // Items and Folders are restamped by canonicalize: a mismatched stamp here means a codec bug.
- for name in [ITEMS_TABLE, FOLDERS_TABLE] {
- if rows_of(&manifest.tables, name).iter().any(|r| str_col(r, MANIFEST_ID_COL) != expected_scope) {
- failed.push("content-scope-mismatch".to_string());
- explain.push(format!("{} carries rows stamped for another manifest", name));
- break;
- }
+ if let Some(name) = [ITEMS_TABLE, FOLDERS_TABLE].into_iter().find(|name| rows_of(&manifest.tables, name).iter().any(|r| str_col(r, MANIFEST_ID_COL) != expected_manifest_id)) {
+ fail("content-manifest-mismatch", format!("{} carries rows stamped for another manifest", name));
}
-
- let items = rows_of(&manifest.tables, ITEMS_TABLE);
- let folders = rows_of(&manifest.tables, FOLDERS_TABLE);
- let tags = rows_of(&manifest.tables, TAGS_TABLE);
- let item_tags = rows_of(&manifest.tables, ITEM_TAGS_TABLE);
- let field_values = rows_of(&manifest.tables, FIELD_VALUES_TABLE);
- let field_defs = rows_of(&manifest.tables, FIELD_DEFINITIONS_TABLE);
-
- let item_ids: std::collections::HashSet<&str> = items.iter().filter_map(|i| str_col(i, ID_COL)).collect();
- let folder_ids: std::collections::HashSet<&str> = folders.iter().filter_map(|f| str_col(f, ID_COL)).collect();
- let tag_ids: std::collections::HashSet<&str> = tags.iter().filter_map(|t| str_col(t, ID_COL)).collect();
- let field_def_ids: std::collections::HashSet<&str> = field_defs.iter().filter_map(|f| str_col(f, ID_COL)).collect();
-
- // Referential integrity.
- for item in items {
- if let Some(folder_id) = str_col(item, FOLDER_ID_COL) {
- if !folder_ids.contains(folder_id) {
- failed.push("item-folder-fk-broken".to_string());
- explain.push(format!("Item {} references missing folder {}", str_col(item, ID_COL).unwrap_or(""), folder_id));
- break;
- }
- }
+ if rows_of(&manifest.tables, LOGOS_TABLE).iter().any(|l| str_col(l, MANIFEST_ID_COL) != expected_manifest_id) {
+ fail("logo-manifest-mismatch", "Logos carry a ManifestId that is not this manifest's own id".to_string());
}
- for it in item_tags {
- if let Some(item_id) = str_col(it, ITEM_ID_COL) {
- if !item_ids.contains(item_id) {
- failed.push("itemtag-item-fk-broken".to_string());
- explain.push(format!("ItemTag {} references missing item {}", str_col(it, ID_COL).unwrap_or(""), item_id));
- break;
- }
- }
- }
- for it in item_tags {
- if let Some(tag_id) = str_col(it, TAG_ID_COL) {
- if !tag_ids.contains(tag_id) {
- failed.push("itemtag-tag-fk-broken".to_string());
- explain.push(format!("ItemTag {} references missing tag {}", str_col(it, ID_COL).unwrap_or(""), tag_id));
- break;
- }
- }
+ for (rule, why) in integrity::violations(&manifest.tables) {
+ fail(&rule, why);
}
- for fv in field_values {
- if let Some(item_id) = str_col(fv, ITEM_ID_COL) {
- if !item_ids.contains(item_id) {
- failed.push("fieldvalue-item-fk-broken".to_string());
- explain.push(format!("FieldValue {} references missing item {}", str_col(fv, ID_COL).unwrap_or(""), item_id));
- break;
- }
+ for (table, rule) in [(ITEMS_TABLE, "item-ids-not-unique"), (FOLDERS_TABLE, "folder-ids-not-unique")] {
+ let rows = rows_of(&manifest.tables, table);
+ if rows.iter().filter_map(|r| str_col(r, ID_COL)).collect::<HashSet<_>>().len() != rows.len() {
+ fail(rule, String::new());
}
}
- for fv in field_values {
- if let Some(field_def_id) = str_col(fv, FIELD_DEFINITION_ID_COL) {
- if !field_def_ids.contains(field_def_id) {
- failed.push("fieldvalue-fielddef-fk-broken".to_string());
- break;
- }
- }
- }
-
- // Uniqueness.
- if item_ids.len() != items.len() {
- failed.push("item-ids-not-unique".to_string());
- }
- if folder_ids.len() != folders.len() {
- failed.push("folder-ids-not-unique".to_string());
- }
-
- /*
- * A logo belongs to exactly one manifest: (ManifestId, Kind, Source) must be UNIQUE in the
- * client schema.
- */
+ // A logo belongs to exactly one manifest: (ManifestId, Kind, Source) is UNIQUE in the client schema.
let logos = rows_of(&manifest.tables, LOGOS_TABLE);
- let logo_keys: std::collections::HashSet<(String, String)> = logos
- .iter()
- .filter_map(|l| Some((logo_kind(l), str_col(l, SOURCE_COL)?.to_lowercase())))
- .collect();
- let logos_with_source = logos.iter().filter(|l| str_col(l, SOURCE_COL).is_some()).count();
- if logo_keys.len() != logos_with_source {
- failed.push("logo-sources-not-unique".to_string());
- }
-
- // Every logo in a manifest must claim that manifest's scope, otherwise the row would materialize
- // into the wrong uniqueness bucket and could collide with the reader's own rows.
- if logos.iter().any(|l| str_col(l, MANIFEST_ID_COL) != expected_scope) {
- failed.push("logo-scope-mismatch".to_string());
- explain.push("Logos carry a ManifestId that is not this manifest's own id".to_string());
+ let logo_keys: HashSet<(String, String)> = logos.iter().filter_map(|l| Some((logo_kind(l), str_col(l, SOURCE_COL)?.to_lowercase()))).collect();
+ if logo_keys.len() != logos.iter().filter(|l| str_col(l, SOURCE_COL).is_some()).count() {
+ fail("logo-sources-not-unique", String::new());
}
- ValidationResult {
- ok: failed.is_empty(),
- failed_rules: failed,
- message: explain.join("; "),
- }
+ ValidationResult::from_failures(failed)
}
-/// Validate a data bucket before upload.
+/// Validate a data bucket before upload: addressed to a manifest, every row claiming that manifest.
pub fn validate_data_bucket(bucket: &DataBucket) -> ValidationResult {
- let mut failed: Vec<String> = Vec::new();
- let mut explain: Vec<String> = Vec::new();
-
+ let mut failed: Vec<(String, String)> = Vec::new();
if bucket.schema_version < 1 {
- return ValidationResult {
- ok: false,
- failed_rules: vec!["dataBucket-schemaVersion-missing".to_string()],
- message: "Data bucket missing schemaVersion".to_string(),
- };
+ return ValidationResult::from_failures(vec![("dataBucket-schemaVersion-missing".to_string(), "Data bucket missing schemaVersion".to_string())]);
}
-
- // A bucket is addressed by the manifest that owns it: without the id there is nothing to write to.
if bucket.manifest_id.is_empty() {
- failed.push("dataBucket-manifestId-missing".to_string());
- explain.push("Data bucket names no manifest".to_string());
- }
-
- let expected_scope = Some(bucket.manifest_id.as_str());
- for (name, rows) in &bucket.tables {
- if !is_manifest_scoped(name) {
- continue;
- }
- if rows.iter().any(|row| str_col(row, MANIFEST_ID_COL) != expected_scope) {
- failed.push("dataBucket-scope-mismatch".to_string());
- explain.push(format!("{} carries rows stamped for another manifest", name));
- break;
- }
+ failed.push(("dataBucket-manifestId-missing".to_string(), "Data bucket names no manifest".to_string()));
}
-
- ValidationResult {
- ok: failed.is_empty(),
- failed_rules: failed,
- message: explain.join("; "),
+ let expected_manifest_id = Some(bucket.manifest_id.as_str());
+ if let Some((name, _)) = bucket.tables.iter().find(|(name, rows)| is_manifest_scoped(name) && rows.iter().any(|row| str_col(row, MANIFEST_ID_COL) != expected_manifest_id)) {
+ failed.push(("dataBucket-manifest-mismatch".to_string(), format!("{} carries rows stamped for another manifest", name)));
}
+ ValidationResult::from_failures(failed)
}
diff --git a/core/rust/src/vault_merge/item_deletes.rs b/core/rust/src/vault_merge/item_deletes.rs
index 3a87cb32d..daab0497b 100644
--- a/core/rust/src/vault_merge/item_deletes.rs
+++ b/core/rust/src/vault_merge/item_deletes.rs
@@ -14,7 +14,7 @@ use chrono::{DateTime, Utc};
use super::get_key;
use crate::common::timestamp::updated_at;
-use crate::vault_codec::row::is_deleted;
+use crate::vault_codec::row::{is_deleted, rows_of};
use crate::vault_codec::{is_bucketed_table, CodecRecord};
use crate::vault_model::names::{ID_COL, ITEMS_TABLE};
use crate::vault_model::{MANIFEST_ID_COL, SYNCABLE_TABLES};
@@ -34,8 +34,7 @@ pub(super) fn resolve_item_deletes(base: &mut Tables, incoming: &mut Tables) {
/// The items `deleting` tombstoned that `other` still holds live and touched late enough for `outlives` to hold.
fn lost_deletes(deleting: &Tables, other: &Tables, outlives: impl Fn(Option<DateTime<Utc>>, Option<DateTime<Utc>>) -> bool) -> HashSet<String> {
let items = |tables: &Tables, deleted: bool| -> HashMap<String, Option<DateTime<Utc>>> {
- let rows = tables.get(ITEMS_TABLE).map(Vec::as_slice).unwrap_or(&[]);
- rows.iter().filter(|row| is_deleted(row) == deleted).map(|row| (item_key(row, ID_COL), updated_at(row))).collect()
+ rows_of(tables, ITEMS_TABLE).iter().filter(|row| is_deleted(row) == deleted).map(|row| (item_key(row, ID_COL), updated_at(row))).collect()
};
let tombstones = items(deleting, true);
let mut live = items(other, false);
@@ -47,7 +46,7 @@ fn lost_deletes(deleting: &Tables, other: &Tables, outlives: impl Fn(Option<Date
// What the surviving side last did to the item: its own row, or any row hanging off it. Bucketed
// tables stay out, since a usage counter ticking is not an edit.
for child in SYNCABLE_TABLES.iter().filter(|table| table.item_child && !is_bucketed_table(table.name)) {
- for row in other.get(child.name).map(Vec::as_slice).unwrap_or(&[]) {
+ for row in rows_of(other, child.name) {
if let Some(activity) = live.get_mut(&item_key(row, child.item_ref_column())) {
*activity = (*activity).max(updated_at(row));
}
diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rs
index 89e24e37b..e7659ba88 100644
--- a/core/rust/src/vault_merge/mod.rs
+++ b/core/rust/src/vault_merge/mod.rs
@@ -1,8 +1,7 @@
//! Vault merge: Last-Write-Wins over manifest JSON + data buckets, rows in, rows out.
//!
-//! This is the merge for the manifest-v1 storage format (used since 0.31.0+). It runs one layer above any concrete
-//! materialization (SQLite or otherwise): both sides arrive in canonical form and the output is
-//! the merged canonical form, which the platform then materializes once.
+//! Both sides arrive in canonical form and the output is the merged canonical form, which the platform then
+//! materializes once.
//!
//! Each manifest is merged independently (the server manifest set is the universe), so a broken
//! manifest can never affect another manifest's rows, and blob columns carry `__blobRef` markers
@@ -21,28 +20,27 @@ mod item_deletes;
#[cfg(test)]
mod tests;
-/// Statistics about what was merged.
-#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+/// How the rows of a merge were decided. The base is the server's side, the incoming side the local vault.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
-#[cfg_attr(feature = "uniffi", derive(uniffi::Record))]
pub struct MergeStats {
- pub tables_processed: u32,
- pub records_from_local: u32,
- pub records_from_server: u32,
- pub records_created_locally: u32,
- pub conflicts: u32,
- pub records_inserted: u32,
+ /// Rows both sides hold where the base row stood (older or same-aged incoming row).
+ pub base_kept: u32,
+ /// Rows both sides hold where the newer incoming row replaced the base row.
+ pub incoming_won: u32,
+ /// Rows only the base holds.
+ pub base_only: u32,
+ /// Rows only the incoming side holds (made offline).
+ pub incoming_only: u32,
}
impl MergeStats {
/// Add another manifest's counters to these, for a whole-vault total.
pub fn add(&mut self, other: &MergeStats) {
- self.tables_processed += other.tables_processed;
- self.records_from_local += other.records_from_local;
- self.records_from_server += other.records_from_server;
- self.records_created_locally += other.records_created_locally;
- self.conflicts += other.conflicts;
- self.records_inserted += other.records_inserted;
+ self.base_kept += other.base_kept;
+ self.incoming_won += other.incoming_won;
+ self.base_only += other.base_only;
+ self.incoming_only += other.incoming_only;
}
}
@@ -161,10 +159,7 @@ fn merge_manifest_pair(
let base_rows = base_entry.unwrap_or_default();
let incoming_rows = incoming_tables.remove(&name).unwrap_or_default();
let merged_rows = match SYNCABLE_TABLES.iter().find(|t| t.name == name) {
- Some(config) => {
- stats.tables_processed += 1;
- merge_rows(config, base_rows, incoming_rows, schema_columns, &mut stats)
- }
+ Some(config) => merge_rows(config, base_rows, incoming_rows, schema_columns, &mut stats),
// Not a syncable table (a skip table, or one from a newer writer): the base wins as-is.
None => base_rows,
};
@@ -177,12 +172,15 @@ fn merge_manifest_pair(
}
}
- // A union of concurrently added multi-value rows can leave two rows at the same ValueIndex, and a
- // delete that won can leave the other side's children under a tombstoned item; re-normalizing the
- // output renumbers the first and drops the second, so a merged manifest is normalized like any other.
+ // A delete that won can leave the other side's children under a tombstoned item, and a union of concurrently
+ // added multi-value rows can leave two rows at the same ValueIndex: repair and normalize the output like any
+ // canonicalized manifest.
+ crate::vault_codec::integrity::repair(&mut merged);
crate::vault_codec::normalize::normalize_row_shapes(&mut merged);
- let mut buckets = unknown_server_buckets;
+ // One bucket per category: the merged rows of the tables this build buckets, plus the server's tables of that
+ // category it does not know. A merged bucket keeps the server's extra top-level keys.
+ let mut buckets: Vec<DataBucket> = Vec::new();
for category in bucket_categories() {
let mut bucket_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();
for table in tables_for_category(category) {
@@ -190,12 +188,15 @@ fn merge_manifest_pair(
bucket_tables.insert(table.to_string(), rows);
}
}
+ for unknown in unknown_server_buckets.iter().filter(|bucket| bucket.category == category) {
+ bucket_tables.extend(unknown.tables.clone());
+ }
if !bucket_tables.is_empty() {
- // A merged bucket keeps the server's extra top-level keys.
let extra = server_bucket_extras.get(category).cloned().unwrap_or_default();
buckets.push(DataBucket { extra, ..DataBucket::new(manifest_id.clone(), category.to_string(), bucket_tables) });
}
}
+ buckets.extend(unknown_server_buckets.into_iter().filter(|bucket| !bucket_categories().contains(&bucket.category.as_str())));
let manifest = Manifest { tables: merged, ..server };
CanonicalManifestMerge { manifest_id, manifest, buckets, stats }
@@ -225,11 +226,10 @@ fn split_known_buckets(buckets: Vec<DataBucket>) -> (HashMap<String, Vec<CodecRe
/// A contentless server manifest's local counterpart passes through whole: every row is an
/// offline-kept row.
fn pass_through(local: Manifest, local_buckets: Vec<DataBucket>) -> CanonicalManifestMerge {
- let mut stats = MergeStats::default();
let row_count = |tables: &HashMap<String, Vec<CodecRecord>>| tables.values().map(|rows| rows.len() as u32).sum::<u32>();
- stats.records_inserted = row_count(&local.tables) + local_buckets.iter().map(|b| row_count(&b.tables)).sum::<u32>();
+ let stats = MergeStats { incoming_only: row_count(&local.tables) + local_buckets.iter().map(|b| row_count(&b.tables)).sum::<u32>(), ..MergeStats::default() };
- CanonicalManifestMerge { manifest_id: local.manifest_id.clone(), manifest: local.clone(), buckets: local_buckets, stats }
+ CanonicalManifestMerge { manifest_id: local.manifest_id.clone(), manifest: local, buckets: local_buckets, stats }
}
/// LWW one table, rows out: base rows in order (replaced where the incoming row is strictly
@@ -267,18 +267,17 @@ fn merge_rows(
let (incoming_ts, base_ts) = (updated_at(incoming), updated_at(&base_record));
match (incoming_ts, base_ts) {
(Some(i_ts), Some(b_ts)) if i_ts > b_ts => {
- stats.conflicts += 1;
- stats.records_from_server += 1;
+ stats.incoming_won += 1;
merged.push(overlay_winner(incoming.clone(), &base_record, &identity_columns, known_columns.as_ref()));
}
_ => {
- stats.records_from_local += 1;
+ stats.base_kept += 1;
merged.push(base_record);
}
}
}
None => {
- stats.records_created_locally += 1;
+ stats.base_only += 1;
merged.push(base_record);
}
}
@@ -286,7 +285,7 @@ fn merge_rows(
for record in &incoming_rows {
if let Some(winner) = incoming_map.remove(&get_key(record, match_columns)) {
- stats.records_inserted += 1;
+ stats.incoming_only += 1;
merged.push(winner.clone());
}
}
@@ -294,7 +293,7 @@ fn merge_rows(
merged
}
-/// A winning incoming row.
+/// The winning incoming row, with the base's identity columns and the columns this build's schema does not know.
fn overlay_winner(mut winner: CodecRecord, base: &CodecRecord, identity_columns: &[&str], known_columns: Option<&HashSet<&str>>) -> CodecRecord {
for (column, value) in base {
let base_wins = identity_columns.contains(&column.as_str()) || known_columns.is_some_and(|known| !known.contains(column.as_str()));
@@ -311,5 +310,5 @@ fn overlay_winner(mut winner: CodecRecord, base: &CodecRecord, identity_columns:
/// case-insensitively (see [`identity_part`]), everything else exactly as spelled.
fn get_key(record: &CodecRecord, columns: &[&str]) -> String {
let parts: Vec<String> = columns.iter().map(|column| record.get(*column).filter(|v| !v.is_null()).map(identity_part).unwrap_or_default()).collect();
- parts.join(":")
+ parts.join("\u{1f}")
}
diff --git a/core/rust/src/vault_merge/tests.rs b/core/rust/src/vault_merge/tests.rs
index 6b02d41f1..4b83a18c1 100644
--- a/core/rust/src/vault_merge/tests.rs
+++ b/core/rust/src/vault_merge/tests.rs
@@ -1,719 +1,380 @@
-//! Tests for the canonical merge.
+//! The canonical merge: last write wins per row, an item deleted as a unit, manifests merged independently.
use super::*;
+use crate::vault_codec::test_support::{restamp, row, PERSONAL_MANIFEST_ID as PERSONAL, SHARED_MANIFEST_ID as SHARED};
use serde_json::json;
-const PERSONAL: &str = "m-personal";
-const SHARED: &str = "m-shared";
const SALT: &str = "0123456789abcdef0123456789abcdef";
+const T_OLD: &str = "2024-01-01T00:00:00Z";
+const T_MID: &str = "2024-01-05T00:00:00Z";
+const T_NEW: &str = "2024-01-09T00:00:00Z";
-fn manifest(manifest_id: &str, tables: HashMap<String, Vec<CodecRecord>>) -> Manifest {
- Manifest {
- schema_version: 1,
- manifest_salt: SALT.to_string(),
- canonicalized_at: "2024-01-01T00:00:00Z".to_string(),
- manifest_id: manifest_id.to_string(),
- name: None,
- tables,
- extra: HashMap::new(),
- }
+fn manifest(manifest_id: &str, tables: &[(&str, Vec<CodecRecord>)]) -> Manifest {
+ Manifest { schema_version: 1, manifest_salt: SALT.to_string(), canonicalized_at: T_OLD.to_string(), manifest_id: manifest_id.to_string(), name: None, tables: tables.iter().map(|(name, rows)| (name.to_string(), rows.clone())).collect(), extra: HashMap::new() }
}
-fn item(manifest_id: &str, id: &str, name: &str, updated_at: &str) -> CodecRecord {
- [
- ("ManifestId".to_string(), json!(manifest_id)),
- ("Id".to_string(), json!(id)),
- ("Name".to_string(), json!(name)),
- ("UpdatedAt".to_string(), json!(updated_at)),
- ]
- .into_iter()
- .collect()
+/// A row in `PERSONAL` with the given columns plus an `UpdatedAt`.
+fn at(updated_at: &str, pairs: &[(&str, serde_json::Value)]) -> CodecRecord {
+ let mut r = row(pairs);
+ r.insert("ManifestId".to_string(), json!(PERSONAL));
+ r.insert("UpdatedAt".to_string(), json!(updated_at));
+ r
+}
+
+fn item(id: &str, name: &str, updated_at: &str) -> CodecRecord {
+ at(updated_at, &[("Id", json!(id)), ("Name", json!(name)), ("IsDeleted", json!(0))])
+}
+
+fn tombstone(id: &str, updated_at: &str) -> CodecRecord {
+ at(updated_at, &[("Id", json!(id)), ("Name", json!("item")), ("IsDeleted", json!(1))])
}
-fn items_manifest(manifest_id: &str, rows: Vec<CodecRecord>) -> Manifest {
- manifest(manifest_id, [("Items".to_string(), rows)].into_iter().collect())
+/// A single-value `FieldValues` row of `item-1` in wire shape (no id).
+fn field(key: &str, value: &str, deleted: bool, updated_at: &str) -> CodecRecord {
+ at(updated_at, &[("ItemId", json!("item-1")), ("FieldKey", json!(key)), ("ValueIndex", json!(0)), ("Value", json!(value)), ("IsDeleted", json!(deleted as i32))])
+}
+
+/// A `login.url` row: multi-value, so it owns its id on the wire.
+fn url(id: &str, index: i64, value: &str, updated_at: &str) -> CodecRecord {
+ at(updated_at, &[("Id", json!(id)), ("ItemId", json!("item-1")), ("FieldKey", json!("login.url")), ("ValueIndex", json!(index)), ("Value", json!(value))])
}
fn schema() -> HashMap<String, Vec<String>> {
+ let columns = |names: &[&str]| names.iter().map(|c| c.to_string()).collect::<Vec<_>>();
[
- ("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Name".to_string(), "IsDeleted".to_string(), "UpdatedAt".to_string()]),
- ("Settings".to_string(), vec!["ManifestId".to_string(), "Key".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),
- ("ItemStats".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "LastUsedAt".to_string(), "UpdatedAt".to_string()]),
- ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string(), "IsDeleted".to_string(), "UpdatedAt".to_string()]),
- ("ItemTags".to_string(), vec!["ManifestId".to_string(), "ItemId".to_string(), "TagId".to_string(), "UpdatedAt".to_string()]),
- ("Attachments".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Blob".to_string(), "UpdatedAt".to_string()]),
+ ("Items".to_string(), columns(&["ManifestId", "Id", "Name", "IsDeleted", "DeletedAt", "UpdatedAt"])),
+ ("TotpCodes".to_string(), columns(&["ManifestId", "Id", "ItemId", "SecretKey", "IsDeleted", "UpdatedAt"])),
+ ("Settings".to_string(), columns(&["ManifestId", "Key", "Value", "UpdatedAt"])),
+ ("ItemStats".to_string(), columns(&["ManifestId", "Id", "LastUsedAt", "UpdatedAt"])),
+ ("FieldValues".to_string(), columns(&["ManifestId", "Id", "ItemId", "FieldKey", "FieldDefinitionId", "ValueIndex", "Value", "IsDeleted", "UpdatedAt"])),
+ ("ItemTags".to_string(), columns(&["ManifestId", "ItemId", "TagId", "UpdatedAt"])),
+ ("Attachments".to_string(), columns(&["ManifestId", "Id", "Blob", "UpdatedAt"])),
]
.into_iter()
.collect()
}
-fn merge_single(server_rows: Vec<CodecRecord>, local_rows: Vec<CodecRecord>) -> CanonicalManifestMerge {
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![items_manifest(PERSONAL, server_rows)],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![items_manifest(PERSONAL, local_rows)],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
- assert_eq!(output.manifests.len(), 1);
- output.manifests.into_iter().next().unwrap()
+fn merge(server: Vec<Manifest>, local: Vec<Manifest>) -> CanonicalMergeOutput {
+ merge_with_buckets(server, vec![], local, vec![])
}
-fn names_of(manifest: &Manifest) -> Vec<String> {
- manifest.tables.get("Items").map(|rows| rows.iter().map(|r| r["Name"].as_str().unwrap().to_string()).collect()).unwrap_or_default()
+fn merge_with_buckets(server: Vec<Manifest>, server_buckets: Vec<DataBucket>, local: Vec<Manifest>, local_buckets: Vec<DataBucket>) -> CanonicalMergeOutput {
+ merge_canonical(CanonicalMergeInput { server_manifests: server, server_buckets, contentless_server_manifest_ids: vec![], local_manifests: local, local_buckets, schema_columns: schema() }).unwrap()
}
-#[test]
-fn local_newer_wins_server_newer_and_tie_keep_base() {
- let merged = merge_single(
- vec![
- item(PERSONAL, "local-wins", "server-old", "2024-01-01T00:00:00Z"),
- item(PERSONAL, "server-wins", "server-new", "2024-01-09T00:00:00Z"),
- item(PERSONAL, "tie", "server-tie", "2024-01-05T00:00:00Z"),
- ],
- vec![
- item(PERSONAL, "local-wins", "local-new", "2024-01-09T00:00:00Z"),
- item(PERSONAL, "server-wins", "local-old", "2024-01-01T00:00:00Z"),
- item(PERSONAL, "tie", "local-tie", "2024-01-05T00:00:00Z"),
- ],
- );
-
- assert_eq!(names_of(&merged.manifest), vec!["local-new", "server-new", "server-tie"], "ties and newer base rows keep the server version");
- assert_eq!(merged.stats.conflicts, 1);
- assert_eq!(merged.stats.records_from_server, 1);
- assert_eq!(merged.stats.records_from_local, 2);
+/// Merge one table of the personal manifest, rows out.
+fn merge_table(table: &str, server_rows: Vec<CodecRecord>, local_rows: Vec<CodecRecord>) -> (Vec<CodecRecord>, MergeStats) {
+ let output = merge(vec![manifest(PERSONAL, &[(table, server_rows)])], vec![manifest(PERSONAL, &[(table, local_rows)])]);
+ assert_eq!(output.manifests.len(), 1);
+ let merged = output.manifests.into_iter().next().unwrap();
+ (merged.manifest.tables[table].clone(), merged.stats)
}
-#[test]
-fn the_same_id_in_two_spellings_is_one_row() {
- // GUIDs are case-insensitive and should be stored/compared as such.
- let merged = merge_single(
- vec![item(PERSONAL, "9d3f7a2c-1b4e-4f80-8a11-2c3d4e5f6a7b", "server-old", "2024-01-01T00:00:00Z")],
- vec![item(PERSONAL, "9D3F7A2C-1B4E-4F80-8A11-2C3D4E5F6A7B", "local-new", "2024-01-09T00:00:00Z")],
- );
-
- assert_eq!(names_of(&merged.manifest), vec!["local-new"], "one row, the newer one");
+fn column<'a>(rows: &'a [CodecRecord], name: &str) -> Vec<&'a str> {
+ rows.iter().filter_map(|r| r[name].as_str()).collect()
}
-#[test]
-fn one_sided_rows_are_kept_on_both_sides() {
- let merged = merge_single(
- vec![item(PERSONAL, "server-only", "on-server", "2024-01-01T00:00:00Z")],
- vec![item(PERSONAL, "local-only", "made-offline", "2024-01-02T00:00:00Z")],
- );
-
- assert_eq!(names_of(&merged.manifest), vec!["on-server", "made-offline"], "base rows first, then offline-created rows");
- assert_eq!(merged.stats.records_created_locally, 1);
- assert_eq!(merged.stats.records_inserted, 1);
+fn sorted(mut values: Vec<&str>) -> Vec<&str> {
+ values.sort();
+ values
}
-#[test]
-fn manifests_merge_independently_same_id_never_crosses() {
- // The structural guarantee of the canonical merge: manifests are the partition, so the
- // same row id in two manifests can never interact, whatever the timestamps say.
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![
- items_manifest(PERSONAL, vec![item(PERSONAL, "same-id", "mine", "2024-01-01T00:00:00Z")]),
- items_manifest(SHARED, vec![item(SHARED, "same-id", "theirs-old", "2024-01-01T00:00:00Z")]),
- ],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![
- items_manifest(PERSONAL, vec![item(PERSONAL, "same-id", "mine", "2024-01-01T00:00:00Z")]),
- items_manifest(SHARED, vec![item(SHARED, "same-id", "theirs-new", "2024-01-09T00:00:00Z")]),
- ],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
+/*
+ * Rows.
+ */
- let by_id: HashMap<&str, &CanonicalManifestMerge> = output.manifests.iter().map(|m| (m.manifest_id.as_str(), m)).collect();
- assert_eq!(names_of(&by_id[PERSONAL].manifest), vec!["mine"], "the personal row is untouched by the shared manifest's newer row");
- assert_eq!(names_of(&by_id[SHARED].manifest), vec!["theirs-new"]);
- assert_eq!(by_id[PERSONAL].stats.conflicts, 0);
+#[test]
+fn the_newer_row_wins_the_server_keeps_ties_and_one_sided_rows_are_kept() {
+ let (rows, stats) = merge_table(
+ "Items",
+ vec![item("local-wins", "server-old", T_OLD), item("server-wins", "server-new", T_NEW), item("tie", "server-tie", T_MID), item("server-only", "on-server", T_OLD)],
+ vec![item("local-wins", "local-new", T_NEW), item("server-wins", "local-old", T_OLD), item("tie", "local-tie", T_MID), item("local-only", "made-offline", T_OLD)],
+ );
+ assert_eq!(column(&rows, "Name"), vec!["local-new", "server-new", "server-tie", "on-server", "made-offline"], "base rows first in base order, then the offline-made rows");
+ assert_eq!(stats, MergeStats { base_kept: 2, incoming_won: 1, base_only: 1, incoming_only: 1 });
}
#[test]
-fn field_values_match_semantically_and_carry_no_surrogate_id() {
- // A single-value FieldValue arrives in wire shape (no Id, position 0) and matches on its
- // natural key: both sides converge to one row holding the winning value, still id-less.
- let field_value = |value: &str, updated_at: &str| -> CodecRecord {
- [
- ("ManifestId".to_string(), json!(PERSONAL)),
- ("ItemId".to_string(), json!("item-1")),
- ("FieldKey".to_string(), json!("username")),
- ("ValueIndex".to_string(), json!(0)),
- ("Value".to_string(), json!(value)),
- ("UpdatedAt".to_string(), json!(updated_at)),
- ]
- .into_iter()
- .collect()
- };
-
- let rows = merge_field_values(vec![field_value("old", "2024-01-01T00:00:00Z")], vec![field_value("new", "2024-01-09T00:00:00Z")]);
- assert_eq!(rows.len(), 1);
- assert!(rows[0].get("Id").is_none(), "a single-value row's id is derived at materialize, never carried on the wire");
- assert_eq!(rows[0]["Value"], json!("new"), "and the row carries the winning value");
-}
+fn ids_compare_case_insensitively() {
+ // iOS and the extension generate uppercase GUIDs; the same id in two spellings is one row, and a differently
+ // cased manifest id is the same manifest.
+ let (rows, _) = merge_table("Items", vec![item("9d3f7a2c-1b4e-4f80-8a11-2c3d4e5f6a7b", "server-old", T_OLD)], vec![item("9D3F7A2C-1B4E-4F80-8A11-2C3D4E5F6A7B", "local-new", T_NEW)]);
+ assert_eq!(column(&rows, "Name"), vec!["local-new"]);
-/// A `login.url` FieldValues row: multi-value, so it owns its id on the wire.
-fn url_value(id: &str, index: i64, value: &str, updated_at: &str) -> CodecRecord {
- [
- ("ManifestId".to_string(), json!(PERSONAL)),
- ("Id".to_string(), json!(id)),
- ("ItemId".to_string(), json!("item-1")),
- ("FieldKey".to_string(), json!("login.url")),
- ("ValueIndex".to_string(), json!(index)),
- ("Value".to_string(), json!(value)),
- ("UpdatedAt".to_string(), json!(updated_at)),
- ]
- .into_iter()
- .collect()
-}
+ let output = merge(vec![manifest(&PERSONAL.to_uppercase(), &[("Items", vec![item("a", "server", T_OLD)])])], vec![manifest(PERSONAL, &[("Items", vec![item("b", "local", T_MID)])])]);
+ assert!(output.dropped_local_manifest_ids.is_empty());
+ assert_eq!(output.manifests[0].manifest.tables["Items"].len(), 2);
-fn merge_field_values(server: Vec<CodecRecord>, local: Vec<CodecRecord>) -> Vec<CodecRecord> {
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), server)].into_iter().collect())],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), local)].into_iter().collect())],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
- output.manifests[0].manifest.tables["FieldValues"].clone()
+ let key_of = |manifest: &str, id: &str| get_key(&row(&[("ManifestId", json!(manifest)), ("Id", json!(id))]), &["ManifestId", "Id"]);
+ assert_eq!(key_of("1dd1a3fd-8e0e-4b3f-9a3a-0f1a2b3c4d5e", "3ff3c50f-a020-4d51-9c5c-2b3c4d5e6f70"), key_of("1DD1A3FD-8E0E-4B3F-9A3A-0F1A2B3C4D5E", "3FF3C50F-A020-4D51-9C5C-2B3C4D5E6F70"));
+ assert_eq!(get_key(&row(&[("Id", json!("x"))]), &["ManifestId", "Id"]), "\u{1f}x", "a column the row does not carry contributes an empty part, so the row still matches its counterpart");
}
#[test]
-fn multi_value_edit_converges_on_the_owned_id() {
- // The same owned id on both sides is the same row edited: LWW, one row out.
- let rows = merge_field_values(
- vec![url_value("url-1", 0, "https://a.example", "2024-01-01T00:00:00Z")],
- vec![url_value("url-1", 0, "https://a-edited.example", "2024-01-09T00:00:00Z")],
- );
+fn single_value_fields_match_on_their_natural_key_without_an_id() {
+ let (rows, _) = merge_table("FieldValues", vec![field("username", "old", false, T_OLD)], vec![field("username", "new", false, T_NEW)]);
assert_eq!(rows.len(), 1);
- assert_eq!(rows[0]["Id"], json!("url-1"), "the row keeps its owned id");
- assert_eq!(rows[0]["Value"], json!("https://a-edited.example"));
+ assert!(!rows[0].contains_key("Id"), "the derived id is never carried on the wire");
+ assert_eq!(rows[0]["Value"], json!("new"));
+
+ // Custom fields carry no FieldKey, only a FieldDefinitionId, which is part of the key: each keeps its own row.
+ let custom = |def: &str, value: &str, updated_at: &str| at(updated_at, &[("Id", json!(format!("cv-{def}"))), ("ItemId", json!("item-1")), ("FieldKey", serde_json::Value::Null), ("FieldDefinitionId", json!(def)), ("Value", json!(value))]);
+ let (rows, _) = merge_table("FieldValues", vec![custom("fd-pin", "1234", T_OLD), custom("fd-member", "M-77", T_OLD)], vec![custom("fd-pin", "9999", T_NEW), custom("fd-member", "M-77", T_OLD)]);
+ assert_eq!(sorted(column(&rows, "Value")), vec!["9999", "M-77"]);
}
#[test]
-fn multi_value_concurrent_adds_both_survive() {
- // Two devices each add a 2nd url while offline: both additions land at position 1 under
- // different owned ids. Position is not identity, so the union keeps both, renumbered.
- let rows = merge_field_values(
- vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-05T00:00:00Z")],
- vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-c", 1, "https://c.example", "2024-01-06T00:00:00Z")],
- );
- let mut values: Vec<&str> = rows.iter().map(|r| r["Value"].as_str().unwrap()).collect();
- values.sort();
- assert_eq!(values, vec!["https://a.example", "https://b.example", "https://c.example"]);
+fn multi_value_fields_match_on_their_owned_id_not_their_position() {
+ let two = vec![url("url-a", 0, "https://a.example", T_OLD), url("url-b", 1, "https://b.example", T_OLD)];
+ // The same owned id on both sides is one row edited; identical sides merge to exactly themselves.
+ let (rows, _) = merge_table("FieldValues", vec![url("url-1", 0, "https://a.example", T_OLD)], vec![url("url-1", 0, "https://a-edited.example", T_NEW)]);
+ assert_eq!((rows.len(), rows[0]["Id"].as_str(), rows[0]["Value"].as_str()), (1, Some("url-1"), Some("https://a-edited.example")));
+ let (rows, _) = merge_table("FieldValues", two.clone(), two.clone());
+ assert_eq!(rows.len(), 2, "the second url must not replace the first");
+
+ // Two devices each add a second url offline: both land at position 1 under different ids and are renumbered.
+ let (rows, _) = merge_table("FieldValues", two.clone(), vec![url("url-a", 0, "https://a.example", T_OLD), url("url-c", 1, "https://c.example", T_MID)]);
+ assert_eq!(sorted(column(&rows, "Value")), vec!["https://a.example", "https://b.example", "https://c.example"]);
let mut indexes: Vec<i64> = rows.iter().map(|r| r["ValueIndex"].as_i64().unwrap()).collect();
indexes.sort();
- assert_eq!(indexes, vec![0, 1, 2], "colliding positions are renumbered in the merged output");
-}
+ assert_eq!(indexes, vec![0, 1, 2]);
-#[test]
-fn multi_value_reorder_does_not_duplicate() {
- // Reordering changes only ValueIndex; the owned id keeps the rows matched, so no row doubles.
- let rows = merge_field_values(
- vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-01T00:00:00Z")],
- vec![url_value("url-b", 0, "https://b.example", "2024-01-09T00:00:00Z"), url_value("url-a", 1, "https://a.example", "2024-01-09T00:00:00Z")],
- );
- assert_eq!(rows.len(), 2);
+ // A reorder changes only ValueIndex; the owned id keeps the rows matched, so nothing doubles.
+ let (rows, _) = merge_table("FieldValues", two, vec![url("url-b", 0, "https://b.example", T_NEW), url("url-a", 1, "https://a.example", T_NEW)]);
let by_id: HashMap<&str, i64> = rows.iter().map(|r| (r["Id"].as_str().unwrap(), r["ValueIndex"].as_i64().unwrap())).collect();
- assert_eq!(by_id["url-b"], 0, "the reorder won");
- assert_eq!(by_id["url-a"], 1);
+ assert_eq!((rows.len(), by_id["url-b"], by_id["url-a"]), (2, 0, 1));
}
#[test]
-fn both_sides_holding_the_same_two_urls_stay_two_rows() {
- // The historic corruption case: before the id-based key, the 2nd url's value replaced the
- // 1st on an ordinary push. Identical sides must merge to exactly themselves.
- let two = vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-02T00:00:00Z")];
- let rows = merge_field_values(two.clone(), two);
- assert_eq!(rows.len(), 2);
- let by_id: HashMap<&str, &str> = rows.iter().map(|r| (r["Id"].as_str().unwrap(), r["Value"].as_str().unwrap())).collect();
- assert_eq!(by_id["url-a"], "https://a.example");
- assert_eq!(by_id["url-b"], "https://b.example");
-}
+fn id_less_rows_converge_on_their_natural_key() {
+ // History rows key on (item, field, ChangedAt): concurrent changes union, the same snapshot is one row.
+ let history = |changed_at: &str, snapshot: &str| at(changed_at, &[("ItemId", json!("item-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!(changed_at)), ("ValueSnapshot", json!(snapshot))]);
+ let shared = history("2024-01-01 10:00:00.000", "both-know-this");
+ let (rows, _) = merge_table("FieldHistories", vec![shared.clone(), history("2024-02-01 10:00:00.000", "server-only")], vec![shared, history("2024-02-01 11:30:00.000", "local-only")]);
+ assert_eq!(sorted(column(&rows, "ValueSnapshot")), vec!["both-know-this", "local-only", "server-only"]);
+ assert!(rows.iter().all(|r| !r.contains_key("Id")));
-#[test]
-fn custom_field_values_of_one_item_do_not_collapse() {
- // Custom fields carry no FieldKey (only a FieldDefinitionId); before FieldDefinitionId joined
- // the key they all collapsed onto one empty-string key.
- let custom = |def: &str, value: &str, updated_at: &str| -> CodecRecord {
- [
- ("ManifestId".to_string(), json!(PERSONAL)),
- ("Id".to_string(), json!(format!("cv-{}", def))),
- ("ItemId".to_string(), json!("item-1")),
- ("FieldKey".to_string(), serde_json::Value::Null),
- ("FieldDefinitionId".to_string(), json!(def)),
- ("Value".to_string(), json!(value)),
- ("UpdatedAt".to_string(), json!(updated_at)),
- ]
- .into_iter()
- .collect()
- };
- let rows = merge_field_values(
- vec![custom("fd-pin", "1234", "2024-01-01T00:00:00Z"), custom("fd-member", "M-77", "2024-01-01T00:00:00Z")],
- vec![custom("fd-pin", "9999", "2024-01-09T00:00:00Z"), custom("fd-member", "M-77", "2024-01-01T00:00:00Z")],
- );
- assert_eq!(rows.len(), 2, "each custom field keeps its own row");
- let mut values: Vec<&str> = rows.iter().map(|r| r["Value"].as_str().unwrap()).collect();
- values.sort();
- assert_eq!(values, vec!["9999", "M-77"]);
+ // Two devices tag the same item with the same tag: one link.
+ let tag_link = |updated_at: &str| at(updated_at, &[("ItemId", json!("item-1")), ("TagId", json!("tag-1"))]);
+ let (rows, _) = merge_table("ItemTags", vec![tag_link(T_OLD)], vec![tag_link(T_NEW)]);
+ assert_eq!(rows.len(), 1);
+ assert!(!rows[0].contains_key("Id"));
}
#[test]
-fn field_histories_union_by_changed_at_and_converge_on_the_same_snapshot() {
- // History rows derive their id from (item, field, ChangedAt): concurrent changes on two
- // devices carry different timestamps and union; the same snapshot on both sides is one row.
- let history = |changed_at: &str, snapshot: &str| -> CodecRecord {
- [
- ("ManifestId".to_string(), json!(PERSONAL)),
- ("ItemId".to_string(), json!("item-1")),
- ("FieldKey".to_string(), json!("login.password")),
- ("ChangedAt".to_string(), json!(changed_at)),
- ("ValueSnapshot".to_string(), json!(snapshot)),
- ("UpdatedAt".to_string(), json!(changed_at)),
- ]
- .into_iter()
- .collect()
- };
- let shared = history("2024-01-01 10:00:00.000", "both-know-this");
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![manifest(PERSONAL, [("FieldHistories".to_string(), vec![shared.clone(), history("2024-02-01 10:00:00.000", "server-only")])].into_iter().collect())],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![manifest(PERSONAL, [("FieldHistories".to_string(), vec![shared, history("2024-02-01 11:30:00.000", "local-only")])].into_iter().collect())],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
- let rows = &output.manifests[0].manifest.tables["FieldHistories"];
- let mut snapshots: Vec<&str> = rows.iter().map(|r| r["ValueSnapshot"].as_str().unwrap()).collect();
- snapshots.sort();
- assert_eq!(snapshots, vec!["both-know-this", "local-only", "server-only"]);
- assert!(rows.iter().all(|r| r.get("Id").is_none()), "merged history rows stay id-less on the wire");
-}
+fn blob_references_and_server_metadata_ride_through() {
+ let attachment = |blob_ref: &str, updated_at: &str| at(updated_at, &[("Id", json!("att-1")), ("Blob", json!({ "__blobRef": blob_ref, "__blobKind": "attachment" }))]);
+ let (rows, _) = merge_table("Attachments", vec![attachment("hash-old", T_OLD)], vec![attachment("hash-new", T_NEW)]);
+ assert_eq!(rows[0]["Blob"], json!({ "__blobRef": "hash-new", "__blobKind": "attachment" }), "no byte payload ever crosses the merge");
-#[test]
-fn item_tags_converge_on_their_natural_key() {
- // Two devices tag the same item with the same tag: the id-less join rows are one row.
- let tag_row = |updated_at: &str| -> CodecRecord {
- [
- ("ManifestId".to_string(), json!(PERSONAL)),
- ("ItemId".to_string(), json!("item-1")),
- ("TagId".to_string(), json!("tag-1")),
- ("UpdatedAt".to_string(), json!(updated_at)),
- ]
- .into_iter()
- .collect()
- };
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![manifest(PERSONAL, [("ItemTags".to_string(), vec![tag_row("2024-01-01T00:00:00Z")])].into_iter().collect())],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![manifest(PERSONAL, [("ItemTags".to_string(), vec![tag_row("2024-01-09T00:00:00Z")])].into_iter().collect())],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
- let rows = &output.manifests[0].manifest.tables["ItemTags"];
- assert_eq!(rows.len(), 1, "the same (item, tag) link is one row on both sides");
- assert!(rows[0].get("Id").is_none());
+ let mut server = manifest(PERSONAL, &[("Items", vec![item("a", "server", T_OLD)])]);
+ server.name = Some("Family".to_string());
+ let output = merge(vec![server], vec![manifest(PERSONAL, &[("Items", vec![item("b", "local", T_MID)])])]);
+ let merged = &output.manifests[0].manifest;
+ assert_eq!((merged.manifest_salt.as_str(), merged.name.as_deref()), (SALT, Some("Family")), "salt and name come from the server manifest");
+ assert!(crate::vault_codec::validate_manifest(merged).ok);
}
-#[test]
-fn bucketed_tables_merge_and_come_back_as_buckets() {
- let setting = |value: &str, updated_at: &str| -> CodecRecord {
- [
- ("ManifestId".to_string(), json!(PERSONAL)),
- ("Key".to_string(), json!("theme")),
- ("Value".to_string(), json!(value)),
- ("UpdatedAt".to_string(), json!(updated_at)),
- ]
- .into_iter()
- .collect()
- };
-
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![items_manifest(PERSONAL, vec![])],
- server_buckets: vec![DataBucket::new(PERSONAL, "settings", [("Settings".to_string(), vec![setting("light", "2024-01-01T00:00:00Z")])].into_iter().collect())],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![items_manifest(PERSONAL, vec![])],
- local_buckets: vec![DataBucket::new(PERSONAL, "settings", [("Settings".to_string(), vec![setting("dark", "2024-01-09T00:00:00Z")])].into_iter().collect())],
- schema_columns: schema(),
- })
- .unwrap();
+/// `item-1` with the given name and trash stamp.
+fn trashable(name: &str, deleted_at: Option<&str>, updated_at: &str) -> CodecRecord {
+ let mut r = item("item-1", name, updated_at);
+ r.insert("DeletedAt".to_string(), json!(deleted_at));
+ r
+}
- let merged = &output.manifests[0];
- assert!(!merged.manifest.tables.contains_key("Settings"), "bucketed tables never surface in manifest.tables");
- assert_eq!(merged.buckets.len(), 1);
- assert_eq!(merged.buckets[0].category, "settings");
- assert_eq!(merged.buckets[0].tables["Settings"][0]["Value"], json!("dark"), "the newer local setting wins inside the bucket");
- assert!(crate::vault_codec::validate_data_bucket(&merged.buckets[0]).ok);
+fn totp(id: &str, secret: &str, deleted: bool, updated_at: &str) -> CodecRecord {
+ at(updated_at, &[("Id", json!(id)), ("ItemId", json!("item-1")), ("SecretKey", json!(secret)), ("IsDeleted", json!(deleted as i32))])
}
-#[test]
-fn an_empty_table_the_server_carried_stays_in_the_merged_manifest() {
- /*
- * The merged manifest is pushed as-is and compared against a baseline canonicalize produced, and
- * canonicalize keeps an empty table. Dropping it here rewrites the whole manifest on every merge.
- */
- let merged = merge_single(vec![], vec![]);
- assert_eq!(merged.manifest.tables.get("Items").map(Vec::len), Some(0), "the server's empty table keeps its place in the merged manifest");
-
- // The empty tables of a bucket the server served come back as that bucket, not as a missing one.
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![items_manifest(PERSONAL, vec![])],
- server_buckets: vec![DataBucket::new(PERSONAL, "settings", [("Settings".to_string(), vec![])].into_iter().collect())],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![items_manifest(PERSONAL, vec![])],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
- let buckets = &output.manifests[0].buckets;
- assert_eq!(buckets.len(), 1, "the served bucket is still emitted when it holds no rows");
- assert_eq!(buckets[0].tables.get("Settings").map(Vec::len), Some(0));
+/// One concurrent-edit scenario: what the server holds and what the local device holds for `item-1` and its codes.
+struct Concurrent {
+ what: String,
+ server: (CodecRecord, Vec<CodecRecord>),
+ local: (CodecRecord, Vec<CodecRecord>),
+ /// `Name|DeletedAt` of the merged item.
+ item: String,
+ live_secrets: Vec<&'static str>,
}
#[test]
-fn unknown_columns_keep_the_server_value_when_local_wins() {
- // A newer client added Items.NewColumn; this client's schema does not know it. The local
- // row winning the LWW must not regress that column.
- let mut base = item(PERSONAL, "row", "server-old", "2024-01-01T00:00:00Z");
- base.insert("NewColumn".to_string(), json!("newer-client-value"));
- let mut incoming = item(PERSONAL, "row", "local-new", "2024-01-09T00:00:00Z");
- incoming.insert("NewColumn".to_string(), json!("stale-carried-value"));
+fn concurrent_edits_to_an_item_and_its_children_settle_row_by_row() {
+ let bank = || trashable("Bank", None, T_OLD);
+ let code = |secret: &str, deleted: bool, updated_at: &str| totp("totp-1", secret, deleted, updated_at);
+ let case = |what: &str, server: (CodecRecord, Vec<CodecRecord>), local: (CodecRecord, Vec<CodecRecord>), item: &str, live_secrets: Vec<&'static str>| Concurrent { what: what.to_string(), server, local, item: item.to_string(), live_secrets };
+ let mut cases = vec![
+ case("a code deleted while the item was renamed", (bank(), vec![code("S", true, T_MID)]), (trashable("Renamed", None, T_NEW), vec![code("S", false, T_OLD)]), "Renamed|-", vec![]),
+ case("a delete newer than an edit", (bank(), vec![code("S", true, T_NEW)]), (bank(), vec![code("edited", false, T_MID)]), "Bank|-", vec![]),
+ case("an edit newer than a delete", (bank(), vec![code("S", true, T_MID)]), (bank(), vec![code("edited", false, T_NEW)]), "Bank|-", vec!["edited"]),
+ case("trash is an ordinary item edit", (trashable("Bank", Some(T_NEW), T_NEW), vec![code("S", false, T_OLD)]), (bank(), vec![code("S", false, T_OLD), totp("totp-2", "added", false, T_MID)]), &format!("Bank|{T_NEW}"), vec!["S", "added"]),
+ case("a restore newer than the trash", (trashable("Bank", Some(T_MID), T_MID), vec![]), (trashable("Bank", None, T_NEW), vec![]), "Bank|-", vec![]),
+ ];
- let merged = merge_single(vec![base], vec![incoming]);
+ // Spellings older or other clients write (space vs 'T', a Z suffix, an offset, .NET's seven digits) compare as instants.
+ for (server_at, local_at, local_wins) in [
+ ("2026-01-02 00:00:00.000", "2026-01-02T00:00:00.001Z", true),
+ ("2026-01-02T00:00:00.000Z", "2026-01-02 00:00:00.000", false),
+ ("2026-01-02 00:00:00.123", "2026-01-02 00:00:00.1234567", true),
+ ("2026-01-02 02:00:00.000", "2026-01-02T03:00:00+02:00", false),
+ ("2026-01-02 00:00:00", "2026-01-02 00:00:01", true),
+ ] {
+ cases.push(case(&format!("server at {server_at} vs local at {local_at}"), (bank(), vec![code("SERVER", false, server_at)]), (bank(), vec![code("LOCAL", false, local_at)]), "Bank|-", vec![if local_wins { "LOCAL" } else { "SERVER" }]));
+ }
- let row = &merged.manifest.tables["Items"][0];
- assert_eq!(row["Name"], json!("local-new"), "known columns follow the winner");
- assert_eq!(row["NewColumn"], json!("newer-client-value"), "unknown columns keep the base value");
+ for c in cases {
+ let side = |(item, codes): (CodecRecord, Vec<CodecRecord>)| manifest(PERSONAL, &[("Items", vec![item]), ("TotpCodes", codes)]);
+ let merged = merge(vec![side(c.server)], vec![side(c.local)]).manifests.remove(0).manifest;
+ let item = &merged.tables["Items"][0];
+ assert_eq!(format!("{}|{}", item["Name"].as_str().unwrap(), item["DeletedAt"].as_str().unwrap_or("-")), c.item, "{}", c.what);
+ assert_eq!(sorted(merged.tables["TotpCodes"].iter().filter(|r| r["IsDeleted"] == json!(0)).map(|r| r["SecretKey"].as_str().unwrap()).collect()), c.live_secrets, "{}", c.what);
+ }
}
+/*
+ * Tables, buckets and manifests.
+ */
+
#[test]
-fn unknown_tables_are_taken_from_the_server_wholesale() {
- let future_row = |payload: &str| -> CodecRecord {
- [("ManifestId".to_string(), json!(PERSONAL)), ("Id".to_string(), json!("f-1")), ("Payload".to_string(), json!(payload))].into_iter().collect()
- };
- let with_future = |payload: Option<&str>| -> Manifest {
- let mut tables: HashMap<String, Vec<CodecRecord>> = [("Items".to_string(), vec![])].into_iter().collect();
- if let Some(p) = payload {
- tables.insert("FutureTable".to_string(), vec![future_row(p)]);
- }
- manifest(PERSONAL, tables)
- };
+fn unknown_columns_and_tables_keep_the_server_version() {
+ // A newer client added Items.NewColumn and a FutureTable this client's schema does not know. A local row winning
+ // LWW must not regress the column, and an unknown table is taken from the server wholesale (a local-only one is dropped).
+ let mut base = item("row", "server-old", T_OLD);
+ base.insert("NewColumn".to_string(), json!("newer-client-value"));
+ let mut incoming = item("row", "local-new", T_NEW);
+ incoming.insert("NewColumn".to_string(), json!("stale-carried-value"));
+ let (rows, _) = merge_table("Items", vec![base], vec![incoming]);
+ assert_eq!((rows[0]["Name"].as_str(), rows[0]["NewColumn"].as_str()), (Some("local-new"), Some("newer-client-value")));
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![with_future(Some("server-version"))],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![with_future(Some("local-carried-version"))],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
+ let future = |payload: &str| manifest(PERSONAL, &[("Items", vec![]), ("FutureTable", vec![at(T_OLD, &[("Id", json!("f-1")), ("Payload", json!(payload))])])]);
+ let output = merge(vec![future("server-version")], vec![future("local-carried-version")]);
assert_eq!(output.manifests[0].manifest.tables["FutureTable"][0]["Payload"], json!("server-version"));
-
- // And a local-only unknown table is dropped, matching the discarded local carrier of today.
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![with_future(None)],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![with_future(Some("local-only"))],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
+ let output = merge(vec![manifest(PERSONAL, &[("Items", vec![])])], vec![future("local-only")]);
assert!(!output.manifests[0].manifest.tables.contains_key("FutureTable"));
}
#[test]
-fn local_manifest_no_longer_served_is_dropped_and_reported() {
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![items_manifest(PERSONAL, vec![])],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![items_manifest(PERSONAL, vec![]), items_manifest(SHARED, vec![item(SHARED, "x", "gone", "2024-01-01T00:00:00Z")])],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
-
- assert_eq!(output.manifests.len(), 1);
- assert_eq!(output.dropped_local_manifest_ids, vec![SHARED.to_string()]);
-}
-
-#[test]
-fn contentless_server_manifest_passes_the_local_one_through() {
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![items_manifest(PERSONAL, vec![])],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![SHARED.to_string()],
- local_manifests: vec![items_manifest(PERSONAL, vec![]), items_manifest(SHARED, vec![item(SHARED, "x", "kept-offline", "2024-01-01T00:00:00Z")])],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
-
- assert!(output.dropped_local_manifest_ids.is_empty());
- let shared = output.manifests.iter().find(|m| m.manifest_id == SHARED).unwrap();
- assert_eq!(names_of(&shared.manifest), vec!["kept-offline"]);
- assert_eq!(shared.stats.records_inserted, 1);
+fn buckets_merge_like_tables_and_come_back_as_buckets() {
+ let setting = |value: &str, updated_at: &str| at(updated_at, &[("Key", json!("theme")), ("Value", json!(value))]);
+ let bucket = |category: &str, table: &str, rows: Vec<CodecRecord>| DataBucket::new(PERSONAL, category, [(table.to_string(), rows)].into_iter().collect());
+ let unknown_row = at(T_OLD, &[("Id", json!("u-1"))]);
+ let mut server_settings = bucket("settings", "Settings", vec![setting("light", T_OLD)]);
+ server_settings.tables.insert("SettingsExtras".to_string(), vec![unknown_row.clone()]);
+
+ let output = merge_with_buckets(
+ vec![manifest(PERSONAL, &[("Items", vec![])])],
+ vec![server_settings, bucket("stats", "ItemStats", vec![])],
+ vec![manifest(PERSONAL, &[("Items", vec![])])],
+ vec![bucket("settings", "Settings", vec![setting("dark", T_NEW)]), bucket("FutureCategory", "FutureRows", vec![unknown_row])],
+ );
+ let merged = &output.manifests[0];
+ assert!(!merged.manifest.tables.contains_key("Settings"), "bucketed tables never surface in manifest.tables");
+ assert_eq!(merged.buckets.iter().filter(|b| b.category == "settings").count(), 1, "one bucket per category, whatever tables it carries");
+ let settings = merged.buckets.iter().find(|b| b.category == "settings").unwrap();
+ assert_eq!(settings.tables["Settings"][0]["Value"], json!("dark"), "the newer local setting wins inside the bucket");
+ assert!(settings.tables.contains_key("SettingsExtras"), "the server's unknown bucket table rides through in its category's bucket");
+ assert!(crate::vault_codec::validate_data_bucket(settings).ok);
+ assert!(merged.buckets.iter().any(|b| b.category == "stats" && b.tables["ItemStats"].is_empty()), "a served bucket is emitted even when it holds no rows");
+ assert!(!merged.buckets.iter().any(|b| b.category == "FutureCategory"), "a local-only unknown bucket is dropped with the rest of the local carrier");
}
#[test]
-fn blob_refs_ride_through_as_opaque_values() {
- let attachment = |blob_ref: &str, updated_at: &str| -> CodecRecord {
- [
- ("ManifestId".to_string(), json!(PERSONAL)),
- ("Id".to_string(), json!("att-1")),
- ("Blob".to_string(), json!({ "__blobRef": blob_ref, "__blobKind": "attachment" })),
- ("UpdatedAt".to_string(), json!(updated_at)),
- ]
- .into_iter()
- .collect()
- };
-
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![manifest(PERSONAL, [("Attachments".to_string(), vec![attachment("hash-old", "2024-01-01T00:00:00Z")])].into_iter().collect())],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![manifest(PERSONAL, [("Attachments".to_string(), vec![attachment("hash-new", "2024-01-09T00:00:00Z")])].into_iter().collect())],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
-
- assert_eq!(output.manifests[0].manifest.tables["Attachments"][0]["Blob"], json!({ "__blobRef": "hash-new", "__blobKind": "attachment" }));
+fn an_empty_table_the_server_carried_keeps_its_place() {
+ // The merged manifest is pushed as-is and compared against a baseline canonicalize produced, which keeps empty
+ // tables; dropping one here would rewrite the whole manifest on every merge.
+ let (rows, _) = merge_table("Items", vec![], vec![]);
+ assert!(rows.is_empty());
}
#[test]
-fn merged_manifest_keeps_server_metadata_and_validates() {
- let mut server = items_manifest(PERSONAL, vec![item(PERSONAL, "a", "server", "2024-01-01T00:00:00Z")]);
- server.name = Some("Family".to_string());
- let local = items_manifest(PERSONAL, vec![item(PERSONAL, "b", "local", "2024-01-02T00:00:00Z")]);
-
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![server],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![local],
- local_buckets: vec![],
- schema_columns: schema(),
- })
- .unwrap();
-
- let merged = &output.manifests[0].manifest;
- assert_eq!(merged.manifest_salt, SALT, "salt comes from the server manifest");
- assert_eq!(merged.name.as_deref(), Some("Family"), "name comes from the server manifest");
- let validation = crate::vault_codec::validate_manifest(merged);
- assert!(validation.ok, "merged manifest must validate: {:?}", validation.failed_rules);
-}
+fn manifests_merge_independently_and_a_lost_one_is_reported() {
+ // Manifests are the partition: the same row id in two manifests never interacts, a local manifest the server no
+ // longer serves is dropped and named, and one the server serves without content passes the local rows through.
+ let output = merge(
+ vec![manifest(PERSONAL, &[("Items", vec![item("same-id", "mine", T_OLD)])]), manifest(SHARED, &[("Items", vec![item("same-id", "theirs-old", T_OLD)])])],
+ vec![manifest(PERSONAL, &[("Items", vec![item("same-id", "mine", T_OLD)])]), manifest(SHARED, &[("Items", vec![item("same-id", "theirs-new", T_NEW)])]), manifest("m-gone", &[("Items", vec![item("x", "gone", T_OLD)])])],
+ );
+ let by_id: HashMap<&str, &CanonicalManifestMerge> = output.manifests.iter().map(|m| (m.manifest_id.as_str(), m)).collect();
+ assert_eq!(column(&by_id[PERSONAL].manifest.tables["Items"], "Name"), vec!["mine"]);
+ assert_eq!(column(&by_id[SHARED].manifest.tables["Items"], "Name"), vec!["theirs-new"]);
+ assert_eq!(by_id[PERSONAL].stats.incoming_won, 0);
+ assert_eq!(output.dropped_local_manifest_ids, vec!["m-gone".to_string()]);
-#[test]
-fn manifest_pairing_is_case_insensitive() {
let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![items_manifest("M-PERSONAL", vec![item("M-PERSONAL", "a", "server", "2024-01-01T00:00:00Z")])],
+ server_manifests: vec![manifest(PERSONAL, &[("Items", vec![])])],
server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![items_manifest("m-personal", vec![item("m-personal", "b", "local", "2024-01-02T00:00:00Z")])],
+ contentless_server_manifest_ids: vec![SHARED.to_string()],
+ local_manifests: vec![manifest(PERSONAL, &[("Items", vec![])]), manifest(SHARED, &[("Items", vec![item("x", "kept-offline", T_OLD)])])],
local_buckets: vec![],
schema_columns: schema(),
})
.unwrap();
-
- assert!(output.dropped_local_manifest_ids.is_empty(), "differently-cased ids are the same manifest");
- assert_eq!(output.manifests[0].manifest.tables["Items"].len(), 2);
-}
-
-#[test]
-fn unknown_category_bucket_tables_ride_through_from_the_server() {
- // A newer writer added a table inside the Settings bucket and a whole new category. The
- // known Settings rows still merge; the unknown tables ride through from the server only.
- let unknown_row: CodecRecord = [("ManifestId".to_string(), json!(PERSONAL)), ("Id".to_string(), json!("u-1"))].into_iter().collect();
- let server_settings = DataBucket::new(
- PERSONAL,
- "Settings",
- [
- ("Settings".to_string(), vec![[("ManifestId".to_string(), json!(PERSONAL)), ("Key".to_string(), json!("theme")), ("Value".to_string(), json!("light")), ("UpdatedAt".to_string(), json!("2024-01-01T00:00:00Z"))].into_iter().collect()]),
- ("SettingsExtras".to_string(), vec![unknown_row.clone()]),
- ]
- .into_iter()
- .collect(),
- );
- let local_future = DataBucket::new(PERSONAL, "FutureCategory", [("FutureRows".to_string(), vec![unknown_row])].into_iter().collect());
-
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![items_manifest(PERSONAL, vec![])],
- server_buckets: vec![server_settings],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![items_manifest(PERSONAL, vec![])],
- local_buckets: vec![local_future],
- schema_columns: schema(),
- })
- .unwrap();
-
- let buckets = &output.manifests[0].buckets;
- let categories: Vec<&str> = buckets.iter().map(|b| b.category.as_str()).collect();
- assert!(categories.contains(&"Settings"), "the known Settings rows still come back as a bucket");
- assert!(buckets.iter().any(|b| b.category == "Settings" && b.tables.contains_key("SettingsExtras")), "the server's unknown bucket table rides through under its category");
- assert!(!categories.contains(&"FutureCategory"), "a local-only unknown bucket is dropped with the rest of the local carrier");
-}
-
-#[test]
-fn keys_compare_guids_case_insensitively_and_survive_a_missing_column() {
- const MANIFEST: &str = "1dd1a3fd-8e0e-4b3f-9a3a-0f1a2b3c4d5e";
- const OTHER_MANIFEST: &str = "2ee2b4fe-9f1f-4c40-8b4b-1a2b3c4d5e6f";
- const ROW: &str = "3ff3c50f-a020-4d51-9c5c-2b3c4d5e6f70";
- let row = |manifest: &str, id: &str| -> CodecRecord {
- HashMap::from([("ManifestId".to_string(), json!(manifest)), ("Id".to_string(), json!(id))])
- };
- let columns = ["ManifestId", "Id"];
-
- // Two writers spelling the same ids in different cases address one row (see `identity_part`).
- assert_eq!(get_key(&row(MANIFEST, ROW), &columns), get_key(&row(&MANIFEST.to_uppercase(), &ROW.to_uppercase()), &columns));
- assert_ne!(get_key(&row(MANIFEST, ROW), &columns), get_key(&row(OTHER_MANIFEST, ROW), &columns));
- assert_eq!(get_key(&HashMap::from([("Id".to_string(), json!(ROW))]), &columns), format!(":{}", ROW));
-}
-
-/// An `Items` row, live or tombstoned.
-fn item_row(manifest_id: &str, id: &str, deleted: bool, updated_at: &str) -> CodecRecord {
- let mut row = item(manifest_id, id, "item", updated_at);
- row.insert("IsDeleted".to_string(), json!(if deleted { 1 } else { 0 }));
- row
-}
-
-/// A single-value `FieldValues` row of `item-1` in wire shape, live or tombstoned.
-fn field_of_item(manifest_id: &str, field_key: &str, value: &str, deleted: bool, updated_at: &str) -> CodecRecord {
- [
- ("ManifestId".to_string(), json!(manifest_id)),
- ("ItemId".to_string(), json!("item-1")),
- ("FieldKey".to_string(), json!(field_key)),
- ("ValueIndex".to_string(), json!(0)),
- ("Value".to_string(), json!(value)),
- ("IsDeleted".to_string(), json!(if deleted { 1 } else { 0 })),
- ("UpdatedAt".to_string(), json!(updated_at)),
- ]
- .into_iter()
- .collect()
-}
-
-fn item_manifest(manifest_id: &str, items: Vec<CodecRecord>, field_values: Vec<CodecRecord>) -> Manifest {
- manifest(manifest_id, [("Items".to_string(), items), ("FieldValues".to_string(), field_values)].into_iter().collect())
-}
-
-fn merge_manifests(server: Vec<Manifest>, local: Vec<Manifest>) -> Vec<CanonicalManifestMerge> {
- merge_canonical(CanonicalMergeInput { server_manifests: server, server_buckets: vec![], contentless_server_manifest_ids: vec![], local_manifests: local, local_buckets: vec![], schema_columns: schema() }).unwrap().manifests
-}
-
-fn live_values(merged: &CanonicalManifestMerge) -> Vec<String> {
- let mut values: Vec<String> = merged.manifest.tables["FieldValues"].iter().filter(|r| r["IsDeleted"] == json!(0)).map(|r| r["Value"].as_str().unwrap().to_string()).collect();
- values.sort();
- values
+ assert!(output.dropped_local_manifest_ids.is_empty());
+ let shared = output.manifests.iter().find(|m| m.manifest_id == SHARED).unwrap();
+ assert_eq!(column(&shared.manifest.tables["Items"], "Name"), vec!["kept-offline"]);
+ assert_eq!(shared.stats.incoming_only, 1);
}
-const T_OLD: &str = "2024-01-01T00:00:00Z";
-const T_DELETE: &str = "2024-01-05T05:00:00Z";
-const T_EDIT: &str = "2024-01-05T05:05:00Z";
+/*
+ * An item is deleted as a unit: a permanent delete only one side made is settled against everything the other
+ * side did to the item (a field edit does not touch the item row), both ways, and the losing side's rows leave.
+ */
-#[test]
-fn a_permanent_delete_does_not_get_its_children_back_from_the_other_side() {
- // The deleting side removed the item's rows; the server still holds them and must not resurrect them.
- let server = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "secret", false, T_OLD)]);
- let local = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_DELETE)], vec![]);
- for (base, incoming) in [(server.clone(), local.clone()), (local, server)] {
- let merged = &merge_manifests(vec![base], vec![incoming])[0];
- assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1), "the delete stands");
- assert!(merged.manifest.tables["FieldValues"].is_empty(), "and the deleted item keeps no rows");
- }
+fn item_manifest(items: Vec<CodecRecord>, field_values: Vec<CodecRecord>) -> Manifest {
+ manifest(PERSONAL, &[("Items", items), ("FieldValues", field_values)])
}
-#[test]
-fn an_item_edited_after_the_other_side_deleted_it_survives_whole() {
- // One side deleted the item at 05:00, tombstoning its rows as the pruner does. The other edited one
- // field at 05:05, which never touches the item row. The edit is the last write, so the item survives,
- // and with every field: the delete's child tombstones must not wipe the fields the edit left alone.
- let deleting = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_DELETE)], vec![
- field_of_item(PERSONAL, "login.username", "user", true, T_DELETE),
- field_of_item(PERSONAL, "login.password", "old-secret", true, T_DELETE),
- ]);
- let editing = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![
- field_of_item(PERSONAL, "login.username", "user", false, T_OLD),
- field_of_item(PERSONAL, "login.password", "new-secret", false, T_EDIT),
- ]);
- for (base, incoming) in [(deleting.clone(), editing.clone()), (editing, deleting)] {
- let merged = &merge_manifests(vec![base], vec![incoming])[0];
- assert_eq!(merged.manifest.tables["Items"].len(), 1);
- assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(0), "the later edit outlives the delete");
- assert_eq!(live_values(merged), vec!["new-secret", "user"], "with the untouched field intact");
- }
+fn live_values(merged: &CanonicalManifestMerge) -> Vec<&str> {
+ sorted(merged.manifest.tables["FieldValues"].iter().filter(|r| r["IsDeleted"] == json!(0)).map(|r| r["Value"].as_str().unwrap()).collect())
}
#[test]
-fn a_delete_newer_than_everything_the_other_side_did_stands() {
- let editing = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "edited", false, T_DELETE)]);
- let deleting = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_EDIT)], vec![]);
- for (base, incoming) in [(deleting.clone(), editing.clone()), (editing, deleting)] {
- let merged = &merge_manifests(vec![base], vec![incoming])[0];
- assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1));
- assert!(merged.manifest.tables["FieldValues"].is_empty());
+fn a_permanent_delete_is_settled_against_the_item_as_a_unit_both_ways() {
+ let deleting = item_manifest(vec![tombstone("item-1", T_MID)], vec![field("login.username", "user", true, T_MID), field("login.password", "old-secret", true, T_MID)]);
+ for (edit_at, survives) in [(T_NEW, true), (T_OLD, false)] {
+ let editing = item_manifest(vec![item("item-1", "item", T_OLD)], vec![field("login.username", "user", false, T_OLD), field("login.password", "new-secret", false, edit_at)]);
+ for (base, incoming) in [(deleting.clone(), editing.clone()), (editing, deleting.clone())] {
+ let output = merge(vec![base], vec![incoming]);
+ let merged = &output.manifests[0];
+ assert_eq!(merged.manifest.tables["Items"].len(), 1);
+ if survives {
+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(0), "the later edit outlives the delete");
+ assert_eq!(live_values(merged), vec!["new-secret", "user"], "with the untouched field intact: the delete's child tombstones do not wipe it");
+ } else {
+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1), "the delete stands");
+ assert!(merged.manifest.tables["FieldValues"].is_empty(), "and the deleted item keeps no rows, however the other side still held them");
+ }
+ }
}
-}
-
-#[test]
-fn a_delete_and_an_edit_at_the_same_instant_keep_the_server_side() {
- let deleting = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_EDIT)], vec![]);
- let editing = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "edited", false, T_EDIT)]);
- let merged = &merge_manifests(vec![deleting.clone()], vec![editing.clone()])[0];
- assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1), "the server's delete keeps the tie");
-
- let merged = &merge_manifests(vec![editing], vec![deleting])[0];
- assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(0), "the server's edit keeps the tie");
- assert_eq!(live_values(merged), vec!["edited"]);
+ // At the same instant the server side keeps the tie, whichever side it is.
+ let deleting = item_manifest(vec![tombstone("item-1", T_NEW)], vec![]);
+ let mut editing = item_manifest(vec![item("item-1", "item", T_OLD)], vec![field("login.password", "edited", false, T_NEW)]);
+ let totp = at(T_NEW, &[("Id", json!("totp-1")), ("ItemId", json!("item-1")), ("SecretKey", json!("S")), ("IsDeleted", json!(0))]);
+ editing.tables.insert("TotpCodes".to_string(), vec![totp]);
+ let won = merge(vec![deleting.clone()], vec![editing.clone()]).manifests.remove(0).manifest;
+ assert_eq!(won.tables["Items"][0]["IsDeleted"], json!(1));
+ assert!(won.tables["FieldValues"].is_empty() && won.tables["TotpCodes"].is_empty(), "a bare tombstone takes the live children with it");
+ let merged = merge(vec![editing], vec![deleting]);
+ assert_eq!(merged.manifests[0].manifest.tables["Items"][0]["IsDeleted"], json!(0));
+ assert_eq!(live_values(&merged.manifests[0]), vec!["edited"]);
}
#[test]
fn a_usage_counter_does_not_outlive_a_delete() {
// ItemStats ticks on every autofill; that is not an edit and must not undo a delete.
- let stats = |updated_at: &str| -> CodecRecord {
- [("ManifestId".to_string(), json!(PERSONAL)), ("Id".to_string(), json!("item-1")), ("LastUsedAt".to_string(), json!(updated_at)), ("UpdatedAt".to_string(), json!(updated_at))].into_iter().collect()
- };
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_DELETE)], vec![])],
- server_buckets: vec![],
- contentless_server_manifest_ids: vec![],
- local_manifests: vec![item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![])],
- local_buckets: vec![DataBucket::new(PERSONAL, "Stats", [("ItemStats".to_string(), vec![stats(T_EDIT)])].into_iter().collect())],
- schema_columns: schema(),
- })
- .unwrap();
+ let stats = at(T_NEW, &[("Id", json!("item-1")), ("LastUsedAt", json!(T_NEW))]);
+ let output = merge_with_buckets(vec![item_manifest(vec![tombstone("item-1", T_MID)], vec![])], vec![], vec![item_manifest(vec![item("item-1", "item", T_OLD)], vec![])], vec![DataBucket::new(PERSONAL, "stats", [("ItemStats".to_string(), vec![stats])].into_iter().collect())]);
let merged = &output.manifests[0];
assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1));
- assert!(merged.buckets.iter().all(|bucket| bucket.tables.get("ItemStats").is_none_or(Vec::is_empty)), "the deleted item's stats row goes with it");
+ assert!(merged.buckets.iter().all(|b| b.tables.get("ItemStats").is_none_or(Vec::is_empty)), "the deleted item's stats row goes with it");
}
#[test]
fn an_item_moved_to_another_manifest_does_not_stay_behind_in_the_one_it_left() {
- // The move leaves a tombstone in the source manifest, so the server's copy there loses instead of
- // surviving the union next to the moved one.
- let server = vec![
- item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "secret", false, T_OLD)]),
- item_manifest(SHARED, vec![], vec![]),
- ];
- let local = vec![
- item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_EDIT)], vec![]),
- item_manifest(SHARED, vec![item_row(SHARED, "item-1", false, T_EDIT)], vec![field_of_item(SHARED, "login.password", "secret", false, T_OLD)]),
- ];
- let merged = merge_manifests(server, local);
- let by_id: HashMap<&str, &CanonicalManifestMerge> = merged.iter().map(|m| (m.manifest_id.as_str(), m)).collect();
+ // The move leaves a tombstone in the source manifest, so the server's copy there loses instead of surviving next to the moved one.
+ let server = vec![item_manifest(vec![item("item-1", "item", T_OLD)], vec![field("login.password", "secret", false, T_OLD)]), manifest(SHARED, &[("Items", vec![]), ("FieldValues", vec![])])];
+ let local = vec![item_manifest(vec![tombstone("item-1", T_NEW)], vec![]), manifest(SHARED, &[("Items", vec![restamp(item("item-1", "item", T_NEW), SHARED)]), ("FieldValues", vec![restamp(field("login.password", "secret", false, T_OLD), SHARED)])])];
+ let output = merge(server, local);
+ let by_id: HashMap<&str, &CanonicalManifestMerge> = output.manifests.iter().map(|m| (m.manifest_id.as_str(), m)).collect();
assert_eq!(by_id[PERSONAL].manifest.tables["Items"][0]["IsDeleted"], json!(1), "the source manifest keeps only the tombstone");
assert!(by_id[PERSONAL].manifest.tables["FieldValues"].is_empty());
assert_eq!(by_id[SHARED].manifest.tables["Items"][0]["IsDeleted"], json!(0));
diff --git a/core/rust/src/vault_model/generated.rs b/core/rust/src/vault_model/generated.rs
index 71e8743e5..060fd9eae 100644
--- a/core/rust/src/vault_model/generated.rs
+++ b/core/rust/src/vault_model/generated.rs
@@ -92,7 +92,7 @@ pub static SINGLE_VALUE_FIELD_KEYS: &[&str] = &["login.email", "login.username",
/// stamped with that manifest's id (`ManifestId`).
pub const ENCRYPTION_KEYS_TABLE: &str = "EncryptionKeys";
-/// The scope column every stamped table carries: the id of the manifest that owns the row.
+/// The column every stamped table carries: the id of the manifest that owns the row.
pub const MANIFEST_ID_COL: &str = "ManifestId";
/// Local bookkeeping table materialize writes into the vault DB: one row per manifest this
@@ -109,7 +109,7 @@ pub const OVERFLOW_TABLE: &str = "CodecOverflows";
pub const OVERFLOW_ROW_ID: &str = "00000000-0000-0000-0000-00000000c0de";
/// All zero GUID used for default values which indicate unstamped rows.
-pub const UNSTAMPED_SCOPE_SENTINEL: &str = "00000000-0000-0000-0000-000000000000";
+pub const UNSTAMPED_MANIFEST_ID: &str = "00000000-0000-0000-0000-000000000000";
/// Rows referenced from inside manifest content: `(target_table, [(referencing_table, column)])`.
/// On a manifest split the referenced rows are reference-copied into the destination manifest.
diff --git a/core/rust/src/vault_model/mod.rs b/core/rust/src/vault_model/mod.rs
index b824d1c74..892b48d46 100644
--- a/core/rust/src/vault_model/mod.rs
+++ b/core/rust/src/vault_model/mod.rs
@@ -107,6 +107,7 @@ mod tests {
assert!(config.manifest_scoped, "{} must be manifest_scoped or it merges across manifests", config.name);
assert_eq!(config.identity_columns().first(), Some(&MANIFEST_ID_COL), "{} must be addressed by its manifest first", config.name);
}
+ assert!(!SYNCABLE_TABLE_NAMES.contains(&OVERFLOW_TABLE), "the overflow carrier rides inside the manifest, it is not synced as a table of its own");
}
#[test]
diff --git a/core/rust/src/vault_pruner/mod.rs b/core/rust/src/vault_pruner/mod.rs
index eaa2d9ac2..5a1e8353c 100644
--- a/core/rust/src/vault_pruner/mod.rs
+++ b/core/rust/src/vault_pruner/mod.rs
@@ -1,69 +1,47 @@
-//! Vault pruner for automatically removing expired trash items.
-//!
-//! This module handles the automatic cleanup of items that have been in the trash
-//! (DeletedAt set) for longer than the retention period (default 30 days).
-//! It generates SQL statements to permanently delete (IsDeleted = true) these items
-//! along with their related entities.
+//! Vault pruner: SQL statements that permanently delete (IsDeleted = true) items that have been in the trash longer
+//! than the retention period, along with their related entities.
use chrono::{DateTime, Duration, Utc};
-use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use crate::common::error::{VaultError, VaultResult};
use crate::sqlite_host::SqlStatement;
use crate::vault_codec::row::{has_bytes, is_deleted, logo_kind, str_col};
+use crate::vault_codec::blob_spec_for;
use crate::vault_codec::{CodecRecord, CodecTableData};
use crate::vault_model::names::{
- DELETED_AT_COL, FILE_DATA_COL, ID_COL, IS_DELETED_COL, ITEMS_TABLE, ITEM_ID_COL, KIND_COL, LOGOS_TABLE,
+ DELETED_AT_COL, FILE_DATA_COL, ID_COL, IS_DELETED_COL, ITEMS_TABLE, KIND_COL, LOGOS_TABLE,
LOGO_ID_COL, LOGO_KIND_FAVICON, UPDATED_AT_COL,
};
-use crate::vault_model::{BLOB_COLUMNS, MANIFEST_ID_COL, SYNCABLE_TABLES, TRASH_RETENTION_DEFAULT_DAYS};
+use crate::vault_model::{BLOB_COLUMNS, MANIFEST_ID_COL, SYNCABLE_TABLES};
/// Input for the prune operation.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
+#[derive(Debug, Clone)]
pub struct PruneInput {
/// Tables from the local database (at minimum, Items table is required)
pub tables: Vec<CodecTableData>,
/// Current time in ISO 8601 UTC format: `YYYY-MM-DDTHH:MM:SS.sssZ`.
- /*
- * Callers: the Rust sync engine (`now_iso_utc()`), Swift `ISO8601DateFormatter().string(from: Date())`,
- * Kotlin `Instant.now().toString()`.
- */
pub current_time: String,
- /// Retention period in days (default: 30)
- #[serde(default = "default_retention_days")]
+ /// Retention period in days.
pub retention_days: u32,
}
-fn default_retention_days() -> u32 {
- TRASH_RETENTION_DEFAULT_DAYS
-}
-
/// Statistics about what was pruned.
-#[derive(Debug, Clone, Default, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-#[cfg_attr(feature = "uniffi", derive(uniffi::Record))]
+#[derive(Debug, Clone, Default)]
pub struct PruneStats {
/// Number of items permanently deleted
pub items_pruned: u32,
/// Rows tombstoned per item-child table, keyed by table name.
- #[serde(default)]
pub child_rows_pruned: HashMap<String, u32>,
/// Number of orphan logos soft-deleted (no remaining active item references them)
- #[serde(default)]
pub logos_pruned: u32,
/// Tombstoned rows whose blob bytes were cleared, keyed by table name.
- #[serde(default)]
pub blobs_cleared: HashMap<String, u32>,
}
/// Output of the prune operation.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
+#[derive(Debug, Clone)]
pub struct PruneOutput {
- /// Whether the prune was successful
- pub success: bool,
/// SQL statements to execute on the local database (in order)
pub statements: Vec<SqlStatement>,
/// Statistics about what was pruned
@@ -71,8 +49,7 @@ pub struct PruneOutput {
}
/// A per-table SELECT query for building `PruneInput`.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[cfg_attr(feature = "uniffi", derive(uniffi::Record))]
+#[derive(Debug, Clone)]
pub struct PruneTableQuery {
/// Table name
pub name: String,
@@ -94,7 +71,7 @@ pub fn get_prune_table_queries() -> Vec<PruneTableQuery> {
// registry is read by the pruner automatically.
for child in item_child_tables() {
let query = match blob_column_for(child.name) {
- Some(blob_col) => format!("SELECT {}, {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", MANIFEST_ID_COL, ID_COL, ITEM_ID_COL, IS_DELETED_COL, blob_col, blob_col, child.name),
+ Some(blob_col) => format!("SELECT {}, {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", MANIFEST_ID_COL, ID_COL, child.item_ref_column(), IS_DELETED_COL, blob_col, blob_col, child.name),
None => format!("SELECT {}, {}, {} FROM {}", MANIFEST_ID_COL, child.item_ref_column(), IS_DELETED_COL, child.name),
};
queries.push(PruneTableQuery { name: child.name.to_string(), query });
@@ -111,9 +88,9 @@ fn item_child_tables() -> impl Iterator<Item = &'static crate::vault_model::Tabl
SYNCABLE_TABLES.iter().filter(|t| t.item_child)
}
-/// The extracted blob column of a table, if it has one (see `vault_model::BLOB_COLUMNS`).
+/// The extracted blob column of a table, if it has one.
fn blob_column_for(table_name: &str) -> Option<&'static str> {
- BLOB_COLUMNS.iter().find(|spec| spec.table == table_name).map(|spec| spec.column)
+ blob_spec_for(table_name).map(|spec| spec.column)
}
/// The `, <blob> = NULL` SET fragment that drops a table's blob bytes, empty for a table without one.
@@ -126,10 +103,7 @@ fn records_of<'a>(tables: &'a [CodecTableData], name: &str) -> Option<&'a [Codec
tables.iter().find(|t| t.name == name).map(|t| t.records.as_slice())
}
-/*
- * The manifest a row lives in. The same id can appear in multiple manifests, so every statement
- * addresses its rows by `(ManifestId, Id)`.
- */
+/// The manifest a row lives in; the same id can appear in several manifests, so statements address `(ManifestId, Id)`.
fn manifest_of(record: &CodecRecord) -> &str {
str_col(record, MANIFEST_ID_COL).unwrap_or("")
}
@@ -150,7 +124,7 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
// Items table is required for both the trash purge and the logo orphan check.
let Some(items) = records_of(&input.tables, ITEMS_TABLE) else {
- return Ok(PruneOutput { success: true, statements, stats });
+ return Ok(PruneOutput { statements, stats });
};
let expired = expired_item_ids(items, cutoff_date);
@@ -166,7 +140,7 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
sweep_orphan_favicons(&input.tables, items, &expired, &now_str, &mut statements, &mut stats);
clear_tombstoned_blobs(&input.tables, &now_str, &mut statements, &mut stats);
- Ok(PruneOutput { success: true, statements, stats })
+ Ok(PruneOutput { statements, stats })
}
/// Pass 1: the `(ManifestId, Id)` of live items whose trash date (`DeletedAt`) lies before the cutoff.
@@ -184,12 +158,7 @@ fn expired_item_ids(items: &[CodecRecord], cutoff_date: DateTime<Utc>) -> Vec<(S
expired
}
-/*
- * Pass 1 cascade: tombstone the rows of every registered item-child table (TableConfig::item_child)
- * for one purged item, so a table added to the registry is swept automatically. A child with an
- * extracted blob column has its bytes dropped in the same statement, leaving the column non-null
- * while reclaiming the storage on the next save.
- */
+/// Pass 1 cascade: tombstone one purged item's rows in every item-child table, dropping blob bytes in the same statement.
fn tombstone_item_children(tables: &[CodecTableData], manifest_id: &str, item_id: &str, now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {
for child in item_child_tables() {
let Some(records) = records_of(tables, child.name) else { continue };
@@ -210,17 +179,7 @@ fn tombstone_item_children(tables: &[CodecTableData], manifest_id: &str, item_id
}
}
-/*
- * Pass 2: orphan logo cleanup. A Logo is orphan when no Item with IsDeleted=0 references it. Items
- * being purged in Pass 1 are treated as effectively deleted so logos they referenced can be
- * reclaimed in the same call.
- *
- * Only `Kind = 'favicon'` rows are swept (a row without a `Kind` predates the column and is a favicon
- * by definition): a favicon is a per-domain cache the client can always refetch, while a built-in or
- * uploaded logo is a choice the user made and expects to find again in their logo library even after
- * the item that first used it is gone. Those are removed only when the user deletes them, and Pass 3
- * then reclaims the bytes.
- */
+/// Pass 2: tombstone favicons no live (and not just purged) item references; built-in and uploaded logos are the user's choice and stay.
fn sweep_orphan_favicons(
tables: &[CodecTableData],
items: &[CodecRecord],
@@ -255,13 +214,7 @@ fn sweep_orphan_favicons(
}
}
-/*
- * Pass 3: sweep tombstoned rows of every blob table (BLOB_COLUMNS) that still carry bytes and empty
- * them in place. Older clients could tombstone an attachment or favicon without clearing its blob,
- * which inflates the encrypted vault for no reason; for an uploaded logo it is the normal path, a
- * user deleting one from their logo library tombstones the row and this pass reclaims the bytes.
- * Rows tombstoned by Pass 1 or Pass 2 in this same call are already cleared there.
- */
+/// Pass 3: clear the blob bytes of tombstoned rows that still carry them.
fn clear_tombstoned_blobs(tables: &[CodecTableData], now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {
for spec in BLOB_COLUMNS {
let (table, blob_col) = (spec.table, spec.column);
diff --git a/core/rust/src/vault_pruner/tests.rs b/core/rust/src/vault_pruner/tests.rs
index 3c75291e3..297cf9d9d 100644
--- a/core/rust/src/vault_pruner/tests.rs
+++ b/core/rust/src/vault_pruner/tests.rs
@@ -1,738 +1,135 @@
+//! The pruner: expired trash is tombstoned with everything hanging off it, orphan favicons and the bytes of
+//! tombstoned blob rows are reclaimed, and every statement names the manifest it addresses.
+
use super::*;
-use crate::vault_codec::test_support::{days_ago_iso, now_iso};
+use crate::vault_codec::test_support::{days_ago_iso, now_iso, restamp, row, table, PERSONAL_MANIFEST_ID as M, SHARED_MANIFEST_ID};
use crate::vault_model::names::{ATTACHMENTS_TABLE, FIELD_HISTORIES_TABLE, FIELD_VALUES_TABLE, ITEM_TAGS_TABLE};
+use serde_json::{json, Value};
-/// The counter of one table in a per-table stats map, 0 when the table was not touched.
-fn count(map: &HashMap<String, u32>, table: &str) -> u32 {
- map.get(table).copied().unwrap_or(0)
+/// An item row in manifest `M`: live, trashed `days_ago` days, or already tombstoned.
+fn item(id: &str, trashed_days_ago: Option<i64>, deleted: bool, logo_id: Option<&str>) -> CodecRecord {
+ row(&[("ManifestId", json!(M)), ("Id", json!(id)), ("UpdatedAt", json!("2024-01-01T00:00:00Z")), ("IsDeleted", json!(deleted as i32)), ("DeletedAt", trashed_days_ago.map(days_ago_iso).map(Value::String).unwrap_or(Value::Null)), ("LogoId", logo_id.map(|l| json!(l)).unwrap_or(Value::Null))])
}
-fn make_item_record(id: &str, deleted_at: Option<&str>, is_deleted: bool) -> CodecRecord {
- let mut record = HashMap::new();
- record.insert("Id".to_string(), serde_json::json!(id));
- record.insert("UpdatedAt".to_string(), serde_json::json!("2024-01-01T00:00:00Z"));
- record.insert("IsDeleted".to_string(), serde_json::json!(if is_deleted { 1 } else { 0 }));
- if let Some(dt) = deleted_at {
- record.insert("DeletedAt".to_string(), serde_json::json!(dt));
- } else {
- record.insert("DeletedAt".to_string(), serde_json::Value::Null);
+/// A row hanging off `item_id` in manifest `M`, with a `Blob` cell when given.
+fn child(id: &str, item_id: &str, deleted: bool, blob: Option<Value>) -> CodecRecord {
+ let mut r = row(&[("ManifestId", json!(M)), ("Id", json!(id)), ("ItemId", json!(item_id)), ("IsDeleted", json!(deleted as i32))]);
+ if let Some(blob) = blob {
+ r.insert("Blob".to_string(), blob);
}
- record
-}
-
-fn make_field_value_record(id: &str, item_id: &str, is_deleted: bool) -> CodecRecord {
- let mut record = HashMap::new();
- record.insert("Id".to_string(), serde_json::json!(id));
- record.insert("ItemId".to_string(), serde_json::json!(item_id));
- record.insert("UpdatedAt".to_string(), serde_json::json!("2024-01-01T00:00:00Z"));
- record.insert("IsDeleted".to_string(), serde_json::json!(if is_deleted { 1 } else { 0 }));
- record
-}
-
-fn make_attachment_record(
- id: &str,
- item_id: &str,
- is_deleted: bool,
- blob: serde_json::Value,
-) -> CodecRecord {
- let mut record = HashMap::new();
- record.insert("Id".to_string(), serde_json::json!(id));
- record.insert("ItemId".to_string(), serde_json::json!(item_id));
- record.insert("UpdatedAt".to_string(), serde_json::json!("2024-01-01T00:00:00Z"));
- record.insert("IsDeleted".to_string(), serde_json::json!(if is_deleted { 1 } else { 0 }));
- record.insert("Blob".to_string(), blob);
- record
-}
-
-fn make_item_with_logo(
- id: &str,
- logo_id: Option<&str>,
- deleted_at: Option<&str>,
- is_deleted: bool,
-) -> CodecRecord {
- let mut record = make_item_record(id, deleted_at, is_deleted);
- match logo_id {
- Some(lid) => record.insert("LogoId".to_string(), serde_json::json!(lid)),
- None => record.insert("LogoId".to_string(), serde_json::Value::Null),
- };
- record
-}
-
-fn make_logo_record(id: &str, is_deleted: bool) -> CodecRecord {
- let mut record = HashMap::new();
- record.insert("Id".to_string(), serde_json::json!(id));
- record.insert("Source".to_string(), serde_json::json!("example.com"));
- record.insert("UpdatedAt".to_string(), serde_json::json!("2024-01-01T00:00:00Z"));
- record.insert("IsDeleted".to_string(), serde_json::json!(if is_deleted { 1 } else { 0 }));
- record
-}
-
-fn make_logo_record_with_blob(id: &str, is_deleted: bool, blob: serde_json::Value) -> CodecRecord {
- let mut record = make_logo_record(id, is_deleted);
- record.insert("FileData".to_string(), blob);
- record
-}
-
-#[test]
-fn test_prune_expired_items() {
- let now_str = now_iso();
- // Create an item deleted 60 days ago
- let old_date = days_ago_iso(60);
-
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![make_item_record("item-1", Some(&old_date), false)],
- },
- CodecTableData {
- name: "FieldValues".to_string(),
- records: vec![make_field_value_record("fv-1", "item-1", false)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert!(output.success);
- assert_eq!(output.stats.items_pruned, 1);
- assert_eq!(count(&output.stats.child_rows_pruned, FIELD_VALUES_TABLE), 1);
- assert!(output.statements.len() >= 2); // At least item + field value updates
-}
-
-#[test]
-fn test_no_prune_recent_items() {
- let now_str = now_iso();
- // Create an item deleted 10 days ago (within retention)
- let recent_date = days_ago_iso(10);
-
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![make_item_record("item-1", Some(&recent_date), false)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert!(output.success);
- assert_eq!(output.stats.items_pruned, 0);
- assert!(output.statements.is_empty());
-}
-
-#[test]
-fn test_no_prune_active_items() {
- let now_str = now_iso();
- // Create an item that's not in trash (DeletedAt is null)
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![make_item_record("item-1", None, false)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert!(output.success);
- assert_eq!(output.stats.items_pruned, 0);
- assert!(output.statements.is_empty());
-}
-
-#[test]
-fn test_no_prune_already_deleted() {
- let now_str = now_iso();
- // Create an item that's already permanently deleted
- let old_date = days_ago_iso(60);
-
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![make_item_record("item-1", Some(&old_date), true)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert!(output.success);
- assert_eq!(output.stats.items_pruned, 0);
- assert!(output.statements.is_empty());
+ r
}
-#[test]
-fn test_prune_json_api() {
- let now_str = now_iso();
- let old_date = days_ago_iso(60);
-
- let input_json = format!(r#"{{
- "tables": [{{
- "name": "Items",
- "records": [{{
- "Id": "item-1",
- "UpdatedAt": "2024-01-01T00:00:00Z",
- "IsDeleted": 0,
- "DeletedAt": "{}"
- }}]
- }}],
- "retentionDays": 30,
- "currentTime": "{}"
- }}"#, old_date, now_str);
-
- let output_json = crate::common::error::json_call(&input_json, prune_vault).unwrap();
- let output: PruneOutput = serde_json::from_str(&output_json).unwrap();
-
- assert!(output.success);
- assert_eq!(output.stats.items_pruned, 1);
+fn logo(id: &str, kind: &str, deleted: bool, file_data: Value) -> CodecRecord {
+ row(&[("ManifestId", json!(M)), ("Id", json!(id)), ("Kind", json!(kind)), ("Source", json!("example.com")), ("IsDeleted", json!(deleted as i32)), ("FileData", file_data)])
}
-fn logo_update_count(output: &PruneOutput) -> usize {
- output.statements.iter()
- .filter(|s| s.sql.starts_with("UPDATE Logos"))
- .count()
+fn prune(tables: Vec<CodecTableData>) -> PruneOutput {
+ prune_vault(PruneInput { tables, current_time: now_iso(), retention_days: 30 }).unwrap()
}
-#[test]
-fn test_orphan_logo_with_no_referencing_items_is_pruned() {
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![],
- },
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![make_logo_record("logo-orphan", false)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert!(output.success);
- assert_eq!(output.stats.logos_pruned, 1);
- assert_eq!(logo_update_count(&output), 1);
-}
-
-#[test]
-fn test_logo_referenced_by_active_item_is_kept() {
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![make_item_with_logo("item-1", Some("logo-1"), None, false)],
- },
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![make_logo_record("logo-1", false)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert!(output.success);
- assert_eq!(output.stats.logos_pruned, 0);
- assert_eq!(logo_update_count(&output), 0);
-}
-
-#[test]
-fn test_logo_referenced_only_by_tombstoned_item_is_pruned() {
- let now_str = now_iso();
- // The tombstoned item (IsDeleted=1) still has LogoId set; the logo
- // should be considered orphan since no active item references it.
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![make_item_with_logo("item-1", Some("logo-1"), None, true)],
- },
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![make_logo_record("logo-1", false)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(output.stats.logos_pruned, 1);
+fn sql_of(output: &PruneOutput) -> Vec<&str> {
+ output.statements.iter().map(|s| s.sql.as_str()).collect()
}
#[test]
-fn test_logo_referenced_only_by_item_being_purged_is_pruned() {
- let now_str = now_iso();
- // Item is in trash older than retention, so Pass 1 will tombstone it,
- // Pass 2 should reclaim its logo in the same call.
- let old_date = days_ago_iso(60);
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![make_item_with_logo("item-1", Some("logo-1"), Some(&old_date), false)],
- },
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![make_logo_record("logo-1", false)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
+fn expired_trash_is_tombstoned_as_a_unit() {
+ let output = prune(vec![
+ table("Items", vec![item("expired", Some(60), false, None)]),
+ table("FieldValues", vec![child("fv-1", "expired", false, None), child("fv-dead", "expired", true, None)]),
+ table("FieldHistories", vec![child("fh-1", "expired", false, None)]),
+ table("ItemTags", vec![child("it-1", "expired", false, None)]),
+ table("Attachments", vec![child("att-1", "expired", false, Some(json!("aGVsbG8=")))]),
+ ]);
assert_eq!(output.stats.items_pruned, 1);
- assert_eq!(output.stats.logos_pruned, 1);
-}
-
-#[test]
-fn test_logo_referenced_by_item_in_recent_trash_is_kept() {
- let now_str = now_iso();
- // Item is in trash but within retention, so it could still be restored,
- // so its logo must be preserved.
- let recent_date = days_ago_iso(10);
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![make_item_with_logo("item-1", Some("logo-1"), Some(&recent_date), false)],
- },
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![make_logo_record("logo-1", false)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(output.stats.items_pruned, 0);
- assert_eq!(output.stats.logos_pruned, 0);
-}
-
-#[test]
-fn test_orphan_logo_pruning_emits_filedata_clear_in_same_statement() {
- // Pass 2 must clear FileData when it tombstones a logo, otherwise the
- // encrypted vault keeps the blob bytes even after the row is "deleted".
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![],
- },
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![make_logo_record_with_blob("logo-orphan", false, serde_json::json!("aGVsbG8="))],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(output.stats.logos_pruned, 1);
- assert!(output.statements.iter().any(|s| s.sql.contains("FileData = NULL")));
-}
-
-#[test]
-fn test_tombstoned_logo_with_blob_bytes_is_swept() {
- // Pass 3 must catch historical logos that are IsDeleted=1 but still carry FileData
- // (e.g. tombstoned by an older client that did not drop FileData).
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![],
- },
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![make_logo_record_with_blob("logo-tombstoned", true, serde_json::json!("aGVsbG8="))],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(count(&output.stats.blobs_cleared, LOGOS_TABLE), 1);
- assert_eq!(output.stats.logos_pruned, 0);
-}
-
-#[test]
-fn test_tombstoned_logo_without_blob_is_not_touched() {
- // Logos already cleared shouldn't generate redundant updates.
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![],
- },
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![make_logo_record_with_blob("logo-tombstoned", true, serde_json::Value::Null)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(count(&output.stats.blobs_cleared, LOGOS_TABLE), 0);
-}
-
-#[test]
-fn test_already_soft_deleted_logo_is_not_re_pruned() {
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![],
- },
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![make_logo_record("logo-1", true)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(output.stats.logos_pruned, 0);
-}
-
-#[test]
-fn test_logo_pruning_skipped_when_logos_table_absent() {
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![make_item_with_logo("item-1", Some("logo-1"), None, false)],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(output.stats.logos_pruned, 0);
- assert_eq!(logo_update_count(&output), 0);
-}
-
-#[test]
-fn test_trash_purge_clears_attachment_blobs() {
- let now_str = now_iso();
- let old_date = days_ago_iso(60);
-
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![make_item_record("item-1", Some(&old_date), false)],
- },
- CodecTableData {
- name: "Attachments".to_string(),
- records: vec![make_attachment_record(
- "att-1",
- "item-1",
- false,
- serde_json::json!("aGVsbG8="),
- )],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(output.stats.items_pruned, 1);
- assert_eq!(count(&output.stats.child_rows_pruned, ATTACHMENTS_TABLE), 1);
- // The trash-purge UPDATE for Attachments should now also clear the blob.
- let attachment_update = output.statements.iter()
- .find(|s| s.sql.starts_with("UPDATE Attachments"))
- .expect("expected an UPDATE Attachments statement");
- assert!(attachment_update.sql.contains("Blob = NULL"),
- "attachment trash purge must drop the blob: {}", attachment_update.sql);
- // The pass-3 sweeper should not also fire for the same row in this call.
- assert_eq!(count(&output.stats.blobs_cleared, ATTACHMENTS_TABLE), 0);
-}
-
-#[test]
-fn test_sweeper_clears_blob_on_already_tombstoned_attachment() {
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![],
- },
- CodecTableData {
- name: "Attachments".to_string(),
- records: vec![make_attachment_record(
- "att-old",
- "item-1",
- true,
- serde_json::json!("aGVsbG8="),
- )],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(count(&output.stats.blobs_cleared, ATTACHMENTS_TABLE), 1);
- let stmt = output.statements.iter()
- .find(|s| s.sql.starts_with("UPDATE Attachments SET Blob = NULL"))
- .expect("expected the sweeper UPDATE");
- // params: [updated_at, attachment_id, manifest_id]
- assert_eq!(stmt.params.len(), 3);
- assert_eq!(stmt.params[1], serde_json::json!("att-old"));
-}
-
-#[test]
-fn test_sweeper_skips_already_empty_blob() {
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![],
- },
- CodecTableData {
- name: "Attachments".to_string(),
- records: vec![
- // Empty string (already cleared): should be skipped.
- make_attachment_record("att-empty-string", "item-1", true, serde_json::json!("")),
- // Empty array form: should also be skipped.
- make_attachment_record("att-empty-array", "item-1", true, serde_json::json!([])),
- // Null Blob: should also be skipped.
- make_attachment_record("att-null", "item-1", true, serde_json::Value::Null),
- ],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(count(&output.stats.blobs_cleared, ATTACHMENTS_TABLE), 0);
- assert!(output.statements.is_empty());
-}
-
-#[test]
-fn test_sweeper_skips_empty_uint8array_object_blob() {
- // An already-cleared blob must not generate a clear statement on every prune.
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![],
- },
- CodecTableData {
- name: "Attachments".to_string(),
- records: vec![make_attachment_record("att-empty-object", "item-1", true, serde_json::json!({}))],
- },
- CodecTableData {
- name: "Logos".to_string(),
- records: vec![make_logo_record_with_blob("logo-empty-object", true, serde_json::json!({}))],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(count(&output.stats.blobs_cleared, ATTACHMENTS_TABLE), 0);
- assert_eq!(count(&output.stats.blobs_cleared, LOGOS_TABLE), 0);
- assert!(output.statements.is_empty());
-}
-
-#[test]
-fn test_sweeper_clears_nonempty_uint8array_object_blob() {
- // Non-empty blobs are serialized as {"0":104,...}.
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![],
- },
- CodecTableData {
- name: "Attachments".to_string(),
- records: vec![make_attachment_record("att-object", "item-1", true, serde_json::json!({"0": 104, "1": 105}))],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(count(&output.stats.blobs_cleared, ATTACHMENTS_TABLE), 1);
-}
-
-#[test]
-fn test_sweeper_skips_active_attachment_with_blob() {
- let now_str = now_iso();
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![],
- },
- CodecTableData {
- name: "Attachments".to_string(),
- records: vec![make_attachment_record(
- "att-active",
- "item-1",
- false,
- serde_json::json!("aGVsbG8="),
- )],
- },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(count(&output.stats.blobs_cleared, ATTACHMENTS_TABLE), 0);
- assert!(output.statements.is_empty());
-}
-
-#[test]
-fn prune_queries_cover_every_item_child_table() {
- let names: Vec<String> = get_prune_table_queries().iter().map(|q| q.name.clone()).collect();
- for child in crate::vault_model::SYNCABLE_TABLES.iter().filter(|t| t.item_child) {
- assert!(names.contains(&child.name.to_string()), "pruner does not read item child table {}", child.name);
+ for (table, expected) in [(FIELD_VALUES_TABLE, 1), (FIELD_HISTORIES_TABLE, 1), (ITEM_TAGS_TABLE, 1), (ATTACHMENTS_TABLE, 1)] {
+ assert_eq!(output.stats.child_rows_pruned.get(table).copied().unwrap_or(0), expected, "{table}");
}
- assert!(names.contains(&ITEMS_TABLE.to_string()));
- assert!(names.contains(&LOGOS_TABLE.to_string()));
+ let sql = sql_of(&output);
+ assert!(sql.iter().any(|s| s.starts_with("UPDATE Items SET IsDeleted = 1")));
+ assert!(sql.iter().any(|s| s.starts_with("UPDATE FieldHistories SET IsDeleted = 1")) && sql.iter().any(|s| s.starts_with("UPDATE ItemTags SET IsDeleted = 1")), "{sql:?}");
+ let attachment = sql.iter().find(|s| s.starts_with("UPDATE Attachments")).expect("the attachment is tombstoned");
+ assert!(attachment.contains("Blob = NULL"), "the blob bytes go in the same statement: {attachment}");
+ assert!(output.stats.blobs_cleared.is_empty(), "the sweeper does not fire again for a row this call tombstoned");
}
#[test]
-fn test_prune_cascades_to_field_histories_and_item_tags() {
- let now_str = now_iso();
- let old_date = days_ago_iso(60);
-
- let mut history = HashMap::new();
- history.insert("ItemId".to_string(), serde_json::json!("item-1"));
- history.insert("IsDeleted".to_string(), serde_json::json!(0));
- let mut item_tag = HashMap::new();
- item_tag.insert("ItemId".to_string(), serde_json::json!("item-1"));
- item_tag.insert("IsDeleted".to_string(), serde_json::json!(0));
-
- let input = PruneInput {
- tables: vec![
- CodecTableData { name: "Items".to_string(), records: vec![make_item_record("item-1", Some(&old_date), false)] },
- CodecTableData { name: "FieldHistories".to_string(), records: vec![history] },
- CodecTableData { name: "ItemTags".to_string(), records: vec![item_tag] },
- ],
- retention_days: 30,
- current_time: now_str,
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(output.stats.items_pruned, 1);
- assert_eq!(count(&output.stats.child_rows_pruned, FIELD_HISTORIES_TABLE), 1);
- assert_eq!(count(&output.stats.child_rows_pruned, ITEM_TAGS_TABLE), 1);
- let sql: Vec<&str> = output.statements.iter().map(|s| s.sql.as_str()).collect();
- assert!(sql.iter().any(|s| s.starts_with("UPDATE FieldHistories SET IsDeleted = 1")), "field history rows must die with their item: {:?}", sql);
- assert!(sql.iter().any(|s| s.starts_with("UPDATE ItemTags SET IsDeleted = 1")), "item tag rows must die with their item: {:?}", sql);
-}
-
-fn in_manifest(mut record: CodecRecord, manifest_id: &str) -> CodecRecord {
- record.insert("ManifestId".to_string(), serde_json::json!(manifest_id));
- record
+fn items_that_are_live_in_recent_trash_or_already_tombstoned_are_left_alone() {
+ for (what, row) in [("live", item("i", None, false, None)), ("recently trashed", item("i", Some(10), false, None)), ("already tombstoned", item("i", Some(60), true, None))] {
+ let output = prune(vec![table("Items", vec![row])]);
+ assert_eq!(output.stats.items_pruned, 0, "{what}");
+ assert!(output.statements.is_empty(), "{what}");
+ }
}
#[test]
-fn a_prune_never_reaches_a_same_id_row_in_another_manifest() {
- let old_date = days_ago_iso(60);
- let input = PruneInput {
- tables: vec![
- CodecTableData {
- name: "Items".to_string(),
- records: vec![
- in_manifest(make_item_record("item-1", Some(&old_date), false), "m-personal"),
- in_manifest(make_item_record("item-1", None, false), "m-shared"),
- ],
- },
- CodecTableData {
- name: "FieldValues".to_string(),
- records: vec![
- in_manifest(make_field_value_record("fv-1", "item-1", false), "m-personal"),
- in_manifest(make_field_value_record("fv-1", "item-1", false), "m-shared"),
- ],
- },
- CodecTableData {
- name: "Attachments".to_string(),
- records: vec![
- in_manifest(make_attachment_record("att-1", "item-2", true, serde_json::json!("aGVsbG8=")), "m-personal"),
- in_manifest(make_attachment_record("att-1", "item-2", false, serde_json::json!("aGVsbG8=")), "m-shared"),
- ],
- },
- ],
- retention_days: 30,
- current_time: now_iso(),
- };
-
- let output = prune_vault(input).unwrap();
-
- assert_eq!(output.stats.items_pruned, 1);
- assert_eq!(count(&output.stats.child_rows_pruned, FIELD_VALUES_TABLE), 1);
- assert_eq!(count(&output.stats.blobs_cleared, ATTACHMENTS_TABLE), 1);
+fn favicons_no_live_item_references_are_swept_with_their_bytes() {
+ // A favicon can be fetched again, so an unreferenced one goes, in the same call that purges the item referencing
+ // it. A built-in or uploaded logo is the user's choice and stays. A logo an item in recent trash uses stays too.
+ let cases: [(&str, Vec<CodecRecord>, u32); 6] = [
+ ("unreferenced", vec![], 1),
+ ("referenced only by a tombstoned item", vec![item("i", None, true, Some("logo"))], 1),
+ ("referenced only by an item being purged", vec![item("i", Some(60), false, Some("logo"))], 1),
+ ("referenced by a live item", vec![item("i", None, false, Some("logo"))], 0),
+ ("referenced by an item in recent trash", vec![item("i", Some(10), false, Some("logo"))], 0),
+ ("already tombstoned", vec![], 0),
+ ];
+ for (what, items, expected) in cases {
+ let deleted = what == "already tombstoned";
+ let output = prune(vec![table("Items", items), table("Logos", vec![logo("logo", "favicon", deleted, json!("aGVsbG8="))])]);
+ assert_eq!(output.stats.logos_pruned, expected, "{what}");
+ if expected == 1 {
+ assert!(sql_of(&output).iter().any(|s| s.starts_with("UPDATE Logos") && s.contains("FileData = NULL")), "{what}: the bytes go with the tombstone");
+ }
+ }
+ let kept = prune(vec![table("Items", vec![]), table("Logos", vec![logo("custom", "custom", false, json!("aGVsbG8=")), logo("builtin", "builtin", false, Value::Null)])]);
+ assert_eq!(kept.stats.logos_pruned, 0, "only favicons are swept");
+ assert_eq!(prune(vec![table("Items", vec![item("i", None, false, Some("logo"))])]).stats.logos_pruned, 0, "no Logos table, nothing to sweep");
+}
+
+#[test]
+fn tombstoned_rows_that_still_carry_blob_bytes_are_emptied_once() {
+ // Older clients tombstoned attachments and logos without clearing their bytes; a cleared row is not touched again.
+ let swept = prune(vec![
+ table("Items", vec![]),
+ table("Attachments", vec![child("att-text", "i", true, Some(json!("aGVsbG8="))), child("att-object", "i", true, Some(json!({ "0": 104, "1": 105 }))), child("att-live", "i", false, Some(json!("aGVsbG8=")))]),
+ table("Logos", vec![logo("logo", "favicon", true, json!("aGVsbG8="))]),
+ ]);
+ assert_eq!(swept.stats.blobs_cleared.get(ATTACHMENTS_TABLE), Some(&2));
+ assert_eq!(swept.stats.blobs_cleared.get(LOGOS_TABLE), Some(&1));
+ assert_eq!(swept.stats.logos_pruned, 0, "a tombstoned logo is not pruned again");
+ let statement = swept.statements.iter().find(|s| s.sql.starts_with("UPDATE Attachments SET Blob = NULL")).unwrap();
+ assert_eq!(statement.params[1], json!("att-text"));
+
+ let empty_forms = [json!(""), json!([]), json!({}), Value::Null];
+ let untouched = prune(vec![table("Items", vec![]), table("Attachments", empty_forms.iter().enumerate().map(|(i, blob)| child(&format!("att-{i}"), "i", true, Some(blob.clone()))).collect()), table("Logos", vec![logo("logo", "favicon", true, json!({}))])]);
+ assert!(untouched.statements.is_empty(), "already-cleared blobs in any spelling generate no statement");
+}
+
+#[test]
+fn a_prune_addresses_rows_by_manifest_and_never_reaches_another_manifests_twin() {
+ let output = prune(vec![
+ table("Items", vec![item("item-1", Some(60), false, None), restamp(item("item-1", None, false, None), SHARED_MANIFEST_ID)]),
+ table("FieldValues", vec![child("fv-1", "item-1", false, None), restamp(child("fv-1", "item-1", false, None), SHARED_MANIFEST_ID)]),
+ table("Attachments", vec![child("att-1", "item-2", true, Some(json!("aGVsbG8="))), restamp(child("att-1", "item-2", false, Some(json!("aGVsbG8="))), SHARED_MANIFEST_ID)]),
+ ]);
+ assert_eq!((output.stats.items_pruned, output.stats.child_rows_pruned[FIELD_VALUES_TABLE], output.stats.blobs_cleared[ATTACHMENTS_TABLE]), (1, 1, 1));
assert_eq!(output.statements.len(), 3);
- for stmt in &output.statements {
- assert!(stmt.sql.ends_with("AND ManifestId = ?") || stmt.sql.contains("AND ManifestId = ? AND"), "statement must name its manifest: {}", stmt.sql);
- assert!(stmt.params.contains(&serde_json::json!("m-personal")), "statement must target the personal manifest: {:?}", stmt.params);
- assert!(!stmt.params.contains(&serde_json::json!("m-shared")), "statement must not reach the shared manifest: {:?}", stmt.params);
+ for statement in &output.statements {
+ assert!(statement.sql.contains("AND ManifestId = ?"), "statement must name its manifest: {}", statement.sql);
+ assert!(statement.params.contains(&json!(M)) && !statement.params.contains(&json!(SHARED_MANIFEST_ID)), "{:?}", statement.params);
}
}
#[test]
-fn every_prune_query_reads_the_manifest_id() {
- for table_query in get_prune_table_queries() {
- assert!(table_query.query.starts_with("SELECT ManifestId, "), "{} must be read with its manifest: {}", table_query.name, table_query.query);
+fn the_prune_queries_read_every_item_child_table_with_its_manifest() {
+ let queries = get_prune_table_queries();
+ let names: Vec<&str> = queries.iter().map(|q| q.name.as_str()).collect();
+ for child in crate::vault_model::SYNCABLE_TABLES.iter().filter(|t| t.item_child) {
+ assert!(names.contains(&child.name), "pruner does not read item child table {}", child.name);
+ }
+ assert!(names.contains(&ITEMS_TABLE) && names.contains(&LOGOS_TABLE));
+ for query in &queries {
+ assert!(query.query.starts_with("SELECT ManifestId, "), "{} must be read with its manifest: {}", query.name, query.query);
}
}
diff --git a/core/rust/src/vault_sharing/mod.rs b/core/rust/src/vault_sharing/mod.rs
deleted file mode 100644
index d8ee5dcf7..000000000
--- a/core/rust/src/vault_sharing/mod.rs
+++ /dev/null
@@ -1,72 +0,0 @@
-//! Sharing write logic for multi-manifest vaults.
-pub mod types;
-
-#[cfg(test)]
-mod tests;
-
-pub use types::{
- ManifestAccessPartition, ManifestAccessRequest, ManifestWriteRecord, ManifestWriteSet,
- ManifestWriteSetRequest, SharedManifestRecord, SkippedManifest, WriteSkipReason,
-};
-
-use crate::vault_model::id_key;
-
-/// Resolve which manifests the next push writes, personal manifest first.
-///
-/// A manifest is written when the vault holds rows for it and this session opened its key.
-pub fn resolve_manifest_write_set(request: ManifestWriteSetRequest) -> ManifestWriteSet {
- let stamped: Vec<String> = request.stamped_manifest_ids.iter().map(|id| id_key(id)).collect();
- let opened: Vec<String> = request.opened_manifest_ids.iter().map(|id| id_key(id)).collect();
-
- let mut records = vec![ManifestWriteRecord {
- manifest_id: request.personal_manifest_id,
- is_personal: true,
- salt: request.personal_manifest_salt,
- can_administer: false,
- }];
- let mut skipped = Vec::new();
-
- for record in &request.held_records {
- let key = id_key(&record.manifest_id);
-
- if !stamped.contains(&key) {
- skipped.push(SkippedManifest { manifest_id: record.manifest_id.clone(), reason: WriteSkipReason::NoRowsInVault });
- continue;
- }
-
- if !opened.contains(&key) {
- skipped.push(SkippedManifest { manifest_id: record.manifest_id.clone(), reason: WriteSkipReason::KeyDidNotOpen });
- continue;
- }
-
- records.push(ManifestWriteRecord {
- manifest_id: record.manifest_id.clone(),
- is_personal: false,
- salt: record.salt.clone(),
- can_administer: record.can_administer,
- });
- }
-
- ManifestWriteSet { records, skipped }
-}
-
-/// Split what the vault holds into what this session cannot write and what the account has lost.
-///
-/// Lost rows can never be written again and a merge keeps local-only rows, so leaving them would make
-/// every following push refuse itself.
-pub fn partition_manifest_access(request: ManifestAccessRequest) -> ManifestAccessPartition {
- let held: Vec<String> = request.manifest_ids_in_vault;
- let writable: Vec<String> = request.writable_manifest_ids.iter().map(|id| id_key(id)).collect();
- let granted: Vec<String> = request.granted_manifest_ids.iter().map(|id| id_key(id)).collect();
-
- let unwritable = held.iter().filter(|id| !writable.contains(&id_key(id))).cloned().collect();
-
- // An empty grant set means nothing is known yet, not that all access was revoked.
- let lost = match granted.is_empty() {
- true => Vec::new(),
- false => held.iter().filter(|id| !granted.contains(&id_key(id))).cloned().collect(),
- };
-
- ManifestAccessPartition { unwritable, lost }
-}
-
diff --git a/core/rust/src/vault_sharing/tests.rs b/core/rust/src/vault_sharing/tests.rs
deleted file mode 100644
index e192a49fa..000000000
--- a/core/rust/src/vault_sharing/tests.rs
+++ /dev/null
@@ -1,166 +0,0 @@
-//! Tests for the sharing policy.
-
-
-use super::*;
-
-/// A held key record for a manifest.
-fn record(manifest_id: &str) -> SharedManifestRecord {
- SharedManifestRecord {
- manifest_id: manifest_id.to_string(),
- salt: "salt-1".to_string(),
- can_administer: true,
- }
-}
-
-/// A write-set request with one personal manifest and the given shared records.
-fn write_set_request(stamped: &[&str], opened: &[&str], held: Vec<SharedManifestRecord>) -> ManifestWriteSetRequest {
- ManifestWriteSetRequest {
- personal_manifest_id: "PERSONAL".to_string(),
- personal_manifest_salt: "personal-salt".to_string(),
- stamped_manifest_ids: stamped.iter().map(|id| id.to_string()).collect(),
- opened_manifest_ids: opened.iter().map(|id| id.to_string()).collect(),
- held_records: held,
- }
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// The write set
-// ─────────────────────────────────────────────────────────────────────────────
-
-#[test]
-fn the_write_set_leads_with_the_personal_manifest() {
- let set = resolve_manifest_write_set(write_set_request(&["PERSONAL", "MAN-1"], &["MAN-1"], vec![record("MAN-1")]));
-
- assert_eq!(set.records.len(), 2);
- assert!(set.records[0].is_personal);
- assert_eq!(set.records[0].manifest_id, "PERSONAL");
- assert_eq!(set.records[0].salt, "personal-salt");
- // A personal manifest is not administered through a group.
- assert!(!set.records[0].can_administer);
-
- assert!(!set.records[1].is_personal);
- assert_eq!(set.records[1].manifest_id, "MAN-1");
- assert_eq!(set.records[1].salt, "salt-1");
- assert!(set.records[1].can_administer);
- assert!(set.skipped.is_empty());
-}
-
-#[test]
-fn a_manifest_with_no_local_rows_is_left_alone_rather_than_emptied() {
- let set = resolve_manifest_write_set(write_set_request(&["PERSONAL"], &["MAN-1"], vec![record("MAN-1")]));
-
- assert_eq!(set.records.len(), 1);
- assert_eq!(set.skipped.len(), 1);
- assert_eq!(set.skipped[0].manifest_id, "MAN-1");
- assert_eq!(set.skipped[0].reason, WriteSkipReason::NoRowsInVault);
-}
-
-#[test]
-fn a_manifest_whose_key_did_not_open_is_not_written() {
- let set = resolve_manifest_write_set(write_set_request(&["PERSONAL", "MAN-1"], &[], vec![record("MAN-1")]));
-
- assert_eq!(set.records.len(), 1);
- assert_eq!(set.skipped[0].reason, WriteSkipReason::KeyDidNotOpen);
-}
-
-#[test]
-fn the_write_set_matches_ids_regardless_of_casing() {
- let set = resolve_manifest_write_set(write_set_request(&["man-1"], &["MAN-1"], vec![record("Man-1")]));
-
- assert_eq!(set.records.len(), 2);
- // The record's own spelling comes back, since that is what addresses the write.
- assert_eq!(set.records[1].manifest_id, "Man-1");
- assert!(set.skipped.is_empty());
-}
-
-#[test]
-fn several_manifests_keep_the_order_they_were_held_in() {
- let held = vec![record("MAN-1"), record("MAN-2"), record("MAN-3")];
- let set = resolve_manifest_write_set(write_set_request(&["PERSONAL", "MAN-1", "MAN-3"], &["MAN-1", "MAN-2", "MAN-3"], held));
-
- let written: Vec<&str> = set.records.iter().map(|r| r.manifest_id.as_str()).collect();
- assert_eq!(written, vec!["PERSONAL", "MAN-1", "MAN-3"]);
- assert_eq!(set.skipped.len(), 1);
- assert_eq!(set.skipped[0].manifest_id, "MAN-2");
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Access partitioning
-// ─────────────────────────────────────────────────────────────────────────────
-
-#[test]
-fn access_splits_unwritable_from_lost() {
- let partition = partition_manifest_access(ManifestAccessRequest {
- manifest_ids_in_vault: vec!["PERSONAL".to_string(), "MAN-1".to_string(), "MAN-GONE".to_string()],
- writable_manifest_ids: vec!["personal".to_string(), "man-1".to_string()],
- granted_manifest_ids: vec!["PERSONAL".to_string(), "MAN-1".to_string()],
- });
-
- assert_eq!(partition.unwritable, vec!["MAN-GONE".to_string()]);
- assert_eq!(partition.lost, vec!["MAN-GONE".to_string()]);
-}
-
-#[test]
-fn nothing_is_lost_while_nothing_is_known() {
- // No snapshot this session, so nothing is known about what the account may open.
- let partition = partition_manifest_access(ManifestAccessRequest {
- manifest_ids_in_vault: vec!["PERSONAL".to_string(), "MAN-1".to_string()],
- writable_manifest_ids: vec!["PERSONAL".to_string()],
- granted_manifest_ids: vec![],
- });
-
- assert_eq!(partition.unwritable, vec!["MAN-1".to_string()]);
- assert!(partition.lost.is_empty());
-}
-
-#[test]
-fn a_manifest_that_is_served_but_did_not_open_is_unwritable_without_being_lost() {
- // A blob that fails to decrypt once is still this account's, so its rows must not be dropped.
- let partition = partition_manifest_access(ManifestAccessRequest {
- manifest_ids_in_vault: vec!["PERSONAL".to_string(), "MAN-1".to_string()],
- writable_manifest_ids: vec!["PERSONAL".to_string()],
- granted_manifest_ids: vec!["PERSONAL".to_string(), "MAN-1".to_string()],
- });
-
- assert_eq!(partition.unwritable, vec!["MAN-1".to_string()]);
- assert!(partition.lost.is_empty());
-}
-
-#[test]
-fn a_vault_that_agrees_with_its_access_reports_nothing() {
- let partition = partition_manifest_access(ManifestAccessRequest {
- manifest_ids_in_vault: vec!["PERSONAL".to_string(), "MAN-1".to_string()],
- writable_manifest_ids: vec!["PERSONAL".to_string(), "MAN-1".to_string()],
- granted_manifest_ids: vec!["PERSONAL".to_string(), "MAN-1".to_string()],
- });
-
- assert!(partition.unwritable.is_empty());
- assert!(partition.lost.is_empty());
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// The wire shape the hosts read
-// ─────────────────────────────────────────────────────────────────────────────
-
-#[test]
-fn the_json_boundary_round_trips() {
- let input = r#"{
- "personalManifestId": "PERSONAL",
- "personalManifestSalt": "personal-salt",
- "stampedManifestIds": ["PERSONAL", "MAN-1"],
- "openedManifestIds": ["MAN-1"],
- "heldRecords": [{ "manifestId": "MAN-1", "salt": "salt-1", "encryptedVek": "ignored", "canAdminister": true }]
- }"#;
-
- let output: serde_json::Value = serde_json::from_str(&crate::common::error::json_call(input, |request| Ok(resolve_manifest_write_set(request))).unwrap()).unwrap();
- assert_eq!(output["records"][1]["manifestId"], "MAN-1");
- assert_eq!(output["records"][1]["isPersonal"], false);
- assert_eq!(output["skipped"].as_array().unwrap().len(), 0);
-}
-
-#[test]
-fn a_skip_reason_is_a_stable_token() {
- let set = resolve_manifest_write_set(write_set_request(&["PERSONAL"], &[], vec![record("MAN-1")]));
- let json = serde_json::to_value(&set).unwrap();
- assert_eq!(json["skipped"][0]["reason"], "noRowsInVault");
-}
diff --git a/core/rust/src/vault_sharing/types.rs b/core/rust/src/vault_sharing/types.rs
deleted file mode 100644
index fe417a79f..000000000
--- a/core/rust/src/vault_sharing/types.rs
+++ /dev/null
@@ -1,104 +0,0 @@
-//! Input and output types for the sharing write logic.
-
-use serde::{Deserialize, Serialize};
-
-/// A shared manifest's key record.
-#[derive(Debug, Clone, Deserialize, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct SharedManifestRecord {
- /// Id of the manifest this record holds the key to.
- pub manifest_id: String,
- /// Salt this manifest's blob hashes are derived with.
- pub salt: String,
- /// Whether this account may publish the manifest's email delivery key.
- #[serde(default)]
- pub can_administer: bool,
-}
-
-/// Input for resolving a push's write set.
-#[derive(Debug, Clone, Deserialize, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct ManifestWriteSetRequest {
- /// The vault's own manifest.
- pub personal_manifest_id: String,
- /// Salt the personal manifest's blob hashes are derived with.
- pub personal_manifest_salt: String,
- /// Every manifest the local vault holds rows for.
- #[serde(default)]
- pub stamped_manifest_ids: Vec<String>,
- /// The shared manifests whose grant this session unwrapped.
- #[serde(default)]
- pub opened_manifest_ids: Vec<String>,
- /// The shared-manifest key records this client holds.
- #[serde(default)]
- pub held_records: Vec<SharedManifestRecord>,
-}
-
-/// Why a manifest is left out of the write.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub enum WriteSkipReason {
- /// The vault holds no rows for it, so writing would delete all its rows.
- NoRowsInVault,
- /// Its grant did not unwrap in this session.
- KeyDidNotOpen,
-}
-
-/// One manifest left out of the write, for the caller to log.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct SkippedManifest {
- /// Id of the manifest that was left out.
- pub manifest_id: String,
- /// Why it was left out.
- pub reason: WriteSkipReason,
-}
-
-/// One manifest the push writes. Carries no key; the caller looks the VEK up by id.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct ManifestWriteRecord {
- /// Id of the manifest to write.
- pub manifest_id: String,
- /// Whether this is the vault's own manifest.
- pub is_personal: bool,
- /// Salt this manifest's blob hashes are derived with.
- pub salt: String,
- /// Whether this account may publish the manifest's email delivery key.
- pub can_administer: bool,
-}
-
-/// The manifests a push writes, personal first, plus what was left out.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct ManifestWriteSet {
- /// The manifests to write, personal manifest first.
- pub records: Vec<ManifestWriteRecord>,
- /// The manifests left out, with the reason for each.
- pub skipped: Vec<SkippedManifest>,
-}
-
-/// Input for splitting a vault by what the account can still open.
-#[derive(Debug, Clone, Deserialize, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct ManifestAccessRequest {
- /// Every manifest the local vault holds rows for.
- #[serde(default)]
- pub manifest_ids_in_vault: Vec<String>,
- /// The manifests this session can write: the personal one plus every grant that opened.
- #[serde(default)]
- pub writable_manifest_ids: Vec<String>,
- /// The manifests the last snapshot served. Empty means nothing is known yet.
- #[serde(default)]
- pub granted_manifest_ids: Vec<String>,
-}
-
-/// What the vault holds but cannot write, and what it holds but has lost access to.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct ManifestAccessPartition {
- /// Held but not writable, which the caller repairs before pushing.
- pub unwritable: Vec<String>,
- /// Held but no longer served to this account, so its rows are deleted.
- pub lost: Vec<String>,
-}
diff --git a/core/rust/src/vault_sync/db.rs b/core/rust/src/vault_sync/db.rs
index b5427ff16..285b589f1 100644
--- a/core/rust/src/vault_sync/db.rs
+++ b/core/rust/src/vault_sync/db.rs
@@ -13,7 +13,7 @@ use crate::sqlite_host::SqlStatement;
use crate::common::timestamp::{now_iso_utc, now_vault_datetime};
use crate::vault_codec::row::{blob_ref_of, inline_bytes};
use crate::vault_codec::{is_skip_table, manifest_scoped_tables, CodecRecord, CodecTableData, MaterializedTables};
-use crate::vault_model::{id_key, ids_equal, MANIFEST_ID_COL, UNSTAMPED_SCOPE_SENTINEL};
+use crate::vault_model::{id_key, ids_equal, MANIFEST_ID_COL, UNSTAMPED_MANIFEST_ID};
pub(crate) type Row = Map<String, Value>;
@@ -57,12 +57,12 @@ pub(crate) fn value_string(value: &Value) -> String {
}
/// A row's cell as text, empty when absent.
-pub(crate) fn cell_string(row: &Row, column: &str) -> String {
+fn cell_string(row: &Row, column: &str) -> String {
row.get(column).map(value_string).unwrap_or_default()
}
/// The names of every user table.
-pub(crate) async fn list_user_tables(host: &Host, db: Db) -> SyncResult<Vec<String>> {
+async fn list_user_tables(host: &Host, db: Db) -> SyncResult<Vec<String>> {
let rows = query(host, db, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name", vec![]).await?;
Ok(rows.iter().filter_map(|row| row.get("name").and_then(Value::as_str).map(str::to_string)).collect())
}
@@ -138,15 +138,12 @@ pub(crate) async fn schema_state(host: &Host, current_migration_id: &str) -> Syn
Ok(if !current_migration_id.is_empty() && local.as_str() < current_migration_id { SchemaState::Stale } else { SchemaState::Current })
}
-pub(crate) async fn has_column(host: &Host, db: Db, table: &str, column: &str) -> SyncResult<bool> {
+/// Whether a table has a column.
+async fn has_column(host: &Host, db: Db, table: &str, column: &str) -> SyncResult<bool> {
Ok(!query(host, db, "SELECT name FROM pragma_table_info(?) WHERE name = ?", vec![json!(table), json!(column)]).await?.is_empty())
}
-/*
- * Every manifest id the rows of this vault name. This reads what the vault holds and nothing else: which
- * manifests may be written is decided by the grants the server served and the keys that opened, and the
- * two are compared so rows naming a manifest outside that set stop the push instead of being left out of it.
- */
+/// Every manifest id the rows of this vault name; the caller compares them with the writable set.
pub(crate) async fn manifest_ids_in_vault(host: &Host, db: Db) -> SyncResult<Vec<String>> {
// Table names come from the registry, never from the database; a vault on an older schema lacks some.
let columns = schema_columns(host, db).await?;
@@ -162,7 +159,7 @@ pub(crate) async fn manifest_ids_in_vault(host: &Host, db: Db) -> SyncResult<Vec
let rows = query(host, db, &format!("{} ORDER BY 1", selects.join(" UNION ")), vec![]).await?;
let mut ids: Vec<String> = Vec::new();
for id in rows.iter().filter_map(|row| row.get(MANIFEST_ID_COL).and_then(Value::as_str)) {
- if !id.is_empty() && !ids_equal(id, UNSTAMPED_SCOPE_SENTINEL) && !ids.iter().any(|known| known == id) {
+ if !id.is_empty() && !ids_equal(id, UNSTAMPED_MANIFEST_ID) && !ids.iter().any(|known| known == id) {
ids.push(id.to_string());
}
}
@@ -177,7 +174,7 @@ pub(crate) async fn insert_materialized(host: &Host, materialized: &Materialized
continue;
}
if !schema_columns.contains_key(&table.name) {
- host.log(LogLevel::Warn, format!("[VaultCodec] Skipping table \"{}\" ({} rows), not present in the schema.", table.name, table.records.len())).await;
+ host.log(LogLevel::Warn, format!("[Pull] Skipping table \"{}\" ({} rows), not present in the schema.", table.name, table.records.len())).await;
continue;
}
@@ -212,9 +209,7 @@ fn rows_rejected(table: &str, error: SyncError) -> SyncError {
}
}
-/// A materialized cell as a bind parameter: blob markers become bytes (NULL while the bytes are not loaded; the row's
-/// hash column says which blob it is),
-/// inline `{ __b64 }` payloads bind as bytes, everything else binds as is.
+/// A materialized cell as a bind parameter: blob markers bind as their bytes (NULL when not loaded), anything else as-is.
fn bind_value(value: &Value, blobs: &HashMap<String, Vec<u8>>) -> Value {
if let Some((reference, _)) = blob_ref_of(value) {
return blobs.get(reference).map(|bytes| inline_bytes(bytes)).unwrap_or(Value::Null);
@@ -256,9 +251,7 @@ pub(crate) async fn set_active_key_for_manifest(host: &Host, manifest_id: &str,
.await
}
-/*
- * What each manifest is called, keyed by lower-cased id.
- */
+/// What each manifest is called, keyed by lower-cased id.
pub(crate) async fn manifest_display_names(host: &Host) -> SyncResult<HashMap<String, String>> {
// A vault whose schema predates the table names none.
if !has_column(host, Db::Local, "Manifests", "Name").await? {
@@ -273,7 +266,7 @@ pub(crate) async fn manifest_display_names(host: &Host) -> SyncResult<HashMap<St
/// Name a manifest in the local vault.
pub(crate) async fn set_manifest_name(host: &Host, manifest_id: &str, name: &str) -> SyncResult<()> {
- exec(host, Db::Local, vec![set_manifest_name_statement(manifest_id, name)]).await
+ exec(host, Db::Local, vec![SqlStatement { sql: UPSERT_MANIFEST_NAME.to_string(), params: vec![json!(id_key(manifest_id)), json!(name)] }]).await
}
/// Name the manifests a database already holds; a manifest it does not hold stays out of it.
@@ -282,11 +275,6 @@ pub(crate) async fn apply_manifest_names(host: &Host, db: Db, names: &HashMap<St
exec(host, db, statements).await
}
-/// The statement that names a manifest.
-pub(crate) fn set_manifest_name_statement(manifest_id: &str, name: &str) -> SqlStatement {
- SqlStatement { sql: UPSERT_MANIFEST_NAME.to_string(), params: vec![json!(id_key(manifest_id)), json!(name)] }
-}
-
/// Prune expired trash items in place. Returns the number of statements executed.
pub(crate) async fn prune_in_place(host: &Host, retention_days: u32) -> SyncResult<usize> {
let mut tables = Vec::new();
diff --git a/core/rust/src/vault_sync/email_routing.rs b/core/rust/src/vault_sync/email_routing.rs
index ba173bcc4..745334d2a 100644
--- a/core/rust/src/vault_sync/email_routing.rs
+++ b/core/rust/src/vault_sync/email_routing.rs
@@ -21,7 +21,7 @@ pub(crate) fn build_email_routing(manifests: &[Manifest], private_email_domains:
for manifest in manifests {
let live_item_ids: HashSet<String> = rows_of(&manifest.tables, ITEMS_TABLE)
.iter()
- .filter(|row| !is_deleted(row) && row.get(DELETED_AT_COL).map_or(true, Value::is_null))
+ .filter(|row| !is_deleted(row) && row.get(DELETED_AT_COL).is_none_or(Value::is_null))
.filter_map(|row| row.get(ID_COL).map(value_string))
.collect();
diff --git a/core/rust/src/vault_sync/engine.rs b/core/rust/src/vault_sync/engine.rs
index 78daffeb3..c7ced40f3 100644
--- a/core/rust/src/vault_sync/engine.rs
+++ b/core/rust/src/vault_sync/engine.rs
@@ -4,22 +4,19 @@ use std::collections::HashMap;
use serde_json::Value;
-use super::db::SchemaState;
use super::errors::{LogoutReason, SyncError, SyncResult};
use super::merge::{self, PullAndMergeOutcome};
use super::pull::{self, PulledVault};
-use super::push::{self, CanonicalizedSet, PushStatus};
+use super::push::{self, CanonicalizedSet, PushStatus, WriteKind};
use super::session::Host;
use super::state::{self, Ctx};
-use super::types::{Db, FailureFields, FullSyncResult, LogLevel, MigrateManifestResult, MigrationKind, MigrationStatusResult, OperationResult, ResolveVaultKeyResult, SessionOutcome, StatusCheckResult, StatusResponse, SyncOperation, SyncRequest};
-use super::{db, http, keys, legacy, sharing};
+use super::types::{FailureFields, FullSyncResult, LogLevel, ResolveVaultKeyResult, StatusCheckResult, StatusResponse, SyncOperation, SyncRequest};
+use super::{db, http, keys, migration, sharing};
use crate::crypto;
use crate::vault_model::{id_key, ids_equal};
-/// How many times a chain of syncs may re-sync after an outdated push before giving up.
-/// This is a auto-healing mechanism in case two clients are pushing at the same time, which could
-/// result in a race condition. The sync mechanism retries to pull/merge/push up to 3 times if the
-/// server refuses any write as outdated.
+/// How many times a chain of syncs may re-sync after an outdated push before giving up. Two clients pushing at the
+/// same time race on the server's revision gate; the loser pulls, merges and pushes again, up to this many times.
const MAX_OUTDATED_RESYNCS: u32 = 3;
/// The pending action type this build carries out.
@@ -28,65 +25,35 @@ const ACTION_ROTATE_MANIFEST_DELIVERY_KEY: &str = "rotate-manifest-delivery-key"
/// Run one operation to completion; the value is the `done` command's result.
pub(crate) async fn run(host: Host, request: SyncRequest) -> Value {
let mut ctx = Ctx::new(host, request);
- match ctx.request.operation {
- SyncOperation::FullSync => {
- let result = full_sync_operation(&mut ctx).await;
- finish(&ctx, result)
- }
- SyncOperation::MigrationStatus => {
- let result = migration_status(&mut ctx).await;
- finish(&ctx, result)
- }
- SyncOperation::MigrateManifest => {
- let result = migrate_manifest(&mut ctx).await;
- finish(&ctx, result)
- }
- SyncOperation::StatusCheck => {
- let result = status_check(&mut ctx).await;
- finish(&ctx, result)
- }
- SyncOperation::ResolveVaultKey => {
- let result = resolve_vault_key(&mut ctx).await;
- finish(&ctx, result)
- }
- SyncOperation::CreateSharedManifest => {
- let result = sharing::create_shared_manifest_operation(&mut ctx).await;
- finish(&ctx, result)
- }
- SyncOperation::InviteToSharedManifest => {
- let result = sharing::invite_to_shared_manifest_operation(&mut ctx).await;
- finish(&ctx, result)
- }
- SyncOperation::UpdateSharedManifest => {
- let result = sharing::update_shared_manifest_operation(&mut ctx).await;
- finish(&ctx, result)
- }
- }
+ let result = match ctx.request.operation {
+ SyncOperation::FullSync => serde_json::to_value(full_sync_operation(&mut ctx).await),
+ SyncOperation::MigrationStatus => serde_json::to_value(migration::migration_status(&mut ctx).await),
+ SyncOperation::MigrateManifest => serde_json::to_value(migration::migrate_manifest(&mut ctx).await),
+ SyncOperation::StatusCheck => serde_json::to_value(status_check(&mut ctx).await),
+ SyncOperation::ResolveVaultKey => serde_json::to_value(resolve_vault_key(&mut ctx).await),
+ SyncOperation::CreateSharedManifest => serde_json::to_value(sharing::create_shared_manifest_operation(&mut ctx).await),
+ SyncOperation::InviteToSharedManifest => serde_json::to_value(sharing::invite_to_shared_manifest_operation(&mut ctx).await),
+ SyncOperation::UpdateSharedManifest => serde_json::to_value(sharing::update_shared_manifest_operation(&mut ctx).await),
+ };
+ finish(&ctx, result.unwrap_or(Value::Null))
}
-/// Attach the sync outcome onto an operation's result and serialize it.
-fn finish<T: OperationResult>(ctx: &Ctx, mut result: T) -> Value {
- *result.session_mut() = SessionOutcome { vault_changed: ctx.vault_changed };
- serde_json::to_value(result).unwrap_or(Value::Null)
+/// Attach the sync outcome (what every operation reports on top of its own) onto a serialized result.
+fn finish(ctx: &Ctx, mut result: Value) -> Value {
+ if let Some(object) = result.as_object_mut() {
+ object.insert("vaultChanged".to_string(), Value::Bool(ctx.vault_changed));
+ }
+ result
}
/// What one pass of the sync decided: a final outcome, or that the whole sync has to re-run.
+#[allow(clippy::large_enum_variant)]
enum Flow {
Done(FullSyncResult),
/// Re-run the sync; `outdated` counts against the resync limit.
Resync { outdated: bool },
}
-/// What the preflight decided: run the sync on this state, or stop with a final outcome.
-enum Preflight {
- Proceed {
- status: StatusResponse,
- needs_pull: bool,
- canonicalize_cache: Option<(u64, CanonicalizedSet)>,
- },
- Finish(FullSyncResult),
-}
-
fn success() -> FullSyncResult {
FullSyncResult { success: true, ..Default::default() }
}
@@ -96,6 +63,11 @@ fn failure(error: &SyncError) -> FullSyncResult {
FullSyncResult { failure: error.into(), ..Default::default() }
}
+/// The outcome of a sync that fell back on the local vault because the server could not be reached.
+fn offline() -> FullSyncResult {
+ FullSyncResult { success: true, was_offline: true, is_offline_mode: true, ..Default::default() }
+}
+
fn logout(reason: LogoutReason) -> FullSyncResult {
FullSyncResult { failure: FailureFields::logout(reason), ..Default::default() }
}
@@ -103,10 +75,10 @@ fn logout(reason: LogoutReason) -> FullSyncResult {
/// Turn an error thrown during a sync into the outcome the host acts on. A transport failure with a local vault
/// to fall back on is not a failure but puts the host in offline mode instead.
async fn map_sync_failure(ctx: &mut Ctx, error: SyncError) -> FullSyncResult {
- ctx.warn(format!("[VaultSync] Sync failed: {}", error)).await;
+ ctx.warn(format!("[Sync] Sync failed: {}", error)).await;
if let SyncError::Network(_) = &error {
if let Ok(true) = ctx.has_local_vault().await {
- return FullSyncResult { success: true, was_offline: true, is_offline_mode: true, ..Default::default() };
+ return offline();
}
}
failure(&error)
@@ -128,123 +100,130 @@ pub(crate) async fn full_sync(ctx: &mut Ctx) -> SyncResult<FullSyncResult> {
Flow::Resync { outdated } => {
if outdated {
if outdated_resyncs >= MAX_OUTDATED_RESYNCS {
- ctx.warn(format!("[VaultSync] The server still refuses the write after {} re-syncs; giving up on this chain.", MAX_OUTDATED_RESYNCS)).await;
+ ctx.warn(format!("[Sync] The server still refuses the write after {} re-syncs; giving up on this chain.", MAX_OUTDATED_RESYNCS)).await;
return Ok(failure(&SyncError::ResyncLimitReached));
}
outdated_resyncs += 1;
}
- ctx.log("[VaultSync] Re-running the sync.").await;
+ ctx.log("[Sync] Re-running the sync.").await;
}
}
}
}
async fn full_sync_once(ctx: &mut Ctx) -> SyncResult<Flow> {
- ctx.log("[VaultSync] Sync started").await;
- let (status, mut needs_pull, canonicalize_cache) = match run_sync_preflight(ctx).await? {
- Preflight::Proceed { status, needs_pull, canonicalize_cache } => (status, needs_pull, canonicalize_cache),
- Preflight::Finish(result) => return Ok(Flow::Done(result)),
+ ctx.log("[Sync] Sync started").await;
+ let status = match check_session(ctx).await? {
+ Ok(status) => status,
+ Err(result) => return Ok(Flow::Done(result)),
};
+ let mut needs_pull = ctx.request.force_pull || server_state_needs_pull(ctx, &status).await?;
+ ctx.log(format!("[Sync] Status received (needsPull {}, isDirty {})", needs_pull, ctx.is_dirty)).await;
+
+ // A hierarchy another device created since this legacy login shows up as a revision change, so it is checked here.
+ if needs_pull || ctx.is_dirty {
+ match keys::ensure_key_chain_accepted(ctx).await {
+ Err(SyncError::KeyOutOfSync) => return Ok(Flow::Done(logout(LogoutReason::PasswordChanged))),
+ other => {
+ other?;
+ }
+ }
+ }
announce_phase(ctx, needs_pull, ctx.is_dirty).await;
- let names_changed_vault = refresh_manifest_names(ctx, &status, needs_pull).await?;
- let grant_sync_changed_vault = apply_server_directed_changes(ctx, &status).await? || names_changed_vault;
- // The rows just reconciled can turn a clean vault dirty after the preflight decided; a dirty vault was checked there.
- if !needs_pull && grant_sync_changed_vault && push::vault_holds_unwritable_manifests(ctx).await {
- needs_pull = true;
- announce_phase(ctx, true, true).await;
+ let names_changed = refresh_manifest_names(ctx, &status, needs_pull).await?;
+ let server_directed_changes = apply_server_directed_changes(ctx, &status).await? || names_changed;
+
+ // A dirty vault on the current schema: drop the dirty flag when nothing changed, and pull first when it holds
+ // rows this session cannot write (a share that was revoked), since a push would refuse them.
+ let mut canonicalize_cache = None;
+ if !needs_pull && ctx.is_dirty && !migration::vault_predates_current_schema(ctx).await? {
+ canonicalize_cache = clear_no_op_mutation(ctx).await;
+ if ctx.is_dirty && push::vault_holds_unwritable_manifests(ctx).await {
+ needs_pull = true;
+ announce_phase(ctx, true, true).await;
+ }
}
let flow = if needs_pull {
- pull_and_materialize_server_vault(ctx, grant_sync_changed_vault).await?
- } else if ctx.is_dirty && !vault_predates_current_schema(ctx).await? {
- // Push path: server and client agree on every revision, so the pending local changes upload as-is.
- match push_local_changes(ctx, canonicalize_cache, false, false).await? {
- Some(vault_changed) => Flow::Done(FullSyncResult { success: true, has_new_vault: vault_changed || grant_sync_changed_vault, ..Default::default() }),
+ pull_and_materialize_server_vault(ctx, server_directed_changes).await?
+ } else if ctx.is_dirty && !migration::vault_predates_current_schema(ctx).await? {
+ // Server and client agree on every revision, so the pending local changes upload as-is.
+ match push_local_changes(ctx, canonicalize_cache, WriteKind::Changes { buckets_only: true }).await? {
+ Some(vault_changed) => Flow::Done(FullSyncResult { success: true, has_new_vault: vault_changed || server_directed_changes, ..Default::default() }),
None => Flow::Resync { outdated: true },
}
} else {
// A vault the codec cannot canonicalize has no way to the server; the upgrade gate takes it from here.
- Flow::Done(pending_migration_result(ctx).await?.unwrap_or_else(success))
+ Flow::Done(migration::pending_migration_result(ctx).await?.unwrap_or_else(success))
};
Ok(match flow {
Flow::Done(mut result) => {
result.server_version = Some(status.server_version.clone());
result.capabilities = status.capabilities.clone();
- ctx.log("[VaultSync] Sync finished").await;
+ ctx.log("[Sync] Sync finished").await;
Flow::Done(result)
}
other => other,
})
}
-/// Tell the host what the sync is about to do, so it can show a pull or push indicator.
-async fn announce_phase(ctx: &Ctx, needs_pull: bool, is_dirty: bool) {
- if needs_pull {
- ctx.host.log(LogLevel::Phase, "pull").await;
- } else if is_dirty {
- ctx.host.log(LogLevel::Phase, "push").await;
- }
-}
-
-/// Sanity checks before the sync touches the vault.
-async fn run_sync_preflight(ctx: &mut Ctx) -> SyncResult<Preflight> {
+/// The checks before the sync touches the vault: a usable session, a reachable server at a compatible version, and
+/// an unchanged password. `Err` carries the final outcome of a sync that stops here.
+async fn check_session(ctx: &mut Ctx) -> SyncResult<Result<StatusResponse, FullSyncResult>> {
if ctx.request.username.is_empty() {
- return Ok(Preflight::Finish(failure(&SyncError::Other("no username in the sync request".to_string()))));
+ return Ok(Err(failure(&SyncError::Other("no username in the sync request".to_string()))));
}
if ctx.encryption_key.is_none() {
- return Ok(Preflight::Finish(failure(&SyncError::VaultLocked)));
+ return Ok(Err(failure(&SyncError::VaultLocked)));
}
-
let status = http::get_status(&ctx.host).await?;
- let mut needs_pull = ctx.request.force_pull || server_state_needs_pull(ctx, &status).await?;
- ctx.log(format!("[VaultSync] Status received (needsPull {}, isDirty {})", needs_pull, ctx.is_dirty)).await;
-
if status.server_version == http::SERVER_UNREACHABLE_VERSION {
- return Ok(Preflight::Finish(enter_offline_mode(ctx).await?));
- }
- if !status.client_version_supported {
- return Ok(Preflight::Finish(logout(LogoutReason::ClientVersionNotSupported)));
+ return Ok(Err(enter_offline_mode(ctx).await?));
}
- if server_too_old(ctx, &status) {
- return Ok(Preflight::Finish(logout(LogoutReason::ServerVersionNotSupported)));
+ if let Some(reason) = version_logout_reason(ctx, &status) {
+ return Ok(Err(logout(reason)));
}
-
// A changed server salt means the password was changed elsewhere, which warrants a logout.
assert_salt_unchanged(ctx, status.srp_salt.as_deref()).await?;
+ Ok(Ok(status))
+}
- // The session key is the VEK (or a legacy account's unlock key) by contract; the host resolved it at login. The one
- // case the sync has to catch itself: this device holds no chain because the account was still legacy when it
- // logged in, and another device created the hierarchy since. That shows up as a revision change, so a pull
- // (or a push about to hit the server) is where it is checked.
- if (needs_pull || ctx.is_dirty) && !keys::has_local_vault_key(&ctx.host).await? && !keys::accept_hierarchy_created_elsewhere(ctx).await? {
- return Ok(Preflight::Finish(logout(LogoutReason::PasswordChanged)));
+/// Tell the host what the sync is about to do, so it can show a pull or push indicator.
+async fn announce_phase(ctx: &Ctx, needs_pull: bool, is_dirty: bool) {
+ if needs_pull {
+ ctx.host.log(LogLevel::Phase, "pull").await;
+ } else if is_dirty {
+ ctx.host.log(LogLevel::Phase, "push").await;
}
+}
- let mut canonicalize_cache = None;
- if ctx.is_dirty && !needs_pull && !vault_predates_current_schema(ctx).await? {
- match push::detect_no_op_mutation(ctx).await {
- Ok((true, _)) => {
- if state::mark_clean(&ctx.host, ctx.mutation_sequence).await? {
- ctx.is_dirty = false;
- }
+/// Clear the dirty flag of a vault whose content equals the last known server state. Returns the canonicalize result
+/// when the vault did change, so the push right after does not canonicalize again.
+async fn clear_no_op_mutation(ctx: &mut Ctx) -> Option<(u64, CanonicalizedSet)> {
+ match push::detect_no_op_mutation(ctx).await {
+ Ok((true, _)) => {
+ if let Ok(true) = state::mark_clean(&ctx.host, ctx.mutation_sequence).await {
+ ctx.is_dirty = false;
}
- Ok((false, set)) => canonicalize_cache = Some((ctx.mutation_sequence, set)),
- Err(error) => ctx.warn(format!("[VaultSync] No-op mutation pre-check failed, proceeding with a normal push: {}", error)).await,
+ None
+ }
+ Ok((false, set)) => Some((ctx.mutation_sequence, set)),
+ Err(error) => {
+ ctx.warn(format!("[Sync] No-op mutation pre-check failed, proceeding with a normal push: {}", error)).await;
+ None
}
}
-
- if ctx.is_dirty && !needs_pull && push::vault_holds_unwritable_manifests(ctx).await {
- needs_pull = true;
- }
-
- Ok(Preflight::Proceed { status, needs_pull, canonicalize_cache })
}
-/// Whether the server is older than the oldest version this client supports.
-fn server_too_old(ctx: &Ctx, status: &StatusResponse) -> bool {
- ctx.request.min_server_version.as_deref().map(|min| !version_gte(&status.server_version, min)).unwrap_or(false)
+/// Why this client and server cannot work together, if they cannot.
+fn version_logout_reason(ctx: &Ctx, status: &StatusResponse) -> Option<LogoutReason> {
+ if !status.client_version_supported {
+ return Some(LogoutReason::ClientVersionNotSupported);
+ }
+ let server_too_old = ctx.request.min_server_version.as_deref().is_some_and(|min| !version_gte(&status.server_version, min));
+ server_too_old.then_some(LogoutReason::ServerVersionNotSupported)
}
/// Whether `version1 >= version2` under the client's SemVer rules: a pre-release sorts below its release, and
@@ -258,11 +237,8 @@ fn version_gte(version1: &str, version2: &str) -> bool {
for index in 0..parts1.len().max(parts2.len()) {
let part1 = parts1.get(index).copied().unwrap_or(0);
let part2 = parts2.get(index).copied().unwrap_or(0);
- if part1 > part2 {
- return true;
- }
- if part1 < part2 {
- return false;
+ if part1 != part2 {
+ return part1 > part2;
}
}
@@ -293,10 +269,10 @@ async fn server_state_needs_pull(ctx: &Ctx, status: &StatusResponse) -> SyncResu
let manifests_to_pull = requiring_pull(&server_manifests, &local_manifests);
let buckets_to_pull = requiring_pull(&server_buckets, &local_buckets);
if manifests_to_pull == 0 && buckets_to_pull == 0 {
- ctx.log("[VaultSync] No pull needed: every manifest and data bucket matches the local revisions.").await;
+ ctx.log("[Sync] No pull needed: every manifest and data bucket matches the local revisions.").await;
return Ok(false);
}
- ctx.log(format!("[VaultSync] Pull needed for {} manifest(s) and {} data bucket(s).", manifests_to_pull, buckets_to_pull)).await;
+ ctx.log(format!("[Sync] Pull needed for {} manifest(s) and {} data bucket(s).", manifests_to_pull, buckets_to_pull)).await;
Ok(true)
}
@@ -322,31 +298,13 @@ async fn enter_offline_mode(ctx: &mut Ctx) -> SyncResult<FullSyncResult> {
let unreachable = SyncError::Network("server unreachable and no local vault to fall back on".to_string());
return Ok(FullSyncResult { was_offline: true, is_offline_mode: true, ..failure(&unreachable) });
}
- Ok(FullSyncResult { success: true, was_offline: true, is_offline_mode: true, ..Default::default() })
-}
-
-/// Where the local vault's schema stands against the current one.
-pub(crate) async fn schema_state(ctx: &mut Ctx) -> SyncResult<SchemaState> {
- let schema = ctx.schema().await?;
- db::schema_state(&ctx.host, &schema.migration_id).await
-}
-
-/// Whether the local vault is still on a schema the codec cannot canonicalize.
-async fn vault_predates_current_schema(ctx: &mut Ctx) -> SyncResult<bool> {
- Ok(schema_state(ctx).await? != SchemaState::Current)
-}
-
-/// Whether the vault still has to run the manifest migration: a stale schema, or a missing account key hierarchy.
-pub(crate) async fn vault_requires_manifest_migration(ctx: &mut Ctx) -> SyncResult<bool> {
- Ok(schema_state(ctx).await? == SchemaState::Stale || !keys::has_local_vault_key(&ctx.host).await?)
+ Ok(offline())
}
-/*
- * Fetch the shared manifest names the status served and store them in the local vault.
- */
+/// Fetch the shared manifest names the status served and store them in the local vault.
async fn refresh_manifest_names(ctx: &mut Ctx, status: &StatusResponse, needs_pull: bool) -> SyncResult<bool> {
ctx.served_manifest_names = status.manifest_revisions.iter().filter_map(|m| Some((id_key(&m.manifest_id), m.encrypted_name.clone()?))).collect();
- if needs_pull || ctx.served_manifest_names.is_empty() || !ctx.has_local_vault().await? || vault_requires_manifest_migration(ctx).await? {
+ if needs_pull || ctx.served_manifest_names.is_empty() || !ctx.has_local_vault().await? || migration::vault_requires_manifest_migration(ctx).await? {
return Ok(false);
}
@@ -366,88 +324,72 @@ async fn refresh_manifest_names(ctx: &mut Ctx, status: &StatusResponse, needs_pu
}
let key = ctx.encryption_key()?;
- db::apply_manifest_names(&ctx.host, Db::Local, &opened).await?;
- let bytes = db::export(&ctx.host, Db::Local).await?;
- ctx.store_vault(&state::encrypt_vault_blob(&bytes, &key)?, false, None, None).await?;
- keys::set_shared_manifest_records(&ctx.host, &records, &key).await?;
- ctx.vault_changed = true;
+ db::apply_manifest_names(&ctx.host, super::types::Db::Local, &opened).await?;
+ ctx.persist_local_vault(false).await?;
+ keys::set_shared_manifest_records(ctx, &records, &key).await?;
ctx.log(format!("[Sharing] Refreshed the name of {} shared manifest(s).", opened.len())).await;
Ok(true)
}
-/// Carry out the work the server has addressed to the current client (e.g. shared groups revocation/rotation actions).
+/// Carry out the pending actions the server queued for this client.
async fn apply_server_directed_changes(ctx: &mut Ctx, status: &StatusResponse) -> SyncResult<bool> {
+ if status.pending_actions.is_empty() || !ctx.has_local_vault().await? || migration::vault_requires_manifest_migration(ctx).await? {
+ return Ok(false);
+ }
let shared = keys::shared_manifest_records(ctx).await?;
- let can_reconcile = ctx.has_local_vault().await? && !vault_requires_manifest_migration(ctx).await?;
let mut vault_changed = false;
-
- if can_reconcile {
- let mut completed = 0usize;
- for action in &status.pending_actions {
- let outcome: SyncResult<(bool, bool)> = async {
- if action.action_type != ACTION_ROTATE_MANIFEST_DELIVERY_KEY {
- ctx.log(format!("[PendingActions] Action type {} is not known to this client; leaving it for one that knows it.", action.action_type)).await;
- return Ok((false, false));
- }
- let record = shared.values().find(|r| action.manifest_id.as_deref().map(|id| ids_equal(id, &r.manifest_id)).unwrap_or(false));
- let Some(record) = record.filter(|r| r.can_administer) else {
- ctx.log(format!("[PendingActions] Manifest {:?} is not open to this session as an administrator; leaving its delivery key rotation for another sync.", action.manifest_id)).await;
- return Ok((false, false));
- };
- let pair = crypto::generate_rsa_key_pair()?;
- db::set_active_key_for_manifest(&ctx.host, &record.manifest_id, &pair.public_key, &pair.private_key).await?;
- ctx.log(format!("[PendingActions] Rotated the mail delivery key of shared manifest {}.", record.manifest_id)).await;
- Ok((true, true))
- }
- .await;
- match outcome {
- Ok((changed, done)) => {
- vault_changed |= changed;
- if done {
- http::delete(&ctx.host, &format!("ClientActions/{}", action.id)).await?;
- completed += 1;
- }
- }
- Err(error) => ctx.warn(format!("[PendingActions] Could not carry out action {} ({}); retrying on the next sync. {}", action.action_type, action.id, error)).await,
+ let mut completed = 0usize;
+ for action in &status.pending_actions {
+ if action.action_type != ACTION_ROTATE_MANIFEST_DELIVERY_KEY {
+ ctx.log(format!("[Sync] Action type {} is not known to this client; leaving it for one that knows it.", action.action_type)).await;
+ continue;
+ }
+ let record = shared.values().find(|r| action.manifest_id.as_deref().is_some_and(|id| ids_equal(id, &r.manifest_id))).filter(|r| r.can_administer);
+ let Some(record) = record else {
+ ctx.log(format!("[Sync] Manifest {:?} is not open to this session as an administrator; leaving its delivery key rotation for another sync.", action.manifest_id)).await;
+ continue;
+ };
+ let rotated: SyncResult<()> = async {
+ let pair = crypto::generate_rsa_key_pair()?;
+ db::set_active_key_for_manifest(&ctx.host, &record.manifest_id, &pair.public_key, &pair.private_key).await?;
+ http::delete(&ctx.host, &format!("ClientActions/{}", action.id)).await
+ }
+ .await;
+ match rotated {
+ Ok(()) => {
+ ctx.log(format!("[Sync] Rotated the mail delivery key of shared manifest {}.", record.manifest_id)).await;
+ vault_changed = true;
+ completed += 1;
}
- }
- if !status.pending_actions.is_empty() {
- ctx.log(format!("[PendingActions] {} of {} action(s) carried out.", completed, status.pending_actions.len())).await;
+ Err(error) => ctx.warn(format!("[Sync] Could not carry out action {} ({}); retrying on the next sync. {}", action.action_type, action.id, error)).await,
}
}
+ ctx.log(format!("[Sync] {} of {} server-directed action(s) carried out.", completed, status.pending_actions.len())).await;
- if !vault_changed {
- return Ok(false);
- }
-
- let key = ctx.encryption_key()?;
- let bytes = db::export(&ctx.host, Db::Local).await?;
- let stored = ctx.store_vault(&state::encrypt_vault_blob(&bytes, &key)?, true, None, None).await?;
- let was_dirty = ctx.is_dirty;
- ctx.is_dirty = true;
- ctx.mutation_sequence = stored.mutation_sequence;
- ctx.vault_changed = true;
- if !was_dirty {
- // The reconciled rows ride out on this very sync, which the host had not announced as a push yet.
- announce_phase(ctx, false, true).await;
+ if vault_changed {
+ let was_dirty = ctx.is_dirty;
+ ctx.persist_local_vault(true).await?;
+ if !was_dirty {
+ // The reconciled rows ride out on this very sync, which the host had not announced as a push yet.
+ announce_phase(ctx, false, true).await;
+ }
}
- Ok(true)
+ Ok(vault_changed)
}
/// Outcome for a sync that stored a freshly pulled vault.
async fn materialized_vault_result(ctx: &mut Ctx, pulled: &PulledVault) -> SyncResult<FullSyncResult> {
- let schema = schema_state(ctx).await?;
- let manifest_migration_required = schema == SchemaState::Stale || !keys::has_local_vault_key(&ctx.host).await?;
- Ok(FullSyncResult { success: true, has_new_vault: true, sqlite_blob_upgrade_required: schema == SchemaState::LegacyChain, manifest_migration_required, pulled_revision: Some(pulled.revision), email_routing: Some(pulled.email_routing.clone()), ..Default::default() })
+ let migration_required = migration::vault_requires_manifest_migration(ctx).await?;
+ let legacy_chain = migration::schema_state(ctx).await? == db::SchemaState::LegacyChain;
+ Ok(FullSyncResult { success: true, has_new_vault: true, sqlite_blob_upgrade_required: legacy_chain, manifest_migration_required: migration_required, pulled_revision: Some(pulled.personal_revision), email_routing: Some(pulled.email_routing.clone()), ..Default::default() })
}
-/// Store a pulled vault as the local one and only then commit its revisions as the local truth. A store the host
-/// refuses (a mutation raced the pull) re-runs the sync instead, which is the returned flow. A new vault that was
-/// never written is stored dirty, so a push that does not land now is retried by the next sync.
+/// Store a pulled vault, then commit its revisions; a refused store (a mutation raced the pull) returns a resync.
+/// A never-written new vault is stored dirty, so a push that does not land now is retried by the next sync.
async fn commit_pulled_vault(ctx: &mut Ctx, pulled: &PulledVault) -> SyncResult<Option<Flow>> {
- let stored = ctx.store_vault(&pulled.encrypted_vault, pulled.needs_first_write, Some(ctx.mutation_sequence), Some(pulled.revision)).await?;
+ let stored = ctx.store_vault(&pulled.encrypted_vault, pulled.needs_first_write, Some(ctx.mutation_sequence), Some(pulled.personal_revision)).await?;
if !stored.success {
- ctx.log("[VaultSync] Mutation detected during sync, re-syncing...").await;
+ ctx.log("[Sync] Mutation detected during sync, re-syncing...").await;
return Ok(Some(Flow::Resync { outdated: false }));
}
if pulled.needs_first_write {
@@ -455,7 +397,7 @@ async fn commit_pulled_vault(ctx: &mut Ctx, pulled: &PulledVault) -> SyncResult<
ctx.mutation_sequence = stored.mutation_sequence;
}
ctx.vault_changed = true;
- pull::commit_revisions(ctx, &pulled.manifest_revisions, &pulled.bucket_revisions).await?;
+ pull::commit_revisions(ctx, pulled).await?;
Ok(None)
}
@@ -465,8 +407,8 @@ async fn store_server_vault(ctx: &mut Ctx, pulled: &PulledVault) -> SyncResult<F
return Ok(resync);
}
if pulled.needs_first_write {
- ctx.log("[VaultSync] Writing the first revision of the new vault.").await;
- if push_local_changes(ctx, None, true, false).await?.is_none() {
+ ctx.log("[Sync] Writing the first revision of the new vault.").await;
+ if push_local_changes(ctx, None, WriteKind::Everything).await?.is_none() {
return Ok(Flow::Resync { outdated: true });
}
}
@@ -474,43 +416,34 @@ async fn store_server_vault(ctx: &mut Ctx, pulled: &PulledVault) -> SyncResult<F
}
/// Pull the server's latest vault and merge it with what is stored locally when needed.
-async fn pull_and_materialize_server_vault(ctx: &mut Ctx, grant_sync_changed_vault: bool) -> SyncResult<Flow> {
+async fn pull_and_materialize_server_vault(ctx: &mut Ctx, server_directed_changes: bool) -> SyncResult<Flow> {
if !ctx.is_dirty || !ctx.has_local_vault().await? {
- return store_pulled_vault(ctx).await;
+ let pulled = pull::pull(ctx).await?;
+ return store_server_vault(ctx, &pulled).await;
}
-
// A dirty vault the codec cannot canonicalize is not merged.
- if vault_predates_current_schema(ctx).await? {
- ctx.warn("[VaultSync] The local vault predates the current storage model, so its pending changes can be neither merged nor uploaded; taking the server's vault.").await;
- return store_pulled_vault(ctx).await;
+ if migration::vault_predates_current_schema(ctx).await? {
+ ctx.warn("[Sync] The local vault predates the current storage model, so its pending changes can be neither merged nor uploaded; taking the server's vault.").await;
+ let pulled = pull::pull(ctx).await?;
+ return store_server_vault(ctx, &pulled).await;
}
- canonical_pull_and_merge(ctx, grant_sync_changed_vault).await
-}
-/// Pull the server's vault and store it as the local one.
-async fn store_pulled_vault(ctx: &mut Ctx) -> SyncResult<Flow> {
- let pulled = pull::pull(ctx).await?;
- store_server_vault(ctx, &pulled).await
-}
-
-/// Canonical-merge path of a dirty pull.
-async fn canonical_pull_and_merge(ctx: &mut Ctx, grant_sync_changed_vault: bool) -> SyncResult<Flow> {
match merge::pull_and_merge(ctx).await? {
PullAndMergeOutcome::LegacyServer { revision } => {
- ctx.warn(format!("[VaultSync] Server vault (revision {}) is still on the legacy storage format while the local vault is migrated; pushing the local vault.", revision)).await;
- if push_local_changes(ctx, None, true, false).await?.is_none() {
+ ctx.warn(format!("[Sync] Server vault (revision {}) is still on the legacy storage format while the local vault is migrated; pushing the local vault.", revision)).await;
+ if push_local_changes(ctx, None, WriteKind::Everything).await?.is_none() {
return Ok(Flow::Resync { outdated: true });
}
- Ok(Flow::Done(FullSyncResult { success: true, has_new_vault: grant_sync_changed_vault, ..Default::default() }))
+ Ok(Flow::Done(FullSyncResult { success: true, has_new_vault: server_directed_changes, ..Default::default() }))
}
PullAndMergeOutcome::ServerOnly(pulled) => store_server_vault(ctx, &pulled).await,
- PullAndMergeOutcome::Merged { pulled, stats, fallback_manifest_ids, dropped_local_manifest_ids, push_canonical } => {
+ PullAndMergeOutcome::Merged { pulled, push_canonical } => {
if let Some(resync) = commit_pulled_vault(ctx, &pulled).await? {
return Ok(resync);
}
- ctx.log(format!("[VaultSync] Canonical vault merge completed: {:?}; {} validation fallback(s), {} dropped local manifest(s).", stats, fallback_manifest_ids.len(), dropped_local_manifest_ids.len())).await;
+ // The merge may have changed any manifest, so the push after it is never bucket-only.
let cache = push_canonical.map(|set| (ctx.mutation_sequence, set));
- if push_local_changes(ctx, cache, false, false).await?.is_none() {
+ if push_local_changes(ctx, cache, WriteKind::Changes { buckets_only: false }).await?.is_none() {
return Ok(Flow::Resync { outdated: true });
}
Ok(Flow::Done(materialized_vault_result(ctx, &pulled).await?))
@@ -518,9 +451,9 @@ async fn canonical_pull_and_merge(ctx: &mut Ctx, grant_sync_changed_vault: bool)
}
}
-/// Push the pending local changes and clear the dirty flag.
-async fn push_local_changes(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)>, force_full_write: bool, create_vault_key: bool) -> SyncResult<Option<bool>> {
- let upload = push::upload_vault(ctx, cache, force_full_write, create_vault_key).await?;
+/// Push the pending local changes and clear the dirty flag. `None` when the server refused the write as outdated.
+async fn push_local_changes(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)>, kind: WriteKind) -> SyncResult<Option<bool>> {
+ let upload = push::upload_vault(ctx, cache, kind).await?;
match upload.status {
PushStatus::Ok => {
state::mark_clean(&ctx.host, upload.mutation_seq_at_start).await?;
@@ -530,138 +463,12 @@ async fn push_local_changes(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)
}
}
-/// Check for any pending migrations of the stored vault.
-async fn pending_migration_result(ctx: &mut Ctx) -> SyncResult<Option<FullSyncResult>> {
- let checked: SyncResult<Option<FullSyncResult>> = async {
- Ok(match schema_state(ctx).await? {
- SchemaState::LegacyChain => Some(FullSyncResult { success: true, sqlite_blob_upgrade_required: true, ..Default::default() }),
- SchemaState::Stale => Some(FullSyncResult { success: true, manifest_migration_required: true, ..Default::default() }),
- SchemaState::Current if !keys::has_local_vault_key(&ctx.host).await? => Some(FullSyncResult { success: true, manifest_migration_required: true, ..Default::default() }),
- SchemaState::Current => None,
- })
- }
- .await;
- match checked {
- Ok(result) => Ok(result),
- Err(error @ SyncError::VaultVersionIncompatible(_)) => Err(error),
- Err(error) => {
- ctx.warn(format!("[VaultSync] Ignoring failed pending migration check: {}", error)).await;
- Ok(None)
- }
- }
-}
-
-/// Classify the pending migration so the upgrade gate knows whether it may run on its own.
-///
-/// A vault still on the frozen sqlite-blob chain classifies as `None`: nothing here applies until the host has
-/// walked that chain. `migrate_manifest` treats the same state as a hard error instead, because reaching the
-/// executor with the chain outstanding is a caller bug, while asking the classifier about it is not.
-async fn migration_status(ctx: &mut Ctx) -> MigrationStatusResult {
- let classified: SyncResult<MigrationKind> = async {
- let schema = schema_state(ctx).await?;
- if schema == SchemaState::LegacyChain {
- return Ok(MigrationKind::None);
- }
- if !keys::has_local_vault_key(&ctx.host).await? {
- // A hierarchy another device created since this device logged in classifies as no migration at all.
- keys::accept_hierarchy_created_elsewhere(ctx).await?;
- }
- if !keys::has_local_vault_key(&ctx.host).await? {
- return Ok(MigrationKind::StorageFormatUpgrade);
- }
- Ok(if schema == SchemaState::Stale { MigrationKind::SchemaRebuild } else { MigrationKind::None })
- }
- .await;
- let kind = match classified {
- Ok(kind) => kind,
- Err(error) => {
- ctx.warn(format!("[ManifestMigration] Could not classify the pending migration, assuming it crosses the storage format: {}", error)).await;
- MigrationKind::StorageFormatUpgrade
- }
- };
- MigrationStatusResult { kind, session: SessionOutcome::default() }
-}
-
-/// Bring the local vault onto the current storage model and push it: a schema rebuild for a migrated account, the
-/// one-time account upgrade (`legacy::upgrade_account_to_manifest_v1`) for an account without a key hierarchy yet.
-async fn migrate_manifest(ctx: &mut Ctx) -> MigrateManifestResult {
- let migrated: SyncResult<bool> = async {
- if !keys::has_local_vault_key(&ctx.host).await? {
- return legacy::upgrade_account_to_manifest_v1(ctx).await;
- }
- migrate_schema(ctx).await
- }
- .await;
- match migrated {
- Ok(pushed) => MigrateManifestResult { success: true, pushed, ..Default::default() },
- Err(error) => {
- ctx.warn(format!("[ManifestMigration] Migration failed: {}", error)).await;
- MigrateManifestResult { failure: (&error).into(), ..Default::default() }
- }
- }
-}
-
-/// The permanent migration: rebuild a stale local schema onto the current one and push it. Returns whether the
-/// push reached the server; a vault already on the current schema counts as pushed.
-pub(crate) async fn migrate_schema(ctx: &mut Ctx) -> SyncResult<bool> {
- match schema_state(ctx).await? {
- SchemaState::Current => Ok(true),
- SchemaState::LegacyChain => Err(SyncError::LegacyUpgradePending),
- SchemaState::Stale => {
- rebuild_local_schema(ctx, None).await?;
- push_migrated_vault(ctx, false).await
- }
- }
-}
-
-/// Rebuild the local vault onto the current schema (a local round-trip through the codec, no server involved) and
-/// store it as a pending change. `stamp_unstamped_into` stamps rows that carry no manifest yet, which only a
-/// sqlite-blob vault has.
-pub(crate) async fn rebuild_local_schema(ctx: &mut Ctx, stamp_unstamped_into: Option<String>) -> SyncResult<()> {
- ctx.log("[ManifestMigration] Migrating local vault onto the current schema (local round-trip, no server involved)...").await;
- let set = push::canonicalize_vault(ctx, stamp_unstamped_into).await?;
- let mut blob_map = HashMap::new();
- for entry in &set.canonicalized.manifests {
- for (hash, blob) in &entry.blobs {
- blob_map.insert(hash.clone(), crate::common::encoding::base64_decode(&blob.bytes_base64)?);
- }
- }
- let manifests: Vec<_> = set.canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect();
- let bytes = pull::materialize_to_sqlite(ctx, &manifests, &set.canonicalized.data_buckets, &blob_map, &HashMap::new()).await?;
- ctx.log(format!("[ManifestMigration] Migration complete: {} blobs re-embedded, {} bytes.", blob_map.len(), bytes.len())).await;
-
- let stored = ctx.store_vault(&state::encrypt_vault_blob(&bytes, &ctx.encryption_key()?)?, true, None, None).await?;
- ctx.mutation_sequence = stored.mutation_sequence;
- ctx.is_dirty = true;
- ctx.vault_changed = true;
- Ok(())
-}
-
-/// Push the migrated vault and clear the dirty flag. A push the server refuses, or that fails, is not a migration
-/// failure: the vault stays dirty and the next sync carries it. Returns whether the push reached the server.
-pub(crate) async fn push_migrated_vault(ctx: &mut Ctx, create_vault_key: bool) -> SyncResult<bool> {
- match push::upload_vault(ctx, None, false, create_vault_key).await {
- Ok(upload) if upload.status == PushStatus::Ok => {
- state::mark_clean(&ctx.host, upload.mutation_seq_at_start).await?;
- Ok(true)
- }
- Ok(upload) => {
- ctx.warn(format!("[ManifestMigration] Migration push did not succeed ({:?}), vault stays dirty for the next sync.", upload.status)).await;
- Ok(false)
- }
- Err(error) => {
- ctx.warn(format!("[ManifestMigration] Migration push failed, vault stays dirty for the next sync: {}", error)).await;
- Ok(false)
- }
- }
-}
-
/// The login-time key resolution; see `keys::resolve_vault_key`.
async fn resolve_vault_key(ctx: &mut Ctx) -> ResolveVaultKeyResult {
match keys::resolve_vault_key(ctx).await {
Ok(has_vault_key) => ResolveVaultKeyResult { success: true, has_vault_key, encryption_key: ctx.encryption_key.clone(), ..Default::default() },
Err(error) => {
- ctx.warn(format!("[VaultSync] Key resolution failed: {}", error)).await;
+ ctx.warn(format!("[Keys] Key resolution failed: {}", error)).await;
ResolveVaultKeyResult { failure: (&error).into(), ..Default::default() }
}
}
@@ -674,14 +481,7 @@ async fn status_check(ctx: &mut Ctx) -> StatusCheckResult {
if status.server_version == http::SERVER_UNREACHABLE_VERSION {
return Ok(StatusCheckResult { success: true, has_dirty_changes: ctx.is_dirty, is_offline: true, ..Default::default() });
}
- let logout_reason = if !status.client_version_supported {
- Some(LogoutReason::ClientVersionNotSupported)
- } else if server_too_old(ctx, &status) {
- Some(LogoutReason::ServerVersionNotSupported)
- } else {
- None
- };
- if let Some(reason) = logout_reason {
+ if let Some(reason) = version_logout_reason(ctx, &status) {
return Ok(StatusCheckResult { failure: FailureFields::logout(reason), server_version: Some(status.server_version), ..Default::default() });
}
assert_salt_unchanged(ctx, status.srp_salt.as_deref()).await?;
diff --git a/core/rust/src/vault_sync/errors.rs b/core/rust/src/vault_sync/errors.rs
index da782d75b..4369a04d6 100644
--- a/core/rust/src/vault_sync/errors.rs
+++ b/core/rust/src/vault_sync/errors.rs
@@ -2,7 +2,6 @@
use serde::{Deserialize, Serialize};
-use super::types::CommandKind;
use crate::common::error::VaultError;
/// Client error codes, shared with the client apps (`AppErrorCodes` in the client core).
@@ -157,9 +156,9 @@ pub enum SyncError {
/// LEGACY: the manifest migration cannot run before the sqlite-blob upgrade chain.
#[error("The vault has to walk the legacy upgrade chain first")]
LegacyUpgradePending,
- /// The host reported a failure for a command.
+ /// The host reported a failure for a command (named by its wire tag, `dbExec`).
#[error("Host command {command} failed: {message}")]
- Host { command: CommandKind, message: String },
+ Host { command: &'static str, message: String },
#[error("JSON error: {0}")]
Json(#[from] serde_json::Error),
/// A codec, merge or crypto failure from the core library.
@@ -211,7 +210,7 @@ impl SyncError {
SyncError::ResyncLimitReached => Failure::Coded(ErrorCode::MergeConflict),
SyncError::UploadRejected(_) | SyncError::MissingBlobs(_) => Failure::Coded(ErrorCode::UploadFailed),
SyncError::LegacyUpgradePending => Failure::Coded(ErrorCode::MigrationCheckFailed),
- SyncError::Host { command, .. } => Failure::Coded(command.storage_error_code()),
+ SyncError::Host { command, .. } => Failure::Coded(storage_error_code(command)),
SyncError::Json(_) => Failure::Coded(ErrorCode::SyncResponseInvalid),
SyncError::Core(_) => Failure::Coded(ErrorCode::SyncCodecFailed),
SyncError::Other(_) => Failure::Coded(ErrorCode::SyncEngineFailed),
@@ -219,15 +218,13 @@ impl SyncError {
}
}
-impl CommandKind {
- /// The client error code a host failure of this command maps to.
- pub fn storage_error_code(self) -> ErrorCode {
- match self {
- CommandKind::DbOpen => ErrorCode::DatabaseInitFailed,
- CommandKind::StateGet | CommandKind::DbQuery | CommandKind::DbExport | CommandKind::VaultLoad => ErrorCode::StorageReadFailed,
- CommandKind::StateSet | CommandKind::StateRemove | CommandKind::DbExec | CommandKind::VaultStore | CommandKind::MarkClean => ErrorCode::StorageWriteFailed,
- CommandKind::Http | CommandKind::Log | CommandKind::Done => ErrorCode::UnknownError,
- }
+/// The client error code a host failure of the named command maps to.
+fn storage_error_code(command: &str) -> ErrorCode {
+ match command {
+ "dbOpen" => ErrorCode::DatabaseInitFailed,
+ "stateGet" | "dbQuery" | "dbExport" | "vaultLoad" => ErrorCode::StorageReadFailed,
+ "stateSet" | "stateRemove" | "dbExec" | "vaultStore" | "markClean" => ErrorCode::StorageWriteFailed,
+ _ => ErrorCode::UnknownError,
}
}
@@ -247,8 +244,8 @@ mod tests {
assert_eq!(SyncError::KeyOutOfSync.failure(), Failure::Coded(ErrorCode::KeyOutOfSync));
assert_eq!(serde_json::to_string(&LogoutReason::PasswordChanged).unwrap(), "\"passwordChanged\"");
assert_eq!(SyncError::Auth.failure(), Failure::Logout(LogoutReason::SessionExpired));
- assert_eq!(SyncError::Host { command: CommandKind::DbOpen, message: String::new() }.failure(), Failure::Coded(ErrorCode::DatabaseInitFailed));
- assert_eq!(SyncError::Host { command: CommandKind::DbExec, message: String::new() }.failure(), Failure::Coded(ErrorCode::StorageWriteFailed));
+ assert_eq!(SyncError::Host { command: "dbOpen", message: String::new() }.failure(), Failure::Coded(ErrorCode::DatabaseInitFailed));
+ assert_eq!(SyncError::Host { command: "dbExec", message: String::new() }.failure(), Failure::Coded(ErrorCode::StorageWriteFailed));
}
#[test]
diff --git a/core/rust/src/vault_sync/http.rs b/core/rust/src/vault_sync/http.rs
index 867978f10..4d53675ce 100644
--- a/core/rust/src/vault_sync/http.rs
+++ b/core/rust/src/vault_sync/http.rs
@@ -43,7 +43,7 @@ fn is_api_error_code(value: &str) -> bool {
}
fn parse<T: DeserializeOwned>(body: &str) -> SyncResult<T> {
- serde_json::from_str(body).map_err(|e| SyncError::Other(format!("Unexpected API response: {}", e)))
+ Ok(serde_json::from_str(body)?)
}
/// Authenticated GET of a v2 endpoint.
@@ -116,17 +116,15 @@ pub(crate) fn with_outdated_server_guard<T>(result: SyncResult<T>) -> SyncResult
/// The server version a status call reports when the server cannot be reached.
pub(crate) const SERVER_UNREACHABLE_VERSION: &str = "0.0.0";
-/// `GET v2/Status`. Anything but an auth failure or a client-version refusal reads as "server unreachable",
-/// reported as server version [`SERVER_UNREACHABLE_VERSION`] so the caller can go offline.
+/// `GET v2/Status`. A transport failure or a 5xx reads as "server unreachable", reported as server version
+/// [`SERVER_UNREACHABLE_VERSION`] so the caller can go offline; every other failure is returned.
pub(crate) async fn get_status(host: &Host) -> SyncResult<StatusResponse> {
- match get::<StatusResponse>(host, "Status", false).await {
- Ok(status) => Ok(status),
- Err(SyncError::Auth) => Err(SyncError::Auth),
- Err(SyncError::ClientUpgradeRequired) => Err(SyncError::ClientUpgradeRequired),
- Err(error) => {
- host.log(LogLevel::Warn, format!("[VaultSync] Status call failed, treating the server as unreachable: {}", error)).await;
+ match with_outdated_server_guard(get::<StatusResponse>(host, "Status", false).await) {
+ Err(error @ (SyncError::Network(_) | SyncError::Timeout(_) | SyncError::Http { status: 500..=599, .. })) => {
+ host.log(LogLevel::Warn, format!("[Sync] Status call failed, treating the server as unreachable: {}", error)).await;
Ok(StatusResponse { client_version_supported: true, server_version: SERVER_UNREACHABLE_VERSION.to_string(), ..Default::default() })
}
+ other => other,
}
}
diff --git a/core/rust/src/vault_sync/keys.rs b/core/rust/src/vault_sync/keys.rs
index 1fd13c119..e19051c41 100644
--- a/core/rust/src/vault_sync/keys.rs
+++ b/core/rust/src/vault_sync/keys.rs
@@ -1,4 +1,4 @@
-//! Key material during a sync.
+//! Key material during a sync: the account's key chain and the grants on shared manifests.
use std::collections::HashMap;
@@ -9,13 +9,13 @@ use super::types::{ManifestDto, SharedManifestDto, VaultKeyGetResponse, VaultKey
use super::http;
use crate::crypto;
-/// Whether this device holds a vault key.
-pub(crate) async fn has_local_vault_key(host: &Host) -> SyncResult<bool> {
+/// Whether this device has the account's key chain cached (the encrypted Account Key).
+pub(crate) async fn has_cached_key_chain(host: &Host) -> SyncResult<bool> {
Ok(state::get::<String>(host, state::ENCRYPTED_ACCOUNT_KEY).await?.is_some())
}
/// `GET v2/VaultKey/Password`: the account's vault key, `None` when the server holds none.
-pub(crate) async fn fetch_vault_key(host: &Host) -> SyncResult<Option<VaultKeyResponse>> {
+async fn fetch_vault_key(host: &Host) -> SyncResult<Option<VaultKeyResponse>> {
match http::get::<VaultKeyGetResponse>(host, http::VAULT_KEY_PASSWORD_ENDPOINT, false).await {
Ok(response) => Ok(response.vault_key),
Err(SyncError::Http { status: 404, .. }) => Ok(None),
@@ -24,40 +24,150 @@ pub(crate) async fn fetch_vault_key(host: &Host) -> SyncResult<Option<VaultKeyRe
}
/// Refuse an Account Key encrypted with an algorithm this build cannot open.
-pub(crate) fn ensure_known_unlock_algorithm(vault_key: &VaultKeyResponse) -> SyncResult<()> {
+fn ensure_known_unlock_algorithm(vault_key: &VaultKeyResponse) -> SyncResult<()> {
if vault_key.algorithm == ALGORITHM_AES256_GCM {
return Ok(());
}
Err(SyncError::VaultVersionIncompatible(format!("the Account Key is encrypted with '{}', which this app cannot open; update the app", vault_key.algorithm)))
}
-/// Persist a server vault-key response's encrypted blobs.
-pub(crate) async fn cache_vault_key_blobs(host: &Host, vault_key: &VaultKeyResponse) -> SyncResult<()> {
- state::set(host, state::ENCRYPTED_ACCOUNT_KEY, &vault_key.encrypted_account_key).await?;
- state::set(host, state::ENCRYPTED_VEK, &vault_key.encrypted_vek).await?;
- match (&vault_key.account_public_key, &vault_key.encrypted_account_private_key) {
- (Some(public), Some(private)) => {
- state::set(host, state::ACCOUNT_PUBLIC_KEY, public).await?;
- state::set(host, state::ENCRYPTED_ACCOUNT_PRIVATE_KEY, private).await?;
+/// The state keys the key chain is cached under; `cache_vault_key_blobs` pairs them with the server fields by position.
+const CHAIN_STATE: [&str; 6] = [state::ENCRYPTED_ACCOUNT_KEY, state::ENCRYPTED_VEK, state::ACCOUNT_PUBLIC_KEY, state::ENCRYPTED_ACCOUNT_PRIVATE_KEY, state::SIGNING_PUBLIC_KEY, state::ENCRYPTED_SIGNING_PRIVATE_KEY];
+
+/// Cache a server vault-key response's encrypted blobs; a half the server does not hold is removed.
+async fn cache_vault_key_blobs(host: &Host, vault_key: &VaultKeyResponse) -> SyncResult<()> {
+ let values = [Some(&vault_key.encrypted_account_key), vault_key.encrypted_vek.as_ref(), vault_key.account_public_key.as_ref(), vault_key.encrypted_account_private_key.as_ref(), vault_key.signing_public_key.as_ref(), vault_key.encrypted_signing_private_key.as_ref()];
+ for (key, value) in CHAIN_STATE.iter().zip(values) {
+ match value {
+ Some(value) => state::set(host, key, value).await?,
+ None => state::remove(host, key).await?,
}
- _ => {
- state::remove(host, state::ACCOUNT_PUBLIC_KEY).await?;
- state::remove(host, state::ENCRYPTED_ACCOUNT_PRIVATE_KEY).await?;
+ }
+ Ok(())
+}
+
+/// Clear the cached key chain: the account has none (legacy), so the unlock key is the vault key.
+async fn clear_cached_chain(host: &Host) -> SyncResult<()> {
+ for key in CHAIN_STATE {
+ state::remove(host, key).await?;
+ }
+ Ok(())
+}
+
+/// Open a key chain with the unlock key or the stored Account Key: stages the Account Key and the account private key
+/// for this run and returns the VEK, which the caller makes the session key.
+async fn open_chain(ctx: &mut Ctx, encrypted_account_key: &str, encrypted_vek: &str, encrypted_private_key: Option<&str>, unlock_key: &str) -> SyncResult<String> {
+ let opened = crypto::open_account_key_chain(unlock_key, encrypted_account_key, encrypted_vek, None)?;
+ ctx.account_key = Some(opened.account_key.to_string());
+ if let Some(encrypted) = encrypted_private_key.filter(|e| !e.is_empty()) {
+ match crypto::open_account_private_key(encrypted, &opened.account_key) {
+ Ok(private_key) => ctx.account_private_key = Some(private_key),
+ Err(error) => ctx.warn(format!("[Keys] The cached account private key did not open; shared grants stay closed. {}", error)).await,
}
}
- match (&vault_key.signing_public_key, &vault_key.encrypted_signing_private_key) {
- (Some(public), Some(private)) => {
- state::set(host, state::SIGNING_PUBLIC_KEY, public).await?;
- state::set(host, state::ENCRYPTED_SIGNING_PRIVATE_KEY, private).await?;
+ Ok(opened.vault_encryption_key.to_string())
+}
+
+/// Open the chain of a server vault-key response with `unlock_key`, refusing an unknown unlock algorithm first.
+async fn open_server_chain(ctx: &mut Ctx, vault_key: &VaultKeyResponse, unlock_key: &str) -> SyncResult<String> {
+ ensure_known_unlock_algorithm(vault_key)?;
+ open_chain(ctx, &vault_key.encrypted_account_key, vault_key.encrypted_vek.as_deref().unwrap_or_default(), vault_key.encrypted_account_private_key.as_deref(), unlock_key).await
+}
+
+/// The login-time key resolution (the `resolveVaultKey` operation): open the server's chain with the session key,
+/// cache it for offline unlock, and make the VEK the session key. Returns whether the account has a chain.
+pub(crate) async fn resolve_vault_key(ctx: &mut Ctx) -> SyncResult<bool> {
+ let unlock_key = ctx.encryption_key()?;
+ ctx.vault_key_probed = true;
+ match http::get::<VaultKeyGetResponse>(&ctx.host, http::VAULT_KEY_PASSWORD_ENDPOINT, false).await {
+ Ok(VaultKeyGetResponse { vault_key: Some(vault_key) }) if vault_key.encrypted_vek.is_some() => {
+ let vek = open_server_chain(ctx, &vault_key, &unlock_key).await?;
+ ctx.set_encryption_key(vek);
+ cache_vault_key_blobs(&ctx.host, &vault_key).await?;
+ ctx.log("[Keys] Opened the account's key chain; the session key is the VEK.").await;
+ Ok(true)
+ }
+ Ok(_) => {
+ clear_cached_chain(&ctx.host).await?;
+ ctx.log("[Keys] The account has no key chain yet (legacy vault); the unlock key is the vault key.").await;
+ Ok(false)
}
- _ => {
- state::remove(host, state::SIGNING_PUBLIC_KEY).await?;
- state::remove(host, state::ENCRYPTED_SIGNING_PRIVATE_KEY).await?;
+ Err(error @ (SyncError::Network(_) | SyncError::Timeout(_) | SyncError::Http { .. })) => {
+ ctx.warn(format!("[Keys] Could not fetch the key chain, opening the cached one: {}", error)).await;
+ let (Some(encrypted_account_key), Some(encrypted_vek)) = (state::get::<String>(&ctx.host, state::ENCRYPTED_ACCOUNT_KEY).await?, state::get::<String>(&ctx.host, state::ENCRYPTED_VEK).await?) else { return Ok(false) };
+ let encrypted_private_key = state::get::<String>(&ctx.host, state::ENCRYPTED_ACCOUNT_PRIVATE_KEY).await?;
+ let vek = open_chain(ctx, &encrypted_account_key, &encrypted_vek, encrypted_private_key.as_deref(), &unlock_key).await?;
+ ctx.set_encryption_key(vek);
+ Ok(true)
}
+ Err(error) => Err(error),
}
- Ok(())
}
+/// Accept a key hierarchy another device created since this legacy-login device last looked (probed once per run).
+/// False only when the session key does not open it, which needs a re-login.
+pub(crate) async fn accept_hierarchy_created_elsewhere(ctx: &mut Ctx) -> SyncResult<bool> {
+ if ctx.vault_key_probed {
+ return Ok(true);
+ }
+ let session_key = ctx.encryption_key()?;
+ // A failed probe fails the run: assuming "no hierarchy" would send a migration write that creates a second one.
+ let fetched = fetch_vault_key(&ctx.host).await?;
+ ctx.vault_key_probed = true;
+ let vault_key = match fetched {
+ Some(vault_key) if vault_key.encrypted_vek.is_some() => vault_key,
+ _ => return Ok(true),
+ };
+ let vek = match open_server_chain(ctx, &vault_key, &session_key).await {
+ Ok(vek) => vek,
+ Err(SyncError::UnlockKeyRejected) => {
+ ctx.warn("[Keys] The session key does not open the key hierarchy the server holds; a re-login is needed.").await;
+ return Ok(false);
+ }
+ // A storage failure or an unreadable stored vault is its own failure, not a key mismatch.
+ Err(error) => return Err(error),
+ };
+ // The chain is cached before the vault is stored under the VEK: hosts hold the unlock key and derive the vault key from it.
+ cache_vault_key_blobs(&ctx.host, &vault_key).await?;
+ if let Some(encrypted_vault) = state::load_vault(&ctx.host).await? {
+ let plaintext = state::decrypt_vault_blob(&encrypted_vault, &session_key)?;
+ state::store_vault_with_key(&ctx.host, &state::encrypt_vault_blob(&plaintext, &vek)?, false, None, None, Some(vek.clone())).await?;
+ }
+ re_encrypt_shared_manifest_records(ctx, &vek).await?;
+ ctx.set_encryption_key(vek);
+ ctx.log("[Keys] Another device created the account's key hierarchy; accepted it and swapped the session key to the VEK.").await;
+ Ok(true)
+}
+
+/// Whether this device holds the key chain, after accepting one another device created; `KeyOutOfSync` when the session key does not open it.
+pub(crate) async fn ensure_key_chain_accepted(ctx: &mut Ctx) -> SyncResult<bool> {
+ if has_cached_key_chain(&ctx.host).await? {
+ return Ok(true);
+ }
+ if !accept_hierarchy_created_elsewhere(ctx).await? {
+ return Err(SyncError::KeyOutOfSync);
+ }
+ has_cached_key_chain(&ctx.host).await
+}
+
+/// The account signing private key, opened on first use from the cached encrypted copy with the session's Account Key.
+/// None when that key or the cached copy is missing or does not open it.
+pub(crate) async fn signing_private_key(ctx: &mut Ctx) -> SyncResult<Option<String>> {
+ if ctx.signing_private_key.is_some() {
+ return Ok(ctx.signing_private_key.clone());
+ }
+ let (Some(account_key), Some(encrypted)) = (ctx.account_key.clone(), state::get::<String>(&ctx.host, state::ENCRYPTED_SIGNING_PRIVATE_KEY).await?) else { return Ok(None) };
+ match crypto::open_account_signing_private_key(&encrypted, &account_key) {
+ Ok(private_key) => ctx.signing_private_key = Some(private_key.to_string()),
+ Err(error) => ctx.warn(format!("[Keys] The cached signing private key did not open; nothing can be signed this run. {}", error)).await,
+ }
+ Ok(ctx.signing_private_key.clone())
+}
+
+/*
+ * Shared manifests: the grants this account holds.
+ */
+
/// The shared-manifest key records, keyed by manifest id.
pub(crate) async fn shared_manifest_records(ctx: &Ctx) -> SyncResult<HashMap<String, SharedManifestDto>> {
let Some(ciphertext) = state::get::<String>(&ctx.host, state::SHARED_MANIFESTS).await? else { return Ok(HashMap::new()) };
@@ -71,10 +181,11 @@ pub(crate) async fn shared_manifest_records(ctx: &Ctx) -> SyncResult<HashMap<Str
}
}
-/// Persist the shared-manifest key records.
-pub(crate) async fn set_shared_manifest_records(host: &Host, records: &HashMap<String, SharedManifestDto>, key: &str) -> SyncResult<()> {
+/// Persist the shared-manifest key records under `key`.
+pub(crate) async fn set_shared_manifest_records(ctx: &Ctx, records: &HashMap<String, SharedManifestDto>, key: &str) -> SyncResult<()> {
let ciphertext = crypto::symmetric_encrypt(&serde_json::to_string(records)?, key)?;
- state::set(host, state::SHARED_MANIFESTS, &ciphertext).await?;
+ state::set(&ctx.host, state::SHARED_MANIFESTS, &ciphertext).await?;
+ ctx.cache_shared_veks(None);
Ok(())
}
@@ -84,7 +195,7 @@ pub(crate) async fn re_encrypt_shared_manifest_records(ctx: &Ctx, new_key: &str)
if records.is_empty() {
return Ok(());
}
- set_shared_manifest_records(&ctx.host, &records, new_key).await
+ set_shared_manifest_records(ctx, &records, new_key).await
}
/// The private key that opens a grant made out to `public_key`.
@@ -125,164 +236,21 @@ pub(crate) async fn open_shared_manifest_vek(ctx: &Ctx, record: &SharedManifestD
}
}
-/// The VEK of every shared manifest this account holds a grant on, keyed by manifest id.
+/// The VEK of every shared manifest this account holds a grant on, keyed by manifest id. Unwrapped once per run.
pub(crate) async fn open_shared_manifest_veks(ctx: &Ctx) -> SyncResult<HashMap<String, String>> {
+ if let Some(cached) = ctx.cached_shared_veks() {
+ return Ok(cached);
+ }
let mut veks = HashMap::new();
for record in shared_manifest_records(ctx).await?.values() {
if let Some(vek) = open_shared_manifest_vek(ctx, record).await? {
veks.insert(record.manifest_id.clone(), vek);
}
}
+ ctx.cache_shared_veks(Some(veks.clone()));
Ok(veks)
}
-/// The cached account-key chain, when this device holds one.
-async fn cached_chain(host: &Host) -> SyncResult<Option<(String, String)>> {
- let Some(encrypted_account_key) = state::get::<String>(host, state::ENCRYPTED_ACCOUNT_KEY).await? else { return Ok(None) };
- let Some(encrypted_vek) = state::get::<String>(host, state::ENCRYPTED_VEK).await? else { return Ok(None) };
- Ok(Some((encrypted_account_key, encrypted_vek)))
-}
-
-/// Whether the transport reached the server and got an answer other than an auth or version refusal.
-fn is_server_unreachable(error: &SyncError) -> bool {
- matches!(error, SyncError::Network(_) | SyncError::Timeout(_) | SyncError::Http { .. })
-}
-
-/// Clear the cached key chain: the account has none (legacy), so the unlock key is the vault key.
-async fn clear_cached_chain(host: &Host) -> SyncResult<()> {
- for key in [state::ENCRYPTED_ACCOUNT_KEY, state::ENCRYPTED_VEK, state::ACCOUNT_PUBLIC_KEY, state::ENCRYPTED_ACCOUNT_PRIVATE_KEY, state::SIGNING_PUBLIC_KEY, state::ENCRYPTED_SIGNING_PRIVATE_KEY] {
- state::remove(host, key).await?;
- }
- Ok(())
-}
-
-/// Open a key chain with an unlock key or the stored Account Key: the VEK becomes the session key and the private key
-/// is staged.
-async fn open_chain(ctx: &mut Ctx, encrypted_account_key: &str, encrypted_vek: &str, encrypted_private_key: Option<&str>, unlock_key: &str) -> SyncResult<()> {
- let opened = crypto::open_account_key_chain(unlock_key, encrypted_account_key, encrypted_vek, None)?;
- ctx.set_encryption_key(opened.vault_encryption_key.to_string());
- ctx.account_key = Some(opened.account_key.to_string());
- stage_account_private_key(ctx, &opened.account_key, encrypted_private_key).await;
- Ok(())
-}
-
-/// The login-time key resolution (the `resolveVaultKey` operation).
-pub(crate) async fn resolve_vault_key(ctx: &mut Ctx) -> SyncResult<bool> {
- let unlock_key = ctx.encryption_key()?;
- let fetched = http::get::<VaultKeyGetResponse>(&ctx.host, http::VAULT_KEY_PASSWORD_ENDPOINT, false).await;
- ctx.vault_key_probed = true;
- match fetched {
- Ok(response) => match response.vault_key {
- Some(vault_key) if vault_key.encrypted_vek.is_some() => {
- ensure_known_unlock_algorithm(&vault_key)?;
- let encrypted_vek = vault_key.encrypted_vek.clone().unwrap_or_default();
- open_chain(ctx, &vault_key.encrypted_account_key, &encrypted_vek, vault_key.encrypted_account_private_key.as_deref(), &unlock_key).await?;
- cache_vault_key_blobs(&ctx.host, &vault_key).await?;
- ctx.log("[VaultSync] Opened the account's key chain; the session key is the VEK.").await;
- Ok(true)
- }
- _ => {
- clear_cached_chain(&ctx.host).await?;
- ctx.log("[VaultSync] The account has no key chain yet (legacy vault); the unlock key is the vault key.").await;
- Ok(false)
- }
- },
- Err(error) if is_server_unreachable(&error) => {
- ctx.warn(format!("[VaultSync] Could not fetch the key chain, opening the cached one: {}", error)).await;
- let Some((encrypted_account_key, encrypted_vek)) = cached_chain(&ctx.host).await? else { return Ok(false) };
- let encrypted_private_key = state::get::<String>(&ctx.host, state::ENCRYPTED_ACCOUNT_PRIVATE_KEY).await?;
- open_chain(ctx, &encrypted_account_key, &encrypted_vek, encrypted_private_key.as_deref(), &unlock_key).await?;
- Ok(true)
- }
- Err(error) => Err(error),
- }
-}
-
-/// The cross-device race: this device holds no key chain (it logged in while the account was still a legacy
-/// vault), and another device may have created the hierarchy since. Probes the server once per run; when the
-/// hierarchy exists, the stored vault is brought under the VEK and the session key swapped, which the host picks up
-/// through the store command. Callers gate this on the absence of a cached chain and on a sync that pulls or
-/// pushes: another device's migration shows up as a revision change, so that is when it becomes visible.
-/// False only when the session key does not open the server's chain, which requires a re-login; any other failure
-/// is returned as itself.
-pub(crate) async fn accept_hierarchy_created_elsewhere(ctx: &mut Ctx) -> SyncResult<bool> {
- if ctx.vault_key_probed {
- return Ok(true);
- }
- ctx.vault_key_probed = true;
- let session_key = ctx.encryption_key()?;
- let vault_key = match fetch_vault_key(&ctx.host).await {
- Ok(result) => result,
- Err(error) => {
- ctx.warn(format!("[VaultSync] Vault key probe failed, deferring the vault key check: {}", error)).await;
- return Ok(true);
- }
- };
- let Some(vault_key) = vault_key else { return Ok(true) };
- let Some(encrypted_vek) = vault_key.encrypted_vek.clone() else { return Ok(true) };
- ensure_known_unlock_algorithm(&vault_key)?;
-
- // Hosts hold the unlock key and derive the vault key from the cached chain.
- let accepted: SyncResult<()> = async {
- let opened = crypto::open_account_key_chain(&session_key, &vault_key.encrypted_account_key, &encrypted_vek, None)?;
- cache_vault_key_blobs(&ctx.host, &vault_key).await?;
- switch_to_vek(ctx, &session_key, &opened.vault_encryption_key).await?;
- ctx.account_key = Some(opened.account_key.to_string());
- stage_account_private_key(ctx, &opened.account_key, vault_key.encrypted_account_private_key.as_deref()).await;
- Ok(())
- }
- .await;
-
- match accepted {
- Ok(()) => {
- ctx.log("[VaultSync] Another device created the account's key hierarchy; accepted it and swapped the session key to the VEK.").await;
- Ok(true)
- }
- Err(SyncError::UnlockKeyRejected) => {
- ctx.warn("[VaultSync] The session key does not open the key hierarchy the server holds; a re-login is needed.").await;
- Ok(false)
- }
- // A storage failure or an unreadable stored vault is its own failure, not a key mismatch.
- Err(error) => Err(error),
- }
-}
-
-/// Swap the session key for the VEK: re-encrypt the stored vault and the shared-manifest records under it, then
-/// report it to the host.
-async fn switch_to_vek(ctx: &mut Ctx, old_key: &str, vek: &str) -> SyncResult<()> {
- if let Some(encrypted_vault) = state::load_vault(&ctx.host).await? {
- let plaintext = state::decrypt_vault_blob(&encrypted_vault, old_key)?;
- let re_encrypted = crypto::symmetric_encrypt_bytes(&plaintext, vek)?;
- state::store_vault_with_key(&ctx.host, &re_encrypted, false, None, None, Some(vek.to_string())).await?;
- }
- re_encrypt_shared_manifest_records(ctx, vek).await?;
- ctx.set_encryption_key(vek.to_string());
- Ok(())
-}
-
-/// Open the account private key with the Account Key and stage it for the grant flows of this run.
-async fn stage_account_private_key(ctx: &mut Ctx, account_key: &str, encrypted_private_key: Option<&str>) {
- let Some(encrypted) = encrypted_private_key.filter(|e| !e.is_empty()) else { return };
- match crypto::open_account_private_key(encrypted, account_key) {
- Ok(private_key) => ctx.account_private_key = Some(private_key),
- Err(error) => ctx.warn(format!("[VaultSync] The cached account private key did not open; shared grants stay closed. {}", error)).await,
- }
-}
-
-/// The account signing private key, opened on first use from the cached encrypted copy with the session's Account Key
-/// (the key the host's chain open returned). None when that key or the cached copy is missing or does not open it.
-pub(crate) async fn signing_private_key(ctx: &mut Ctx) -> SyncResult<Option<String>> {
- if ctx.signing_private_key.is_some() {
- return Ok(ctx.signing_private_key.clone());
- }
- let (Some(account_key), Some(encrypted)) = (ctx.account_key.clone(), state::get::<String>(&ctx.host, state::ENCRYPTED_SIGNING_PRIVATE_KEY).await?) else { return Ok(None) };
- match crypto::open_account_signing_private_key(&encrypted, &account_key) {
- Ok(private_key) => ctx.signing_private_key = Some(private_key.to_string()),
- Err(error) => ctx.warn(format!("[VaultSync] The cached signing private key did not open; nothing can be signed this run. {}", error)).await,
- }
- Ok(ctx.signing_private_key.clone())
-}
-
/// Whether the grant a manifest was served with carries a valid signature by its signer.
pub(crate) fn grant_signature_verifies(dto: &ManifestDto, recipient_public_key: &str, algorithm: &str, encrypted_vek: &str) -> bool {
let (Some(signer_user_id), Some(signer_public_key), Some(signature)) = (dto.grant_signer_user_id.as_deref(), dto.grant_signer_public_key.as_deref(), dto.grant_signature.as_deref()) else { return false };
diff --git a/core/rust/src/vault_sync/legacy.rs b/core/rust/src/vault_sync/legacy.rs
index 72c82818e..de70bd695 100644
--- a/core/rust/src/vault_sync/legacy.rs
+++ b/core/rust/src/vault_sync/legacy.rs
@@ -1,7 +1,7 @@
//! LEGACY: what old sqlite-blob accounts need, including their one-time upgrade from sqlite-blob to manifest-v1
-//! ([`upgrade_account_to_manifest_v1`]). Remove this module once every account has upgraded, together with its callers:
-//! `pull::pull` (the legacy snapshot branch), `merge::pull_and_merge` (the legacy server branch),
-//! `db::schema_state` (the frozen chain check) and `engine::migrate_manifest` (the branch without a vault key).
+//! ([`upgrade_account_to_manifest_v1`]). Remove this module once every account has upgraded, together with the
+//! `Snapshot::LegacySqliteBlob` arm in `pull.rs` and `merge.rs`, `db::schema_state`'s frozen chain check and the
+//! branch without a vault key in `migration::migrate_manifest`.
use std::collections::HashMap;
@@ -9,11 +9,12 @@ use serde_json::Value;
use super::db::{self, SchemaState};
use super::errors::{SyncError, SyncResult};
-use super::pull::{self, email_routing_of, PulledVault};
-use super::push::resolve_personal_manifest_id;
+use super::migration;
+use super::pull::{self, PulledVault};
+use super::push::{resolve_personal_manifest_id, WriteKind};
use super::state::{self, Ctx};
use super::types::{Db, GetResponse};
-use super::{engine, keys};
+use super::keys;
use crate::common::encoding::base64_decode;
use crate::sqlite_host::SqlStatement;
use crate::vault_codec::row::inline_bytes;
@@ -21,11 +22,6 @@ use crate::vault_codec::row::inline_bytes;
/// The `storageFormat` of a legacy sqlite-blob snapshot; an absent value means the same.
pub(crate) const STORAGE_FORMAT_SQLITE_BLOB: &str = "sqlite-blob";
-/// Whether a snapshot is still on the legacy sqlite-blob format.
-pub(crate) fn is_legacy_sqlite_blob_snapshot(snapshot: &GetResponse) -> bool {
- matches!(snapshot.storage_format.as_deref(), None | Some(STORAGE_FORMAT_SQLITE_BLOB))
-}
-
/// Take a legacy snapshot apart for local storage: the blob passes through untouched, the manifest-v1 fingerprints
/// are reset, and the personal manifest id is recorded for the migration push.
pub(crate) async fn open_legacy_snapshot(ctx: &Ctx, snapshot: &GetResponse) -> SyncResult<PulledVault> {
@@ -36,8 +32,8 @@ pub(crate) async fn open_legacy_snapshot(ctx: &Ctx, snapshot: &GetResponse) -> S
state::set(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID, personal).await?;
manifest_revisions.insert(personal.clone(), revision);
}
- ctx.log("[V2Pull] Legacy sqlite-blob pass-through (user not yet migrated), returning the blob as-is.").await;
- Ok(PulledVault { encrypted_vault: snapshot.legacy_vault_blob.clone().unwrap_or_default(), revision, email_routing: email_routing_of(snapshot), manifest_revisions, bucket_revisions: HashMap::new(), needs_first_write: false })
+ ctx.log("[Pull] Legacy sqlite-blob pass-through (user not yet migrated), returning the blob as-is.").await;
+ Ok(PulledVault { encrypted_vault: snapshot.legacy_vault_blob.clone().unwrap_or_default(), personal_revision: revision, email_routing: snapshot.email_routing.clone().unwrap_or_default(), manifest_revisions, bucket_revisions: HashMap::new(), needs_first_write: false })
}
/*
@@ -51,51 +47,46 @@ const BLOB_TYPED_COLUMNS: &str = "SELECT m.name AS TableName, p.name AS ColumnNa
/// rows stamped with the personal manifest, and the push that carries it creates the account key hierarchy. Returns
/// whether that push reached the server.
pub(crate) async fn upgrade_account_to_manifest_v1(ctx: &mut Ctx) -> SyncResult<bool> {
- if engine::schema_state(ctx).await? == SchemaState::LegacyChain {
+ if migration::schema_state(ctx).await? == SchemaState::LegacyChain {
return Err(SyncError::LegacyUpgradePending);
}
// Another device may have created the hierarchy since this one logged in. Accepting it swaps the session key to the VEK, which the baseline pull below needs.
- if !keys::accept_hierarchy_created_elsewhere(ctx).await? {
- return Err(SyncError::KeyOutOfSync);
- }
+ keys::ensure_key_chain_accepted(ctx).await?;
record_server_baseline_if_missing(ctx).await?;
- if keys::has_local_vault_key(&ctx.host).await? {
+ if keys::has_cached_key_chain(&ctx.host).await? {
// The account turned out to be upgraded already (accepted above, or pulled with the baseline); a schema rebuild is all that can remain.
- return engine::migrate_schema(ctx).await;
+ return migration::migrate_schema(ctx).await;
}
- if engine::schema_state(ctx).await? == SchemaState::LegacyChain {
+ if migration::schema_state(ctx).await? == SchemaState::LegacyChain {
// The baseline pull stored a server vault that is still on the chain.
return Err(SyncError::LegacyUpgradePending);
}
let personal = resolve_personal_manifest_id(ctx).await?;
decode_base64_text_in_blob_columns(ctx).await?;
- engine::rebuild_local_schema(ctx, Some(personal)).await?;
- engine::push_migrated_vault(ctx, true).await
+ migration::rebuild_local_schema(ctx, Some(personal)).await?;
+ migration::push_migrated_vault(ctx, WriteKind::AccountKeyMigration).await
}
-/// A session that logged in through a client predating the manifest storage format never pulled through this engine,
-/// so it holds neither the personal manifest id nor the revision baseline, and the upgrade push needs both (the
-/// server refuses a write whose revision it does not know). Calling this method before the push fixes this.
+/// Pull the personal manifest id and revision baseline the upgrade push needs, when a pre-manifest session lacks them.
async fn record_server_baseline_if_missing(ctx: &mut Ctx) -> SyncResult<()> {
if state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await?.is_some() {
return Ok(());
}
- ctx.log("[ManifestMigration] The session predates the manifest storage format and never pulled; fetching the server vault for the personal manifest id and the revision baseline.").await;
+ ctx.log("[Migration] The session predates the manifest storage format and never pulled; fetching the server vault for the personal manifest id and the revision baseline.").await;
let pulled = pull::pull(ctx).await?;
if ctx.is_dirty && ctx.has_local_vault().await? {
- ctx.warn("[ManifestMigration] The local vault has pending changes; keeping it and recording only the server's revisions so the migration push carries them.").await;
- return pull::commit_revisions(ctx, &pulled.manifest_revisions, &pulled.bucket_revisions).await;
+ ctx.warn("[Migration] The local vault has pending changes; keeping it and recording only the server's revisions so the migration push carries them.").await;
+ return pull::commit_revisions(ctx, &pulled).await;
}
- if !ctx.store_vault(&pulled.encrypted_vault, false, Some(ctx.mutation_sequence), Some(pulled.revision)).await?.success {
+ if !ctx.store_vault(&pulled.encrypted_vault, false, Some(ctx.mutation_sequence), Some(pulled.personal_revision)).await?.success {
return Err(SyncError::Other("a mutation raced the baseline pull; run the migration again".to_string()));
}
ctx.vault_changed = true;
- pull::commit_revisions(ctx, &pulled.manifest_revisions, &pulled.bucket_revisions).await
+ pull::commit_revisions(ctx, &pulled).await
}
-/// Decode base64 TEXT in BLOB columns back to bytes, in place. The 0.30.x merges (Android, iOS, Blazor) bound every
-/// value as text, so the BLOB cells of each row the server copy won were stored as base64 TEXT. The upgrade push then
-/// carries real bytes to every device. Text that is not base64 is left as-is.
+/// Decode base64 TEXT in BLOB columns back to bytes, in place; text that is not base64 is left as-is.
+/// The 0.30.x merges (Android, iOS, Blazor) bound every value as text, storing BLOB cells as base64 TEXT.
async fn decode_base64_text_in_blob_columns(ctx: &Ctx) -> SyncResult<()> {
let mut decoded = 0usize;
for column in db::query(&ctx.host, Db::Local, BLOB_TYPED_COLUMNS, vec![]).await? {
@@ -112,7 +103,7 @@ async fn decode_base64_text_in_blob_columns(ctx: &Ctx) -> SyncResult<()> {
db::exec(&ctx.host, Db::Local, statements).await?;
}
if decoded > 0 {
- ctx.warn(format!("[ManifestMigration] Decoded {} base64 TEXT cells in BLOB columns back to bytes (written by a 0.30.x merge).", decoded)).await;
+ ctx.warn(format!("[Migration] Decoded {} base64 TEXT cells in BLOB columns back to bytes (written by a 0.30.x merge).", decoded)).await;
}
Ok(())
}
@@ -137,7 +128,7 @@ const LEGACY_VAULT_VERSIONS: &[(u32, &str)] = &[
/// The revision of the last entry in the legacy chain.
fn latest_legacy_revision() -> u32 {
- LEGACY_VAULT_VERSIONS.last().map(|(revision, _)| *revision).unwrap_or(0)
+ LEGACY_VAULT_VERSIONS[LEGACY_VAULT_VERSIONS.len() - 1].0
}
/// The data version embedded in an EF migration id (`20250101000000_2.0.0-Name`).
@@ -174,20 +165,16 @@ mod tests {
use super::*;
#[test]
- fn migration_ids_yield_their_version() {
+ fn migration_stamps_map_onto_the_frozen_chain() {
assert_eq!(extract_version_from_migration_id("20250101000000_2.0.0-Squashed").as_deref(), Some("2.0.0"));
assert_eq!(extract_version_from_migration_id("20250101000000_Initial"), None);
assert_eq!(legacy_revision_for("1.7.0"), Ok(12));
- assert_eq!(legacy_revision_for("2.0.0"), Ok(13));
- assert_eq!(legacy_revision_for("2.1.0"), Ok(13));
+ assert_eq!(legacy_revision_for("2.1.0"), Ok(13), "an unknown but well-formed version reads as the latest");
assert!(legacy_revision_for("nope").is_err());
- }
- #[test]
- fn stamps_before_2_0_0_are_on_the_chain() {
- assert_eq!(stamp_predates_manifest_schema("20250101000000_1.7.0-Old").unwrap(), true);
- assert_eq!(stamp_predates_manifest_schema("20250101000000_2.0.0-Squashed").unwrap(), false);
- assert_eq!(stamp_predates_manifest_schema("20260101000000_2.1.0-Later").unwrap(), false);
+ assert!(stamp_predates_manifest_schema("20250101000000_1.7.0-Old").unwrap());
+ assert!(!stamp_predates_manifest_schema("20250101000000_2.0.0-Squashed").unwrap());
+ assert!(!stamp_predates_manifest_schema("20260101000000_2.1.0-Later").unwrap());
assert!(matches!(stamp_predates_manifest_schema("20250101000000_Initial"), Err(SyncError::Other(_))));
}
}
diff --git a/core/rust/src/vault_sync/merge.rs b/core/rust/src/vault_sync/merge.rs
index 1d752101f..2992fce70 100644
--- a/core/rust/src/vault_sync/merge.rs
+++ b/core/rust/src/vault_sync/merge.rs
@@ -5,10 +5,10 @@ use std::collections::{HashMap, HashSet};
use crate::vault_model::{id_key, ids_equal};
use super::errors::SyncResult;
-use super::pull::{self, OpenedManifestSet, PulledVault};
-use super::push::{canonicalize_vault, CanonicalizedSet, ManifestRecord};
+use super::pull::{self, OpenedManifestSet, PulledVault, Snapshot};
+use super::push::{canonicalize_vault, CanonicalizedSet};
use super::state::{self, Ctx};
-use super::types::EmailRoutingDto;
+use super::write_set::ManifestRecord;
use super::legacy;
use crate::vault_codec::{self, BlobEntry, CanonicalizedVault, DataBucket, Manifest};
use crate::vault_merge::{merge_canonical, CanonicalManifestMerge, CanonicalMergeInput, MergeStats};
@@ -17,14 +17,9 @@ use crate::vault_merge::{merge_canonical, CanonicalManifestMerge, CanonicalMerge
pub(crate) enum PullAndMergeOutcome {
/// LEGACY: the server's vault is still a sqlite blob at this revision; the caller pushes the local vault over it.
LegacyServer { revision: i64 },
- Merged {
- pulled: PulledVault,
- /// The merge counters summed over every manifest.
- stats: MergeStats,
- fallback_manifest_ids: Vec<String>,
- dropped_local_manifest_ids: Vec<String>,
- push_canonical: Option<CanonicalizedSet>,
- },
+ /// The merged vault, with the canonical set the push right after it writes from (none when a manifest fell back
+ /// to the server's version, which the push then has to canonicalize again).
+ Merged { pulled: PulledVault, push_canonical: Option<CanonicalizedSet> },
/// The merge failed, so the server's vault stands as pulled and the local changes are dropped.
ServerOnly(PulledVault),
}
@@ -32,23 +27,24 @@ pub(crate) enum PullAndMergeOutcome {
/// Pull the latest snapshot and merge the local vault onto it at canonical level, one manifest at a time.
pub(crate) async fn pull_and_merge(ctx: &mut Ctx) -> SyncResult<PullAndMergeOutcome> {
let vek = ctx.encryption_key()?;
- ctx.log("[V2Merge] Fetching vault snapshot for canonical merge (GET /v2/Vault)...").await;
- let snapshot = pull::fetch_snapshot(ctx).await?;
-
- // LEGACY: a server still on the sqlite-blob format cannot merge with a manifest-v1 vault; the caller pushes over it.
- if legacy::is_legacy_sqlite_blob_snapshot(&snapshot) {
- let legacy = legacy::open_legacy_snapshot(ctx, &snapshot).await?;
- pull::commit_revisions(ctx, &legacy.manifest_revisions, &legacy.bucket_revisions).await?;
- return Ok(PullAndMergeOutcome::LegacyServer { revision: legacy.revision });
- }
- let email_routing = pull::email_routing_of(&snapshot);
+ ctx.log("[Merge] Fetching vault snapshot for canonical merge (GET /v2/Vault)...").await;
+ let snapshot = match pull::fetch_snapshot(ctx).await? {
+ // LEGACY: a server still on the sqlite-blob format cannot merge with a manifest-v1 vault; the caller pushes over it.
+ Snapshot::LegacySqliteBlob(snapshot) => {
+ let legacy = legacy::open_legacy_snapshot(ctx, &snapshot).await?;
+ pull::commit_revisions(ctx, &legacy).await?;
+ return Ok(PullAndMergeOutcome::LegacyServer { revision: legacy.personal_revision });
+ }
+ Snapshot::Manifest(snapshot) => snapshot,
+ };
+ let email_routing = snapshot.email_routing.clone().unwrap_or_default();
let opened = pull::open_manifests_and_record_sync_state(ctx, &snapshot, &vek).await?;
let local_side = canonicalize_vault(ctx, None).await?;
let local_side = match ground_moved_under_canonicalize(&local_side.manifest_records, &opened) {
None => local_side,
Some(moved) => {
- ctx.warn(format!("[V2Merge] This snapshot changed {}; canonicalizing the local vault again so its blob hashes match the server's.", moved)).await;
+ ctx.warn(format!("[Merge] This snapshot changed {}; canonicalizing the local vault again so its blob hashes match the server's.", moved)).await;
canonicalize_vault(ctx, None).await?
}
};
@@ -57,7 +53,7 @@ pub(crate) async fn pull_and_merge(ctx: &mut Ctx) -> SyncResult<PullAndMergeOutc
Ok(outcome) => Ok(outcome),
Err(merge_error) => {
// The merge-failure fallback, from the same snapshot: the server vault stands, local changes are dropped.
- ctx.warn(format!("[V2Merge] Canonical merge failed, falling back to the server vault: {}", merge_error)).await;
+ ctx.warn(format!("[Merge] Canonical merge failed, falling back to the server vault: {}", merge_error)).await;
let sqlite_bytes = pull::materialize_to_sqlite(ctx, &opened.manifests(), &opened.data_buckets, &opened.blob_map, &opened.manifest_names).await?;
Ok(PullAndMergeOutcome::ServerOnly(opened.pulled_vault(state::encrypt_vault_blob(&sqlite_bytes, &vek)?, email_routing)))
}
@@ -84,9 +80,9 @@ fn ground_moved_under_canonicalize(records: &[ManifestRecord], opened: &OpenedMa
}
/// Merge the local side against the opened manifests, validate per manifest, and materialize.
-async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet, local_side: CanonicalizedSet, vek: &str, email_routing: EmailRoutingDto) -> SyncResult<PullAndMergeOutcome> {
+async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet, local_side: CanonicalizedSet, vek: &str, email_routing: super::types::EmailRoutingDto) -> SyncResult<PullAndMergeOutcome> {
let schema = ctx.schema().await?;
- ctx.log(format!("[V2Merge] Merging {} local manifest(s) onto {} server manifest(s)...", local_side.canonicalized.manifests.len(), opened.resolved.len())).await;
+ ctx.log(format!("[Merge] Merging {} local manifest(s) onto {} server manifest(s)...", local_side.canonicalized.manifests.len(), opened.resolved.len())).await;
let merge_output = merge_canonical(CanonicalMergeInput {
server_manifests: opened.manifests(),
server_buckets: opened.data_buckets.clone(),
@@ -105,28 +101,27 @@ async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet,
let mut manifests: Vec<Manifest> = Vec::new();
let mut data_buckets: Vec<DataBucket> = Vec::new();
- let mut fallback_manifest_ids = Vec::new();
+ let mut fallbacks = 0usize;
let mut stats = MergeStats::default();
for entry in merge_output.manifests {
- let failure = validate_merged_manifest(&entry);
- if let Some(failure) = &failure {
+ if let Some(failure) = validate_merged_manifest(&entry) {
if !contentless.contains(&id_key(&entry.manifest_id)) {
- ctx.warn(format!("[V2Merge] Merged manifest {} failed validation ({}); the server's version stands and local changes to it are dropped.", entry.manifest_id, failure)).await;
- fallback_manifest_ids.push(entry.manifest_id.clone());
+ ctx.warn(format!("[Merge] Merged manifest {} failed validation ({}); the server's version stands and local changes to it are dropped.", entry.manifest_id, failure)).await;
+ fallbacks += 1;
if let Some(server) = server_manifest_by_id.get(&id_key(&entry.manifest_id)) {
manifests.push((*server).clone());
data_buckets.extend(server_buckets_by_id.get(&id_key(&entry.manifest_id)).cloned().unwrap_or_default());
}
continue;
}
- ctx.warn(format!("[V2Merge] Pass-through manifest {} failed validation ({}); keeping its local rows.", entry.manifest_id, failure)).await;
+ ctx.warn(format!("[Merge] Pass-through manifest {} failed validation ({}); keeping its local rows.", entry.manifest_id, failure)).await;
}
manifests.push(entry.manifest);
data_buckets.extend(entry.buckets);
stats.add(&entry.stats);
}
for dropped in &merge_output.dropped_local_manifest_ids {
- ctx.warn(format!("[V2Merge] Local manifest {} is no longer served; its rows are dropped from the merged vault.", dropped)).await;
+ ctx.warn(format!("[Merge] Local manifest {} is no longer served; its rows are dropped from the merged vault.", dropped)).await;
}
// Blob bytes for materialize: the server download plus everything the local canonicalize extracted.
@@ -144,17 +139,14 @@ async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet,
let sqlite_bytes = pull::materialize_to_sqlite(ctx, &manifests, &data_buckets, &blob_map, &opened.manifest_names).await?;
let encrypted_vault = state::encrypt_vault_blob(&sqlite_bytes, vek)?;
- ctx.log(format!("[V2Merge] Canonical merge complete: {} conflict(s), {} offline row(s) kept, {} validation fallback(s), {} dropped local manifest(s).", stats.conflicts, stats.records_inserted, fallback_manifest_ids.len(), merge_output.dropped_local_manifest_ids.len())).await;
+ ctx.log(format!("[Merge] Canonical merge complete: {} local row(s) won, {} local-only row(s) kept, {} validation fallback(s), {} dropped local manifest(s).", stats.incoming_won, stats.incoming_only, fallbacks, merge_output.dropped_local_manifest_ids.len())).await;
- let push_canonical = merge_output_for_push(&manifests, &data_buckets, merged_blobs, &local_side.manifest_records, !fallback_manifest_ids.is_empty());
- Ok(PullAndMergeOutcome::Merged { pulled: opened.pulled_vault(encrypted_vault, email_routing), stats, fallback_manifest_ids, dropped_local_manifest_ids: merge_output.dropped_local_manifest_ids, push_canonical })
+ let push_canonical = if fallbacks == 0 { merge_output_for_push(&manifests, &data_buckets, merged_blobs, &local_side.manifest_records) } else { None };
+ Ok(PullAndMergeOutcome::Merged { pulled: opened.pulled_vault(encrypted_vault, email_routing), push_canonical })
}
-/// The merged vault in the shape the push writes from.
-fn merge_output_for_push(manifests: &[Manifest], data_buckets: &[DataBucket], blobs_by_manifest: HashMap<String, HashMap<String, BlobEntry>>, manifest_records: &[ManifestRecord], had_fallbacks: bool) -> Option<CanonicalizedSet> {
- if had_fallbacks {
- return None;
- }
+/// The merged vault in the shape the push writes from, when it covers exactly the manifests this session writes.
+fn merge_output_for_push(manifests: &[Manifest], data_buckets: &[DataBucket], blobs_by_manifest: HashMap<String, HashMap<String, BlobEntry>>, manifest_records: &[ManifestRecord]) -> Option<CanonicalizedSet> {
let merged_ids: HashSet<String> = manifests.iter().map(|m| id_key(&m.manifest_id)).collect();
let record_ids: Vec<String> = manifest_records.iter().map(|r| id_key(&r.manifest_id)).collect();
if merged_ids.len() != record_ids.len() || record_ids.iter().any(|id| !merged_ids.contains(id)) {
diff --git a/core/rust/src/vault_sync/migration.rs b/core/rust/src/vault_sync/migration.rs
new file mode 100644
index 000000000..c48ee6341
--- /dev/null
+++ b/core/rust/src/vault_sync/migration.rs
@@ -0,0 +1,149 @@
+//! The local storage-model migrations: where the vault's schema stands, the rebuild of a stale one, and the
+//! operations the upgrade gate drives (`migrationStatus`, `migrateManifest`).
+
+use std::collections::HashMap;
+
+use super::db::{self, SchemaState};
+use super::errors::{Failure, SyncError, SyncResult};
+use super::push::{self, PushStatus, WriteKind};
+use super::state::{self, Ctx};
+use super::types::{FullSyncResult, MigrateManifestResult, MigrationKind, MigrationStatusResult};
+use super::{keys, legacy, pull};
+
+/// Where the local vault's schema stands against the current one.
+pub(crate) async fn schema_state(ctx: &mut Ctx) -> SyncResult<SchemaState> {
+ let schema = ctx.schema().await?;
+ db::schema_state(&ctx.host, &schema.migration_id).await
+}
+
+/// Whether the local vault is still on a schema the codec cannot canonicalize.
+pub(crate) async fn vault_predates_current_schema(ctx: &mut Ctx) -> SyncResult<bool> {
+ Ok(schema_state(ctx).await? != SchemaState::Current)
+}
+
+/// Whether the vault still has to run the manifest migration: a stale schema, or a missing account key hierarchy.
+pub(crate) async fn vault_requires_manifest_migration(ctx: &mut Ctx) -> SyncResult<bool> {
+ Ok(schema_state(ctx).await? == SchemaState::Stale || !keys::has_cached_key_chain(&ctx.host).await?)
+}
+
+/// The sync outcome for a vault that still has a migration pending, `None` when it has none.
+pub(crate) async fn pending_migration_result(ctx: &mut Ctx) -> SyncResult<Option<FullSyncResult>> {
+ let checked: SyncResult<Option<FullSyncResult>> = async {
+ Ok(match schema_state(ctx).await? {
+ SchemaState::LegacyChain => Some(FullSyncResult { success: true, sqlite_blob_upgrade_required: true, ..Default::default() }),
+ SchemaState::Stale => Some(FullSyncResult { success: true, manifest_migration_required: true, ..Default::default() }),
+ SchemaState::Current if !keys::has_cached_key_chain(&ctx.host).await? => Some(FullSyncResult { success: true, manifest_migration_required: true, ..Default::default() }),
+ SchemaState::Current => None,
+ })
+ }
+ .await;
+ match checked {
+ Ok(result) => Ok(result),
+ Err(error @ SyncError::VaultVersionIncompatible(_)) => Err(error),
+ Err(error) => {
+ ctx.warn(format!("[Migration] Ignoring failed pending migration check: {}", error)).await;
+ Ok(None)
+ }
+ }
+}
+
+/// Classify the pending migration so the upgrade gate knows whether it may run on its own.
+pub(crate) async fn migration_status(ctx: &mut Ctx) -> MigrationStatusResult {
+ let classified: SyncResult<MigrationKind> = async {
+ let schema = schema_state(ctx).await?;
+ if schema == SchemaState::LegacyChain {
+ return Ok(MigrationKind::None);
+ }
+ if !keys::has_cached_key_chain(&ctx.host).await? {
+ // A hierarchy another device created since this device logged in classifies as no migration at all.
+ keys::accept_hierarchy_created_elsewhere(ctx).await?;
+ }
+ if !keys::has_cached_key_chain(&ctx.host).await? {
+ return Ok(MigrationKind::StorageFormatUpgrade);
+ }
+ Ok(if schema == SchemaState::Stale { MigrationKind::SchemaRebuild } else { MigrationKind::None })
+ }
+ .await;
+ let kind = match classified {
+ Ok(kind) => kind,
+ Err(error) => {
+ ctx.warn(format!("[Migration] Could not classify the pending migration, assuming it crosses the storage format: {}", error)).await;
+ MigrationKind::StorageFormatUpgrade
+ }
+ };
+ MigrationStatusResult { kind }
+}
+
+/// Bring the local vault onto the current storage model and push it: a schema rebuild for a migrated account, the
+/// one-time account upgrade (`legacy::upgrade_account_to_manifest_v1`) for an account without a key hierarchy yet.
+pub(crate) async fn migrate_manifest(ctx: &mut Ctx) -> MigrateManifestResult {
+ let migrated: SyncResult<bool> = async {
+ if !keys::has_cached_key_chain(&ctx.host).await? {
+ return legacy::upgrade_account_to_manifest_v1(ctx).await;
+ }
+ migrate_schema(ctx).await
+ }
+ .await;
+ match migrated {
+ Ok(pushed) => MigrateManifestResult { success: true, pushed, ..Default::default() },
+ Err(error) => {
+ ctx.warn(format!("[Migration] Migration failed: {}", error)).await;
+ MigrateManifestResult { failure: (&error).into(), ..Default::default() }
+ }
+ }
+}
+
+/// The permanent migration: rebuild a stale local schema onto the current one and push it. Returns whether the
+/// push reached the server; a vault already on the current schema counts as pushed.
+pub(crate) async fn migrate_schema(ctx: &mut Ctx) -> SyncResult<bool> {
+ match schema_state(ctx).await? {
+ SchemaState::Current => Ok(true),
+ SchemaState::LegacyChain => Err(SyncError::LegacyUpgradePending),
+ SchemaState::Stale => {
+ rebuild_local_schema(ctx, None).await?;
+ push_migrated_vault(ctx, WriteKind::Changes { buckets_only: false }).await
+ }
+ }
+}
+
+/// Rebuild the local vault onto the current schema locally and store it as a pending change; `stamp_unstamped_into`
+/// stamps rows that carry no manifest yet (sqlite-blob vaults only).
+pub(crate) async fn rebuild_local_schema(ctx: &mut Ctx, stamp_unstamped_into: Option<String>) -> SyncResult<()> {
+ ctx.log("[Migration] Migrating local vault onto the current schema (local round-trip, no server involved)...").await;
+ let set = push::canonicalize_vault(ctx, stamp_unstamped_into).await?;
+ let mut blob_map = HashMap::new();
+ for entry in &set.canonicalized.manifests {
+ for (hash, blob) in &entry.blobs {
+ blob_map.insert(hash.clone(), crate::common::encoding::base64_decode(&blob.bytes_base64)?);
+ }
+ }
+ let manifests: Vec<_> = set.canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect();
+ let bytes = pull::materialize_to_sqlite(ctx, &manifests, &set.canonicalized.data_buckets, &blob_map, &HashMap::new()).await?;
+ ctx.log(format!("[Migration] Migration complete: {} blobs re-embedded, {} bytes.", blob_map.len(), bytes.len())).await;
+
+ let stored = ctx.store_vault(&state::encrypt_vault_blob(&bytes, &ctx.encryption_key()?)?, true, None, None).await?;
+ ctx.mutation_sequence = stored.mutation_sequence;
+ ctx.is_dirty = true;
+ ctx.vault_changed = true;
+ Ok(())
+}
+
+/// Push the migrated vault and clear the dirty flag. A refused or failed push leaves the vault dirty for the next sync,
+/// except a failure that ends the session. Returns whether the push reached the server.
+pub(crate) async fn push_migrated_vault(ctx: &mut Ctx, kind: WriteKind) -> SyncResult<bool> {
+ match push::upload_vault(ctx, None, kind).await {
+ Ok(upload) if upload.status == PushStatus::Ok => {
+ state::mark_clean(&ctx.host, upload.mutation_seq_at_start).await?;
+ Ok(true)
+ }
+ Ok(upload) => {
+ ctx.warn(format!("[Migration] Migration push did not succeed ({:?}), vault stays dirty for the next sync.", upload.status)).await;
+ Ok(false)
+ }
+ Err(error) if matches!(error.failure(), Failure::Logout(_)) => Err(error),
+ Err(error) => {
+ ctx.warn(format!("[Migration] Migration push failed, vault stays dirty for the next sync: {}", error)).await;
+ Ok(false)
+ }
+ }
+}
diff --git a/core/rust/src/vault_sync/mod.rs b/core/rust/src/vault_sync/mod.rs
index f3c7b2503..2dbb01ff0 100644
--- a/core/rust/src/vault_sync/mod.rs
+++ b/core/rust/src/vault_sync/mod.rs
@@ -9,12 +9,14 @@ mod http;
mod keys;
mod legacy;
mod merge;
+mod migration;
mod pull;
mod push;
mod session;
mod sharing;
pub(crate) mod state;
pub(crate) mod types;
+mod write_set;
#[cfg(test)]
mod tests;
diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rs
index afbec70f3..31ef85e52 100644
--- a/core/rust/src/vault_sync/pull.rs
+++ b/core/rust/src/vault_sync/pull.rs
@@ -46,14 +46,14 @@ impl OpenedManifestSet {
/// The opened set as a vault ready to become the local one.
pub fn pulled_vault(&self, encrypted_vault: String, email_routing: EmailRoutingDto) -> PulledVault {
- PulledVault { encrypted_vault, revision: self.personal_revision, email_routing, manifest_revisions: self.manifest_revisions.clone(), bucket_revisions: self.bucket_revisions.clone(), needs_first_write: self.personal_needs_first_write }
+ PulledVault { encrypted_vault, personal_revision: self.personal_revision, email_routing, manifest_revisions: self.manifest_revisions.clone(), bucket_revisions: self.bucket_revisions.clone(), needs_first_write: self.personal_needs_first_write }
}
}
/// A pulled vault, encrypted for local storage, with the revisions that become the local truth once it is stored.
pub(crate) struct PulledVault {
pub encrypted_vault: String,
- pub revision: i64,
+ pub personal_revision: i64,
pub email_routing: EmailRoutingDto,
pub manifest_revisions: HashMap<String, i64>,
pub bucket_revisions: HashMap<String, i64>,
@@ -61,28 +61,26 @@ pub(crate) struct PulledVault {
pub needs_first_write: bool,
}
-/// `GET v2/Vault`.
-pub(crate) async fn fetch_snapshot(ctx: &Ctx) -> SyncResult<GetResponse> {
- let snapshot = http::with_outdated_server_guard(http::get::<GetResponse>(&ctx.host, http::VAULT_ENDPOINT, true).await)?;
- ensure_known_storage_format(&snapshot)?;
- Ok(snapshot)
-}
-
/// The `storageFormat` a manifest-v1 snapshot declares.
const STORAGE_FORMAT_MANIFEST: &str = "manifest";
-/// Refuse a snapshot in a storage format newer than this build knows, instead of reading it as legacy.
-fn ensure_known_storage_format(snapshot: &GetResponse) -> SyncResult<()> {
+/// A server snapshot by storage format.
+pub(crate) enum Snapshot {
+ Manifest(GetResponse),
+ /// LEGACY: the account's vault is still one sqlite blob.
+ LegacySqliteBlob(GetResponse),
+}
+
+/// `GET v2/Vault`. A snapshot in a storage format newer than this build knows is refused, never read as legacy.
+pub(crate) async fn fetch_snapshot(ctx: &Ctx) -> SyncResult<Snapshot> {
+ let snapshot = http::with_outdated_server_guard(http::get::<GetResponse>(&ctx.host, http::VAULT_ENDPOINT, true).await)?;
match snapshot.storage_format.as_deref() {
- None | Some(legacy::STORAGE_FORMAT_SQLITE_BLOB) | Some(STORAGE_FORMAT_MANIFEST) => Ok(()),
+ None | Some(legacy::STORAGE_FORMAT_SQLITE_BLOB) => Ok(Snapshot::LegacySqliteBlob(snapshot)),
+ Some(STORAGE_FORMAT_MANIFEST) => Ok(Snapshot::Manifest(snapshot)),
Some(format) => Err(SyncError::VaultVersionIncompatible(format!("vault has storage format {}, this app reads up to {}; update the app", format, STORAGE_FORMAT_MANIFEST))),
}
}
-pub(crate) fn email_routing_of(snapshot: &GetResponse) -> EmailRoutingDto {
- snapshot.email_routing.clone().unwrap_or_default()
-}
-
fn base64_chars(size_bytes: i64) -> usize {
((size_bytes.max(0) as usize).div_ceil(3)) * 4
}
@@ -132,17 +130,17 @@ fn select_personal_manifest(snapshot: &GetResponse) -> Option<&ManifestDto> {
/// Pull the latest vault: fetch, open, materialize and re-encrypt for local storage.
pub(crate) async fn pull(ctx: &mut Ctx) -> SyncResult<PulledVault> {
let vek = ctx.encryption_key()?;
- ctx.log("[V2Pull] Step 1/4: fetching vault snapshot (GET /v2/Vault)...").await;
- let snapshot = fetch_snapshot(ctx).await?;
- if legacy::is_legacy_sqlite_blob_snapshot(&snapshot) {
- return legacy::open_legacy_snapshot(ctx, &snapshot).await;
- }
+ ctx.log("[Pull] Fetching vault snapshot (GET /v2/Vault)...").await;
+ let snapshot = match fetch_snapshot(ctx).await? {
+ Snapshot::LegacySqliteBlob(snapshot) => return legacy::open_legacy_snapshot(ctx, &snapshot).await,
+ Snapshot::Manifest(snapshot) => snapshot,
+ };
- ctx.log("[V2Pull] Step 2/4: manifest format: decrypting and reassembling local SQLite...").await;
+ ctx.log("[Pull] Decrypting the manifests and reassembling the local SQLite...").await;
let opened = open_manifests_and_record_sync_state(ctx, &snapshot, &vek).await?;
let sqlite_bytes = materialize_to_sqlite(ctx, &opened.manifests(), &opened.data_buckets, &opened.blob_map, &opened.manifest_names).await?;
- ctx.log(format!("[V2Pull] Step 3/4: materialized SQLite ({} bytes); re-encrypting for local storage...", sqlite_bytes.len())).await;
- Ok(opened.pulled_vault(state::encrypt_vault_blob(&sqlite_bytes, &vek)?, email_routing_of(&snapshot)))
+ ctx.log(format!("[Pull] Materialized SQLite ({} bytes); re-encrypting for local storage...", sqlite_bytes.len())).await;
+ Ok(opened.pulled_vault(state::encrypt_vault_blob(&sqlite_bytes, &vek)?, snapshot.email_routing.clone().unwrap_or_default()))
}
/// Open every manifest a snapshot carries, personal first, and record the snapshot as this device's sync state.
@@ -150,7 +148,7 @@ pub(crate) async fn pull(ctx: &mut Ctx) -> SyncResult<PulledVault> {
pub(crate) async fn open_manifests_and_record_sync_state(ctx: &mut Ctx, snapshot: &GetResponse, vek: &str) -> SyncResult<OpenedManifestSet> {
let personal_dto = select_personal_manifest(snapshot).ok_or_else(|| SyncError::Snapshot("server returned no personal manifest, refusing to assemble".to_string()))?;
// Registration creates the personal manifest without content at revision 0; any later revision without content is damage.
- let personal_needs_first_write = personal_dto.blob.as_deref().unwrap_or("").is_empty();
+ let personal_needs_first_write = !personal_dto.has_content();
if personal_needs_first_write && personal_dto.revision != 0 {
return Err(SyncError::Snapshot(format!("server returned no manifest blob at revision {}, nothing to assemble", personal_dto.revision)));
}
@@ -166,15 +164,19 @@ pub(crate) async fn open_manifests_and_record_sync_state(ctx: &mut Ctx, snapshot
for dto in ordered {
let is_personal = dto.manifest_id == personal_dto.manifest_id;
let manifest_key = resolve_manifest_vek(ctx, dto, &personal_dto.manifest_id, vek, is_personal).await?;
- if is_personal && personal_needs_first_write {
- ctx.log("[V2Pull] Personal manifest has no content yet (new account); starting from an empty vault.").await;
- let entry = empty_personal_manifest(ctx, dto, &manifest_key).await?;
+ if is_personal {
+ let entry = if personal_needs_first_write {
+ ctx.log("[Pull] Personal manifest has no content yet (new account); starting from an empty vault.").await;
+ empty_personal_manifest(ctx, dto, &manifest_key).await?
+ } else {
+ open_logged_manifest(ctx, dto, &manifest_key, true).await?
+ };
state::set(&ctx.host, state::VAULT_MANIFEST_SALT, &entry.manifest.manifest_salt).await?;
state::set(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID, &entry.manifest_id).await?;
resolved.push(entry);
continue;
}
- if dto.blob.as_deref().unwrap_or("").is_empty() {
+ if !dto.has_content() {
// A shared manifest served without content yet (created but never written); its grant and revision are still tracked.
let (encrypted_vek, account_public_key, algorithm) = grant_of(dto).ok_or_else(|| SyncError::Snapshot(format!("shared manifest {} was served without content and without a grant, refusing to assemble", dto.manifest_id)))?;
let encrypted_name = served_encrypted_name(ctx, &previous_records, &dto.manifest_id);
@@ -183,30 +185,19 @@ pub(crate) async fn open_manifests_and_record_sync_state(ctx: &mut Ctx, snapshot
continue;
}
- ctx.log(format!("[V2Pull] Verifying ciphertext hash; decrypting + opening {}...", if is_personal { "personal manifest".to_string() } else { format!("shared manifest {}", dto.manifest_id) })).await;
- let entry = open_manifest(dto, &manifest_key, is_personal)?;
- let table_summary: Vec<String> = entry.manifest.tables.iter().map(|(t, rows)| format!("{}={}", t, rows.len())).collect();
- ctx.log(format!("[V2Pull] Manifest {} opened (content hash verified): tables: {}", entry.manifest_id, table_summary.join(", "))).await;
-
- if is_personal {
- state::set(&ctx.host, state::VAULT_MANIFEST_SALT, &entry.manifest.manifest_salt).await?;
- state::set(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID, &entry.manifest_id).await?;
- resolved.push(entry);
- continue;
- }
-
+ let entry = open_logged_manifest(ctx, dto, &manifest_key, false).await?;
let (encrypted_vek, account_public_key, algorithm) = grant_of(dto).ok_or_else(|| SyncError::Snapshot(format!("shared manifest {} carries no grant this account can re-open, refusing to assemble", entry.manifest_id)))?;
let encrypted_name = served_encrypted_name(ctx, &previous_records, &entry.manifest_id);
if let Some(ciphertext) = &encrypted_name {
match open_manifest_name(ciphertext, &entry.manifest_id, &manifest_key) {
Some(name) => drop(manifest_names.insert(id_key(&entry.manifest_id), name)),
- None => ctx.warn(format!("[V2Pull] The name of shared manifest {} did not open with its key; leaving it unnamed.", entry.manifest_id)).await,
+ None => ctx.warn(format!("[Pull] The name of shared manifest {} did not open with its key; leaving it unnamed.", entry.manifest_id)).await,
}
}
shared_records.insert(entry.manifest_id.clone(), SharedManifestDto { manifest_id: entry.manifest_id.clone(), encrypted_vek, account_public_key, algorithm, salt: entry.manifest.manifest_salt.clone(), encrypted_name, can_administer: dto.can_administer });
resolved.push(entry);
}
- keys::set_shared_manifest_records(&ctx.host, &shared_records, vek).await?;
+ keys::set_shared_manifest_records(ctx, &shared_records, vek).await?;
let personal_revision = resolved[0].revision;
@@ -216,14 +207,14 @@ pub(crate) async fn open_manifests_and_record_sync_state(ctx: &mut Ctx, snapshot
let mut manifest_revisions: HashMap<String, i64> = resolved.iter().map(|m| (m.manifest_id.clone(), m.revision)).collect();
manifest_revisions.extend(contentless_revisions.iter().map(|(id, rev)| (id.clone(), *rev)));
- let blob_map = download_referenced_blobs(ctx, &resolved, vek).await?;
+ let blob_map = download_referenced_blobs(ctx, &resolved).await?;
// An empty personal manifest has no server content to compare a push against, so it gets no baseline.
for entry in resolved.iter().filter(|m| !(m.is_personal && personal_needs_first_write)) {
pulled_fingerprints.insert(state::fingerprint_manifest_key(&entry.manifest_id), entry.content_fingerprint.clone());
}
state::set(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS, &pulled_fingerprints).await?;
- ctx.log(format!("[V2Pull] Stored {} content fingerprint baseline(s) for push-side change detection.", pulled_fingerprints.len())).await;
+ ctx.log(format!("[Pull] Stored {} content fingerprint baseline(s) for push-side change detection.", pulled_fingerprints.len())).await;
Ok(OpenedManifestSet {
resolved,
@@ -238,7 +229,16 @@ pub(crate) async fn open_manifests_and_record_sync_state(ctx: &mut Ctx, snapshot
})
}
-/// The encrypted name of a manifest.
+/// Open one manifest with content, logging what it holds.
+async fn open_logged_manifest(ctx: &Ctx, dto: &ManifestDto, manifest_key: &str, is_personal: bool) -> SyncResult<ResolvedManifest> {
+ ctx.log(format!("[Pull] Verifying ciphertext hash; decrypting + opening {}...", if is_personal { "personal manifest".to_string() } else { format!("shared manifest {}", dto.manifest_id) })).await;
+ let entry = open_manifest(dto, manifest_key, is_personal)?;
+ let table_summary: Vec<String> = entry.manifest.tables.iter().map(|(t, rows)| format!("{}={}", t, rows.len())).collect();
+ ctx.log(format!("[Pull] Manifest {} opened: tables: {}", entry.manifest_id, table_summary.join(", "))).await;
+ Ok(entry)
+}
+
+/// A manifest's encrypted name: the one the status served, else the one last recorded.
fn served_encrypted_name(ctx: &Ctx, previous: &HashMap<String, SharedManifestDto>, manifest_id: &str) -> Option<String> {
ctx.served_manifest_names
.get(&id_key(manifest_id))
@@ -256,11 +256,11 @@ pub(crate) fn encrypt_manifest_name(name: &str, manifest_id: &str, manifest_key:
Ok(crypto::symmetric_encrypt_with_aad(name, manifest_key, &crypto::aad::manifest_name(manifest_id))?)
}
-/// Commit a snapshot's revision maps as the local believed-current revisions, manifests and buckets together.
-pub(crate) async fn commit_revisions(ctx: &Ctx, manifest_revisions: &HashMap<String, i64>, bucket_revisions: &HashMap<String, i64>) -> SyncResult<()> {
- state::set(&ctx.host, state::SERVER_MANIFEST_REVISIONS, manifest_revisions).await?;
- state::set(&ctx.host, state::VAULT_BUCKET_REVISIONS, bucket_revisions).await?;
- ctx.log(format!("[V2Pull] Stored local manifest revisions from snapshot: {:?}; bucket revisions: {:?}.", manifest_revisions, bucket_revisions)).await;
+/// Commit a pulled vault's revisions as the local believed-current revisions, manifests and buckets together.
+pub(crate) async fn commit_revisions(ctx: &Ctx, pulled: &PulledVault) -> SyncResult<()> {
+ state::set(&ctx.host, state::SERVER_MANIFEST_REVISIONS, &pulled.manifest_revisions).await?;
+ state::set(&ctx.host, state::VAULT_BUCKET_REVISIONS, &pulled.bucket_revisions).await?;
+ ctx.log(format!("[Pull] Stored local manifest revisions from snapshot: {:?}; bucket revisions: {:?}.", pulled.manifest_revisions, pulled.bucket_revisions)).await;
Ok(())
}
@@ -281,7 +281,7 @@ async fn open_data_buckets(ctx: &Ctx, snapshot: &GetResponse, resolved: &[Resolv
return Err(SyncError::Snapshot(format!("{} declares a different address (manifest {}, category \"{}\") inside its encrypted payload, refusing to assemble", label, bucket.manifest_id, bucket.category)));
}
let rows: usize = bucket.tables.values().map(Vec::len).sum();
- ctx.log(format!("[V2Pull] Data bucket {} opened: {} rows (revision {:?}).", label, rows, dto.revision)).await;
+ ctx.log(format!("[Pull] Data bucket {} opened: {} rows (revision {:?}).", label, rows, dto.revision)).await;
fingerprints.insert(state::fingerprint_bucket_key(&dto.manifest_id, &dto.category), vault_codec::compute_content_fingerprint(&bucket_json));
if let Some(revision) = dto.revision {
revisions.insert(state::bucket_revision_key(&dto.manifest_id, &dto.category), revision);
@@ -292,7 +292,7 @@ async fn open_data_buckets(ctx: &Ctx, snapshot: &GetResponse, resolved: &[Resolv
}
/// Fetch every referenced blob not cached locally, decrypt it, and prune the cache to the referenced set.
-async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], fallback_vek: &str) -> SyncResult<HashMap<String, Vec<u8>>> {
+async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest]) -> SyncResult<HashMap<String, Vec<u8>>> {
let mut owners: HashMap<String, &ResolvedManifest> = HashMap::new();
let mut refs: Vec<StoredBlobRef> = Vec::new();
for entry in resolved {
@@ -307,7 +307,7 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], fal
let mut cache: HashMap<String, EncryptedBlob> = state::get(&ctx.host, state::VAULT_BLOB_CIPHER_CACHE).await?.unwrap_or_default();
let missing: Vec<StoredBlobRef> = refs.iter().filter(|r| !cache.contains_key(&r.hash)).cloned().collect();
- ctx.log(format!("[V2Pull] Blob refs: {} referenced, {} cached locally, {} to download.", refs.len(), refs.len() - missing.len(), missing.len())).await;
+ ctx.log(format!("[Pull] Blob refs: {} referenced, {} cached locally, {} to download.", refs.len(), refs.len() - missing.len(), missing.len())).await;
// The server stores blobs per manifest, so each batch names the manifest that owns its hashes.
let mut batches: Vec<(String, Vec<StoredBlobRef>)> = Vec::new();
@@ -318,7 +318,7 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], fal
let batch_count = batches.len();
for (index, (manifest_id, chunk)) in batches.into_iter().enumerate() {
let blobs = http::post::<_, BlobDownloadResponse>(&ctx.host, BLOBS_DOWNLOAD_ENDPOINT, &BlobHashesRequest { manifest_id, hashes: chunk.iter().map(|r| r.hash.clone()).collect() }, true).await?.blobs;
- ctx.log(format!("[V2Pull] Downloaded blob batch {}/{}: requested {}, received {}.", index + 1, batch_count, chunk.len(), blobs.len())).await;
+ ctx.log(format!("[Pull] Downloaded blob batch {}/{}: requested {}, received {}.", index + 1, batch_count, chunk.len(), blobs.len())).await;
for dto in blobs {
cache.insert(dto.hash, EncryptedBlob { encrypted_data_base64: dto.encrypted_data_base64, encrypted_blob_key: dto.encrypted_blob_key });
}
@@ -328,16 +328,16 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], fal
let mut blob_map = HashMap::new();
for reference in &refs {
let Some(encrypted) = cache.get(&reference.hash) else {
- ctx.warn(format!("[V2Sync] Referenced {} blob {} was not served; its row stays not loaded and keeps the reference.", reference.category, reference.hash)).await;
+ ctx.warn(format!("[Pull] Referenced {} blob {} was not served; its row stays not loaded and keeps the reference.", reference.category, reference.hash)).await;
continue;
};
- let (key, manifest_id) = owners.get(&reference.hash).map(|o| (o.vek.as_str(), o.manifest_id.as_str())).unwrap_or((fallback_vek, ""));
- match blob_keys::decrypt_blob(encrypted, key, manifest_id, &reference.hash) {
+ let owner = owners[&reference.hash];
+ match blob_keys::decrypt_blob(encrypted, &owner.vek, &owner.manifest_id, &reference.hash) {
Ok(bytes) => {
blob_map.insert(reference.hash.clone(), bytes);
pruned_cache.insert(reference.hash.clone(), encrypted.clone());
}
- Err(_) => ctx.warn(format!("[V2Sync] Referenced {} blob {} did not decrypt with the manifest key; its row stays not loaded and keeps the reference.", reference.category, reference.hash)).await,
+ Err(_) => ctx.warn(format!("[Pull] Referenced {} blob {} did not decrypt with the manifest key; its row stays not loaded and keeps the reference.", reference.category, reference.hash)).await,
}
}
state::set(&ctx.host, state::VAULT_BLOB_CIPHER_CACHE, &pruned_cache).await?;
@@ -347,13 +347,13 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], fal
/// Materialize manifests and buckets into a fresh database and return its bytes.
pub(crate) async fn materialize_to_sqlite(ctx: &mut Ctx, manifests: &[Manifest], data_buckets: &[DataBucket], blob_map: &HashMap<String, Vec<u8>>, opened_names: &HashMap<String, String>) -> SyncResult<Vec<u8>> {
- ctx.log(format!("[V2Pull] {} blobs decrypted; running codec reassembly into a fresh SQLite ({} manifest(s) combined)...", blob_map.len(), manifests.len())).await;
+ ctx.log(format!("[Pull] {} blobs decrypted; running codec reassembly into a fresh SQLite ({} manifest(s) combined)...", blob_map.len(), manifests.len())).await;
let schema = ctx.schema().await?;
let materialized = vault_codec::materialize_as_sqlite(MaterializeInput { manifests: manifests.to_vec(), data_buckets: data_buckets.to_vec(), schema_columns: schema.columns.clone() })?;
let overflow_tables = materialized.overflow.tables.len() + materialized.overflow.bucket_tables.values().map(HashMap::len).sum::<usize>();
if overflow_tables > 0 || !materialized.overflow.columns.is_empty() {
- ctx.warn(format!("[V2Pull] Newer-schema data preserved as overflow: {} unknown table(s), unknown columns on [{}].", overflow_tables, materialized.overflow.columns.keys().cloned().collect::<Vec<_>>().join(", "))).await;
+ ctx.warn(format!("[Pull] Newer-schema data preserved as overflow: {} unknown table(s), unknown columns on [{}].", overflow_tables, materialized.overflow.columns.keys().cloned().collect::<Vec<_>>().join(", "))).await;
}
// Use a fresh staging database for every materialize.
@@ -365,7 +365,7 @@ pub(crate) async fn materialize_to_sqlite(ctx: &mut Ctx, manifests: &[Manifest],
names.extend(opened_names.clone());
db::apply_manifest_names(&ctx.host, Db::Staging, &names).await?;
let bytes = db::export(&ctx.host, Db::Staging).await?;
- ctx.log(format!("[V2Pull] Codec reassembly complete: {} bytes.", bytes.len())).await;
+ ctx.log(format!("[Pull] Codec reassembly complete: {} bytes.", bytes.len())).await;
Ok(bytes)
}
diff --git a/core/rust/src/vault_sync/push.rs b/core/rust/src/vault_sync/push.rs
index 4b1e8fba5..5981d5095 100644
--- a/core/rust/src/vault_sync/push.rs
+++ b/core/rust/src/vault_sync/push.rs
@@ -2,18 +2,18 @@
//! changed under the key of the manifest that owns it, upload the blobs the server lacks, and `POST v2/Vault`.
use std::collections::{HashMap, HashSet};
-use std::ops::Deref;
-use crate::vault_model::{id_key, ids_equal, OVERFLOW_TABLE, TRASH_RETENTION_DEFAULT_DAYS};
+use crate::vault_model::names::ITEMS_TABLE;
+use crate::vault_model::{id_key, OVERFLOW_TABLE, TRASH_RETENTION_DEFAULT_DAYS};
use super::email_routing::build_email_routing;
use super::errors::{SyncError, SyncResult};
use super::state::{self, Ctx};
use super::types::{BlobDto, BlobHashesRequest, BlobRef, BlobUploadRequest, BucketRevision, BucketWrite, Db, ManifestRevision, ManifestWrite, MissingBlobsResponse, VaultWriteMigration, VaultWriteRequest, VaultWriteResponse, VaultWriteStatus, ALGORITHM_RSA_OAEP_SHA256};
use super::blob_keys::{self, EncryptedBlob};
+use super::write_set::{self, ManifestRecord, SkipReason};
use super::{db, http, keys};
use crate::crypto;
use crate::vault_codec::{self, BlobEntry, CanonicalizeInput, CanonicalizedVault, DataBucket, Manifest, ManifestSpec};
-use crate::vault_sharing::{self, ManifestAccessRequest, ManifestWriteRecord, ManifestWriteSetRequest, SharedManifestRecord};
const BLOBS_ENDPOINT: &str = "Vault/blobs";
const BLOBS_MISSING_ENDPOINT: &str = "Vault/blobs/missing";
@@ -21,20 +21,23 @@ const BLOBS_MISSING_ENDPOINT: &str = "Vault/blobs/missing";
/// The mutation scope that requires a full manifest push.
const MANIFEST_SCOPE: &str = "Main";
-/// One manifest this vault can write, resolved from local state before canonicalizing: the write record the
-/// sharing logic produced plus the key it encrypts under. Derefs to the record so its fields read directly.
-#[derive(Debug, Clone)]
-pub(crate) struct ManifestRecord {
- pub record: ManifestWriteRecord,
- /// The key this manifest encrypts with; None for the personal manifest, whose content key the push supplies.
- pub vek: Option<String>,
+/// What a push writes.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum WriteKind {
+ /// What changed against the last known server state. With `buckets_only`, a host that recorded only
+ /// bucket-scoped mutations gets away with writing those buckets, never canonicalizing the manifests.
+ Changes { buckets_only: bool },
+ /// Every manifest and bucket, whatever the fingerprints say: the server never saw this vault.
+ Everything,
+ /// The one-time account upgrade: creates the key hierarchy and encrypts every personal blob again under the
+ /// new key, so the whole vault is written.
+ AccountKeyMigration,
}
-impl Deref for ManifestRecord {
- type Target = ManifestWriteRecord;
-
- fn deref(&self) -> &ManifestWriteRecord {
- &self.record
+impl WriteKind {
+ /// Whether elements whose fingerprint matches the server baseline are written anyway.
+ fn writes_unchanged(self) -> bool {
+ !matches!(self, WriteKind::Changes { .. })
}
}
@@ -57,7 +60,7 @@ pub(crate) enum PushStatus {
pub(crate) struct UploadOutcome {
pub status: PushStatus,
pub mutation_seq_at_start: u64,
- /// Whether the stored vault changed under the upload (pruned or re-keyed).
+ /// Whether the stored vault was rewritten under the upload (pruned or keypair added).
pub vault_changed: bool,
}
@@ -74,9 +77,9 @@ pub(crate) async fn canonicalize_vault(ctx: &Ctx, stamp_unstamped_into: Option<S
Ok(CanonicalizedSet { canonicalized, manifest_records })
}
-/// Every manifest this vault can write, personal manifest first. The core decides which.
+/// Every manifest this vault can write, personal manifest first.
async fn resolve_manifest_records(ctx: &Ctx) -> SyncResult<Vec<ManifestRecord>> {
- let manifest_salt = match state::get::<String>(&ctx.host, state::VAULT_MANIFEST_SALT).await? {
+ let personal_salt = match state::get::<String>(&ctx.host, state::VAULT_MANIFEST_SALT).await? {
Some(salt) => salt,
None => {
let salt = vault_codec::generate_manifest_salt();
@@ -84,31 +87,17 @@ async fn resolve_manifest_records(ctx: &Ctx) -> SyncResult<Vec<ManifestRecord>>
salt
}
};
- let shared_veks = keys::open_shared_manifest_veks(ctx).await?;
+ let personal_id = resolve_personal_manifest_id(ctx).await?;
+ let stamped = db::manifest_ids_in_vault(&ctx.host, Db::Local).await?;
+ let opened = keys::open_shared_manifest_veks(ctx).await?;
let held = keys::shared_manifest_records(ctx).await?;
- let write_set = vault_sharing::resolve_manifest_write_set(ManifestWriteSetRequest {
- personal_manifest_id: resolve_personal_manifest_id(ctx).await?,
- personal_manifest_salt: manifest_salt,
- stamped_manifest_ids: db::manifest_ids_in_vault(&ctx.host, Db::Local).await?,
- opened_manifest_ids: shared_veks.keys().cloned().collect(),
- held_records: held.values().map(|r| SharedManifestRecord { manifest_id: r.manifest_id.clone(), salt: r.salt.clone(), can_administer: r.can_administer }).collect(),
- });
- for skipped in &write_set.skipped {
- let why = match skipped.reason {
- vault_sharing::WriteSkipReason::NoRowsInVault => "has no rows in this vault; leaving it out of the write rather than emptying it server-side",
- vault_sharing::WriteSkipReason::KeyDidNotOpen => "did not open; leaving it out of the write",
+ let (records, skipped) = write_set::resolve_write_set(&personal_id, &personal_salt, &stamped, &opened, &held);
+ for (manifest_id, reason) in skipped {
+ let why = match reason {
+ SkipReason::NoRowsInVault => "has no rows in this vault; leaving it out of the write rather than emptying it server-side",
+ SkipReason::KeyDidNotOpen => "did not open; leaving it out of the write",
};
- ctx.log(format!("[V2Push] Shared manifest {} {}.", skipped.manifest_id, why)).await;
- }
-
- let vek_by_id: HashMap<String, String> = shared_veks.iter().map(|(id, vek)| (id_key(id), vek.clone())).collect();
- let mut records = Vec::new();
- for record in write_set.records {
- let vek = if record.is_personal { None } else { vek_by_id.get(&id_key(&record.manifest_id)).cloned() };
- if !record.is_personal && vek.is_none() {
- return Err(SyncError::Other(format!("manifest {} is in the write set without a key, refusing to write it", record.manifest_id)));
- }
- records.push(ManifestRecord { record, vek });
+ ctx.log(format!("[Push] Shared manifest {} {}.", manifest_id, why)).await;
}
Ok(records)
}
@@ -118,6 +107,13 @@ pub(crate) async fn resolve_personal_manifest_id(ctx: &Ctx) -> SyncResult<String
state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await?.ok_or_else(|| SyncError::Other("no personal manifest id available (no snapshot baseline recorded); pull once before pushing".to_string()))
}
+/// The key each writable manifest encrypts with, the personal one under the session key.
+async fn resolve_write_keys(ctx: &Ctx) -> SyncResult<HashMap<String, String>> {
+ let mut keys_by_manifest = keys::open_shared_manifest_veks(ctx).await?;
+ keys_by_manifest.insert(resolve_personal_manifest_id(ctx).await?, ctx.encryption_key()?);
+ Ok(keys_by_manifest)
+}
+
/// Whether the local vault is canonically identical to the last-known server state. Returns the canonicalize
/// result too, so a push right after does not canonicalize a second time.
pub(crate) async fn detect_no_op_mutation(ctx: &Ctx) -> SyncResult<(bool, CanonicalizedSet)> {
@@ -140,13 +136,10 @@ pub(crate) async fn detect_no_op_mutation(ctx: &Ctx) -> SyncResult<(bool, Canoni
}
/// The manifests the local vault holds rows for that this session cannot write.
-pub(crate) async fn find_unwritable_manifests(ctx: &Ctx) -> SyncResult<Vec<String>> {
- let mut writable: Vec<String> = keys::open_shared_manifest_veks(ctx).await?.keys().cloned().collect();
- if let Some(personal) = state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await? {
- writable.push(personal);
- }
- let partition = vault_sharing::partition_manifest_access(ManifestAccessRequest { manifest_ids_in_vault: db::manifest_ids_in_vault(&ctx.host, Db::Local).await?, writable_manifest_ids: writable, granted_manifest_ids: Vec::new() });
- Ok(partition.unwritable)
+async fn find_unwritable_manifests(ctx: &Ctx) -> SyncResult<Vec<String>> {
+ let mut writable: Vec<String> = keys::open_shared_manifest_veks(ctx).await?.into_keys().collect();
+ writable.extend(state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await?);
+ Ok(write_set::unwritable_manifests(&db::manifest_ids_in_vault(&ctx.host, Db::Local).await?, &writable))
}
/// Whether the vault holds rows this session cannot write, logging why; a failed check reads as "no".
@@ -154,117 +147,125 @@ pub(crate) async fn vault_holds_unwritable_manifests(ctx: &Ctx) -> bool {
match find_unwritable_manifests(ctx).await {
Ok(unwritable) if unwritable.is_empty() => false,
Ok(unwritable) => {
- ctx.warn(format!("[VaultSync] Vault holds rows for manifest(s) this session cannot write ({}); pulling before the push.", unwritable.join(", "))).await;
+ ctx.warn(format!("[Push] Vault holds rows for manifest(s) this session cannot write ({}); pulling before the push.", unwritable.join(", "))).await;
true
}
Err(error) => {
- ctx.warn(format!("[VaultSync] Could not check which manifests this session can write; leaving the pull decision to the revisions. {}", error)).await;
+ ctx.warn(format!("[Push] Could not check which manifests this session can write; leaving the pull decision to the revisions. {}", error)).await;
false
}
}
}
-/// The key each manifest's data buckets are encrypted with.
-async fn resolve_bucket_write_keys(ctx: &Ctx, personal_vek: &str) -> SyncResult<HashMap<String, String>> {
- let mut keys_by_manifest = HashMap::new();
- keys_by_manifest.insert(resolve_personal_manifest_id(ctx).await?, personal_vek.to_string());
- keys_by_manifest.extend(keys::open_shared_manifest_veks(ctx).await?);
- Ok(keys_by_manifest)
-}
-
/*
* The upload as the flow drives it.
*/
-/// Upload the stored vault: bucket-only when every pending mutation is bucket-scoped, full otherwise. Fails
-/// with `KeyOutOfSync` when the session key does not open a hierarchy another device created meanwhile.
-pub(crate) async fn upload_vault(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)>, force_full_write: bool, create_vault_key: bool) -> SyncResult<UploadOutcome> {
+/// Upload the stored vault. Fails with `KeyOutOfSync` when the session key does not open a hierarchy another
+/// device created meanwhile; an account without a hierarchy gets the migration write whatever `kind` asked for.
+pub(crate) async fn upload_vault(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)>, kind: WriteKind) -> SyncResult<UploadOutcome> {
let mutation_seq_at_start = ctx.mutation_sequence;
- if !keys::has_local_vault_key(&ctx.host).await? && !keys::accept_hierarchy_created_elsewhere(ctx).await? {
- return Err(SyncError::KeyOutOfSync);
- }
-
- let create_vault_key = create_vault_key || !keys::has_local_vault_key(&ctx.host).await?;
- let scopes = ctx.request.dirty_scopes.clone();
- // A missing delivery keypair is created on the full path only, as publishing it takes a manifest write.
- let bucket_only = !force_full_write && !create_vault_key && !scopes.is_empty() && !scopes.iter().any(|s| s == MANIFEST_SCOPE) && !personal_delivery_key_missing(ctx).await?;
+ let kind = if keys::ensure_key_chain_accepted(ctx).await? { kind } else { WriteKind::AccountKeyMigration };
- let (status, vault_changed) = if bucket_only {
- (upload_dirty_buckets_only(ctx, &scopes).await?, false)
- } else {
- let cached = cache.filter(|(seq, _)| *seq == ctx.mutation_sequence).map(|(_, set)| set);
- upload_new_vault_to_server(ctx, cached, force_full_write, create_vault_key).await?
- };
+ if kind == (WriteKind::Changes { buckets_only: true }) {
+ if let Some(categories) = bucket_only_categories(ctx).await? {
+ let status = write_dirty_buckets(ctx, &categories).await?;
+ return Ok(UploadOutcome { status, mutation_seq_at_start, vault_changed: false });
+ }
+ }
+ let cached = cache.filter(|(seq, _)| *seq == ctx.mutation_sequence).map(|(_, set)| set);
+ let (status, vault_changed) = write_whole_vault(ctx, cached, kind).await?;
Ok(UploadOutcome { status, mutation_seq_at_start, vault_changed })
}
-/// Push only the data buckets named by the dirty scopes, no manifest upload.
-async fn upload_dirty_buckets_only(ctx: &mut Ctx, scopes: &[String]) -> SyncResult<PushStatus> {
- let key = ctx.encryption_key()?;
- if state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await?.is_none() {
- ctx.warn("[V2Push] No manifest to address the bucket write to, falling back to a full vault upload.").await;
- return Ok(upload_new_vault_to_server(ctx, None, false, false).await?.0);
+/// The bucket categories a bucket-only write covers, or `None` when the write has to go through the manifests: no
+/// scopes recorded, a manifest scope among them, an unknown scope, or a personal delivery key still to be published.
+async fn bucket_only_categories(ctx: &Ctx) -> SyncResult<Option<Vec<String>>> {
+ let scopes = &ctx.request.dirty_scopes;
+ if scopes.is_empty() || scopes.iter().any(|s| s == MANIFEST_SCOPE) {
+ return Ok(None);
+ }
+ let layout = vault_codec::bucket_layout();
+ let mut categories: Vec<String> = Vec::new();
+ for scope in scopes {
+ if !layout.iter().any(|entry| &entry.category == scope) {
+ ctx.warn(format!("[Push] Unknown bucket scope \"{}\", writing the whole vault instead.", scope)).await;
+ return Ok(None);
+ }
+ if !categories.contains(scope) {
+ categories.push(scope.clone());
+ }
}
+ // A missing delivery keypair is created on the full path only, as publishing it takes a manifest write.
+ Ok(if personal_delivery_key_missing(ctx).await? { None } else { Some(categories) })
+}
+
+/// Write the data buckets of `categories` and nothing else: no manifest is canonicalized.
+async fn write_dirty_buckets(ctx: &Ctx, categories: &[String]) -> SyncResult<PushStatus> {
if vault_holds_unwritable_manifests(ctx).await {
return Ok(PushStatus::Outdated);
}
+ let write_keys = resolve_write_keys(ctx).await?;
+ let mut buckets: Vec<DataBucket> = Vec::new();
+ for category in categories {
+ let mut names: Vec<String> = vault_codec::tables_for_category(category).into_iter().map(str::to_string).collect();
+ names.push(OVERFLOW_TABLE.to_string());
+ let tables = db::read_named_tables(&ctx.host, Db::Local, &names).await?;
+ buckets.extend(vault_codec::extract_buckets(category.clone(), write_keys.keys().cloned().collect(), tables)?);
+ }
- let layout = vault_codec::bucket_layout();
- let write_keys = resolve_bucket_write_keys(ctx, &key).await?;
- let mut seen = HashSet::new();
- for category in scopes.iter().filter(|s| seen.insert((*s).clone())) {
- let Some(spec) = layout.iter().find(|entry| &entry.category == category) else {
- ctx.warn(format!("[V2Push] Unknown bucket scope \"{}\", falling back to full vault upload.", category)).await;
- return Ok(upload_new_vault_to_server(ctx, None, false, false).await?.0);
- };
- let tables = read_bucket_tables(ctx, &spec.tables).await?;
- let buckets = vault_codec::extract_buckets(category.clone(), write_keys.keys().cloned().collect(), tables)?;
- for bucket in buckets {
- let Some(vek) = write_keys.get(&bucket.manifest_id) else { continue };
- let (status, revision) = push_data_bucket_only(ctx, &bucket, vek).await?;
- if status != PushStatus::Ok {
- return Ok(PushStatus::Outdated);
- }
- ctx.log(format!("[V2Push] Bucket-only push for \"{}\" of manifest {} done (bucket revision {}).", category, bucket.manifest_id, revision)).await;
- }
+ let baselines = PushBaselines::load(ctx).await?;
+ let mut written = WrittenFingerprints::default();
+ let writes = encrypt_changed_buckets(ctx, &buckets, &write_keys, &baselines, WriteKind::Changes { buckets_only: true }, &mut written).await?;
+ if writes.is_empty() {
+ ctx.log("[Push] No bucket changed versus the server baselines; skipping upload.").await;
+ return Ok(PushStatus::Ok);
+ }
+ let payload = VaultWriteRequest { username: ctx.request.username.clone(), manifests: Vec::new(), buckets: writes, email_routing: None, migration: None };
+ let response: VaultWriteResponse = http::with_outdated_server_guard(http::post(&ctx.host, http::VAULT_ENDPOINT, &payload, true).await)?;
+ if response.status != VaultWriteStatus::Ok {
+ ctx.warn("[Push] Bucket-only write outdated; pulling and merging before the next attempt.").await;
+ return Ok(PushStatus::Outdated);
}
+ commit_push_baselines(ctx, baselines, &response.manifest_revisions, &response.bucket_revisions, written).await?;
+ ctx.log(format!("[Push] Bucket-only write of {} bucket(s) done.", response.bucket_revisions.len())).await;
Ok(PushStatus::Ok)
}
-/// Upload a new version of the vault, pruning expired trash items and creating a missing personal delivery keypair
-/// first. Returns the status and whether the stored vault changed (pruned, keypair added or re-keyed).
-async fn upload_new_vault_to_server(ctx: &mut Ctx, cached: Option<CanonicalizedSet>, force_full_write: bool, create_vault_key: bool) -> SyncResult<(PushStatus, bool)> {
+/// Write the whole vault: prune expired trash and create a missing personal delivery keypair first, then the gated
+/// write. Returns the status and whether the stored vault was rewritten (pruned or keypair added).
+async fn write_whole_vault(ctx: &mut Ctx, cached: Option<CanonicalizedSet>, kind: WriteKind) -> SyncResult<(PushStatus, bool)> {
let key_before = ctx.encryption_key()?;
let mut cached = cached;
let mut vault_rewritten = false;
match db::prune_in_place(&ctx.host, TRASH_RETENTION_DEFAULT_DAYS).await {
Ok(0) => {}
Ok(count) => {
- ctx.log(format!("[VaultMerge] Pruned expired items from trash ({} SQL statements executed)", count)).await;
+ ctx.log(format!("[Push] Pruned expired items from trash ({} SQL statements executed).", count)).await;
vault_rewritten = true;
cached = None;
}
- Err(error) => ctx.warn(format!("[VaultSync] Failed to prune vault, continuing with upload: {}", error)).await,
+ Err(error) => ctx.warn(format!("[Push] Failed to prune vault, continuing with upload: {}", error)).await,
}
if personal_delivery_key_missing(ctx).await? {
- create_personal_delivery_key(ctx).await?;
+ let personal_manifest_id = resolve_personal_manifest_id(ctx).await?;
+ let pair = crypto::generate_rsa_key_pair()?;
+ db::set_active_key_for_manifest(&ctx.host, &personal_manifest_id, &pair.public_key, &pair.private_key).await?;
+ ctx.log(format!("[Push] Personal manifest {} had no mail delivery keypair; created one.", personal_manifest_id)).await;
vault_rewritten = true;
cached = None;
}
- let (status, new_key) = push(ctx, cached, create_vault_key, force_full_write).await?;
+ let (status, new_key) = http::with_outdated_server_guard(write_manifests_and_buckets(ctx, cached, kind).await)?;
if let Some(new_key) = new_key {
// Migration succeeded: from now on the session key is the VEK, not the password-derived key.
keys::re_encrypt_shared_manifest_records(ctx, &new_key).await?;
ctx.set_encryption_key(new_key);
}
- // Re-encrypt and persist locally only when the stored blob went stale or the encryption key changed.
- let key_after = ctx.encryption_key()?;
- if vault_rewritten || key_after != key_before {
- let bytes = db::export(&ctx.host, Db::Local).await?;
- ctx.store_vault(&state::encrypt_vault_blob(&bytes, &key_after)?, false, None, None).await?;
- ctx.vault_changed = true;
+ // Store the vault again only when it changed or its key did.
+ if vault_rewritten || ctx.encryption_key()? != key_before {
+ ctx.persist_local_vault(false).await?;
}
Ok((status, vault_rewritten))
}
@@ -278,24 +279,8 @@ async fn personal_delivery_key_missing(ctx: &Ctx) -> SyncResult<bool> {
Ok(db::active_public_key_for_manifest(&ctx.host, &personal_manifest_id).await?.is_none())
}
-/// Create the personal manifest's mail delivery keypair in the local vault; the manifest write publishes its public half.
-async fn create_personal_delivery_key(ctx: &Ctx) -> SyncResult<()> {
- let personal_manifest_id = resolve_personal_manifest_id(ctx).await?;
- let pair = crypto::generate_rsa_key_pair()?;
- db::set_active_key_for_manifest(&ctx.host, &personal_manifest_id, &pair.public_key, &pair.private_key).await?;
- ctx.log(format!("[V2Push] Personal manifest {} had no mail delivery keypair; created one.", personal_manifest_id)).await;
- Ok(())
-}
-
-/// The tables of one bucket category read from the local vault, as extract_buckets takes them.
-async fn read_bucket_tables(ctx: &Ctx, category_tables: &[String]) -> SyncResult<HashMap<String, Vec<vault_codec::CodecRecord>>> {
- let mut names: Vec<String> = category_tables.to_vec();
- names.push(OVERFLOW_TABLE.to_string());
- db::read_named_tables(&ctx.host, Db::Local, &names).await
-}
-
/*
- * The full write.
+ * The gated write.
*/
/// The local baselines a write gates against and, on success, advances.
@@ -326,15 +311,6 @@ impl PushBaselines {
}
}
-/// What decides whether an unchanged element still goes into the write.
-#[derive(Clone, Copy)]
-struct WriteGate {
- /// Write every manifest and bucket whatever its fingerprint.
- force_full_write: bool,
- /// Upload every personal blob again, replacing the server copy (e.g. a KEK/VEK migration push changed its key).
- overwrite_personal_blobs: bool,
-}
-
/// One manifest of the write: its record, canonical content and the key it encrypts under.
struct Candidate<'a> {
record: &'a ManifestRecord,
@@ -429,68 +405,62 @@ struct EncryptedPayload {
async fn encrypt_payload(ctx: &Ctx, label: &str, plaintext: &str, key: &str, aad: &[u8]) -> SyncResult<EncryptedPayload> {
let packed = vault_codec::pack_payload(plaintext)?;
let ciphertext = crypto::symmetric_encrypt_bytes_with_aad(&packed, key, aad)?;
- ctx.log(format!("[V2Push] {}: raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(packed.len()), format_kb(ciphertext.len()))).await;
+ ctx.log(format!("[Push] {}: raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(packed.len()), format_kb(ciphertext.len()))).await;
let hash = vault_codec::compute_ciphertext_hash(&ciphertext);
Ok(EncryptedPayload { ciphertext, hash })
}
-/// Canonicalize, gate by content fingerprint, encrypt and `POST v2/Vault`. Returns the outcome and, on a KEK/VEK
+/// Canonicalize, gate by content fingerprint, encrypt and `POST v2/Vault`. Returns the outcome and, on a
/// migration push, the new content key the session switches to.
-pub(crate) async fn push(ctx: &mut Ctx, cached: Option<CanonicalizedSet>, create_vault_key: bool, force_full_write: bool) -> SyncResult<(PushStatus, Option<String>)> {
- http::with_outdated_server_guard(push_internal(ctx, cached, create_vault_key, force_full_write).await)
-}
-
-async fn push_internal(ctx: &mut Ctx, cached: Option<CanonicalizedSet>, create_vault_key: bool, force_full_write: bool) -> SyncResult<(PushStatus, Option<String>)> {
+async fn write_manifests_and_buckets(ctx: &mut Ctx, cached: Option<CanonicalizedSet>, kind: WriteKind) -> SyncResult<(PushStatus, Option<String>)> {
let vek = ctx.encryption_key()?;
let unwritable = find_unwritable_manifests(ctx).await?;
if !unwritable.is_empty() {
- ctx.warn(format!("[V2Push] Vault holds rows for manifest(s) this session cannot write ({}); refusing the write until a pull restores them.", unwritable.join(", "))).await;
+ ctx.warn(format!("[Push] Vault holds rows for manifest(s) this session cannot write ({}); refusing the write until a pull restores them.", unwritable.join(", "))).await;
return Ok((PushStatus::Outdated, None));
}
- let migration = start_account_key_migration(ctx, &vek, create_vault_key).await?;
+ let migration = start_account_key_migration(ctx, &vek, kind).await?;
let content_key = migration.as_ref().map(|m| m.content_key.clone()).unwrap_or_else(|| vek.clone());
- let gate = WriteGate { force_full_write, overwrite_personal_blobs: migration.is_some() };
let set = match cached {
Some(set) => {
- ctx.log("[V2Push] Reusing the canonicalize result already produced for this vault.").await;
+ ctx.log("[Push] Reusing the canonicalize result already produced for this vault.").await;
set
}
None => canonicalize_vault(ctx, None).await?,
};
let CanonicalizedSet { canonicalized, manifest_records } = set;
- if manifest_records.is_empty() {
- return Err(SyncError::Other("no manifest records to push".to_string()));
- }
- ctx.log(format!("[V2Push] Canonicalize produced {} manifest(s) + {} data bucket(s).", canonicalized.manifests.len(), canonicalized.data_buckets.len())).await;
+ ctx.log(format!("[Push] Canonicalize produced {} manifest(s) + {} data bucket(s).", canonicalized.manifests.len(), canonicalized.data_buckets.len())).await;
let baselines = PushBaselines::load(ctx).await?;
let candidates = collect_candidates(&canonicalized, &manifest_records, &content_key, &baselines);
let blobs = collect_upload_blobs(&candidates)?;
- if gate.overwrite_personal_blobs {
+ if migration.is_some() {
refuse_overwrite_without_bytes(&candidates, &blobs)?;
}
+ let keys_by_manifest: HashMap<String, String> = candidates.iter().map(|c| (c.record.manifest_id.clone(), c.vek.clone())).collect();
let mut written = WrittenFingerprints::default();
- let bucket_writes = encrypt_changed_buckets(ctx, &canonicalized.data_buckets, &candidates, &baselines, gate, &mut written).await?;
+ let bucket_writes = encrypt_changed_buckets(ctx, &canonicalized.data_buckets, &keys_by_manifest, &baselines, kind, &mut written).await?;
// An upgrade push signs with the signing key it creates, which the server stores in the same write.
let signing_key = match &migration {
Some(migration) => Some(migration.signing_private_key.clone()),
None => keys::signing_private_key(ctx).await?,
};
- let manifest_writes = encrypt_changed_manifests(ctx, &candidates, &baselines, gate, signing_key.as_deref(), &mut written).await?;
+ let manifest_writes = encrypt_changed_manifests(ctx, &candidates, &baselines, kind, signing_key.as_deref(), &mut written).await?;
if manifest_writes.is_empty() && bucket_writes.is_empty() {
- ctx.log("[V2Push] No content changes detected (every manifest and data bucket matches the server baselines); skipping upload.").await;
+ ctx.log("[Push] No content changes detected (every manifest and data bucket matches the server baselines); skipping upload.").await;
return Ok((PushStatus::Ok, None));
}
- let mut uploaded = upload_missing_blobs(ctx, &blobs, &baselines, gate).await?;
+ let overwrite_personal_blobs = migration.is_some();
+ let mut uploaded = upload_missing_blobs(ctx, &blobs, &baselines, overwrite_personal_blobs).await?;
let routed_manifests: Vec<_> = canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect();
let email_routing = build_email_routing(&routed_manifests, &ctx.request.private_email_domains, &baselines.manifest_revisions);
let payload = VaultWriteRequest { username: ctx.request.username.clone(), manifests: manifest_writes, buckets: bucket_writes, email_routing: Some(email_routing), migration: migration.as_ref().map(|m| VaultWriteMigration { account_keys: Some(m.account_keys.clone()) }) };
- let response = write_vault(ctx, &payload, &blobs, gate, &mut uploaded).await?;
+ let response = write_vault(ctx, &payload, &blobs, overwrite_personal_blobs, &mut uploaded).await?;
if response.status != VaultWriteStatus::Ok {
// All-or-nothing: a single stale manifest or bucket rejected the whole write; the caller pulls, merges and retries.
@@ -501,19 +471,19 @@ async fn push_internal(ctx: &mut Ctx, cached: Option<CanonicalizedSet>, create_v
commit_blob_baselines(ctx, &blobs, &uploaded).await?;
if let Some(migration) = &migration {
complete_account_key_migration(ctx, migration).await?;
- ctx.log("[V2Push] Account-key migration complete: hierarchy created server-side, blob chain cached locally.").await;
+ ctx.log("[Push] Account-key migration complete: hierarchy created server-side, blob chain cached locally.").await;
}
Ok((PushStatus::Ok, migration.map(|m| m.content_key.clone())))
}
-/// Generate the account key hierarchy for a migration push, when one is due. The session key of a legacy account is
-/// its unlock key, which the hierarchy derives its KEK from.
-async fn start_account_key_migration(ctx: &Ctx, unlock_key: &str, create_vault_key: bool) -> SyncResult<Option<LegacyAccountKeyMigration>> {
- if !create_vault_key {
+/// Generate the account key hierarchy for a migration push. The session key of a legacy account is its unlock key,
+/// which the hierarchy derives its KEK from.
+async fn start_account_key_migration(ctx: &Ctx, unlock_key: &str, kind: WriteKind) -> SyncResult<Option<LegacyAccountKeyMigration>> {
+ if kind != WriteKind::AccountKeyMigration {
return Ok(None);
}
let hierarchy = crypto::create_account_key_hierarchy(unlock_key)?;
- ctx.log("[V2Push] Account-key migration: generated new VEK, AK, account keypair and signing keypair; vault content and all blobs will be re-encrypted and re-uploaded.").await;
+ ctx.log("[Push] Account-key migration: generated new VEK, AK, account keypair and signing keypair; vault content and all blobs will be re-encrypted and re-uploaded.").await;
Ok(Some(LegacyAccountKeyMigration {
content_key: hierarchy.vault_encryption_key.clone(),
account_keys: hierarchy.account_keys.clone(),
@@ -550,37 +520,29 @@ fn collect_upload_blobs(candidates: &[Candidate]) -> SyncResult<UploadBlobs> {
Ok(blobs)
}
-/// An overwriting push uploads every personal blob again. A blob whose bytes are not loaded on this device would keep
-/// its old server copy (on a migration: under the old key, never being able to open again) while its manifest moves on: refuse this on client instead.
-/// TODO: when implementing full lazy-load of blobs on client, review this logic as well.
+/// A migration push uploads every personal blob again under the new key. A blob whose bytes are not loaded on this
+/// device would keep its old server copy under the old key, never to open again, while its manifest moves on: refuse.
fn refuse_overwrite_without_bytes(candidates: &[Candidate], blobs: &UploadBlobs) -> SyncResult<()> {
- let not_loaded: Vec<String> = candidates
- .iter()
- .filter(|candidate| candidate.record.is_personal)
- .flat_map(|candidate| candidate.manifest.referenced_blobs())
- .map(|(hash, _)| hash)
- .filter(|hash| !blobs.entries.contains_key(hash))
- .collect();
- if not_loaded.is_empty() {
+ let not_loaded = candidates.iter().filter(|c| c.record.is_personal).flat_map(|c| c.manifest.referenced_blobs()).filter(|(hash, _)| !blobs.entries.contains_key(hash)).count();
+ if not_loaded == 0 {
return Ok(());
}
- Err(SyncError::UploadRejected(vec![format!("{} blob(s) of the personal manifest are not loaded on this device, so they cannot be uploaded again; sync again when they load", not_loaded.len())]))
+ Err(SyncError::UploadRejected(vec![format!("{} blob(s) of the personal manifest are not loaded on this device, so they cannot be uploaded again; sync again when they load", not_loaded)]))
}
/// Gate, validate, pack and encrypt each changed bucket under the key of the manifest that owns it.
-async fn encrypt_changed_buckets(ctx: &Ctx, buckets: &[DataBucket], candidates: &[Candidate<'_>], baselines: &PushBaselines, gate: WriteGate, written: &mut WrittenFingerprints) -> SyncResult<Vec<BucketWrite>> {
- let key_by_manifest: HashMap<&str, &str> = candidates.iter().map(|c| (c.record.manifest_id.as_str(), c.vek.as_str())).collect();
+async fn encrypt_changed_buckets(ctx: &Ctx, buckets: &[DataBucket], keys_by_manifest: &HashMap<String, String>, baselines: &PushBaselines, kind: WriteKind, written: &mut WrittenFingerprints) -> SyncResult<Vec<BucketWrite>> {
let mut writes = Vec::new();
for bucket in buckets {
let label = format!("Data bucket \"{}\" of manifest {}", bucket.category, bucket.manifest_id);
let fingerprint_key = state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category);
let (plaintext, fingerprint) = fingerprinted(bucket)?;
- if !gate.force_full_write && !gate.overwrite_personal_blobs && baselines.unchanged(&fingerprint_key, &fingerprint) {
- ctx.log(format!("[V2Push] {} unchanged versus server baseline, leaving it out of this write.", label)).await;
+ if !kind.writes_unchanged() && baselines.unchanged(&fingerprint_key, &fingerprint) {
+ ctx.log(format!("[Push] {} unchanged versus server baseline, leaving it out of this write.", label)).await;
continue;
}
- let Some(bucket_key) = key_by_manifest.get(bucket.manifest_id.as_str()) else {
- ctx.warn(format!("[V2Push] {} names a manifest this vault cannot write; leaving it out of this write.", label)).await;
+ let Some(bucket_key) = keys_by_manifest.get(&bucket.manifest_id) else {
+ ctx.warn(format!("[Push] {} names a manifest this vault cannot write; leaving it out of this write.", label)).await;
continue;
};
let validation = vault_codec::validate_data_bucket(bucket);
@@ -595,15 +557,14 @@ async fn encrypt_changed_buckets(ctx: &Ctx, buckets: &[DataBucket], candidates:
}
/// Gate, validate, pack and encrypt every changed candidate into the write batch, each with its own key.
-async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], baselines: &PushBaselines, gate: WriteGate, signing_key: Option<&str>, written: &mut WrittenFingerprints) -> SyncResult<Vec<ManifestWrite>> {
+async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], baselines: &PushBaselines, kind: WriteKind, signing_key: Option<&str>, written: &mut WrittenFingerprints) -> SyncResult<Vec<ManifestWrite>> {
let mut writes = Vec::new();
for candidate in candidates {
let label = candidate.label();
let fingerprint_key = state::fingerprint_manifest_key(&candidate.record.manifest_id);
let (plaintext, fingerprint) = fingerprinted(candidate.manifest)?;
- let rekeyed = gate.overwrite_personal_blobs && candidate.record.is_personal;
- if !gate.force_full_write && !rekeyed && baselines.unchanged(&fingerprint_key, &fingerprint) {
- ctx.log(format!("[V2Push] {} unchanged versus server baseline, leaving it out of this write.", label)).await;
+ if !kind.writes_unchanged() && baselines.unchanged(&fingerprint_key, &fingerprint) {
+ ctx.log(format!("[Push] {} unchanged versus server baseline, leaving it out of this write.", label)).await;
continue;
}
let validation = vault_codec::validate_manifest(candidate.manifest);
@@ -611,7 +572,7 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], base
if candidate.record.is_personal {
return Err(SyncError::UploadRejected(vec![format!("Manifest validation failed: {}. {}", validation.failed_rules.join(", "), validation.message).trim().to_string()]));
}
- ctx.warn(format!("[V2Push] {} failed validation ({}), dropping it from this write.", label, validation.failed_rules.join(", "))).await;
+ ctx.warn(format!("[Push] {} failed validation ({}), dropping it from this write.", label, validation.failed_rules.join(", "))).await;
continue;
}
let encrypted = encrypt_payload(ctx, &label, &plaintext, &candidate.vek, &crypto::aad::manifest(&candidate.record.manifest_id)).await?;
@@ -620,14 +581,14 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], base
let may_publish = candidate.record.is_personal || candidate.record.can_administer;
let manifest_key = if may_publish { db::active_public_key_for_manifest(&ctx.host, &candidate.record.manifest_id).await? } else { None };
if may_publish && manifest_key.is_none() && !candidate.record.is_personal {
- ctx.warn(format!("[V2Push] {} is missing its email keypair; its aliases stay personal until sharing is re-enabled.", label)).await;
+ ctx.warn(format!("[Push] {} is missing its email keypair; its aliases stay personal until sharing is re-enabled.", label)).await;
}
// The server only accepts a new delivery key signed by the caller; a key that is already published needs none.
let manifest_key_signature = match (&manifest_key, signing_key) {
(Some(public_key), Some(signing_key)) => Some(crypto::signing::sign(signing_key, &crypto::signing::delivery_key_message(&candidate.record.manifest_id, public_key, candidate.current_revision))?),
(Some(_), None) => {
- ctx.warn(format!("[V2Push] {} publishes its email key without a signing key; the server refuses it if the key is new.", label)).await;
+ ctx.warn(format!("[Push] {} publishes its email key without a signing key; the server refuses it if the key is new.", label)).await;
None
}
_ => None,
@@ -639,7 +600,7 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], base
manifest_blob: encrypted.ciphertext,
manifest_ciphertext_hash: encrypted.hash,
current_revision: candidate.current_revision,
- credentials_count: candidate.manifest.tables.get("Items").map(Vec::len).unwrap_or(0),
+ credentials_count: candidate.manifest.tables.get(ITEMS_TABLE).map(Vec::len).unwrap_or(0),
blob_references: blob_refs,
delivery_public_key_algorithm: manifest_key.as_ref().map(|_| ALGORITHM_RSA_OAEP_SHA256.to_string()),
delivery_public_key: manifest_key,
@@ -652,36 +613,39 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], base
/// Encrypt and upload only the blobs the server does not already have. On a migration push every personal
/// blob is re-encrypted under the new key and overwritten. Returns hash to ciphertext for the local cache.
-async fn upload_missing_blobs(ctx: &Ctx, blobs: &UploadBlobs, baselines: &PushBaselines, gate: WriteGate) -> SyncResult<HashMap<String, EncryptedBlob>> {
+async fn upload_missing_blobs(ctx: &Ctx, blobs: &UploadBlobs, baselines: &PushBaselines, overwrite_personal_blobs: bool) -> SyncResult<HashMap<String, EncryptedBlob>> {
let personal_hashes = blobs.hashes(true);
let shared_hashes = blobs.hashes(false);
let unknown_to_server = |hashes: &[String]| -> Vec<String> { hashes.iter().filter(|h| !baselines.known_server_hashes.contains(*h)).cloned().collect() };
- let (personal_to_upload, shared_to_upload) = if gate.overwrite_personal_blobs {
+ let (personal_to_upload, shared_to_upload) = if overwrite_personal_blobs {
(personal_hashes.clone(), missing_on_server(ctx, blobs, &unknown_to_server(&shared_hashes)).await?)
} else {
let missing: HashSet<String> = missing_on_server(ctx, blobs, &unknown_to_server(&blobs.order)).await?.into_iter().collect();
(personal_hashes.iter().filter(|h| missing.contains(*h)).cloned().collect(), shared_hashes.iter().filter(|h| missing.contains(*h)).cloned().collect())
};
- ctx.log(format!("[V2Push] Blob diff: {} blobs, uploading {} personal + {} shared{}.", blobs.order.len(), personal_to_upload.len(), shared_to_upload.len(), if gate.overwrite_personal_blobs { " (every personal blob overwritten)" } else { "" })).await;
+ ctx.log(format!("[Push] Blob diff: {} blobs, uploading {} personal + {} shared{}.", blobs.order.len(), personal_to_upload.len(), shared_to_upload.len(), if overwrite_personal_blobs { " (every personal blob overwritten)" } else { "" })).await;
- let mut uploaded = upload_blobs(ctx, blobs, &personal_to_upload, gate.overwrite_personal_blobs).await?;
+ let mut uploaded = upload_blobs(ctx, blobs, &personal_to_upload, overwrite_personal_blobs).await?;
uploaded.extend(upload_blobs(ctx, blobs, &shared_to_upload, false).await?);
Ok(uploaded)
}
-/// Ask the server which of the given hashes it lacks, one request per manifest that owns them.
+/// Ask the server which of the given hashes it lacks, per manifest that owns them and within the server's per-request cap.
async fn missing_on_server(ctx: &Ctx, blobs: &UploadBlobs, hashes: &[String]) -> SyncResult<Vec<String>> {
let mut missing = Vec::new();
for (manifest_id, hashes) in blobs.by_manifest(hashes) {
- missing.extend(http::post::<_, MissingBlobsResponse>(&ctx.host, BLOBS_MISSING_ENDPOINT, &BlobHashesRequest { manifest_id, hashes }, false).await?.missing);
+ for chunk in hashes.chunks(http::BLOB_HASH_REQUEST_MAX_COUNT) {
+ let request = BlobHashesRequest { manifest_id: manifest_id.clone(), hashes: chunk.to_vec() };
+ missing.extend(http::post::<_, MissingBlobsResponse>(&ctx.host, BLOBS_MISSING_ENDPOINT, &request, false).await?.missing);
+ }
}
Ok(missing)
}
/// `POST v2/Vault`. When the server reports blobs it lacks (stale local knowledge of its blob set), upload
/// them and retry the identical write once; blobs this client cannot supply fail the push.
-async fn write_vault(ctx: &Ctx, payload: &VaultWriteRequest, blobs: &UploadBlobs, gate: WriteGate, uploaded: &mut HashMap<String, EncryptedBlob>) -> SyncResult<VaultWriteResponse> {
+async fn write_vault(ctx: &Ctx, payload: &VaultWriteRequest, blobs: &UploadBlobs, overwrite_personal_blobs: bool, uploaded: &mut HashMap<String, EncryptedBlob>) -> SyncResult<VaultWriteResponse> {
let response: VaultWriteResponse = http::post(&ctx.host, http::VAULT_ENDPOINT, payload, true).await?;
if response.missing_blob_hashes.is_empty() {
return Ok(response);
@@ -690,9 +654,9 @@ async fn write_vault(ctx: &Ctx, payload: &VaultWriteRequest, blobs: &UploadBlobs
if !unsatisfiable.is_empty() {
return Err(SyncError::MissingBlobs(unsatisfiable));
}
- ctx.warn(format!("[V2Sync] Server reported {} missing blob(s); uploading and retrying once.", response.missing_blob_hashes.len())).await;
+ ctx.warn(format!("[Push] Server reported {} missing blob(s); uploading and retrying once.", response.missing_blob_hashes.len())).await;
let (missing_personal, missing_shared): (Vec<String>, Vec<String>) = response.missing_blob_hashes.iter().cloned().partition(|h| blobs.entries[h].from_personal);
- uploaded.extend(upload_blobs(ctx, blobs, &missing_personal, gate.overwrite_personal_blobs).await?);
+ uploaded.extend(upload_blobs(ctx, blobs, &missing_personal, overwrite_personal_blobs).await?);
uploaded.extend(upload_blobs(ctx, blobs, &missing_shared, false).await?);
let response: VaultWriteResponse = http::post(&ctx.host, http::VAULT_ENDPOINT, payload, true).await?;
if !response.missing_blob_hashes.is_empty() {
@@ -733,7 +697,7 @@ async fn commit_blob_baselines(ctx: &Ctx, blobs: &UploadBlobs, uploaded: &HashMa
state::set(&ctx.host, state::VAULT_BLOB_CIPHER_CACHE, &new_cache).await
}
-/// Store the hierarchy a migration push just committed: cache the encrypted chain and stage the private key.
+/// Store the hierarchy a migration push just committed: cache the encrypted chain and stage the private keys.
async fn complete_account_key_migration(ctx: &mut Ctx, migration: &LegacyAccountKeyMigration) -> SyncResult<()> {
let blobs = &migration.account_keys;
state::set(&ctx.host, state::ENCRYPTED_ACCOUNT_KEY, &blobs.encrypted_account_key).await?;
@@ -762,52 +726,9 @@ async fn upload_blobs(ctx: &Ctx, blobs: &UploadBlobs, hashes: &[String], overwri
}
for batch in http::batch_by_transfer_cost(dtos, |dto| dto.encrypted_data_base64.len()) {
let chars: usize = batch.iter().map(|dto| dto.encrypted_data_base64.len()).sum();
- ctx.log(format!("[V2Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(chars))).await;
+ ctx.log(format!("[Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(chars))).await;
http::post_no_content(&ctx.host, BLOBS_ENDPOINT, &BlobUploadRequest { manifest_id: manifest_id.clone(), blobs: batch, overwrite }).await?;
}
}
Ok(encrypted_blobs)
}
-
-/*
- * The bucket-only write.
- */
-
-/*
- * Single-data-bucket upload through the unified write. A stale revision reports `Outdated` returns early.
- */
-async fn push_data_bucket_only(ctx: &Ctx, bucket: &DataBucket, vek: &str) -> SyncResult<(PushStatus, i64)> {
- http::with_outdated_server_guard(push_data_bucket_only_internal(ctx, bucket, vek).await)
-}
-
-async fn push_data_bucket_only_internal(ctx: &Ctx, bucket: &DataBucket, vek: &str) -> SyncResult<(PushStatus, i64)> {
- let label = format!("Bucket \"{}\" of manifest {} (bucket-only)", bucket.category, bucket.manifest_id);
- let fingerprint_key = state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category);
- let baselines = PushBaselines::load(ctx).await?;
- let (plaintext, fingerprint) = fingerprinted(bucket)?;
- if baselines.unchanged(&fingerprint_key, &fingerprint) {
- ctx.log(format!("[V2Push] {} unchanged versus server baseline, skipping upload.", label)).await;
- return Ok((PushStatus::Ok, baselines.bucket_revision(bucket)));
- }
- let encrypted = encrypt_payload(ctx, &label, &plaintext, vek, &crypto::aad::bucket(&bucket.manifest_id, &bucket.category)).await?;
-
- let current_revision = baselines.bucket_revision(bucket);
- let payload = VaultWriteRequest {
- username: ctx.request.username.clone(),
- manifests: Vec::new(),
- buckets: vec![BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: encrypted.ciphertext, ciphertext_hash: encrypted.hash, current_revision }],
- email_routing: None,
- migration: None,
- };
- let response: VaultWriteResponse = http::post(&ctx.host, http::VAULT_ENDPOINT, &payload, true).await?;
- let reported = response.bucket_revisions.iter().find(|b| ids_equal(&b.manifest_id, &bucket.manifest_id) && b.category == bucket.category).map(|b| b.revision);
- if response.status != VaultWriteStatus::Ok {
- ctx.warn(format!("[V2Push] {} outdated (server at revision {}, we assumed {}); pulling and merging before the next attempt.", label, reported.unwrap_or(current_revision), current_revision)).await;
- return Ok((PushStatus::Outdated, reported.unwrap_or(current_revision)));
- }
-
- let new_revision = reported.unwrap_or(current_revision + 1);
- let advanced = BucketRevision { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), revision: new_revision };
- commit_push_baselines(ctx, baselines, &[], &[advanced], HashMap::from([(fingerprint_key, fingerprint)])).await?;
- Ok((PushStatus::Ok, new_revision))
-}
diff --git a/core/rust/src/vault_sync/session.rs b/core/rust/src/vault_sync/session.rs
index 2cc412027..c767c3915 100644
--- a/core/rust/src/vault_sync/session.rs
+++ b/core/rust/src/vault_sync/session.rs
@@ -40,7 +40,7 @@ impl Host {
/// Like [`Host::call`], also returning the raw bytes the host attached to its response, if any.
pub async fn call_with_bytes<R: DeserializeOwned>(&self, command: Command) -> SyncResult<(R, Option<Vec<u8>>)> {
- let kind = command.kind();
+ let kind = command.name();
let (response, bytes) = CommandFuture { slot: self.slot.clone(), command: Some(command), sent: false }.await;
if let Some(error) = response.get("error").and_then(Value::as_str) {
return Err(SyncError::Host { command: kind, message: error.to_string() });
diff --git a/core/rust/src/vault_sync/sharing.rs b/core/rust/src/vault_sync/sharing.rs
index 05099a2f5..81951a59c 100644
--- a/core/rust/src/vault_sync/sharing.rs
+++ b/core/rust/src/vault_sync/sharing.rs
@@ -1,4 +1,4 @@
-//! The sharing operations that take vault keys: creating a group's shared manifest and inviting a member to one.
+//! The sharing operations that take vault keys: creating a group's shared manifest, updating one and inviting a member to one.
use std::collections::HashMap;
@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
use super::errors::{SyncError, SyncResult};
use super::state::{self, Ctx};
use super::types::{Db, FailureFields, SharedManifestDto, SharingOperationResult, SharingParams, ALGORITHM_RSA_OAEP_SHA256};
-use super::{db, engine, http, keys, pull};
+use super::{db, engine, http, keys, migration, pull};
use crate::crypto;
use crate::vault_codec;
use crate::vault_model::{id_key, ids_equal};
@@ -208,7 +208,7 @@ async fn create_shared_manifest(ctx: &mut Ctx) -> Outcome {
if state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await?.is_none() {
return Ok(Err(Refusal::VaultUpgradeRequired));
}
- if !ctx.has_local_vault().await? || engine::vault_requires_manifest_migration(ctx).await? {
+ if !ctx.has_local_vault().await? || migration::vault_requires_manifest_migration(ctx).await? {
return Ok(Err(Refusal::VaultUpgradeRequired));
}
@@ -243,7 +243,7 @@ async fn create_shared_manifest(ctx: &mut Ctx) -> Outcome {
can_administer: true,
},
);
- keys::set_shared_manifest_records(&ctx.host, &records, &key).await?;
+ keys::set_shared_manifest_records(ctx, &records, &key).await?;
let mut revisions: HashMap<String, i64> = state::get(&ctx.host, state::SERVER_MANIFEST_REVISIONS).await?.unwrap_or_default();
revisions.insert(manifest_id.clone(), response.revision_number);
@@ -256,17 +256,13 @@ async fn create_shared_manifest(ctx: &mut Ctx) -> Outcome {
db::set_active_key_for_manifest(&ctx.host, &manifest_id, &delivery_keys.public_key, &delivery_keys.private_key).await?;
// The name and keypair ride out on the host's next sync, which the dirty flag asks for.
- let bytes = db::export(&ctx.host, Db::Local).await?;
- let stored = ctx.store_vault(&state::encrypt_vault_blob(&bytes, &key)?, true, None, None).await?;
- ctx.is_dirty = true;
- ctx.mutation_sequence = stored.mutation_sequence;
- ctx.vault_changed = true;
+ ctx.persist_local_vault(true).await?;
ctx.log(format!("[Sharing] Created shared manifest {} for group {}.", manifest_id, group.group_id)).await;
Ok(Ok(manifest_id))
}
-/// The details an update can change (e.g. the name of a manifest / shared vault).
+/// The details an update can change (e.g. the name of a shared manifest).
async fn update_shared_manifest(ctx: &mut Ctx) -> Outcome {
let key = ctx.encryption_key()?;
let target = params(ctx)?;
@@ -274,25 +270,23 @@ async fn update_shared_manifest(ctx: &mut Ctx) -> Outcome {
let name = target.name.as_deref().map(str::trim).filter(|name| !name.is_empty()).ok_or_else(|| SyncError::Other("The update changes nothing".to_string()))?.to_string();
check_name_length(&name)?;
- let mut records = keys::shared_manifest_records(ctx).await?;
- let record = records.values().find(|record| ids_equal(&record.manifest_id, &manifest_id)).cloned().ok_or_else(|| SyncError::Other("This account holds no key for the shared manifest".to_string()))?;
+ let record = held_record(ctx, &manifest_id).await?.ok_or_else(|| SyncError::Other("This account holds no key for the shared manifest".to_string()))?;
let manifest_vek = keys::open_shared_manifest_vek(ctx, &record).await?.ok_or_else(|| SyncError::Other("The key of the shared manifest did not open".to_string()))?;
// The server decides who may change a shared manifest; it only ever sees the name encrypted.
let encrypted_name = pull::encrypt_manifest_name(&name, &record.manifest_id, &manifest_vek)?;
http::post_no_content(&ctx.host, &format!("Groups/{}/manifests/{}", target.group_id, record.manifest_id), &UpdateSharedManifestRequest { encrypted_name: Some(&encrypted_name) }).await?;
- if let Some(held) = records.values_mut().find(|held| ids_equal(&held.manifest_id, &manifest_id)) {
+ let mut records = keys::shared_manifest_records(ctx).await?;
+ if let Some(held) = records.get_mut(&record.manifest_id) {
held.encrypted_name = Some(encrypted_name);
}
- keys::set_shared_manifest_records(&ctx.host, &records, &key).await?;
+ keys::set_shared_manifest_records(ctx, &records, &key).await?;
// Show the new name right away; every other member fetches it on their next sync.
if ctx.has_local_vault().await? {
db::apply_manifest_names(&ctx.host, Db::Local, &HashMap::from([(id_key(&record.manifest_id), name)])).await?;
- let bytes = db::export(&ctx.host, Db::Local).await?;
- ctx.store_vault(&state::encrypt_vault_blob(&bytes, &key)?, false, None, None).await?;
- ctx.vault_changed = true;
+ ctx.persist_local_vault(false).await?;
}
ctx.log(format!("[Sharing] Updated shared manifest {}.", record.manifest_id)).await;
diff --git a/core/rust/src/vault_sync/state.rs b/core/rust/src/vault_sync/state.rs
index fd11af98c..da4a57bd8 100644
--- a/core/rust/src/vault_sync/state.rs
+++ b/core/rust/src/vault_sync/state.rs
@@ -1,6 +1,7 @@
-//! Engine state: the run context.
+//! Engine state: the run context and the values the engine persists through the host.
use std::collections::HashMap;
+use std::sync::Mutex;
use serde::de::DeserializeOwned;
use serde::Serialize;
@@ -30,6 +31,8 @@ pub(crate) struct Ctx {
pub vault_key_probed: bool,
pub served_manifest_names: HashMap<String, String>,
has_local_vault: bool,
+ /// The VEK of every shared manifest this account holds a grant on, unwrapped once per run.
+ shared_veks: Mutex<Option<HashMap<String, String>>>,
}
/// What a fresh staging database tells about the current client schema.
@@ -55,6 +58,7 @@ impl Ctx {
vault_key_probed: false,
served_manifest_names: HashMap::new(),
has_local_vault: false,
+ shared_veks: Mutex::new(None),
host,
request,
}
@@ -84,6 +88,20 @@ impl Ctx {
store_vault_with_key(&self.host, encrypted_blob, mark_dirty, expected_mutation_seq, revision, self.new_encryption_key.clone()).await
}
+ /// Write the local database as the at-rest vault under the session key. `dirty` records it as a pending change
+ /// the next push carries; a clean store only refreshes what the host shows.
+ pub async fn persist_local_vault(&mut self, dirty: bool) -> SyncResult<()> {
+ let key = self.encryption_key()?;
+ let bytes = db::export(&self.host, Db::Local).await?;
+ let stored = self.store_vault(&encrypt_vault_blob(&bytes, &key)?, dirty, None, None).await?;
+ if dirty {
+ self.is_dirty = true;
+ self.mutation_sequence = stored.mutation_sequence;
+ }
+ self.vault_changed = true;
+ Ok(())
+ }
+
/// The current schema.
pub async fn schema(&mut self) -> SyncResult<SchemaInfo> {
if let Some(schema) = &self.schema {
@@ -95,6 +113,16 @@ impl Ctx {
Ok(schema)
}
+ /// The shared-manifest keys unwrapped earlier in this run, if any.
+ pub fn cached_shared_veks(&self) -> Option<HashMap<String, String>> {
+ self.shared_veks.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).clone()
+ }
+
+ /// Remember the unwrapped shared-manifest keys; `None` forgets them (the records changed).
+ pub fn cache_shared_veks(&self, veks: Option<HashMap<String, String>>) {
+ *self.shared_veks.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = veks;
+ }
+
pub async fn log(&self, message: impl Into<String>) {
self.host.log(LogLevel::Log, message).await;
}
@@ -145,7 +173,7 @@ pub(crate) async fn remove(host: &Host, key: &str) -> SyncResult<()> {
Ok(())
}
-/// The record key of one data bucket's revision and fingerprint.
+/// The record key of one data bucket's revision.
pub(crate) fn bucket_revision_key(manifest_id: &str, category: &str) -> String {
format!("{}:{}", manifest_id, category)
}
diff --git a/core/rust/src/vault_sync/tests/blobs.rs b/core/rust/src/vault_sync/tests/blobs.rs
new file mode 100644
index 000000000..48c43dcf7
--- /dev/null
+++ b/core/rust/src/vault_sync/tests/blobs.rs
@@ -0,0 +1,178 @@
+//! Blobs (logos, attachments) are stored per manifest, each encrypted under a key of its own. Their bytes are
+//! optional locally: a blob that is not loaded (not served, or not decrypting with the key) neither fails a pull or
+//! a merge nor loses its reference on the next push.
+
+use serde_json::{json, Value};
+
+use super::fake_server::FakeServer;
+use super::test_host::{open_schema_db, query, TestHost};
+use super::{insert_attachment, insert_delivery_key, insert_item, insert_logo, rename_item, synced, Synced, ITEM_A, ITEM_B, PERSONAL_MANIFEST_ID};
+use crate::crypto;
+use crate::vault_codec;
+use crate::vault_sync::blob_keys;
+
+/// A server vault holding one item with a logo and an attachment.
+fn vault_with_blobs(db: &rusqlite::Connection) {
+ insert_item(db, ITEM_A, "Server item");
+ insert_delivery_key(db);
+ insert_logo(db, ITEM_A, &[1, 2, 3, 4]);
+ insert_attachment(db, ITEM_A, Some(&[9u8; 64]));
+}
+
+/// The blob hashes the last vault write referenced, sorted.
+fn written_hashes(host: &TestHost) -> Vec<String> {
+ let mut hashes: Vec<String> = host.last_vault_write()["manifests"][0]["blobReferences"].as_array().unwrap().iter().map(|r| r["hash"].as_str().unwrap().to_string()).collect();
+ hashes.sort();
+ hashes
+}
+
+/// How a device can end up with a blob it holds a reference to but no bytes for.
+#[derive(Clone, Copy, Debug)]
+enum NotLoaded {
+ /// The server never served the blob.
+ Withheld,
+ /// The server served it encrypted under a key this device does not hold.
+ Undecryptable,
+}
+
+/// A device that pulled `vault_with_blobs` while none of its blobs loaded.
+fn synced_without_blobs(how: NotLoaded) -> Synced {
+ let vek = crypto::generate_key_base64();
+ let salt = vault_codec::generate_manifest_salt();
+ let db = open_schema_db();
+ vault_with_blobs(&db);
+ let server = FakeServer::new();
+ match how {
+ NotLoaded::Withheld => {
+ server.borrow_mut().publish(&db, &vek, &salt, 7);
+ server.borrow_mut().faults.withhold_blobs = true;
+ }
+ NotLoaded::Undecryptable => server.borrow_mut().publish_with_blob_key(&db, &vek, &salt, 7, &crypto::generate_key_base64()),
+ }
+ let mut host = TestHost::logged_in(&vek, &server);
+ let pulled = host.sync();
+ assert_eq!(pulled["success"], true, "a blob that does not load must not fail the pull ({:?}): {}", how, pulled);
+ assert_eq!(host.item_names(), vec!["Server item"]);
+ let not_loaded = query(&host.local, "SELECT Id FROM Attachments WHERE IsDeleted = 0 AND Blob IS NULL AND BlobHash IS NOT NULL", &[]).unwrap();
+ assert_eq!(not_loaded.len(), 1, "the attachment row is there, not loaded, and knows its blob");
+ Synced { server, host, db, vek, salt }
+}
+
+#[test]
+fn a_pulled_blob_opens_through_its_own_key() {
+ let s = synced(vault_with_blobs);
+ let loaded: Vec<Value> = query(&s.host.local, "SELECT hex(Blob) AS Bytes FROM Attachments WHERE IsDeleted = 0", &[]).unwrap().into_iter().map(|row| row["Bytes"].clone()).collect();
+ assert_eq!(loaded, vec![json!("09".repeat(64))]);
+}
+
+#[test]
+fn pushed_blobs_are_encrypted_under_their_own_key_and_stored_with_their_manifest() {
+ let mut s = synced(|db| insert_item(db, ITEM_A, "Server item"));
+
+ s.host.edit(|db| insert_logo(db, ITEM_A, &[1, 2, 3, 4]));
+ let result = s.host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ let missing_checks = s.host.requests_to("Vault/blobs/missing");
+ assert_eq!(missing_checks.len(), 1);
+ assert_eq!(missing_checks[0].body.as_ref().unwrap()["manifestId"], PERSONAL_MANIFEST_ID);
+ let uploads = s.host.requests_to("Vault/blobs");
+ assert_eq!(uploads.len(), 1);
+ let upload = uploads[0].body.as_ref().unwrap();
+ assert_eq!(upload["manifestId"], PERSONAL_MANIFEST_ID);
+ assert_eq!(upload["blobs"].as_array().unwrap().len(), 1);
+ let hash = upload["blobs"][0]["hash"].as_str().unwrap();
+ assert_eq!(s.server.borrow().stored_blob_hashes(), vec![hash.to_string()]);
+
+ // The uploaded blob opens through its own key at its own address (the key scheme itself is `blob_keys`' to test).
+ let uploaded: blob_keys::EncryptedBlob = serde_json::from_value(upload["blobs"][0].clone()).unwrap();
+ assert_eq!(blob_keys::decrypt_blob(&uploaded, &s.vek, PERSONAL_MANIFEST_ID, hash).unwrap(), vec![1u8, 2, 3, 4]);
+ s.server.borrow().assert_converged(&s.host);
+}
+
+#[test]
+fn the_missing_blob_check_stays_within_the_servers_hash_cap() {
+ let mut s = synced(|db| insert_item(db, ITEM_A, "Server item"));
+
+ s.host.edit(|db| {
+ for i in 0..1001u32 {
+ db.execute("INSERT INTO Attachments (ManifestId, Id, ItemId, Filename, Blob, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'f.bin', ?, ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, format!("cccccccc-0000-4000-8000-{:012}", i), ITEM_A, i.to_le_bytes().to_vec(), super::now(), super::now()]).unwrap();
+ }
+ });
+ let result = s.host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ let checks: Vec<usize> = s.host.requests_to("Vault/blobs/missing").iter().map(|r| r.body.as_ref().unwrap()["hashes"].as_array().unwrap().len()).collect();
+ assert_eq!(checks, vec![1000, 1]);
+ assert_eq!(s.server.borrow().stored_blob_hashes().len(), 1001);
+}
+
+#[test]
+fn a_blob_that_did_not_load_keeps_its_reference_on_the_next_push() {
+ for how in [NotLoaded::Withheld, NotLoaded::Undecryptable] {
+ let mut s = synced_without_blobs(how);
+ let server_hashes = s.server.borrow().stored_blob_hashes();
+
+ s.host.edit(|db| rename_item(db, "Renamed"));
+ let pushed = s.host.sync();
+
+ assert_eq!(pushed["success"], true, "{:?}: {}", how, pushed);
+ assert_eq!(written_hashes(&s.host), server_hashes, "{:?}: a push may not drop the reference to a blob that is not loaded", how);
+ assert!(s.host.requests_to("Vault/blobs").is_empty(), "{:?}: there are no bytes to upload for a blob this device never loaded", how);
+ s.server.borrow().assert_converged(&s.host);
+ }
+}
+
+#[test]
+fn a_merge_keeps_the_reference_to_a_blob_that_is_not_loaded() {
+ let mut s = synced_without_blobs(NotLoaded::Withheld);
+ let server_hashes = s.server.borrow().stored_blob_hashes();
+
+ // A local edit, while the server moved on to revision 8 with another item added.
+ s.host.edit(|db| rename_item(db, "Renamed locally"));
+ insert_item(&s.db, ITEM_B, "Added elsewhere");
+ s.publish(8);
+ let merged = s.host.sync();
+
+ assert_eq!(merged["success"], true, "{}", merged);
+ assert_eq!(s.host.item_names(), vec!["Added elsewhere", "Renamed locally"], "the merge went through, it did not fall back to the server vault");
+ assert_eq!(written_hashes(&s.host), server_hashes);
+ s.server.borrow().assert_converged(&s.host);
+}
+
+#[test]
+fn deleting_an_attachment_that_is_not_loaded_releases_its_reference() {
+ let mut s = synced_without_blobs(NotLoaded::Withheld);
+
+ s.host.edit(|db| db.execute("UPDATE Attachments SET IsDeleted = 1, Blob = NULL, UpdatedAt = '2099-01-01 00:00:00.000'", []).map(drop).unwrap());
+ let pushed = s.host.sync();
+
+ assert_eq!(pushed["success"], true, "{}", pushed);
+ assert_eq!(written_hashes(&s.host).len(), 1, "only the logo's reference is left");
+}
+
+#[test]
+fn a_save_that_changed_nothing_is_not_written_again() {
+ let mut s = synced_without_blobs(NotLoaded::Withheld);
+
+ // The local hash column may not read as a content change.
+ s.host.edit(|_| {});
+ let result = s.host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert!(s.host.vault_writes().is_empty(), "the manifest did not change, so it is not written again");
+}
+
+#[test]
+fn an_attachment_row_that_references_no_blob_does_not_fail_the_pull() {
+ // A row some client pushed without a blob (an empty file, a writer bug) is one attachment that cannot be opened,
+ // not a vault that cannot be pulled any more.
+ let s = synced(|db| {
+ vault_with_blobs(db);
+ db.execute("UPDATE Attachments SET Blob = NULL", []).unwrap();
+ });
+
+ let rows = query(&s.host.local, "SELECT Filename, Blob, BlobHash FROM Attachments WHERE IsDeleted = 0", &[]).unwrap();
+ assert_eq!(rows.len(), 1);
+ assert_eq!((&rows[0]["Blob"], &rows[0]["BlobHash"]), (&Value::Null, &Value::Null));
+}
diff --git a/core/rust/src/vault_sync/tests/fake_server.rs b/core/rust/src/vault_sync/tests/fake_server.rs
new file mode 100644
index 000000000..c7f3f4df2
--- /dev/null
+++ b/core/rust/src/vault_sync/tests/fake_server.rs
@@ -0,0 +1,418 @@
+//! An in-process model of the v2 vault API, so a test describes server state instead of scripting HTTP bodies.
+//!
+//! The rules are the controller's: `POST Vault` is all-or-nothing on revisions (a write names the revision it
+//! builds on, the server accepts it only when that is the current one and advances by one), blobs are stored per
+//! manifest and a write referencing a blob the server lacks is answered with `missingBlobHashes`, the account
+//! key hierarchy is created at most once, and a blob hash request carries at most 1000 hashes.
+
+use std::cell::RefCell;
+use std::collections::{BTreeMap, HashMap};
+use std::rc::Rc;
+
+use rusqlite::Connection;
+use serde_json::{json, Value};
+
+use super::test_host::{read_tables, TestHost};
+use super::PERSONAL_MANIFEST_ID;
+use crate::crypto;
+use crate::vault_codec::{self, CanonicalizeInput, ManifestSpec};
+use crate::vault_sync::blob_keys;
+
+/// The server's `VaultWriteLimits.MaxHashesPerRequest`.
+const MAX_HASHES_PER_REQUEST: usize = 1000;
+
+/// A shared handle to the server, so the host's responder and the test both reach it.
+pub type Server = Rc<RefCell<FakeServer>>;
+
+/// One manifest as the server stores it.
+#[derive(Debug, Clone, Default)]
+pub struct ServerManifest {
+ pub blob: Option<String>,
+ pub ciphertext_hash: Option<String>,
+ pub revision: i64,
+ pub blob_references: Vec<Value>,
+ /// Fields of the manifest dto beyond content and revision: key type, grant, administration flag.
+ pub access: Value,
+}
+
+/// One data bucket as the server stores it.
+#[derive(Debug, Clone)]
+pub struct ServerBucket {
+ pub blob: String,
+ pub ciphertext_hash: String,
+ pub revision: i64,
+}
+
+/// LEGACY: the account's vault while it is still a sqlite blob.
+#[derive(Debug, Clone)]
+pub struct LegacyVault {
+ pub blob: String,
+ pub revision: i64,
+}
+
+/// Ways the server misbehaves on purpose.
+#[derive(Debug, Default)]
+pub struct Faults {
+ /// Every request fails at the transport level.
+ pub unreachable: bool,
+ /// `GET Status` answers with this HTTP status and an empty body.
+ pub status_http: Option<u16>,
+ /// Blob downloads return nothing, as if the blobs were never stored.
+ pub withhold_blobs: bool,
+ /// The next `n` vault writes are refused as outdated without being applied.
+ pub refuse_writes: u32,
+ /// Vault writes are not answered at all (the host sees a transport failure).
+ pub drop_writes: bool,
+}
+
+pub struct FakeServer {
+ pub version: String,
+ pub srp_salt: String,
+ pub personal_manifest_id: String,
+ pub manifests: BTreeMap<String, ServerManifest>,
+ pub buckets: BTreeMap<(String, String), ServerBucket>,
+ /// Blob store: manifest id -> hash -> the blob dto as the download endpoint serves it.
+ pub blobs: HashMap<String, HashMap<String, Value>>,
+ /// The `vaultKey` of `GET VaultKey/Password`; `Null` for an account without a key hierarchy.
+ pub vault_key: Value,
+ pub legacy: Option<LegacyVault>,
+ pub pending_actions: Vec<Value>,
+ pub capabilities: Value,
+ pub faults: Faults,
+}
+
+impl FakeServer {
+ pub fn new() -> Server {
+ Rc::new(RefCell::new(Self {
+ version: "0.31.0".to_string(),
+ srp_salt: "salt".to_string(),
+ personal_manifest_id: PERSONAL_MANIFEST_ID.to_string(),
+ manifests: BTreeMap::new(),
+ buckets: BTreeMap::new(),
+ blobs: HashMap::new(),
+ vault_key: Value::Null,
+ legacy: None,
+ pending_actions: Vec::new(),
+ capabilities: json!({ "sharing": "on" }),
+ faults: Faults::default(),
+ }))
+ }
+
+ /*
+ * Seeding.
+ */
+
+ /// Store `db` as the personal manifest at `revision`, encrypted under `vek`, every bucket at the same revision.
+ pub fn publish(&mut self, db: &Connection, vek: &str, salt: &str, revision: i64) {
+ self.publish_with_blob_key(db, vek, salt, revision, vek);
+ }
+
+ /// [`publish`](Self::publish) with the blobs encrypted under another key, to serve blobs this device cannot open.
+ pub fn publish_with_blob_key(&mut self, db: &Connection, vek: &str, salt: &str, revision: i64, blob_key: &str) {
+ let personal = self.personal_manifest_id.clone();
+ let canonicalized = vault_codec::canonicalize_from_sqlite(CanonicalizeInput {
+ tables: read_tables(db),
+ canonicalized_at: "2026-09-11T00:00:00.000Z".to_string(),
+ manifests: vec![ManifestSpec { manifest_id: personal.clone(), manifest_salt: salt.to_string(), name: None }],
+ stamp_unstamped_into: None,
+ })
+ .unwrap();
+ let entry = &canonicalized.manifests[0];
+ let (blob, hash) = encrypt_payload(&serde_json::to_string(&entry.manifest).unwrap(), vek, &crypto::aad::manifest(&personal));
+ let store = self.blobs.entry(personal.clone()).or_default();
+ let mut references = Vec::new();
+ for (blob_hash, blob_entry) in &entry.blobs {
+ let bytes = crate::common::encoding::base64_decode(&blob_entry.bytes_base64).unwrap();
+ let encrypted = blob_keys::encrypt_blob(&bytes, blob_key, &personal, blob_hash).unwrap();
+ store.insert(blob_hash.clone(), json!({ "hash": blob_hash, "category": blob_entry.kind, "encryptedDataBase64": encrypted.encrypted_data_base64, "encryptedBlobKey": encrypted.encrypted_blob_key }));
+ references.push(json!({ "hash": blob_hash, "category": blob_entry.kind, "sizeBytes": bytes.len() }));
+ }
+ let access = self.manifests.get(&personal).map(|m| m.access.clone()).unwrap_or_else(|| json!({ "canAdminister": true, "keyType": "account-key" }));
+ self.manifests.insert(personal.clone(), ServerManifest { blob: Some(blob), ciphertext_hash: Some(hash), revision, blob_references: references, access });
+ for bucket in &canonicalized.data_buckets {
+ let (blob, hash) = encrypt_payload(&serde_json::to_string(bucket).unwrap(), vek, &crypto::aad::bucket(&bucket.manifest_id, &bucket.category));
+ self.buckets.insert((bucket.manifest_id.clone(), bucket.category.clone()), ServerBucket { blob, ciphertext_hash: hash, revision });
+ }
+ }
+
+ /// A personal manifest the server created without content (registration), at `revision`.
+ pub fn publish_contentless_personal(&mut self, revision: i64) {
+ let personal = self.personal_manifest_id.clone();
+ self.manifests.insert(personal, ServerManifest { revision, access: json!({ "canAdminister": false, "keyType": "account-key" }), ..Default::default() });
+ }
+
+ /// LEGACY: hold the account as a sqlite blob under `unlock_key`.
+ pub fn publish_legacy(&mut self, db: &Connection, unlock_key: &str, revision: i64) {
+ let bytes = db.serialize(rusqlite::MAIN_DB).unwrap().to_vec();
+ self.legacy = Some(LegacyVault { blob: crypto::symmetric_encrypt_bytes(&bytes, unlock_key).unwrap(), revision });
+ }
+
+ /// Serve a shared manifest exactly as given (a manifest dto), for the grant tests.
+ pub fn add_manifest_dto(&mut self, dto: Value) {
+ let id = dto["manifestId"].as_str().unwrap().to_string();
+ let mut access = dto.clone();
+ for key in ["manifestId", "blob", "ciphertextHash", "revision", "blobReferences"] {
+ access.as_object_mut().unwrap().remove(key);
+ }
+ let blob = dto["blob"].as_str().filter(|b| !b.is_empty()).map(str::to_string);
+ self.manifests.insert(id, ServerManifest { blob, ciphertext_hash: dto["ciphertextHash"].as_str().map(str::to_string), revision: dto["revision"].as_i64().unwrap_or(0), blob_references: Vec::new(), access });
+ }
+
+ /// Hold the account's key hierarchy, as `GET VaultKey/Password` serves it.
+ pub fn set_vault_key(&mut self, hierarchy: &crypto::AccountKeyHierarchy) {
+ let blobs = &hierarchy.account_keys;
+ let account_keys = json!({ "encryptedAccountKey": blobs.encrypted_account_key, "encryptedVek": blobs.encrypted_vek, "accountPublicKey": blobs.account_public_key, "encryptedAccountPrivateKey": blobs.encrypted_account_private_key, "signingPublicKey": blobs.signing_public_key, "encryptedSigningPrivateKey": blobs.encrypted_signing_private_key });
+ self.vault_key = vault_key_dto(account_keys.as_object().unwrap());
+ }
+
+ /*
+ * Reading back.
+ */
+
+ pub fn personal(&self) -> &ServerManifest {
+ &self.manifests[&self.personal_manifest_id]
+ }
+
+ /// The personal manifest decrypted with `vek`.
+ pub fn open_personal(&self, vek: &str) -> Value {
+ decrypt_manifest(self.personal().blob.as_deref().expect("the personal manifest has content"), vek, &self.personal_manifest_id)
+ }
+
+ /// The personal manifest's `category` bucket decrypted with `vek`.
+ pub fn open_bucket(&self, category: &str, vek: &str) -> Value {
+ let bucket = &self.buckets[&(self.personal_manifest_id.clone(), category.to_string())];
+ decrypt_payload(&bucket.blob, vek, &crypto::aad::bucket(&self.personal_manifest_id, category))
+ }
+
+ /// The hashes the personal manifest's blob store holds, sorted.
+ pub fn stored_blob_hashes(&self) -> Vec<String> {
+ let mut hashes: Vec<String> = self.blobs.get(&self.personal_manifest_id).map(|store| store.keys().cloned().collect()).unwrap_or_default();
+ hashes.sort();
+ hashes
+ }
+
+ /// Assert that the host's local vault canonicalizes to exactly what the server holds for the personal
+ /// manifest and its buckets: the convergence check every multi-device scenario ends with.
+ pub fn assert_converged(&self, host: &TestHost) {
+ let server_manifest = self.open_personal(&host.vault_key);
+ let salt = server_manifest["manifestSalt"].as_str().unwrap();
+ let local = vault_codec::canonicalize_from_sqlite(CanonicalizeInput {
+ tables: read_tables(&host.local),
+ canonicalized_at: String::new(),
+ manifests: vec![ManifestSpec { manifest_id: self.personal_manifest_id.clone(), manifest_salt: salt.to_string(), name: None }],
+ stamp_unstamped_into: None,
+ })
+ .unwrap();
+ let fingerprint = |value: &Value| vault_codec::compute_content_fingerprint(&value.to_string());
+ assert_eq!(fingerprint(&serde_json::to_value(&local.manifests[0].manifest).unwrap()), fingerprint(&server_manifest), "the local vault and the server's personal manifest differ:\nlocal {}\nserver {}", serde_json::to_value(&local.manifests[0].manifest).unwrap(), server_manifest);
+ for bucket in &local.data_buckets {
+ assert_eq!(fingerprint(&serde_json::to_value(bucket).unwrap()), fingerprint(&self.open_bucket(&bucket.category, &host.vault_key)), "bucket {} differs", bucket.category);
+ }
+ }
+
+ /*
+ * The API.
+ */
+
+ /// Answer one request, `None` for a transport failure.
+ pub fn respond(&mut self, method: &str, path: &str, body: Option<&Value>) -> Option<(u16, Value)> {
+ if self.faults.unreachable {
+ return None;
+ }
+ match (method, path) {
+ ("GET", "Status") => Some(match self.faults.status_http {
+ Some(status) => (status, json!({})),
+ None => (200, self.status()),
+ }),
+ ("GET", "Vault") => Some((200, self.vault())),
+ ("POST", "Vault") => self.write_vault(body?),
+ ("POST", "Vault/blobs") => Some((200, self.upload_blobs(body?))),
+ ("POST", "Vault/blobs/missing" | "Vault/blobs/download") if body?["hashes"].as_array().is_some_and(|h| h.len() > MAX_HASHES_PER_REQUEST) => Some((400, json!({ "code": "INVALID_REQUEST", "statusCode": 400 }))),
+ ("POST", "Vault/blobs/missing") => Some((200, self.missing_blobs(body?))),
+ ("POST", "Vault/blobs/download") => Some((200, self.download_blobs(body?))),
+ ("GET", "VaultKey/Password") => Some((200, json!({ "vaultKey": self.vault_key }))),
+ ("DELETE", path) if path.starts_with("ClientActions/") => {
+ let id = &path["ClientActions/".len()..];
+ self.pending_actions.retain(|action| action["id"] != id);
+ Some((200, json!({})))
+ }
+ _ => None,
+ }
+ }
+
+ /// The `GET Status` body.
+ pub fn status(&self) -> Value {
+ let (mut manifest_revisions, bucket_revisions) = self.current_revisions();
+ for (entry, m) in manifest_revisions.iter_mut().zip(self.manifests.values()) {
+ if let Some(name) = m.access.get("encryptedName").filter(|n| !n.is_null()) {
+ entry["encryptedName"] = name.clone();
+ }
+ }
+ let legacy_revisions = self.legacy.as_ref().map(|l| vec![json!({ "manifestId": self.personal_manifest_id, "revision": l.revision })]);
+ json!({
+ "clientVersionSupported": true,
+ "serverVersion": self.version,
+ "manifestRevisions": legacy_revisions.unwrap_or(manifest_revisions),
+ "bucketRevisions": bucket_revisions,
+ "personalManifestId": self.personal_manifest_id,
+ "srpSalt": self.srp_salt,
+ "capabilities": self.capabilities,
+ "pendingActions": self.pending_actions,
+ })
+ }
+
+ /// The `GET Vault` body.
+ pub fn vault(&self) -> Value {
+ let email_routing = json!({ "privateEmailDomainList": ["private.io"], "publicEmailDomainList": [], "hiddenPrivateEmailDomainList": [], "emailAddressList": [] });
+ if let Some(legacy) = &self.legacy {
+ return json!({ "storageFormat": "sqlite-blob", "legacyVaultBlob": legacy.blob, "legacyRevision": legacy.revision, "personalManifestId": self.personal_manifest_id, "emailRouting": email_routing });
+ }
+ let manifests: Vec<Value> = self.manifests.iter().map(|(id, m)| {
+ let mut dto = json!({ "manifestId": id, "blob": m.blob, "ciphertextHash": m.ciphertext_hash, "revision": m.revision, "blobReferences": m.blob_references });
+ for (key, value) in m.access.as_object().unwrap() {
+ dto[key] = value.clone();
+ }
+ dto
+ }).collect();
+ let buckets: Vec<Value> = self.buckets.iter().map(|((manifest_id, category), b)| json!({ "manifestId": manifest_id, "category": category, "blob": b.blob, "ciphertextHash": b.ciphertext_hash, "revision": b.revision })).collect();
+ json!({ "storageFormat": "manifest", "personalManifestId": self.personal_manifest_id, "manifests": manifests, "buckets": buckets, "emailRouting": email_routing })
+ }
+
+ /// The manifest and bucket revisions the server currently holds.
+ fn current_revisions(&self) -> (Vec<Value>, Vec<Value>) {
+ let manifests = self.manifests.iter().map(|(id, m)| json!({ "manifestId": id, "revision": m.revision })).collect();
+ let buckets = self.buckets.iter().map(|((manifest_id, category), b)| json!({ "manifestId": manifest_id, "category": category, "revision": b.revision })).collect();
+ (manifests, buckets)
+ }
+
+ fn write_vault(&mut self, body: &Value) -> Option<(u16, Value)> {
+ if self.faults.drop_writes {
+ return None;
+ }
+ let manifests = body["manifests"].as_array().cloned().unwrap_or_default();
+ let buckets = body["buckets"].as_array().cloned().unwrap_or_default();
+
+ // Like the controller: a write naming a manifest the caller cannot reach is refused as a whole.
+ if manifests.iter().chain(&buckets).any(|w| !self.manifests.contains_key(w["manifestId"].as_str().unwrap())) {
+ return Some((404, json!({ "code": "SHARED_MANIFEST_NOT_FOUND", "statusCode": 404 })));
+ }
+
+ let stale_manifest = manifests.iter().any(|w| self.manifests.get(w["manifestId"].as_str().unwrap()).map(|m| m.revision) != w["currentRevision"].as_i64());
+ let stale_bucket = buckets.iter().any(|w| self.buckets.get(&(w["manifestId"].as_str().unwrap().to_string(), w["category"].as_str().unwrap().to_string())).map(|b| b.revision).unwrap_or(0) != w["currentRevision"].as_i64().unwrap_or(-1));
+ if stale_manifest || stale_bucket || self.faults.refuse_writes > 0 {
+ self.faults.refuse_writes = self.faults.refuse_writes.saturating_sub(1);
+ let (manifest_revisions, bucket_revisions) = self.current_revisions();
+ return Some((200, json!({ "status": "outdated", "manifestRevisions": manifest_revisions, "bucketRevisions": bucket_revisions, "missingBlobHashes": [] })));
+ }
+
+ let missing: Vec<Value> = manifests
+ .iter()
+ .flat_map(|w| {
+ let store = self.blobs.get(w["manifestId"].as_str().unwrap());
+ w["blobReferences"].as_array().cloned().unwrap_or_default().into_iter().filter(move |r| !store.is_some_and(|s| s.contains_key(r["hash"].as_str().unwrap()))).map(|r| r["hash"].clone())
+ })
+ .collect();
+ if !missing.is_empty() {
+ return Some((200, json!({ "status": "ok", "manifestRevisions": [], "bucketRevisions": [], "missingBlobHashes": missing })));
+ }
+
+ if let Some(account_keys) = body["migration"]["accountKeys"].as_object() {
+ if !self.vault_key.is_null() {
+ return Some((400, json!({ "code": "VAULT_KEY_ALREADY_EXISTS", "statusCode": 400 })));
+ }
+ self.vault_key = vault_key_dto(account_keys);
+ // The upgrade write moves the account onto the manifest format.
+ self.legacy = None;
+ }
+
+ let mut manifest_revisions = Vec::new();
+ for write in &manifests {
+ let id = write["manifestId"].as_str().unwrap();
+ let references = write["blobReferences"].as_array().cloned().unwrap_or_default().into_iter().map(|r| {
+ let size = r.get("sizeBytes").cloned().unwrap_or_else(|| json!(self.stored_blob_size(id, r["hash"].as_str().unwrap())));
+ json!({ "hash": r["hash"], "category": r["category"], "sizeBytes": size })
+ });
+ let references = references.collect();
+ let entry = self.manifests.get_mut(id).unwrap();
+ entry.blob = Some(write["manifestBlob"].as_str().unwrap().to_string());
+ entry.ciphertext_hash = Some(write["manifestCiphertextHash"].as_str().unwrap().to_string());
+ entry.blob_references = references;
+ entry.revision += 1;
+ manifest_revisions.push(json!({ "manifestId": id, "revision": entry.revision }));
+ }
+ let mut bucket_revisions = Vec::new();
+ for write in &buckets {
+ let key = (write["manifestId"].as_str().unwrap().to_string(), write["category"].as_str().unwrap().to_string());
+ let revision = write["currentRevision"].as_i64().unwrap() + 1;
+ self.buckets.insert(key.clone(), ServerBucket { blob: write["blob"].as_str().unwrap().to_string(), ciphertext_hash: write["ciphertextHash"].as_str().unwrap().to_string(), revision });
+ bucket_revisions.push(json!({ "manifestId": key.0, "category": key.1, "revision": revision }));
+ }
+ Some((200, json!({ "status": "ok", "manifestRevisions": manifest_revisions, "bucketRevisions": bucket_revisions, "missingBlobHashes": [] })))
+ }
+
+ /// The decoded size of a stored blob's encrypted data, 0 when unknown.
+ fn stored_blob_size(&self, manifest_id: &str, hash: &str) -> usize {
+ let data = self.blobs.get(manifest_id).and_then(|store| store.get(hash)).and_then(|blob| blob["encryptedDataBase64"].as_str());
+ data.and_then(|data| crate::common::encoding::base64_decode(data).ok()).map_or(0, |bytes| bytes.len())
+ }
+
+ fn upload_blobs(&mut self, body: &Value) -> Value {
+ let store = self.blobs.entry(body["manifestId"].as_str().unwrap().to_string()).or_default();
+ let overwrite = body["overwrite"].as_bool().unwrap_or(false);
+ let mut accepted = 0;
+ for blob in body["blobs"].as_array().cloned().unwrap_or_default() {
+ let hash = blob["hash"].as_str().unwrap().to_string();
+ if overwrite || !store.contains_key(&hash) {
+ store.insert(hash, blob);
+ accepted += 1;
+ }
+ }
+ json!({ "acceptedCount": accepted })
+ }
+
+ fn missing_blobs(&self, body: &Value) -> Value {
+ let store = self.blobs.get(body["manifestId"].as_str().unwrap());
+ let missing: Vec<Value> = body["hashes"].as_array().cloned().unwrap_or_default().into_iter().filter(|h| !store.is_some_and(|s| s.contains_key(h.as_str().unwrap()))).collect();
+ json!({ "missing": missing })
+ }
+
+ fn download_blobs(&self, body: &Value) -> Value {
+ if self.faults.withhold_blobs {
+ return json!({ "blobs": [] });
+ }
+ let store = self.blobs.get(body["manifestId"].as_str().unwrap());
+ let blobs: Vec<Value> = body["hashes"].as_array().cloned().unwrap_or_default().into_iter().filter_map(|h| store.and_then(|s| s.get(h.as_str().unwrap())).cloned()).collect();
+ json!({ "blobs": blobs })
+ }
+}
+
+/*
+ * Payload crypto, as the engine does it.
+ */
+
+/// A manifest or bucket payload packed and encrypted under `vek`, with its ciphertext hash.
+fn encrypt_payload(json: &str, vek: &str, aad: &[u8]) -> (String, String) {
+ let packed = vault_codec::pack_payload(json).unwrap();
+ let blob = crypto::symmetric_encrypt_bytes_with_aad(&packed, vek, aad).unwrap();
+ let hash = vault_codec::compute_ciphertext_hash(&blob);
+ (blob, hash)
+}
+
+/// The `vaultKey` dto of `GET VaultKey/Password` around the account key fields.
+fn vault_key_dto(account_keys: &serde_json::Map<String, Value>) -> Value {
+ let mut dto = json!({ "type": "password", "algorithm": "aes256-gcm", "salt": "salt", "encryptionType": "Argon2Id", "encryptionSettings": "{}" });
+ for (key, value) in account_keys {
+ dto[key] = value.clone();
+ }
+ dto
+}
+
+/// A manifest blob (as written or as stored) decrypted and unpacked.
+pub fn decrypt_manifest(blob: &str, vek: &str, manifest_id: &str) -> Value {
+ decrypt_payload(blob, vek, &crypto::aad::manifest(manifest_id))
+}
+
+fn decrypt_payload(blob: &str, vek: &str, aad: &[u8]) -> Value {
+ let plain = crypto::symmetric_decrypt_bytes_with_aad(&crate::common::encoding::base64_decode(blob).unwrap(), vek, aad).unwrap();
+ serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap()
+}
diff --git a/core/rust/src/vault_sync/tests/grants.rs b/core/rust/src/vault_sync/tests/grants.rs
new file mode 100644
index 000000000..d1851499c
--- /dev/null
+++ b/core/rust/src/vault_sync/tests/grants.rs
@@ -0,0 +1,48 @@
+//! A shared manifest granted in a format this build does not know ends the session with "update the app", like a
+//! newer vault format. A grant-format change ships with a minimum client version bump, so this only catches a missed bump.
+
+use serde_json::{json, Value};
+
+use super::test_host::TestHost;
+use super::{server_with_item, SHARED_MANIFEST_ID};
+use crate::crypto;
+
+/// Sync a fresh device against a personal vault plus the given shared manifest dto.
+fn sync_with_shared_manifest(shared: Value) -> (TestHost, Value) {
+ let vek = crypto::generate_key_base64();
+ let server = server_with_item(&vek);
+ server.borrow_mut().add_manifest_dto(shared);
+ let mut host = TestHost::logged_in(&vek, &server);
+ let result = host.sync();
+ (host, result)
+}
+
+fn grant(algorithm: &str, key_type: &str, blob: &str) -> Value {
+ json!({ "manifestId": SHARED_MANIFEST_ID, "blob": blob, "revision": 3, "blobReferences": [], "canAdminister": false, "keyType": key_type, "algorithm": algorithm, "encryptedVek": "future-grant", "accountPublicKey": "future-key" })
+}
+
+#[test]
+fn an_unknown_grant_algorithm_or_key_type_asks_for_an_app_update() {
+ for (what, shared) in [
+ ("unknown algorithm", grant("future-x", "grant-key", "bm90LXJlYWRhYmxl")),
+ ("unknown key type", grant("rsa-oaep-sha256", "future-key-type", "bm90LXJlYWRhYmxl")),
+ ("unknown algorithm on a contentless manifest", grant("future-x", "grant-key", "")),
+ ] {
+ let (host, result) = sync_with_shared_manifest(shared);
+ assert_eq!(result["success"], false, "{}: {}", what, result);
+ assert_eq!(result["logoutReason"], "vaultVersionIncompatible", "{}: an app too old for the grant format is told to update: {}", what, result);
+ assert!(host.item_names().is_empty(), "{}: nothing is materialized, not even the personal vault", what);
+ assert!(host.store_calls.is_empty());
+ }
+}
+
+#[test]
+fn a_shared_manifest_without_key_type_or_algorithm_is_refused_never_guessed() {
+ for missing in ["keyType", "algorithm"] {
+ let mut shared = grant("rsa-oaep-sha256", "grant-key", "bm90LXJlYWRhYmxl");
+ shared.as_object_mut().unwrap().remove(missing);
+ let (host, result) = sync_with_shared_manifest(shared);
+ assert_eq!(result["success"], false, "missing {}: {}", missing, result);
+ assert!(host.item_names().is_empty(), "missing {}: nothing is materialized", missing);
+ }
+}
diff --git a/core/rust/src/vault_sync/tests/item_move.rs b/core/rust/src/vault_sync/tests/item_move.rs
index 168e2941c..aa966455c 100644
--- a/core/rust/src/vault_sync/tests/item_move.rs
+++ b/core/rust/src/vault_sync/tests/item_move.rs
@@ -2,66 +2,60 @@
use rusqlite::Connection;
-use super::test_host::{complete_schema_sql, open_schema_db};
+use super::test_host::open_schema_db;
+use super::{ITEM_A, PERSONAL_MANIFEST_ID, SHARED_MANIFEST_ID};
-const PERSONAL: &str = "11111111-1111-4111-8111-111111111111";
-const SHARED: &str = "22222222-2222-4222-8222-222222222222";
-const ITEM: &str = "33333333-3333-4333-8333-333333333333";
const T_CREATED: &str = "2024-01-01 00:00:00.000";
+const FIELD_VALUE: &str = "eeeeeeee-0000-4000-8000-000000000001";
+const TOTP: &str = "44444444-4444-4444-8444-444444444441";
/// A vault holding one personal item with a password and a TOTP code.
fn vault_with_item() -> Connection {
- let db = open_schema_db(&complete_schema_sql());
- db.execute("INSERT INTO Items (ManifestId, Id, Name, ItemType, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, ?2, 'Bank', 'Login', ?3, ?3, 0)", (PERSONAL, ITEM, T_CREATED)).unwrap();
- db.execute("INSERT INTO FieldValues (ManifestId, Id, ItemId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, 'fv-1', ?2, 'login.password', 'secret', 0, 0, ?3, ?3, 0)", (PERSONAL, ITEM, T_CREATED)).unwrap();
- db.execute("INSERT INTO TotpCodes (ManifestId, Id, ItemId, Name, SecretKey, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, 'totp-1', ?2, 'code', 'ABC', ?3, ?3, 0)", (PERSONAL, ITEM, T_CREATED)).unwrap();
+ let db = open_schema_db();
+ db.execute("INSERT INTO Items (ManifestId, Id, Name, ItemType, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, ?2, 'Bank', 'Login', ?3, ?3, 0)", (PERSONAL_MANIFEST_ID, ITEM_A, T_CREATED)).unwrap();
+ db.execute("INSERT INTO FieldValues (ManifestId, Id, ItemId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, ?2, ?3, 'login.password', 'secret', 0, 0, ?4, ?4, 0)", (PERSONAL_MANIFEST_ID, FIELD_VALUE, ITEM_A, T_CREATED)).unwrap();
+ db.execute("INSERT INTO TotpCodes (ManifestId, Id, ItemId, Name, SecretKey, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, ?2, ?3, 'code', 'ABC', ?4, ?4, 0)", (PERSONAL_MANIFEST_ID, TOTP, ITEM_A, T_CREATED)).unwrap();
db
}
fn move_item(db: &Connection, from: &str, to: &str, at: &str) {
- assert_eq!(db.execute("UPDATE Items SET ManifestId = ?1, UpdatedAt = ?2 WHERE Id = ?3 AND ManifestId = ?4", (to, at, ITEM, from)).unwrap(), 1);
+ assert_eq!(db.execute("UPDATE Items SET ManifestId = ?1, UpdatedAt = ?2 WHERE Id = ?3 AND ManifestId = ?4", (to, at, ITEM_A, from)).unwrap(), 1);
}
/// `(IsDeleted, UpdatedAt)` of the item row in `manifest`, when it has one.
fn item_in(db: &Connection, manifest: &str) -> Option<(i64, String)> {
- db.query_row("SELECT IsDeleted, UpdatedAt FROM Items WHERE ManifestId = ?1 AND Id = ?2", (manifest, ITEM), |row| Ok((row.get(0)?, row.get(1)?))).ok()
+ db.query_row("SELECT IsDeleted, UpdatedAt FROM Items WHERE ManifestId = ?1 AND Id = ?2", (manifest, ITEM_A), |row| Ok((row.get(0)?, row.get(1)?))).ok()
}
fn child_rows_in(db: &Connection, manifest: &str) -> i64 {
- let count = |table: &str| -> i64 { db.query_row(&format!("SELECT COUNT(*) FROM {} WHERE ManifestId = ?1 AND ItemId = ?2", table), (manifest, ITEM), |row| row.get(0)).unwrap() };
+ let count = |table: &str| -> i64 { db.query_row(&format!("SELECT COUNT(*) FROM {} WHERE ManifestId = ?1 AND ItemId = ?2", table), (manifest, ITEM_A), |row| row.get(0)).unwrap() };
count("FieldValues") + count("TotpCodes")
}
#[test]
-fn moving_an_item_leaves_a_tombstone_in_the_manifest_it_left() {
- // Without it the merge, a union per manifest, keeps the server's copy in the source manifest and the item ends up in both.
+fn moving_an_item_leaves_a_tombstone_in_the_manifest_it_left_and_can_move_back() {
+ // Without the tombstone the merge, a union per manifest, keeps the server's copy in the source manifest and the item ends up in both.
let db = vault_with_item();
- move_item(&db, PERSONAL, SHARED, "2024-02-01 00:00:00.000");
+ move_item(&db, PERSONAL_MANIFEST_ID, SHARED_MANIFEST_ID, "2024-02-01 00:00:00.000");
- assert_eq!(item_in(&db, SHARED), Some((0, "2024-02-01 00:00:00.000".to_string())));
- assert_eq!(child_rows_in(&db, SHARED), 2, "the item's rows follow it");
- assert_eq!(item_in(&db, PERSONAL), Some((1, "2024-02-01 00:00:00.000".to_string())), "a tombstone stamped with the move stays behind");
- assert_eq!(child_rows_in(&db, PERSONAL), 0);
-}
+ assert_eq!(item_in(&db, SHARED_MANIFEST_ID), Some((0, "2024-02-01 00:00:00.000".to_string())));
+ assert_eq!(child_rows_in(&db, SHARED_MANIFEST_ID), 2, "the item's rows follow it");
+ assert_eq!(item_in(&db, PERSONAL_MANIFEST_ID), Some((1, "2024-02-01 00:00:00.000".to_string())), "a tombstone stamped with the move stays behind");
+ assert_eq!(child_rows_in(&db, PERSONAL_MANIFEST_ID), 0);
-#[test]
-fn an_item_can_move_back_into_the_manifest_it_left() {
// The tombstone holds the primary key the item returns to, so the move back clears it first.
- let db = vault_with_item();
- move_item(&db, PERSONAL, SHARED, "2024-02-01 00:00:00.000");
- move_item(&db, SHARED, PERSONAL, "2024-03-01 00:00:00.000");
-
- assert_eq!(item_in(&db, PERSONAL), Some((0, "2024-03-01 00:00:00.000".to_string())));
- assert_eq!(child_rows_in(&db, PERSONAL), 2);
- assert_eq!(item_in(&db, SHARED), Some((1, "2024-03-01 00:00:00.000".to_string())), "and the shared manifest now holds the tombstone");
+ move_item(&db, SHARED_MANIFEST_ID, PERSONAL_MANIFEST_ID, "2024-03-01 00:00:00.000");
+ assert_eq!(item_in(&db, PERSONAL_MANIFEST_ID), Some((0, "2024-03-01 00:00:00.000".to_string())));
+ assert_eq!(child_rows_in(&db, PERSONAL_MANIFEST_ID), 2);
+ assert_eq!(item_in(&db, SHARED_MANIFEST_ID), Some((1, "2024-03-01 00:00:00.000".to_string())), "and the shared manifest now holds the tombstone");
}
#[test]
fn a_live_item_with_the_same_id_in_the_destination_is_never_cleared() {
// Two manifests holding the same item id is a supported state; only a tombstone gives way to a move.
let db = vault_with_item();
- db.execute("INSERT INTO Items (ManifestId, Id, Name, ItemType, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, ?2, 'Theirs', 'Login', ?3, ?3, 0)", (SHARED, ITEM, T_CREATED)).unwrap();
- let moved = db.execute("UPDATE Items SET ManifestId = ?1 WHERE Id = ?2 AND ManifestId = ?3", (SHARED, ITEM, PERSONAL));
+ db.execute("INSERT INTO Items (ManifestId, Id, Name, ItemType, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, ?2, 'Theirs', 'Login', ?3, ?3, 0)", (SHARED_MANIFEST_ID, ITEM_A, T_CREATED)).unwrap();
+ let moved = db.execute("UPDATE Items SET ManifestId = ?1 WHERE Id = ?2 AND ManifestId = ?3", (SHARED_MANIFEST_ID, ITEM_A, PERSONAL_MANIFEST_ID));
assert!(moved.is_err(), "the move is refused rather than overwriting the other item");
- assert_eq!(item_in(&db, SHARED).map(|(deleted, _)| deleted), Some(0));
+ assert_eq!(item_in(&db, SHARED_MANIFEST_ID).map(|(deleted, _)| deleted), Some(0));
}
diff --git a/core/rust/src/vault_sync/tests/keys.rs b/core/rust/src/vault_sync/tests/keys.rs
new file mode 100644
index 000000000..d0fde903b
--- /dev/null
+++ b/core/rust/src/vault_sync/tests/keys.rs
@@ -0,0 +1,168 @@
+//! Key resolution at login, and the cross-device race where another device creates the key hierarchy.
+
+use serde_json::json;
+
+use super::fake_server::FakeServer;
+use super::test_host::{open_schema_db, TestHost};
+use super::{insert_item, legacy_device, server_with_item, state, ITEM_A, PERSONAL_MANIFEST_ID};
+use crate::crypto;
+use crate::vault_codec;
+use crate::vault_sync::types::Command;
+
+/// Login: the host hands the password-derived key to `resolveVaultKey`; the server's chain opens with it, is
+/// cached for offline unlock, and the VEK comes back as the key to store.
+#[test]
+fn resolve_vault_key_opens_the_chain_from_the_server() {
+ let unlock_key = crypto::generate_key_base64();
+ let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
+ let server = FakeServer::new();
+ server.borrow_mut().set_vault_key(&hierarchy);
+ let mut host = TestHost::new(&unlock_key);
+ host.serve(&server);
+
+ let result = host.run("resolveVaultKey");
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["hasVaultKey"], true);
+ assert_eq!(result["encryptionKey"], hierarchy.vault_encryption_key);
+ assert_eq!(host.state[state::ENCRYPTED_ACCOUNT_PRIVATE_KEY], hierarchy.account_keys.encrypted_account_private_key);
+ assert_eq!(host.state[state::ENCRYPTED_ACCOUNT_KEY], hierarchy.account_keys.encrypted_account_key);
+ assert_eq!(host.state[state::SIGNING_PUBLIC_KEY], hierarchy.account_keys.signing_public_key);
+ assert_eq!(host.state[state::ENCRYPTED_SIGNING_PRIVATE_KEY], hierarchy.account_keys.encrypted_signing_private_key);
+ assert!(host.store_calls.is_empty(), "resolving a key never touches the stored vault");
+
+ // A device that stores the Account Key (Login with Mobile) resolves with it as well as with the unlock key.
+ let account_key = crypto::unwrap_account_key(&hierarchy.account_keys.encrypted_account_key, &unlock_key).unwrap();
+ let mut host = TestHost::new(&account_key);
+ host.serve(&server);
+ assert_eq!(host.run("resolveVaultKey")["encryptionKey"], hierarchy.vault_encryption_key);
+}
+
+/// A legacy account has no chain: the password-derived key is the vault key and any stale cached chain is dropped.
+#[test]
+fn resolve_vault_key_keeps_the_unlock_key_for_a_legacy_account() {
+ let unlock_key = crypto::generate_key_base64();
+ let mut host = TestHost::new(&unlock_key);
+ host.serve(&FakeServer::new());
+ host.set_state(state::ENCRYPTED_ACCOUNT_KEY, json!("stale"));
+
+ let result = host.run("resolveVaultKey");
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["hasVaultKey"], false);
+ assert_eq!(result["encryptionKey"], unlock_key);
+ assert!(!host.state.contains_key(state::ENCRYPTED_ACCOUNT_KEY));
+}
+
+/// Offline, the cached chain opens with the password-derived key (a re-login after a forced logout).
+#[test]
+fn resolve_vault_key_opens_the_cached_chain_when_the_server_is_unreachable() {
+ let unlock_key = crypto::generate_key_base64();
+ let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
+ let mut host = TestHost::new(&unlock_key);
+ host.set_state(state::ENCRYPTED_ACCOUNT_KEY, json!(hierarchy.account_keys.encrypted_account_key));
+ host.set_state(state::ENCRYPTED_VEK, json!(hierarchy.account_keys.encrypted_vek));
+
+ let result = host.run("resolveVaultKey");
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["hasVaultKey"], true);
+ assert_eq!(result["encryptionKey"], hierarchy.vault_encryption_key);
+}
+
+/// A key that does not open the chain is a rejected unlock key (E-206); a chain whose VEK does not open under its
+/// own account key is damage, not a wrong password (E-207); an unknown wrap algorithm asks for an app update.
+#[test]
+fn resolve_vault_key_tells_the_failure_modes_apart() {
+ let unlock_key = crypto::generate_key_base64();
+ let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
+ let server = FakeServer::new();
+ server.borrow_mut().set_vault_key(&hierarchy);
+
+ let mut host = TestHost::new(&crypto::generate_key_base64());
+ host.serve(&server);
+ let result = host.run("resolveVaultKey");
+ assert_eq!(result["errorCode"], "E-206", "{}", result);
+ assert!(result.get("encryptionKey").is_none());
+
+ let mut damaged = hierarchy.clone();
+ damaged.account_keys.encrypted_vek = crypto::wrap_key(&crypto::generate_key_base64(), &crypto::generate_key_base64(), crypto::aad::PERSONAL_VEK).unwrap();
+ server.borrow_mut().set_vault_key(&damaged);
+ let mut host = TestHost::new(&unlock_key);
+ host.serve(&server);
+ assert_eq!(host.run("resolveVaultKey")["errorCode"], "E-207");
+
+ server.borrow_mut().set_vault_key(&hierarchy);
+ server.borrow_mut().vault_key["algorithm"] = json!("future-x");
+ let mut host = TestHost::new(&unlock_key);
+ host.serve(&server);
+ let result = host.run("resolveVaultKey");
+ assert_eq!(result["logoutReason"], "vaultVersionIncompatible", "{}", result);
+ assert!(!host.state.contains_key(state::ENCRYPTED_ACCOUNT_KEY), "nothing is cached from a chain this build cannot open");
+}
+
+/// The cross-device race: this device logged in while the account was legacy (no cached chain, unlock key session), and
+/// another device created the hierarchy since. The pull accepts it, and the stored vault reaches the host
+/// together with the VEK it is now encrypted under.
+#[test]
+fn a_hierarchy_created_on_another_device_is_accepted_on_the_next_pull() {
+ let unlock_key = crypto::generate_key_base64();
+ let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
+ let vek = hierarchy.vault_encryption_key.clone();
+ let server = server_with_item(&vek);
+ server.borrow_mut().set_vault_key(&hierarchy);
+ let mut host = TestHost::new(&unlock_key);
+ host.serve(&server);
+
+ let result = host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(host.vault_key, vek, "the store carried the VEK, so the host switched to it before opening the blob");
+ assert!(matches!(&host.store_calls[0], Command::VaultStore { encryption_key: Some(key), .. } if *key == vek));
+ assert_eq!(host.item_names(), vec!["Server item"]);
+ assert_eq!(host.state[state::ENCRYPTED_ACCOUNT_KEY], hierarchy.account_keys.encrypted_account_key);
+ assert_eq!(host.state[state::ACCOUNT_PUBLIC_KEY], hierarchy.account_keys.account_public_key);
+ assert_eq!(host.rekeyed_stores, 1, "the chain is cached before the vault is stored under the VEK");
+}
+
+/// The sync trusts the key it is given: a device with a cached chain is never probed, and an unlock key handed to it
+/// is not silently upgraded. Resolving the key is the host's job.
+#[test]
+fn a_device_with_a_cached_chain_is_not_probed_and_its_session_key_is_left_alone() {
+ let unlock_key = crypto::generate_key_base64();
+ let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
+ let vek = hierarchy.vault_encryption_key.clone();
+ let server = FakeServer::new();
+ let db = open_schema_db();
+ server.borrow_mut().publish(&db, &vek, &vault_codec::generate_manifest_salt(), 4);
+ let mut host = TestHost::new(&vek);
+ host.serve(&server);
+ host.store_local_as_blob();
+ host.set_state(state::ENCRYPTED_ACCOUNT_KEY, json!(hierarchy.account_keys.encrypted_account_key));
+ host.set_state(state::ENCRYPTED_VEK, json!(hierarchy.account_keys.encrypted_vek));
+ host.set_state(state::SERVER_MANIFEST_REVISIONS, json!({ PERSONAL_MANIFEST_ID: 3 }));
+
+ host.sync_as(&unlock_key);
+
+ assert!(host.store_calls.iter().all(|c| matches!(c, Command::VaultStore { encryption_key: None, .. })), "no key swap reaches the host");
+ assert!(host.requests_to("VaultKey/Password").is_empty(), "a device with a cached chain is never probed");
+}
+
+/// A legacy account (unlock key session, no chain) is probed for a vault key only when the sync pulls or pushes,
+/// and then once per run.
+#[test]
+fn a_legacy_account_is_probed_for_a_vault_key_only_when_it_pulls_or_pushes() {
+ let unlock_key = crypto::generate_key_base64();
+ let server = FakeServer::new();
+ server.borrow_mut().publish_contentless_personal(3);
+ let mut host = legacy_device(&unlock_key, &server, |_| {});
+
+ let result = host.sync();
+ assert_eq!(result["success"], true, "{}", result);
+ assert!(host.requests_to("VaultKey/Password").is_empty(), "clean and in sync: not probed");
+
+ host.edit(|db| insert_item(db, ITEM_A, "Local item"));
+ let result = host.sync();
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(host.requests_to("VaultKey/Password").len(), 1, "dirty: probed once");
+}
diff --git a/core/rust/src/vault_sync/tests/merge.rs b/core/rust/src/vault_sync/tests/merge.rs
new file mode 100644
index 000000000..94cbedf4b
--- /dev/null
+++ b/core/rust/src/vault_sync/tests/merge.rs
@@ -0,0 +1,264 @@
+//! Two devices editing the same vault: the other device pushes first, this one syncs its offline edits and merges.
+//! Every scenario ends with the convergence check, which is what catches a merge dropping rows or keys. Which row wins
+//! is `vault_merge`'s to test; the scenarios here are the ones that cross the SQLite boundary.
+//!
+//! Known gap, not covered here: an item moved to a shared manifest by one device while another added a TOTP code to
+//! it offline. The move leaves a tombstone newer than the offline add, so the delete stands and the new code is lost.
+
+use rusqlite::Connection;
+use serde_json::{json, Value};
+
+use super::test_host::{column, exec, query};
+use super::{insert_item_at, synced, Synced, ITEM_A, PERSONAL_MANIFEST_ID};
+use crate::sqlite_host::SqlStatement;
+
+const TOTP_A: &str = "44444444-4444-4444-8444-444444444441";
+const TOTP_B: &str = "44444444-4444-4444-8444-444444444442";
+const TOTP_C: &str = "44444444-4444-4444-8444-444444444443";
+const PASSKEY: &str = "55555555-5555-4555-8555-555555555555";
+const PASSKEY_B: &str = "55555555-5555-4555-8555-555555555556";
+const FOLDER: &str = "66666666-6666-4666-8666-666666666666";
+const GONE: &str = "77777777-7777-4777-8777-777777777777";
+const JWK: &str = "{\"kty\":\"EC\",\"crv\":\"P-256\",\"d\":\"secret-d\",\"x\":\"x\",\"y\":\"y\"}";
+/// The PrfKey every fixture passkey holds: the bytes 0 to 31.
+const PRF_KEY: [u8; 32] = {
+ let mut key = [0u8; 32];
+ let mut i = 0;
+ while i < key.len() {
+ key[i] = i as u8;
+ i += 1;
+ }
+ key
+};
+/// [`PRF_KEY`] as SQLite `hex()` prints it.
+const PRF_KEY_HEX: &str = "000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F";
+
+/// Four instants, oldest first, fixed for the whole run and recent enough that nothing in trash expires during a push.
+fn t(step: i64) -> String {
+ static BASE: std::sync::OnceLock<chrono::DateTime<chrono::Utc>> = std::sync::OnceLock::new();
+ let base = BASE.get_or_init(chrono::Utc::now);
+ (*base - chrono::Duration::days(4 - step)).format(crate::common::timestamp::VAULT_DATETIME).to_string()
+}
+
+fn sql(db: &Connection, statement: &str, params: &[Value]) {
+ exec(db, &[SqlStatement { sql: statement.to_string(), params: params.to_vec() }]).unwrap();
+}
+
+fn insert_totp(db: &Connection, item: &str, id: &str, secret: &str, at: &str) {
+ sql(db, "INSERT INTO TotpCodes (ManifestId, Id, ItemId, Name, SecretKey, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'code', ?, ?, ?, 0)", &[json!(PERSONAL_MANIFEST_ID), json!(id), json!(item), json!(secret), json!(t(0)), json!(at)]);
+}
+
+fn insert_password(db: &Connection, item: &str, value: &str, at: &str) {
+ sql(db, "INSERT INTO FieldValues (ManifestId, Id, ItemId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'login.password', ?, 0, 0, ?, ?, 0)", &[json!(PERSONAL_MANIFEST_ID), json!(crate::vault_sync::db::new_id()), json!(item), json!(value), json!(t(0)), json!(at)]);
+}
+
+fn bytes(data: &[u8]) -> Value {
+ json!({ "__b64": crate::common::encoding::base64_encode(data) })
+}
+
+/// A passkey with the given `PrivateKey` cell (TEXT or `{ __b64 }` bytes) and fixed byte columns.
+fn insert_passkey(db: &Connection, id: &str, private_key: Value, display_name: &str, at: &str) {
+ sql(
+ db,
+ "INSERT INTO Passkeys (ManifestId, Id, ItemId, RpId, UserHandle, PublicKey, PrivateKey, PrfKey, DisplayName, AdditionalData, CredentialId, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'example.com', ?, ?, ?, ?, ?, NULL, ?, ?, ?, 0)",
+ &[json!(PERSONAL_MANIFEST_ID), json!(id), json!(ITEM_A), bytes(&[0, 1, 2, 255, 254, 0, 7]), json!("{\"kty\":\"EC\",\"x\":\"pub\"}"), private_key, bytes(&PRF_KEY), json!(display_name), bytes(&[0x55; 16]), json!(t(0)), json!(at)],
+ );
+}
+
+/// Both devices after their last common sync: one item with a password, a TOTP code and a passkey.
+fn synced_pair() -> Synced {
+ synced(|db| {
+ insert_item_at(db, ITEM_A, "Bank", &t(0));
+ insert_password(db, ITEM_A, "pw-0", &t(0));
+ insert_totp(db, ITEM_A, TOTP_A, "SECRET-A", &t(0));
+ insert_passkey(db, PASSKEY, json!(JWK), "me", &t(0));
+ })
+}
+
+/// The other device makes `other_device` and pushes it; this device makes `this_device` offline and syncs. Returns the merged vault.
+fn merge(s: &mut Synced, other_device: impl FnOnce(&Connection), this_device: impl FnOnce(&Connection)) -> &Connection {
+ other_device(&s.db);
+ s.publish(8);
+ s.host.edit(this_device);
+ let result = s.host.sync();
+ assert_eq!(result["success"], true, "{} {:?}", result, s.host.logs);
+ s.server.borrow().assert_converged(&s.host);
+ &s.host.local
+}
+
+fn live_totp_secrets(db: &Connection) -> Vec<String> {
+ column(db, "SELECT SecretKey FROM TotpCodes WHERE IsDeleted = 0")
+}
+
+fn live_passkeys(db: &Connection) -> Vec<String> {
+ column(db, "SELECT DisplayName FROM Passkeys WHERE IsDeleted = 0")
+}
+
+fn live_field_values(db: &Connection) -> Vec<String> {
+ column(db, "SELECT Value FROM FieldValues WHERE IsDeleted = 0")
+}
+
+/// `Name|IsDeleted|DeletedAt|FolderId` of the item.
+fn item_state(db: &Connection) -> String {
+ column(db, &format!("SELECT Name || '|' || IsDeleted || '|' || ifnull(DeletedAt, '-') || '|' || ifnull(FolderId, '-') FROM Items WHERE Id = '{}'", ITEM_A)).join(",")
+}
+
+/*
+ * Concurrent edits.
+ */
+
+#[test]
+fn child_rows_added_on_both_devices_are_kept() {
+ let mut s = synced_pair();
+ let merged = merge(&mut s, |db| insert_totp(db, ITEM_A, TOTP_B, "SERVER-NEW", &t(1)), |db| insert_totp(db, ITEM_A, TOTP_C, "LOCAL-NEW", &t(2)));
+ assert_eq!(live_totp_secrets(merged), vec!["LOCAL-NEW", "SECRET-A", "SERVER-NEW"]);
+}
+
+#[test]
+fn a_child_row_added_offline_survives_the_other_device_editing_the_item() {
+ let mut s = synced_pair();
+ let merged = merge(
+ &mut s,
+ |db| {
+ sql(db, "UPDATE Items SET Name = 'Server name', UpdatedAt = ? WHERE Id = ?", &[json!(t(3)), json!(ITEM_A)]);
+ sql(db, "UPDATE FieldValues SET Value = 'pw-server', UpdatedAt = ? WHERE FieldKey = 'login.password'", &[json!(t(3))]);
+ },
+ |db| {
+ insert_totp(db, ITEM_A, TOTP_B, "LOCAL-NEW", &t(1));
+ insert_passkey(db, PASSKEY_B, json!(JWK), "mobile", &t(1));
+ },
+ );
+ assert_eq!(live_totp_secrets(merged), vec!["LOCAL-NEW", "SECRET-A"]);
+ assert_eq!(live_passkeys(merged), vec!["me", "mobile"]);
+ assert_eq!(live_field_values(merged), vec!["pw-server"]);
+ assert_eq!(item_state(merged), "Server name|0|-|-");
+}
+
+#[test]
+fn an_item_moved_into_a_folder_the_other_device_deleted_still_merges() {
+ let mut s = synced(|db| {
+ insert_item_at(db, ITEM_A, "Bank", &t(0));
+ insert_totp(db, ITEM_A, TOTP_A, "SECRET-A", &t(0));
+ sql(db, "INSERT INTO Folders (ManifestId, Id, Name, CreatedAt, UpdatedAt, IsDeleted, Weight) VALUES (?, ?, 'Work', ?, ?, 0, 0)", &[json!(PERSONAL_MANIFEST_ID), json!(FOLDER), json!(t(0)), json!(t(0))]);
+ });
+ let merged = merge(&mut s, |db| sql(db, "UPDATE Folders SET IsDeleted = 1, UpdatedAt = ? WHERE Id = ?", &[json!(t(1)), json!(FOLDER)]), |db| sql(db, "UPDATE Items SET FolderId = ?, UpdatedAt = ? WHERE Id = ?", &[json!(FOLDER), json!(t(2)), json!(ITEM_A)]));
+ assert_eq!(item_state(merged), format!("Bank|0|-|{}", FOLDER));
+ assert_eq!(live_totp_secrets(merged), vec!["SECRET-A"]);
+}
+
+#[test]
+fn a_merged_vault_syncs_again_without_a_write() {
+ let mut s = synced_pair();
+ merge(&mut s, |db| insert_totp(db, ITEM_A, TOTP_B, "SERVER-NEW", &t(1)), |db| insert_passkey(db, PASSKEY_B, json!(JWK), "mobile", &t(1)));
+ let writes_before = s.host.vault_writes().len();
+
+ s.host.edit(|_| {});
+ let again = s.host.sync();
+
+ assert_eq!(again["success"], true, "{}", again);
+ assert_eq!(s.host.vault_writes().len(), writes_before, "nothing changed, nothing is written");
+}
+
+/*
+ * Spellings older or other clients write.
+ */
+
+#[test]
+fn uppercase_guids_from_an_older_client_match_their_lowercase_twins() {
+ let mut s = synced_pair();
+ let merged = merge(
+ &mut s,
+ |db| insert_totp(db, ITEM_A, TOTP_B, "SERVER-NEW", &t(1)),
+ |db| {
+ // The local client wrote its rows in uppercase (as an older iOS build did).
+ for table in ["Items", "TotpCodes", "Passkeys", "FieldValues"] {
+ sql(db, &format!("UPDATE {} SET Id = upper(Id), ManifestId = upper(ManifestId)", table), &[]);
+ }
+ for table in ["TotpCodes", "Passkeys", "FieldValues"] {
+ sql(db, &format!("UPDATE {} SET ItemId = upper(ItemId)", table), &[]);
+ }
+ sql(db, "UPDATE TotpCodes SET SecretKey = 'LOCAL', UpdatedAt = ?", &[json!(t(2))]);
+ },
+ );
+ assert_eq!(live_totp_secrets(merged), vec!["LOCAL", "SERVER-NEW"]);
+ assert_eq!(live_passkeys(merged), vec!["me"]);
+ assert_eq!(column(merged, "SELECT COUNT(*) FROM Items"), vec!["1"]);
+}
+
+#[test]
+fn passkey_bytes_survive_a_merge_with_text_and_blob_private_keys() {
+ for (private_key, expected_type) in [(json!(JWK), "text"), (bytes(JWK.as_bytes()), "blob")] {
+ let mut s = synced_pair();
+ // The server side touches the original passkey, so both a local-only and a server-won row cross the merge.
+ let merged = merge(&mut s, |db| sql(db, "UPDATE Passkeys SET DisplayName = 'server', UpdatedAt = ? WHERE Id = ?", &[json!(t(2)), json!(PASSKEY)]), |db| insert_passkey(db, PASSKEY_B, private_key.clone(), "mobile", &t(1)));
+ let rows = query(merged, "SELECT DisplayName, typeof(PrivateKey) AS pt, CAST(PrivateKey AS TEXT) AS pk, hex(UserHandle) AS uh, hex(PrfKey) AS prf, hex(CredentialId) AS cid, typeof(UserHandle) AS ut FROM Passkeys ORDER BY DisplayName", &[]).unwrap();
+ assert_eq!(rows.len(), 2);
+ for row in &rows {
+ assert_eq!(row["uh"], json!("000102FFFE0007"), "UserHandle bytes kept, a NUL included");
+ assert_eq!(row["ut"], json!("blob"));
+ assert_eq!(row["prf"], json!(PRF_KEY_HEX));
+ assert_eq!(row["cid"], json!("55".repeat(16)));
+ assert_eq!(row["pk"], json!(JWK));
+ }
+ let mobile = rows.iter().find(|r| r["DisplayName"] == json!("mobile")).unwrap();
+ assert_eq!(mobile["pt"], json!(expected_type), "the PrivateKey storage class is kept as the writer chose it");
+ }
+}
+
+#[test]
+fn a_passkey_without_a_prf_key_merges_against_one_that_gained_it() {
+ // The local row wins as a whole, so the PrfKey it holds comes along.
+ let mut s = synced_pair();
+ let merged = merge(
+ &mut s,
+ |db| sql(db, "UPDATE Passkeys SET PrfKey = NULL, UpdatedAt = ? WHERE Id = ?", &[json!(t(1)), json!(PASSKEY)]),
+ |db| sql(db, "UPDATE Passkeys SET DisplayName = 'local', UpdatedAt = ? WHERE Id = ?", &[json!(t(2)), json!(PASSKEY)]),
+ );
+ assert_eq!(column(merged, "SELECT DisplayName || '|' || ifnull(hex(PrfKey), 'null') FROM Passkeys"), vec![format!("local|{PRF_KEY_HEX}")]);
+}
+
+/*
+ * Rows real vaults carry that must never fail the whole merge (which would drop every local change).
+ */
+
+#[test]
+fn stray_rows_on_the_local_side_never_fail_the_whole_merge() {
+ let strays: [(&str, Stray); 5] = [
+ ("an orphan TOTP code under an item that is gone", |db| insert_totp(db, GONE, TOTP_B, "ORPHAN", &t(1))),
+ ("an orphan field value", |db| insert_password(db, GONE, "orphan", &t(1))),
+ ("an item tag without its tag", |db| sql(db, "INSERT INTO ItemTags (ManifestId, ItemId, TagId, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, '88888888-8888-4888-8888-888888888888', ?, ?, 0)", &[json!(PERSONAL_MANIFEST_ID), json!(ITEM_A), json!(t(1)), json!(t(1))])),
+ ("an item pointing at a folder neither side has", |db| sql(db, "UPDATE Items SET FolderId = ?, Name = 'Edited', UpdatedAt = ? WHERE Id = ?", &[json!(FOLDER), json!(t(2)), json!(ITEM_A)])),
+ ("an empty-string FolderId", |db| sql(db, "UPDATE Items SET FolderId = '', Name = 'Edited', UpdatedAt = ? WHERE Id = ?", &[json!(t(2)), json!(ITEM_A)])),
+ ];
+ for (what, stray) in strays {
+ let mut s = synced_pair();
+ let merged = merge(
+ &mut s,
+ |db| insert_totp(db, ITEM_A, TOTP_B, "SERVER-NEW", &t(1)),
+ |db| {
+ stray(db);
+ insert_totp(db, ITEM_A, TOTP_C, "LOCAL-NEW", &t(2));
+ },
+ );
+ assert!(live_totp_secrets(merged).contains(&"LOCAL-NEW".to_string()), "{} dropped every local change", what);
+ }
+}
+
+/// A local edit that leaves a stray row behind.
+type Stray = fn(&Connection);
+
+#[test]
+fn custom_fields_whose_definitions_are_gone_keep_every_value() {
+ // The definition link is cut, the values stay apart: without a definition nothing says the field holds one value.
+ let mut s = synced_pair();
+ let merged = merge(
+ &mut s,
+ |db| insert_totp(db, ITEM_A, TOTP_B, "SERVER-NEW", &t(1)),
+ |db| {
+ for (def, value) in [("99999999-9999-4999-8999-999999999991", "one"), ("99999999-9999-4999-8999-999999999992", "two")] {
+ sql(db, "INSERT INTO FieldValues (ManifestId, Id, ItemId, FieldDefinitionId, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, ?, 0, 0, ?, ?, 0)", &[json!(PERSONAL_MANIFEST_ID), json!(crate::vault_sync::db::new_id()), json!(ITEM_A), json!(def), json!(value), json!(t(1)), json!(t(1))]);
+ }
+ },
+ );
+ assert_eq!(live_field_values(merged), vec!["one", "pw-0", "two"]);
+}
diff --git a/core/rust/src/vault_sync/tests/merge_edge_cases.rs b/core/rust/src/vault_sync/tests/merge_edge_cases.rs
deleted file mode 100644
index 676fc4dc3..000000000
--- a/core/rust/src/vault_sync/tests/merge_edge_cases.rs
+++ /dev/null
@@ -1,528 +0,0 @@
-//! Merge edge cases run through the whole dirty-pull pipeline on the real client schema: both devices'
-//! SQLite canonicalized, merged per manifest, validated, materialized and inserted into a fresh database.
-
-use std::collections::HashMap;
-
-use rusqlite::Connection;
-use serde_json::{json, Value};
-
-use super::test_host::{complete_schema_sql, open_schema_db, query};
-use crate::common::encoding::base64_decode;
-use crate::sqlite_host::{self, SqlStatement};
-use crate::vault_codec::{self, CanonicalizeInput, CanonicalizedVault, CodecTableData, Manifest, ManifestSpec, MaterializeInput};
-use crate::vault_merge::{merge_canonical, CanonicalMergeInput};
-
-const PERSONAL: &str = "11111111-1111-4111-8111-111111111111";
-const SHARED: &str = "22222222-2222-4222-8222-222222222222";
-const SALT: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
-const ITEM: &str = "33333333-3333-4333-8333-333333333333";
-const TOTP_A: &str = "44444444-4444-4444-8444-444444444441";
-const TOTP_B: &str = "44444444-4444-4444-8444-444444444442";
-const PASSKEY: &str = "55555555-5555-4555-8555-555555555555";
-const FOLDER: &str = "66666666-6666-4666-8666-666666666666";
-const T0: &str = "2026-01-01 00:00:00.000";
-const T1: &str = "2026-01-02 00:00:00.000";
-const T2: &str = "2026-01-03 00:00:00.000";
-const T3: &str = "2026-01-04 00:00:00.000";
-
-/// What one merge produced: the materialized vault, or the reason the engine would drop the local changes.
-type MergeResult = Result<Connection, String>;
-
-/// A fresh client database; foreign keys are checked once after the inserts, like the engine's staging load.
-fn new_db() -> Connection {
- let db = open_schema_db(&complete_schema_sql());
- db.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();
- db
-}
-
-fn read_tables(conn: &Connection) -> Vec<CodecTableData> {
- let names: Vec<String> = query(conn, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name", &[]).unwrap().iter().map(|r| r["name"].as_str().unwrap().to_string()).collect();
- names.into_iter().filter(|name| !vault_codec::is_skip_table(name)).map(|name| CodecTableData { records: query(conn, &format!("SELECT * FROM \"{}\"", name), &[]).unwrap().into_iter().map(|row| row.into_iter().collect()).collect(), name }).collect()
-}
-
-fn schema_columns(conn: &Connection) -> HashMap<String, Vec<String>> {
- let sql = "SELECT m.name AS TableName, p.name AS ColumnName FROM sqlite_master m JOIN pragma_table_info(m.name) p WHERE m.type = 'table' ORDER BY m.name, p.cid";
- let mut out: HashMap<String, Vec<String>> = HashMap::new();
- for row in query(conn, sql, &[]).unwrap() {
- out.entry(row["TableName"].as_str().unwrap().to_string()).or_default().push(row["ColumnName"].as_str().unwrap().to_string());
- }
- out
-}
-
-fn canon(conn: &Connection) -> Result<CanonicalizedVault, String> {
- canon_manifests(conn, &[PERSONAL])
-}
-
-fn canon_manifests(conn: &Connection, manifest_ids: &[&str]) -> Result<CanonicalizedVault, String> {
- let manifests = manifest_ids.iter().map(|id| ManifestSpec { manifest_id: id.to_string(), manifest_salt: SALT.to_string(), name: None }).collect();
- vault_codec::canonicalize_from_sqlite(CanonicalizeInput { tables: read_tables(conn), canonicalized_at: "2026-02-01T00:00:00.000Z".to_string(), manifests, stamp_unstamped_into: None }).map_err(|e| format!("canonicalize: {}", e))
-}
-
-/// The engine's dirty pull (`vault_sync::merge`) minus HTTP and encryption: any `Err` is a merge the engine
-/// answers by keeping the server vault and dropping the local changes.
-fn merge_vaults(server: &Connection, local: &Connection) -> MergeResult {
- let server_side = canon(server)?;
- let local_side = canon(local)?;
- merge_canonical_sides(server_side.manifests.iter().map(|m| m.manifest.clone()).collect(), server_side.clone(), local_side)
-}
-
-fn merge_canonical_sides(server_manifests: Vec<Manifest>, server_side: CanonicalizedVault, local_side: CanonicalizedVault) -> MergeResult {
- let schema = schema_columns(&new_db());
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests,
- server_buckets: server_side.data_buckets.clone(),
- contentless_server_manifest_ids: vec![],
- local_manifests: local_side.manifests.iter().map(|m| m.manifest.clone()).collect(),
- local_buckets: local_side.data_buckets.clone(),
- schema_columns: schema.clone(),
- })
- .map_err(|e| format!("merge: {}", e))?;
-
- let mut manifests = Vec::new();
- let mut buckets = Vec::new();
- for entry in output.manifests {
- let validation = vault_codec::validate_manifest(&entry.manifest);
- if !validation.ok {
- return Err(format!("validation fallback: {}", validation.failed_rules.join(", ")));
- }
- for bucket in &entry.buckets {
- let validation = vault_codec::validate_data_bucket(bucket);
- if !validation.ok {
- return Err(format!("bucket validation fallback: {}", validation.failed_rules.join(", ")));
- }
- }
- manifests.push(entry.manifest);
- buckets.extend(entry.buckets);
- }
-
- let mut blobs: HashMap<String, Vec<u8>> = HashMap::new();
- for side in [&server_side, &local_side] {
- for manifest in &side.manifests {
- for (hash, blob) in &manifest.blobs {
- blobs.insert(hash.clone(), base64_decode(&blob.bytes_base64).unwrap());
- }
- }
- }
-
- let materialized = vault_codec::materialize_as_sqlite(MaterializeInput { manifests, data_buckets: buckets, schema_columns: schema.clone() }).map_err(|e| format!("materialize: {}", e))?;
- let db = new_db();
- for table in materialized.tables.iter().filter(|t| !t.records.is_empty() && schema.contains_key(&t.name)) {
- let statements: Vec<SqlStatement> = table
- .records
- .iter()
- .map(|row| {
- let mut columns: Vec<&String> = row.keys().collect();
- columns.sort();
- let params = columns.iter().map(|c| bind(&row[*c], &blobs)).collect();
- let quoted: Vec<String> = columns.iter().map(|c| format!("\"{}\"", c)).collect();
- SqlStatement { sql: format!("INSERT INTO \"{}\" ({}) VALUES ({})", table.name, quoted.join(", "), vec!["?"; columns.len()].join(", ")), params }
- })
- .collect();
- sqlite_host::exec(&db, &statements).map_err(|e| format!("insert {}: {}", table.name, e))?;
- }
- let violations = query(&db, "PRAGMA foreign_key_check", &[]).unwrap();
- if !violations.is_empty() {
- return Err(format!("foreign key check: {:?}", violations));
- }
- Ok(db)
-}
-
-/// Bind like `vault_sync::db::bind_value`: a blob reference becomes its bytes, NULL when not at hand.
-fn bind(value: &Value, blobs: &HashMap<String, Vec<u8>>) -> Value {
- match value.get("__blobRef").and_then(Value::as_str) {
- Some(hash) => blobs.get(hash).map(|bytes| json!({ "__b64": crate::common::encoding::base64_encode(bytes) })).unwrap_or(Value::Null),
- None => value.clone(),
- }
-}
-
-fn exec(db: &Connection, sql: &str, params: &[Value]) {
- sqlite_host::exec(db, &[SqlStatement { sql: sql.to_string(), params: params.to_vec() }]).unwrap();
-}
-
-fn insert_item(db: &Connection, id: &str, name: &str, at: &str) {
- exec(db, "INSERT INTO Items (ManifestId, Id, Name, ItemType, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'Login', ?, ?, 0)", &[json!(PERSONAL), json!(id), json!(name), json!(T0), json!(at)]);
-}
-
-fn insert_totp(db: &Connection, id: &str, item: &str, secret: &str, at: &str, deleted: i64) {
- exec(db, "INSERT INTO TotpCodes (ManifestId, Id, ItemId, Name, SecretKey, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'code', ?, ?, ?, ?)", &[json!(PERSONAL), json!(id), json!(item), json!(secret), json!(T0), json!(at), json!(deleted)]);
-}
-
-fn insert_password(db: &Connection, item: &str, value: &str, at: &str) {
- exec(db, "INSERT INTO FieldValues (ManifestId, Id, ItemId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'login.password', ?, 0, 0, ?, ?, 0)", &[json!(PERSONAL), json!(crate::vault_sync::db::new_id()), json!(item), json!(value), json!(T0), json!(at)]);
-}
-
-/// A passkey with the given `PrivateKey` cell (TEXT or `{ __b64 }` bytes) and fixed byte columns.
-fn insert_passkey(db: &Connection, id: &str, item: &str, private_key: Value, display_name: &str, at: &str, deleted: i64) {
- exec(
- db,
- "INSERT INTO Passkeys (ManifestId, Id, ItemId, RpId, UserHandle, PublicKey, PrivateKey, PrfKey, DisplayName, AdditionalData, CredentialId, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'example.com', ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?)",
- &[json!(PERSONAL), json!(id), json!(item), user_handle(), json!("{\"kty\":\"EC\",\"x\":\"pub\"}"), private_key, prf_key(), json!(display_name), credential_id(), json!(T0), json!(at), json!(deleted)],
- );
-}
-
-fn user_handle() -> Value {
- json!({ "__b64": crate::common::encoding::base64_encode(&[0u8, 1, 2, 255, 254, 0, 7]) })
-}
-
-fn prf_key() -> Value {
- json!({ "__b64": crate::common::encoding::base64_encode(&(0u8..32).collect::<Vec<u8>>()) })
-}
-
-fn credential_id() -> Value {
- json!({ "__b64": crate::common::encoding::base64_encode(&[0x55u8; 16]) })
-}
-
-const JWK: &str = "{\"kty\":\"EC\",\"crv\":\"P-256\",\"d\":\"secret-d\",\"x\":\"x\",\"y\":\"y\"}";
-
-/// Both devices after their last common sync: one item with a password, a TOTP code and a passkey.
-fn synced_pair() -> (Connection, Connection) {
- let build = || {
- let db = new_db();
- insert_item(&db, ITEM, "Bank", T0);
- insert_password(&db, ITEM, "pw-0", T0);
- insert_totp(&db, TOTP_A, ITEM, "SECRET-A", T0, 0);
- insert_passkey(&db, PASSKEY, ITEM, json!(JWK), "me", T0, 0);
- db
- };
- let server = build();
- // Copy the server's rows, so both sides carry identical ids (the derived FieldValues id included).
- let local = new_db();
- for table in read_tables(&server).into_iter().filter(|t| !t.records.is_empty() && t.name != "__EFMigrationsHistory") {
- for row in table.records {
- let columns: Vec<&String> = row.keys().collect();
- let quoted: Vec<String> = columns.iter().map(|c| format!("\"{}\"", c)).collect();
- let sql = format!("INSERT INTO \"{}\" ({}) VALUES ({})", table.name, quoted.join(", "), vec!["?"; columns.len()].join(", "));
- exec(&local, &sql, &columns.iter().map(|c| row[*c].clone()).collect::<Vec<_>>());
- }
- }
- (server, local)
-}
-
-fn strings(db: &Connection, sql: &str) -> Vec<String> {
- let mut out: Vec<String> = query(db, sql, &[]).unwrap().iter().map(|r| r.values().map(|v| v.as_str().map(str::to_string).unwrap_or_else(|| v.to_string())).collect::<Vec<_>>().join("|")).collect();
- out.sort();
- out
-}
-
-fn live_totp_secrets(db: &Connection) -> Vec<String> {
- strings(db, "SELECT SecretKey FROM TotpCodes WHERE IsDeleted = 0")
-}
-
-fn live_passkeys(db: &Connection) -> Vec<String> {
- strings(db, "SELECT DisplayName FROM Passkeys WHERE IsDeleted = 0")
-}
-
-fn item_state(db: &Connection) -> Vec<String> {
- strings(db, &format!("SELECT Name || '|' || IsDeleted || '|' || ifnull(DeletedAt, '-') || '|' || ifnull(FolderId, '-') AS s FROM Items WHERE Id = '{}'", ITEM))
-}
-
-/*
- * Concurrent edits.
- */
-
-#[test]
-fn totp_added_on_both_devices_keeps_both() {
- let (server, local) = synced_pair();
- insert_totp(&server, TOTP_B, ITEM, "SERVER-NEW", T1, 0);
- insert_totp(&local, "44444444-4444-4444-8444-444444444443", ITEM, "LOCAL-NEW", T2, 0);
- let merged = merge_vaults(&server, &local).unwrap();
- assert_eq!(live_totp_secrets(&merged), vec!["LOCAL-NEW", "SECRET-A", "SERVER-NEW"]);
-}
-
-#[test]
-fn totp_deleted_on_one_device_while_the_other_edits_the_item() {
- let (server, local) = synced_pair();
- exec(&server, "UPDATE TotpCodes SET IsDeleted = 1, UpdatedAt = ? WHERE Id = ?", &[json!(T1), json!(TOTP_A)]);
- exec(&local, "UPDATE Items SET Name = 'Bank renamed', UpdatedAt = ? WHERE Id = ?", &[json!(T2), json!(ITEM)]);
- let merged = merge_vaults(&server, &local).unwrap();
- assert!(live_totp_secrets(&merged).is_empty(), "the delete of the code stands");
- assert_eq!(item_state(&merged), vec!["Bank renamed|0|-|-"]);
-}
-
-#[test]
-fn totp_added_locally_while_the_server_edits_the_item_and_password() {
- let (server, local) = synced_pair();
- exec(&server, "UPDATE Items SET Name = 'Server name', UpdatedAt = ? WHERE Id = ?", &[json!(T3), json!(ITEM)]);
- exec(&server, "UPDATE FieldValues SET Value = 'pw-server', UpdatedAt = ? WHERE FieldKey = 'login.password'", &[json!(T3)]);
- insert_totp(&local, TOTP_B, ITEM, "LOCAL-NEW", T1, 0);
- let merged = merge_vaults(&server, &local).unwrap();
- assert_eq!(live_totp_secrets(&merged), vec!["LOCAL-NEW", "SECRET-A"]);
- assert_eq!(strings(&merged, "SELECT Value FROM FieldValues WHERE IsDeleted = 0"), vec!["pw-server"]);
-}
-
-#[test]
-fn passkey_created_on_mobile_while_the_extension_edits_the_item() {
- let (server, local) = synced_pair();
- exec(&server, "UPDATE Items SET Name = 'Extension edit', UpdatedAt = ? WHERE Id = ?", &[json!(T2), json!(ITEM)]);
- exec(&server, "UPDATE FieldValues SET Value = 'pw-ext', UpdatedAt = ? WHERE FieldKey = 'login.password'", &[json!(T2)]);
- insert_passkey(&local, "55555555-5555-4555-8555-555555555556", ITEM, json!(JWK), "mobile", T1, 0);
- let merged = merge_vaults(&server, &local).unwrap();
- assert_eq!(live_passkeys(&merged), vec!["me", "mobile"]);
- assert_eq!(item_state(&merged), vec!["Extension edit|0|-|-"]);
- assert_eq!(strings(&merged, "SELECT Value FROM FieldValues WHERE IsDeleted = 0"), vec!["pw-ext"]);
-}
-
-#[test]
-fn passkey_soft_deleted_versus_updated_is_last_write_wins_both_ways() {
- for (delete_at, rename_at, expected) in [(T2, T1, vec![]), (T1, T2, vec!["renamed".to_string()])] {
- let (server, local) = synced_pair();
- exec(&server, "UPDATE Passkeys SET IsDeleted = 1, UpdatedAt = ? WHERE Id = ?", &[json!(delete_at), json!(PASSKEY)]);
- exec(&local, "UPDATE Passkeys SET DisplayName = 'renamed', UpdatedAt = ? WHERE Id = ?", &[json!(rename_at), json!(PASSKEY)]);
- let merged = merge_vaults(&server, &local).unwrap();
- assert_eq!(live_passkeys(&merged), expected, "delete at {}, rename at {}", delete_at, rename_at);
- }
-}
-
-#[test]
-fn permanent_delete_on_one_side_and_a_later_totp_on_the_other_keeps_the_item_whole() {
- let (server, local) = synced_pair();
- // Server device purges the item: tombstone plus child tombstones, as the pruner writes them.
- exec(&server, "UPDATE Items SET IsDeleted = 1, DeletedAt = ?, UpdatedAt = ? WHERE Id = ?", &[json!(T1), json!(T1), json!(ITEM)]);
- for table in ["TotpCodes", "Passkeys", "FieldValues"] {
- exec(&server, &format!("UPDATE {} SET IsDeleted = 1, UpdatedAt = ? WHERE ItemId = ?", table), &[json!(T1), json!(ITEM)]);
- }
- insert_totp(&local, TOTP_B, ITEM, "LOCAL-NEW", T2, 0);
- let merged = merge_vaults(&server, &local).unwrap();
- assert_eq!(item_state(&merged), vec!["Bank|0|-|-"]);
- assert_eq!(live_totp_secrets(&merged), vec!["LOCAL-NEW", "SECRET-A"]);
- assert_eq!(live_passkeys(&merged), vec!["me"]);
- assert_eq!(strings(&merged, "SELECT Value FROM FieldValues WHERE IsDeleted = 0"), vec!["pw-0"]);
-}
-
-#[test]
-fn permanent_delete_newer_than_the_other_sides_edits_drops_the_item_and_its_secrets() {
- let (server, local) = synced_pair();
- insert_totp(&local, TOTP_B, ITEM, "LOCAL-NEW", T1, 0);
- exec(&server, "UPDATE Items SET IsDeleted = 1, DeletedAt = ?, UpdatedAt = ? WHERE Id = ?", &[json!(T2), json!(T2), json!(ITEM)]);
- let merged = merge_vaults(&server, &local).unwrap();
- assert_eq!(strings(&merged, "SELECT IsDeleted FROM Items"), vec!["1"]);
- assert!(live_totp_secrets(&merged).is_empty() && live_passkeys(&merged).is_empty());
-}
-
-#[test]
-fn trash_on_one_side_and_a_passkey_on_the_other_keeps_both_changes() {
- let (server, local) = synced_pair();
- exec(&server, "UPDATE Items SET DeletedAt = ?, UpdatedAt = ? WHERE Id = ?", &[json!(T2), json!(T2), json!(ITEM)]);
- insert_passkey(&local, "55555555-5555-4555-8555-555555555556", ITEM, json!(JWK), "mobile", T1, 0);
- let merged = merge_vaults(&server, &local).unwrap();
- assert_eq!(item_state(&merged), vec![format!("Bank|0|{}|-", T2)]);
- assert_eq!(live_passkeys(&merged), vec!["me", "mobile"]);
-}
-
-#[test]
-fn restore_from_trash_newer_than_a_trash_on_the_other_side_wins() {
- let (server, local) = synced_pair();
- exec(&server, "UPDATE Items SET DeletedAt = ?, UpdatedAt = ? WHERE Id = ?", &[json!(T1), json!(T1), json!(ITEM)]);
- exec(&local, "UPDATE Items SET DeletedAt = NULL, UpdatedAt = ? WHERE Id = ?", &[json!(T2), json!(ITEM)]);
- let merged = merge_vaults(&server, &local).unwrap();
- assert_eq!(item_state(&merged), vec!["Bank|0|-|-"]);
-}
-
-#[test]
-fn item_moved_into_a_folder_the_other_side_deleted_still_merges() {
- let (server, local) = synced_pair();
- for db in [&server, &local] {
- exec(db, "INSERT INTO Folders (ManifestId, Id, Name, CreatedAt, UpdatedAt, IsDeleted, Weight) VALUES (?, ?, 'Work', ?, ?, 0, 0)", &[json!(PERSONAL), json!(FOLDER), json!(T0), json!(T0)]);
- }
- exec(&server, "UPDATE Folders SET IsDeleted = 1, UpdatedAt = ? WHERE Id = ?", &[json!(T1), json!(FOLDER)]);
- exec(&local, "UPDATE Items SET FolderId = ?, UpdatedAt = ? WHERE Id = ?", &[json!(FOLDER), json!(T2), json!(ITEM)]);
- let merged = merge_vaults(&server, &local).unwrap();
- assert_eq!(item_state(&merged), vec![format!("Bank|0|-|{}", FOLDER)]);
- assert_eq!(live_totp_secrets(&merged), vec!["SECRET-A"]);
-}
-
-/*
- * Spellings older or other clients write.
- */
-
-#[test]
-fn timestamp_spellings_compare_as_instants() {
- // Space vs 'T', a Z suffix, an offset, and seven-digit .NET precision.
- for (server_at, local_at, local_wins) in [
- ("2026-01-02 00:00:00.000", "2026-01-02T00:00:00.001Z", true),
- ("2026-01-02T00:00:00.000Z", "2026-01-02 00:00:00.000", false),
- ("2026-01-02 00:00:00.123", "2026-01-02 00:00:00.1234567", true),
- ("2026-01-02 02:00:00.000", "2026-01-02T03:00:00+02:00", false),
- ("2026-01-02 00:00:00", "2026-01-02 00:00:01", true),
- ] {
- let (server, local) = synced_pair();
- exec(&server, "UPDATE TotpCodes SET SecretKey = 'SERVER', UpdatedAt = ? WHERE Id = ?", &[json!(server_at), json!(TOTP_A)]);
- exec(&local, "UPDATE TotpCodes SET SecretKey = 'LOCAL', UpdatedAt = ? WHERE Id = ?", &[json!(local_at), json!(TOTP_A)]);
- let merged = merge_vaults(&server, &local).unwrap();
- assert_eq!(live_totp_secrets(&merged), vec![if local_wins { "LOCAL" } else { "SERVER" }], "server {} vs local {}", server_at, local_at);
- }
-}
-
-#[test]
-fn uppercase_guids_from_an_older_client_match_their_lowercase_twins() {
- let (server, local) = synced_pair();
- // The local client wrote its rows in uppercase (as an older iOS build did).
- for table in ["Items", "TotpCodes", "Passkeys", "FieldValues"] {
- exec(&local, &format!("UPDATE {} SET Id = upper(Id), ManifestId = upper(ManifestId)", table), &[]);
- }
- for table in ["TotpCodes", "Passkeys", "FieldValues"] {
- exec(&local, &format!("UPDATE {} SET ItemId = upper(ItemId)", table), &[]);
- }
- exec(&local, "UPDATE TotpCodes SET SecretKey = 'LOCAL', UpdatedAt = ?", &[json!(T2)]);
- insert_totp(&server, TOTP_B, ITEM, "SERVER-NEW", T1, 0);
- let merged = merge_vaults(&server, &local).unwrap();
- assert_eq!(live_totp_secrets(&merged), vec!["LOCAL", "SERVER-NEW"]);
- assert_eq!(live_passkeys(&merged), vec!["me"]);
- assert_eq!(strings(&merged, "SELECT COUNT(*) FROM Items"), vec!["1"]);
-}
-
-#[test]
-fn an_empty_string_folder_id_is_read_as_no_folder() {
- let (server, local) = synced_pair();
- exec(&local, "UPDATE Items SET FolderId = '', Name = 'Edited', UpdatedAt = ? WHERE Id = ?", &[json!(T2), json!(ITEM)]);
- let merged = merge_vaults(&server, &local);
- assert!(merged.is_ok(), "an empty FolderId must not fail the whole merge and drop the local edits: {:?}", merged.err());
-}
-
-/*
- * Passkey byte columns.
- */
-
-#[test]
-fn passkey_bytes_survive_merge_unchanged_with_text_and_blob_private_keys() {
- let blob_private_key = json!({ "__b64": crate::common::encoding::base64_encode(JWK.as_bytes()) });
- for (private_key, expected_type) in [(json!(JWK), "text"), (blob_private_key, "blob")] {
- let (server, local) = synced_pair();
- insert_passkey(&local, "55555555-5555-4555-8555-555555555556", ITEM, private_key.clone(), "mobile", T1, 0);
- // The server side touches the original passkey, so both a local-only and a server-won row cross the merge.
- exec(&server, "UPDATE Passkeys SET DisplayName = 'server', UpdatedAt = ? WHERE Id = ?", &[json!(T2), json!(PASSKEY)]);
- let merged = merge_vaults(&server, &local).unwrap();
- let rows = query(&merged, "SELECT DisplayName, typeof(PrivateKey) AS pt, CAST(PrivateKey AS TEXT) AS pk, hex(UserHandle) AS uh, hex(PrfKey) AS prf, hex(CredentialId) AS cid, typeof(UserHandle) AS ut FROM Passkeys ORDER BY DisplayName", &[]).unwrap();
- assert_eq!(rows.len(), 2);
- for row in &rows {
- assert_eq!(row["uh"], json!("000102FFFE0007"), "UserHandle bytes kept, a NUL included");
- assert_eq!(row["ut"], json!("blob"));
- assert_eq!(row["prf"], json!((0u8..32).map(|b| format!("{:02X}", b)).collect::<String>()));
- assert_eq!(row["cid"], json!("55".repeat(16)));
- assert_eq!(row["pk"], json!(JWK));
- }
- let mobile = rows.iter().find(|r| r["DisplayName"] == json!("mobile")).unwrap();
- assert_eq!(mobile["pt"], json!(expected_type), "the PrivateKey storage class is kept as the writer chose it");
- }
-}
-
-#[test]
-fn a_passkey_without_a_prf_key_merges_against_one_that_gained_it() {
- let (server, local) = synced_pair();
- exec(&server, "UPDATE Passkeys SET PrfKey = NULL, UpdatedAt = ? WHERE Id = ?", &[json!(T1), json!(PASSKEY)]);
- exec(&local, "UPDATE Passkeys SET DisplayName = 'local', UpdatedAt = ? WHERE Id = ?", &[json!(T2), json!(PASSKEY)]);
- let merged = merge_vaults(&server, &local).unwrap();
- // The local row wins as a whole, so its PrfKey comes along.
- assert_eq!(strings(&merged, "SELECT DisplayName || '|' || ifnull(hex(PrfKey), 'null') FROM Passkeys"), vec![format!("local|{}", (0u8..32).map(|b| format!("{:02X}", b)).collect::<String>())]);
-}
-
-/*
- * Rows real vaults carry that must never fail the whole merge.
- */
-
-#[test]
-fn an_unknown_column_on_a_totp_row_survives_a_local_win() {
- let (server, local) = synced_pair();
- exec(&local, "UPDATE TotpCodes SET SecretKey = 'LOCAL', UpdatedAt = ? WHERE Id = ?", &[json!(T2), json!(TOTP_A)]);
- let mut server_side = canon(&server).unwrap();
- let totps = server_side.manifests[0].manifest.tables.get_mut("TotpCodes").unwrap();
- totps[0].insert("FutureColumn".to_string(), json!("from a newer writer"));
- let local_side = canon(&local).unwrap();
- let server_manifests = server_side.manifests.iter().map(|m| m.manifest.clone()).collect();
- let schema = schema_columns(&new_db());
- let output = merge_canonical(CanonicalMergeInput {
- server_manifests,
- server_buckets: server_side.data_buckets.clone(),
- contentless_server_manifest_ids: vec![],
- local_manifests: local_side.manifests.iter().map(|m| m.manifest.clone()).collect(),
- local_buckets: local_side.data_buckets.clone(),
- schema_columns: schema,
- })
- .unwrap();
- let merged_totp = &output.manifests[0].manifest.tables["TotpCodes"][0];
- assert_eq!(merged_totp["SecretKey"], json!("LOCAL"));
- assert_eq!(merged_totp["FutureColumn"], json!("from a newer writer"));
-}
-
-#[test]
-fn an_orphan_totp_on_the_local_side_does_not_fail_the_whole_merge() {
- let (server, local) = synced_pair();
- // A code left behind by an older client under an item that is gone from both sides.
- insert_totp(&local, TOTP_B, "77777777-7777-4777-8777-777777777777", "ORPHAN", T1, 0);
- insert_totp(&local, "44444444-4444-4444-8444-444444444443", ITEM, "LOCAL-NEW", T1, 0);
- let merged = merge_vaults(&server, &local);
- let merged = merged.unwrap_or_else(|e| panic!("one orphan row dropped every local change: {}", e));
- assert!(live_totp_secrets(&merged).contains(&"LOCAL-NEW".to_string()));
-}
-
-#[test]
-fn an_item_pointing_at_a_folder_neither_side_has_does_not_fail_the_whole_merge() {
- let (server, local) = synced_pair();
- exec(&local, "UPDATE Items SET FolderId = ?, Name = 'Edited', UpdatedAt = ? WHERE Id = ?", &[json!(FOLDER), json!(T2), json!(ITEM)]);
- insert_totp(&local, TOTP_B, ITEM, "LOCAL-NEW", T2, 0);
- let merged = merge_vaults(&server, &local);
- assert!(merged.is_ok(), "a dangling FolderId fails validation and drops the new TOTP code: {:?}", merged.err());
-}
-
-#[test]
-fn a_merged_vault_merges_onto_itself_without_changes() {
- let (server, local) = synced_pair();
- insert_totp(&local, TOTP_B, ITEM, "LOCAL-NEW", T2, 0);
- insert_passkey(&local, "55555555-5555-4555-8555-555555555556", ITEM, json!(JWK), "mobile", T1, 0);
- let merged = merge_vaults(&server, &local).unwrap();
- let again = merge_vaults(&merged, &merged).unwrap();
- for sql in ["SELECT Id, SecretKey, UpdatedAt FROM TotpCodes", "SELECT Id, hex(UserHandle), hex(PrfKey), PrivateKey, UpdatedAt FROM Passkeys", "SELECT Id, Value FROM FieldValues"] {
- assert_eq!(strings(&merged, sql), strings(&again, sql), "{}", sql);
- }
-}
-
-#[test]
-fn an_orphan_field_value_on_the_local_side_does_not_fail_the_whole_merge() {
- let (server, local) = synced_pair();
- insert_password(&local, "77777777-7777-4777-8777-777777777777", "orphan", T1);
- insert_totp(&local, TOTP_B, ITEM, "LOCAL-NEW", T1, 0);
- let merged = merge_vaults(&server, &local).unwrap_or_else(|e| panic!("one orphan field value dropped every local change: {}", e));
- assert_eq!(live_totp_secrets(&merged), vec!["LOCAL-NEW", "SECRET-A"]);
-}
-
-#[test]
-fn an_item_tag_without_its_tag_does_not_fail_the_whole_merge() {
- let (server, local) = synced_pair();
- exec(&local, "INSERT INTO ItemTags (ManifestId, ItemId, TagId, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, '88888888-8888-4888-8888-888888888888', ?, ?, 0)", &[json!(PERSONAL), json!(ITEM), json!(T1), json!(T1)]);
- insert_totp(&local, TOTP_B, ITEM, "LOCAL-NEW", T1, 0);
- let merged = merge_vaults(&server, &local).unwrap_or_else(|e| panic!("one dangling item tag dropped every local change: {}", e));
- assert_eq!(live_totp_secrets(&merged), vec!["LOCAL-NEW", "SECRET-A"]);
-}
-
-#[test]
-fn a_local_vault_with_a_dangling_folder_id_can_still_be_pushed() {
- let (_, local) = synced_pair();
- exec(&local, "UPDATE Items SET FolderId = ? WHERE Id = ?", &[json!(FOLDER), json!(ITEM)]);
- let canonical = canon(&local).unwrap();
- let validation = vault_codec::validate_manifest(&canonical.manifests[0].manifest);
- assert!(validation.ok, "push refused: {:?}", validation.failed_rules);
-}
-
-#[test]
-#[ignore = "a custom field whose FieldDefinition is gone fails validation (fieldvalue-fielddef-fk-broken) and drops all local changes; nulling the id instead would collapse every such field of the item into one row"]
-fn custom_fields_whose_definitions_are_gone_do_not_fail_the_whole_merge() {
- let (server, local) = synced_pair();
- for (def, value) in [("99999999-9999-4999-8999-999999999991", "one"), ("99999999-9999-4999-8999-999999999992", "two")] {
- exec(&local, "INSERT INTO FieldValues (ManifestId, Id, ItemId, FieldDefinitionId, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, ?, 0, 0, ?, ?, 0)", &[json!(PERSONAL), json!(crate::vault_sync::db::new_id()), json!(ITEM), json!(def), json!(value), json!(T1), json!(T1)]);
- }
- let merged = merge_vaults(&server, &local).unwrap_or_else(|e| panic!("{}", e));
- assert_eq!(strings(&merged, "SELECT Value FROM FieldValues WHERE IsDeleted = 0"), vec!["one", "pw-0", "two"]);
-}
-
-#[test]
-#[ignore = "a move to another manifest leaves a tombstone newer than the other device's offline TOTP add, so the delete stands and the new code is dropped from both manifests"]
-fn totp_added_offline_survives_the_other_device_moving_the_item_to_a_shared_vault() {
- let (server, local) = synced_pair();
- insert_totp(&local, TOTP_B, ITEM, "LOCAL-NEW", T1, 0);
- // The other device moves the item to the shared vault later; the schema trigger carries its rows along.
- exec(&server, "UPDATE Items SET ManifestId = ?, UpdatedAt = ? WHERE Id = ? AND ManifestId = ?", &[json!(SHARED), json!(T2), json!(ITEM), json!(PERSONAL)]);
- let server_side = canon_manifests(&server, &[PERSONAL, SHARED]).unwrap();
- let local_side = canon_manifests(&local, &[PERSONAL, SHARED]).unwrap();
- let merged = merge_canonical_sides(server_side.manifests.iter().map(|m| m.manifest.clone()).collect(), server_side.clone(), local_side).unwrap_or_else(|e| panic!("{}", e));
- assert!(live_totp_secrets(&merged).contains(&"LOCAL-NEW".to_string()), "codes now: {:?}", strings(&merged, "SELECT ManifestId || ':' || SecretKey FROM TotpCodes WHERE IsDeleted = 0"));
-}
diff --git a/core/rust/src/vault_sync/tests/migration.rs b/core/rust/src/vault_sync/tests/migration.rs
new file mode 100644
index 000000000..a8fcbb6e1
--- /dev/null
+++ b/core/rust/src/vault_sync/tests/migration.rs
@@ -0,0 +1,169 @@
+//! The storage-format migrations: the one-time account upgrade of a legacy sqlite-blob account, and the schema
+//! rebuild of a vault written by an older client.
+
+use serde_json::json;
+
+use super::fake_server::{decrypt_manifest, FakeServer};
+use super::test_host::{open_schema_db, TestHost};
+use super::{insert_item, latest_migration_id, legacy_device, state, ATTACHMENT, ITEM_A, ITEM_B, PERSONAL_MANIFEST_ID};
+use crate::crypto;
+use crate::vault_codec;
+
+#[test]
+fn legacy_account_without_vault_key_reports_the_manifest_migration() {
+ let unlock_key = crypto::generate_key_base64();
+ let server = FakeServer::new();
+ server.borrow_mut().publish(&open_schema_db(), &unlock_key, &vault_codec::generate_manifest_salt(), 3);
+ let mut host = legacy_device(&unlock_key, &server, |_| {});
+
+ let result = host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["manifestMigrationRequired"], true);
+ assert_eq!(host.run("migrationStatus")["kind"], "storageFormatUpgrade");
+}
+
+#[test]
+fn manifest_migration_generates_the_key_hierarchy_and_pushes() {
+ let unlock_key = crypto::generate_key_base64();
+ let server = FakeServer::new();
+ server.borrow_mut().publish_contentless_personal(3);
+ let mut host = legacy_device(&unlock_key, &server, |db| insert_item(db, ITEM_A, "Old item"));
+
+ let result = host.run("migrateManifest");
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["pushed"], true);
+ let new_key = host.vault_key.clone();
+ assert_ne!(new_key, unlock_key, "the host receives the new VEK through the store command");
+ let write = host.last_vault_write();
+ let account_keys = &write["migration"]["accountKeys"];
+ assert!(account_keys["encryptedAccountKey"].is_string(), "the migration push carries the key hierarchy");
+ assert!(crypto::unwrap_key(account_keys["encryptedAccountKey"].as_str().unwrap(), &unlock_key, crypto::aad::ACCOUNT_KEY).is_err(), "the legacy vault key does not wrap the Account Key directly");
+ let opened = crypto::open_account_key_chain(&unlock_key, account_keys["encryptedAccountKey"].as_str().unwrap(), account_keys["encryptedVek"].as_str().unwrap(), None).unwrap();
+ assert_eq!(*opened.vault_encryption_key, new_key);
+ let signed = crypto::signing::account_public_key_message(account_keys["accountPublicKey"].as_str().unwrap());
+ assert!(crypto::signing::verify(account_keys["signingPublicKey"].as_str().unwrap(), &signed, account_keys["accountPublicKeySignature"].as_str().unwrap()), "the upgrade push carries a signing key that signed the account public key");
+ assert!(host.state.contains_key(state::ENCRYPTED_ACCOUNT_KEY));
+ assert!(host.state.contains_key(state::ENCRYPTED_ACCOUNT_PRIVATE_KEY));
+ assert_eq!(host.state[state::ENCRYPTED_SIGNING_PRIVATE_KEY], account_keys["encryptedSigningPrivateKey"]);
+ assert_eq!(host.rekeyed_stores, 1, "the chain is cached before the vault is stored under the VEK");
+ assert!(!server.borrow().vault_key.is_null(), "the server holds the hierarchy from now on");
+ server.borrow().assert_converged(&host);
+}
+
+#[test]
+fn account_upgrade_decodes_base64_text_that_0_30_merges_left_in_blob_columns() {
+ let unlock_key = crypto::generate_key_base64();
+ let bytes = vec![0x00u8, 0xFF, 0xC3, 0x28, 0x10];
+ let text = crate::common::encoding::base64_encode(&bytes);
+ let server = FakeServer::new();
+ server.borrow_mut().publish_contentless_personal(3);
+ let mut host = legacy_device(&unlock_key, &server, |db| {
+ insert_item(db, ITEM_A, "Item");
+ db.execute("INSERT INTO Passkeys (ManifestId, Id, ItemId, RpId, UserHandle, PublicKey, PrivateKey, PrfKey, DisplayName, AdditionalData, CredentialId, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, 'bbbbbbbb-0000-4000-8000-000000000001', ?, 'example.com', ?, 'pub', 'priv', 'not base64!', 'Passkey', NULL, NULL, ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, ITEM_A, text, super::now(), super::now()]).unwrap();
+ db.execute("INSERT INTO Attachments (ManifestId, Id, ItemId, Filename, Blob, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'file.bin', ?, ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, ATTACHMENT, ITEM_A, text, super::now(), super::now()]).unwrap();
+ });
+
+ let result = host.run("migrateManifest");
+
+ assert_eq!(result["success"], true, "{} {:?}", result, host.logs);
+ let cell = |sql: &str| host.local.query_row(sql, [], |row| Ok((row.get::<_, String>(0)?, row.get_ref(1)?.as_bytes().map(<[u8]>::to_vec).ok()))).unwrap();
+ assert_eq!(cell("SELECT typeof(UserHandle), UserHandle FROM Passkeys"), ("blob".to_string(), Some(bytes.clone())));
+ assert_eq!(cell("SELECT typeof(Blob), Blob FROM Attachments"), ("blob".to_string(), Some(bytes)));
+ assert_eq!(cell("SELECT typeof(PrfKey), PrfKey FROM Passkeys"), ("text".to_string(), Some(b"not base64!".to_vec())), "text that is not base64 is kept as-is");
+}
+
+#[test]
+fn a_key_migration_push_is_refused_while_a_personal_blob_is_not_loaded() {
+ // The migration encrypts every personal blob again under the new key; one it has no bytes for would stay under
+ // the old key and never open again. The vault stays dirty and the migration runs again once the blob loads.
+ let unlock_key = crypto::generate_key_base64();
+ let server = FakeServer::new();
+ server.borrow_mut().publish_contentless_personal(3);
+ let mut host = legacy_device(&unlock_key, &server, |db| {
+ insert_item(db, ITEM_A, "Old item");
+ db.execute("INSERT INTO Attachments (ManifestId, Id, ItemId, Filename, Blob, BlobHash, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'passport.pdf', NULL, 'hash-of-a-blob-that-is-not-loaded', ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, ATTACHMENT, ITEM_A, super::now(), super::now()]).unwrap();
+ });
+
+ let result = host.run("migrateManifest");
+
+ assert_eq!(result["pushed"], false, "{}", result);
+ assert!(host.vault_writes().is_empty(), "nothing was written with the new key");
+ assert_eq!(host.vault_key, unlock_key, "and the session keeps its key");
+}
+
+#[test]
+fn schema_rebuild_of_a_stale_vault_pushes_without_touching_the_key_hierarchy() {
+ let vek = crypto::generate_key_base64();
+ let server = FakeServer::new();
+ server.borrow_mut().publish_contentless_personal(3);
+ let mut host = legacy_device(&vek, &server, |db| {
+ insert_item(db, ITEM_A, "Kept item");
+ // A vault written by an older client: same tables, older schema stamp (still past the frozen sqlite-blob chain).
+ db.execute_batch("DELETE FROM __EFMigrationsHistory; INSERT INTO __EFMigrationsHistory (MigrationId, ProductVersion) VALUES ('20250101000000_2.0.0-Stale', '9.0.0');").unwrap();
+ });
+ host.set_state(state::ENCRYPTED_ACCOUNT_KEY, json!("wrapped"));
+ let current_stamp = latest_migration_id(&open_schema_db());
+
+ assert_eq!(host.run("migrationStatus")["kind"], "schemaRebuild");
+ let result = host.run("migrateManifest");
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["pushed"], true);
+ assert_eq!(latest_migration_id(&host.local), current_stamp, "the local vault is rebuilt onto the current schema");
+ assert_eq!(host.item_names(), vec!["Kept item"]);
+ assert!(host.requests_to("VaultKey/Password").is_empty(), "a migrated account is not probed for a key hierarchy");
+ assert!(host.last_vault_write()["migration"].is_null(), "no key hierarchy is created");
+ assert_eq!(host.vault_key, vek, "the session key stays the VEK");
+}
+
+/// A device as a client predating the manifest storage format leaves it: a sqlite blob under the unlock key, no personal
+/// manifest id and no revision baseline, against a server that still holds the account as a legacy vault.
+fn pre_format_session(unlock_key: &str, dirty: bool) -> TestHost {
+ let server = FakeServer::new();
+ let server_db = open_schema_db();
+ insert_item(&server_db, ITEM_B, "Server item");
+ server.borrow_mut().publish_legacy(&server_db, unlock_key, 3);
+ server.borrow_mut().publish_contentless_personal(3);
+ let mut host = TestHost::new(unlock_key);
+ host.serve(&server);
+ insert_item(&host.local, ITEM_A, "Local item");
+ host.store_local_as_blob();
+ host.is_dirty = dirty;
+ host
+}
+
+#[test]
+fn manifest_migration_of_a_pre_format_session_pulls_the_server_vault_first() {
+ let unlock_key = crypto::generate_key_base64();
+ let mut host = pre_format_session(&unlock_key, false);
+
+ let result = host.run("migrateManifest");
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["pushed"], true);
+ assert_eq!(host.requests_to("Vault").iter().filter(|r| r.method == "GET").count(), 1, "the baseline is pulled once");
+ assert_eq!(host.state[state::VAULT_PERSONAL_MANIFEST_ID], PERSONAL_MANIFEST_ID);
+ assert_eq!(host.item_names(), vec!["Server item"], "a clean local vault is replaced by the server's, like a login does");
+ let write = host.last_vault_write();
+ assert_eq!(write["manifests"][0]["currentRevision"], 3, "the migration push names the revision the server holds");
+ assert!(write["migration"]["accountKeys"]["encryptedAccountKey"].is_string());
+ assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 4);
+}
+
+#[test]
+fn manifest_migration_of_a_dirty_pre_format_session_keeps_the_local_vault() {
+ let unlock_key = crypto::generate_key_base64();
+ let mut host = pre_format_session(&unlock_key, true);
+
+ let result = host.run("migrateManifest");
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["pushed"], true);
+ assert_eq!(host.item_names(), vec!["Local item"], "pending local changes are not thrown away");
+ let write = host.last_vault_write();
+ assert_eq!(write["manifests"][0]["currentRevision"], 3, "the baseline still comes from the server");
+ let manifest = decrypt_manifest(write["manifests"][0]["manifestBlob"].as_str().unwrap(), &host.vault_key, PERSONAL_MANIFEST_ID);
+ assert!(manifest.to_string().contains("Local item"), "the push carries the local changes");
+}
diff --git a/core/rust/src/vault_sync/tests/mod.rs b/core/rust/src/vault_sync/tests/mod.rs
index b359a8f30..1dfb2c876 100644
--- a/core/rust/src/vault_sync/tests/mod.rs
+++ b/core/rust/src/vault_sync/tests/mod.rs
@@ -1,1078 +1,135 @@
-//! Engine tests: the whole sync driven through the command loop against a real SQLite host.
-
+//! Engine tests: the whole sync driven through the command loop, one real SQLite host per device against an
+//! in-process model of the v2 server. A test reads as "the server holds X, the device does Y, sync, assert".
+//!
+//! `fake_server` is the server, `test_host` is a device, this module holds the row fixtures they share.
+
+mod blobs;
+mod fake_server;
+mod grants;
mod item_move;
-mod merge_edge_cases;
+mod keys;
+mod merge;
+mod migration;
+mod sync;
mod test_host;
-mod unloaded_blobs;
-mod unsupported_grant;
-
-use std::collections::HashMap;
-use serde_json::{json, Value};
+use rusqlite::Connection;
+use serde_json::json;
-use self::test_host::{query, TestHost};
+use self::fake_server::{FakeServer, Server};
+use self::test_host::{open_schema_db, query, TestHost};
use crate::crypto;
-use crate::vault_codec::{self, CanonicalizeInput, CodecTableData, ManifestSpec};
-use crate::vault_sync::session::SyncSession;
+use crate::vault_codec;
use crate::vault_sync::state;
-use crate::vault_sync::types::Command;
-
-const PERSONAL_MANIFEST_ID: &str = "11111111-1111-4111-8111-111111111111";
-const USERNAME: &str = "tester";
-
-fn request(operation: &str, key: &str, dirty: bool, mutation_sequence: u64) -> String {
- json!({
- "operation": operation,
- "username": USERNAME,
- "encryptionKey": key,
- "isDirty": dirty,
- "mutationSequence": mutation_sequence,
- "dirtyScopes": if dirty { vec!["Main"] } else { vec![] },
- "privateEmailDomains": ["private.io"],
- "minServerVersion": "0.12.0-dev",
- })
- .to_string()
-}
-
-fn insert_item(conn: &rusqlite::Connection, id: &str, name: &str, manifest_id: &str) {
- let now = crate::common::timestamp::now_vault_datetime();
- conn.execute(
- "INSERT INTO Items (Id, ManifestId, Name, ItemType, FolderId, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'Login', NULL, ?, ?, 0)",
- rusqlite::params![id, manifest_id, name, now, now],
- )
- .unwrap();
-}
-
-/// Give a manifest the mail delivery keypair every written vault carries.
-fn insert_delivery_key(conn: &rusqlite::Connection, manifest_id: &str) {
- let now = crate::common::timestamp::now_vault_datetime();
- conn.execute(
- "INSERT INTO EncryptionKeys (Id, ManifestId, PublicKey, PrivateKey, IsPrimary, CreatedAt, UpdatedAt, IsDeleted) VALUES ('dddddddd-0000-4000-8000-000000000001', ?, 'public', 'private', 1, ?, ?, 0)",
- rusqlite::params![manifest_id, now, now],
- )
- .unwrap();
-}
-
-/// The active delivery keys of a manifest in a connection.
-fn active_delivery_keys(conn: &rusqlite::Connection, manifest_id: &str) -> Vec<String> {
- query(conn, "SELECT PublicKey FROM EncryptionKeys WHERE ManifestId = ? AND IsPrimary = 1 AND IsDeleted = 0", &[json!(manifest_id)]).unwrap().iter().map(|r| r["PublicKey"].as_str().unwrap().to_string()).collect()
-}
-
-/// Read every user table of a connection into the codec's input shape.
-fn read_tables(conn: &rusqlite::Connection) -> Vec<CodecTableData> {
- let names: Vec<String> = query(conn, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name", &[]).unwrap().iter().map(|r| r["name"].as_str().unwrap().to_string()).collect();
- names
- .into_iter()
- .map(|name| CodecTableData { records: query(conn, &format!("SELECT * FROM \"{}\"", name), &[]).unwrap().into_iter().map(|row| row.into_iter().collect()).collect(), name })
- .collect()
-}
-
-/// A server-side snapshot of the given database as a manifest-v1 personal manifest encrypted under `vek`.
-fn snapshot_of(conn: &rusqlite::Connection, vek: &str, revision: i64, salt: &str) -> (Value, Value) {
- let canonicalized = vault_codec::canonicalize_from_sqlite(CanonicalizeInput {
- tables: read_tables(conn),
- canonicalized_at: "2026-09-11T00:00:00.000Z".to_string(),
- manifests: vec![ManifestSpec { manifest_id: PERSONAL_MANIFEST_ID.to_string(), manifest_salt: salt.to_string(), name: None }],
- stamp_unstamped_into: None,
- })
- .unwrap();
- let manifest_json = serde_json::to_string(&canonicalized.manifests[0].manifest).unwrap();
- let blob = crypto::symmetric_encrypt_bytes_with_aad(&vault_codec::pack_payload(&manifest_json).unwrap(), vek, &crypto::aad::manifest(PERSONAL_MANIFEST_ID)).unwrap();
- let ciphertext_hash = vault_codec::compute_ciphertext_hash(&blob);
- let mut buckets = Vec::new();
- for bucket in &canonicalized.data_buckets {
- let bucket_json = serde_json::to_string(bucket).unwrap();
- let bucket_blob = crypto::symmetric_encrypt_bytes_with_aad(&vault_codec::pack_payload(&bucket_json).unwrap(), vek, &crypto::aad::bucket(&bucket.manifest_id, &bucket.category)).unwrap();
- buckets.push(json!({ "manifestId": bucket.manifest_id, "category": bucket.category, "blob": bucket_blob, "ciphertextHash": vault_codec::compute_ciphertext_hash(&bucket_blob), "revision": revision }));
- }
- let status = json!({
- "clientVersionSupported": true,
- "serverVersion": "0.31.0",
- "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": revision }],
- "bucketRevisions": canonicalized.data_buckets.iter().map(|b| json!({ "manifestId": b.manifest_id, "category": b.category, "revision": revision })).collect::<Vec<_>>(),
- "personalManifestId": PERSONAL_MANIFEST_ID,
- "srpSalt": "salt",
- "capabilities": { "sharing": "on" },
- });
- let vault = json!({
- "storageFormat": "manifest",
- "personalManifestId": PERSONAL_MANIFEST_ID,
- "manifests": [{ "manifestId": PERSONAL_MANIFEST_ID, "blob": blob, "ciphertextHash": ciphertext_hash, "revision": revision, "blobReferences": [], "canAdminister": true, "keyType": "account-key" }],
- "buckets": buckets,
- "emailRouting": { "privateEmailDomainList": ["private.io"], "publicEmailDomainList": [], "hiddenPrivateEmailDomainList": [], "emailAddressList": [] },
- });
- (status, vault)
-}
-
-fn item_names(conn: &rusqlite::Connection) -> Vec<String> {
- let mut names: Vec<String> = query(conn, "SELECT Name FROM Items WHERE IsDeleted = 0", &[]).unwrap().iter().map(|r| r["Name"].as_str().unwrap().to_string()).collect();
- names.sort();
- names
-}
-
-#[test]
-fn fresh_client_pulls_and_materializes_the_server_vault() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
-
- // The server holds a vault with one item; the client has an empty database and no baselines.
- let server_db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);
- let (status, vault) = snapshot_of(&server_db, &vek, 7, &vault_codec::generate_manifest_salt());
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
-
- let session = SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap();
- let result = host.drive(&session);
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["hasNewVault"], true);
- assert_eq!(result["vaultChanged"], true);
- assert_eq!(result["pulledRevision"], 7);
- assert_eq!(result["serverVersion"], "0.31.0");
- assert_eq!(result["manifestMigrationRequired"], false);
- assert_eq!(item_names(&host.local), vec!["Server item"]);
- assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 7);
- assert_eq!(host.state[state::VAULT_PERSONAL_MANIFEST_ID], PERSONAL_MANIFEST_ID);
- assert!(host.state[state::VAULT_CONTENT_FINGERPRINTS].as_object().unwrap().contains_key(&format!("manifest:{}", PERSONAL_MANIFEST_ID)));
- assert!(host.requests_to("Vault").len() == 1);
-}
-
-/// The status and snapshot of an account whose personal manifest the server holds without content.
-fn contentless_personal_snapshot(revision: i64) -> (Value, Value) {
- let status = json!({
- "clientVersionSupported": true,
- "serverVersion": "0.31.0",
- "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": revision }],
- "bucketRevisions": [],
- "personalManifestId": PERSONAL_MANIFEST_ID,
- "srpSalt": "salt",
- "capabilities": {},
- });
- let vault = json!({
- "storageFormat": "manifest",
- "personalManifestId": PERSONAL_MANIFEST_ID,
- "manifests": [{ "manifestId": PERSONAL_MANIFEST_ID, "blob": null, "ciphertextHash": null, "revision": revision, "blobReferences": [], "canAdminister": false, "keyType": "account-key" }],
- "buckets": [],
- "emailRouting": { "privateEmailDomainList": ["private.io"], "publicEmailDomainList": [], "hiddenPrivateEmailDomainList": [], "emailAddressList": [] },
- });
- (status, vault)
-}
-
-#[test]
-fn new_account_starts_from_an_empty_vault_and_writes_its_first_revision() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- let (status, vault) = contentless_personal_snapshot(0);
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 1 }], "bucketRevisions": [], "missingBlobHashes": [] }));
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["hasNewVault"], true);
- assert!(item_names(&host.local).is_empty());
- let posts: Vec<_> = host.requests_to("Vault").into_iter().filter(|r| r.method == "POST").collect();
- assert_eq!(posts.len(), 1, "the new vault is written once");
- let body = posts[0].body.as_ref().unwrap();
- assert_eq!(body["manifests"][0]["manifestId"], PERSONAL_MANIFEST_ID);
- assert_eq!(body["manifests"][0]["currentRevision"], 0);
- assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 1);
- assert!(!host.is_dirty);
-
- // The written manifest opens with the VEK and names the personal manifest.
- let blob = body["manifests"][0]["manifestBlob"].as_str().unwrap();
- let plain = crypto::symmetric_decrypt_bytes_with_aad(&crate::common::encoding::base64_decode(blob).unwrap(), &vek, &crypto::aad::manifest(PERSONAL_MANIFEST_ID)).unwrap();
- let manifest: Value = serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap();
- assert_eq!(manifest["manifestId"], PERSONAL_MANIFEST_ID);
- assert_eq!(manifest["manifestSalt"], host.state[state::VAULT_MANIFEST_SALT]);
-}
-
-#[test]
-fn new_account_whose_first_write_fails_retries_it_on_the_next_sync() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- let (status, vault) = contentless_personal_snapshot(0);
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
-
- // No response for the write: the vault stays dirty and the sync goes offline on the stored empty vault.
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
- assert_eq!(result["success"], true, "{}", result);
- assert!(host.is_dirty, "the unwritten new vault stays dirty");
-
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 1 }], "bucketRevisions": [], "missingBlobHashes": [] }));
- let mutation_sequence = host.mutation_sequence;
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, true, mutation_sequence)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- let posts: Vec<_> = host.requests_to("Vault").into_iter().filter(|r| r.method == "POST").collect();
- assert_eq!(posts.last().unwrap().body.as_ref().unwrap()["manifests"][0]["currentRevision"], 0);
- assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 1);
- assert!(!host.is_dirty);
-}
-#[test]
-fn contentless_personal_manifest_past_revision_zero_is_refused() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- let (status, vault) = contentless_personal_snapshot(4);
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
+pub(crate) use crate::vault_codec::test_support::{PERSONAL_MANIFEST_ID, SHARED_MANIFEST_ID};
+pub const ITEM_A: &str = "aaaaaaaa-0000-4000-8000-000000000001";
+pub const ITEM_B: &str = "aaaaaaaa-0000-4000-8000-000000000002";
+pub const ITEM_C: &str = "aaaaaaaa-0000-4000-8000-000000000003";
+pub const LOGO: &str = "bbbbbbbb-0000-4000-8000-000000000001";
+pub const ATTACHMENT: &str = "cccccccc-0000-4000-8000-000000000001";
- assert_eq!(result["success"], false, "{}", result);
- assert!(host.requests_to("Vault").iter().all(|r| r.method != "POST"), "nothing is written over a damaged manifest");
- assert!(host.store_calls.is_empty());
+/// The current instant in the vault's row timestamp format.
+pub fn now() -> String {
+ crate::common::timestamp::now_vault_datetime()
}
-#[test]
-fn unknown_storage_format_is_refused_not_read_as_legacy() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- let server_db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);
- let (status, mut vault) = snapshot_of(&server_db, &vek, 7, &vault_codec::generate_manifest_salt());
- vault["storageFormat"] = json!("manifest-v2");
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- assert_eq!(result["success"], false, "{}", result);
- assert_eq!(result["logoutReason"], "vaultVersionIncompatible", "an app too old for the format is told to update");
- assert!(item_names(&host.local).is_empty(), "nothing is materialized");
- assert!(host.requests_to("Vault").iter().all(|r| r.method == "GET"), "no legacy migration push");
+/// `days` days before now in the vault's row timestamp format.
+pub fn days_ago(days: i64) -> String {
+ (chrono::Utc::now() - chrono::Duration::days(days)).format(crate::common::timestamp::VAULT_DATETIME).to_string()
}
-#[test]
-fn clean_client_in_sync_does_nothing() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));
- host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));
- host.store_local_as_blob();
- host.respond("GET", "Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 3 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));
-
- let session = SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap();
- let result = host.drive(&session);
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["hasNewVault"], false);
- assert!(host.requests_to("Vault").is_empty());
- assert!(host.store_calls.is_empty());
+pub fn insert_item(conn: &Connection, id: &str, name: &str) {
+ insert_item_at(conn, id, name, &now());
}
-#[test]
-fn dirty_client_pushes_only_what_changed() {
- let vek = crypto::generate_key_base64();
- let salt = vault_codec::generate_manifest_salt();
- let mut host = TestHost::new(&vek);
-
- // Pull once to establish baselines, then mutate locally and push.
- let server_db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);
- let (status, vault) = snapshot_of(&server_db, &vek, 7, &salt);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- host.respond("GET", "Status", status.clone());
- host.respond("GET", "Vault", vault);
- host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000002", "Local item", PERSONAL_MANIFEST_ID);
- host.store_local_as_blob();
- host.mutation_sequence = 1;
- host.is_dirty = true;
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 8 }], "bucketRevisions": [], "missingBlobHashes": [] }));
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, true, 1)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- let writes = host.requests_to("Vault");
- let posts: Vec<_> = writes.iter().filter(|r| r.method == "POST").collect();
- assert_eq!(posts.len(), 1, "one write expected");
- let body = posts[0].body.as_ref().unwrap();
- assert_eq!(body["username"], USERNAME);
- assert_eq!(body["manifests"].as_array().unwrap().len(), 1, "only the personal manifest changed");
- assert_eq!(body["manifests"][0]["currentRevision"], 7);
- assert_eq!(body["manifests"][0]["credentialsCount"], 2);
- assert_eq!(body["buckets"].as_array().unwrap().len(), 0, "unchanged buckets stay out of the write");
- assert_eq!(body["emailRouting"]["coveredManifestIds"][0], PERSONAL_MANIFEST_ID);
- assert_eq!(body["emailRouting"]["baseRevisions"][0]["revision"], 7, "routing carries the revision it was built from");
- assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 8);
- assert_eq!(host.mark_clean_calls, vec![1]);
- assert!(!host.is_dirty);
-
- // The written manifest decrypts with the VEK and carries both items.
- let blob = body["manifests"][0]["manifestBlob"].as_str().unwrap();
- let plain = crypto::symmetric_decrypt_bytes_with_aad(&crate::common::encoding::base64_decode(blob).unwrap(), &vek, &crypto::aad::manifest(PERSONAL_MANIFEST_ID)).unwrap();
- let manifest: Value = serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap();
- assert_eq!(manifest["tables"]["Items"].as_array().unwrap().len(), 2);
-}
-
-#[test]
-fn pushed_blobs_name_the_manifest_that_owns_them() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- let server_db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);
- let (status, vault) = snapshot_of(&server_db, &vek, 7, &vault_codec::generate_manifest_salt());
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
- host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- let now = crate::common::timestamp::now_vault_datetime();
- host.local
- .execute(
- "INSERT INTO Logos (ManifestId, Id, Source, FileData, Kind, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, 'example.com', ?, 'favicon', ?, ?, 0)",
- rusqlite::params![PERSONAL_MANIFEST_ID, "bbbbbbbb-0000-4000-8000-000000000001", vec![1u8, 2, 3, 4], now, now],
- )
- .unwrap();
- host.local.execute("UPDATE Items SET LogoId = 'bbbbbbbb-0000-4000-8000-000000000001'", []).unwrap();
- host.store_local_as_blob();
- host.mutation_sequence = 1;
- host.is_dirty = true;
- host.respond_with(Box::new(|method, path, body| (method == "POST" && path == "Vault/blobs/missing").then(|| (200, json!({ "missing": body.map(|b| b["hashes"].clone()).unwrap_or_default() })))));
- host.respond("POST", "Vault/blobs", json!({ "acceptedCount": 1 }));
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 8 }], "bucketRevisions": [], "missingBlobHashes": [] }));
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, true, 1)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- let missing_checks = host.requests_to("Vault/blobs/missing");
- assert_eq!(missing_checks.len(), 1);
- assert_eq!(missing_checks[0].body.as_ref().unwrap()["manifestId"], PERSONAL_MANIFEST_ID);
- let uploads = host.requests_to("Vault/blobs");
- assert_eq!(uploads.len(), 1);
- let upload = uploads[0].body.as_ref().unwrap();
- assert_eq!(upload["manifestId"], PERSONAL_MANIFEST_ID);
- assert_eq!(upload["blobs"].as_array().unwrap().len(), 1);
- assert_eq!(upload["blobs"][0]["hash"], missing_checks[0].body.as_ref().unwrap()["hashes"][0]);
-
- // The bytes are encrypted with the blob's own key, which travels encrypted with the VEK.
- let ciphertext = crate::common::encoding::base64_decode(upload["blobs"][0]["encryptedDataBase64"].as_str().unwrap()).unwrap();
- assert!(crypto::symmetric_decrypt_bytes(&ciphertext, &vek).is_err());
- let hash = upload["blobs"][0]["hash"].as_str().unwrap();
- let blob_key = crypto::unwrap_key(upload["blobs"][0]["encryptedBlobKey"].as_str().unwrap(), &vek, &crypto::aad::blob_key(PERSONAL_MANIFEST_ID, hash)).unwrap();
- assert_eq!(crypto::symmetric_decrypt_bytes_with_aad(&ciphertext, &blob_key, &crypto::aad::blob_data(PERSONAL_MANIFEST_ID, hash)).unwrap(), vec![1u8, 2, 3, 4]);
-}
-
-#[test]
-fn no_op_mutation_clears_the_dirty_flag_without_a_write() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- let server_db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);
- let (status, vault) = snapshot_of(&server_db, &vek, 7, &vault_codec::generate_manifest_salt());
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
- host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- // Marked dirty, nothing actually changed.
- host.mutation_sequence = 1;
- host.is_dirty = true;
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, true, 1)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert!(host.requests.iter().all(|r| r.method != "POST"), "no write for a no-op mutation");
- assert_eq!(host.mark_clean_calls, vec![1]);
- assert!(!host.is_dirty);
-}
-
-/// A host that pulled a server vault at revision 7 holding one item and no mail delivery keypair.
-fn host_with_keyless_vault(vek: &str) -> TestHost {
- let mut host = TestHost::new(vek);
- let server_db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);
- let (status, vault) = snapshot_of(&server_db, vek, 7, &vault_codec::generate_manifest_salt());
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
- host.drive(&SyncSession::new(&request("fullSync", vek, false, 0)).unwrap());
- assert!(active_delivery_keys(&host.local, PERSONAL_MANIFEST_ID).is_empty());
- host
-}
-
-/// Mark the local vault changed and sync it with the given dirty scopes; returns the result and the last vault write.
-fn push_local_edit(host: &mut TestHost, vek: &str, name: &str, scopes: &[&str]) -> (Value, Value) {
- host.local.execute("UPDATE Items SET Name = ?, UpdatedAt = '2099-01-01 00:00:00.000'", [name]).unwrap();
- host.store_local_as_blob();
- host.mutation_sequence += 1;
- host.is_dirty = true;
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 8 }], "bucketRevisions": [], "missingBlobHashes": [] }));
- let mut sync = serde_json::from_str::<Value>(&request("fullSync", vek, true, host.mutation_sequence)).unwrap();
- sync["dirtyScopes"] = json!(scopes);
- let result = host.drive(&SyncSession::new(&sync.to_string()).unwrap());
- let write = host.requests_to("Vault").into_iter().rfind(|r| r.method == "POST").map(|r| r.body.clone().unwrap()).unwrap_or(Value::Null);
- (result, write)
-}
-
-#[test]
-fn push_creates_and_publishes_a_missing_personal_delivery_key() {
- let vek = crypto::generate_key_base64();
- let mut host = host_with_keyless_vault(&vek);
-
- let (result, write) = push_local_edit(&mut host, &vek, "Renamed", &["Main"]);
-
- assert_eq!(result["success"], true, "{}", result);
- let keys = active_delivery_keys(&host.local, PERSONAL_MANIFEST_ID);
- assert_eq!(keys.len(), 1, "the push created exactly one keypair");
- assert_eq!(write["manifests"][0]["deliveryPublicKey"], json!(keys[0]), "the write publishes the new public key");
- assert_eq!(write["manifests"][0]["deliveryPublicKeyAlgorithm"], json!("rsa-oaep-sha256"), "the write states the key's algorithm");
-
- // The keypair travels inside the manifest, so the other devices get the private half.
- let blob = write["manifests"][0]["manifestBlob"].as_str().unwrap();
- let plain = crypto::symmetric_decrypt_bytes_with_aad(&crate::common::encoding::base64_decode(blob).unwrap(), &vek, &crypto::aad::manifest(PERSONAL_MANIFEST_ID)).unwrap();
- let manifest: Value = serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap();
- assert_eq!(manifest["tables"]["EncryptionKeys"].as_array().unwrap().len(), 1);
-
- // The stored vault holds it too, so a reload does not read the keypair as deleted.
- let stored = state::decrypt_vault_blob(host.vault_blob.as_ref().unwrap(), &host.vault_key).unwrap();
- assert_eq!(active_delivery_keys(&test_host::open_from_bytes(&stored), PERSONAL_MANIFEST_ID), keys);
-
- // The next push keeps the same keypair.
- let (again, second_write) = push_local_edit(&mut host, &vek, "Renamed again", &["Main"]);
- assert_eq!(again["success"], true, "{}", again);
- assert_eq!(active_delivery_keys(&host.local, PERSONAL_MANIFEST_ID), keys);
- assert_eq!(second_write["manifests"][0]["deliveryPublicKey"], json!(keys[0]));
-}
-
-#[test]
-fn bucket_only_push_writes_the_manifest_while_the_personal_delivery_key_is_missing() {
- let vek = crypto::generate_key_base64();
- let mut host = host_with_keyless_vault(&vek);
- let category = vault_codec::bucket_layout()[0].category.clone();
-
- let (result, write) = push_local_edit(&mut host, &vek, "Renamed", &[category.as_str()]);
-
- assert_eq!(result["success"], true, "{}", result);
- let keys = active_delivery_keys(&host.local, PERSONAL_MANIFEST_ID);
- assert_eq!(keys.len(), 1);
- assert_eq!(write["manifests"][0]["deliveryPublicKey"], json!(keys[0]), "the full write publishes the key a bucket-only write could not");
-}
-
-#[test]
-fn outdated_push_merges_the_server_change_and_retries() {
- let vek = crypto::generate_key_base64();
- let salt = vault_codec::generate_manifest_salt();
- let mut host = TestHost::new(&vek);
- let server_db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);
- let (status, vault) = snapshot_of(&server_db, &vek, 7, &salt);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
- host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- // Another client adds an item on the server (revision 8) while this one adds a different item locally.
- insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000003", "Other device item", PERSONAL_MANIFEST_ID);
- let (status8, vault8) = snapshot_of(&server_db, &vek, 8, &salt);
- insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000002", "Local item", PERSONAL_MANIFEST_ID);
- host.store_local_as_blob();
- host.mutation_sequence = 1;
- host.is_dirty = true;
-
- host.responders.clear();
- host.respond("GET", "Status", status8);
- host.respond("GET", "Vault", vault8);
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
- host.respond_with(Box::new(|method, path, body| {
- if method != "POST" || path != "Vault" {
- return None;
- }
- let current = body.and_then(|b| b["manifests"][0]["currentRevision"].as_i64()).unwrap_or(-1);
- Some(if current == 8 {
- (200, json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 9 }], "bucketRevisions": [], "missingBlobHashes": [] }))
- } else {
- (200, json!({ "status": "outdated", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 8 }], "bucketRevisions": [], "missingBlobHashes": [] }))
- })
- }));
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, true, 1)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["hasNewVault"], true);
- assert_eq!(item_names(&host.local), vec!["Local item", "Other device item", "Server item"]);
- assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 9);
- let posts: Vec<_> = host.requests_to("Vault").into_iter().filter(|r| r.method == "POST").collect();
- assert_eq!(posts.last().unwrap().body.as_ref().unwrap()["manifests"][0]["currentRevision"], 8);
- let merged: Value = {
- let blob = posts.last().unwrap().body.as_ref().unwrap()["manifests"][0]["manifestBlob"].as_str().unwrap();
- let plain = crypto::symmetric_decrypt_bytes_with_aad(&crate::common::encoding::base64_decode(blob).unwrap(), &vek, &crypto::aad::manifest(PERSONAL_MANIFEST_ID)).unwrap();
- serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap()
- };
- assert_eq!(merged["tables"]["Items"].as_array().unwrap().len(), 3);
-}
-
-fn insert_item_stats(conn: &rusqlite::Connection, item_id: &str, use_count: i64) {
- let now = crate::common::timestamp::now_vault_datetime();
- conn.execute(
- "INSERT INTO ItemStats (ManifestId, Id, UseCount, AutofillCount, CopyCount, PasskeyAuthCount, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 0, 0, 0, ?, ?, 0)",
- rusqlite::params![PERSONAL_MANIFEST_ID, item_id, use_count, now, now],
- )
- .unwrap();
-}
-
-#[test]
-fn outdated_bucket_only_push_merges_the_server_bucket_instead_of_overwriting_it() {
- const ITEM_A: &str = "aaaaaaaa-0000-4000-8000-000000000001";
- const ITEM_B: &str = "aaaaaaaa-0000-4000-8000-000000000002";
- let vek = crypto::generate_key_base64();
- let salt = vault_codec::generate_manifest_salt();
- let mut host = TestHost::new(&vek);
- let server_db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&server_db, ITEM_A, "Item A", PERSONAL_MANIFEST_ID);
- insert_item(&server_db, ITEM_B, "Item B", PERSONAL_MANIFEST_ID);
- let (status7, vault7) = snapshot_of(&server_db, &vek, 7, &salt);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- host.respond("GET", "Status", status7.clone());
- host.respond("GET", "Vault", vault7);
- host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- // Another device writes the stats of item B after this client's status check, while this one used item A.
- insert_item_stats(&server_db, ITEM_B, 5);
- let (status8, vault8) = snapshot_of(&server_db, &vek, 8, &salt);
- insert_item_stats(&host.local, ITEM_A, 1);
- host.store_local_as_blob();
- host.mutation_sequence = 1;
- host.is_dirty = true;
-
- let status_calls = std::rc::Rc::new(std::cell::Cell::new(0u32));
- host.responders.clear();
- host.respond_with(Box::new(move |method, path, _| {
- if method != "GET" || path != "Status" {
- return None;
- }
- status_calls.set(status_calls.get() + 1);
- Some((200, if status_calls.get() == 1 { status7.clone() } else { status8.clone() }))
- }));
- host.respond("GET", "Vault", vault8);
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
- host.respond_with(Box::new(|method, path, body| {
- if method != "POST" || path != "Vault" {
- return None;
- }
- let current = body.and_then(|b| b["buckets"][0]["currentRevision"].as_i64()).unwrap_or(-1);
- let revision = if current == 8 { 9 } else { 8 };
- Some((200, json!({ "status": if current == 8 { "ok" } else { "outdated" }, "manifestRevisions": [], "bucketRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "category": "stats", "revision": revision }], "missingBlobHashes": [] })))
- }));
-
- let mut bucket_request: Value = serde_json::from_str(&request("fullSync", &vek, true, 1)).unwrap();
- bucket_request["dirtyScopes"] = json!(["stats"]);
- let result = host.drive(&SyncSession::new(&bucket_request.to_string()).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- let posts: Vec<_> = host.requests_to("Vault").into_iter().filter(|r| r.method == "POST").collect();
- let accepted = posts.last().unwrap().body.as_ref().unwrap();
- assert_eq!(accepted["buckets"][0]["currentRevision"], 8);
- let bucket: Value = {
- let aad = crypto::aad::bucket(accepted["buckets"][0]["manifestId"].as_str().unwrap(), accepted["buckets"][0]["category"].as_str().unwrap());
- let plain = crypto::symmetric_decrypt_bytes_with_aad(&crate::common::encoding::base64_decode(accepted["buckets"][0]["blob"].as_str().unwrap()).unwrap(), &vek, &aad).unwrap();
- serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap()
- };
- assert_eq!(bucket["tables"]["ItemStats"].as_array().unwrap().len(), 2, "the other device's stats must survive: {}", bucket);
- let tail: Vec<String> = host.requests.iter().rev().take(4).rev().map(|r| format!("{} {}", r.method, r.path)).collect();
- assert_eq!(tail, vec!["POST Vault", "GET Status", "GET Vault", "POST Vault"], "the refused write must be followed by a pull before the next attempt");
- assert_eq!(posts.len(), 2);
- assert_eq!(posts[0].body.as_ref().unwrap()["buckets"][0]["currentRevision"], 7);
+pub fn insert_item_at(conn: &Connection, id: &str, name: &str, updated_at: &str) {
+ conn.execute("INSERT INTO Items (Id, ManifestId, Name, ItemType, FolderId, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'Login', NULL, ?, ?, 0)", rusqlite::params![id, PERSONAL_MANIFEST_ID, name, days_ago(30), updated_at]).unwrap();
}
-#[test]
-fn unreachable_server_with_a_local_vault_goes_offline() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.store_local_as_blob();
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["wasOffline"], true);
- assert_eq!(result["isOfflineMode"], true);
+/// Give the personal manifest the mail delivery keypair every written vault carries.
+pub fn insert_delivery_key(conn: &Connection) {
+ conn.execute("INSERT INTO EncryptionKeys (Id, ManifestId, PublicKey, PrivateKey, IsPrimary, CreatedAt, UpdatedAt, IsDeleted) VALUES ('dddddddd-0000-4000-8000-000000000001', ?, 'public', 'private', 1, ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, now(), now()]).unwrap();
}
-#[test]
-fn expired_session_requires_logout() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.respond_with(Box::new(|_, path, _| if path == "Status" { Some((401, json!({}))) } else { None }));
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- assert_eq!(result["success"], false);
- assert_eq!(result["requiresLogout"], true);
- assert_eq!(result["logoutReason"], "sessionExpired");
+/// The active delivery public keys of a manifest.
+pub fn active_delivery_keys(conn: &Connection, manifest_id: &str) -> Vec<String> {
+ query(conn, "SELECT PublicKey FROM EncryptionKeys WHERE ManifestId = ? AND IsPrimary = 1 AND IsDeleted = 0", &[json!(manifest_id)]).unwrap().iter().map(|r| r["PublicKey"].as_str().unwrap().to_string()).collect()
}
-#[test]
-fn password_changed_elsewhere_requires_logout() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.state.insert(state::UNLOCK_KEY_DERIVATION_PARAMS.to_string(), json!({ "salt": "old-salt", "encryptionType": "Argon2Id", "encryptionSettings": "{}" }));
- host.respond("GET", "Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "new-salt" }));
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- assert_eq!(result["requiresLogout"], true);
- assert_eq!(result["logoutReason"], "passwordChanged");
+pub fn insert_logo(conn: &Connection, item_id: &str, bytes: &[u8]) {
+ conn.execute("INSERT INTO Logos (ManifestId, Id, Source, FileData, Kind, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, 'example.com', ?, 'favicon', ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, LOGO, bytes, now(), now()]).unwrap();
+ conn.execute("UPDATE Items SET LogoId = ? WHERE Id = ?", rusqlite::params![LOGO, item_id]).unwrap();
}
-#[test]
-fn legacy_account_without_vault_key_reports_the_manifest_migration() {
- let unlock_key = crypto::generate_key_base64();
- let mut host = TestHost::new(&unlock_key);
- host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));
- host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));
- host.store_local_as_blob();
- host.respond("GET", "Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 3 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["manifestMigrationRequired"], true);
-
- let status = host.drive(&SyncSession::new(&request("migrationStatus", &unlock_key, false, 0)).unwrap());
- assert_eq!(status["kind"], "storageFormatUpgrade");
+pub fn insert_attachment(conn: &Connection, item_id: &str, bytes: Option<&[u8]>) {
+ conn.execute("INSERT INTO Attachments (ManifestId, Id, ItemId, Filename, Blob, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'passport.pdf', ?, ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, ATTACHMENT, item_id, bytes, now(), now()]).unwrap();
}
-#[test]
-fn manifest_migration_generates_the_key_hierarchy_and_pushes() {
- let unlock_key = crypto::generate_key_base64();
- let salt = vault_codec::generate_manifest_salt();
- let mut host = TestHost::new(&unlock_key);
- insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000001", "Old item", PERSONAL_MANIFEST_ID);
- host.store_local_as_blob();
- host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));
- host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));
- host.state.insert(state::VAULT_MANIFEST_SALT.to_string(), json!(salt));
- host.respond_with(Box::new(|method, path, _| if method == "GET" && path == "VaultKey/Password" { Some((200, json!({ "vaultKey": null }))) } else { None }));
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "bucketRevisions": [], "missingBlobHashes": [] }));
-
- let result = host.drive(&SyncSession::new(&request("migrateManifest", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["pushed"], true);
- let new_key = host.vault_key.clone();
- assert_ne!(new_key, unlock_key, "the host receives the new VEK through the store command");
- let posts: Vec<_> = host.requests_to("Vault").into_iter().filter(|r| r.method == "POST").collect();
- let body = posts[0].body.as_ref().unwrap();
- assert!(body["migration"]["accountKeys"]["encryptedAccountKey"].is_string(), "the migration push carries the key hierarchy");
- assert!(crypto::unwrap_key(body["migration"]["accountKeys"]["encryptedAccountKey"].as_str().unwrap(), &unlock_key, crypto::aad::ACCOUNT_KEY).is_err(), "the legacy vault key does not wrap the Account Key directly");
- let opened = crypto::open_account_key_chain(&unlock_key, body["migration"]["accountKeys"]["encryptedAccountKey"].as_str().unwrap(), body["migration"]["accountKeys"]["encryptedVek"].as_str().unwrap(), None).unwrap();
- assert_eq!(*opened.vault_encryption_key, new_key);
- let account_keys = &body["migration"]["accountKeys"];
- let signed = crypto::signing::account_public_key_message(account_keys["accountPublicKey"].as_str().unwrap());
- assert!(crypto::signing::verify(account_keys["signingPublicKey"].as_str().unwrap(), &signed, account_keys["accountPublicKeySignature"].as_str().unwrap()), "the upgrade push carries a signing key that signed the account public key");
- assert!(host.state.contains_key(state::ENCRYPTED_ACCOUNT_KEY));
- assert!(host.state.contains_key(state::ENCRYPTED_ACCOUNT_PRIVATE_KEY));
- assert_eq!(host.state[state::ENCRYPTED_SIGNING_PRIVATE_KEY], account_keys["encryptedSigningPrivateKey"]);
- assert_eq!(host.rekeyed_stores_found_the_chain, vec![true], "the chain is cached before the vault is stored under the VEK");
+pub fn insert_item_stats(conn: &Connection, item_id: &str, use_count: i64) {
+ conn.execute("INSERT INTO ItemStats (ManifestId, Id, UseCount, AutofillCount, CopyCount, PasskeyAuthCount, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 0, 0, 0, ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, item_id, use_count, now(), now()]).unwrap();
}
-#[test]
-fn account_upgrade_decodes_base64_text_that_0_30_merges_left_in_blob_columns() {
- let unlock_key = crypto::generate_key_base64();
- let mut host = TestHost::new(&unlock_key);
- let item = "aaaaaaaa-0000-4000-8000-000000000001";
- let bytes = vec![0x00u8, 0xFF, 0xC3, 0x28, 0x10];
- let text = crate::common::encoding::base64_encode(&bytes);
- let now = crate::common::timestamp::now_vault_datetime();
- insert_item(&host.local, item, "Item", PERSONAL_MANIFEST_ID);
- host.local.execute("INSERT INTO Passkeys (ManifestId, Id, ItemId, RpId, UserHandle, PublicKey, PrivateKey, PrfKey, DisplayName, AdditionalData, CredentialId, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, 'bbbbbbbb-0000-4000-8000-000000000001', ?, 'example.com', ?, 'pub', 'priv', 'not base64!', 'Passkey', NULL, NULL, ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, item, text, now, now]).unwrap();
- host.local.execute("INSERT INTO Attachments (ManifestId, Id, ItemId, Filename, Blob, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, 'cccccccc-0000-4000-8000-000000000001', ?, 'file.bin', ?, ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, item, text, now, now]).unwrap();
- host.store_local_as_blob();
- host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));
- host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));
- host.state.insert(state::VAULT_MANIFEST_SALT.to_string(), json!(vault_codec::generate_manifest_salt()));
- host.respond_with(Box::new(|method, path, _| if method == "GET" && path == "VaultKey/Password" { Some((200, json!({ "vaultKey": null }))) } else { None }));
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
- host.respond("POST", "Vault/blobs", json!({}));
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "bucketRevisions": [], "missingBlobHashes": [] }));
-
- let result = host.drive(&SyncSession::new(&request("migrateManifest", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{} {:?}", result, host.logs);
- let cell = |sql: &str| host.local.query_row(sql, [], |row| Ok((row.get::<_, String>(0)?, row.get_ref(1)?.as_bytes().map(<[u8]>::to_vec).ok()))).unwrap();
- assert_eq!(cell("SELECT typeof(UserHandle), UserHandle FROM Passkeys"), ("blob".to_string(), Some(bytes.clone())));
- assert_eq!(cell("SELECT typeof(Blob), Blob FROM Attachments"), ("blob".to_string(), Some(bytes)));
- assert_eq!(cell("SELECT typeof(PrfKey), PrfKey FROM Passkeys"), ("text".to_string(), Some(b"not base64!".to_vec())), "text that is not base64 is kept as-is");
+/// Rename every item, stamped now: a local edit newer than the synced state.
+pub fn rename_item(conn: &Connection, name: &str) {
+ conn.execute("UPDATE Items SET Name = ?, UpdatedAt = ?", rusqlite::params![name, now()]).unwrap();
}
/// The latest EF migration stamp of a database.
-fn latest_migration_id(conn: &rusqlite::Connection) -> String {
+pub fn latest_migration_id(conn: &Connection) -> String {
query(conn, "SELECT MigrationId FROM __EFMigrationsHistory ORDER BY MigrationId DESC LIMIT 1", &[]).unwrap()[0]["MigrationId"].as_str().unwrap().to_string()
}
-#[test]
-fn schema_rebuild_of_a_stale_vault_pushes_without_touching_the_key_hierarchy() {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- let current_stamp = latest_migration_id(&host.local);
- insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000001", "Kept item", PERSONAL_MANIFEST_ID);
- // A vault written by an older client: same tables, older schema stamp (still past the frozen sqlite-blob chain).
- host.local.execute_batch("DELETE FROM __EFMigrationsHistory; INSERT INTO __EFMigrationsHistory (MigrationId, ProductVersion) VALUES ('20250101000000_2.0.0-Stale', '9.0.0');").unwrap();
- host.store_local_as_blob();
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));
- host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));
- host.state.insert(state::VAULT_MANIFEST_SALT.to_string(), json!(vault_codec::generate_manifest_salt()));
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "bucketRevisions": [], "missingBlobHashes": [] }));
-
- let status = host.drive(&SyncSession::new(&request("migrationStatus", &vek, false, 0)).unwrap());
- assert_eq!(status["kind"], "schemaRebuild");
-
- let result = host.drive(&SyncSession::new(&request("migrateManifest", &vek, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["pushed"], true);
- assert_eq!(latest_migration_id(&host.local), current_stamp, "the local vault is rebuilt onto the current schema");
- assert_eq!(item_names(&host.local), vec!["Kept item"]);
- assert!(host.requests_to("VaultKey/Password").is_empty(), "a migrated account is not probed for a key hierarchy");
- let posts: Vec<_> = host.requests_to("Vault").into_iter().filter(|r| r.method == "POST").collect();
- assert!(posts[0].body.as_ref().unwrap()["migration"].is_null(), "no key hierarchy is created");
- assert_eq!(host.vault_key, vek, "the session key stays the VEK");
+/// A server holding a vault and a device that pulled it: the starting point of most tests. `db` is the plaintext
+/// the server was published from, which a test edits as "the other device" before publishing again.
+pub struct Synced {
+ pub server: Server,
+ pub host: TestHost,
+ pub db: Connection,
+ pub vek: String,
+ pub salt: String,
}
-/// A legacy sqlite-blob snapshot of the given database, as the server serves an account that has not migrated.
-fn legacy_snapshot_of(conn: &rusqlite::Connection, unlock_key: &str, revision: i64) -> Value {
- let bytes = conn.serialize(rusqlite::MAIN_DB).unwrap().to_vec();
- json!({
- "storageFormat": "sqlite-blob",
- "legacyVaultBlob": crypto::symmetric_encrypt_bytes(&bytes, unlock_key).unwrap(),
- "legacyRevision": revision,
- "personalManifestId": PERSONAL_MANIFEST_ID,
- "emailRouting": { "privateEmailDomainList": ["private.io"], "publicEmailDomainList": [], "hiddenPrivateEmailDomainList": [], "emailAddressList": [] },
- })
+/// A server at revision 7 holding one item named "Server item", encrypted under `vek`.
+pub fn server_with_item(vek: &str) -> Server {
+ let db = open_schema_db();
+ insert_item(&db, ITEM_A, "Server item");
+ let server = FakeServer::new();
+ server.borrow_mut().publish(&db, vek, &vault_codec::generate_manifest_salt(), 7);
+ server
}
-/// A host as a client predating the manifest storage format leaves it: a sqlite blob under the unlock key, no personal
-/// manifest id and no revision baseline, against a server that still holds the account as a legacy vault.
-fn pre_format_session_host(unlock_key: &str, local_item: &str, server_item: &str) -> TestHost {
+/// A legacy device: a sqlite-blob vault under the unlock key, no key chain, the server's revision baseline recorded.
+pub fn legacy_device(unlock_key: &str, server: &Server, seed: impl FnOnce(&Connection)) -> TestHost {
let mut host = TestHost::new(unlock_key);
- insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000001", local_item, PERSONAL_MANIFEST_ID);
+ host.serve(server);
+ seed(&host.local);
host.store_local_as_blob();
- let server_db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000002", server_item, PERSONAL_MANIFEST_ID);
- host.respond("GET", "Vault", legacy_snapshot_of(&server_db, unlock_key, 3));
- host.respond_with(Box::new(|method, path, _| if method == "GET" && path == "VaultKey/Password" { Some((200, json!({ "vaultKey": null }))) } else { None }));
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "bucketRevisions": [], "missingBlobHashes": [] }));
+ host.set_state(state::SERVER_MANIFEST_REVISIONS, json!({ PERSONAL_MANIFEST_ID: 3 }));
+ host.set_state(state::VAULT_PERSONAL_MANIFEST_ID, json!(PERSONAL_MANIFEST_ID));
+ host.set_state(state::VAULT_MANIFEST_SALT, json!(vault_codec::generate_manifest_salt()));
host
}
-#[test]
-fn manifest_migration_of_a_pre_format_session_pulls_the_server_vault_first() {
- let unlock_key = crypto::generate_key_base64();
- let mut host = pre_format_session_host(&unlock_key, "Local item", "Server item");
-
- let result = host.drive(&SyncSession::new(&request("migrateManifest", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["pushed"], true);
- assert_eq!(host.requests_to("Vault").iter().filter(|r| r.method == "GET").count(), 1, "the baseline is pulled once");
- assert_eq!(host.state[state::VAULT_PERSONAL_MANIFEST_ID], PERSONAL_MANIFEST_ID);
- assert_eq!(item_names(&host.local), vec!["Server item"], "a clean local vault is replaced by the server's, like a login does");
- let posts: Vec<_> = host.requests_to("Vault").into_iter().filter(|r| r.method == "POST").collect();
- let body = posts[0].body.as_ref().unwrap();
- assert_eq!(body["manifests"][0]["currentRevision"], 3, "the migration push names the revision the server holds");
- assert!(body["migration"]["accountKeys"]["encryptedAccountKey"].is_string());
- assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 4);
-}
-
-#[test]
-fn manifest_migration_of_a_dirty_pre_format_session_keeps_the_local_vault() {
- let unlock_key = crypto::generate_key_base64();
- let mut host = pre_format_session_host(&unlock_key, "Local item", "Server item");
- host.is_dirty = true;
-
- let result = host.drive(&SyncSession::new(&request("migrateManifest", &unlock_key, true, 1)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["pushed"], true);
- assert_eq!(host.state[state::VAULT_PERSONAL_MANIFEST_ID], PERSONAL_MANIFEST_ID);
- assert_eq!(item_names(&host.local), vec!["Local item"], "pending local changes are not thrown away");
- let posts: Vec<_> = host.requests_to("Vault").into_iter().filter(|r| r.method == "POST").collect();
- let body = posts[0].body.as_ref().unwrap();
- assert_eq!(body["manifests"][0]["currentRevision"], 3, "the baseline still comes from the server");
- let manifest_json = crypto::symmetric_decrypt_bytes_with_aad(&crate::common::encoding::base64_decode(body["manifests"][0]["manifestBlob"].as_str().unwrap()).unwrap(), &host.vault_key, &crypto::aad::manifest(PERSONAL_MANIFEST_ID)).unwrap();
- assert!(vault_codec::unpack_payload(&manifest_json).unwrap().contains("Local item"), "the push carries the local changes");
-}
-
-#[test]
-fn session_reports_when_a_response_is_missing() {
- let session = SyncSession::new(&request("fullSync", "key", false, 0)).unwrap();
- let first: Value = serde_json::from_str(&session.next_command().unwrap()).unwrap();
- assert_eq!(first["kind"], "log", "the engine announces the sync before touching the host");
- assert!(session.next_command().is_err(), "the session refuses to advance without a response");
- session.resume("{}", None).unwrap();
- let second: Value = serde_json::from_str(&session.next_command().unwrap()).unwrap();
- assert_eq!(second["kind"], "http");
- assert_eq!(second["path"], "Status");
-}
-
-#[test]
-fn state_helpers_key_buckets_and_fingerprints_like_the_client() {
- assert_eq!(state::bucket_revision_key("m", "settings"), "m:settings");
- assert_eq!(state::fingerprint_bucket_key("m", "stats"), "bucket:m:stats");
- assert_eq!(state::fingerprint_manifest_key("m"), "manifest:m");
- let _unused: HashMap<String, String> = HashMap::new();
-}
-
-#[test]
-fn status_check_reports_newer_server_state_without_touching_the_vault() {
+/// A server at revision 7 holding the vault `seed` builds, pulled by a logged-in device.
+pub fn synced(seed: impl FnOnce(&Connection)) -> Synced {
let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));
- host.respond("GET", "Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));
-
- let result = host.drive(&SyncSession::new(&request("statusCheck", &vek, true, 2)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["hasNewerVault"], true);
- assert_eq!(result["hasDirtyChanges"], true);
- assert_eq!(result["isOffline"], false);
- assert!(host.store_calls.is_empty());
- assert!(host.requests.iter().all(|r| r.path == "Status"));
-}
-
-/// The `GET v2/VaultKey/Password` answer for a hierarchy the server holds.
-fn vault_key_body(hierarchy: &crypto::AccountKeyHierarchy) -> Value {
- let blobs = &hierarchy.account_keys;
- json!({ "vaultKey": { "type": "password", "encryptedAccountKey": blobs.encrypted_account_key, "algorithm": "aes256-gcm", "encryptedVek": blobs.encrypted_vek, "accountPublicKey": blobs.account_public_key, "encryptedAccountPrivateKey": blobs.encrypted_account_private_key, "signingPublicKey": blobs.signing_public_key, "encryptedSigningPrivateKey": blobs.encrypted_signing_private_key, "salt": "salt", "encryptionType": "Argon2Id", "encryptionSettings": "{}" } })
-}
-
-/// The cross-device race: this device logged in while the account was legacy (no cached chain, unlock key session), and
-/// another device created the hierarchy since. The pull accepts it, and the stored vault reaches the host
-/// together with the VEK it is now encrypted under.
-#[test]
-fn a_hierarchy_created_on_another_device_is_accepted_on_the_next_pull() {
- let unlock_key = crypto::generate_key_base64();
- let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
- let vek = hierarchy.vault_encryption_key.clone();
- let mut host = TestHost::new(&unlock_key);
-
- let server_db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);
- let (status, vault) = snapshot_of(&server_db, &vek, 7, &vault_codec::generate_manifest_salt());
- host.respond("GET", "Status", status);
- host.respond("GET", "VaultKey/Password", vault_key_body(&hierarchy));
- host.respond("GET", "Vault", vault);
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(host.vault_key, vek, "the store carried the VEK, so the host switched to it before opening the blob");
- assert!(matches!(&host.store_calls[0], Command::VaultStore { encryption_key: Some(key), .. } if *key == vek));
- assert_eq!(item_names(&host.local), vec!["Server item"]);
- assert_eq!(host.state[state::ENCRYPTED_ACCOUNT_KEY], hierarchy.account_keys.encrypted_account_key);
- assert_eq!(host.state[state::ACCOUNT_PUBLIC_KEY], hierarchy.account_keys.account_public_key);
- assert_eq!(host.rekeyed_stores_found_the_chain, vec![true], "the chain is cached before the vault is stored under the VEK");
-}
-
-/// An Account Key encrypted with an algorithm this build does not know asks for an app update instead of a wrong password.
-#[test]
-fn resolve_vault_key_with_an_unknown_unlock_algorithm_asks_for_an_app_update() {
- let unlock_key = crypto::generate_key_base64();
- let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
- let mut host = TestHost::new(&unlock_key);
- let mut body = vault_key_body(&hierarchy);
- body["vaultKey"]["algorithm"] = json!("future-x");
- host.respond("GET", "VaultKey/Password", body);
-
- let result = host.drive(&SyncSession::new(&request("resolveVaultKey", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], false, "{}", result);
- assert_eq!(result["logoutReason"], "vaultVersionIncompatible", "{}", result);
- assert!(!host.state.contains_key(state::ENCRYPTED_ACCOUNT_KEY), "nothing is cached from a chain this build cannot open");
-}
-
-/// Login: the host hands the password-derived key to `resolveVaultKey`; the server's chain opens with it, is
-/// cached for offline unlock, and the VEK comes back as the key to store.
-#[test]
-fn resolve_vault_key_opens_the_chain_from_the_server() {
- let unlock_key = crypto::generate_key_base64();
- let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
- let mut host = TestHost::new(&unlock_key);
- host.respond("GET", "VaultKey/Password", vault_key_body(&hierarchy));
-
- let result = host.drive(&SyncSession::new(&request("resolveVaultKey", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["hasVaultKey"], true);
- assert_eq!(result["encryptionKey"], hierarchy.vault_encryption_key);
- assert_eq!(host.state[state::ENCRYPTED_ACCOUNT_PRIVATE_KEY], hierarchy.account_keys.encrypted_account_private_key);
- assert_eq!(host.state[state::ENCRYPTED_ACCOUNT_KEY], hierarchy.account_keys.encrypted_account_key);
- assert_eq!(host.state[state::SIGNING_PUBLIC_KEY], hierarchy.account_keys.signing_public_key);
- assert_eq!(host.state[state::ENCRYPTED_SIGNING_PRIVATE_KEY], hierarchy.account_keys.encrypted_signing_private_key);
- assert!(host.store_calls.is_empty(), "resolving a key never touches the stored vault");
-}
-
-/// A device that stores the Account Key (Login with Mobile from a phone that converted its keychain) resolves with it
-/// as well as with the unlock key.
-#[test]
-fn resolve_vault_key_opens_the_chain_with_a_stored_account_key() {
- let unlock_key = crypto::generate_key_base64();
- let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
- let account_key = crypto::unwrap_account_key(&hierarchy.account_keys.encrypted_account_key, &unlock_key).unwrap();
- let mut host = TestHost::new(&account_key);
- host.respond("GET", "VaultKey/Password", vault_key_body(&hierarchy));
-
- let result = host.drive(&SyncSession::new(&request("resolveVaultKey", &account_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["encryptionKey"], hierarchy.vault_encryption_key);
-}
-
-/// A legacy account has no chain: the password-derived key is the vault key and any stale cached chain is dropped.
-#[test]
-fn resolve_vault_key_keeps_the_unlock_key_for_a_legacy_account() {
- let unlock_key = crypto::generate_key_base64();
- let mut host = TestHost::new(&unlock_key);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("stale"));
- host.respond("GET", "VaultKey/Password", json!({ "vaultKey": null }));
-
- let result = host.drive(&SyncSession::new(&request("resolveVaultKey", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["hasVaultKey"], false);
- assert_eq!(result["encryptionKey"], unlock_key);
- assert!(!host.state.contains_key(state::ENCRYPTED_ACCOUNT_KEY));
-}
-
-/// Offline, the cached chain opens with the password-derived key (a re-login after a forced logout).
-#[test]
-fn resolve_vault_key_opens_the_cached_chain_when_the_server_is_unreachable() {
- let unlock_key = crypto::generate_key_base64();
- let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
- let mut host = TestHost::new(&unlock_key);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!(hierarchy.account_keys.encrypted_account_key));
- host.state.insert(state::ENCRYPTED_VEK.to_string(), json!(hierarchy.account_keys.encrypted_vek));
-
- let result = host.drive(&SyncSession::new(&request("resolveVaultKey", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(result["hasVaultKey"], true);
- assert_eq!(result["encryptionKey"], hierarchy.vault_encryption_key);
-}
-
-/// A key that does not open the chain is reported as a rejected unlock key, never silently kept.
-#[test]
-fn resolve_vault_key_refuses_a_key_that_does_not_open_the_chain() {
- let hierarchy = crypto::create_account_key_hierarchy(&crypto::generate_key_base64()).unwrap();
- let wrong_unlock_key = crypto::generate_key_base64();
- let mut host = TestHost::new(&wrong_unlock_key);
- host.respond("GET", "VaultKey/Password", vault_key_body(&hierarchy));
-
- let result = host.drive(&SyncSession::new(&request("resolveVaultKey", &wrong_unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], false);
- assert_eq!(result["errorCode"], "E-206");
- assert!(result.get("encryptionKey").is_none());
-}
-
-/// A chain whose VEK does not open under its own account key is not reported as a wrong password.
-#[test]
-fn resolve_vault_key_tells_an_unreadable_chain_apart_from_a_wrong_password() {
- let unlock_key = crypto::generate_key_base64();
- let mut hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
- hierarchy.account_keys.encrypted_vek = crypto::wrap_key(&crypto::generate_key_base64(), &crypto::generate_key_base64(), crypto::aad::PERSONAL_VEK).unwrap();
- let mut host = TestHost::new(&unlock_key);
- host.respond("GET", "VaultKey/Password", vault_key_body(&hierarchy));
-
- let result = host.drive(&SyncSession::new(&request("resolveVaultKey", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], false);
- assert_eq!(result["errorCode"], "E-207");
-}
-
-/// The sync trusts the key it is given: an unlock key handed to a device that caches the chain is not upgraded any more,
-/// so the run fails to open the vault instead of silently swapping keys. Resolving the key is the host's job.
-#[test]
-fn an_unlock_key_session_on_a_migrated_device_is_not_upgraded_by_the_sync() {
- let unlock_key = crypto::generate_key_base64();
- let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
- let vek = hierarchy.vault_encryption_key.clone();
- let mut host = TestHost::new(&vek);
- host.store_local_as_blob();
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!(hierarchy.account_keys.encrypted_account_key));
- host.state.insert(state::ENCRYPTED_VEK.to_string(), json!(hierarchy.account_keys.encrypted_vek));
- host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));
- host.respond("GET", "Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));
-
- host.drive(&SyncSession::new(&request("fullSync", &unlock_key, false, 0)).unwrap());
-
- assert!(host.store_calls.iter().all(|c| matches!(c, Command::VaultStore { encryption_key: None, .. })), "no key swap reaches the host");
- assert!(host.requests_to("VaultKey/Password").is_empty(), "a device with a cached chain is never probed");
-}
-
-/// The VEK itself opens nothing in the chain and stands as the session key.
-#[test]
-fn a_vek_session_key_is_left_alone() {
- let unlock_key = crypto::generate_key_base64();
- let hierarchy = crypto::create_account_key_hierarchy(&unlock_key).unwrap();
- let vek = hierarchy.vault_encryption_key.clone();
- let mut host = TestHost::new(&vek);
- host.store_local_as_blob();
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!(hierarchy.account_keys.encrypted_account_key));
- host.state.insert(state::ENCRYPTED_VEK.to_string(), json!(hierarchy.account_keys.encrypted_vek));
- host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));
- host.respond("GET", "Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 3 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert!(host.store_calls.is_empty());
-}
-
-/// A legacy account that is clean and in sync is not probed for a vault key: the probe belongs to a pull or a push.
-#[test]
-fn clean_legacy_account_in_sync_is_not_probed_for_a_vault_key() {
- let unlock_key = crypto::generate_key_base64();
- let mut host = TestHost::new(&unlock_key);
- host.store_local_as_blob();
- host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));
- host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));
- host.respond("GET", "Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 3 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert!(host.requests_to("VaultKey/Password").is_empty());
-}
-
-/// A dirty legacy account probes the server for a vault key once per run, not once per step.
-#[test]
-fn dirty_legacy_account_probes_for_a_vault_key_once_per_run() {
- let unlock_key = crypto::generate_key_base64();
let salt = vault_codec::generate_manifest_salt();
- let mut host = TestHost::new(&unlock_key);
- insert_item(&host.local, "aaaaaaaa-0000-4000-8000-000000000001", "Local item", PERSONAL_MANIFEST_ID);
- host.store_local_as_blob();
- host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));
- host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));
- host.state.insert(state::VAULT_MANIFEST_SALT.to_string(), json!(salt));
- host.respond("GET", "Status", json!({ "clientVersionSupported": true, "serverVersion": "0.31.0", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 3 }], "personalManifestId": PERSONAL_MANIFEST_ID, "srpSalt": "salt" }));
- host.respond("GET", "VaultKey/Password", json!({ "vaultKey": null }));
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "bucketRevisions": [], "missingBlobHashes": [] }));
-
- let result = host.drive(&SyncSession::new(&request("fullSync", &unlock_key, true, 1)).unwrap());
-
- assert_eq!(result["success"], true, "{}", result);
- assert_eq!(host.requests_to("VaultKey/Password").len(), 1);
+ let db = open_schema_db();
+ seed(&db);
+ let server = FakeServer::new();
+ server.borrow_mut().publish(&db, &vek, &salt, 7);
+ let mut host = TestHost::logged_in(&vek, &server);
+ let pulled = host.sync();
+ assert_eq!(pulled["success"], true, "{}", pulled);
+ host.requests.clear();
+ host.store_calls.clear();
+ Synced { server, host, db, vek, salt }
+}
+
+impl Synced {
+ /// Publish `db` as the other device's push at `revision`.
+ pub fn publish(&self, revision: i64) {
+ self.server.borrow_mut().publish(&self.db, &self.vek, &self.salt, revision);
+ }
}
-
diff --git a/core/rust/src/vault_sync/tests/sync.rs b/core/rust/src/vault_sync/tests/sync.rs
new file mode 100644
index 000000000..39f58e6bc
--- /dev/null
+++ b/core/rust/src/vault_sync/tests/sync.rs
@@ -0,0 +1,367 @@
+//! The sync flow: pull, push, the no-op and outdated paths, and how a session ends.
+
+use serde_json::{json, Value};
+
+use super::fake_server::{decrypt_manifest, FakeServer};
+use super::test_host::{request_json, TestHost, USERNAME};
+use super::{active_delivery_keys, insert_item, insert_item_stats, rename_item, server_with_item, state, synced, ITEM_A, ITEM_B, ITEM_C, PERSONAL_MANIFEST_ID, SHARED_MANIFEST_ID};
+use crate::crypto;
+use crate::vault_codec;
+use crate::vault_sync::session::SyncSession;
+
+#[test]
+fn fresh_client_pulls_and_materializes_the_server_vault() {
+ let vek = crypto::generate_key_base64();
+ let server = server_with_item(&vek);
+ let mut host = TestHost::logged_in(&vek, &server);
+
+ let result = host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["hasNewVault"], true);
+ assert_eq!(result["vaultChanged"], true);
+ assert_eq!(result["pulledRevision"], 7);
+ assert_eq!(result["serverVersion"], "0.31.0");
+ assert_eq!(result["manifestMigrationRequired"], false);
+ assert_eq!(host.item_names(), vec!["Server item"]);
+ assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 7);
+ assert_eq!(host.state[state::VAULT_PERSONAL_MANIFEST_ID], PERSONAL_MANIFEST_ID);
+ // Pinned: the persisted state key formats, which a stored vault from an earlier build must still match.
+ let keys = |name: &str| -> Vec<String> { host.state[name].as_object().unwrap().keys().cloned().collect::<std::collections::BTreeSet<_>>().into_iter().collect() };
+ let p = PERSONAL_MANIFEST_ID;
+ assert_eq!(keys(state::VAULT_CONTENT_FINGERPRINTS), vec![format!("bucket:{p}:settings"), format!("bucket:{p}:stats"), format!("manifest:{p}")]);
+ assert_eq!(keys(state::VAULT_BUCKET_REVISIONS), vec![format!("{p}:settings"), format!("{p}:stats")]);
+ assert_eq!(host.request_log(), vec!["GET Status", "GET Vault"]);
+ server.borrow().assert_converged(&host);
+}
+
+#[test]
+fn new_account_starts_from_an_empty_vault_and_writes_its_first_revision() {
+ let vek = crypto::generate_key_base64();
+ let server = FakeServer::new();
+ server.borrow_mut().publish_contentless_personal(0);
+ let mut host = TestHost::logged_in(&vek, &server);
+
+ let result = host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["hasNewVault"], true);
+ assert!(host.item_names().is_empty());
+ assert_eq!(host.vault_writes().len(), 1, "the new vault is written once");
+ let write = host.last_vault_write();
+ assert_eq!(write["manifests"][0]["manifestId"], PERSONAL_MANIFEST_ID);
+ assert_eq!(write["manifests"][0]["currentRevision"], 0);
+ assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 1);
+ assert!(!host.is_dirty);
+
+ // The written manifest opens with the VEK and names the personal manifest.
+ let manifest = decrypt_manifest(write["manifests"][0]["manifestBlob"].as_str().unwrap(), &vek, PERSONAL_MANIFEST_ID);
+ assert_eq!(manifest["manifestId"], PERSONAL_MANIFEST_ID);
+ assert_eq!(manifest["manifestSalt"], host.state[state::VAULT_MANIFEST_SALT]);
+ server.borrow().assert_converged(&host);
+}
+
+#[test]
+fn new_account_whose_first_write_fails_retries_it_on_the_next_sync() {
+ let vek = crypto::generate_key_base64();
+ let server = FakeServer::new();
+ server.borrow_mut().publish_contentless_personal(0);
+ server.borrow_mut().faults.drop_writes = true;
+ let mut host = TestHost::logged_in(&vek, &server);
+
+ // The write never gets an answer: the vault stays dirty and the sync goes offline on the stored empty vault.
+ let result = host.sync();
+ assert_eq!(result["success"], true, "{}", result);
+ assert!(host.is_dirty, "the unwritten new vault stays dirty");
+
+ server.borrow_mut().faults.drop_writes = false;
+ let result = host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(host.last_vault_write()["manifests"][0]["currentRevision"], 0);
+ assert_eq!(host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 1);
+ assert!(!host.is_dirty);
+}
+
+#[test]
+fn contentless_personal_manifest_past_revision_zero_is_refused() {
+ let server = FakeServer::new();
+ server.borrow_mut().publish_contentless_personal(4);
+ let mut host = TestHost::logged_in(&crypto::generate_key_base64(), &server);
+
+ let result = host.sync();
+
+ assert_eq!(result["success"], false, "{}", result);
+ assert!(host.vault_writes().is_empty(), "nothing is written over a damaged manifest");
+ assert!(host.store_calls.is_empty());
+}
+
+#[test]
+fn unknown_storage_format_is_refused_not_read_as_legacy() {
+ let vek = crypto::generate_key_base64();
+ let server = server_with_item(&vek);
+ let mut vault = server.borrow().vault();
+ vault["storageFormat"] = json!("manifest-v2");
+ let mut host = TestHost::logged_in(&vek, &server);
+ host.respond("GET", "Vault", vault);
+
+ let result = host.sync();
+
+ assert_eq!(result["success"], false, "{}", result);
+ assert_eq!(result["logoutReason"], "vaultVersionIncompatible", "an app too old for the format is told to update");
+ assert!(host.item_names().is_empty(), "nothing is materialized");
+ assert!(host.vault_writes().is_empty(), "no legacy migration push");
+}
+
+#[test]
+fn clean_client_in_sync_does_nothing() {
+ let mut s = synced(|db| insert_item(db, ITEM_A, "Server item"));
+
+ let result = s.host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["hasNewVault"], false);
+ assert_eq!(s.host.request_log(), vec!["GET Status"]);
+ assert!(s.host.store_calls.is_empty());
+}
+
+#[test]
+fn dirty_client_pushes_only_what_changed() {
+ let mut s = synced(|db| insert_item(db, ITEM_A, "Server item"));
+
+ s.host.edit(|db| insert_item(db, ITEM_B, "Local item"));
+ let result = s.host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ let writes = s.host.vault_writes();
+ assert_eq!(writes.len(), 1, "one write expected");
+ let write = &writes[0];
+ assert_eq!(write["username"], USERNAME);
+ assert_eq!(write["manifests"].as_array().unwrap().len(), 1, "only the personal manifest changed");
+ assert_eq!(write["manifests"][0]["currentRevision"], 7);
+ assert_eq!(write["manifests"][0]["credentialsCount"], 2);
+ assert_eq!(write["buckets"].as_array().unwrap().len(), 0, "unchanged buckets stay out of the write");
+ assert_eq!(write["emailRouting"]["coveredManifestIds"][0], PERSONAL_MANIFEST_ID);
+ assert_eq!(write["emailRouting"]["baseRevisions"][0]["revision"], 7, "routing carries the revision it was built from");
+ assert_eq!(s.host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 8);
+ assert_eq!(s.host.mark_clean_calls, vec![1]);
+ assert!(!s.host.is_dirty);
+
+ // The written manifest decrypts with the VEK and carries both items.
+ let manifest = s.server.borrow().open_personal(&s.vek);
+ assert_eq!(manifest["tables"]["Items"].as_array().unwrap().len(), 2);
+ s.server.borrow().assert_converged(&s.host);
+}
+
+#[test]
+fn no_op_mutation_clears_the_dirty_flag_without_a_write() {
+ let mut s = synced(|db| insert_item(db, ITEM_A, "Server item"));
+
+ // Marked dirty, nothing actually changed.
+ s.host.edit(|_| {});
+ let result = s.host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert!(s.host.vault_writes().is_empty(), "no write for a no-op mutation");
+ assert_eq!(s.host.mark_clean_calls, vec![1]);
+ assert!(!s.host.is_dirty);
+}
+
+#[test]
+fn push_creates_and_publishes_a_missing_personal_delivery_key() {
+ let mut s = synced(|db| insert_item(db, ITEM_A, "Server item"));
+ assert!(active_delivery_keys(&s.host.local, PERSONAL_MANIFEST_ID).is_empty());
+
+ s.host.edit(|db| rename_item(db, "Renamed"));
+ let result = s.host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ let keys = active_delivery_keys(&s.host.local, PERSONAL_MANIFEST_ID);
+ assert_eq!(keys.len(), 1, "the push created exactly one keypair");
+ let write = s.host.last_vault_write();
+ assert_eq!(write["manifests"][0]["deliveryPublicKey"], json!(keys[0]), "the write publishes the new public key");
+ assert_eq!(write["manifests"][0]["deliveryPublicKeyAlgorithm"], json!("rsa-oaep-sha256"), "the write states the key's algorithm");
+
+ // The keypair travels inside the manifest (so other devices get the private half) and sits in the stored vault.
+ assert_eq!(s.server.borrow().open_personal(&s.vek)["tables"]["EncryptionKeys"].as_array().unwrap().len(), 1);
+ assert_eq!(active_delivery_keys(&s.host.stored_vault(), PERSONAL_MANIFEST_ID), keys);
+
+ // The next push keeps the same keypair.
+ s.host.edit(|db| rename_item(db, "Renamed again"));
+ let again = s.host.sync();
+ assert_eq!(again["success"], true, "{}", again);
+ assert_eq!(active_delivery_keys(&s.host.local, PERSONAL_MANIFEST_ID), keys);
+ assert_eq!(s.host.last_vault_write()["manifests"][0]["deliveryPublicKey"], json!(keys[0]));
+}
+
+#[test]
+fn a_server_queued_delivery_key_rotation_is_carried_out_and_acknowledged() {
+ // Only a known action for a manifest this session administers is carried out and deleted; the others stay queued.
+ let mut s = synced(|db| insert_item(db, ITEM_A, "Server item"));
+ let record = json!({ "manifestId": SHARED_MANIFEST_ID, "encryptedVek": "x", "accountPublicKey": "x", "algorithm": "rsa-oaep-sha256", "salt": "x", "canAdminister": true });
+ let records = json!({ SHARED_MANIFEST_ID: record });
+ s.host.set_state(state::SHARED_MANIFESTS, json!(crypto::symmetric_encrypt(&records.to_string(), &s.vek).unwrap()));
+ s.server.borrow_mut().pending_actions = vec![
+ json!({ "id": "a-rotate", "type": "rotate-manifest-delivery-key", "manifestId": SHARED_MANIFEST_ID }),
+ json!({ "id": "a-unknown", "type": "future-action", "manifestId": SHARED_MANIFEST_ID }),
+ json!({ "id": "a-foreign", "type": "rotate-manifest-delivery-key", "manifestId": "33333333-3333-4333-8333-333333333333" }),
+ ];
+
+ let result = s.host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(s.host.request_log().into_iter().filter(|r| r.starts_with("DELETE")).collect::<Vec<_>>(), vec!["DELETE ClientActions/a-rotate"]);
+ let left: Vec<Value> = s.server.borrow().pending_actions.iter().map(|a| a["id"].clone()).collect();
+ assert_eq!(left, vec![json!("a-unknown"), json!("a-foreign")]);
+ assert_eq!(active_delivery_keys(&s.host.local, SHARED_MANIFEST_ID).len(), 1, "the shared manifest got a fresh delivery keypair");
+}
+
+#[test]
+fn bucket_only_push_writes_the_manifest_while_the_personal_delivery_key_is_missing() {
+ let mut s = synced(|db| insert_item(db, ITEM_A, "Server item"));
+ let category = vault_codec::bucket_layout()[0].category.clone();
+
+ s.host.edit(|db| insert_item_stats(db, ITEM_A, 1));
+ let result = s.host.sync_scoped(&[category.as_str()]);
+
+ assert_eq!(result["success"], true, "{}", result);
+ let keys = active_delivery_keys(&s.host.local, PERSONAL_MANIFEST_ID);
+ assert_eq!(keys.len(), 1);
+ assert_eq!(s.host.last_vault_write()["manifests"][0]["deliveryPublicKey"], json!(keys[0]), "the full write publishes the key a bucket-only write could not");
+}
+
+#[test]
+fn outdated_push_merges_the_server_change_and_retries() {
+ let mut s = synced(|db| insert_item(db, ITEM_A, "Server item"));
+
+ // Another device adds an item on the server (revision 8) after this one's status check, while this one adds a different item.
+ let stale_status = s.server.borrow().status();
+ insert_item(&s.db, ITEM_C, "Other device item");
+ s.publish(8);
+ s.host.respond_once("GET", "Status", stale_status);
+ s.host.edit(|db| insert_item(db, ITEM_B, "Local item"));
+
+ let result = s.host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["hasNewVault"], true);
+ assert_eq!(s.host.item_names(), vec!["Local item", "Other device item", "Server item"]);
+ assert_eq!(s.host.state[state::SERVER_MANIFEST_REVISIONS][PERSONAL_MANIFEST_ID], 9);
+ let writes = s.host.vault_writes();
+ assert_eq!(writes.len(), 2, "the refused write and the one after the merge");
+ assert_eq!(writes[0]["manifests"][0]["currentRevision"], 7);
+ assert_eq!(writes[1]["manifests"][0]["currentRevision"], 8);
+ assert_eq!(s.server.borrow().open_personal(&s.vek)["tables"]["Items"].as_array().unwrap().len(), 3);
+ s.server.borrow().assert_converged(&s.host);
+}
+
+#[test]
+fn outdated_bucket_only_push_merges_the_server_bucket_instead_of_overwriting_it() {
+ let mut s = synced(|db| {
+ insert_item(db, ITEM_A, "Item A");
+ insert_item(db, ITEM_B, "Item B");
+ });
+
+ // Another device writes the stats of item B after this client's status check, while this one used item A.
+ let stale_status = s.server.borrow().status();
+ insert_item_stats(&s.db, ITEM_B, 5);
+ s.publish(8);
+ s.host.respond_once("GET", "Status", stale_status);
+ s.host.edit(|db| insert_item_stats(db, ITEM_A, 1));
+
+ let result = s.host.sync_scoped(&["stats"]);
+
+ assert_eq!(result["success"], true, "{}", result);
+ let writes = s.host.vault_writes();
+ assert_eq!(writes.len(), 2);
+ assert_eq!(writes[0]["buckets"][0]["currentRevision"], 7);
+ assert_eq!(writes[1]["buckets"][0]["currentRevision"], 8);
+ let bucket = s.server.borrow().open_bucket("stats", &s.vek);
+ assert_eq!(bucket["tables"]["ItemStats"].as_array().unwrap().len(), 2, "the other device's stats must survive: {}", bucket);
+ let tail: Vec<String> = s.host.request_log().into_iter().rev().take(4).rev().collect();
+ assert_eq!(tail, vec!["POST Vault", "GET Status", "GET Vault", "POST Vault"], "the refused write must be followed by a pull before the next attempt");
+ // The first attempt created the missing delivery keypair; the write after the merge carries it although the host
+ // only recorded a bucket scope, so the server ends up with everything the merge changed.
+ assert_eq!(writes[1]["manifests"].as_array().unwrap().len(), 1);
+ s.server.borrow().assert_converged(&s.host);
+}
+
+#[test]
+fn a_write_the_server_keeps_refusing_gives_up_after_three_resyncs() {
+ let mut s = synced(|db| insert_item(db, ITEM_A, "Server item"));
+ s.server.borrow_mut().faults.refuse_writes = 10;
+
+ s.host.edit(|db| insert_item(db, ITEM_B, "Local item"));
+ let result = s.host.sync();
+
+ assert_eq!(result["success"], false, "{}", result);
+ assert_eq!(result["errorCode"], "E-702");
+ assert_eq!(s.host.vault_writes().len(), 4, "the first attempt plus three re-syncs");
+ assert!(s.host.is_dirty, "the local change is kept for the next sync");
+}
+
+#[test]
+fn unreachable_server_with_a_local_vault_goes_offline() {
+ let mut host = TestHost::new(&crypto::generate_key_base64());
+ host.store_local_as_blob();
+
+ let result = host.sync();
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["wasOffline"], true);
+ assert_eq!(result["isOfflineMode"], true);
+}
+
+#[test]
+fn expired_session_requires_logout() {
+ let server = FakeServer::new();
+ server.borrow_mut().faults.status_http = Some(401);
+ let mut host = TestHost::logged_in(&crypto::generate_key_base64(), &server);
+
+ let result = host.sync();
+
+ assert_eq!(result["success"], false);
+ assert_eq!(result["requiresLogout"], true);
+ assert_eq!(result["logoutReason"], "sessionExpired");
+}
+
+#[test]
+fn password_changed_elsewhere_requires_logout() {
+ let server = FakeServer::new();
+ server.borrow_mut().srp_salt = "new-salt".to_string();
+ let mut host = TestHost::logged_in(&crypto::generate_key_base64(), &server);
+ host.set_state(state::UNLOCK_KEY_DERIVATION_PARAMS, json!({ "salt": "old-salt", "encryptionType": "Argon2Id", "encryptionSettings": "{}" }));
+
+ let result = host.sync();
+
+ assert_eq!(result["requiresLogout"], true);
+ assert_eq!(result["logoutReason"], "passwordChanged");
+}
+
+#[test]
+fn status_check_reports_newer_server_state_without_touching_the_vault() {
+ let mut s = synced(|db| insert_item(db, ITEM_A, "Server item"));
+ s.publish(8);
+ s.host.is_dirty = true;
+
+ let result = s.host.run("statusCheck");
+
+ assert_eq!(result["success"], true, "{}", result);
+ assert_eq!(result["hasNewerVault"], true);
+ assert_eq!(result["hasDirtyChanges"], true);
+ assert_eq!(result["isOffline"], false);
+ assert!(s.host.store_calls.is_empty());
+ assert_eq!(s.host.request_log(), vec!["GET Status"]);
+}
+
+#[test]
+fn session_reports_when_a_response_is_missing() {
+ let session = SyncSession::new(&request_json("fullSync", "key", false, 0).to_string()).unwrap();
+ let first: Value = serde_json::from_str(&session.next_command().unwrap()).unwrap();
+ assert_eq!(first["kind"], "log", "the engine announces the sync before touching the host");
+ assert!(session.next_command().is_err(), "the session refuses to advance without a response");
+ session.resume("{}", None).unwrap();
+ let second: Value = serde_json::from_str(&session.next_command().unwrap()).unwrap();
+ assert_eq!(second["kind"], "http");
+ assert_eq!(second["path"], "Status");
+}
diff --git a/core/rust/src/vault_sync/tests/test_host.rs b/core/rust/src/vault_sync/tests/test_host.rs
index 6a7b45c98..fae5d9fb2 100644
--- a/core/rust/src/vault_sync/tests/test_host.rs
+++ b/core/rust/src/vault_sync/tests/test_host.rs
@@ -1,15 +1,20 @@
-//! A reference host for the engine tests: real SQLite (rusqlite), in-memory state, scripted HTTP.
+//! A reference host for the engine tests: one device with real SQLite (rusqlite), in-memory state and HTTP
+//! answered by a [`FakeServer`](super::fake_server::FakeServer) or a scripted responder.
use std::collections::HashMap;
use rusqlite::{Connection, MAIN_DB};
use serde_json::{json, Map, Value};
+use super::fake_server::Server;
use crate::crypto;
-use crate::common::encoding::base64_decode;
use crate::sqlite_host::{self, SqlStatement};
-use crate::vault_sync::types::{Command, Db};
+use crate::vault_codec::{self, CodecTableData};
use crate::vault_sync::session::SyncSession;
+use crate::vault_sync::state;
+use crate::vault_sync::types::{Command, Db};
+
+pub const USERNAME: &str = "tester";
/// One recorded HTTP request.
#[derive(Debug, Clone)]
@@ -19,7 +24,7 @@ pub struct RecordedRequest {
pub body: Option<Value>,
}
-/// A scripted HTTP answer: status and JSON body for a (method, path) match.
+/// A scripted HTTP answer: status and JSON body for a (method, path) match, `None` to pass.
pub type Responder = Box<dyn Fn(&str, &str, Option<&Value>) -> Option<(u16, Value)>>;
pub struct TestHost {
@@ -27,16 +32,17 @@ pub struct TestHost {
pub staging: Option<Connection>,
pub state: HashMap<String, Value>,
pub vault_blob: Option<String>,
+ /// The key the stored vault is encrypted under; the session key `sync` and `run` hand to the engine.
pub vault_key: String,
pub mutation_sequence: u64,
pub is_dirty: bool,
pub mark_clean_calls: Vec<u64>,
pub store_calls: Vec<Command>,
- pub rekeyed_stores_found_the_chain: Vec<bool>,
+ /// Stores under a new key; the host asserts each one found the key chain cached first.
+ pub rekeyed_stores: usize,
pub requests: Vec<RecordedRequest>,
pub responders: Vec<Responder>,
pub logs: Vec<String>,
- pub schema_sql: String,
}
/// The current client schema, read from the TypeScript source of truth.
@@ -52,22 +58,49 @@ pub fn complete_schema_sql() -> String {
pub fn open_from_bytes(bytes: &[u8]) -> Connection {
let mut conn = Connection::open_in_memory().unwrap();
sqlite_host::deserialize_into(&mut conn, bytes).unwrap();
+ // Foreign keys off, like the app hosts, so a test can write the stray rows real clients produce.
+ conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();
conn
}
-/// A fresh database on the client schema similar to the app hosts.
-pub fn open_schema_db(schema_sql: &str) -> Connection {
+/// A fresh database on the client schema, foreign keys off like the app hosts.
+pub fn open_schema_db() -> Connection {
let conn = Connection::open_in_memory().unwrap();
+ conn.execute_batch(&complete_schema_sql()).unwrap();
+ // The schema turns foreign keys on; the engine checks them once after the inserts, like the app hosts.
conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();
- conn.execute_batch(schema_sql).unwrap();
conn
}
+/// Every user table of a connection in the codec's input shape.
+pub fn read_tables(conn: &Connection) -> Vec<CodecTableData> {
+ let names: Vec<String> = query(conn, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name", &[]).unwrap().iter().map(|r| r["name"].as_str().unwrap().to_string()).collect();
+ names
+ .into_iter()
+ .filter(|name| !vault_codec::is_skip_table(name))
+ .map(|name| CodecTableData { records: query(conn, &format!("SELECT * FROM \"{}\"", name), &[]).unwrap().into_iter().map(|row| row.into_iter().collect()).collect(), name })
+ .collect()
+}
+
+/// The names of the live items in a database, sorted.
+pub fn item_names(conn: &Connection) -> Vec<String> {
+ let mut names: Vec<String> = query(conn, "SELECT Name FROM Items WHERE IsDeleted = 0", &[]).unwrap().iter().map(|r| r["Name"].as_str().unwrap().to_string()).collect();
+ names.sort();
+ names
+}
+
+/// The first column of every row of `sql`, as text, sorted.
+pub fn column(conn: &Connection, sql: &str) -> Vec<String> {
+ let mut out: Vec<String> = query(conn, sql, &[]).unwrap().iter().map(|r| r.values().next().map(|v| v.as_str().map(str::to_string).unwrap_or_else(|| v.to_string())).unwrap_or_default()).collect();
+ out.sort();
+ out
+}
+
impl TestHost {
+ /// A device holding an empty vault database under `vault_key`, with no at-rest blob stored yet.
pub fn new(vault_key: &str) -> Self {
- let schema_sql = complete_schema_sql();
Self {
- local: open_schema_db(&schema_sql),
+ local: open_schema_db(),
staging: None,
state: HashMap::new(),
vault_blob: None,
@@ -76,21 +109,41 @@ impl TestHost {
is_dirty: false,
mark_clean_calls: Vec::new(),
store_calls: Vec::new(),
- rekeyed_stores_found_the_chain: Vec::new(),
+ rekeyed_stores: 0,
requests: Vec::new(),
responders: Vec::new(),
logs: Vec::new(),
- schema_sql,
}
}
- /// Answer requests whose method and path match.
+ /// A device under `vault_key` that talks to `server` and holds a cached key chain, like a device that logged in.
+ pub fn logged_in(vault_key: &str, server: &Server) -> Self {
+ let mut host = Self::new(vault_key);
+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
+ host.serve(server);
+ host
+ }
+
+ /// Route every request this host makes to `server`.
+ pub fn serve(&mut self, server: &Server) {
+ let server = server.clone();
+ self.responders.push(Box::new(move |method, path, body| server.borrow_mut().respond(method, path, body)));
+ }
+
+ /// Answer requests whose method and path match, ahead of the server.
pub fn respond(&mut self, method: &'static str, path: &'static str, body: Value) {
- self.responders.push(Box::new(move |m, p, _| if m == method && p == path { Some((200, body.clone())) } else { None }));
+ self.responders.insert(0, Box::new(move |m, p, _| if m == method && p == path { Some((200, body.clone())) } else { None }));
}
- pub fn respond_with(&mut self, responder: Responder) {
- self.responders.push(responder);
+ /// Answer the next matching request once, ahead of the server; the server answers from then on.
+ pub fn respond_once(&mut self, method: &'static str, path: &'static str, body: Value) {
+ let spent = std::cell::Cell::new(false);
+ self.responders.insert(0, Box::new(move |m, p, _| if m == method && p == path && !spent.replace(true) { Some((200, body.clone())) } else { None }));
+ }
+
+ /// Record a value in the engine's persisted state.
+ pub fn set_state(&mut self, key: &str, value: Value) {
+ self.state.insert(key.to_string(), value);
}
/// Encrypt the local database as the at-rest blob, the way the app stores it.
@@ -99,6 +152,49 @@ impl TestHost {
self.vault_blob = Some(crypto::symmetric_encrypt_bytes(&bytes, &self.vault_key).unwrap());
}
+ /// Change the local vault the way the app does: write rows, store the blob, count the mutation, mark dirty.
+ pub fn edit(&mut self, change: impl FnOnce(&Connection)) {
+ change(&self.local);
+ self.store_local_as_blob();
+ self.mutation_sequence += 1;
+ self.is_dirty = true;
+ }
+
+ /// A full sync with the dirty state this host is in, under its own vault key.
+ pub fn sync(&mut self) -> Value {
+ let key = self.vault_key.clone();
+ self.sync_as(&key)
+ }
+
+ /// A full sync under another session key than the vault is stored with.
+ pub fn sync_as(&mut self, key: &str) -> Value {
+ let scopes: Vec<&str> = if self.is_dirty { vec!["Main"] } else { vec![] };
+ self.sync_scoped_as(key, &scopes)
+ }
+
+ /// A full sync that names the dirty scopes the app recorded.
+ pub fn sync_scoped(&mut self, scopes: &[&str]) -> Value {
+ let key = self.vault_key.clone();
+ self.sync_scoped_as(&key, scopes)
+ }
+
+ fn sync_scoped_as(&mut self, key: &str, scopes: &[&str]) -> Value {
+ let mut request = request_json("fullSync", key, self.is_dirty, self.mutation_sequence);
+ request["dirtyScopes"] = json!(scopes);
+ self.drive(&SyncSession::new(&request.to_string()).unwrap())
+ }
+
+ /// Any other operation (`migrationStatus`, `migrateManifest`, `statusCheck`, `resolveVaultKey`) under the vault key.
+ pub fn run(&mut self, operation: &str) -> Value {
+ let key = self.vault_key.clone();
+ self.run_as(operation, &key)
+ }
+
+ pub fn run_as(&mut self, operation: &str, key: &str) -> Value {
+ let request = request_json(operation, key, self.is_dirty, self.mutation_sequence);
+ self.drive(&SyncSession::new(&request.to_string()).unwrap())
+ }
+
/// Drive a session to completion and return the `done` result.
pub fn drive(&mut self, session: &SyncSession) -> Value {
loop {
@@ -124,9 +220,7 @@ impl TestHost {
json!({})
}
Command::DbOpen { .. } => {
- let conn = open_schema_db(&self.schema_sql);
- conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();
- self.staging = Some(conn);
+ self.staging = Some(open_schema_db());
json!({})
}
Command::DbQuery { db, .. } | Command::DbExec { db, .. } | Command::DbExport { db } if db == Db::Local && self.vault_blob.is_none() => {
@@ -151,9 +245,9 @@ impl TestHost {
self.store_calls.push(Command::VaultStore { encrypted_blob: String::new(), mark_dirty, encryption_key: encryption_key.clone(), expected_mutation_seq, revision });
if let Some(key) = encryption_key {
// The app hosts hold the unlock key and derive the vault key from the cached chain.
- let found_the_chain = self.state.contains_key("encryptedAccountKey") && self.state.contains_key("encryptedVek");
+ let found_the_chain = self.state.contains_key(state::ENCRYPTED_ACCOUNT_KEY) && self.state.contains_key(state::ENCRYPTED_VEK);
assert!(found_the_chain, "a vault stored under a new key needs the chain that opens it cached first");
- self.rekeyed_stores_found_the_chain.push(found_the_chain);
+ self.rekeyed_stores += 1;
self.vault_key = key;
}
// Like the app hosts: a sync store is refused after a mutation, and either kind may mark the vault dirty.
@@ -189,7 +283,7 @@ impl TestHost {
/// Store a blob and reload `local` from it, as the contract requires.
fn store_blob(&mut self, encrypted_blob: &str) {
self.vault_blob = Some(encrypted_blob.to_string());
- let bytes = crypto::symmetric_decrypt_bytes(&base64_decode(encrypted_blob).unwrap(), &self.vault_key).expect("stored blob decrypts with the vault key");
+ let bytes = state::decrypt_vault_blob(encrypted_blob, &self.vault_key).expect("stored blob decrypts with the vault key");
self.local = open_from_bytes(&bytes);
}
@@ -200,9 +294,51 @@ impl TestHost {
}
}
+ /*
+ * Readers.
+ */
+
+ pub fn item_names(&self) -> Vec<String> {
+ item_names(&self.local)
+ }
+
pub fn requests_to(&self, path: &str) -> Vec<&RecordedRequest> {
self.requests.iter().filter(|r| r.path == path).collect()
}
+
+ /// The `method path` of every request made, in order.
+ pub fn request_log(&self) -> Vec<String> {
+ self.requests.iter().map(|r| format!("{} {}", r.method, r.path)).collect()
+ }
+
+ /// The bodies of every `POST Vault`, in order.
+ pub fn vault_writes(&self) -> Vec<Value> {
+ self.requests.iter().filter(|r| r.method == "POST" && r.path == "Vault").map(|r| r.body.clone().unwrap()).collect()
+ }
+
+ pub fn last_vault_write(&self) -> Value {
+ self.vault_writes().pop().expect("a vault write")
+ }
+
+ /// The stored at-rest vault decrypted and opened.
+ pub fn stored_vault(&self) -> Connection {
+ let bytes = state::decrypt_vault_blob(self.vault_blob.as_ref().expect("a stored vault"), &self.vault_key).unwrap();
+ open_from_bytes(&bytes)
+ }
+}
+
+/// The JSON of a sync request.
+pub fn request_json(operation: &str, key: &str, dirty: bool, mutation_sequence: u64) -> Value {
+ json!({
+ "operation": operation,
+ "username": USERNAME,
+ "encryptionKey": key,
+ "isDirty": dirty,
+ "mutationSequence": mutation_sequence,
+ "dirtyScopes": if dirty { vec!["Main"] } else { vec![] },
+ "privateEmailDomains": ["private.io"],
+ "minServerVersion": "0.12.0-dev",
+ })
}
pub fn query(conn: &Connection, sql: &str, params: &[Value]) -> Result<Vec<Map<String, Value>>, String> {
diff --git a/core/rust/src/vault_sync/tests/unloaded_blobs.rs b/core/rust/src/vault_sync/tests/unloaded_blobs.rs
deleted file mode 100644
index 9668c0251..000000000
--- a/core/rust/src/vault_sync/tests/unloaded_blobs.rs
+++ /dev/null
@@ -1,262 +0,0 @@
-//! Blob bytes are optional locally: a blob that is not loaded (not served, or not decrypting with the key) neither
-//! fails a pull or a merge nor loses its reference on the next push.
-
-use serde_json::{json, Value};
-
-use super::test_host::{self, query, TestHost};
-use super::{insert_delivery_key, insert_item, item_names, read_tables, request, PERSONAL_MANIFEST_ID};
-use crate::crypto;
-use crate::vault_codec::{self, CanonicalizeInput, ManifestSpec};
-use crate::vault_sync::blob_keys;
-use crate::vault_sync::session::SyncSession;
-use crate::vault_sync::state;
-
-const ITEM_ID: &str = "aaaaaaaa-0000-4000-8000-000000000001";
-
-/// One blob as the server holds it.
-struct ServerBlob {
- hash: String,
- kind: String,
- ciphertext: String,
- encrypted_blob_key: String,
-}
-
-impl ServerBlob {
- /// The blob as the download endpoint serves it.
- fn served(&self) -> Value {
- json!({ "hash": self.hash, "category": self.kind, "encryptedDataBase64": self.ciphertext, "encryptedBlobKey": self.encrypted_blob_key })
- }
-}
-
-/// A server vault holding one item with a logo and an attachment.
-fn server_db(host: &TestHost) -> rusqlite::Connection {
- let db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&db, ITEM_ID, "Server item", PERSONAL_MANIFEST_ID);
- insert_delivery_key(&db, PERSONAL_MANIFEST_ID);
- let now = crate::common::timestamp::now_vault_datetime();
- db.execute("INSERT INTO Logos (ManifestId, Id, Source, FileData, Kind, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, 'bbbbbbbb-0000-4000-8000-000000000001', 'example.com', ?, 'favicon', ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, vec![1u8, 2, 3, 4], now, now]).unwrap();
- db.execute("UPDATE Items SET LogoId = 'bbbbbbbb-0000-4000-8000-000000000001'", []).unwrap();
- db.execute("INSERT INTO Attachments (ManifestId, Id, ItemId, Filename, Blob, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, 'cccccccc-0000-4000-8000-000000000001', ?, 'passport.pdf', ?, ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, ITEM_ID, vec![9u8; 64], now, now]).unwrap();
- db
-}
-
-/// The status and vault responses of a server holding `conn` at `revision`, and its blobs, their keys encrypted with `blob_key`.
-fn snapshot(conn: &rusqlite::Connection, vek: &str, blob_key: &str, salt: &str, revision: i64) -> (Value, Value, Vec<ServerBlob>) {
- let canonicalized = vault_codec::canonicalize_from_sqlite(CanonicalizeInput {
- tables: read_tables(conn),
- canonicalized_at: "2026-09-11T00:00:00.000Z".to_string(),
- manifests: vec![ManifestSpec { manifest_id: PERSONAL_MANIFEST_ID.to_string(), manifest_salt: salt.to_string(), name: None }],
- stamp_unstamped_into: None,
- })
- .unwrap();
- let entry = &canonicalized.manifests[0];
- let blob = crypto::symmetric_encrypt_bytes_with_aad(&vault_codec::pack_payload(&serde_json::to_string(&entry.manifest).unwrap()).unwrap(), vek, &crypto::aad::manifest(PERSONAL_MANIFEST_ID)).unwrap();
- let blobs: Vec<ServerBlob> = entry
- .blobs
- .iter()
- .map(|(hash, b)| {
- let encrypted = blob_keys::encrypt_blob(&crate::common::encoding::base64_decode(&b.bytes_base64).unwrap(), blob_key, PERSONAL_MANIFEST_ID, hash).unwrap();
- ServerBlob { hash: hash.clone(), kind: b.kind.clone(), ciphertext: encrypted.encrypted_data_base64, encrypted_blob_key: encrypted.encrypted_blob_key }
- })
- .collect();
- let references: Vec<Value> = blobs.iter().map(|b| json!({ "hash": b.hash, "category": b.kind, "sizeBytes": 64 })).collect();
- let status = json!({
- "clientVersionSupported": true,
- "serverVersion": "0.31.0",
- "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": revision }],
- "bucketRevisions": [],
- "personalManifestId": PERSONAL_MANIFEST_ID,
- "srpSalt": "salt",
- "capabilities": { "sharing": "on" },
- });
- let vault = json!({
- "storageFormat": "manifest",
- "personalManifestId": PERSONAL_MANIFEST_ID,
- "manifests": [{ "manifestId": PERSONAL_MANIFEST_ID, "blob": blob, "ciphertextHash": vault_codec::compute_ciphertext_hash(&blob), "revision": revision, "blobReferences": references, "canAdminister": true, "keyType": "account-key" }],
- "buckets": [],
- "emailRouting": { "privateEmailDomainList": ["private.io"], "publicEmailDomainList": [], "hiddenPrivateEmailDomainList": [], "emailAddressList": [] },
- });
- (status, vault, blobs)
-}
-
-/// A host that pulled the server vault at revision 7 while none of its blobs loaded: not served at all, or (with
-/// `serve_undecryptable`) served under a key this device does not hold.
-fn host_with_unloaded_blobs(vek: &str, serve_undecryptable: bool) -> (TestHost, Vec<ServerBlob>) {
- let mut host = TestHost::new(vek);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- let blob_key = if serve_undecryptable { crypto::generate_key_base64() } else { vek.to_string() };
- let (status, vault, blobs) = snapshot(&server_db(&host), vek, &blob_key, &vault_codec::generate_manifest_salt(), 7);
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
- let served: Vec<Value> = if serve_undecryptable { blobs.iter().map(ServerBlob::served).collect() } else { Vec::new() };
- host.respond("POST", "Vault/blobs/download", json!({ "blobs": served }));
-
- let pulled = host.drive(&SyncSession::new(&request("fullSync", vek, false, 0)).unwrap());
- assert_eq!(pulled["success"], true, "a blob that does not load must not fail the pull: {}", pulled);
- assert_eq!(item_names(&host.local), vec!["Server item"]);
- let not_loaded = query(&host.local, "SELECT Id FROM Attachments WHERE IsDeleted = 0 AND Blob IS NULL AND BlobHash IS NOT NULL", &[]).unwrap();
- assert_eq!(not_loaded.len(), 1, "the attachment row is there, not loaded, and knows its blob");
- (host, blobs)
-}
-
-/// Mark the local vault changed and sync; the server reports every hash it is asked about as missing.
-fn sync_dirty(host: &mut TestHost, vek: &str) -> Value {
- host.store_local_as_blob();
- host.mutation_sequence += 1;
- host.is_dirty = true;
- host.respond_with(Box::new(|method, path, body| (method == "POST" && path == "Vault/blobs/missing").then(|| (200, json!({ "missing": body.map(|b| b["hashes"].clone()).unwrap_or_default() })))));
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 9 }], "bucketRevisions": [], "missingBlobHashes": [] }));
- host.drive(&SyncSession::new(&request("fullSync", vek, true, host.mutation_sequence)).unwrap())
-}
-
-/// The blob hashes the last vault write referenced.
-fn written_hashes(host: &TestHost) -> Vec<String> {
- let write = host.requests_to("Vault").into_iter().filter(|r| r.method == "POST").next_back().expect("a vault write").body.clone().unwrap();
- write["manifests"][0]["blobReferences"].as_array().unwrap().iter().map(|r| r["hash"].as_str().unwrap().to_string()).collect()
-}
-
-fn sorted_hashes(blobs: &[ServerBlob]) -> Vec<String> {
- let mut hashes: Vec<String> = blobs.iter().map(|b| b.hash.clone()).collect();
- hashes.sort();
- hashes
-}
-
-#[test]
-fn blobs_the_server_does_not_serve_neither_fail_the_pull_nor_lose_their_reference() {
- let vek = crypto::generate_key_base64();
- let (mut host, blobs) = host_with_unloaded_blobs(&vek, false);
-
- host.local.execute("UPDATE Items SET Name = 'Renamed', UpdatedAt = '2099-01-01 00:00:00.000'", []).unwrap();
- let pushed = sync_dirty(&mut host, &vek);
-
- assert_eq!(pushed["success"], true, "{}", pushed);
- assert_eq!(written_hashes(&host), sorted_hashes(&blobs), "a push may not drop the reference to a blob that is not loaded");
- assert!(host.requests_to("Vault/blobs").is_empty(), "there are no bytes to upload for a blob this device never loaded");
-}
-
-/// Pull the server vault at revision 7 with `served` as the download response, and return the loaded attachment bytes.
-fn pull_attachment_bytes(vek: &str, serve: impl Fn(&[ServerBlob]) -> Vec<Value>) -> Vec<Value> {
- let mut host = TestHost::new(vek);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- let (status, vault, blobs) = snapshot(&server_db(&host), vek, vek, &vault_codec::generate_manifest_salt(), 7);
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
- host.respond("POST", "Vault/blobs/download", json!({ "blobs": serve(&blobs) }));
-
- let pulled = host.drive(&SyncSession::new(&request("fullSync", vek, false, 0)).unwrap());
- assert_eq!(pulled["success"], true, "{}", pulled);
- query(&host.local, "SELECT hex(Blob) AS Bytes FROM Attachments WHERE IsDeleted = 0", &[]).unwrap().into_iter().map(|row| row["Bytes"].clone()).collect()
-}
-
-#[test]
-fn a_pulled_blob_opens_through_its_own_key() {
- let vek = crypto::generate_key_base64();
- let loaded = pull_attachment_bytes(&vek, |blobs| blobs.iter().map(ServerBlob::served).collect());
- assert_eq!(loaded, vec![json!("09".repeat(64))]);
-}
-
-#[test]
-fn blobs_that_do_not_decrypt_neither_fail_the_pull_nor_lose_their_reference() {
- let vek = crypto::generate_key_base64();
- let (mut host, blobs) = host_with_unloaded_blobs(&vek, true);
-
- host.local.execute("UPDATE Items SET Name = 'Renamed', UpdatedAt = '2099-01-01 00:00:00.000'", []).unwrap();
- let pushed = sync_dirty(&mut host, &vek);
-
- assert_eq!(pushed["success"], true, "{}", pushed);
- assert_eq!(written_hashes(&host), sorted_hashes(&blobs));
-}
-
-#[test]
-fn a_merge_keeps_the_reference_to_a_blob_that_is_not_loaded() {
- let vek = crypto::generate_key_base64();
- let (mut host, blobs) = host_with_unloaded_blobs(&vek, false);
-
- // A local edit, while the server moved on to revision 8 with another item added.
- host.local.execute("UPDATE Items SET Name = 'Renamed locally', UpdatedAt = '2099-01-01 00:00:00.000'", []).unwrap();
- let salt = host.state[state::VAULT_MANIFEST_SALT].as_str().unwrap().to_string();
- let server = server_db(&host);
- insert_item(&server, "aaaaaaaa-0000-4000-8000-000000000002", "Added elsewhere", PERSONAL_MANIFEST_ID);
- let (status, vault, _) = snapshot(&server, &vek, &vek, &salt, 8);
- host.responders.clear();
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
- host.respond("POST", "Vault/blobs/download", json!({ "blobs": [] }));
-
- let merged = sync_dirty(&mut host, &vek);
-
- assert_eq!(merged["success"], true, "{}", merged);
- assert_eq!(item_names(&host.local), vec!["Added elsewhere", "Renamed locally"], "the merge went through, it did not fall back to the server vault");
- assert_eq!(written_hashes(&host), sorted_hashes(&blobs));
-}
-
-#[test]
-fn deleting_an_attachment_that_is_not_loaded_releases_its_reference() {
- let vek = crypto::generate_key_base64();
- let (mut host, _) = host_with_unloaded_blobs(&vek, false);
-
- host.local.execute("UPDATE Attachments SET IsDeleted = 1, Blob = NULL, UpdatedAt = '2099-01-01 00:00:00.000'", []).unwrap();
- let pushed = sync_dirty(&mut host, &vek);
-
- assert_eq!(pushed["success"], true, "{}", pushed);
- assert_eq!(written_hashes(&host).len(), 1, "only the logo's reference is left");
-}
-
-#[test]
-fn a_save_that_changed_nothing_is_not_written_again() {
- let vek = crypto::generate_key_base64();
- let (mut host, _) = host_with_unloaded_blobs(&vek, false);
-
- // The local hash column may not read as a content change.
- let result = sync_dirty(&mut host, &vek);
-
- assert_eq!(result["success"], true, "{}", result);
- let rewritten = host.requests_to("Vault").iter().filter(|r| r.method == "POST").any(|r| !r.body.as_ref().unwrap()["manifests"].as_array().unwrap().is_empty());
- assert!(!rewritten, "the manifest did not change, so it is not written again");
-}
-
-#[test]
-fn a_key_migration_push_is_refused_while_a_personal_blob_is_not_loaded() {
- // The migration encrypts every personal blob again under the new key; one it has no bytes for would stay under
- // the old key and never open again. The vault stays dirty and the migration runs again once the blob loads.
- let unlock_key = crypto::generate_key_base64();
- let mut host = TestHost::new(&unlock_key);
- insert_item(&host.local, ITEM_ID, "Old item", PERSONAL_MANIFEST_ID);
- let now = crate::common::timestamp::now_vault_datetime();
- host.local.execute("INSERT INTO Attachments (ManifestId, Id, ItemId, Filename, Blob, BlobHash, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, 'cccccccc-0000-4000-8000-000000000001', ?, 'passport.pdf', NULL, 'hash-of-a-blob-that-is-not-loaded', ?, ?, 0)", rusqlite::params![PERSONAL_MANIFEST_ID, ITEM_ID, now, now]).unwrap();
- host.store_local_as_blob();
- host.state.insert(state::SERVER_MANIFEST_REVISIONS.to_string(), json!({ PERSONAL_MANIFEST_ID: 3 }));
- host.state.insert(state::VAULT_PERSONAL_MANIFEST_ID.to_string(), json!(PERSONAL_MANIFEST_ID));
- host.state.insert(state::VAULT_MANIFEST_SALT.to_string(), json!(vault_codec::generate_manifest_salt()));
- host.respond_with(Box::new(|method, path, _| (method == "GET" && path == "VaultKey/Password").then(|| (200, json!({ "vaultKey": null })))));
- host.respond("POST", "Vault/blobs/missing", json!({ "missing": [] }));
- host.respond("POST", "Vault", json!({ "status": "ok", "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 4 }], "bucketRevisions": [], "missingBlobHashes": [] }));
-
- let result = host.drive(&SyncSession::new(&request("migrateManifest", &unlock_key, false, 0)).unwrap());
-
- assert_eq!(result["pushed"], false, "{}", result);
- assert!(host.requests_to("Vault").iter().all(|r| r.method != "POST"), "nothing was written with the new key");
- assert_eq!(host.vault_key, unlock_key, "and the session keeps its key");
-}
-
-#[test]
-fn an_attachment_row_that_references_no_blob_does_not_fail_the_pull() {
- // A row some client pushed without a blob (an empty file, a writer bug) is one attachment that cannot be opened,
- // not a vault that cannot be pulled any more.
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- let server = server_db(&host);
- server.execute("UPDATE Attachments SET Blob = NULL", []).unwrap();
- let (status, vault, _) = snapshot(&server, &vek, &vek, &vault_codec::generate_manifest_salt(), 7);
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
- host.respond("POST", "Vault/blobs/download", json!({ "blobs": [] }));
-
- let pulled = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
-
- assert_eq!(pulled["success"], true, "{}", pulled);
- let rows = query(&host.local, "SELECT Filename, Blob, BlobHash FROM Attachments WHERE IsDeleted = 0", &[]).unwrap();
- assert_eq!(rows.len(), 1);
- assert_eq!((&rows[0]["Blob"], &rows[0]["BlobHash"]), (&Value::Null, &Value::Null));
-}
diff --git a/core/rust/src/vault_sync/tests/unsupported_grant.rs b/core/rust/src/vault_sync/tests/unsupported_grant.rs
deleted file mode 100644
index f0258ad71..000000000
--- a/core/rust/src/vault_sync/tests/unsupported_grant.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-//! A manifest granted in a format this build does not know ends the session with "update the app", like a newer
-//! vault format. A grant-format change ships with a minimum client version bump, so this only catches a missed bump.
-
-use serde_json::{json, Value};
-
-use super::test_host::{self, TestHost};
-use super::{insert_item, item_names, request, snapshot_of, PERSONAL_MANIFEST_ID};
-use crate::crypto;
-use crate::vault_codec;
-use crate::vault_sync::session::SyncSession;
-use crate::vault_sync::state;
-
-const SHARED_MANIFEST_ID: &str = "22222222-2222-4222-8222-222222222222";
-
-/// Sync a fresh client against a personal snapshot plus a shared manifest granted under `algorithm` and `key_type`.
-fn sync_with_shared_grant(algorithm: &str, key_type: &str, blob: &str) -> (TestHost, Value) {
- sync_with_shared_manifest(json!({ "manifestId": SHARED_MANIFEST_ID, "blob": blob, "revision": 3, "blobReferences": [], "canAdminister": false, "keyType": key_type, "algorithm": algorithm, "encryptedVek": "future-grant", "accountPublicKey": "future-key" }))
-}
-
-/// Sync a fresh client against a personal snapshot plus the given shared manifest entry.
-fn sync_with_shared_manifest(shared: Value) -> (TestHost, Value) {
- let vek = crypto::generate_key_base64();
- let mut host = TestHost::new(&vek);
- host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
- let server_db = test_host::open_schema_db(&host.schema_sql);
- insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);
- let (mut status, mut vault) = snapshot_of(&server_db, &vek, 7, &vault_codec::generate_manifest_salt());
- vault["manifests"].as_array_mut().unwrap().push(shared);
- status["manifestRevisions"].as_array_mut().unwrap().push(json!({ "manifestId": SHARED_MANIFEST_ID, "revision": 3 }));
- host.respond("GET", "Status", status);
- host.respond("GET", "Vault", vault);
- let result = host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
- (host, result)
-}
-
-fn assert_told_to_update(host: &TestHost, result: &Value) {
- assert_eq!(result["success"], false, "{}", result);
- assert_eq!(result["logoutReason"], "vaultVersionIncompatible", "an app too old for the grant format is told to update: {}", result);
- assert!(item_names(&host.local).is_empty(), "nothing is materialized, not even the personal vault");
- assert!(host.store_calls.is_empty());
-}
-
-#[test]
-fn unknown_grant_algorithm_asks_for_an_app_update() {
- let (host, result) = sync_with_shared_grant("future-x", "grant-key", "bm90LXJlYWRhYmxl");
- assert_told_to_update(&host, &result);
-}
-
-#[test]
-fn unknown_grant_key_type_asks_for_an_app_update() {
- let (host, result) = sync_with_shared_grant("rsa-oaep-sha256", "future-key-type", "bm90LXJlYWRhYmxl");
- assert_told_to_update(&host, &result);
-}
-
-#[test]
-fn unknown_grant_algorithm_on_a_contentless_shared_manifest_asks_for_an_app_update() {
- let (host, result) = sync_with_shared_grant("future-x", "grant-key", "");
- assert_told_to_update(&host, &result);
-}
-
-/// A shared manifest without a key type or grant algorithm is refused, never guessed.
-#[test]
-fn shared_manifest_without_key_type_or_algorithm_is_refused() {
- for missing in ["keyType", "algorithm"] {
- let mut shared = json!({ "manifestId": SHARED_MANIFEST_ID, "blob": "bm90LXJlYWRhYmxl", "revision": 3, "blobReferences": [], "canAdminister": false, "keyType": "grant-key", "algorithm": "rsa-oaep-sha256", "encryptedVek": "grant", "accountPublicKey": "key" });
- shared.as_object_mut().unwrap().remove(missing);
- let (host, result) = sync_with_shared_manifest(shared);
- assert_eq!(result["success"], false, "missing {}: {}", missing, result);
- assert!(item_names(&host.local).is_empty(), "missing {}: nothing is materialized", missing);
- }
-}
diff --git a/core/rust/src/vault_sync/types.rs b/core/rust/src/vault_sync/types.rs
index f1474e564..ca94ac56d 100644
--- a/core/rust/src/vault_sync/types.rs
+++ b/core/rust/src/vault_sync/types.rs
@@ -1,7 +1,6 @@
//! Every JSON shape the engine exchanges.
use std::collections::HashMap;
-use std::fmt;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
@@ -70,27 +69,11 @@ pub struct SyncRequest {
pub force_pull: bool,
#[serde(default)]
pub min_server_version: Option<String>,
- #[serde(default)]
- pub is_offline_mode: bool,
/// The target of a sharing operation; absent for every other operation.
#[serde(default)]
pub sharing: Option<SharingParams>,
}
-/// What every operation reports on top of its own outcome.
-#[derive(Debug, Clone, Default, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct SessionOutcome {
- /// Whether the stored vault changed; hosts reload what reads it (autofill stores, open UI).
- #[serde(default)]
- pub vault_changed: bool,
-}
-
-/// The result of one operation; the engine fills in the session outcome once the run is over.
-pub(crate) trait OperationResult: Serialize {
- fn session_mut(&mut self) -> &mut SessionOutcome;
-}
-
/// How a failed operation reports itself: a coded error the host translates, or a forced logout.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -123,6 +106,11 @@ impl From<&SyncError> for FailureFields {
}
}
+/*
+ * Operation results. The engine adds `vaultChanged` (whether the stored vault changed, so hosts reload what reads it)
+ * to every result when the run is over.
+ */
+
/// Outcome of a full sync.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -143,22 +131,13 @@ pub struct FullSyncResult {
pub pulled_revision: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub email_routing: Option<EmailRoutingDto>,
- #[serde(flatten)]
- pub session: SessionOutcome,
-}
-
-impl OperationResult for FullSyncResult {
- fn session_mut(&mut self) -> &mut SessionOutcome {
- &mut self.session
- }
}
/// The pending local migration, as the upgrade gate classifies it.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum MigrationKind {
- /// Nothing to migrate, or nothing that may be migrated yet: a vault still on the frozen sqlite-blob chain
- /// classifies as `None` because that chain has to bring it to 2.0.0 before either kind below can apply.
+ /// Nothing to migrate, or nothing yet (a vault still on the frozen sqlite-blob chain).
None,
/// The local schema predates the current full schema; rebuilt locally, no server involved.
SchemaRebuild,
@@ -171,14 +150,6 @@ pub enum MigrationKind {
#[serde(rename_all = "camelCase")]
pub struct MigrationStatusResult {
pub kind: MigrationKind,
- #[serde(flatten)]
- pub session: SessionOutcome,
-}
-
-impl OperationResult for MigrationStatusResult {
- fn session_mut(&mut self) -> &mut SessionOutcome {
- &mut self.session
- }
}
/// Outcome of the manifest migration.
@@ -190,14 +161,6 @@ pub struct MigrateManifestResult {
pub pushed: bool,
#[serde(flatten)]
pub failure: FailureFields,
- #[serde(flatten)]
- pub session: SessionOutcome,
-}
-
-impl OperationResult for MigrateManifestResult {
- fn session_mut(&mut self) -> &mut SessionOutcome {
- &mut self.session
- }
}
/// Outcome of the login-time key resolution: the vault key the host stores as its session key (the VEK behind
@@ -212,14 +175,6 @@ pub struct ResolveVaultKeyResult {
pub encryption_key: Option<String>,
#[serde(flatten)]
pub failure: FailureFields,
- #[serde(flatten)]
- pub session: SessionOutcome,
-}
-
-impl OperationResult for ResolveVaultKeyResult {
- fn session_mut(&mut self) -> &mut SessionOutcome {
- &mut self.session
- }
}
/// Outcome of a sharing operation.
@@ -236,14 +191,6 @@ pub struct SharingOperationResult {
/// words for. `INVITE_RECIPIENT_NOT_READY` is also reported when the engine sees the recipient has no key yet.
#[serde(flatten)]
pub failure: FailureFields,
- #[serde(flatten)]
- pub session: SessionOutcome,
-}
-
-impl OperationResult for SharingOperationResult {
- fn session_mut(&mut self) -> &mut SessionOutcome {
- &mut self.session
- }
}
/// Outcome of the lightweight status check.
@@ -260,14 +207,6 @@ pub struct StatusCheckResult {
pub server_version: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub capabilities: Option<HashMap<String, String>>,
- #[serde(flatten)]
- pub session: SessionOutcome,
-}
-
-impl OperationResult for StatusCheckResult {
- fn session_mut(&mut self) -> &mut SessionOutcome {
- &mut self.session
- }
}
/*
@@ -356,63 +295,26 @@ pub enum Command {
}
impl Command {
- pub fn kind(&self) -> CommandKind {
+ /// The command's wire tag (`dbExec`), as error reports name it.
+ pub fn name(&self) -> &'static str {
match self {
- Command::Http { .. } => CommandKind::Http,
- Command::StateGet { .. } => CommandKind::StateGet,
- Command::StateSet { .. } => CommandKind::StateSet,
- Command::StateRemove { .. } => CommandKind::StateRemove,
- Command::DbOpen { .. } => CommandKind::DbOpen,
- Command::DbQuery { .. } => CommandKind::DbQuery,
- Command::DbExec { .. } => CommandKind::DbExec,
- Command::DbExport { .. } => CommandKind::DbExport,
- Command::VaultStore { .. } => CommandKind::VaultStore,
- Command::VaultLoad => CommandKind::VaultLoad,
- Command::MarkClean { .. } => CommandKind::MarkClean,
- Command::Log { .. } => CommandKind::Log,
- Command::Done { .. } => CommandKind::Done,
+ Command::Http { .. } => "http",
+ Command::StateGet { .. } => "stateGet",
+ Command::StateSet { .. } => "stateSet",
+ Command::StateRemove { .. } => "stateRemove",
+ Command::DbOpen { .. } => "dbOpen",
+ Command::DbQuery { .. } => "dbQuery",
+ Command::DbExec { .. } => "dbExec",
+ Command::DbExport { .. } => "dbExport",
+ Command::VaultStore { .. } => "vaultStore",
+ Command::VaultLoad => "vaultLoad",
+ Command::MarkClean { .. } => "markClean",
+ Command::Log { .. } => "log",
+ Command::Done { .. } => "done",
}
}
}
-/// The kind of a command, for error reports.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum CommandKind {
- Http,
- StateGet,
- StateSet,
- StateRemove,
- DbOpen,
- DbQuery,
- DbExec,
- DbExport,
- VaultStore,
- VaultLoad,
- MarkClean,
- Log,
- Done,
-}
-
-impl fmt::Display for CommandKind {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.write_str(match self {
- CommandKind::Http => "http",
- CommandKind::StateGet => "stateGet",
- CommandKind::StateSet => "stateSet",
- CommandKind::StateRemove => "stateRemove",
- CommandKind::DbOpen => "dbOpen",
- CommandKind::DbQuery => "dbQuery",
- CommandKind::DbExec => "dbExec",
- CommandKind::DbExport => "dbExport",
- CommandKind::VaultStore => "vaultStore",
- CommandKind::VaultLoad => "vaultLoad",
- CommandKind::MarkClean => "markClean",
- CommandKind::Log => "log",
- CommandKind::Done => "done",
- })
- }
-}
-
/*
* Responses.
*/
@@ -495,8 +397,6 @@ pub struct StatusResponse {
#[serde(default)]
pub bucket_revisions: Vec<BucketRevision>,
#[serde(default)]
- pub personal_manifest_id: Option<String>,
- #[serde(default)]
pub srp_salt: Option<String>,
#[serde(default)]
pub capabilities: Option<HashMap<String, String>>,
@@ -529,8 +429,6 @@ pub struct PendingAction {
pub action_type: String,
#[serde(default)]
pub manifest_id: Option<String>,
- #[serde(default)]
- pub payload: Option<String>,
}
/// `GET v2/Vault`.
@@ -586,6 +484,13 @@ pub struct ManifestDto {
pub key_version: i64,
}
+impl ManifestDto {
+ /// Whether the server served this manifest with content.
+ pub fn has_content(&self) -> bool {
+ self.blob.as_deref().is_some_and(|blob| !blob.is_empty())
+ }
+}
+
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BucketDto {
@@ -772,8 +677,6 @@ pub struct VaultKeyGetResponse {
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct VaultKeyResponse {
- #[serde(default, rename = "type")]
- pub key_type: String,
#[serde(default)]
pub encrypted_account_key: String,
#[serde(default)]
@@ -788,12 +691,6 @@ pub struct VaultKeyResponse {
pub encrypted_signing_private_key: Option<String>,
#[serde(default)]
pub encrypted_vek: Option<String>,
- #[serde(default)]
- pub salt: String,
- #[serde(default)]
- pub encryption_type: String,
- #[serde(default)]
- pub encryption_settings: String,
}
/// A shared manifest as this account holds it: the grant on its key plus what the last pull learned about it.
@@ -818,18 +715,18 @@ pub struct SharedManifestDto {
mod tests {
use super::*;
- /// The failure fields flatten into every result exactly where the four loose fields used to sit.
+ /// The failure fields flatten into the top level of every result.
#[test]
fn failure_fields_flatten_into_the_result_wire_shape() {
let coded = FailureFields::from(&SyncError::VaultLocked);
let full = serde_json::to_string(&FullSyncResult { failure: coded, ..Default::default() }).unwrap();
- assert_eq!(full, r#"{"success":false,"hasNewVault":false,"wasOffline":false,"sqliteBlobUpgradeRequired":false,"manifestMigrationRequired":false,"error":"No encryption key available","errorCode":"E-202","requiresLogout":false,"isOfflineMode":false,"vaultChanged":false}"#);
+ assert_eq!(full, r#"{"success":false,"hasNewVault":false,"wasOffline":false,"sqliteBlobUpgradeRequired":false,"manifestMigrationRequired":false,"error":"No encryption key available","errorCode":"E-202","requiresLogout":false,"isOfflineMode":false}"#);
let round_trip: FullSyncResult = serde_json::from_str(&full).unwrap();
assert_eq!(serde_json::to_string(&round_trip).unwrap(), full);
let logout = FailureFields::logout(LogoutReason::SessionExpired);
let migrate = serde_json::to_string(&MigrateManifestResult { failure: logout.clone(), ..Default::default() }).unwrap();
- assert_eq!(migrate, r#"{"success":false,"pushed":false,"logoutReason":"sessionExpired","requiresLogout":true,"vaultChanged":false}"#);
+ assert_eq!(migrate, r#"{"success":false,"pushed":false,"logoutReason":"sessionExpired","requiresLogout":true}"#);
let round_trip: MigrateManifestResult = serde_json::from_str(&migrate).unwrap();
assert_eq!(round_trip.failure, logout);
diff --git a/core/rust/src/vault_sync/write_set.rs b/core/rust/src/vault_sync/write_set.rs
new file mode 100644
index 000000000..5265d29d3
--- /dev/null
+++ b/core/rust/src/vault_sync/write_set.rs
@@ -0,0 +1,85 @@
+//! Which manifests a push writes, and which rows it may not: a row stamped for a manifest this session cannot
+//! write has no namespace to go to, so a push holding one is refused until a pull restores the vault.
+
+use std::collections::HashMap;
+
+use super::types::SharedManifestDto;
+use crate::vault_model::{id_key, ids_equal};
+
+/// One manifest this session can write, personal manifest first in every write set.
+#[derive(Debug, Clone)]
+pub(crate) struct ManifestRecord {
+ pub manifest_id: String,
+ pub is_personal: bool,
+ /// The salt this manifest's blob hashes are derived with.
+ pub salt: String,
+ /// Whether this account may publish the manifest's email delivery key.
+ pub can_administer: bool,
+ /// The key this manifest encrypts with; None for the personal manifest, whose key the push supplies.
+ pub vek: Option<String>,
+}
+
+/// Why a held shared manifest is left out of a write.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum SkipReason {
+ /// The vault holds no rows for it, so writing it would empty it server-side.
+ NoRowsInVault,
+ /// Its grant did not unwrap in this session.
+ KeyDidNotOpen,
+}
+
+/// The manifests the next push writes, personal first, plus the held shared manifests left out and why. A shared
+/// manifest is written when the vault holds rows for it (`stamped`) and its key opened (`opened`, id to VEK).
+pub(crate) fn resolve_write_set(personal_manifest_id: &str, personal_salt: &str, stamped: &[String], opened: &HashMap<String, String>, held: &HashMap<String, SharedManifestDto>) -> (Vec<ManifestRecord>, Vec<(String, SkipReason)>) {
+ let mut records = vec![ManifestRecord { manifest_id: personal_manifest_id.to_string(), is_personal: true, salt: personal_salt.to_string(), can_administer: false, vek: None }];
+ let mut skipped = Vec::new();
+ let mut held: Vec<&SharedManifestDto> = held.values().collect();
+ held.sort_by(|a, b| a.manifest_id.cmp(&b.manifest_id));
+ for record in held {
+ let vek = opened.iter().find(|(id, _)| ids_equal(id, &record.manifest_id)).map(|(_, vek)| vek.clone());
+ match vek {
+ _ if !stamped.iter().any(|id| ids_equal(id, &record.manifest_id)) => skipped.push((record.manifest_id.clone(), SkipReason::NoRowsInVault)),
+ None => skipped.push((record.manifest_id.clone(), SkipReason::KeyDidNotOpen)),
+ Some(vek) => records.push(ManifestRecord { manifest_id: record.manifest_id.clone(), is_personal: false, salt: record.salt.clone(), can_administer: record.can_administer, vek: Some(vek) }),
+ }
+ }
+ (records, skipped)
+}
+
+/// The manifests the vault holds rows for that are not in `writable`.
+pub(crate) fn unwritable_manifests(in_vault: &[String], writable: &[String]) -> Vec<String> {
+ in_vault.iter().filter(|id| !writable.iter().any(|w| id_key(w) == id_key(id))).cloned().collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn held(ids: &[&str]) -> HashMap<String, SharedManifestDto> {
+ ids.iter().map(|id| (id.to_string(), SharedManifestDto { manifest_id: id.to_string(), encrypted_vek: String::new(), account_public_key: String::new(), algorithm: String::new(), salt: format!("salt-{}", id), encrypted_name: None, can_administer: true })).collect()
+ }
+
+ fn opened(ids: &[&str]) -> HashMap<String, String> {
+ ids.iter().map(|id| (id.to_string(), format!("vek-{}", id))).collect()
+ }
+
+ fn strings(ids: &[&str]) -> Vec<String> {
+ ids.iter().map(|id| id.to_string()).collect()
+ }
+
+ #[test]
+ fn the_write_set_leads_with_the_personal_manifest_and_names_what_it_left_out() {
+ let (records, skipped) = resolve_write_set("PERSONAL", "personal-salt", &strings(&["PERSONAL", "man-1", "MAN-3"]), &opened(&["MAN-1", "MAN-2"]), &held(&["MAN-1", "MAN-2", "MAN-3"]));
+
+ let written: Vec<(&str, bool, &str, Option<&str>)> = records.iter().map(|r| (r.manifest_id.as_str(), r.is_personal, r.salt.as_str(), r.vek.as_deref())).collect();
+ assert_eq!(written, vec![("PERSONAL", true, "personal-salt", None), ("MAN-1", false, "salt-MAN-1", Some("vek-MAN-1"))], "ids match regardless of casing and the record's own spelling comes back");
+ assert!(!records[0].can_administer, "a personal manifest is not administered through a group");
+ assert_eq!(skipped, vec![("MAN-2".to_string(), SkipReason::NoRowsInVault), ("MAN-3".to_string(), SkipReason::KeyDidNotOpen)]);
+ }
+
+ #[test]
+ fn rows_of_a_manifest_this_session_cannot_write_are_reported() {
+ assert_eq!(unwritable_manifests(&strings(&["PERSONAL", "MAN-1", "MAN-GONE"]), &strings(&["personal", "man-1"])), vec!["MAN-GONE".to_string()]);
+ assert!(unwritable_manifests(&strings(&["PERSONAL"]), &strings(&["PERSONAL"])).is_empty());
+ }
+}