Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Refactor Rust to share logic between modules (#2159)
Leendert de Borstd540322976de
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rsindex 985808132..93879090c 100644--- a/core/rust/src/vault_codec/canonicalize.rs+++ b/core/rust/src/vault_codec/canonicalize.rs@@ -16,18 +16,20 @@ use serde_json::{json, Value};use super::normalize::{normalize_id_spelling, normalize_row_shapes};use super::hash::salted_blob_hash;+use super::row::{blob_ref, inline_b64, str_col};use super::scoped_assets::{normalize_logo_scope, reconcile_logo_references};use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedManifest, CanonicalizedVault, CodecOverflow, DataBucket, Manifest, ManifestSpec, CodecRecord};use super::sharing::{clone_referenced_rows, partition_by_manifest, prune_unreferenced_logos, referenced_tables};use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_bucketed_table, is_skip_table, is_unstamped_scope,- manifest_scoped_tables, row_identity, MANIFEST_ID_COL, OVERFLOW_TABLE, SCHEMA_VERSION,+ manifest_scoped_tables, row_identity, SCHEMA_VERSION,};use crate::error::VaultResult;use crate::vault_model::names::LOGOS_TABLE;+use crate::vault_model::{ids_equal, MANIFEST_ID_COL, OVERFLOW_TABLE};/// Canonicalize normalized tables into the split resources: the manifest, one data bucket per declared-/// category (see [`BUCKET_TABLES`](super::types::BUCKET_TABLES)), and the content-addressed blob map.+/// category (see [`BUCKET_TABLES`](crate::vault_model::BUCKET_TABLES)), and the content-addressed blob map.pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<CanonicalizedVault> {let writing_spec = match input.manifests.first() {Some(spec) if !spec.manifest_id.is_empty() => spec.clone(),@@ -213,11 +215,11 @@ fn group_category_rows(tables: HashMap<String, Vec<CodecRecord>>, manifest_ids:/// The manifest a bucketed row belongs to, spelled the way `manifest_ids` spells it, or `None` when the row/// names no manifest or one this vault does not carry.fn owning_manifest(row: &CodecRecord, manifest_ids: &[String]) -> Option<String> {- let stamp = row.get(MANIFEST_ID_COL).and_then(|value| value.as_str())?;+ let stamp = str_col(row, MANIFEST_ID_COL)?;if is_unstamped_scope(Some(stamp)) {return None;}- manifest_ids.iter().find(|id| id.eq_ignore_ascii_case(stamp)).cloned()+ manifest_ids.iter().find(|id| ids_equal(id, stamp)).cloned()}/// For legacy sqlite-blob migration: the manifest that unstamped rows are adopted into.@@ -254,7 +256,7 @@ fn reject_unstamped_rows(tables: &HashMap<String, Vec<CodecRecord>>) -> VaultRes/// True when a row carries no usable `ManifestId`: absent, JSON null, a non-string, or the empty string.fn is_unstamped(row: &CodecRecord) -> bool {- is_unstamped_scope(row.get(MANIFEST_ID_COL).and_then(|value| value.as_str()))+ is_unstamped_scope(str_col(row, MANIFEST_ID_COL))}/// Extract `table`'s blob column (if it owns one) into `blobs`, returning the rewritten rows.@@ -279,7 +281,7 @@ fn extract_blob_cell(kind: &str,blobs: &mut HashMap<String, BlobEntry>,) -> serde_json::Value {- let b64 = match cell.and_then(|v| v.get("__b64")).and_then(|v| v.as_str()) {+ let b64 = match cell.and_then(inline_b64) {Some(s) => s,None => return serde_json::Value::Null,};@@ -295,7 +297,7 @@ fn extract_blob_cell(bytes_base64: b64.to_string(),});- json!({ "__blobRef": hash, "__blobKind": kind })+ blob_ref(&hash, kind)}/// Re-attach overflow columns.diff --git a/core/rust/src/vault_codec/hash.rs b/core/rust/src/vault_codec/hash.rsindex 67a8a6122..d5a6ed5eb 100644--- a/core/rust/src/vault_codec/hash.rs+++ b/core/rust/src/vault_codec/hash.rs@@ -8,25 +8,20 @@//! one platform and re-hashed by another would fail integrity even when byte-identical. This module is//! the single canonical-serialization contract every binding reproduces.//!-//! [`canonical_json`] is intentionally byte-compatible with the TypeScript `canonicalize()` the-//! clients used before the codec moved into Rust:-//! - object keys sorted ascending, recursively;-//! - arrays kept in order;-//! - primitives serialized exactly as `JSON.stringify` would (serde_json matches JS for the-//! escaping + integer cases that occur in vault data).+//! [`canonical_json`] is that contract: object keys sorted ascending, recursively; arrays kept in+//! order; primitives as serde_json renders them, which matches `JSON.stringify` for the escaping and+//! integer cases that occur in vault data, so a hash computed here agrees with one a client computed.use sha2::{Digest, Sha256};-use crate::encoding::{format_uuid, hex_decode, hex_encode_lower};+use crate::encoding::{hex_decode, hex_encode_lower, uuid_from_bytes};/// A UUIDv8 derived from a string: the first 16 bytes of its sha256, version and variant bits set.pub fn derived_uuid(material: &str) -> String {let digest = Sha256::digest(material.as_bytes());let mut bytes = [0u8; 16];bytes.copy_from_slice(&digest[..16]);- bytes[6] = (bytes[6] & 0x0f) | 0x80;- bytes[8] = (bytes[8] & 0x3f) | 0x80;- format_uuid(&bytes)+ uuid_from_bytes(bytes, 8)}/// SHA-256 of arbitrary bytes, returned as lowercase hex.diff --git a/core/rust/src/vault_codec/manifest.rs b/core/rust/src/vault_codec/manifest.rsindex 7e6573729..f02db1e6f 100644--- a/core/rust/src/vault_codec/manifest.rs+++ b/core/rust/src/vault_codec/manifest.rs@@ -4,7 +4,10 @@ use std::collections::HashMap;use serde::{Deserialize, Serialize};+use super::row::str_col;use super::types::SCHEMA_VERSION;+use crate::vault_model::names::ID_COL;+use crate::vault_model::OVERFLOW_ROW_ID;/// A codec record is a map of column names to JSON values.pub type CodecRecord = HashMap<String, serde_json::Value>;@@ -155,7 +158,7 @@ impl CodecOverflow {/// Render this overflow as the single `OVERFLOW_TABLE` row the platform inserts into the vault DB.pub fn to_table_records(&self) -> Vec<CodecRecord> {let mut row: CodecRecord = HashMap::new();- row.insert("Id".to_string(), serde_json::Value::String(super::types::OVERFLOW_ROW_ID.to_string()));+ row.insert(ID_COL.to_string(), serde_json::Value::String(OVERFLOW_ROW_ID.to_string()));row.insert("Data".to_string(), serde_json::Value::String(serde_json::to_string(self).unwrap_or_default()));vec![row]}@@ -166,8 +169,7 @@ impl CodecOverflow {pub fn from_table_records(records: &[CodecRecord]) -> Self {records.first()- .and_then(|row| row.get("Data"))- .and_then(|v| v.as_str())+ .and_then(|row| str_col(row, "Data")).and_then(|json| serde_json::from_str(json).ok()).unwrap_or_default()}@@ -190,12 +192,12 @@ pub struct MaterializedTables {#[serde(rename_all = "camelCase")]pub struct CanonicalizeInput {/// All local vault tables (a plain `SELECT *` read from the vault DB). When the read includes the- /// [`OVERFLOW_TABLE`](super::types::OVERFLOW_TABLE) row written by the last materialize, its overflow+ /// [`OVERFLOW_TABLE`](crate::vault_model::OVERFLOW_TABLE) row written by the last materialize, its overflow/// (a newer writer's tables/columns this schema can't hold) is re-merged automatically.pub tables: Vec<CodecTableData>,pub canonicalized_at: String,pub manifests: Vec<ManifestSpec>,- /// For legacy sqlite-blob migration: the manifest that unstamped rows are adopted into.+ /// For legacy sqlite-blob migration: the manifest that unstamped rows are adopted into./// TODO: delete this field once the migration is complete.#[serde(default)]pub adopt_unstamped_into: Option<String>,@@ -209,7 +211,7 @@ pub struct ExtractBucketsInput {/// Every manifest this vault can write. Each gets a bucket back; rows naming anything else are dropped.pub manifest_ids: Vec<String>,/// The category's tables as read from the local vault, plus the- /// [`OVERFLOW_TABLE`](super::types::OVERFLOW_TABLE) row so a newer writer's columns/tables re-merge.+ /// [`OVERFLOW_TABLE`](crate::vault_model::OVERFLOW_TABLE) row so a newer writer's columns/tables re-merge.pub tables: HashMap<String, Vec<CodecRecord>>,}diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rsindex c3dc6bd8c..39e49eba2 100644--- a/core/rust/src/vault_codec/materialize.rs+++ b/core/rust/src/vault_codec/materialize.rs@@ -4,9 +4,6 @@//! projection of the canonical dataset, not an authoritative destination. Future targets would add//! sibling `materialize_as_*` entry points. The inverse direction lives in `canonicalize`.//!-//! Rust does not handle actual SQLite operations. It only emits the destination table rows as-is-//! to the caller and let the caller handle the actual SQLite database creation and data insertion.-//!//! Forward compatibility: the caller supplies its local schema (`schema_columns`), and anything a//! newer writer put in the manifest that this schema cannot hold (whole unknown tables or unknown//! columns on known tables) is split into [`CodecOverflow`] instead of being emitted (which would@@ -20,8 +17,10 @@ use std::collections::{HashMap, HashSet};use serde_json::json;use super::manifest::{CodecOverflow, CodecRecord, CodecTableData, Manifest, MaterializeInput, MaterializedTables};-use super::types::{is_local_only_table, row_identity, MANIFEST_ID_COL, MANIFESTS_TABLE, OVERFLOW_TABLE};+use super::types::{is_local_only_table, row_identity};use crate::error::{VaultError, VaultResult};+use crate::vault_model::names::ID_COL;+use crate::vault_model::{MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_TABLE};/// Materialize the vault's manifests into the table set the platform inserts. Every manifest arrives/// in one list, each carrying its own data buckets; they are combined into a single table set with@@ -112,7 +111,7 @@ fn manifest_bookkeeping_records(manifests: &[Manifest]) -> Vec<CodecRecord> {continue;}let mut row: CodecRecord = HashMap::new();- row.insert("Id".to_string(), json!(id));+ row.insert(ID_COL.to_string(), json!(id));row.insert("Name".to_string(), manifest.name.as_deref().map(|n| json!(n)).unwrap_or(serde_json::Value::Null));super::normalize::normalize_row_id_spelling(&mut row);records.push(row);diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rsindex d13019f5f..793dce5d3 100644--- a/core/rust/src/vault_codec/mod.rs+++ b/core/rust/src/vault_codec/mod.rs@@ -16,6 +16,8 @@ mod materialize;pub(crate) mod row;mod scoped_assets;mod sharing;+#[cfg(test)]+pub(crate) mod test_support;mod types;mod validate;@@ -34,7 +36,7 @@ pub use manifest::{pub use materialize::materialize_as_sqlite;pub use scoped_assets::logo_id_for;pub use sharing::extract_encryption_key_for_public_key;-pub use types::{bucket_categories, identity_part, is_bucketed_table, tables_for_category, MANIFEST_ID_COL, OVERFLOW_TABLE};+pub use types::{bucket_categories, identity_part, is_bucketed_table, tables_for_category};pub use validate::{validate_data_bucket, validate_manifest, ValidationResult};/// The bucket layout: every category and the tables it owns, in declaration order.diff --git a/core/rust/src/vault_codec/normalize.rs b/core/rust/src/vault_codec/normalize.rsindex 42d1907b9..2f0d48443 100644--- a/core/rust/src/vault_codec/normalize.rs+++ b/core/rust/src/vault_codec/normalize.rs@@ -6,9 +6,9 @@ use serde_json::json;use super::manifest::CodecRecord;use super::row::{str_col, truthy};-use super::types::{is_guid, is_id_column, MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS};+use super::types::{is_guid, is_id_column};use crate::timestamp::updated_at;-use crate::vault_model::id_key;+use crate::vault_model::{id_key, MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS};use crate::vault_model::names::{CHANGED_AT_COL, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_KEY_COL,FIELD_VALUES_TABLE, ID_COL, IS_MULTI_VALUE_COL, ITEM_ID_COL, ITEM_TAGS_TABLE, TAG_ID_COL, VALUE_INDEX_COL,@@ -37,7 +37,7 @@ pub fn field_history_id_for(manifest_id: &str, item_id: &str, field_key: &str, f/// A UUIDv8 from `sha256(namespace | manifest | item | field | tail)`, ids lowercased.fn derived_field_id(namespace: &str, manifest_id: &str, item_id: &str, field_key: &str, field_definition_id: &str, tail: &str) -> String {let field = field_discriminator_of(field_key, field_definition_id);- super::hash::derived_uuid(&format!("{}\n{}\n{}\n{}\n{}", namespace, manifest_id.to_lowercase(), item_id.to_lowercase(), field, tail))+ super::hash::derived_uuid(&format!("{}\n{}\n{}\n{}\n{}", namespace, id_key(manifest_id), id_key(item_id), field, tail))}/// Lowercase every id in `tables` which is the normalized spelling expected by all AliasVault clients.@@ -117,7 +117,7 @@ fn normalize_field_values(rows: &mut Vec<CodecRecord>, multi_value_defs: &HashSelet mut removed: HashSet<usize> = HashSet::new();for ((manifest, _, _), mut positions) in groups {- let multi_value = rows[positions[0]].get(FIELD_KEY_COL).and_then(|v| v.as_str()).is_some_and(|key| MULTI_VALUE_FIELD_KEYS.contains(&key.to_lowercase().as_str()))+ let multi_value = str_col(&rows[positions[0]], FIELD_KEY_COL).is_some_and(|key| MULTI_VALUE_FIELD_KEYS.contains(&key.to_lowercase().as_str()))|| lower_str(&rows[positions[0]], FIELD_DEFINITION_ID_COL).is_some_and(|def| multi_value_defs.contains(&(manifest.clone(), def)));if multi_value {@@ -232,7 +232,7 @@ fn field_discriminator_of(field_key: &str, field_definition_id: &str) -> String}fn has_id(row: &CodecRecord) -> bool {- row.get(ID_COL).and_then(|v| v.as_str()).is_some_and(|s| !s.is_empty())+ str_col(row, ID_COL).is_some_and(|s| !s.is_empty())}fn value_index_of(row: &CodecRecord) -> Option<i64> {diff --git a/core/rust/src/vault_codec/row.rs b/core/rust/src/vault_codec/row.rsindex 100f48529..beb27d071 100644--- a/core/rust/src/vault_codec/row.rs+++ b/core/rust/src/vault_codec/row.rs@@ -1,12 +1,23 @@//! Reading cells out of a row as the platforms hand them over: SQLite booleans arrive as 0/1-//! numbers, JSON booleans or "1"/"true" strings, and a missing table reads as no rows.+//! numbers, JSON booleans or "1"/"true" strings, byte cells as `{ "__b64" }` (inline bytes) or+//! `{ "__blobRef", "__blobKind" }` (an extracted blob) markers, and a missing table reads as no rows.use std::collections::HashMap;-use serde_json::Value;+use serde_json::{json, Value};use super::manifest::CodecRecord;-use crate::vault_model::names::IS_DELETED_COL;+use crate::encoding::base64_encode;+use crate::vault_model::names::{IS_DELETED_COL, KIND_COL, LOGO_KIND_FAVICON};++/// The key of an inline byte cell: `{ "__b64": <base64> }`.+pub(crate) const INLINE_BYTES_KEY: &str = "__b64";++/// The hash key of an extracted blob cell: `{ "__blobRef": <hash>, "__blobKind": <kind> }`.+pub(crate) const BLOB_REF_KEY: &str = "__blobRef";++/// The kind key of an extracted blob cell.+pub(crate) const BLOB_KIND_KEY: &str = "__blobKind";/// A cell as a string slice, `None` when absent or not a string.pub(crate) fn str_col<'a>(row: &'a CodecRecord, column: &str) -> Option<&'a str> {@@ -33,6 +44,55 @@ pub(crate) fn rows_of<'a>(tables: &'a HashMap<String, Vec<CodecRecord>>, table:tables.get(table).map(Vec::as_slice).unwrap_or(&[])}+/// A logo `Kind` in its comparison form: trimmed and lowercased, empty meaning [`LOGO_KIND_FAVICON`]+/// (the row was written before the column existed, when every logo was a favicon).+pub(crate) fn normalize_logo_kind(kind: &str) -> String {+ let trimmed = kind.trim();+ if trimmed.is_empty() { LOGO_KIND_FAVICON.to_string() } else { trimmed.to_lowercase() }+}++/// A logo row's `Kind`, normalized (see [`normalize_logo_kind`]).+pub(crate) fn logo_kind(row: &CodecRecord) -> String {+ normalize_logo_kind(str_col(row, KIND_COL).unwrap_or(""))+}++/// Bytes as the inline `{ "__b64" }` cell every platform binds and reads.+pub(crate) fn inline_bytes(bytes: &[u8]) -> Value {+ json!({ INLINE_BYTES_KEY: base64_encode(bytes) })+}++/// The base64 text of an inline `{ "__b64" }` cell, `None` for any other value.+pub(crate) fn inline_b64(value: &Value) -> Option<&str> {+ value.get(INLINE_BYTES_KEY).and_then(Value::as_str)+}++/// An extracted blob cell: the content hash and kind that stand in for the bytes inside a manifest.+pub(crate) fn blob_ref(hash: &str, kind: &str) -> Value {+ json!({ BLOB_REF_KEY: hash, BLOB_KIND_KEY: kind })+}++/// The `(hash, kind)` of an extracted blob cell, `None` for any other value.+pub(crate) fn blob_ref_of(value: &Value) -> Option<(&str, Option<&str>)> {+ let hash = value.get(BLOB_REF_KEY).and_then(Value::as_str)?;+ Some((hash, value.get(BLOB_KIND_KEY).and_then(Value::as_str)))+}++/// True when a cell holds bytes: a non-empty inline `{ __b64 }`, an extracted `{ __blobRef }`, or any+/// other non-empty string, array or object (the 1-byte presence marker the prune queries select).+/// A tombstoned row drops its bytes (NULL), which is present but carries nothing.+pub(crate) fn has_bytes(value: Option<&Value>) -> bool {+ match value {+ None | Some(Value::Null) => false,+ Some(Value::String(s)) => !s.is_empty(),+ Some(Value::Array(a)) => !a.is_empty(),+ Some(Value::Object(o)) => match o.get(INLINE_BYTES_KEY).and_then(Value::as_str) {+ Some(b64) => !b64.is_empty(),+ None => !o.is_empty(),+ },+ Some(_) => true,+ }+}+#[cfg(test)]mod tests {use super::*;@@ -48,4 +108,26 @@ mod tests {}assert!(!truthy(None));}++ #[test]+ fn byte_markers_round_trip_and_presence_reads_every_shape() {+ assert_eq!(inline_b64(&inline_bytes(&[1, 2, 3])), Some("AQID"));+ assert_eq!(blob_ref_of(&blob_ref("hash", "favicon")), Some(("hash", Some("favicon"))));+ assert_eq!(blob_ref_of(&inline_bytes(&[1])), None);+ assert!(has_bytes(Some(&inline_bytes(&[1]))));+ assert!(!has_bytes(Some(&inline_bytes(&[]))));+ assert!(has_bytes(Some(&blob_ref("hash", "attachment"))));+ assert!(has_bytes(Some(&json!("x"))) && has_bytes(Some(&json!([1]))) && has_bytes(Some(&json!({ "0": 1 }))));+ for empty in [json!(""), json!([]), json!({}), json!(null)] {+ assert!(!has_bytes(Some(&empty)), "{empty} holds no bytes");+ }+ assert!(!has_bytes(None));+ }++ #[test]+ fn logo_kind_normalizes_case_whitespace_and_absence() {+ assert_eq!(normalize_logo_kind(" Custom "), "custom");+ assert_eq!(normalize_logo_kind(""), LOGO_KIND_FAVICON);+ assert_eq!(logo_kind(&HashMap::new()), LOGO_KIND_FAVICON);+ }}diff --git a/core/rust/src/vault_codec/scoped_assets.rs b/core/rust/src/vault_codec/scoped_assets.rsindex b35e1932c..cff61d5d1 100644--- a/core/rust/src/vault_codec/scoped_assets.rs+++ b/core/rust/src/vault_codec/scoped_assets.rs@@ -20,10 +20,9 @@ use std::collections::{HashMap, HashSet};use serde_json::{json, Value};use super::manifest::CodecRecord;-use super::row::{is_deleted, str_col};-use super::types::MANIFEST_ID_COL;-use crate::vault_model::id_key;+use super::row::{has_bytes, is_deleted, logo_kind, normalize_logo_kind, str_col};use crate::vault_model::names::{FILE_DATA_COL, ID_COL, ITEMS_TABLE, KIND_COL, LOGOS_TABLE, LOGO_ID_COL, LOGO_KIND_CUSTOM, LOGO_KIND_FAVICON, SOURCE_COL, UPDATED_AT_COL};+use crate::vault_model::{id_key, ids_equal, MANIFEST_ID_COL};/// Domain-separation prefix for favicon ids. It predates the `Kind` column and is kept verbatim so/// every favicon row that already exists keeps its id: changing it would re-derive the logo id of@@ -50,34 +49,23 @@ pub fn logo_id_for(manifest_id: &str, kind: &str, source: &str) -> String {/// client can add a kind without this one having to know about it: ids stay stable and distinct either/// way, and the rows simply round-trip until this client learns to render them.fn namespace_for_kind(kind: &str) -> String {- match normalize_kind(kind) {+ match normalize_logo_kind(kind) {k if k == LOGO_KIND_FAVICON => FAVICON_ID_NAMESPACE.to_string(),k => format!("aliasvault:logo:{}:v1", k),}}-/// A row's kind, lowercased so `Kind` matching is case-insensitive like the rest of the natural key.-/// Absent or empty means [`LOGO_KIND_FAVICON`]: the row was written before the column existed, when-/// every logo was a favicon.-fn normalize_kind(kind: &str) -> String {- let trimmed = kind.trim();- if trimmed.is_empty() {- return LOGO_KIND_FAVICON.to_string();- }- trimmed.to_lowercase()-}-/// True when a row holds an image the user supplied themselves ([`LOGO_KIND_CUSTOM`]) rather than one the/// client can produce again on its own: a favicon it can refetch from the domain, a built-in logo it/// draws from the catalog. Only that first group is worth keeping around once nothing references it.pub(super) fn is_custom_logo(row: &CodecRecord) -> bool {- normalize_kind(str_col(row, KIND_COL).unwrap_or("")) == LOGO_KIND_CUSTOM+ logo_kind(row) == LOGO_KIND_CUSTOM}/// The `(kind, source)` natural key of a row, or `None` when it carries no `Source` to key on.fn natural_key(row: &CodecRecord) -> Option<(String, String)> {let source = str_col(row, SOURCE_COL)?.to_lowercase();- Some((normalize_kind(str_col(row, KIND_COL).unwrap_or("")), source))+ Some((logo_kind(row), source))}/// Normalize one table set's logo rows to `scope` (the owning manifest's id): stamp `ManifestId`,@@ -196,7 +184,7 @@ pub(super) fn reconcile_logo_references(tables: &mut HashMap<String, Vec<CodecRelet scope_value = json!(scope);for missing_id in missing {// The referenced row as it exists in its original scope, if it exists at all.- let Some(origin) = all_logos.iter().find(|row| str_col(row, ID_COL).is_some_and(|id| id.eq_ignore_ascii_case(&missing_id))) else { continue };+ let Some(origin) = all_logos.iter().find(|row| str_col(row, ID_COL).is_some_and(|id| ids_equal(id, &missing_id))) else { continue };let Some((kind, source)) = natural_key(origin) else { continue };if let Some(existing_id) = id_by_key.get(&(kind.clone(), source.clone())).cloned() {@@ -272,15 +260,8 @@ fn is_better_logo(candidate: &CodecRecord, incumbent: &CodecRecord) -> bool {str_col(candidate, ID_COL).unwrap_or("") > str_col(incumbent, ID_COL).unwrap_or("")}-/// True when `FileData` holds actual bytes: a non-empty inline `{ __b64 }` or an extracted-/// `{ __blobRef }`. A tombstoned row drops its bytes (NULL) which is present but carries no image and must not beat a row that has one.+/// True when `FileData` holds actual bytes; a tombstoned row's NULL must not beat a row with an image.fn has_file_data(row: &CodecRecord) -> bool {- match row.get(FILE_DATA_COL) {- None | Some(Value::Null) => false,- Some(value) => match value.get("__b64").and_then(|v| v.as_str()) {- Some(b64) => !b64.is_empty(),- None => value.get("__blobRef").is_some(),- },- }+ has_bytes(row.get(FILE_DATA_COL))}diff --git a/core/rust/src/vault_codec/sharing.rs b/core/rust/src/vault_codec/sharing.rsindex 8481ea85b..d2223b4dc 100644--- a/core/rust/src/vault_codec/sharing.rs+++ b/core/rust/src/vault_codec/sharing.rs@@ -34,20 +34,16 @@ use std::collections::{HashMap, HashSet};use serde_json::{json, Value};use super::manifest::{CodecRecord, Manifest, ManifestSpec};-use super::row::{str_col, truthy};+use super::row::{is_deleted, rows_of, str_col};use super::scoped_assets::{is_custom_logo, normalize_logo_scope, reconcile_logo_references};-use super::types::{- is_bucketed_table, is_local_only_table, is_manifest_scoped, is_personal_table, manifest_scoped_tables, row_identity, ENCRYPTION_KEYS_TABLE,- MANIFEST_ID_COL,-};+use super::types::{is_bucketed_table, is_local_only_table, is_manifest_scoped, is_personal_table, manifest_scoped_tables, row_identity};use crate::error::{VaultError, VaultResult};use crate::vault_model::names::{- FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_VALUES_TABLE, FOLDERS_TABLE,- FOLDER_ID_COL, ID_COL, IS_DELETED_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, LOGOS_TABLE,- LOGO_ID_COL, PARENT_FOLDER_ID_COL, PUBLIC_KEY_COL, TAGS_TABLE, TAG_ID_COL,+ ENCRYPTION_KEYS_TABLE, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_VALUES_TABLE, FOLDERS_TABLE,+ FOLDER_ID_COL, ID_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, LOGOS_TABLE, LOGO_ID_COL, PARENT_FOLDER_ID_COL, PUBLIC_KEY_COL,+ TAGS_TABLE, TAG_ID_COL,};--use crate::vault_model::REFERENCED_TABLES;+use crate::vault_model::{ids_equal, MANIFEST_ID_COL, REFERENCED_TABLES};/// The tables a caller must snapshot before routing, so the referenced content can be copied out of/// them afterwards: every [`REFERENCED_TABLES`] target plus `Logos`.@@ -141,12 +137,12 @@ pub(super) fn partition_by_manifest(// Where a row goes, read straight off its `ManifestId` stamp.let route = |row: &CodecRecord| -> Route {let Some(scope) = str_col(row, MANIFEST_ID_COL) else { return Route::Gone };- if scope.eq_ignore_ascii_case(writing_manifest_id) {+ if ids_equal(scope, writing_manifest_id) {return Route::Base;}manifest_to_spec.iter()- .find(|(id, _)| id.eq_ignore_ascii_case(scope))+ .find(|(id, _)| ids_equal(id, scope)).map(|(_, index)| Route::Partition(*index)).unwrap_or(Route::Gone)};@@ -156,10 +152,7 @@ pub(super) fn partition_by_manifest(* same namespace. Both halves, because `Folders.ParentFolderId` resolves through the composite key:* the same id in another manifest is another folder, not this one's parent.*/- let folder_identities: HashSet<(String, String)> = tables- .get(FOLDERS_TABLE)- .map(Vec::as_slice)- .unwrap_or(&[])+ let folder_identities: HashSet<(String, String)> = rows_of(tables, FOLDERS_TABLE).iter().filter_map(|row| Some((str_col(row, MANIFEST_ID_COL)?.to_string(), str_col(row, ID_COL)?.to_string()))).collect();@@ -358,7 +351,7 @@ pub(super) fn clone_referenced_rows(tables: &mut HashMap<String, Vec<CodecRecordlet mut missing: Vec<String> = referencing.iter()- .flat_map(|(ref_table, ref_column)| tables.get(*ref_table).map(Vec::as_slice).unwrap_or(&[]).iter().filter_map(move |row| str_col(row, ref_column)))+ .flat_map(|(ref_table, ref_column)| rows_of(tables, ref_table).iter().filter_map(move |row| str_col(row, ref_column))).filter(|id| !present.contains(*id)).map(str::to_string).collect();@@ -496,7 +489,7 @@ fn claim_manifest_scope(tables: &mut HashMap<String, Vec<CodecRecord>>, manifest/// owns.fn retain_own_encryption_keys(tables: &mut HashMap<String, Vec<CodecRecord>>, manifest_id: &str) {if let Some(rows) = tables.get_mut(ENCRYPTION_KEYS_TABLE) {- rows.retain(|row| str_col(row, MANIFEST_ID_COL) == Some(manifest_id));+ rows.retain(|row| str_col(row, MANIFEST_ID_COL).is_some_and(|scope| ids_equal(scope, manifest_id)));}}@@ -613,7 +606,7 @@ pub fn extract_encryption_key_for_public_key(manifest: &Manifest, public_key: &s.tables.get(ENCRYPTION_KEYS_TABLE)?.iter()- .find(|row| str_col(row, PUBLIC_KEY_COL) == Some(public_key) && str_col(row, MANIFEST_ID_COL) == scope && !truthy(row.get(IS_DELETED_COL)))+ .find(|row| str_col(row, PUBLIC_KEY_COL) == Some(public_key) && str_col(row, MANIFEST_ID_COL) == scope && !is_deleted(row)).cloned()}diff --git a/core/rust/src/vault_codec/sharing_tests.rs b/core/rust/src/vault_codec/sharing_tests.rsindex 12bfeb30b..18dab51c6 100644--- a/core/rust/src/vault_codec/sharing_tests.rs+++ b/core/rust/src/vault_codec/sharing_tests.rs@@ -6,11 +6,11 @@//! local vault holds every row) and a recipient (whose personal manifest knows nothing of the share).use super::*;+use super::test_support::{b64, row, table};use super::tests::{fitting_schema, materialize_manifests, materialize_input, stamp_unstamped};use super::types::{is_bucketed_table, is_personal_table, manifest_scoped_tables, SCHEMA_VERSION};use crate::vault_model::names::LOGO_KIND_FAVICON;-use base64::engine::general_purpose::STANDARD as BASE64;-use base64::Engine;+use crate::vault_model::OVERFLOW_TABLE;use serde_json::json;use std::collections::{HashMap, HashSet};@@ -21,18 +21,6 @@ const SALT_SHARED: &str = "ffeeddccbbaa99887766554433221100ffeeddccbbaa998877665/// no NULL-scope convention anywhere in the format.const PERSONAL_M: &str = "m-personal";-fn b64(bytes: &[u8]) -> String {- BASE64.encode(bytes)-}--fn row(pairs: &[(&str, serde_json::Value)]) -> CodecRecord {- pairs.iter().map(|(k, v)| (k.to_string(), v.clone())).collect()-}--fn table(name: &str, records: Vec<CodecRecord>) -> CodecTableData {- CodecTableData { name: name.to_string(), records }-}-/// A spec for `folder_id`, whose manifest id is derived as `m-<folder_id>` so every test can predict/// the scope stamp of a partition's rows.fn spec(folder_id: &str) -> ManifestSpec {diff --git a/core/rust/src/vault_codec/test_support.rs b/core/rust/src/vault_codec/test_support.rsnew file mode 100644index 000000000..557730f96--- /dev/null+++ b/core/rust/src/vault_codec/test_support.rs@@ -0,0 +1,28 @@+//! Fixtures shared by the codec, sharing and pruner tests.++use super::manifest::{CodecRecord, CodecTableData};++/// Standard base64 of `bytes`, the inline byte spelling the tests hand the codec.+pub(crate) fn b64(bytes: &[u8]) -> String {+ crate::encoding::base64_encode(bytes)+}++/// A row from `(column, value)` pairs.+pub(crate) fn row(pairs: &[(&str, serde_json::Value)]) -> CodecRecord {+ pairs.iter().map(|(k, v)| (k.to_string(), v.clone())).collect()+}++/// A table with the given rows.+pub(crate) fn table(name: &str, records: Vec<CodecRecord>) -> CodecTableData {+ CodecTableData { name: name.to_string(), records }+}++/// The current instant in the ISO form the prune input takes.+pub(crate) fn now_iso() -> String {+ crate::timestamp::now_iso_utc()+}++/// The instant `days` days ago in the ISO form the prune input takes.+pub(crate) fn days_ago_iso(days: i64) -> String {+ crate::timestamp::iso_utc(&(chrono::Utc::now() - chrono::Duration::days(days)))+}diff --git a/core/rust/src/vault_codec/tests.rs b/core/rust/src/vault_codec/tests.rsindex c82408464..3f5500a0a 100644--- a/core/rust/src/vault_codec/tests.rs+++ b/core/rust/src/vault_codec/tests.rs@@ -1,20 +1,12 @@//! Unit tests for vault_codec, covering the round-trip contract.use super::*;-use super::types::{bucket_category_for, MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_ROW_ID, SCHEMA_VERSION};+use super::test_support::{b64, row};+use super::types::{bucket_category_for, SCHEMA_VERSION};use crate::vault_model::names::LOGO_KIND_FAVICON;-use base64::engine::general_purpose::STANDARD as BASE64;-use base64::Engine;+use crate::vault_model::{MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_ROW_ID, OVERFLOW_TABLE};use serde_json::json;use std::collections::{HashMap, HashSet};-fn b64(bytes: &[u8]) -> String {- BASE64.encode(bytes)-}--fn row(pairs: &[(&str, serde_json::Value)]) -> CodecRecord {- pairs.iter().map(|(k, v)| (k.to_string(), v.clone())).collect()-}-/// The rows of `table` inside the data bucket for `category` (empty if absent).fn bucket_rows<'a>(out: &'a CanonicalizedVault, category: &str, table: &str) -> &'a [CodecRecord] {out.data_buckets@@ -88,7 +80,7 @@ fn bucket_layout_matches_bucket_tables_source_of_truth() {#[test]fn every_bucketed_table_is_manifest_scoped_with_a_composite_identity() {// The invariant that makes per-manifest buckets work at all: a bucket is addressed by `(manifest_id, category)`, so a row that carries no manifest cannot be routed into one.- for (table, category) in super::types::BUCKET_TABLES {+ for (table, category) in crate::vault_model::BUCKET_TABLES {assert!(crate::vault_model::SYNCABLE_TABLES.iter().any(|t| t.name == *table),"bucketed table {} ({}) is not registered in SYNCABLE_TABLES, so it would never row-merge",@@ -101,7 +93,7 @@ fn every_bucketed_table_is_manifest_scoped_with_a_composite_identity() {table,category);- let mut expected = vec![super::types::MANIFEST_ID_COL];+ let mut expected = vec![MANIFEST_ID_COL];expected.extend_from_slice(super::types::primary_key_columns_for(table));assert_eq!(super::types::identity_columns_for(table),diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rsindex 545fb2688..128400729 100644--- a/core/rust/src/vault_codec/types.rs+++ b/core/rust/src/vault_codec/types.rs@@ -2,16 +2,10 @@//!//! The datamodel registry data (tables, keys, bucket layout, blob columns, sentinels) lives in//! [`crate::vault_model`], generated from the TypeScript source of truth in-//! core/models/src/vault/VaultTableRegistry.ts; this module re-exports it for the codec and adds-//! the codec-owned accessors on top.--pub use crate::vault_model::{- BLOB_COLUMNS, BUCKET_TABLES, ENCRYPTION_KEYS_TABLE, MANIFESTS_TABLE, MANIFEST_ID_COL,- MULTI_VALUE_FIELD_KEYS, OVERFLOW_ROW_ID, OVERFLOW_TABLE, PERSONAL_TABLES, SKIP_TABLES,- UNSTAMPED_SCOPE_SENTINEL,-};+//! core/models/src/vault/VaultTableRegistry.ts; this module adds the codec-owned accessors on top.use crate::vault_model::names::ID_COL;+use crate::vault_model::{ids_equal, BLOB_COLUMNS, BUCKET_TABLES, MANIFEST_ID_COL, OVERFLOW_TABLE, PERSONAL_TABLES, SKIP_TABLES, SYNCABLE_TABLES, UNSTAMPED_SCOPE_SENTINEL};/// Manifest / metadata schema version.pub const SCHEMA_VERSION: u32 = 1;@@ -85,7 +79,7 @@ pub fn tables_for_category(category: &str) -> Vec<&'static str> {pub fn is_unstamped_scope(scope: Option<&str>) -> bool {match scope {None => true,- Some(value) => value.is_empty() || value.eq_ignore_ascii_case(UNSTAMPED_SCOPE_SENTINEL),+ Some(value) => value.is_empty() || ids_equal(value, UNSTAMPED_SCOPE_SENTINEL),}}@@ -100,30 +94,27 @@ pub fn is_bucketed_table(table_name: &str) -> bool {}/// Get the primary key columns for a table.-pub fn primary_key_columns_for(table_name: &str) -> &'static [&'static str] {- crate::vault_model::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key_columns).unwrap_or(&["Id"])+pub(crate) fn primary_key_columns_for(table_name: &str) -> &'static [&'static str] {+ SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key_columns).unwrap_or(&[ID_COL])}/// True when `table_name`'s rows are namespaced per manifest.pub fn is_manifest_scoped(table_name: &str) -> bool {- crate::vault_model::SYNCABLE_TABLES.iter().any(|t| t.name == table_name && t.manifest_scoped)+ SYNCABLE_TABLES.iter().any(|t| t.name == table_name && t.manifest_scoped)}/// Every manifest-scoped table, in registry order.pub fn manifest_scoped_tables() -> Vec<&'static str> {- crate::vault_model::SYNCABLE_TABLES.iter().filter(|t| t.manifest_scoped).map(|t| t.name).collect()+ SYNCABLE_TABLES.iter().filter(|t| t.manifest_scoped).map(|t| t.name).collect()}/// The columns that together identify one row of `table_name`.-pub fn identity_columns_for(table_name: &str) -> Vec<&'static str> {- crate::vault_model::SYNCABLE_TABLES- .iter()- .find(|t| t.name == table_name)- .map(|t| t.identity_columns())- .unwrap_or_else(|| vec!["Id"])+pub(crate) fn identity_columns_for(table_name: &str) -> Vec<&'static str> {+ SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.identity_columns()).unwrap_or_else(|| vec![ID_COL])}-/// Stable string key identifying `row` within `table_name`.+/// Stable string key identifying `row` within `table_name`: the identity parts joined by `\u{1f}`, a+/// null or absent column skipped. See also `vault_merge::get_key`, whose empty-part rule the merge relies on.pub fn row_identity(table_name: &str, row: &super::manifest::CodecRecord) -> Option<String> {let mut columns = identity_columns_for(table_name);if !columns.contains(&MANIFEST_ID_COL) && row.get(MANIFEST_ID_COL).filter(|value| !value.is_null()).is_some() {diff --git a/core/rust/src/vault_codec/validate.rs b/core/rust/src/vault_codec/validate.rsindex 4bd930509..44e14a0f1 100644--- a/core/rust/src/vault_codec/validate.rs+++ b/core/rust/src/vault_codec/validate.rs@@ -5,11 +5,13 @@use serde::{Deserialize, Serialize};use super::manifest::{DataBucket, Manifest};-use super::row::{rows_of, str_col};+use super::row::{logo_kind, rows_of, str_col};+use super::types::{is_bucketed_table, is_manifest_scoped, is_personal_table};use crate::vault_model::names::{- FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_VALUES_TABLE, FOLDERS_TABLE, FOLDER_ID_COL, ID_COL,- ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, KIND_COL, LOGOS_TABLE, LOGO_KIND_FAVICON, SOURCE_COL, TAGS_TABLE, TAG_ID_COL,+ ENCRYPTION_KEYS_TABLE, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_VALUES_TABLE, FOLDERS_TABLE, FOLDER_ID_COL, ID_COL,+ ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, LOGOS_TABLE, SOURCE_COL, TAGS_TABLE, TAG_ID_COL,};+use crate::vault_model::MANIFEST_ID_COL;/// Outcome of a structural validation run.#[derive(Debug, Clone, Serialize, Deserialize)]@@ -53,14 +55,14 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {* personal-only table may not leave the user's own vault at all.*/for name in manifest.tables.keys().filter(|name| !manifest.tables[*name].is_empty()) {- if super::types::is_bucketed_table(name) {+ if is_bucketed_table(name) {failed.push("manifest-carries-bucketed-table".to_string());explain.push(format!("Manifest carries bucketed table {}, which belongs in its data bucket", name));break;}}for name in manifest.tables.keys().filter(|name| !manifest.tables[*name].is_empty()) {- if super::types::is_personal_table(name) {+ if is_personal_table(name) {failed.push("manifest-carries-personal-table".to_string());explain.push(format!("Manifest carries personal table {}", name));break;@@ -71,14 +73,14 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {* Every EncryptionKeys row must be stamped with the manifest's own id.*/let expected_scope = Some(manifest.manifest_id.as_str());- if rows_of(&manifest.tables, super::types::ENCRYPTION_KEYS_TABLE).iter().any(|r| str_col(r, super::types::MANIFEST_ID_COL) != expected_scope) {+ if rows_of(&manifest.tables, ENCRYPTION_KEYS_TABLE).iter().any(|r| str_col(r, MANIFEST_ID_COL) != expected_scope) {failed.push("encryption-keys-scope-mismatch".to_string());explain.push("EncryptionKeys carries rows stamped for another manifest".to_string());}// Items and Folders are restamped by canonicalize: a mismatched stamp here means a codec bug.for name in [ITEMS_TABLE, FOLDERS_TABLE] {- if rows_of(&manifest.tables, name).iter().any(|r| str_col(r, super::types::MANIFEST_ID_COL) != expected_scope) {+ if rows_of(&manifest.tables, name).iter().any(|r| str_col(r, MANIFEST_ID_COL) != expected_scope) {failed.push("content-scope-mismatch".to_string());explain.push(format!("{} carries rows stamped for another manifest", name));break;@@ -160,7 +162,7 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {let logos = rows_of(&manifest.tables, LOGOS_TABLE);let logo_keys: std::collections::HashSet<(String, String)> = logos.iter()- .filter_map(|l| Some((str_col(l, KIND_COL).unwrap_or(LOGO_KIND_FAVICON).to_lowercase(), str_col(l, SOURCE_COL)?.to_lowercase())))+ .filter_map(|l| Some((logo_kind(l), str_col(l, SOURCE_COL)?.to_lowercase()))).collect();let logos_with_source = logos.iter().filter(|l| str_col(l, SOURCE_COL).is_some()).count();if logo_keys.len() != logos_with_source {@@ -169,7 +171,7 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {// Every logo in a manifest must claim that manifest's scope, otherwise the row would materialize// into the wrong uniqueness bucket and could collide with the reader's own rows.- if logos.iter().any(|l| str_col(l, super::types::MANIFEST_ID_COL) != expected_scope) {+ if logos.iter().any(|l| str_col(l, MANIFEST_ID_COL) != expected_scope) {failed.push("logo-scope-mismatch".to_string());explain.push("Logos carry a ManifestId that is not this manifest's own id".to_string());}@@ -202,10 +204,10 @@ pub fn validate_data_bucket(bucket: &DataBucket) -> ValidationResult {let expected_scope = Some(bucket.manifest_id.as_str());for (name, rows) in &bucket.tables {- if !super::types::is_manifest_scoped(name) {+ if !is_manifest_scoped(name) {continue;}- if rows.iter().any(|row| str_col(row, super::types::MANIFEST_ID_COL) != expected_scope) {+ if rows.iter().any(|row| str_col(row, MANIFEST_ID_COL) != expected_scope) {failed.push("dataBucket-scope-mismatch".to_string());explain.push(format!("{} carries rows stamped for another manifest", name));break;diff --git a/core/rust/src/vault_merge/canonical.rs b/core/rust/src/vault_merge/canonical.rsdeleted file mode 100644index 3af08090a..000000000--- a/core/rust/src/vault_merge/canonical.rs+++ /dev/null@@ -1,834 +0,0 @@-//! Canonical-level vault merge: LWW over manifest JSON + data buckets, rows in, rows out.-//!-//! This is the merge for the manifest-v1 storage format (used since 0.31.0+). It runs one layer above any concrete-//! materialization (SQLite or otherwise): both sides arrive in canonical form and the output is-//! the merged canonical form, which the platform then materializes once.-//!-//! Each manifest is merged independently (the server manifest set is the universe), so a broken-//! manifest can never affect another manifest's rows, and blob columns carry `__blobRef` markers-//! rather than bytes, so no byte payload ever crosses the merge.--use std::collections::{BTreeSet, HashMap, HashSet};--use serde::{Deserialize, Serialize};--use super::{get_key, MergeStats};-use crate::error::VaultResult;-use crate::timestamp::updated_at;-use crate::vault_model::{id_key, TableConfig, SYNCABLE_TABLES};-use crate::vault_codec::{bucket_categories, is_bucketed_table, tables_for_category, CodecRecord, DataBucket, Manifest};--/// Input of the canonical merge. The server side is the base (kept on ties); the local side is the-/// incoming set produced by `canonicalize_from_sqlite`.-#[derive(Debug, Clone, Serialize, Deserialize)]-#[serde(rename_all = "camelCase")]-pub struct CanonicalMergeInput {- pub server_manifests: Vec<Manifest>,- pub server_buckets: Vec<DataBucket>,- #[serde(default)]- pub contentless_server_manifest_ids: Vec<String>,- pub local_manifests: Vec<Manifest>,- pub local_buckets: Vec<DataBucket>,- pub schema_columns: HashMap<String, Vec<String>>,-}--/// One manifest's merged result.-#[derive(Debug, Clone, Serialize, Deserialize)]-#[serde(rename_all = "camelCase")]-pub struct CanonicalManifestMerge {- pub manifest_id: String,- pub manifest: Manifest,- pub buckets: Vec<DataBucket>,- pub stats: MergeStats,-}--/// Output of the canonical merge.-#[derive(Debug, Clone, Serialize, Deserialize)]-#[serde(rename_all = "camelCase")]-pub struct CanonicalMergeOutput {- pub manifests: Vec<CanonicalManifestMerge>,- pub dropped_local_manifest_ids: Vec<String>,-}--/// Merge the local canonical vault onto the server canonical vault (the base), per manifest.-pub fn merge_canonical(input: CanonicalMergeInput) -> VaultResult<CanonicalMergeOutput> {- let CanonicalMergeInput { server_manifests, server_buckets, contentless_server_manifest_ids, local_manifests, local_buckets, schema_columns } = input;-- let mut local_by_id: HashMap<String, Manifest> = local_manifests.into_iter().map(|m| (id_key(&m.manifest_id), m)).collect();- let mut local_buckets_by_id = group_buckets(local_buckets);- let mut server_buckets_by_id = group_buckets(server_buckets);-- let mut manifests: Vec<CanonicalManifestMerge> = Vec::new();- for server_manifest in server_manifests {- let key = id_key(&server_manifest.manifest_id);- let local = local_by_id.remove(&key);- let local_buckets = local_buckets_by_id.remove(&key).unwrap_or_default();- let server_buckets = server_buckets_by_id.remove(&key).unwrap_or_default();- manifests.push(merge_manifest_pair(server_manifest, server_buckets, local, local_buckets, &schema_columns));- }-- // A contentless server manifest has no base; the local counterpart passes through whole.- for manifest_id in contentless_server_manifest_ids {- let key = id_key(&manifest_id);- if let Some(local) = local_by_id.remove(&key) {- manifests.push(pass_through(local, local_buckets_by_id.remove(&key).unwrap_or_default()));- }- }-- let mut dropped_local_manifest_ids: Vec<String> = local_by_id.into_values().map(|m| m.manifest_id).collect();- dropped_local_manifest_ids.sort();-- Ok(CanonicalMergeOutput { manifests, dropped_local_manifest_ids })-}--/// Group buckets by their manifest id (lowercased).-fn group_buckets(buckets: Vec<DataBucket>) -> HashMap<String, Vec<DataBucket>> {- let mut grouped: HashMap<String, Vec<DataBucket>> = HashMap::new();- for bucket in buckets {- grouped.entry(id_key(&bucket.manifest_id)).or_default().push(bucket);- }- grouped-}--/// Merge one manifest: LWW over the flattened table view of both sides, then split the bucketed-/// tables back out. The merged `Manifest` is the server one with only its tables replaced.-fn merge_manifest_pair(- server: Manifest,- server_buckets: Vec<DataBucket>,- local: Option<Manifest>,- local_buckets: Vec<DataBucket>,- schema_columns: &HashMap<String, Vec<String>>,-) -> CanonicalManifestMerge {- let manifest_id = server.manifest_id.clone();- let mut stats = MergeStats::default();-- // A bucket of a category this build does not know cannot be flattened without losing its- // category; the server's ride through as-is and the local ones are dropped with the rest of- // the local carrier, exactly as the base-wins rule treats every unknown table.- let (server_bucket_tables, unknown_server_buckets) = split_known_buckets(server_buckets);- let (local_bucket_tables, _) = split_known_buckets(local_buckets);-- let mut base_tables = server.tables.clone();- base_tables.extend(server_bucket_tables);- let mut incoming_tables = local.map(|m| m.tables).unwrap_or_default();- incoming_tables.extend(local_bucket_tables);-- let table_names: BTreeSet<String> = base_tables.keys().chain(incoming_tables.keys()).cloned().collect();-- let mut merged: HashMap<String, Vec<CodecRecord>> = HashMap::new();- for name in table_names {- let base_entry = base_tables.remove(&name);- let base_carried_table = base_entry.is_some();- let base_rows = base_entry.unwrap_or_default();- let incoming_rows = incoming_tables.remove(&name).unwrap_or_default();- let merged_rows = match SYNCABLE_TABLES.iter().find(|t| t.name == name) {- Some(config) => {- stats.tables_processed += 1;- merge_rows(config, base_rows, incoming_rows, schema_columns, &mut stats)- }- // Not a syncable table (a skip table, or one from a newer writer): the base wins as-is.- None => base_rows,- };- /*- * A table the base carried stays in the output even when it merged to nothing, so the merged- * manifest keeps the base's shape.- */- if base_carried_table || !merged_rows.is_empty() {- merged.insert(name, merged_rows);- }- }-- // A union of concurrently added multi-value rows can leave two rows at the same ValueIndex;- // re-normalizing the output renumbers them, so a merged manifest is normalized like any other.- crate::vault_codec::normalize::normalize_row_shapes(&mut merged);-- let mut buckets = unknown_server_buckets;- for category in bucket_categories() {- let mut bucket_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();- for table in tables_for_category(category) {- if let Some(rows) = merged.remove(table) {- bucket_tables.insert(table.to_string(), rows);- }- }- if !bucket_tables.is_empty() {- buckets.push(DataBucket::new(manifest_id.clone(), category.to_string(), bucket_tables));- }- }-- let manifest = Manifest { tables: merged, ..server };- CanonicalManifestMerge { manifest_id, manifest, buckets, stats }-}--/// Split buckets into the flattened rows of the tables this build buckets itself, and leftover-/// buckets holding every other table (a newer writer's), kept under their served category.-fn split_known_buckets(buckets: Vec<DataBucket>) -> (HashMap<String, Vec<CodecRecord>>, Vec<DataBucket>) {- let mut tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();- let mut unknown: Vec<DataBucket> = Vec::new();- for bucket in buckets {- let mut unknown_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();- for (name, rows) in bucket.tables {- if is_bucketed_table(&name) {- tables.entry(name).or_default().extend(rows);- } else {- unknown_tables.insert(name, rows);- }- }- if !unknown_tables.is_empty() {- unknown.push(DataBucket::new(bucket.manifest_id, bucket.category, unknown_tables));- }- }- (tables, unknown)-}--/// A contentless server manifest's local counterpart passes through whole: every row is an-/// offline-kept row.-fn pass_through(local: Manifest, local_buckets: Vec<DataBucket>) -> CanonicalManifestMerge {- let mut stats = MergeStats::default();- let row_count = |tables: &HashMap<String, Vec<CodecRecord>>| tables.values().map(|rows| rows.len() as u32).sum::<u32>();- stats.records_inserted = row_count(&local.tables) + local_buckets.iter().map(|b| row_count(&b.tables)).sum::<u32>();-- CanonicalManifestMerge { manifest_id: local.manifest_id.clone(), manifest: local.clone(), buckets: local_buckets, stats }-}--/// LWW one table, rows out: base rows in order (replaced where the incoming row is strictly-/// newer), then incoming-only rows in first-occurrence order.-fn merge_rows(- config: &TableConfig,- base_rows: Vec<CodecRecord>,- incoming_rows: Vec<CodecRecord>,- schema_columns: &HashMap<String, Vec<String>>,- stats: &mut MergeStats,-) -> Vec<CodecRecord> {- let identity_columns = config.identity_columns();- // Rows arrive normalized to the wire shape here, so a canonical-only key may rely on stripped- // derived ids (see the FieldValues registry comment).- let match_columns: &[&str] = if config.canonical_key_columns.is_empty() { &identity_columns } else { config.canonical_key_columns };- let known_columns: Option<HashSet<&str>> = schema_columns.get(config.name).map(|cols| cols.iter().map(String::as_str).collect());-- // Winner per match key among incoming rows; on duplicates the latest UpdatedAt wins.- let mut incoming_map: HashMap<String, &CodecRecord> = HashMap::new();- for record in &incoming_rows {- let key = get_key(record, match_columns);- match incoming_map.get(&key) {- Some(existing) if updated_at(record) <= updated_at(existing) => {}- _ => {- incoming_map.insert(key, record);- }- }- }-- let mut merged: Vec<CodecRecord> = Vec::with_capacity(base_rows.len());- for base_record in base_rows {- let key = get_key(&base_record, match_columns);- match incoming_map.remove(&key) {- Some(incoming) => {- let (incoming_ts, base_ts) = (updated_at(incoming), updated_at(&base_record));- match (incoming_ts, base_ts) {- (Some(i_ts), Some(b_ts)) if i_ts > b_ts => {- stats.conflicts += 1;- stats.records_from_server += 1;- merged.push(overlay_winner(incoming.clone(), &base_record, &identity_columns, known_columns.as_ref()));- }- _ => {- stats.records_from_local += 1;- merged.push(base_record);- }- }- }- None => {- stats.records_created_locally += 1;- merged.push(base_record);- }- }- }-- for record in &incoming_rows {- if let Some(winner) = incoming_map.remove(&get_key(record, match_columns)) {- stats.records_inserted += 1;- merged.push(winner.clone());- }- }-- merged-}--/// A winning incoming row.-fn overlay_winner(mut winner: CodecRecord, base: &CodecRecord, identity_columns: &[&str], known_columns: Option<&HashSet<&str>>) -> CodecRecord {- for (column, value) in base {- let base_wins = identity_columns.contains(&column.as_str()) || known_columns.is_some_and(|known| !known.contains(column.as_str()));- if base_wins {- winner.insert(column.clone(), value.clone());- }- }- winner-}--#[cfg(test)]-mod tests {- use super::*;- use serde_json::json;-- const PERSONAL: &str = "m-personal";- const SHARED: &str = "m-shared";- const SALT: &str = "0123456789abcdef0123456789abcdef";-- fn manifest(manifest_id: &str, tables: HashMap<String, Vec<CodecRecord>>) -> Manifest {- Manifest {- schema_version: 1,- manifest_salt: SALT.to_string(),- canonicalized_at: "2024-01-01T00:00:00Z".to_string(),- manifest_id: manifest_id.to_string(),- name: None,- tables,- extra: HashMap::new(),- }- }-- fn item(manifest_id: &str, id: &str, name: &str, updated_at: &str) -> CodecRecord {- [- ("ManifestId".to_string(), json!(manifest_id)),- ("Id".to_string(), json!(id)),- ("Name".to_string(), json!(name)),- ("UpdatedAt".to_string(), json!(updated_at)),- ]- .into_iter()- .collect()- }-- fn items_manifest(manifest_id: &str, rows: Vec<CodecRecord>) -> Manifest {- manifest(manifest_id, [("Items".to_string(), rows)].into_iter().collect())- }-- fn schema() -> HashMap<String, Vec<String>> {- [- ("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Name".to_string(), "UpdatedAt".to_string()]),- ("Settings".to_string(), vec!["ManifestId".to_string(), "Key".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),- ("ItemStats".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "LastUsedAt".to_string(), "UpdatedAt".to_string()]),- ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),- ("ItemTags".to_string(), vec!["ManifestId".to_string(), "ItemId".to_string(), "TagId".to_string(), "UpdatedAt".to_string()]),- ("Attachments".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Blob".to_string(), "UpdatedAt".to_string()]),- ]- .into_iter()- .collect()- }-- fn merge_single(server_rows: Vec<CodecRecord>, local_rows: Vec<CodecRecord>) -> CanonicalManifestMerge {- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![items_manifest(PERSONAL, server_rows)],- server_buckets: vec![],- contentless_server_manifest_ids: vec![],- local_manifests: vec![items_manifest(PERSONAL, local_rows)],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();- assert_eq!(output.manifests.len(), 1);- output.manifests.into_iter().next().unwrap()- }-- fn names_of(manifest: &Manifest) -> Vec<String> {- manifest.tables.get("Items").map(|rows| rows.iter().map(|r| r["Name"].as_str().unwrap().to_string()).collect()).unwrap_or_default()- }-- #[test]- fn local_newer_wins_server_newer_and_tie_keep_base() {- let merged = merge_single(- vec![- item(PERSONAL, "local-wins", "server-old", "2024-01-01T00:00:00Z"),- item(PERSONAL, "server-wins", "server-new", "2024-01-09T00:00:00Z"),- item(PERSONAL, "tie", "server-tie", "2024-01-05T00:00:00Z"),- ],- vec![- item(PERSONAL, "local-wins", "local-new", "2024-01-09T00:00:00Z"),- item(PERSONAL, "server-wins", "local-old", "2024-01-01T00:00:00Z"),- item(PERSONAL, "tie", "local-tie", "2024-01-05T00:00:00Z"),- ],- );-- assert_eq!(names_of(&merged.manifest), vec!["local-new", "server-new", "server-tie"], "ties and newer base rows keep the server version");- assert_eq!(merged.stats.conflicts, 1);- assert_eq!(merged.stats.records_from_server, 1);- assert_eq!(merged.stats.records_from_local, 2);- }-- #[test]- fn the_same_id_in_two_spellings_is_one_row() {- // GUIDs are case-insensitive and should be stored/compared as such.- let merged = merge_single(- vec![item(PERSONAL, "9d3f7a2c-1b4e-4f80-8a11-2c3d4e5f6a7b", "server-old", "2024-01-01T00:00:00Z")],- vec![item(PERSONAL, "9D3F7A2C-1B4E-4F80-8A11-2C3D4E5F6A7B", "local-new", "2024-01-09T00:00:00Z")],- );-- assert_eq!(names_of(&merged.manifest), vec!["local-new"], "one row, the newer one");- }-- #[test]- fn one_sided_rows_are_kept_on_both_sides() {- let merged = merge_single(- vec![item(PERSONAL, "server-only", "on-server", "2024-01-01T00:00:00Z")],- vec![item(PERSONAL, "local-only", "made-offline", "2024-01-02T00:00:00Z")],- );-- assert_eq!(names_of(&merged.manifest), vec!["on-server", "made-offline"], "base rows first, then offline-created rows");- assert_eq!(merged.stats.records_created_locally, 1);- assert_eq!(merged.stats.records_inserted, 1);- }-- #[test]- fn manifests_merge_independently_same_id_never_crosses() {- // The structural guarantee of the canonical merge: manifests are the partition, so the- // same row id in two manifests can never interact, whatever the timestamps say.- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![- items_manifest(PERSONAL, vec![item(PERSONAL, "same-id", "mine", "2024-01-01T00:00:00Z")]),- items_manifest(SHARED, vec![item(SHARED, "same-id", "theirs-old", "2024-01-01T00:00:00Z")]),- ],- server_buckets: vec![],- contentless_server_manifest_ids: vec![],- local_manifests: vec![- items_manifest(PERSONAL, vec![item(PERSONAL, "same-id", "mine", "2024-01-01T00:00:00Z")]),- items_manifest(SHARED, vec![item(SHARED, "same-id", "theirs-new", "2024-01-09T00:00:00Z")]),- ],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();-- let by_id: HashMap<&str, &CanonicalManifestMerge> = output.manifests.iter().map(|m| (m.manifest_id.as_str(), m)).collect();- assert_eq!(names_of(&by_id[PERSONAL].manifest), vec!["mine"], "the personal row is untouched by the shared manifest's newer row");- assert_eq!(names_of(&by_id[SHARED].manifest), vec!["theirs-new"]);- assert_eq!(by_id[PERSONAL].stats.conflicts, 0);- }-- #[test]- fn field_values_match_semantically_and_carry_no_surrogate_id() {- // A single-value FieldValue arrives in wire shape (no Id, position 0) and matches on its- // natural key: both sides converge to one row holding the winning value, still id-less.- let field_value = |value: &str, updated_at: &str| -> CodecRecord {- [- ("ManifestId".to_string(), json!(PERSONAL)),- ("ItemId".to_string(), json!("item-1")),- ("FieldKey".to_string(), json!("username")),- ("ValueIndex".to_string(), json!(0)),- ("Value".to_string(), json!(value)),- ("UpdatedAt".to_string(), json!(updated_at)),- ]- .into_iter()- .collect()- };-- let rows = merge_field_values(vec![field_value("old", "2024-01-01T00:00:00Z")], vec![field_value("new", "2024-01-09T00:00:00Z")]);- assert_eq!(rows.len(), 1);- assert!(rows[0].get("Id").is_none(), "a single-value row's id is derived at materialize, never carried on the wire");- assert_eq!(rows[0]["Value"], json!("new"), "and the row carries the winning value");- }-- /// A `login.url` FieldValues row: multi-value, so it owns its id on the wire.- fn url_value(id: &str, index: i64, value: &str, updated_at: &str) -> CodecRecord {- [- ("ManifestId".to_string(), json!(PERSONAL)),- ("Id".to_string(), json!(id)),- ("ItemId".to_string(), json!("item-1")),- ("FieldKey".to_string(), json!("login.url")),- ("ValueIndex".to_string(), json!(index)),- ("Value".to_string(), json!(value)),- ("UpdatedAt".to_string(), json!(updated_at)),- ]- .into_iter()- .collect()- }-- fn merge_field_values(server: Vec<CodecRecord>, local: Vec<CodecRecord>) -> Vec<CodecRecord> {- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), server)].into_iter().collect())],- server_buckets: vec![],- contentless_server_manifest_ids: vec![],- local_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), local)].into_iter().collect())],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();- output.manifests[0].manifest.tables["FieldValues"].clone()- }-- #[test]- fn multi_value_edit_converges_on_the_owned_id() {- // The same owned id on both sides is the same row edited: LWW, one row out.- let rows = merge_field_values(- vec![url_value("url-1", 0, "https://a.example", "2024-01-01T00:00:00Z")],- vec![url_value("url-1", 0, "https://a-edited.example", "2024-01-09T00:00:00Z")],- );- assert_eq!(rows.len(), 1);- assert_eq!(rows[0]["Id"], json!("url-1"), "the row keeps its owned id");- assert_eq!(rows[0]["Value"], json!("https://a-edited.example"));- }-- #[test]- fn multi_value_concurrent_adds_both_survive() {- // Two devices each add a 2nd url while offline: both additions land at position 1 under- // different owned ids. Position is not identity, so the union keeps both, renumbered.- let rows = merge_field_values(- vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-05T00:00:00Z")],- vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-c", 1, "https://c.example", "2024-01-06T00:00:00Z")],- );- let mut values: Vec<&str> = rows.iter().map(|r| r["Value"].as_str().unwrap()).collect();- values.sort();- assert_eq!(values, vec!["https://a.example", "https://b.example", "https://c.example"]);- let mut indexes: Vec<i64> = rows.iter().map(|r| r["ValueIndex"].as_i64().unwrap()).collect();- indexes.sort();- assert_eq!(indexes, vec![0, 1, 2], "colliding positions are renumbered in the merged output");- }-- #[test]- fn multi_value_reorder_does_not_duplicate() {- // Reordering changes only ValueIndex; the owned id keeps the rows matched, so no row doubles.- let rows = merge_field_values(- vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-01T00:00:00Z")],- vec![url_value("url-b", 0, "https://b.example", "2024-01-09T00:00:00Z"), url_value("url-a", 1, "https://a.example", "2024-01-09T00:00:00Z")],- );- assert_eq!(rows.len(), 2);- let by_id: HashMap<&str, i64> = rows.iter().map(|r| (r["Id"].as_str().unwrap(), r["ValueIndex"].as_i64().unwrap())).collect();- assert_eq!(by_id["url-b"], 0, "the reorder won");- assert_eq!(by_id["url-a"], 1);- }-- #[test]- fn both_sides_holding_the_same_two_urls_stay_two_rows() {- // The historic corruption case: before the id-based key, the 2nd url's value replaced the- // 1st on an ordinary push. Identical sides must merge to exactly themselves.- let two = vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-02T00:00:00Z")];- let rows = merge_field_values(two.clone(), two);- assert_eq!(rows.len(), 2);- let by_id: HashMap<&str, &str> = rows.iter().map(|r| (r["Id"].as_str().unwrap(), r["Value"].as_str().unwrap())).collect();- assert_eq!(by_id["url-a"], "https://a.example");- assert_eq!(by_id["url-b"], "https://b.example");- }-- #[test]- fn custom_field_values_of_one_item_do_not_collapse() {- // Custom fields carry no FieldKey (only a FieldDefinitionId); before FieldDefinitionId joined- // the key they all collapsed onto one empty-string key.- let custom = |def: &str, value: &str, updated_at: &str| -> CodecRecord {- [- ("ManifestId".to_string(), json!(PERSONAL)),- ("Id".to_string(), json!(format!("cv-{}", def))),- ("ItemId".to_string(), json!("item-1")),- ("FieldKey".to_string(), serde_json::Value::Null),- ("FieldDefinitionId".to_string(), json!(def)),- ("Value".to_string(), json!(value)),- ("UpdatedAt".to_string(), json!(updated_at)),- ]- .into_iter()- .collect()- };- let rows = merge_field_values(- vec![custom("fd-pin", "1234", "2024-01-01T00:00:00Z"), custom("fd-member", "M-77", "2024-01-01T00:00:00Z")],- vec![custom("fd-pin", "9999", "2024-01-09T00:00:00Z"), custom("fd-member", "M-77", "2024-01-01T00:00:00Z")],- );- assert_eq!(rows.len(), 2, "each custom field keeps its own row");- let mut values: Vec<&str> = rows.iter().map(|r| r["Value"].as_str().unwrap()).collect();- values.sort();- assert_eq!(values, vec!["9999", "M-77"]);- }-- #[test]- fn field_histories_union_by_changed_at_and_converge_on_the_same_snapshot() {- // History rows derive their id from (item, field, ChangedAt): concurrent changes on two- // devices carry different timestamps and union; the same snapshot on both sides is one row.- let history = |changed_at: &str, snapshot: &str| -> CodecRecord {- [- ("ManifestId".to_string(), json!(PERSONAL)),- ("ItemId".to_string(), json!("item-1")),- ("FieldKey".to_string(), json!("login.password")),- ("ChangedAt".to_string(), json!(changed_at)),- ("ValueSnapshot".to_string(), json!(snapshot)),- ("UpdatedAt".to_string(), json!(changed_at)),- ]- .into_iter()- .collect()- };- let shared = history("2024-01-01 10:00:00.000", "both-know-this");- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![manifest(PERSONAL, [("FieldHistories".to_string(), vec![shared.clone(), history("2024-02-01 10:00:00.000", "server-only")])].into_iter().collect())],- server_buckets: vec![],- contentless_server_manifest_ids: vec![],- local_manifests: vec![manifest(PERSONAL, [("FieldHistories".to_string(), vec![shared, history("2024-02-01 11:30:00.000", "local-only")])].into_iter().collect())],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();- let rows = &output.manifests[0].manifest.tables["FieldHistories"];- let mut snapshots: Vec<&str> = rows.iter().map(|r| r["ValueSnapshot"].as_str().unwrap()).collect();- snapshots.sort();- assert_eq!(snapshots, vec!["both-know-this", "local-only", "server-only"]);- assert!(rows.iter().all(|r| r.get("Id").is_none()), "merged history rows stay id-less on the wire");- }-- #[test]- fn item_tags_converge_on_their_natural_key() {- // Two devices tag the same item with the same tag: the id-less join rows are one row.- let tag_row = |updated_at: &str| -> CodecRecord {- [- ("ManifestId".to_string(), json!(PERSONAL)),- ("ItemId".to_string(), json!("item-1")),- ("TagId".to_string(), json!("tag-1")),- ("UpdatedAt".to_string(), json!(updated_at)),- ]- .into_iter()- .collect()- };- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![manifest(PERSONAL, [("ItemTags".to_string(), vec![tag_row("2024-01-01T00:00:00Z")])].into_iter().collect())],- server_buckets: vec![],- contentless_server_manifest_ids: vec![],- local_manifests: vec![manifest(PERSONAL, [("ItemTags".to_string(), vec![tag_row("2024-01-09T00:00:00Z")])].into_iter().collect())],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();- let rows = &output.manifests[0].manifest.tables["ItemTags"];- assert_eq!(rows.len(), 1, "the same (item, tag) link is one row on both sides");- assert!(rows[0].get("Id").is_none());- }-- #[test]- fn bucketed_tables_merge_and_come_back_as_buckets() {- let setting = |value: &str, updated_at: &str| -> CodecRecord {- [- ("ManifestId".to_string(), json!(PERSONAL)),- ("Key".to_string(), json!("theme")),- ("Value".to_string(), json!(value)),- ("UpdatedAt".to_string(), json!(updated_at)),- ]- .into_iter()- .collect()- };-- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![items_manifest(PERSONAL, vec![])],- server_buckets: vec![DataBucket::new(PERSONAL, "Settings", [("Settings".to_string(), vec![setting("light", "2024-01-01T00:00:00Z")])].into_iter().collect())],- contentless_server_manifest_ids: vec![],- local_manifests: vec![items_manifest(PERSONAL, vec![])],- local_buckets: vec![DataBucket::new(PERSONAL, "Settings", [("Settings".to_string(), vec![setting("dark", "2024-01-09T00:00:00Z")])].into_iter().collect())],- schema_columns: schema(),- })- .unwrap();-- let merged = &output.manifests[0];- assert!(!merged.manifest.tables.contains_key("Settings"), "bucketed tables never surface in manifest.tables");- assert_eq!(merged.buckets.len(), 1);- assert_eq!(merged.buckets[0].category, "Settings");- assert_eq!(merged.buckets[0].tables["Settings"][0]["Value"], json!("dark"), "the newer local setting wins inside the bucket");- assert!(crate::vault_codec::validate_data_bucket(&merged.buckets[0]).ok);- }-- #[test]- fn an_empty_table_the_server_carried_stays_in_the_merged_manifest() {- /*- * The merged manifest is pushed as-is and compared against a baseline canonicalize produced, and- * canonicalize keeps an empty table. Dropping it here rewrites the whole manifest on every merge.- */- let merged = merge_single(vec![], vec![]);- assert_eq!(merged.manifest.tables.get("Items").map(Vec::len), Some(0), "the server's empty table keeps its place in the merged manifest");-- // The empty tables of a bucket the server served come back as that bucket, not as a missing one.- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![items_manifest(PERSONAL, vec![])],- server_buckets: vec![DataBucket::new(PERSONAL, "Settings", [("Settings".to_string(), vec![])].into_iter().collect())],- contentless_server_manifest_ids: vec![],- local_manifests: vec![items_manifest(PERSONAL, vec![])],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();- let buckets = &output.manifests[0].buckets;- assert_eq!(buckets.len(), 1, "the served bucket is still emitted when it holds no rows");- assert_eq!(buckets[0].tables.get("Settings").map(Vec::len), Some(0));- }-- #[test]- fn unknown_columns_keep_the_server_value_when_local_wins() {- // A newer client added Items.NewColumn; this client's schema does not know it. The local- // row winning the LWW must not regress that column.- let mut base = item(PERSONAL, "row", "server-old", "2024-01-01T00:00:00Z");- base.insert("NewColumn".to_string(), json!("newer-client-value"));- let mut incoming = item(PERSONAL, "row", "local-new", "2024-01-09T00:00:00Z");- incoming.insert("NewColumn".to_string(), json!("stale-carried-value"));-- let merged = merge_single(vec![base], vec![incoming]);-- let row = &merged.manifest.tables["Items"][0];- assert_eq!(row["Name"], json!("local-new"), "known columns follow the winner");- assert_eq!(row["NewColumn"], json!("newer-client-value"), "unknown columns keep the base value");- }-- #[test]- fn unknown_tables_are_taken_from_the_server_wholesale() {- let future_row = |payload: &str| -> CodecRecord {- [("ManifestId".to_string(), json!(PERSONAL)), ("Id".to_string(), json!("f-1")), ("Payload".to_string(), json!(payload))].into_iter().collect()- };- let with_future = |payload: Option<&str>| -> Manifest {- let mut tables: HashMap<String, Vec<CodecRecord>> = [("Items".to_string(), vec![])].into_iter().collect();- if let Some(p) = payload {- tables.insert("FutureTable".to_string(), vec![future_row(p)]);- }- manifest(PERSONAL, tables)- };-- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![with_future(Some("server-version"))],- server_buckets: vec![],- contentless_server_manifest_ids: vec![],- local_manifests: vec![with_future(Some("local-carried-version"))],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();- assert_eq!(output.manifests[0].manifest.tables["FutureTable"][0]["Payload"], json!("server-version"));-- // And a local-only unknown table is dropped, matching the discarded local carrier of today.- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![with_future(None)],- server_buckets: vec![],- contentless_server_manifest_ids: vec![],- local_manifests: vec![with_future(Some("local-only"))],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();- assert!(!output.manifests[0].manifest.tables.contains_key("FutureTable"));- }-- #[test]- fn local_manifest_no_longer_served_is_dropped_and_reported() {- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![items_manifest(PERSONAL, vec![])],- server_buckets: vec![],- contentless_server_manifest_ids: vec![],- local_manifests: vec![items_manifest(PERSONAL, vec![]), items_manifest(SHARED, vec![item(SHARED, "x", "gone", "2024-01-01T00:00:00Z")])],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();-- assert_eq!(output.manifests.len(), 1);- assert_eq!(output.dropped_local_manifest_ids, vec![SHARED.to_string()]);- }-- #[test]- fn contentless_server_manifest_passes_the_local_one_through() {- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![items_manifest(PERSONAL, vec![])],- server_buckets: vec![],- contentless_server_manifest_ids: vec![SHARED.to_string()],- local_manifests: vec![items_manifest(PERSONAL, vec![]), items_manifest(SHARED, vec![item(SHARED, "x", "kept-offline", "2024-01-01T00:00:00Z")])],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();-- assert!(output.dropped_local_manifest_ids.is_empty());- let shared = output.manifests.iter().find(|m| m.manifest_id == SHARED).unwrap();- assert_eq!(names_of(&shared.manifest), vec!["kept-offline"]);- assert_eq!(shared.stats.records_inserted, 1);- }-- #[test]- fn blob_refs_ride_through_as_opaque_values() {- let attachment = |blob_ref: &str, updated_at: &str| -> CodecRecord {- [- ("ManifestId".to_string(), json!(PERSONAL)),- ("Id".to_string(), json!("att-1")),- ("Blob".to_string(), json!({ "__blobRef": blob_ref, "__blobKind": "attachment" })),- ("UpdatedAt".to_string(), json!(updated_at)),- ]- .into_iter()- .collect()- };-- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![manifest(PERSONAL, [("Attachments".to_string(), vec![attachment("hash-old", "2024-01-01T00:00:00Z")])].into_iter().collect())],- server_buckets: vec![],- contentless_server_manifest_ids: vec![],- local_manifests: vec![manifest(PERSONAL, [("Attachments".to_string(), vec![attachment("hash-new", "2024-01-09T00:00:00Z")])].into_iter().collect())],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();-- assert_eq!(output.manifests[0].manifest.tables["Attachments"][0]["Blob"], json!({ "__blobRef": "hash-new", "__blobKind": "attachment" }));- }-- #[test]- fn merged_manifest_keeps_server_metadata_and_validates() {- let mut server = items_manifest(PERSONAL, vec![item(PERSONAL, "a", "server", "2024-01-01T00:00:00Z")]);- server.name = Some("Family".to_string());- let local = items_manifest(PERSONAL, vec![item(PERSONAL, "b", "local", "2024-01-02T00:00:00Z")]);-- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![server],- server_buckets: vec![],- contentless_server_manifest_ids: vec![],- local_manifests: vec![local],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();-- let merged = &output.manifests[0].manifest;- assert_eq!(merged.manifest_salt, SALT, "salt comes from the server manifest");- assert_eq!(merged.name.as_deref(), Some("Family"), "name comes from the server manifest");- let validation = crate::vault_codec::validate_manifest(merged);- assert!(validation.ok, "merged manifest must validate: {:?}", validation.failed_rules);- }-- #[test]- fn manifest_pairing_is_case_insensitive() {- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![items_manifest("M-PERSONAL", vec![item("M-PERSONAL", "a", "server", "2024-01-01T00:00:00Z")])],- server_buckets: vec![],- contentless_server_manifest_ids: vec![],- local_manifests: vec![items_manifest("m-personal", vec![item("m-personal", "b", "local", "2024-01-02T00:00:00Z")])],- local_buckets: vec![],- schema_columns: schema(),- })- .unwrap();-- assert!(output.dropped_local_manifest_ids.is_empty(), "differently-cased ids are the same manifest");- assert_eq!(output.manifests[0].manifest.tables["Items"].len(), 2);- }-- #[test]- fn unknown_category_bucket_tables_ride_through_from_the_server() {- // A newer writer added a table inside the Settings bucket and a whole new category. The- // known Settings rows still merge; the unknown tables ride through from the server only.- let unknown_row: CodecRecord = [("ManifestId".to_string(), json!(PERSONAL)), ("Id".to_string(), json!("u-1"))].into_iter().collect();- let server_settings = DataBucket::new(- PERSONAL,- "Settings",- [- ("Settings".to_string(), vec![[("ManifestId".to_string(), json!(PERSONAL)), ("Key".to_string(), json!("theme")), ("Value".to_string(), json!("light")), ("UpdatedAt".to_string(), json!("2024-01-01T00:00:00Z"))].into_iter().collect()]),- ("SettingsExtras".to_string(), vec![unknown_row.clone()]),- ]- .into_iter()- .collect(),- );- let local_future = DataBucket::new(PERSONAL, "FutureCategory", [("FutureRows".to_string(), vec![unknown_row])].into_iter().collect());-- let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![items_manifest(PERSONAL, vec![])],- server_buckets: vec![server_settings],- contentless_server_manifest_ids: vec![],- local_manifests: vec![items_manifest(PERSONAL, vec![])],- local_buckets: vec![local_future],- schema_columns: schema(),- })- .unwrap();-- let buckets = &output.manifests[0].buckets;- let categories: Vec<&str> = buckets.iter().map(|b| b.category.as_str()).collect();- assert!(categories.contains(&"Settings"), "the known Settings rows still come back as a bucket");- assert!(buckets.iter().any(|b| b.category == "Settings" && b.tables.contains_key("SettingsExtras")), "the server's unknown bucket table rides through under its category");- assert!(!categories.contains(&"FutureCategory"), "a local-only unknown bucket is dropped with the rest of the local carrier");- }-}diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rsindex affc136ff..4accd1160 100644--- a/core/rust/src/vault_merge/mod.rs+++ b/core/rust/src/vault_merge/mod.rs@@ -1,22 +1,24 @@-//! Vault merge logic using Last-Write-Wins (LWW) strategy.+//! Vault merge: Last-Write-Wins over manifest JSON + data buckets, rows in, rows out.+//!+//! This is the merge for the manifest-v1 storage format (used since 0.31.0+). It runs one layer above any concrete+//! materialization (SQLite or otherwise): both sides arrive in canonical form and the output is+//! the merged canonical form, which the platform then materializes once.+//!+//! Each manifest is merged independently (the server manifest set is the universe), so a broken+//! manifest can never affect another manifest's rows, and blob columns carry `__blobRef` markers+//! rather than bytes, so no byte payload ever crosses the merge.-mod canonical;+use std::collections::{BTreeSet, HashMap, HashSet};use serde::{Deserialize, Serialize};-pub use canonical::{merge_canonical, CanonicalManifestMerge, CanonicalMergeInput, CanonicalMergeOutput};+use crate::error::VaultResult;+use crate::timestamp::updated_at;+use crate::vault_model::{id_key, TableConfig, SYNCABLE_TABLES};+use crate::vault_codec::{bucket_categories, identity_part, is_bucketed_table, tables_for_category, CodecRecord, DataBucket, Manifest};-/// A record is a map of column names to JSON values.-pub type Record = crate::vault_codec::CodecRecord;--/// A SQL statement with its parameter values.-#[derive(Debug, Clone, Serialize, Deserialize)]-pub struct SqlStatement {- /// The SQL query with ? placeholders- pub sql: String,- /// Parameter values in order- pub params: Vec<serde_json::Value>,-}+#[cfg(test)]+mod tests;/// Statistics about what was merged.#[derive(Debug, Clone, Default, Serialize, Deserialize)]@@ -31,35 +33,268 @@ pub struct MergeStats {pub records_inserted: u32,}-/// Stable string key over `columns`. A column the record does not carry contributes an empty part,-/// so a row missing one still matches its counterpart rather than dropping out of the merge. See-/// [`crate::vault_codec::identity_part`]: ids compare case-insensitively, everything else exactly as spelled.-fn get_key(record: &Record, columns: &[&str]) -> String {- columns- .iter()- .map(|column| record.get(*column).filter(|v| !v.is_null()).map(crate::vault_codec::identity_part).unwrap_or_default())- .collect::<Vec<_>>()- .join(":")+impl MergeStats {+ /// Add another manifest's counters to these, for a whole-vault total.+ pub fn add(&mut self, other: &MergeStats) {+ self.tables_processed += other.tables_processed;+ self.records_from_local += other.records_from_local;+ self.records_from_server += other.records_from_server;+ self.records_created_locally += other.records_created_locally;+ self.conflicts += other.conflicts;+ self.records_inserted += other.records_inserted;+ }}-#[cfg(test)]-mod tests {- use super::*;- use std::collections::HashMap;-- #[test]- fn keys_compare_guids_case_insensitively_and_survive_a_missing_column() {- const MANIFEST: &str = "1dd1a3fd-8e0e-4b3f-9a3a-0f1a2b3c4d5e";- const OTHER_MANIFEST: &str = "2ee2b4fe-9f1f-4c40-8b4b-1a2b3c4d5e6f";- const ROW: &str = "3ff3c50f-a020-4d51-9c5c-2b3c4d5e6f70";- let row = |manifest: &str, id: &str| -> Record {- HashMap::from([("ManifestId".to_string(), serde_json::json!(manifest)), ("Id".to_string(), serde_json::json!(id))])+/// Input of the canonical merge. The server side is the base (kept on ties); the local side is the+/// incoming set produced by `canonicalize_from_sqlite`.+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct CanonicalMergeInput {+ pub server_manifests: Vec<Manifest>,+ pub server_buckets: Vec<DataBucket>,+ #[serde(default)]+ pub contentless_server_manifest_ids: Vec<String>,+ pub local_manifests: Vec<Manifest>,+ pub local_buckets: Vec<DataBucket>,+ pub schema_columns: HashMap<String, Vec<String>>,+}++/// One manifest's merged result.+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct CanonicalManifestMerge {+ pub manifest_id: String,+ pub manifest: Manifest,+ pub buckets: Vec<DataBucket>,+ pub stats: MergeStats,+}++/// Output of the canonical merge.+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct CanonicalMergeOutput {+ pub manifests: Vec<CanonicalManifestMerge>,+ pub dropped_local_manifest_ids: Vec<String>,+}++/// Merge the local canonical vault onto the server canonical vault (the base), per manifest.+pub fn merge_canonical(input: CanonicalMergeInput) -> VaultResult<CanonicalMergeOutput> {+ let CanonicalMergeInput { server_manifests, server_buckets, contentless_server_manifest_ids, local_manifests, local_buckets, schema_columns } = input;++ let mut local_by_id: HashMap<String, Manifest> = local_manifests.into_iter().map(|m| (id_key(&m.manifest_id), m)).collect();+ let mut local_buckets_by_id = group_buckets(local_buckets);+ let mut server_buckets_by_id = group_buckets(server_buckets);++ let mut manifests: Vec<CanonicalManifestMerge> = Vec::new();+ for server_manifest in server_manifests {+ let key = id_key(&server_manifest.manifest_id);+ let local = local_by_id.remove(&key);+ let local_buckets = local_buckets_by_id.remove(&key).unwrap_or_default();+ let server_buckets = server_buckets_by_id.remove(&key).unwrap_or_default();+ manifests.push(merge_manifest_pair(server_manifest, server_buckets, local, local_buckets, &schema_columns));+ }++ // A contentless server manifest has no base; the local counterpart passes through whole.+ for manifest_id in contentless_server_manifest_ids {+ let key = id_key(&manifest_id);+ if let Some(local) = local_by_id.remove(&key) {+ manifests.push(pass_through(local, local_buckets_by_id.remove(&key).unwrap_or_default()));+ }+ }++ let mut dropped_local_manifest_ids: Vec<String> = local_by_id.into_values().map(|m| m.manifest_id).collect();+ dropped_local_manifest_ids.sort();++ Ok(CanonicalMergeOutput { manifests, dropped_local_manifest_ids })+}++/// Group buckets by their manifest id (lowercased).+fn group_buckets(buckets: Vec<DataBucket>) -> HashMap<String, Vec<DataBucket>> {+ let mut grouped: HashMap<String, Vec<DataBucket>> = HashMap::new();+ for bucket in buckets {+ grouped.entry(id_key(&bucket.manifest_id)).or_default().push(bucket);+ }+ grouped+}++/// Merge one manifest: LWW over the flattened table view of both sides, then split the bucketed+/// tables back out. The merged `Manifest` is the server one with only its tables replaced.+fn merge_manifest_pair(+ server: Manifest,+ server_buckets: Vec<DataBucket>,+ local: Option<Manifest>,+ local_buckets: Vec<DataBucket>,+ schema_columns: &HashMap<String, Vec<String>>,+) -> CanonicalManifestMerge {+ let manifest_id = server.manifest_id.clone();+ let mut stats = MergeStats::default();++ // A bucket of a category this build does not know cannot be flattened without losing its+ // category; the server's ride through as-is and the local ones are dropped with the rest of+ // the local carrier, exactly as the base-wins rule treats every unknown table.+ let (server_bucket_tables, unknown_server_buckets) = split_known_buckets(server_buckets);+ let (local_bucket_tables, _) = split_known_buckets(local_buckets);++ let mut base_tables = server.tables.clone();+ base_tables.extend(server_bucket_tables);+ let mut incoming_tables = local.map(|m| m.tables).unwrap_or_default();+ incoming_tables.extend(local_bucket_tables);++ let table_names: BTreeSet<String> = base_tables.keys().chain(incoming_tables.keys()).cloned().collect();++ let mut merged: HashMap<String, Vec<CodecRecord>> = HashMap::new();+ for name in table_names {+ let base_entry = base_tables.remove(&name);+ let base_carried_table = base_entry.is_some();+ let base_rows = base_entry.unwrap_or_default();+ let incoming_rows = incoming_tables.remove(&name).unwrap_or_default();+ let merged_rows = match SYNCABLE_TABLES.iter().find(|t| t.name == name) {+ Some(config) => {+ stats.tables_processed += 1;+ merge_rows(config, base_rows, incoming_rows, schema_columns, &mut stats)+ }+ // Not a syncable table (a skip table, or one from a newer writer): the base wins as-is.+ None => base_rows,};- let columns = ["ManifestId", "Id"];+ /*+ * A table the base carried stays in the output even when it merged to nothing, so the merged+ * manifest keeps the base's shape.+ */+ if base_carried_table || !merged_rows.is_empty() {+ merged.insert(name, merged_rows);+ }+ }- // Two writers spelling the same ids in different cases address one row (see `identity_part`).- assert_eq!(get_key(&row(MANIFEST, ROW), &columns), get_key(&row(&MANIFEST.to_uppercase(), &ROW.to_uppercase()), &columns));- assert_ne!(get_key(&row(MANIFEST, ROW), &columns), get_key(&row(OTHER_MANIFEST, ROW), &columns));- assert_eq!(get_key(&HashMap::from([("Id".to_string(), serde_json::json!(ROW))]), &columns), format!(":{}", ROW));+ // A union of concurrently added multi-value rows can leave two rows at the same ValueIndex;+ // re-normalizing the output renumbers them, so a merged manifest is normalized like any other.+ crate::vault_codec::normalize::normalize_row_shapes(&mut merged);++ let mut buckets = unknown_server_buckets;+ for category in bucket_categories() {+ let mut bucket_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();+ for table in tables_for_category(category) {+ if let Some(rows) = merged.remove(table) {+ bucket_tables.insert(table.to_string(), rows);+ }+ }+ if !bucket_tables.is_empty() {+ buckets.push(DataBucket::new(manifest_id.clone(), category.to_string(), bucket_tables));+ }}++ let manifest = Manifest { tables: merged, ..server };+ CanonicalManifestMerge { manifest_id, manifest, buckets, stats }+}++/// Split buckets into the flattened rows of the tables this build buckets itself, and leftover+/// buckets holding every other table (a newer writer's), kept under their served category.+fn split_known_buckets(buckets: Vec<DataBucket>) -> (HashMap<String, Vec<CodecRecord>>, Vec<DataBucket>) {+ let mut tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();+ let mut unknown: Vec<DataBucket> = Vec::new();+ for bucket in buckets {+ let mut unknown_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();+ for (name, rows) in bucket.tables {+ if is_bucketed_table(&name) {+ tables.entry(name).or_default().extend(rows);+ } else {+ unknown_tables.insert(name, rows);+ }+ }+ if !unknown_tables.is_empty() {+ unknown.push(DataBucket::new(bucket.manifest_id, bucket.category, unknown_tables));+ }+ }+ (tables, unknown)+}++/// A contentless server manifest's local counterpart passes through whole: every row is an+/// offline-kept row.+fn pass_through(local: Manifest, local_buckets: Vec<DataBucket>) -> CanonicalManifestMerge {+ let mut stats = MergeStats::default();+ let row_count = |tables: &HashMap<String, Vec<CodecRecord>>| tables.values().map(|rows| rows.len() as u32).sum::<u32>();+ stats.records_inserted = row_count(&local.tables) + local_buckets.iter().map(|b| row_count(&b.tables)).sum::<u32>();++ CanonicalManifestMerge { manifest_id: local.manifest_id.clone(), manifest: local.clone(), buckets: local_buckets, stats }+}++/// LWW one table, rows out: base rows in order (replaced where the incoming row is strictly+/// newer), then incoming-only rows in first-occurrence order.+fn merge_rows(+ config: &TableConfig,+ base_rows: Vec<CodecRecord>,+ incoming_rows: Vec<CodecRecord>,+ schema_columns: &HashMap<String, Vec<String>>,+ stats: &mut MergeStats,+) -> Vec<CodecRecord> {+ let identity_columns = config.identity_columns();+ // Rows arrive normalized to the wire shape here, so a canonical-only key may rely on stripped+ // derived ids (see the FieldValues registry comment).+ let match_columns: &[&str] = if config.canonical_key_columns.is_empty() { &identity_columns } else { config.canonical_key_columns };+ let known_columns: Option<HashSet<&str>> = schema_columns.get(config.name).map(|cols| cols.iter().map(String::as_str).collect());++ // Winner per match key among incoming rows; on duplicates the latest UpdatedAt wins.+ let mut incoming_map: HashMap<String, &CodecRecord> = HashMap::new();+ for record in &incoming_rows {+ let key = get_key(record, match_columns);+ match incoming_map.get(&key) {+ Some(existing) if updated_at(record) <= updated_at(existing) => {}+ _ => {+ incoming_map.insert(key, record);+ }+ }+ }++ let mut merged: Vec<CodecRecord> = Vec::with_capacity(base_rows.len());+ for base_record in base_rows {+ let key = get_key(&base_record, match_columns);+ match incoming_map.remove(&key) {+ Some(incoming) => {+ let (incoming_ts, base_ts) = (updated_at(incoming), updated_at(&base_record));+ match (incoming_ts, base_ts) {+ (Some(i_ts), Some(b_ts)) if i_ts > b_ts => {+ stats.conflicts += 1;+ stats.records_from_server += 1;+ merged.push(overlay_winner(incoming.clone(), &base_record, &identity_columns, known_columns.as_ref()));+ }+ _ => {+ stats.records_from_local += 1;+ merged.push(base_record);+ }+ }+ }+ None => {+ stats.records_created_locally += 1;+ merged.push(base_record);+ }+ }+ }++ for record in &incoming_rows {+ if let Some(winner) = incoming_map.remove(&get_key(record, match_columns)) {+ stats.records_inserted += 1;+ merged.push(winner.clone());+ }+ }++ merged+}++/// A winning incoming row.+fn overlay_winner(mut winner: CodecRecord, base: &CodecRecord, identity_columns: &[&str], known_columns: Option<&HashSet<&str>>) -> CodecRecord {+ for (column, value) in base {+ let base_wins = identity_columns.contains(&column.as_str()) || known_columns.is_some_and(|known| !known.contains(column.as_str()));+ if base_wins {+ winner.insert(column.clone(), value.clone());+ }+ }+ winner+}++/// Stable string key over `columns`. A column the record does not carry contributes an empty part,+/// so a row missing one still matches its counterpart rather than dropping out of the merge; that is+/// why this is not `vault_codec::row_identity`, which skips absent parts. Ids compare+/// case-insensitively (see [`identity_part`]), everything else exactly as spelled.+fn get_key(record: &CodecRecord, columns: &[&str]) -> String {+ let parts: Vec<String> = columns.iter().map(|column| record.get(*column).filter(|v| !v.is_null()).map(identity_part).unwrap_or_default()).collect();+ parts.join(":")}diff --git a/core/rust/src/vault_merge/tests.rs b/core/rust/src/vault_merge/tests.rsnew file mode 100644index 000000000..f4bb3d567--- /dev/null+++ b/core/rust/src/vault_merge/tests.rs@@ -0,0 +1,584 @@+//! Tests for the canonical merge.++use super::*;+use serde_json::json;++const PERSONAL: &str = "m-personal";+const SHARED: &str = "m-shared";+const SALT: &str = "0123456789abcdef0123456789abcdef";++fn manifest(manifest_id: &str, tables: HashMap<String, Vec<CodecRecord>>) -> Manifest {+ Manifest {+ schema_version: 1,+ manifest_salt: SALT.to_string(),+ canonicalized_at: "2024-01-01T00:00:00Z".to_string(),+ manifest_id: manifest_id.to_string(),+ name: None,+ tables,+ extra: HashMap::new(),+ }+}++fn item(manifest_id: &str, id: &str, name: &str, updated_at: &str) -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(manifest_id)),+ ("Id".to_string(), json!(id)),+ ("Name".to_string(), json!(name)),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect()+}++fn items_manifest(manifest_id: &str, rows: Vec<CodecRecord>) -> Manifest {+ manifest(manifest_id, [("Items".to_string(), rows)].into_iter().collect())+}++fn schema() -> HashMap<String, Vec<String>> {+ [+ ("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Name".to_string(), "UpdatedAt".to_string()]),+ ("Settings".to_string(), vec!["ManifestId".to_string(), "Key".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),+ ("ItemStats".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "LastUsedAt".to_string(), "UpdatedAt".to_string()]),+ ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),+ ("ItemTags".to_string(), vec!["ManifestId".to_string(), "ItemId".to_string(), "TagId".to_string(), "UpdatedAt".to_string()]),+ ("Attachments".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Blob".to_string(), "UpdatedAt".to_string()]),+ ]+ .into_iter()+ .collect()+}++fn merge_single(server_rows: Vec<CodecRecord>, local_rows: Vec<CodecRecord>) -> CanonicalManifestMerge {+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![items_manifest(PERSONAL, server_rows)],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![items_manifest(PERSONAL, local_rows)],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();+ assert_eq!(output.manifests.len(), 1);+ output.manifests.into_iter().next().unwrap()+}++fn names_of(manifest: &Manifest) -> Vec<String> {+ manifest.tables.get("Items").map(|rows| rows.iter().map(|r| r["Name"].as_str().unwrap().to_string()).collect()).unwrap_or_default()+}++#[test]+fn local_newer_wins_server_newer_and_tie_keep_base() {+ let merged = merge_single(+ vec![+ item(PERSONAL, "local-wins", "server-old", "2024-01-01T00:00:00Z"),+ item(PERSONAL, "server-wins", "server-new", "2024-01-09T00:00:00Z"),+ item(PERSONAL, "tie", "server-tie", "2024-01-05T00:00:00Z"),+ ],+ vec![+ item(PERSONAL, "local-wins", "local-new", "2024-01-09T00:00:00Z"),+ item(PERSONAL, "server-wins", "local-old", "2024-01-01T00:00:00Z"),+ item(PERSONAL, "tie", "local-tie", "2024-01-05T00:00:00Z"),+ ],+ );++ assert_eq!(names_of(&merged.manifest), vec!["local-new", "server-new", "server-tie"], "ties and newer base rows keep the server version");+ assert_eq!(merged.stats.conflicts, 1);+ assert_eq!(merged.stats.records_from_server, 1);+ assert_eq!(merged.stats.records_from_local, 2);+}++#[test]+fn the_same_id_in_two_spellings_is_one_row() {+ // GUIDs are case-insensitive and should be stored/compared as such.+ let merged = merge_single(+ vec![item(PERSONAL, "9d3f7a2c-1b4e-4f80-8a11-2c3d4e5f6a7b", "server-old", "2024-01-01T00:00:00Z")],+ vec![item(PERSONAL, "9D3F7A2C-1B4E-4F80-8A11-2C3D4E5F6A7B", "local-new", "2024-01-09T00:00:00Z")],+ );++ assert_eq!(names_of(&merged.manifest), vec!["local-new"], "one row, the newer one");+}++#[test]+fn one_sided_rows_are_kept_on_both_sides() {+ let merged = merge_single(+ vec![item(PERSONAL, "server-only", "on-server", "2024-01-01T00:00:00Z")],+ vec![item(PERSONAL, "local-only", "made-offline", "2024-01-02T00:00:00Z")],+ );++ assert_eq!(names_of(&merged.manifest), vec!["on-server", "made-offline"], "base rows first, then offline-created rows");+ assert_eq!(merged.stats.records_created_locally, 1);+ assert_eq!(merged.stats.records_inserted, 1);+}++#[test]+fn manifests_merge_independently_same_id_never_crosses() {+ // The structural guarantee of the canonical merge: manifests are the partition, so the+ // same row id in two manifests can never interact, whatever the timestamps say.+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![+ items_manifest(PERSONAL, vec![item(PERSONAL, "same-id", "mine", "2024-01-01T00:00:00Z")]),+ items_manifest(SHARED, vec![item(SHARED, "same-id", "theirs-old", "2024-01-01T00:00:00Z")]),+ ],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![+ items_manifest(PERSONAL, vec![item(PERSONAL, "same-id", "mine", "2024-01-01T00:00:00Z")]),+ items_manifest(SHARED, vec![item(SHARED, "same-id", "theirs-new", "2024-01-09T00:00:00Z")]),+ ],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();++ let by_id: HashMap<&str, &CanonicalManifestMerge> = output.manifests.iter().map(|m| (m.manifest_id.as_str(), m)).collect();+ assert_eq!(names_of(&by_id[PERSONAL].manifest), vec!["mine"], "the personal row is untouched by the shared manifest's newer row");+ assert_eq!(names_of(&by_id[SHARED].manifest), vec!["theirs-new"]);+ assert_eq!(by_id[PERSONAL].stats.conflicts, 0);+}++#[test]+fn field_values_match_semantically_and_carry_no_surrogate_id() {+ // A single-value FieldValue arrives in wire shape (no Id, position 0) and matches on its+ // natural key: both sides converge to one row holding the winning value, still id-less.+ let field_value = |value: &str, updated_at: &str| -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(PERSONAL)),+ ("ItemId".to_string(), json!("item-1")),+ ("FieldKey".to_string(), json!("username")),+ ("ValueIndex".to_string(), json!(0)),+ ("Value".to_string(), json!(value)),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect()+ };++ let rows = merge_field_values(vec![field_value("old", "2024-01-01T00:00:00Z")], vec![field_value("new", "2024-01-09T00:00:00Z")]);+ assert_eq!(rows.len(), 1);+ assert!(rows[0].get("Id").is_none(), "a single-value row's id is derived at materialize, never carried on the wire");+ assert_eq!(rows[0]["Value"], json!("new"), "and the row carries the winning value");+}++/// A `login.url` FieldValues row: multi-value, so it owns its id on the wire.+fn url_value(id: &str, index: i64, value: &str, updated_at: &str) -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(PERSONAL)),+ ("Id".to_string(), json!(id)),+ ("ItemId".to_string(), json!("item-1")),+ ("FieldKey".to_string(), json!("login.url")),+ ("ValueIndex".to_string(), json!(index)),+ ("Value".to_string(), json!(value)),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect()+}++fn merge_field_values(server: Vec<CodecRecord>, local: Vec<CodecRecord>) -> Vec<CodecRecord> {+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), server)].into_iter().collect())],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), local)].into_iter().collect())],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();+ output.manifests[0].manifest.tables["FieldValues"].clone()+}++#[test]+fn multi_value_edit_converges_on_the_owned_id() {+ // The same owned id on both sides is the same row edited: LWW, one row out.+ let rows = merge_field_values(+ vec![url_value("url-1", 0, "https://a.example", "2024-01-01T00:00:00Z")],+ vec![url_value("url-1", 0, "https://a-edited.example", "2024-01-09T00:00:00Z")],+ );+ assert_eq!(rows.len(), 1);+ assert_eq!(rows[0]["Id"], json!("url-1"), "the row keeps its owned id");+ assert_eq!(rows[0]["Value"], json!("https://a-edited.example"));+}++#[test]+fn multi_value_concurrent_adds_both_survive() {+ // Two devices each add a 2nd url while offline: both additions land at position 1 under+ // different owned ids. Position is not identity, so the union keeps both, renumbered.+ let rows = merge_field_values(+ vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-05T00:00:00Z")],+ vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-c", 1, "https://c.example", "2024-01-06T00:00:00Z")],+ );+ let mut values: Vec<&str> = rows.iter().map(|r| r["Value"].as_str().unwrap()).collect();+ values.sort();+ assert_eq!(values, vec!["https://a.example", "https://b.example", "https://c.example"]);+ let mut indexes: Vec<i64> = rows.iter().map(|r| r["ValueIndex"].as_i64().unwrap()).collect();+ indexes.sort();+ assert_eq!(indexes, vec![0, 1, 2], "colliding positions are renumbered in the merged output");+}++#[test]+fn multi_value_reorder_does_not_duplicate() {+ // Reordering changes only ValueIndex; the owned id keeps the rows matched, so no row doubles.+ let rows = merge_field_values(+ vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-01T00:00:00Z")],+ vec![url_value("url-b", 0, "https://b.example", "2024-01-09T00:00:00Z"), url_value("url-a", 1, "https://a.example", "2024-01-09T00:00:00Z")],+ );+ assert_eq!(rows.len(), 2);+ let by_id: HashMap<&str, i64> = rows.iter().map(|r| (r["Id"].as_str().unwrap(), r["ValueIndex"].as_i64().unwrap())).collect();+ assert_eq!(by_id["url-b"], 0, "the reorder won");+ assert_eq!(by_id["url-a"], 1);+}++#[test]+fn both_sides_holding_the_same_two_urls_stay_two_rows() {+ // The historic corruption case: before the id-based key, the 2nd url's value replaced the+ // 1st on an ordinary push. Identical sides must merge to exactly themselves.+ let two = vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-02T00:00:00Z")];+ let rows = merge_field_values(two.clone(), two);+ assert_eq!(rows.len(), 2);+ let by_id: HashMap<&str, &str> = rows.iter().map(|r| (r["Id"].as_str().unwrap(), r["Value"].as_str().unwrap())).collect();+ assert_eq!(by_id["url-a"], "https://a.example");+ assert_eq!(by_id["url-b"], "https://b.example");+}++#[test]+fn custom_field_values_of_one_item_do_not_collapse() {+ // Custom fields carry no FieldKey (only a FieldDefinitionId); before FieldDefinitionId joined+ // the key they all collapsed onto one empty-string key.+ let custom = |def: &str, value: &str, updated_at: &str| -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(PERSONAL)),+ ("Id".to_string(), json!(format!("cv-{}", def))),+ ("ItemId".to_string(), json!("item-1")),+ ("FieldKey".to_string(), serde_json::Value::Null),+ ("FieldDefinitionId".to_string(), json!(def)),+ ("Value".to_string(), json!(value)),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect()+ };+ let rows = merge_field_values(+ vec![custom("fd-pin", "1234", "2024-01-01T00:00:00Z"), custom("fd-member", "M-77", "2024-01-01T00:00:00Z")],+ vec![custom("fd-pin", "9999", "2024-01-09T00:00:00Z"), custom("fd-member", "M-77", "2024-01-01T00:00:00Z")],+ );+ assert_eq!(rows.len(), 2, "each custom field keeps its own row");+ let mut values: Vec<&str> = rows.iter().map(|r| r["Value"].as_str().unwrap()).collect();+ values.sort();+ assert_eq!(values, vec!["9999", "M-77"]);+}++#[test]+fn field_histories_union_by_changed_at_and_converge_on_the_same_snapshot() {+ // History rows derive their id from (item, field, ChangedAt): concurrent changes on two+ // devices carry different timestamps and union; the same snapshot on both sides is one row.+ let history = |changed_at: &str, snapshot: &str| -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(PERSONAL)),+ ("ItemId".to_string(), json!("item-1")),+ ("FieldKey".to_string(), json!("login.password")),+ ("ChangedAt".to_string(), json!(changed_at)),+ ("ValueSnapshot".to_string(), json!(snapshot)),+ ("UpdatedAt".to_string(), json!(changed_at)),+ ]+ .into_iter()+ .collect()+ };+ let shared = history("2024-01-01 10:00:00.000", "both-know-this");+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![manifest(PERSONAL, [("FieldHistories".to_string(), vec![shared.clone(), history("2024-02-01 10:00:00.000", "server-only")])].into_iter().collect())],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![manifest(PERSONAL, [("FieldHistories".to_string(), vec![shared, history("2024-02-01 11:30:00.000", "local-only")])].into_iter().collect())],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();+ let rows = &output.manifests[0].manifest.tables["FieldHistories"];+ let mut snapshots: Vec<&str> = rows.iter().map(|r| r["ValueSnapshot"].as_str().unwrap()).collect();+ snapshots.sort();+ assert_eq!(snapshots, vec!["both-know-this", "local-only", "server-only"]);+ assert!(rows.iter().all(|r| r.get("Id").is_none()), "merged history rows stay id-less on the wire");+}++#[test]+fn item_tags_converge_on_their_natural_key() {+ // Two devices tag the same item with the same tag: the id-less join rows are one row.+ let tag_row = |updated_at: &str| -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(PERSONAL)),+ ("ItemId".to_string(), json!("item-1")),+ ("TagId".to_string(), json!("tag-1")),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect()+ };+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![manifest(PERSONAL, [("ItemTags".to_string(), vec![tag_row("2024-01-01T00:00:00Z")])].into_iter().collect())],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![manifest(PERSONAL, [("ItemTags".to_string(), vec![tag_row("2024-01-09T00:00:00Z")])].into_iter().collect())],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();+ let rows = &output.manifests[0].manifest.tables["ItemTags"];+ assert_eq!(rows.len(), 1, "the same (item, tag) link is one row on both sides");+ assert!(rows[0].get("Id").is_none());+}++#[test]+fn bucketed_tables_merge_and_come_back_as_buckets() {+ let setting = |value: &str, updated_at: &str| -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(PERSONAL)),+ ("Key".to_string(), json!("theme")),+ ("Value".to_string(), json!(value)),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect()+ };++ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![items_manifest(PERSONAL, vec![])],+ server_buckets: vec![DataBucket::new(PERSONAL, "Settings", [("Settings".to_string(), vec![setting("light", "2024-01-01T00:00:00Z")])].into_iter().collect())],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![items_manifest(PERSONAL, vec![])],+ local_buckets: vec![DataBucket::new(PERSONAL, "Settings", [("Settings".to_string(), vec![setting("dark", "2024-01-09T00:00:00Z")])].into_iter().collect())],+ schema_columns: schema(),+ })+ .unwrap();++ let merged = &output.manifests[0];+ assert!(!merged.manifest.tables.contains_key("Settings"), "bucketed tables never surface in manifest.tables");+ assert_eq!(merged.buckets.len(), 1);+ assert_eq!(merged.buckets[0].category, "Settings");+ assert_eq!(merged.buckets[0].tables["Settings"][0]["Value"], json!("dark"), "the newer local setting wins inside the bucket");+ assert!(crate::vault_codec::validate_data_bucket(&merged.buckets[0]).ok);+}++#[test]+fn an_empty_table_the_server_carried_stays_in_the_merged_manifest() {+ /*+ * The merged manifest is pushed as-is and compared against a baseline canonicalize produced, and+ * canonicalize keeps an empty table. Dropping it here rewrites the whole manifest on every merge.+ */+ let merged = merge_single(vec![], vec![]);+ assert_eq!(merged.manifest.tables.get("Items").map(Vec::len), Some(0), "the server's empty table keeps its place in the merged manifest");++ // The empty tables of a bucket the server served come back as that bucket, not as a missing one.+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![items_manifest(PERSONAL, vec![])],+ server_buckets: vec![DataBucket::new(PERSONAL, "Settings", [("Settings".to_string(), vec![])].into_iter().collect())],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![items_manifest(PERSONAL, vec![])],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();+ let buckets = &output.manifests[0].buckets;+ assert_eq!(buckets.len(), 1, "the served bucket is still emitted when it holds no rows");+ assert_eq!(buckets[0].tables.get("Settings").map(Vec::len), Some(0));+}++#[test]+fn unknown_columns_keep_the_server_value_when_local_wins() {+ // A newer client added Items.NewColumn; this client's schema does not know it. The local+ // row winning the LWW must not regress that column.+ let mut base = item(PERSONAL, "row", "server-old", "2024-01-01T00:00:00Z");+ base.insert("NewColumn".to_string(), json!("newer-client-value"));+ let mut incoming = item(PERSONAL, "row", "local-new", "2024-01-09T00:00:00Z");+ incoming.insert("NewColumn".to_string(), json!("stale-carried-value"));++ let merged = merge_single(vec![base], vec![incoming]);++ let row = &merged.manifest.tables["Items"][0];+ assert_eq!(row["Name"], json!("local-new"), "known columns follow the winner");+ assert_eq!(row["NewColumn"], json!("newer-client-value"), "unknown columns keep the base value");+}++#[test]+fn unknown_tables_are_taken_from_the_server_wholesale() {+ let future_row = |payload: &str| -> CodecRecord {+ [("ManifestId".to_string(), json!(PERSONAL)), ("Id".to_string(), json!("f-1")), ("Payload".to_string(), json!(payload))].into_iter().collect()+ };+ let with_future = |payload: Option<&str>| -> Manifest {+ let mut tables: HashMap<String, Vec<CodecRecord>> = [("Items".to_string(), vec![])].into_iter().collect();+ if let Some(p) = payload {+ tables.insert("FutureTable".to_string(), vec![future_row(p)]);+ }+ manifest(PERSONAL, tables)+ };++ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![with_future(Some("server-version"))],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![with_future(Some("local-carried-version"))],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();+ assert_eq!(output.manifests[0].manifest.tables["FutureTable"][0]["Payload"], json!("server-version"));++ // And a local-only unknown table is dropped, matching the discarded local carrier of today.+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![with_future(None)],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![with_future(Some("local-only"))],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();+ assert!(!output.manifests[0].manifest.tables.contains_key("FutureTable"));+}++#[test]+fn local_manifest_no_longer_served_is_dropped_and_reported() {+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![items_manifest(PERSONAL, vec![])],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![items_manifest(PERSONAL, vec![]), items_manifest(SHARED, vec![item(SHARED, "x", "gone", "2024-01-01T00:00:00Z")])],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();++ assert_eq!(output.manifests.len(), 1);+ assert_eq!(output.dropped_local_manifest_ids, vec![SHARED.to_string()]);+}++#[test]+fn contentless_server_manifest_passes_the_local_one_through() {+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![items_manifest(PERSONAL, vec![])],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![SHARED.to_string()],+ local_manifests: vec![items_manifest(PERSONAL, vec![]), items_manifest(SHARED, vec![item(SHARED, "x", "kept-offline", "2024-01-01T00:00:00Z")])],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();++ assert!(output.dropped_local_manifest_ids.is_empty());+ let shared = output.manifests.iter().find(|m| m.manifest_id == SHARED).unwrap();+ assert_eq!(names_of(&shared.manifest), vec!["kept-offline"]);+ assert_eq!(shared.stats.records_inserted, 1);+}++#[test]+fn blob_refs_ride_through_as_opaque_values() {+ let attachment = |blob_ref: &str, updated_at: &str| -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(PERSONAL)),+ ("Id".to_string(), json!("att-1")),+ ("Blob".to_string(), json!({ "__blobRef": blob_ref, "__blobKind": "attachment" })),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect()+ };++ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![manifest(PERSONAL, [("Attachments".to_string(), vec![attachment("hash-old", "2024-01-01T00:00:00Z")])].into_iter().collect())],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![manifest(PERSONAL, [("Attachments".to_string(), vec![attachment("hash-new", "2024-01-09T00:00:00Z")])].into_iter().collect())],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();++ assert_eq!(output.manifests[0].manifest.tables["Attachments"][0]["Blob"], json!({ "__blobRef": "hash-new", "__blobKind": "attachment" }));+}++#[test]+fn merged_manifest_keeps_server_metadata_and_validates() {+ let mut server = items_manifest(PERSONAL, vec![item(PERSONAL, "a", "server", "2024-01-01T00:00:00Z")]);+ server.name = Some("Family".to_string());+ let local = items_manifest(PERSONAL, vec![item(PERSONAL, "b", "local", "2024-01-02T00:00:00Z")]);++ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![server],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![local],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();++ let merged = &output.manifests[0].manifest;+ assert_eq!(merged.manifest_salt, SALT, "salt comes from the server manifest");+ assert_eq!(merged.name.as_deref(), Some("Family"), "name comes from the server manifest");+ let validation = crate::vault_codec::validate_manifest(merged);+ assert!(validation.ok, "merged manifest must validate: {:?}", validation.failed_rules);+}++#[test]+fn manifest_pairing_is_case_insensitive() {+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![items_manifest("M-PERSONAL", vec![item("M-PERSONAL", "a", "server", "2024-01-01T00:00:00Z")])],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![items_manifest("m-personal", vec![item("m-personal", "b", "local", "2024-01-02T00:00:00Z")])],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();++ assert!(output.dropped_local_manifest_ids.is_empty(), "differently-cased ids are the same manifest");+ assert_eq!(output.manifests[0].manifest.tables["Items"].len(), 2);+}++#[test]+fn unknown_category_bucket_tables_ride_through_from_the_server() {+ // A newer writer added a table inside the Settings bucket and a whole new category. The+ // known Settings rows still merge; the unknown tables ride through from the server only.+ let unknown_row: CodecRecord = [("ManifestId".to_string(), json!(PERSONAL)), ("Id".to_string(), json!("u-1"))].into_iter().collect();+ let server_settings = DataBucket::new(+ PERSONAL,+ "Settings",+ [+ ("Settings".to_string(), vec![[("ManifestId".to_string(), json!(PERSONAL)), ("Key".to_string(), json!("theme")), ("Value".to_string(), json!("light")), ("UpdatedAt".to_string(), json!("2024-01-01T00:00:00Z"))].into_iter().collect()]),+ ("SettingsExtras".to_string(), vec![unknown_row.clone()]),+ ]+ .into_iter()+ .collect(),+ );+ let local_future = DataBucket::new(PERSONAL, "FutureCategory", [("FutureRows".to_string(), vec![unknown_row])].into_iter().collect());++ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![items_manifest(PERSONAL, vec![])],+ server_buckets: vec![server_settings],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![items_manifest(PERSONAL, vec![])],+ local_buckets: vec![local_future],+ schema_columns: schema(),+ })+ .unwrap();++ let buckets = &output.manifests[0].buckets;+ let categories: Vec<&str> = buckets.iter().map(|b| b.category.as_str()).collect();+ assert!(categories.contains(&"Settings"), "the known Settings rows still come back as a bucket");+ assert!(buckets.iter().any(|b| b.category == "Settings" && b.tables.contains_key("SettingsExtras")), "the server's unknown bucket table rides through under its category");+ assert!(!categories.contains(&"FutureCategory"), "a local-only unknown bucket is dropped with the rest of the local carrier");+}++#[test]+fn keys_compare_guids_case_insensitively_and_survive_a_missing_column() {+ const MANIFEST: &str = "1dd1a3fd-8e0e-4b3f-9a3a-0f1a2b3c4d5e";+ const OTHER_MANIFEST: &str = "2ee2b4fe-9f1f-4c40-8b4b-1a2b3c4d5e6f";+ const ROW: &str = "3ff3c50f-a020-4d51-9c5c-2b3c4d5e6f70";+ let row = |manifest: &str, id: &str| -> CodecRecord {+ HashMap::from([("ManifestId".to_string(), json!(manifest)), ("Id".to_string(), json!(id))])+ };+ let columns = ["ManifestId", "Id"];++ // Two writers spelling the same ids in different cases address one row (see `identity_part`).+ assert_eq!(get_key(&row(MANIFEST, ROW), &columns), get_key(&row(&MANIFEST.to_uppercase(), &ROW.to_uppercase()), &columns));+ assert_ne!(get_key(&row(MANIFEST, ROW), &columns), get_key(&row(OTHER_MANIFEST, ROW), &columns));+ assert_eq!(get_key(&HashMap::from([("Id".to_string(), json!(ROW))]), &columns), format!(":{}", ROW));+}diff --git a/core/rust/src/vault_pruner/mod.rs b/core/rust/src/vault_pruner/mod.rsindex 443bd9a75..b7378c769 100644--- a/core/rust/src/vault_pruner/mod.rs+++ b/core/rust/src/vault_pruner/mod.rs@@ -10,31 +10,20 @@ use serde::{Deserialize, Serialize};use std::collections::{HashMap, HashSet};use crate::error::{VaultError, VaultResult};-use crate::vault_codec::row::{is_deleted, str_col};-use crate::vault_merge::SqlStatement;+use crate::sqlite_host::SqlStatement;+use crate::vault_codec::row::{has_bytes, is_deleted, logo_kind, str_col};+use crate::vault_codec::{CodecRecord, CodecTableData};use crate::vault_model::names::{DELETED_AT_COL, FILE_DATA_COL, ID_COL, IS_DELETED_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_STATS_TABLE, KIND_COL, LOGOS_TABLE,LOGO_ID_COL, LOGO_KIND_FAVICON, UPDATED_AT_COL,};-use crate::vault_model::{BLOB_COLUMNS, SYNCABLE_TABLES};--/// A record is a map of column names to JSON values.-pub type Record = crate::vault_codec::CodecRecord;--/// Data for a single table.-#[derive(Debug, Clone, Serialize, Deserialize)]-pub struct TableData {- /// Table name- pub name: String,- /// All records in this table- pub records: Vec<Record>,-}+use crate::vault_model::{BLOB_COLUMNS, SYNCABLE_TABLES, TRASH_RETENTION_DEFAULT_DAYS};/// Input for the prune operation.#[derive(Debug, Clone, Serialize, Deserialize)]pub struct PruneInput {/// Tables from the local database (at minimum, Items table is required)- pub tables: Vec<TableData>,+ pub tables: Vec<CodecTableData>,/// Current time in ISO 8601 UTC format: `YYYY-MM-DDTHH:MM:SS.sssZ`./** Callers: JavaScript `new Date().toISOString()`, C# `DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ")`,@@ -47,7 +36,7 @@ pub struct PruneInput {}fn default_retention_days() -> u32 {- crate::vault_model::TRASH_RETENTION_DEFAULT_DAYS+ TRASH_RETENTION_DEFAULT_DAYS}/// Statistics about what was pruned.@@ -136,7 +125,7 @@ fn blob_clear_fragment(table_name: &str) -> String {}/// The records of the named table, `None` when the input does not carry it.-fn records_of<'a>(tables: &'a [TableData], name: &str) -> Option<&'a [Record]> {+fn records_of<'a>(tables: &'a [CodecTableData], name: &str) -> Option<&'a [CodecRecord]> {tables.iter().find(|t| t.name == name).map(|t| t.records.as_slice())}@@ -176,7 +165,7 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {}/// Pass 1: the ids of live items whose trash date (`DeletedAt`) lies before the cutoff.-fn expired_item_ids(items: &[Record], cutoff_date: DateTime<Utc>) -> Vec<String> {+fn expired_item_ids(items: &[CodecRecord], cutoff_date: DateTime<Utc>) -> Vec<String> {let mut expired = Vec::new();for item in items.iter().filter(|item| !is_deleted(item)) {let Some(deleted_at) = str_col(item, DELETED_AT_COL) else { continue };@@ -196,7 +185,7 @@ fn expired_item_ids(items: &[Record], cutoff_date: DateTime<Utc>) -> Vec<String>* extracted blob column has its bytes dropped in the same statement, leaving the column non-null* while reclaiming the storage on the next save.*/-fn tombstone_item_children(tables: &[TableData], item_id: &str, now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {+fn tombstone_item_children(tables: &[CodecTableData], item_id: &str, now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {for child in item_child_tables() {let Some(records) = records_of(tables, child.name) else { continue };let match_col = item_ref_column(child.name);@@ -228,8 +217,8 @@ fn tombstone_item_children(tables: &[TableData], item_id: &str, now_str: &str, s* then reclaims the bytes.*/fn sweep_orphan_favicons(- tables: &[TableData],- items: &[Record],+ tables: &[CodecTableData],+ items: &[CodecRecord],expired: &[String],now_str: &str,statements: &mut Vec<SqlStatement>,@@ -246,8 +235,7 @@ fn sweep_orphan_favicons(.collect();for logo in logos.iter().filter(|logo| !is_deleted(logo)) {- let kind = str_col(logo, KIND_COL).unwrap_or(LOGO_KIND_FAVICON);- if !kind.trim().is_empty() && !kind.eq_ignore_ascii_case(LOGO_KIND_FAVICON) {+ if logo_kind(logo) != LOGO_KIND_FAVICON {continue;}let Some(logo_id) = str_col(logo, ID_COL) else { continue };@@ -269,10 +257,10 @@ fn sweep_orphan_favicons(* user deleting one from their logo library tombstones the row and this pass reclaims the bytes.* Rows tombstoned by Pass 1 or Pass 2 in this same call are already cleared there.*/-fn clear_tombstoned_blobs(tables: &[TableData], now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {+fn clear_tombstoned_blobs(tables: &[CodecTableData], now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {for (table, blob_col, _) in BLOB_COLUMNS {let Some(records) = records_of(tables, table) else { continue };- let stale = records.iter().filter(|row| is_deleted(row) && value_has_bytes(row.get(*blob_col)));+ let stale = records.iter().filter(|row| is_deleted(row) && has_bytes(row.get(*blob_col)));for row in stale {let Some(id) = str_col(row, ID_COL) else { continue };statements.push(SqlStatement {@@ -284,20 +272,8 @@ fn clear_tombstoned_blobs(tables: &[TableData], now_str: &str, statements: &mut}}-/// True if a JSON value represents non-empty blob bytes.-fn value_has_bytes(value: Option<&serde_json::Value>) -> bool {- match value {- None => false,- Some(serde_json::Value::Null) => false,- Some(serde_json::Value::String(s)) => !s.is_empty(),- Some(serde_json::Value::Array(a)) => !a.is_empty(),- Some(serde_json::Value::Object(o)) => !o.is_empty(),- Some(_) => true,- }-}-/// Count related records that match a foreign key value and are not already deleted.-fn count_related_records(records: &[Record], fk_column: &str, fk_value: &str) -> u32 {+fn count_related_records(records: &[CodecRecord], fk_column: &str, fk_value: &str) -> u32 {records.iter().filter(|r| str_col(r, fk_column) == Some(fk_value) && !is_deleted(r)).count() as u32}diff --git a/core/rust/src/vault_pruner/tests.rs b/core/rust/src/vault_pruner/tests.rsindex cb8c97484..d9e393af5 100644--- a/core/rust/src/vault_pruner/tests.rs+++ b/core/rust/src/vault_pruner/tests.rs@@ -1,5 +1,5 @@use super::*;-use chrono::Utc;+use crate::vault_codec::test_support::{days_ago_iso, now_iso};use crate::vault_model::names::{ATTACHMENTS_TABLE, FIELD_HISTORIES_TABLE, FIELD_VALUES_TABLE, ITEM_TAGS_TABLE};/// The counter of one table in a per-table stats map, 0 when the table was not touched.@@ -7,7 +7,7 @@ fn count(map: &HashMap<String, u32>, table: &str) -> u32 {map.get(table).copied().unwrap_or(0)}-fn make_item_record(id: &str, deleted_at: Option<&str>, is_deleted: bool) -> Record {+fn make_item_record(id: &str, deleted_at: Option<&str>, is_deleted: bool) -> CodecRecord {let mut record = HashMap::new();record.insert("Id".to_string(), serde_json::json!(id));record.insert("UpdatedAt".to_string(), serde_json::json!("2024-01-01T00:00:00Z"));@@ -20,7 +20,7 @@ fn make_item_record(id: &str, deleted_at: Option<&str>, is_deleted: bool) -> Recrecord}-fn make_field_value_record(id: &str, item_id: &str, is_deleted: bool) -> Record {+fn make_field_value_record(id: &str, item_id: &str, is_deleted: bool) -> CodecRecord {let mut record = HashMap::new();record.insert("Id".to_string(), serde_json::json!(id));record.insert("ItemId".to_string(), serde_json::json!(item_id));@@ -34,7 +34,7 @@ fn make_attachment_record(item_id: &str,is_deleted: bool,blob: serde_json::Value,-) -> Record {+) -> CodecRecord {let mut record = HashMap::new();record.insert("Id".to_string(), serde_json::json!(id));record.insert("ItemId".to_string(), serde_json::json!(item_id));@@ -49,7 +49,7 @@ fn make_item_with_logo(logo_id: Option<&str>,deleted_at: Option<&str>,is_deleted: bool,-) -> Record {+) -> CodecRecord {let mut record = make_item_record(id, deleted_at, is_deleted);match logo_id {Some(lid) => record.insert("LogoId".to_string(), serde_json::json!(lid)),@@ -58,7 +58,7 @@ fn make_item_with_logo(record}-fn make_logo_record(id: &str, is_deleted: bool) -> Record {+fn make_logo_record(id: &str, is_deleted: bool) -> CodecRecord {let mut record = HashMap::new();record.insert("Id".to_string(), serde_json::json!(id));record.insert("Source".to_string(), serde_json::json!("example.com"));@@ -67,7 +67,7 @@ fn make_logo_record(id: &str, is_deleted: bool) -> Record {record}-fn make_logo_record_with_blob(id: &str, is_deleted: bool, blob: serde_json::Value) -> Record {+fn make_logo_record_with_blob(id: &str, is_deleted: bool, blob: serde_json::Value) -> CodecRecord {let mut record = make_logo_record(id, is_deleted);record.insert("FileData".to_string(), blob);record@@ -75,18 +75,17 @@ fn make_logo_record_with_blob(id: &str, is_deleted: bool, blob: serde_json::Valu#[test]fn test_prune_expired_items() {- let now = Utc::now();- let now_str = now.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();// Create an item deleted 60 days ago- let old_date = (now - Duration::days(60)).format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let old_date = days_ago_iso(60);let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![make_item_record("item-1", Some(&old_date), false)],},- TableData {+ CodecTableData {name: "FieldValues".to_string(),records: vec![make_field_value_record("fv-1", "item-1", false)],},@@ -105,14 +104,13 @@ fn test_prune_expired_items() {#[test]fn test_no_prune_recent_items() {- let now = Utc::now();- let now_str = now.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();// Create an item deleted 10 days ago (within retention)- let recent_date = (now - Duration::days(10)).format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let recent_date = days_ago_iso(10);let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![make_item_record("item-1", Some(&recent_date), false)],},@@ -130,11 +128,11 @@ fn test_no_prune_recent_items() {#[test]fn test_no_prune_active_items() {- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();// Create an item that's not in trash (DeletedAt is null)let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![make_item_record("item-1", None, false)],},@@ -152,14 +150,13 @@ fn test_no_prune_active_items() {#[test]fn test_no_prune_already_deleted() {- let now = Utc::now();- let now_str = now.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();// Create an item that's already permanently deleted- let old_date = (now - Duration::days(60)).format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let old_date = days_ago_iso(60);let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![make_item_record("item-1", Some(&old_date), true)],},@@ -177,9 +174,8 @@ fn test_no_prune_already_deleted() {#[test]fn test_prune_json_api() {- let now = Utc::now();- let now_str = now.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();- let old_date = (now - Duration::days(60)).format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();+ let old_date = days_ago_iso(60);let input_json = format!(r#"{{"tables": [{{@@ -210,14 +206,14 @@ fn logo_update_count(output: &PruneOutput) -> usize {#[test]fn test_orphan_logo_with_no_referencing_items_is_pruned() {- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![],},- TableData {+ CodecTableData {name: "Logos".to_string(),records: vec![make_logo_record("logo-orphan", false)],},@@ -235,14 +231,14 @@ fn test_orphan_logo_with_no_referencing_items_is_pruned() {#[test]fn test_logo_referenced_by_active_item_is_kept() {- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![make_item_with_logo("item-1", Some("logo-1"), None, false)],},- TableData {+ CodecTableData {name: "Logos".to_string(),records: vec![make_logo_record("logo-1", false)],},@@ -260,16 +256,16 @@ fn test_logo_referenced_by_active_item_is_kept() {#[test]fn test_logo_referenced_only_by_tombstoned_item_is_pruned() {- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();// The tombstoned item (IsDeleted=1) still has LogoId set; the logo// should be considered orphan since no active item references it.let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![make_item_with_logo("item-1", Some("logo-1"), None, true)],},- TableData {+ CodecTableData {name: "Logos".to_string(),records: vec![make_logo_record("logo-1", false)],},@@ -285,18 +281,17 @@ fn test_logo_referenced_only_by_tombstoned_item_is_pruned() {#[test]fn test_logo_referenced_only_by_item_being_purged_is_pruned() {- let now = Utc::now();- let now_str = now.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();// Item is in trash older than retention, so Pass 1 will tombstone it,// Pass 2 should reclaim its logo in the same call.- let old_date = (now - Duration::days(60)).format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let old_date = days_ago_iso(60);let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![make_item_with_logo("item-1", Some("logo-1"), Some(&old_date), false)],},- TableData {+ CodecTableData {name: "Logos".to_string(),records: vec![make_logo_record("logo-1", false)],},@@ -313,18 +308,17 @@ fn test_logo_referenced_only_by_item_being_purged_is_pruned() {#[test]fn test_logo_referenced_by_item_in_recent_trash_is_kept() {- let now = Utc::now();- let now_str = now.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();// Item is in trash but within retention, so it could still be restored,// so its logo must be preserved.- let recent_date = (now - Duration::days(10)).format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let recent_date = days_ago_iso(10);let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![make_item_with_logo("item-1", Some("logo-1"), Some(&recent_date), false)],},- TableData {+ CodecTableData {name: "Logos".to_string(),records: vec![make_logo_record("logo-1", false)],},@@ -343,14 +337,14 @@ fn test_logo_referenced_by_item_in_recent_trash_is_kept() {fn test_orphan_logo_pruning_emits_filedata_clear_in_same_statement() {// Pass 2 must clear FileData when it tombstones a logo, otherwise the// encrypted vault keeps the blob bytes even after the row is "deleted".- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![],},- TableData {+ CodecTableData {name: "Logos".to_string(),records: vec![make_logo_record_with_blob("logo-orphan", false, serde_json::json!("aGVsbG8="))],},@@ -369,14 +363,14 @@ fn test_orphan_logo_pruning_emits_filedata_clear_in_same_statement() {fn test_tombstoned_logo_with_blob_bytes_is_swept() {// Pass 3 must catch historical logos that are IsDeleted=1 but still carry FileData// (e.g. tombstoned by an older client that did not drop FileData).- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![],},- TableData {+ CodecTableData {name: "Logos".to_string(),records: vec![make_logo_record_with_blob("logo-tombstoned", true, serde_json::json!("aGVsbG8="))],},@@ -394,14 +388,14 @@ fn test_tombstoned_logo_with_blob_bytes_is_swept() {#[test]fn test_tombstoned_logo_without_blob_is_not_touched() {// Logos already cleared shouldn't generate redundant updates.- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![],},- TableData {+ CodecTableData {name: "Logos".to_string(),records: vec![make_logo_record_with_blob("logo-tombstoned", true, serde_json::Value::Null)],},@@ -417,14 +411,14 @@ fn test_tombstoned_logo_without_blob_is_not_touched() {#[test]fn test_already_soft_deleted_logo_is_not_re_pruned() {- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![],},- TableData {+ CodecTableData {name: "Logos".to_string(),records: vec![make_logo_record("logo-1", true)],},@@ -440,10 +434,10 @@ fn test_already_soft_deleted_logo_is_not_re_pruned() {#[test]fn test_logo_pruning_skipped_when_logos_table_absent() {- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![make_item_with_logo("item-1", Some("logo-1"), None, false)],},@@ -460,17 +454,16 @@ fn test_logo_pruning_skipped_when_logos_table_absent() {#[test]fn test_trash_purge_clears_attachment_blobs() {- let now = Utc::now();- let now_str = now.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();- let old_date = (now - Duration::days(60)).format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();+ let old_date = days_ago_iso(60);let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![make_item_record("item-1", Some(&old_date), false)],},- TableData {+ CodecTableData {name: "Attachments".to_string(),records: vec![make_attachment_record("att-1",@@ -500,14 +493,14 @@ fn test_trash_purge_clears_attachment_blobs() {#[test]fn test_sweeper_clears_blob_on_already_tombstoned_attachment() {- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![],},- TableData {+ CodecTableData {name: "Attachments".to_string(),records: vec![make_attachment_record("att-old",@@ -534,14 +527,14 @@ fn test_sweeper_clears_blob_on_already_tombstoned_attachment() {#[test]fn test_sweeper_skips_already_empty_blob() {- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![],},- TableData {+ CodecTableData {name: "Attachments".to_string(),records: vec![// Empty string (already cleared): should be skipped.@@ -566,18 +559,18 @@ fn test_sweeper_skips_already_empty_blob() {#[test]fn test_sweeper_skips_empty_uint8array_object_blob() {// An already-cleared blob must not generate a clear statement on every prune.- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![],},- TableData {+ CodecTableData {name: "Attachments".to_string(),records: vec![make_attachment_record("att-empty-object", "item-1", true, serde_json::json!({}))],},- TableData {+ CodecTableData {name: "Logos".to_string(),records: vec![make_logo_record_with_blob("logo-empty-object", true, serde_json::json!({}))],},@@ -596,14 +589,14 @@ fn test_sweeper_skips_empty_uint8array_object_blob() {#[test]fn test_sweeper_clears_nonempty_uint8array_object_blob() {// Non-empty blobs are serialized as {"0":104,...}.- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![],},- TableData {+ CodecTableData {name: "Attachments".to_string(),records: vec![make_attachment_record("att-object", "item-1", true, serde_json::json!({"0": 104, "1": 105}))],},@@ -619,14 +612,14 @@ fn test_sweeper_clears_nonempty_uint8array_object_blob() {#[test]fn test_sweeper_skips_active_attachment_with_blob() {- let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();let input = PruneInput {tables: vec![- TableData {+ CodecTableData {name: "Items".to_string(),records: vec![],},- TableData {+ CodecTableData {name: "Attachments".to_string(),records: vec![make_attachment_record("att-active",@@ -658,9 +651,8 @@ fn prune_queries_cover_every_item_child_table() {#[test]fn test_prune_cascades_to_field_histories_and_item_tags() {- let now = Utc::now();- let now_str = now.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();- let old_date = (now - Duration::days(60)).format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();+ let now_str = now_iso();+ let old_date = days_ago_iso(60);let mut history = HashMap::new();history.insert("ItemId".to_string(), serde_json::json!("item-1"));@@ -671,9 +663,9 @@ fn test_prune_cascades_to_field_histories_and_item_tags() {let input = PruneInput {tables: vec![- TableData { name: "Items".to_string(), records: vec![make_item_record("item-1", Some(&old_date), false)] },- TableData { name: "FieldHistories".to_string(), records: vec![history] },- TableData { name: "ItemTags".to_string(), records: vec![item_tag] },+ CodecTableData { name: "Items".to_string(), records: vec![make_item_record("item-1", Some(&old_date), false)] },+ CodecTableData { name: "FieldHistories".to_string(), records: vec![history] },+ CodecTableData { name: "ItemTags".to_string(), records: vec![item_tag] },],retention_days: 30,current_time: now_str,diff --git a/core/rust/src/vault_sync/db.rs b/core/rust/src/vault_sync/db.rsindex 35604f839..018263b76 100644--- a/core/rust/src/vault_sync/db.rs+++ b/core/rust/src/vault_sync/db.rs@@ -6,19 +6,20 @@ use serde_json::{json, Map, Value};use super::errors::{SyncError, SyncResult};use super::session::Host;-use super::types::{Ack, Command, Db, DbBytes, DbRows, LogLevel, SqlStatement};+use super::types::{Ack, Command, Db, DbBytes, DbRows, LogLevel};use super::legacy;-use crate::encoding::{base64_decode, base64_encode};+use crate::encoding::{base64_decode, base64_encode, uuid_from_bytes};+use crate::sqlite_host::SqlStatement;+use crate::timestamp::{now_iso_utc, now_vault_datetime};+use crate::vault_codec::row::{blob_ref_of, inline_bytes};use crate::vault_codec::{CodecRecord, CodecTableData, MaterializedTables};+use crate::vault_model::{id_key, ids_equal, UNSTAMPED_SCOPE_SENTINEL};pub(crate) type Row = Map<String, Value>;/// Rows per `dbExec` batch when bulk-loading a materialized vault.const INSERT_BATCH_ROWS: usize = 400;-/// The id a row carries while it belongs to no manifest yet.-pub const UNSTAMPED_MANIFEST_ID: &str = "00000000-0000-0000-0000-000000000000";-pub(crate) async fn query(host: &Host, db: Db, sql: &str, params: Vec<Value>) -> SyncResult<Vec<Row>> {let response: DbRows = host.call(Command::DbQuery { db, sql: sql.to_string(), params }).await?;Ok(response.rows)@@ -48,10 +49,6 @@ pub(crate) async fn export(host: &Host, db: Db) -> SyncResult<Vec<u8>> {base64_decode(&response.bytes).map_err(|_| SyncError::Other("host returned invalid database bytes".to_string()))}-pub(crate) fn b64_param(bytes: &[u8]) -> Value {- json!({ "__b64": base64_encode(bytes) })-}-/// A cell as text: strings as they are, null as empty, anything else in its JSON form.pub(crate) fn value_string(value: &Value) -> String {match value {@@ -164,7 +161,7 @@ pub(crate) async fn manifest_ids_in_vault(host: &Host, db: Db) -> SyncResult<Vecfor table in stamped_tables(host, db).await? {let rows = query(host, db, &format!("SELECT DISTINCT ManifestId FROM \"{}\" WHERE ManifestId IS NOT NULL AND ManifestId != ''", table), vec![]).await?;for id in rows.iter().filter_map(|row| row.get("ManifestId").and_then(Value::as_str)) {- if !id.eq_ignore_ascii_case(UNSTAMPED_MANIFEST_ID) && !ids.iter().any(|known| known == id) {+ if !ids_equal(id, UNSTAMPED_SCOPE_SENTINEL) && !ids.iter().any(|known| known == id) {ids.push(id.to_string());}}@@ -216,45 +213,33 @@ pub(crate) async fn insert_materialized(host: &Host, materialized: &Materialized/// A materialized cell as a bind parameter: blob markers become bytes (NULL when the bytes are missing),/// inline `{ __b64 }` payloads bind as bytes, everything else binds as is.fn bind_value(value: &Value, blobs: &HashMap<String, Vec<u8>>) -> Value {- if let Some(reference) = value.get("__blobRef").and_then(Value::as_str) {- return blobs.get(reference).map(|bytes| b64_param(bytes)).unwrap_or(Value::Null);+ if let Some((reference, _)) = blob_ref_of(value) {+ return blobs.get(reference).map(|bytes| inline_bytes(bytes)).unwrap_or(Value::Null);}value.clone()}-/// The current timestamp in the vault's row format (`yyyy-MM-dd HH:mm:ss.SSS`, UTC).-pub(crate) fn now() -> String {- crate::timestamp::now_vault_datetime()-}--/// The current instant as ISO-8601 UTC with milliseconds.-pub(crate) fn now_iso() -> String {- crate::timestamp::now_iso_utc()-}-/// A random lowercase UUID v4.pub(crate) fn new_id() -> String {let mut bytes = [0u8; 16];crate::rng::fill_random(&mut bytes);- bytes[6] = (bytes[6] & 0x0f) | 0x40;- bytes[8] = (bytes[8] & 0x3f) | 0x80;- crate::encoding::format_uuid(&bytes)-}--pub(crate) const GET_ACTIVE_KEY_FOR_MANIFEST: &str = "SELECT x.Id, x.PublicKey, x.PrivateKey, x.IsPrimary FROM EncryptionKeys x WHERE x.ManifestId = ? AND x.IsPrimary = 1 AND x.IsDeleted = 0 LIMIT 1";-pub(crate) const GET_ACCOUNT_KEY_BY_PUBLIC_KEY: &str = "SELECT x.PublicKey, x.PrivateKey, x.IsPrimary FROM EncryptionKeys x WHERE x.ManifestId = ? AND x.PublicKey = ? AND x.IsDeleted = 0 LIMIT 1";-pub(crate) const DEMOTE_KEYS_FOR_MANIFEST: &str = "UPDATE EncryptionKeys SET IsPrimary = 0, UpdatedAt = ? WHERE ManifestId = ? AND IsPrimary = 1";-pub(crate) const INSERT_KEY_FOR_MANIFEST: &str = "INSERT INTO EncryptionKeys (Id, ManifestId, PublicKey, PrivateKey, IsPrimary, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, 1, ?, ?, 0)";-pub(crate) const INSERT_FOLDER: &str = "INSERT INTO Folders (Id, Name, ParentFolderId, ManifestId, Weight, IsDeleted, CreatedAt, UpdatedAt) VALUES (?, ?, ?, COALESCE((SELECT ManifestId FROM Folders WHERE Id = ?), ?), 0, 0, ?, ?)";-pub(crate) const RESTAMP_SUBTREE_FOLDERS: &str = "UPDATE Folders SET ManifestId = ? WHERE Id IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";-pub(crate) const RESTAMP_SUBTREE_ITEMS: &str = "UPDATE Items SET ManifestId = ? WHERE FolderId IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";-pub(crate) const FIND_ITEMS_WITH_FOREIGN_LOGO: &str = "SELECT i.Id, i.ManifestId, origin.Kind, origin.Source FROM Items i INNER JOIN Logos origin ON origin.Id = i.LogoId LEFT JOIN Logos own ON own.Id = i.LogoId AND own.ManifestId = i.ManifestId WHERE i.LogoId IS NOT NULL AND own.Id IS NULL AND i.IsDeleted = 0";-pub(crate) const GET_LOGO_ID_FOR_KEY: &str = "SELECT Id FROM Logos WHERE ManifestId = ? AND Kind = ? AND Source = ? AND IsDeleted = 0 LIMIT 1";-pub(crate) const GET_BEST_LOGO_FOR_KEY: &str = "SELECT FileData, MimeType, Name FROM Logos WHERE Kind = ? AND Source = ? AND IsDeleted = 0 ORDER BY (FileData IS NOT NULL AND LENGTH(FileData) > 0) DESC, UpdatedAt DESC LIMIT 1";-pub(crate) const UPSERT_LOGO: &str = "INSERT INTO Logos (Id, Kind, Source, ManifestId, FileData, MimeType, Name, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0) ON CONFLICT(ManifestId, Id) DO UPDATE SET FileData = excluded.FileData, MimeType = excluded.MimeType, Name = COALESCE(excluded.Name, Logos.Name), UpdatedAt = excluded.UpdatedAt, IsDeleted = 0";-pub(crate) const REPOINT_ITEM_LOGO: &str = "UPDATE Items SET LogoId = ? WHERE Id = ? AND ManifestId = ?";-pub(crate) const RENDERED_MANIFEST_FOLDER: &str = "SELECT Id FROM Folders WHERE ManifestId = ? AND IsDeleted = 0 AND ParentFolderId IS NULL";-pub(crate) const MANIFEST_ROOT_FOLDER_NAMES: &str = "SELECT ManifestId, Name FROM Folders WHERE IsDeleted = 0 AND ManifestId IS NOT NULL AND UPPER(Id) = UPPER(ManifestId)";+ uuid_from_bytes(bytes, 4)+}++const GET_ACTIVE_KEY_FOR_MANIFEST: &str = "SELECT x.Id, x.PublicKey, x.PrivateKey, x.IsPrimary FROM EncryptionKeys x WHERE x.ManifestId = ? AND x.IsPrimary = 1 AND x.IsDeleted = 0 LIMIT 1";+const GET_ACCOUNT_KEY_BY_PUBLIC_KEY: &str = "SELECT x.PublicKey, x.PrivateKey, x.IsPrimary FROM EncryptionKeys x WHERE x.ManifestId = ? AND x.PublicKey = ? AND x.IsDeleted = 0 LIMIT 1";+const DEMOTE_KEYS_FOR_MANIFEST: &str = "UPDATE EncryptionKeys SET IsPrimary = 0, UpdatedAt = ? WHERE ManifestId = ? AND IsPrimary = 1";+const INSERT_KEY_FOR_MANIFEST: &str = "INSERT INTO EncryptionKeys (Id, ManifestId, PublicKey, PrivateKey, IsPrimary, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, 1, ?, ?, 0)";+const INSERT_FOLDER: &str = "INSERT INTO Folders (Id, Name, ParentFolderId, ManifestId, Weight, IsDeleted, CreatedAt, UpdatedAt) VALUES (?, ?, ?, COALESCE((SELECT ManifestId FROM Folders WHERE Id = ?), ?), 0, 0, ?, ?)";+const RESTAMP_SUBTREE_FOLDERS: &str = "UPDATE Folders SET ManifestId = ? WHERE Id IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";+const RESTAMP_SUBTREE_ITEMS: &str = "UPDATE Items SET ManifestId = ? WHERE FolderId IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";+const FIND_ITEMS_WITH_FOREIGN_LOGO: &str = "SELECT i.Id, i.ManifestId, origin.Kind, origin.Source FROM Items i INNER JOIN Logos origin ON origin.Id = i.LogoId LEFT JOIN Logos own ON own.Id = i.LogoId AND own.ManifestId = i.ManifestId WHERE i.LogoId IS NOT NULL AND own.Id IS NULL AND i.IsDeleted = 0";+const GET_LOGO_ID_FOR_KEY: &str = "SELECT Id FROM Logos WHERE ManifestId = ? AND Kind = ? AND Source = ? AND IsDeleted = 0 LIMIT 1";+const GET_BEST_LOGO_FOR_KEY: &str = "SELECT FileData, MimeType, Name FROM Logos WHERE Kind = ? AND Source = ? AND IsDeleted = 0 ORDER BY (FileData IS NOT NULL AND LENGTH(FileData) > 0) DESC, UpdatedAt DESC LIMIT 1";+const UPSERT_LOGO: &str = "INSERT INTO Logos (Id, Kind, Source, ManifestId, FileData, MimeType, Name, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0) ON CONFLICT(ManifestId, Id) DO UPDATE SET FileData = excluded.FileData, MimeType = excluded.MimeType, Name = COALESCE(excluded.Name, Logos.Name), UpdatedAt = excluded.UpdatedAt, IsDeleted = 0";+const REPOINT_ITEM_LOGO: &str = "UPDATE Items SET LogoId = ? WHERE Id = ? AND ManifestId = ?";+const RENDERED_MANIFEST_FOLDER: &str = "SELECT Id FROM Folders WHERE ManifestId = ? AND IsDeleted = 0 AND ParentFolderId IS NULL";+const MANIFEST_ROOT_FOLDER_NAMES: &str = "SELECT ManifestId, Name FROM Folders WHERE IsDeleted = 0 AND ManifestId IS NOT NULL AND UPPER(Id) = UPPER(ManifestId)";/// The active mail delivery keypair of a manifest, when it has one.pub(crate) async fn active_key_for_manifest(host: &Host, manifest_id: &str) -> SyncResult<Option<Row>> {@@ -269,7 +254,7 @@ pub(crate) async fn account_private_key_for(host: &Host, personal_manifest_id: &/// Make a keypair the manifest's active one, demoting (never deleting) whatever it supersedes.pub(crate) async fn set_active_key_for_manifest(host: &Host, manifest_id: &str, public_key: &str, private_key: &str) -> SyncResult<()> {- let now = now();+ let now = now_vault_datetime();exec(host,Db::Local,@@ -290,7 +275,7 @@ pub(crate) async fn manifest_display_names(host: &Host) -> SyncResult<HashMap<Stlet rows = query(host, Db::Local, MANIFEST_ROOT_FOLDER_NAMES, vec![]).await?;Ok(rows.iter()- .filter_map(|row| Some((row.get("ManifestId")?.as_str()?.to_lowercase(), row.get("Name")?.as_str()?.to_string())))+ .filter_map(|row| Some((id_key(row.get("ManifestId")?.as_str()?), row.get("Name")?.as_str()?.to_string()))).collect())}@@ -301,8 +286,8 @@ pub(crate) async fn has_rendered_manifest_folder(host: &Host, manifest_id: &str)/// Create the folder a shared manifest is rendered as and pull everything under it into the manifest.pub(crate) async fn render_manifest_folder(host: &Host, manifest_id: &str, name: &str, active_manifest_id: &str) -> SyncResult<()> {- let folder_id = manifest_id.to_lowercase();- let now = now();+ let folder_id = id_key(manifest_id);+ let now = now_vault_datetime();exec(host,Db::Local,@@ -349,11 +334,11 @@ pub(crate) async fn prune_in_place(host: &Host, retention_days: u32) -> SyncResulet mut tables = Vec::new();for table_query in crate::vault_pruner::get_prune_table_queries() {let rows = query(host, Db::Local, &table_query.query, vec![]).await?;- tables.push(crate::vault_pruner::TableData { name: table_query.name, records: rows.into_iter().map(|row| row.into_iter().collect()).collect() });+ tables.push(CodecTableData { name: table_query.name, records: rows.into_iter().map(row_to_record).collect() });}- let output = crate::vault_pruner::prune_vault(crate::vault_pruner::PruneInput { tables, current_time: now_iso(), retention_days })?;+ let output = crate::vault_pruner::prune_vault(crate::vault_pruner::PruneInput { tables, current_time: now_iso_utc(), retention_days })?;let count = output.statements.len();- exec(host, Db::Local, output.statements.into_iter().map(|s| SqlStatement { sql: s.sql, params: s.params }).collect()).await?;+ exec(host, Db::Local, output.statements).await?;Ok(count)}@@ -369,12 +354,4 @@ mod tests {assert_eq!(&id[14..15], "4");assert!(matches!(&id[19..20], "8" | "9" | "a" | "b"));}-- #[test]- fn timestamps_use_the_vault_format() {- let stamp = now();- assert_eq!(stamp.len(), 23);- assert_eq!(&stamp[10..11], " ");- assert!(now_iso().ends_with('Z'));- }}diff --git a/core/rust/src/vault_sync/email_routing.rs b/core/rust/src/vault_sync/email_routing.rsindex c6e181d22..8ec4a5a81 100644--- a/core/rust/src/vault_sync/email_routing.rs+++ b/core/rust/src/vault_sync/email_routing.rs@@ -6,29 +6,30 @@ use serde_json::Value;use super::types::{ClaimedEmailAddress, EmailRoutingPush};use super::db::value_string;-use crate::vault_codec::row::{rows_of, truthy};+use crate::vault_codec::row::{is_deleted, rows_of, str_col, truthy};use crate::vault_codec::Manifest;+use crate::vault_model::names::{DELETED_AT_COL, FIELD_KEY_COL, FIELD_VALUES_TABLE, ID_COL, ITEMS_TABLE, ITEM_ID_COL};/// The field key of an item's login email.const FIELD_KEY_LOGIN_EMAIL: &str = "login.email";/// Build the routing set from the canonicalized manifests, the user's own included.-pub fn build_email_routing(manifests: &[Manifest], private_email_domains: &[String]) -> EmailRoutingPush {+pub(crate) fn build_email_routing(manifests: &[Manifest], private_email_domains: &[String]) -> EmailRoutingPush {let mut by_pair: HashMap<String, ClaimedEmailAddress> = HashMap::new();let mut order: Vec<String> = Vec::new();for manifest in manifests {- let live_item_ids: HashSet<String> = rows_of(&manifest.tables, "Items")+ let live_item_ids: HashSet<String> = rows_of(&manifest.tables, ITEMS_TABLE).iter()- .filter(|row| !truthy(row.get("IsDeleted")) && row.get("DeletedAt").map_or(true, Value::is_null))- .filter_map(|row| row.get("Id").map(value_string))+ .filter(|row| !is_deleted(row) && row.get(DELETED_AT_COL).map_or(true, Value::is_null))+ .filter_map(|row| row.get(ID_COL).map(value_string)).collect();- for field_value in rows_of(&manifest.tables, "FieldValues") {- if field_value.get("FieldKey").and_then(Value::as_str) != Some(FIELD_KEY_LOGIN_EMAIL) || truthy(field_value.get("IsDeleted")) {+ for field_value in rows_of(&manifest.tables, FIELD_VALUES_TABLE) {+ if str_col(field_value, FIELD_KEY_COL) != Some(FIELD_KEY_LOGIN_EMAIL) || is_deleted(field_value) {continue;}- let item_id = field_value.get("ItemId").map(value_string).unwrap_or_default();+ let item_id = field_value.get(ITEM_ID_COL).map(value_string).unwrap_or_default();if !live_item_ids.contains(&item_id) {continue;}diff --git a/core/rust/src/vault_sync/engine.rs b/core/rust/src/vault_sync/engine.rsindex 53bcc07fe..045b5bf38 100644--- a/core/rust/src/vault_sync/engine.rs+++ b/core/rust/src/vault_sync/engine.rs@@ -12,8 +12,9 @@ use super::push::{self, CanonicalizedSet, PushStatus};use super::session::Host;use super::state::{self, Ctx};use super::types::{Db, FailureFields, FullSyncResult, LogLevel, MigrateManifestResult, MigrationKind, MigrationStatusResult, OperationResult, ResolveVaultKeyResult, SessionOutcome, StatusCheckResult, StatusResponse, SyncOperation, SyncRequest};-use super::{db, http, keys, legacy, version};+use super::{db, http, keys, legacy};use crate::crypto;+use crate::vault_model::ids_equal;/// How many times a chain of syncs may re-sync after an outdated push before giving up./// This is a auto-healing mechanism in case two clients are pushing at the same time, which could@@ -230,7 +231,43 @@ async fn run_sync_preflight(ctx: &mut Ctx) -> SyncResult<Preflight> {/// Whether the server is older than the oldest version this client supports.fn server_too_old(ctx: &Ctx, status: &StatusResponse) -> bool {- ctx.request.min_server_version.as_deref().map(|min| !version::version_gte(&status.server_version, min)).unwrap_or(false)+ ctx.request.min_server_version.as_deref().map(|min| !version_gte(&status.server_version, min)).unwrap_or(false)+}++/// Whether `version1 >= version2` under the client's SemVer rules: a pre-release sorts below its release, and+/// two pre-releases compare lexically.+fn version_gte(version1: &str, version2: &str) -> bool {+ let (core1, pre1) = split_version(version1);+ let (core2, pre2) = split_version(version2);+ let parts1: Vec<u64> = core1.split('.').map(|part| part.parse().unwrap_or(0)).collect();+ let parts2: Vec<u64> = core2.split('.').map(|part| part.parse().unwrap_or(0)).collect();++ for index in 0..parts1.len().max(parts2.len()) {+ let part1 = parts1.get(index).copied().unwrap_or(0);+ let part2 = parts2.get(index).copied().unwrap_or(0);+ if part1 > part2 {+ return true;+ }+ if part1 < part2 {+ return false;+ }+ }++ match (pre1, pre2) {+ (None, Some(_)) => true,+ (Some(_), None) => false,+ (None, None) => true,+ (Some(a), Some(b)) => a >= b,+ }+}++/// A version's `core` and optional pre-release tag (`0.12.0-dev`).+fn split_version(version: &str) -> (&str, Option<&str>) {+ match version.split_once('-') {+ Some((core, pre)) if !pre.is_empty() => (core, Some(pre)),+ Some((core, _)) => (core, None),+ None => (version, None),+ }}/// Whether the client has to pull and re-materialize: any manifest or data bucket changed on the server.@@ -318,7 +355,7 @@ async fn apply_server_directed_changes(ctx: &mut Ctx, status: &StatusResponse) -ctx.log(format!("[PendingActions] Action type {} is not known to this client; leaving it for one that knows it.", action.action_type)).await;return Ok((false, false));}- let record = shared.values().find(|r| action.manifest_id.as_deref().map(|id| id.eq_ignore_ascii_case(&r.manifest_id)).unwrap_or(false));+ let record = shared.values().find(|r| action.manifest_id.as_deref().map(|id| ids_equal(id, &r.manifest_id)).unwrap_or(false));let Some(record) = record.filter(|r| r.can_administer) else {ctx.log(format!("[PendingActions] Manifest {:?} is not open to this session as an administrator; leaving its delivery key rotation for another sync.", action.manifest_id)).await;return Ok((false, false));@@ -633,4 +670,14 @@ mod tests {assert_eq!(logout.error_code, None);assert!(logout.requires_logout);}++ #[test]+ fn semver_rules_match_the_client() {+ assert!(version_gte("0.12.0", "0.12.0-dev"));+ assert!(!version_gte("0.12.0-dev", "0.12.0"));+ assert!(version_gte("0.13.1", "0.12.0-dev"));+ assert!(!version_gte("0.11.9", "0.12.0-dev"));+ assert!(version_gte("1.0", "1.0.0"));+ assert!(version_gte("0.12.0-beta", "0.12.0-alpha"));+ }}diff --git a/core/rust/src/vault_sync/errors.rs b/core/rust/src/vault_sync/errors.rsindex d1ad9c40b..0c15af5cf 100644--- a/core/rust/src/vault_sync/errors.rs+++ b/core/rust/src/vault_sync/errors.rs@@ -181,7 +181,6 @@ impl SyncError {SyncError::Other(_) => Failure::Coded(ErrorCode::SyncEngineFailed),}}-}impl CommandKind {diff --git a/core/rust/src/vault_sync/http.rs b/core/rust/src/vault_sync/http.rsindex c4c5920f6..41634b51c 100644--- a/core/rust/src/vault_sync/http.rs+++ b/core/rust/src/vault_sync/http.rs@@ -59,6 +59,9 @@ pub(crate) async fn delete(host: &Host, path: &str) -> SyncResult<()> {/// The v2 vault endpoint (`GET`/`POST v2/Vault`).pub(crate) const VAULT_ENDPOINT: &str = "Vault";+/// The v2 vault key endpoint (`GET v2/VaultKey/Password`): the account's password-wrapped key chain.+pub(crate) const VAULT_KEY_PASSWORD_ENDPOINT: &str = "VaultKey/Password";+/// Max base64 characters in one blob transfer request or response body.pub(crate) const BLOB_TRANSFER_BATCH_MAX_CHARS: usize = 4 * 1024 * 1024;diff --git a/core/rust/src/vault_sync/keys.rs b/core/rust/src/vault_sync/keys.rsindex 897f973ce..5f91644cc 100644--- a/core/rust/src/vault_sync/keys.rs+++ b/core/rust/src/vault_sync/keys.rs@@ -5,7 +5,7 @@ use std::collections::HashMap;use super::errors::{SyncError, SyncResult};use super::session::Host;use super::state::{self, Ctx};-use super::types::{SharedManifestRecord, VaultKeyGetResponse, VaultKeyResponse, ALGORITHM_RSA_OAEP_SHA256};+use super::types::{SharedManifestDto, VaultKeyGetResponse, VaultKeyResponse, ALGORITHM_RSA_OAEP_SHA256};use super::{db, http};use crate::crypto;@@ -16,7 +16,7 @@ pub(crate) async fn has_local_vault_key(host: &Host) -> SyncResult<bool> {/// `GET v2/VaultKey/Password`: the account's vault key, `None` when the server holds none.pub(crate) async fn fetch_vault_key(host: &Host) -> SyncResult<Option<VaultKeyResponse>> {- match http::get::<VaultKeyGetResponse>(host, "VaultKey/Password", false).await {+ match http::get::<VaultKeyGetResponse>(host, http::VAULT_KEY_PASSWORD_ENDPOINT, false).await {Ok(response) => Ok(response.vault_key),Err(SyncError::Http { status: 404, .. }) => Ok(None),Err(error) => Err(error),@@ -41,7 +41,7 @@ pub(crate) async fn cache_vault_key_blobs(host: &Host, vault_key: &VaultKeyRespo}/// The shared-manifest key records, keyed by manifest id.-pub(crate) async fn shared_manifest_records(ctx: &Ctx) -> SyncResult<HashMap<String, SharedManifestRecord>> {+pub(crate) async fn shared_manifest_records(ctx: &Ctx) -> SyncResult<HashMap<String, SharedManifestDto>> {let Some(ciphertext) = state::get::<String>(&ctx.host, state::SHARED_MANIFESTS).await? else { return Ok(HashMap::new()) };let Some(key) = &ctx.encryption_key else { return Ok(HashMap::new()) };match crypto::symmetric_decrypt(&ciphertext, key).and_then(|json| serde_json::from_str(&json).map_err(Into::into)) {@@ -54,7 +54,7 @@ pub(crate) async fn shared_manifest_records(ctx: &Ctx) -> SyncResult<HashMap<Str}/// Persist the shared-manifest key records.-pub(crate) async fn set_shared_manifest_records(host: &Host, records: &HashMap<String, SharedManifestRecord>, key: &str) -> SyncResult<()> {+pub(crate) async fn set_shared_manifest_records(host: &Host, records: &HashMap<String, SharedManifestDto>, key: &str) -> SyncResult<()> {let ciphertext = crypto::symmetric_encrypt(&serde_json::to_string(records)?, key)?;state::set(host, state::SHARED_MANIFESTS, &ciphertext).await?;Ok(())@@ -87,7 +87,7 @@ pub(crate) fn decrypt_manifest_vek(encrypted_vek: &str, private_key_jwk: &str) -}/// Unwrap one shared manifest's VEK from the grant this account holds on it.-pub(crate) async fn open_shared_manifest_vek(ctx: &Ctx, record: &SharedManifestRecord) -> SyncResult<Option<String>> {+pub(crate) async fn open_shared_manifest_vek(ctx: &Ctx, record: &SharedManifestDto) -> SyncResult<Option<String>> {if record.algorithm != ALGORITHM_RSA_OAEP_SHA256 {ctx.warn(format!("[Sharing] Manifest {} grants its key under an unsupported algorithm \"{}\" (newer server?); leaving it closed.", record.manifest_id, record.algorithm)).await;return Ok(None);@@ -151,7 +151,7 @@ async fn open_chain(ctx: &mut Ctx, encrypted_account_key: &str, encrypted_vek: &/// hierarchy is created later by the migration push. Returns whether the account has a chain.pub(crate) async fn resolve_vault_key(ctx: &mut Ctx) -> SyncResult<bool> {let kek = ctx.encryption_key()?;- let fetched = http::get::<VaultKeyGetResponse>(&ctx.host, "VaultKey/Password", false).await;+ let fetched = http::get::<VaultKeyGetResponse>(&ctx.host, http::VAULT_KEY_PASSWORD_ENDPOINT, false).await;ctx.vault_key_probed = true;match fetched {Ok(response) => match response.vault_key {diff --git a/core/rust/src/vault_sync/legacy.rs b/core/rust/src/vault_sync/legacy.rsindex d7cb2b084..763e5ea79 100644--- a/core/rust/src/vault_sync/legacy.rs+++ b/core/rust/src/vault_sync/legacy.rs@@ -1,5 +1,7 @@-//! LEGACY: everything only a vault still on the sqlite-blob storage format needs. TODO: remove this module and all callers once-//! all users have migrated to the manifest-v1 storage model.+//! LEGACY: everything only a vault still on the sqlite-blob storage format needs. Remove this module once every+//! account has migrated to the manifest-v1 storage model, together with its callers:+//! `pull::pull` (the legacy snapshot branch), `merge::pull_and_merge` (the legacy server branch),+//! `db::schema_state` (the frozen chain check) and `engine::migrate_manifest` (the branch without a vault key).use std::collections::HashMap;diff --git a/core/rust/src/vault_sync/merge.rs b/core/rust/src/vault_sync/merge.rsindex 9f09729db..08f6553e0 100644--- a/core/rust/src/vault_sync/merge.rs+++ b/core/rust/src/vault_sync/merge.rs@@ -3,8 +3,6 @@use std::collections::{HashMap, HashSet};-use serde::{Deserialize, Serialize};-use crate::vault_model::{id_key, ids_equal};use super::errors::{SyncError, SyncResult};use super::pull::{self, OpenedManifestSet, PulledVault};@@ -12,39 +10,18 @@ use super::push::{canonicalize_vault, CanonicalizedSet, ManifestRecord};use super::state::{self, Ctx};use super::types::EmailRoutingDto;use super::legacy;+use crate::vault_codec::row::blob_ref_of;use crate::vault_codec::{self, BlobEntry, CanonicalizedVault, DataBucket, Manifest};use crate::vault_merge::{merge_canonical, CanonicalManifestMerge, CanonicalMergeInput, MergeStats};-/// Aggregate merge statistics across all manifests.-#[derive(Debug, Clone, Default, Serialize, Deserialize)]-#[serde(rename_all = "camelCase")]-pub struct MergeSummary {- pub tables_processed: u32,- pub records_from_local: u32,- pub records_from_server: u32,- pub records_created_locally: u32,- pub conflicts: u32,- pub records_inserted: u32,-}--impl MergeSummary {- fn add(&mut self, stats: &MergeStats) {- self.tables_processed += stats.tables_processed;- self.records_from_local += stats.records_from_local;- self.records_from_server += stats.records_from_server;- self.records_created_locally += stats.records_created_locally;- self.conflicts += stats.conflicts;- self.records_inserted += stats.records_inserted;- }-}-/// What a dirty pull (pull and merge) produced.pub(crate) enum PullAndMergeOutcome {/// LEGACY: the server's vault is still a sqlite blob at this revision; the caller pushes the local vault over it.LegacyServer { revision: i64 },Merged {pulled: PulledVault,- stats: MergeSummary,+ /// The merge counters summed over every manifest.+ stats: MergeStats,fallback_manifest_ids: Vec<String>,dropped_local_manifest_ids: Vec<String>,push_canonical: Option<CanonicalizedSet>,@@ -130,7 +107,7 @@ async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet,let mut manifests: Vec<Manifest> = Vec::new();let mut data_buckets: Vec<DataBucket> = Vec::new();let mut fallback_manifest_ids = Vec::new();- let mut stats = MergeSummary::default();+ let mut stats = MergeStats::default();for entry in merge_output.manifests {let failure = validate_merged_manifest(&entry);if let Some(failure) = &failure {@@ -220,8 +197,7 @@ async fn resolve_merged_blob_refs(ctx: &Ctx, manifests: &[Manifest], blob_map: &for rows in manifest.tables.values() {for row in rows {for value in row.values() {- let Some(reference) = value.get("__blobRef").and_then(serde_json::Value::as_str) else { continue };- let kind = value.get("__blobKind").and_then(serde_json::Value::as_str);+ let Some((reference, kind)) = blob_ref_of(value) else { continue };let Some(bytes) = blob_map.get(reference) else {if kind == Some("attachment") && is_personal {return Err(SyncError::MergeFailed(format!("merged vault references attachment blob {} with no bytes available, refusing to materialize an incomplete vault", reference)));diff --git a/core/rust/src/vault_sync/mod.rs b/core/rust/src/vault_sync/mod.rsindex 33c743ef5..3c530da11 100644--- a/core/rust/src/vault_sync/mod.rs+++ b/core/rust/src/vault_sync/mod.rs@@ -1,23 +1,20 @@//! The vault sync engine that all AliasVault clients use to synchronize their vaults with the server.pub(crate) mod db;-pub(crate) mod email_routing;+mod email_routing;mod engine;-pub mod errors;-pub(crate) mod http;-pub(crate) mod keys;+pub(crate) mod errors;+mod http;+mod keys;mod legacy;mod merge;mod pull;mod push;-pub mod session;+mod session;pub(crate) mod state;-pub mod types;-pub mod version;+pub(crate) mod types;#[cfg(test)]mod tests;-pub use errors::{ErrorCode, Failure, LogoutReason, SyncError};pub use session::SyncSession;-pub use types::{Command, CommandKind, Db, FullSyncResult, HttpMethod, LogLevel, MigrateManifestResult, MigrationKind, MigrationStatusResult, ResolveVaultKeyResult, SessionOutcome, SessionUpdates, SqlStatement, StatusCheckResult, SyncOperation, SyncRequest};diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rsindex 760de4c6c..f715d07c5 100644--- a/core/rust/src/vault_sync/pull.rs+++ b/core/rust/src/vault_sync/pull.rs@@ -6,7 +6,7 @@ use std::collections::HashMap;use crate::vault_model::ids_equal;use super::errors::{SyncError, SyncResult};use super::state::{self, Ctx};-use super::types::{self, BlobDto, BlobHashesRequest, Db, EmailRoutingDto, GetResponse, ManifestDto, SharedManifestRecord, StoredBlobRef};+use super::types::{self, BlobDto, BlobHashesRequest, Db, EmailRoutingDto, GetResponse, ManifestDto, SharedManifestDto, StoredBlobRef};use super::{db, http, keys, legacy};use crate::crypto;use crate::vault_codec::{self, DataBucket, Manifest, MaterializeInput};@@ -120,7 +120,7 @@ pub(crate) async fn open_manifests_and_record_sync_state(ctx: &mut Ctx, snapshotlet mut pulled_fingerprints: HashMap<String, String> = HashMap::new();let mut resolved: Vec<ResolvedManifest> = Vec::new();- let mut shared_records: HashMap<String, SharedManifestRecord> = HashMap::new();+ let mut shared_records: HashMap<String, SharedManifestDto> = HashMap::new();let mut contentless_revisions: HashMap<String, i64> = HashMap::new();let ordered: Vec<&ManifestDto> = std::iter::once(personal_dto).chain(snapshot.manifests.iter().filter(|m| m.manifest_id != personal_dto.manifest_id)).collect();@@ -130,7 +130,7 @@ pub(crate) async fn open_manifests_and_record_sync_state(ctx: &mut Ctx, snapshotif dto.blob.as_deref().unwrap_or("").is_empty() {// A shared manifest served without content yet (created but never written); its grant and revision are still tracked.let (encrypted_vek, encryption_public_key, algorithm) = grant_of(dto).ok_or_else(|| SyncError::Snapshot(format!("shared manifest {} was served without content and without a grant, refusing to assemble", dto.manifest_id)))?;- shared_records.insert(dto.manifest_id.clone(), SharedManifestRecord { manifest_id: dto.manifest_id.clone(), encrypted_vek, encryption_public_key, algorithm, salt: vault_codec::generate_manifest_salt(), name: None, can_administer: dto.can_administer });+ shared_records.insert(dto.manifest_id.clone(), SharedManifestDto { manifest_id: dto.manifest_id.clone(), encrypted_vek, encryption_public_key, algorithm, salt: vault_codec::generate_manifest_salt(), name: None, can_administer: dto.can_administer });contentless_revisions.insert(dto.manifest_id.clone(), dto.revision);continue;}@@ -148,7 +148,7 @@ pub(crate) async fn open_manifests_and_record_sync_state(ctx: &mut Ctx, snapshot}let (encrypted_vek, encryption_public_key, algorithm) = grant_of(dto).ok_or_else(|| SyncError::Snapshot(format!("shared manifest {} carries no grant this account can re-open, refusing to assemble", entry.manifest_id)))?;- shared_records.insert(entry.manifest_id.clone(), SharedManifestRecord { manifest_id: entry.manifest_id.clone(), encrypted_vek, encryption_public_key, algorithm, salt: entry.manifest.manifest_salt.clone(), name: entry.manifest.name.clone(), can_administer: dto.can_administer });+ shared_records.insert(entry.manifest_id.clone(), SharedManifestDto { manifest_id: entry.manifest_id.clone(), encrypted_vek, encryption_public_key, algorithm, salt: entry.manifest.manifest_salt.clone(), name: entry.manifest.name.clone(), can_administer: dto.can_administer });resolved.push(entry);}keys::set_shared_manifest_records(&ctx.host, &shared_records, vek).await?;diff --git a/core/rust/src/vault_sync/push.rs b/core/rust/src/vault_sync/push.rsindex 75cc0d2f8..070ae84b6 100644--- a/core/rust/src/vault_sync/push.rs+++ b/core/rust/src/vault_sync/push.rs@@ -2,8 +2,9 @@//! changed under the key of the manifest that owns it, upload the blobs the server lacks, and `POST v2/Vault`.use std::collections::{HashMap, HashSet};+use std::ops::Deref;-use crate::vault_model::{id_key, ids_equal};+use crate::vault_model::{id_key, ids_equal, OVERFLOW_TABLE, TRASH_RETENTION_DEFAULT_DAYS};use super::email_routing::build_email_routing;use super::errors::{SyncError, SyncResult};use super::state::{self, Ctx};@@ -11,26 +12,29 @@ use super::types::{BlobDto, BlobHashesRequest, BlobRef, BlobUploadRequest, Buckeuse super::{db, http, keys};use crate::crypto;use crate::vault_codec::{self, BlobEntry, CanonicalizeInput, CanonicalizedVault, DataBucket, Manifest, ManifestSpec};-use crate::vault_sharing::{self, ManifestAccessRequest, ManifestWriteSetRequest, SharedManifestRecord as SharingRecord};+use crate::vault_sharing::{self, ManifestAccessRequest, ManifestWriteRecord, ManifestWriteSetRequest, SharedManifestRecord};const BLOBS_ENDPOINT: &str = "Vault/blobs";const BLOBS_MISSING_ENDPOINT: &str = "Vault/blobs/missing";-/// Days an item stays in the trash before a push prunes it.-const TRASH_RETENTION_DAYS: u32 = 30;/// The mutation scope that requires a full manifest push.const MANIFEST_SCOPE: &str = "Main";-/// One manifest this vault can write, resolved from local state before canonicalizing.+/// One manifest this vault can write, resolved from local state before canonicalizing: the write record the+/// sharing logic produced plus the key it encrypts under. Derefs to the record so its fields read directly.#[derive(Debug, Clone)]pub(crate) struct ManifestRecord {- pub manifest_id: String,- pub is_personal: bool,- pub salt: String,+ pub record: ManifestWriteRecord,/// The key this manifest encrypts with; None for the personal manifest, whose content key the push supplies.pub vek: Option<String>,- pub name: Option<String>,- pub can_administer: bool,+}++impl Deref for ManifestRecord {+ type Target = ManifestWriteRecord;++ fn deref(&self) -> &ManifestWriteRecord {+ &self.record+ }}/// The canonicalized vault plus the records it was split against, personal manifest first.@@ -65,7 +69,7 @@ pub(crate) async fn canonicalize_vault(ctx: &Ctx, adopt_unstamped_into: Option<Slet tables = db::read_tables(&ctx.host, Db::Local).await?;let manifest_records = resolve_manifest_records(ctx).await?;let manifests: Vec<ManifestSpec> = manifest_records.iter().map(|r| ManifestSpec { manifest_id: r.manifest_id.clone(), manifest_salt: r.salt.clone(), name: r.name.clone() }).collect();- let canonicalized = vault_codec::canonicalize_from_sqlite(CanonicalizeInput { tables, canonicalized_at: db::now_iso(), manifests, adopt_unstamped_into })?;+ let canonicalized = vault_codec::canonicalize_from_sqlite(CanonicalizeInput { tables, canonicalized_at: crate::timestamp::now_iso_utc(), manifests, adopt_unstamped_into })?;Ok(CanonicalizedSet { canonicalized, manifest_records })}@@ -86,7 +90,7 @@ async fn resolve_manifest_records(ctx: &Ctx) -> SyncResult<Vec<ManifestRecord>>personal_manifest_salt: manifest_salt,stamped_manifest_ids: db::manifest_ids_in_vault(&ctx.host, Db::Local).await?,opened_manifest_ids: shared_veks.keys().cloned().collect(),- held_records: held.values().map(|r| SharingRecord { manifest_id: r.manifest_id.clone(), salt: r.salt.clone(), name: r.name.clone(), can_administer: r.can_administer }).collect(),+ held_records: held.values().map(|r| SharedManifestRecord { manifest_id: r.manifest_id.clone(), salt: r.salt.clone(), name: r.name.clone(), can_administer: r.can_administer }).collect(),display_names: db::manifest_display_names(&ctx.host).await?,});for skipped in &write_set.skipped {@@ -104,7 +108,7 @@ async fn resolve_manifest_records(ctx: &Ctx) -> SyncResult<Vec<ManifestRecord>>if !record.is_personal && vek.is_none() {return Err(SyncError::Other(format!("manifest {} is in the write set without a key, refusing to write it", record.manifest_id)));}- records.push(ManifestRecord { manifest_id: record.manifest_id, is_personal: record.is_personal, salt: record.salt, vek, name: record.name, can_administer: record.can_administer });+ records.push(ManifestRecord { record, vek });}Ok(records)}@@ -232,7 +236,7 @@ async fn upload_new_vault_to_server(ctx: &mut Ctx, cached: Option<CanonicalizedSlet key_before = ctx.encryption_key()?;let mut cached = cached;let mut vault_pruned = false;- match db::prune_in_place(&ctx.host, TRASH_RETENTION_DAYS).await {+ match db::prune_in_place(&ctx.host, TRASH_RETENTION_DEFAULT_DAYS).await {Ok(0) => {}Ok(count) => {ctx.log(format!("[VaultMerge] Pruned expired items from trash ({} SQL statements executed)", count)).await;@@ -262,7 +266,7 @@ async fn upload_new_vault_to_server(ctx: &mut Ctx, cached: Option<CanonicalizedS/// The tables of one bucket category read from the local vault, as extract_buckets takes them.async fn read_bucket_tables(ctx: &Ctx, category_tables: &[String]) -> SyncResult<HashMap<String, Vec<vault_codec::CodecRecord>>> {let mut names: Vec<String> = category_tables.to_vec();- names.push(vault_codec::OVERFLOW_TABLE.to_string());+ names.push(OVERFLOW_TABLE.to_string());db::read_named_tables(&ctx.host, Db::Local, &names).await}diff --git a/core/rust/src/vault_sync/state.rs b/core/rust/src/vault_sync/state.rsindex 1b8df7570..2c491ab31 100644--- a/core/rust/src/vault_sync/state.rs+++ b/core/rust/src/vault_sync/state.rs@@ -107,19 +107,19 @@ impl Ctx {* Persisted state.*/-pub const SERVER_MANIFEST_REVISIONS: &str = "serverManifestRevisions";-pub const VAULT_BUCKET_REVISIONS: &str = "vaultBucketRevisions";-pub const VAULT_MANIFEST_SALT: &str = "vaultManifestSalt";-pub const VAULT_PERSONAL_MANIFEST_ID: &str = "vaultPersonalManifestId";-pub const VAULT_CONTENT_FINGERPRINTS: &str = "vaultContentFingerprints";-pub const VAULT_BLOB_CIPHER_CACHE: &str = "vaultBlobCipherCache";-pub const VAULT_SERVER_BLOB_HASHES: &str = "vaultServerBlobHashes";-pub const SHARED_MANIFESTS: &str = "sharedManifests";-pub const ENCRYPTED_VEK: &str = "encryptedVek";-pub const ENCRYPTED_ACCOUNT_KEY: &str = "encryptedAccountKey";-pub const ACCOUNT_PUBLIC_KEY: &str = "accountPublicKey";-pub const ENCRYPTED_ACCOUNT_PRIVATE_KEY: &str = "encryptedAccountPrivateKey";-pub const ENCRYPTION_KEY_DERIVATION_PARAMS: &str = "encryptionKeyDerivationParams";+pub(crate) const SERVER_MANIFEST_REVISIONS: &str = "serverManifestRevisions";+pub(crate) const VAULT_BUCKET_REVISIONS: &str = "vaultBucketRevisions";+pub(crate) const VAULT_MANIFEST_SALT: &str = "vaultManifestSalt";+pub(crate) const VAULT_PERSONAL_MANIFEST_ID: &str = "vaultPersonalManifestId";+pub(crate) const VAULT_CONTENT_FINGERPRINTS: &str = "vaultContentFingerprints";+pub(crate) const VAULT_BLOB_CIPHER_CACHE: &str = "vaultBlobCipherCache";+pub(crate) const VAULT_SERVER_BLOB_HASHES: &str = "vaultServerBlobHashes";+pub(crate) const SHARED_MANIFESTS: &str = "sharedManifests";+pub(crate) const ENCRYPTED_VEK: &str = "encryptedVek";+pub(crate) const ENCRYPTED_ACCOUNT_KEY: &str = "encryptedAccountKey";+pub(crate) const ACCOUNT_PUBLIC_KEY: &str = "accountPublicKey";+pub(crate) const ENCRYPTED_ACCOUNT_PRIVATE_KEY: &str = "encryptedAccountPrivateKey";+pub(crate) const ENCRYPTION_KEY_DERIVATION_PARAMS: &str = "encryptionKeyDerivationParams";/// Read a value, `None` when absent or null.pub(crate) async fn get<T: DeserializeOwned>(host: &Host, key: &str) -> SyncResult<Option<T>> {@@ -143,15 +143,15 @@ pub(crate) async fn remove(host: &Host, key: &str) -> SyncResult<()> {}/// The record key of one data bucket's revision and fingerprint.-pub fn bucket_revision_key(manifest_id: &str, category: &str) -> String {+pub(crate) fn bucket_revision_key(manifest_id: &str, category: &str) -> String {format!("{}:{}", manifest_id, category)}-pub fn fingerprint_manifest_key(manifest_id: &str) -> String {+pub(crate) fn fingerprint_manifest_key(manifest_id: &str) -> String {format!("manifest:{}", manifest_id)}-pub fn fingerprint_bucket_key(manifest_id: &str, category: &str) -> String {+pub(crate) fn fingerprint_bucket_key(manifest_id: &str, category: &str) -> String {format!("bucket:{}:{}", manifest_id, category)}diff --git a/core/rust/src/vault_sync/tests/mod.rs b/core/rust/src/vault_sync/tests/mod.rsindex abf71e034..995fe39ec 100644--- a/core/rust/src/vault_sync/tests/mod.rs+++ b/core/rust/src/vault_sync/tests/mod.rs@@ -31,7 +31,7 @@ fn request(operation: &str, key: &str, dirty: bool, mutation_sequence: u64) -> S}fn insert_item(conn: &rusqlite::Connection, id: &str, name: &str, manifest_id: &str) {- let now = crate::vault_sync::db::now();+ let now = crate::timestamp::now_vault_datetime();conn.execute("INSERT INTO Items (Id, ManifestId, Name, ItemType, FolderId, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, 'Login', NULL, ?, ?, 0)",rusqlite::params![id, manifest_id, name, now, now],@@ -181,7 +181,7 @@ fn dirty_client_pushes_only_what_changed() {// The written manifest decrypts with the VEK and carries both items.let blob = body["manifests"][0]["manifestBlob"].as_str().unwrap();- let plain = crypto::symmetric_decrypt_bytes(&base64::Engine::decode(&base64::engine::general_purpose::STANDARD, blob).unwrap(), &vek).unwrap();+ let plain = crypto::symmetric_decrypt_bytes(&crate::encoding::base64_decode(blob).unwrap(), &vek).unwrap();let manifest: Value = serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap();assert_eq!(manifest["tables"]["Items"].as_array().unwrap().len(), 2);}@@ -256,7 +256,7 @@ fn outdated_push_merges_the_server_change_and_retries() {assert_eq!(posts.last().unwrap().body.as_ref().unwrap()["manifests"][0]["currentRevision"], 8);let merged: Value = {let blob = posts.last().unwrap().body.as_ref().unwrap()["manifests"][0]["manifestBlob"].as_str().unwrap();- let plain = crypto::symmetric_decrypt_bytes(&base64::Engine::decode(&base64::engine::general_purpose::STANDARD, blob).unwrap(), &vek).unwrap();+ let plain = crypto::symmetric_decrypt_bytes(&crate::encoding::base64_decode(blob).unwrap(), &vek).unwrap();serde_json::from_str(&vault_codec::unpack_payload(&plain).unwrap()).unwrap()};assert_eq!(merged["tables"]["Items"].as_array().unwrap().len(), 3);diff --git a/core/rust/src/vault_sync/tests/test_host.rs b/core/rust/src/vault_sync/tests/test_host.rsindex 5b05ff4fe..06a9d756b 100644--- a/core/rust/src/vault_sync/tests/test_host.rs+++ b/core/rust/src/vault_sync/tests/test_host.rs@@ -2,14 +2,13 @@use std::collections::HashMap;-use base64::engine::general_purpose::STANDARD as BASE64;-use base64::Engine;use rusqlite::{Connection, MAIN_DB};use serde_json::{json, Map, Value};use crate::crypto;-use crate::sqlite_host;-use crate::vault_sync::types::{Command, Db, SqlStatement};+use crate::encoding::{base64_decode, base64_encode};+use crate::sqlite_host::{self, SqlStatement};+use crate::vault_sync::types::{Command, Db};use crate::vault_sync::session::SyncSession;/// One recorded HTTP request.@@ -60,7 +59,6 @@ pub fn open_schema_db(schema_sql: &str) -> Connection {let conn = Connection::open_in_memory().unwrap();conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();conn.execute_batch(schema_sql).unwrap();- conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();conn}@@ -127,7 +125,7 @@ impl TestHost {self.staging = Some(match bytes {None => open_schema_db(&self.schema_sql),Some(b64) => {- let conn = open_from_bytes(&BASE64.decode(b64).unwrap());+ let conn = open_from_bytes(&base64_decode(&b64).unwrap());conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();conn}@@ -146,7 +144,7 @@ impl TestHost {Ok(()) => json!({}),Err(error) => json!({ "error": error }),},- Command::DbExport { db } => json!({ "bytes": BASE64.encode(self.db(db).serialize(MAIN_DB).unwrap().to_vec()) }),+ Command::DbExport { db } => json!({ "bytes": base64_encode(&self.db(db).serialize(MAIN_DB).unwrap()) }),Command::VaultStore { encrypted_blob, mark_dirty, encryption_key, expected_mutation_seq, revision } => {self.store_calls.push(Command::VaultStore { encrypted_blob: String::new(), mark_dirty, encryption_key: encryption_key.clone(), expected_mutation_seq, revision });if let Some(key) = encryption_key {@@ -189,7 +187,7 @@ impl TestHost {/// Store a blob and reload `local` from it, as the contract requires.fn adopt_blob(&mut self, encrypted_blob: &str) {self.vault_blob = Some(encrypted_blob.to_string());- let bytes = crypto::symmetric_decrypt_bytes(&BASE64.decode(encrypted_blob).unwrap(), &self.vault_key).expect("stored blob decrypts with the vault key");+ let bytes = crypto::symmetric_decrypt_bytes(&base64_decode(encrypted_blob).unwrap(), &self.vault_key).expect("stored blob decrypts with the vault key");self.local = open_from_bytes(&bytes);}diff --git a/core/rust/src/vault_sync/types.rs b/core/rust/src/vault_sync/types.rsindex e1a8a39d0..dc1a9a205 100644--- a/core/rust/src/vault_sync/types.rs+++ b/core/rust/src/vault_sync/types.rs@@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize};use serde_json::{Map, Value};use super::errors::{ErrorCode, Failure, LogoutReason, SyncError};+use crate::sqlite_host::SqlStatement;/// The operation a session runs.#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]@@ -267,14 +268,6 @@ pub enum LogLevel {Phase,}-/// One parameterized SQL statement for the host to run.-#[derive(Debug, Clone, Serialize, Deserialize)]-pub struct SqlStatement {- pub sql: String,- #[serde(default)]- pub params: Vec<Value>,-}-/// What the engine can ask the host to do.#[derive(Debug, Clone, Serialize, Deserialize)]#[serde(tag = "kind", rename_all = "camelCase", rename_all_fields = "camelCase")]@@ -515,16 +508,12 @@ pub struct PendingAction {#[derive(Debug, Clone, Default, Deserialize)]#[serde(rename_all = "camelCase")]pub struct GetResponse {- #[serde(default)]- pub status: i32,/// LEGACY: 0 = sqlite-blob, 1 = manifest-v1; absent on servers predating the field.#[serde(default)]pub storage_format: Option<i32>,#[serde(default)]pub legacy_vault_blob: Option<String>,#[serde(default)]- pub version: Option<String>,- #[serde(default)]pub legacy_revision: Option<i64>,#[serde(default)]pub personal_manifest_id: Option<String>,@@ -725,10 +714,11 @@ pub struct VaultKeyResponse {#[serde(default)]pub encryption_settings: String,}+/// A shared manifest as this account holds it: the grant on its key plus what the last pull learned about it.#[derive(Debug, Clone, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]-pub struct SharedManifestRecord {+pub struct SharedManifestDto {pub manifest_id: String,pub encrypted_vek: String,pub encryption_public_key: String,diff --git a/core/rust/src/vault_sync/version.rs b/core/rust/src/vault_sync/version.rsdeleted file mode 100644index 79b355af6..000000000--- a/core/rust/src/vault_sync/version.rs+++ /dev/null@@ -1,51 +0,0 @@-//! Version rules: which server versions this client supports.--/// Whether `version1 >= version2` under the client's SemVer rules: a pre-release sorts below its release, and-/// two pre-releases compare lexically.-pub fn version_gte(version1: &str, version2: &str) -> bool {- let (core1, pre1) = split(version1);- let (core2, pre2) = split(version2);- let parts1: Vec<u64> = core1.split('.').map(|part| part.parse().unwrap_or(0)).collect();- let parts2: Vec<u64> = core2.split('.').map(|part| part.parse().unwrap_or(0)).collect();-- for index in 0..parts1.len().max(parts2.len()) {- let part1 = parts1.get(index).copied().unwrap_or(0);- let part2 = parts2.get(index).copied().unwrap_or(0);- if part1 > part2 {- return true;- }- if part1 < part2 {- return false;- }- }-- match (pre1, pre2) {- (None, Some(_)) => true,- (Some(_), None) => false,- (None, None) => true,- (Some(a), Some(b)) => a >= b,- }-}--fn split(version: &str) -> (&str, Option<&str>) {- match version.split_once('-') {- Some((core, pre)) if !pre.is_empty() => (core, Some(pre)),- Some((core, _)) => (core, None),- None => (version, None),- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn semver_rules_match_the_client() {- assert!(version_gte("0.12.0", "0.12.0-dev"));- assert!(!version_gte("0.12.0-dev", "0.12.0"));- assert!(version_gte("0.13.1", "0.12.0-dev"));- assert!(!version_gte("0.11.9", "0.12.0-dev"));- assert!(version_gte("1.0", "1.0.0"));- assert!(version_gte("0.12.0-beta", "0.12.0-alpha"));- }-}diff --git a/core/rust/src/wasm.rs b/core/rust/src/wasm.rsindex ec666acc3..145e48a78 100644--- a/core/rust/src/wasm.rs+++ b/core/rust/src/wasm.rs@@ -1,24 +1,14 @@-//! WASM bindings for browser extension.+//! WASM bindings for web appsuse serde::Serialize;use wasm_bindgen::prelude::*;-use crate::credential_matcher::{- filter_credentials, CredentialMatcherInput, CredentialMatcherOutput,-};+use crate::credential_matcher::{filter_credentials, CredentialMatcherInput, CredentialMatcherOutput};use crate::password_generator::{available_languages, generate_password};-use crate::vault_codec::{- self, CanonicalizeInput, DataBucket, ExtractBucketsInput, Manifest, MaterializeInput,-};+use crate::vault_codec::{self, CanonicalizeInput, DataBucket, ExtractBucketsInput, Manifest, MaterializeInput};use crate::vault_merge::{merge_canonical, CanonicalMergeInput, CanonicalMergeOutput};use crate::vault_sharing::{self, ManifestAccessRequest, ManifestWriteSetRequest};-use crate::vault_pruner::{prune_vault, PruneInput, PruneOutput};--#[wasm_bindgen]-extern "C" {- #[wasm_bindgen(js_namespace = console)]- pub fn log(s: &str);-}+use crate::vault_pruner::prune_vault;/// Initialize panic hook for better error messages.#[wasm_bindgen(start)]@@ -26,14 +16,14 @@ pub fn init() {console_error_panic_hook::set_once();}-// ═══════════════════════════════════════════════════════════════════════════════-// Version-// ═══════════════════════════════════════════════════════════════════════════════+/// A JS error carrying the core's error message.+fn js_err(error: impl std::fmt::Display) -> JsValue {+ JsValue::from_str(&error.to_string())+}-/// Get the version of the aliasvault-core library.-#[wasm_bindgen(js_name = getCoreVersion)]-pub fn get_core_version_js() -> String {- crate::get_core_version().to_string()+/// Serialize a core value to a JsValue with Rust maps rendered as plain JS objects and absent optionals as null.+fn to_js<T: Serialize>(value: &T) -> Result<JsValue, JsValue> {+ value.serialize(&serde_wasm_bindgen::Serializer::new().serialize_maps_as_objects(true).serialize_missing_as_null(true)).map_err(js_err)}// ═══════════════════════════════════════════════════════════════════════════════@@ -52,43 +42,20 @@ pub fn get_syncable_table_names_js() -> Vec<String> {/// manifests + data buckets, one entry per server manifest, rows out instead of SQL statements.#[wasm_bindgen(js_name = mergeCanonical)]pub fn merge_canonical_js(input: JsValue) -> Result<JsValue, JsValue> {- let input: CanonicalMergeInput = serde_wasm_bindgen::from_value(input)- .map_err(|e| JsValue::from_str(&format!("Failed to parse canonical merge input: {}", e)))?;-- let output: CanonicalMergeOutput = merge_canonical(input)- .map_err(|e| JsValue::from_str(&format!("Canonical merge failed: {}", e)))?;-- codec_to_js(&output)+ let input: CanonicalMergeInput = serde_wasm_bindgen::from_value(input).map_err(js_err)?;+ let output: CanonicalMergeOutput = merge_canonical(input).map_err(js_err)?;+ to_js(&output)}// ═══════════════════════════════════════════════════════════════════════════════// Vault Pruner WASM Bindings// ═══════════════════════════════════════════════════════════════════════════════-/// Prune expired items from trash.-///-/// Items with DeletedAt older than retention_days are marked as permanently deleted (IsDeleted = true).-/// Default retention is 30 days.-///-/// Takes a JsValue (PruneInput) and returns a JsValue (PruneOutput).-#[wasm_bindgen(js_name = pruneVault)]-pub fn prune_vault_js(input: JsValue) -> Result<JsValue, JsValue> {- let input: PruneInput = serde_wasm_bindgen::from_value(input)- .map_err(|e| JsValue::from_str(&format!("Failed to parse input: {}", e)))?;-- let output: PruneOutput = prune_vault(input)- .map_err(|e| JsValue::from_str(&format!("Prune failed: {}", e)))?;-- serde_wasm_bindgen::to_value(&output)- .map_err(|e| JsValue::from_str(&format!("Failed to serialize output: {}", e)))-}--/// Prune vault using JSON strings (alternative API).-///-/// Takes a JSON string and returns a JSON string.+/// Prune expired items from trash (items with DeletedAt older than retention_days, default 30).+/// Input: `PruneInput` JSON. Output: `PruneOutput` JSON.#[wasm_bindgen(js_name = pruneVaultJson)]pub fn prune_vault_json_js(input_json: &str) -> Result<String, JsValue> {- crate::error::json_call(input_json, prune_vault).map_err(|e| JsValue::from_str(&format!("Prune failed: {}", e)))+ crate::error::json_call(input_json, prune_vault).map_err(js_err)}/// Get the per-table SELECT queries used to build prune input.@@ -97,21 +64,13 @@ pub fn prune_vault_json_js(input_json: &str) -> Result<String, JsValue> {/// 1-byte presence marker to avoid serializing large binary data to JSON.#[wasm_bindgen(js_name = getPruneTableQueries)]pub fn get_prune_table_queries_js() -> Result<JsValue, JsValue> {- serde_wasm_bindgen::to_value(&crate::vault_pruner::get_prune_table_queries())- .map_err(|e| JsValue::from_str(&format!("Failed to serialize output: {}", e)))+ serde_wasm_bindgen::to_value(&crate::vault_pruner::get_prune_table_queries()).map_err(js_err)}// ═══════════════════════════════════════════════════════════════════════════════// Vault Codec WASM Bindings (manifest-v1 storage format)// ═══════════════════════════════════════════════════════════════════════════════-/// Serialize a codec output to a JsValue with Rust maps rendered as plain JS objects.-fn codec_to_js<T: serde::Serialize>(value: &T) -> Result<JsValue, JsValue> {- value- .serialize(&serde_wasm_bindgen::Serializer::new().serialize_maps_as_objects(true).serialize_missing_as_null(true))- .map_err(|e| JsValue::from_str(&format!("Failed to serialize codec output: {}", e)))-}-/// The `Logos.Id` to use for `source` inside the manifest with id `manifestId`. Every platform derives/// logo ids through this so independent writers produce the same row instead of colliding on/// `UNIQUE(ManifestId, Kind, Source)`.@@ -138,39 +97,33 @@ pub fn vault_codec_logo_id_for_js(manifest_id: String, kind: String, source: Str/// Input: `CanonicalizeInput`. Output: `CanonicalizedVault`.#[wasm_bindgen(js_name = vaultCodecCanonicalizeFromSqlite)]pub fn vault_codec_canonicalize_from_sqlite_js(input: JsValue) -> Result<JsValue, JsValue> {- let input: CanonicalizeInput = serde_wasm_bindgen::from_value(input)- .map_err(|e| JsValue::from_str(&format!("Failed to parse canonicalize_from_sqlite input: {}", e)))?;- let output = vault_codec::canonicalize_from_sqlite(input)- .map_err(|e| JsValue::from_str(&format!("canonicalize_from_sqlite failed: {}", e)))?;- codec_to_js(&output)+ let input: CanonicalizeInput = serde_wasm_bindgen::from_value(input).map_err(js_err)?;+ let output = vault_codec::canonicalize_from_sqlite(input).map_err(js_err)?;+ to_js(&output)}/// Materialize the manifest + data buckets into the table set the platform inserts into a fresh schema DB./// Input: `MaterializeInput`. Output: `MaterializedTables`.#[wasm_bindgen(js_name = vaultCodecMaterializeAsSqlite)]pub fn vault_codec_materialize_as_sqlite_js(input: JsValue) -> Result<JsValue, JsValue> {- let input: MaterializeInput = serde_wasm_bindgen::from_value(input)- .map_err(|e| JsValue::from_str(&format!("Failed to parse materialize_as_sqlite input: {}", e)))?;- let output = vault_codec::materialize_as_sqlite(input)- .map_err(|e| JsValue::from_str(&format!("materialize_as_sqlite failed: {}", e)))?;- codec_to_js(&output)+ let input: MaterializeInput = serde_wasm_bindgen::from_value(input).map_err(js_err)?;+ let output = vault_codec::materialize_as_sqlite(input).map_err(js_err)?;+ to_js(&output)}/// Build a bucket category's data buckets, one per manifest this vault writes: rows route by the manifest/// each one names. Input: `{ category, manifestIds, tables: { <name>: [rows] } }`. Output: `DataBucket[]`.#[wasm_bindgen(js_name = vaultCodecExtractBuckets)]pub fn vault_codec_extract_buckets_js(input: JsValue) -> Result<JsValue, JsValue> {- let input: ExtractBucketsInput = serde_wasm_bindgen::from_value(input)- .map_err(|e| JsValue::from_str(&format!("Failed to parse extract-buckets input: {}", e)))?;- let buckets = vault_codec::extract_buckets(input.category, input.manifest_ids, input.tables)- .map_err(|e| JsValue::from_str(&format!("extract_buckets failed: {}", e)))?;- codec_to_js(&buckets)+ let input: ExtractBucketsInput = serde_wasm_bindgen::from_value(input).map_err(js_err)?;+ let buckets = vault_codec::extract_buckets(input.category, input.manifest_ids, input.tables).map_err(js_err)?;+ to_js(&buckets)}/// The bucket layout: `[{ category, tables: [<name>] }]`. Source of truth for platform bucket-only sync.#[wasm_bindgen(js_name = vaultCodecBucketLayout)]pub fn vault_codec_bucket_layout_js() -> Result<JsValue, JsValue> {- codec_to_js(&vault_codec::bucket_layout())+ to_js(&vault_codec::bucket_layout())}/// The name of the client-local SQLite table that carries the codec overflow inside the vault DB.@@ -178,7 +131,7 @@ pub fn vault_codec_bucket_layout_js() -> Result<JsValue, JsValue> {/// unknown columns/tables re-merge and survive.#[wasm_bindgen(js_name = vaultCodecOverflowTable)]pub fn vault_codec_overflow_table_js() -> String {- vault_codec::OVERFLOW_TABLE.to_string()+ crate::vault_model::OVERFLOW_TABLE.to_string()}/// Generate a fresh 32-byte per-manifest blob-hashing salt (lowercase hex).@@ -190,33 +143,27 @@ pub fn vault_codec_generate_manifest_salt_js() -> String {/// Pack a payload JSON string into gzip(envelope{contentHash, payload}). Encryption is done by platform.#[wasm_bindgen(js_name = vaultCodecPackPayload)]pub fn vault_codec_pack_payload_js(payload_json: &str) -> Result<Vec<u8>, JsValue> {- vault_codec::pack_payload(payload_json)- .map_err(|e| JsValue::from_str(&format!("pack_payload failed: {}", e)))+ vault_codec::pack_payload(payload_json).map_err(js_err)}/// Unpack a (decrypted) payload: gunzip > verify content hash > return payload JSON string.#[wasm_bindgen(js_name = vaultCodecUnpackPayload)]pub fn vault_codec_unpack_payload_js(plain_bytes: &[u8]) -> Result<String, JsValue> {- vault_codec::unpack_payload(plain_bytes)- .map_err(|e| JsValue::from_str(&format!("unpack_payload failed: {}", e)))+ vault_codec::unpack_payload(plain_bytes).map_err(js_err)}/// Structurally validate a manifest. Input: `Manifest`. Output: `ValidationResult`.#[wasm_bindgen(js_name = vaultCodecValidateManifest)]pub fn vault_codec_validate_manifest_js(manifest: JsValue) -> Result<JsValue, JsValue> {- let m: Manifest = serde_wasm_bindgen::from_value(manifest)- .map_err(|e| JsValue::from_str(&format!("Failed to parse manifest: {}", e)))?;- serde_wasm_bindgen::to_value(&vault_codec::validate_manifest(&m))- .map_err(|e| JsValue::from_str(&format!("Failed to serialize result: {}", e)))+ let m: Manifest = serde_wasm_bindgen::from_value(manifest).map_err(js_err)?;+ serde_wasm_bindgen::to_value(&vault_codec::validate_manifest(&m)).map_err(js_err)}/// Validate a data bucket. Input: `DataBucket`. Output: `ValidationResult`.#[wasm_bindgen(js_name = vaultCodecValidateDataBucket)]pub fn vault_codec_validate_data_bucket_js(data_bucket: JsValue) -> Result<JsValue, JsValue> {- let b: DataBucket = serde_wasm_bindgen::from_value(data_bucket)- .map_err(|e| JsValue::from_str(&format!("Failed to parse data bucket: {}", e)))?;- serde_wasm_bindgen::to_value(&vault_codec::validate_data_bucket(&b))- .map_err(|e| JsValue::from_str(&format!("Failed to serialize result: {}", e)))+ let b: DataBucket = serde_wasm_bindgen::from_value(data_bucket).map_err(js_err)?;+ serde_wasm_bindgen::to_value(&vault_codec::validate_data_bucket(&b)).map_err(js_err)}/// SHA-256 (lowercase hex) of a base64 ciphertext string.@@ -237,79 +184,57 @@ pub fn vault_codec_compute_content_fingerprint_js(payload_json: &str) -> String/// delivery keypair on a shared manifest).#[wasm_bindgen(js_name = vaultCodecExtractEncryptionKeyForPublicKey)]pub fn vault_codec_extract_encryption_key_for_public_key_js(manifest: JsValue, public_key: &str) -> Result<JsValue, JsValue> {- let m: Manifest = serde_wasm_bindgen::from_value(manifest)- .map_err(|e| JsValue::from_str(&format!("Failed to parse manifest: {}", e)))?;- codec_to_js(&vault_codec::extract_encryption_key_for_public_key(&m, public_key))+ let m: Manifest = serde_wasm_bindgen::from_value(manifest).map_err(js_err)?;+ to_js(&vault_codec::extract_encryption_key_for_public_key(&m, public_key))}// ═══════════════════════════════════════════════════════════════════════════════// Vault Sharing WASM Bindings// ═══════════════════════════════════════════════════════════════════════════════-/// Serialize a sharing output to a JsValue.-fn sharing_to_js<T: serde::Serialize>(value: &T) -> Result<JsValue, JsValue> {- value- .serialize(&serde_wasm_bindgen::Serializer::new().serialize_maps_as_objects(true).serialize_missing_as_null(true))- .map_err(|e| JsValue::from_str(&format!("Failed to serialize sharing output: {}", e)))-}-/// Resolve which manifests the next push writes, personal manifest first./// Input: `ManifestWriteSetRequest`. Output: `ManifestWriteSet`.#[wasm_bindgen(js_name = vaultSharingResolveManifestWriteSet)]pub fn vault_sharing_resolve_manifest_write_set_js(input: JsValue) -> Result<JsValue, JsValue> {- let input: ManifestWriteSetRequest = serde_wasm_bindgen::from_value(input)- .map_err(|e| JsValue::from_str(&format!("Failed to parse manifest-write-set request: {}", e)))?;- sharing_to_js(&vault_sharing::resolve_manifest_write_set(input))+ let input: ManifestWriteSetRequest = serde_wasm_bindgen::from_value(input).map_err(js_err)?;+ to_js(&vault_sharing::resolve_manifest_write_set(input))}/// Split what the vault holds into what cannot be written and what access was lost./// Input: `ManifestAccessRequest`. Output: `ManifestAccessPartition`.#[wasm_bindgen(js_name = vaultSharingPartitionManifestAccess)]pub fn vault_sharing_partition_manifest_access_js(input: JsValue) -> Result<JsValue, JsValue> {- let input: ManifestAccessRequest = serde_wasm_bindgen::from_value(input)- .map_err(|e| JsValue::from_str(&format!("Failed to parse manifest-access request: {}", e)))?;- sharing_to_js(&vault_sharing::partition_manifest_access(input))+ let input: ManifestAccessRequest = serde_wasm_bindgen::from_value(input).map_err(js_err)?;+ to_js(&vault_sharing::partition_manifest_access(input))}// ═══════════════════════════════════════════════════════════════════════════════// Credential Matcher WASM Bindings// ═══════════════════════════════════════════════════════════════════════════════-/// Filter credentials for autofill.-///-/// Takes a JsValue (CredentialMatcherInput) and returns a JsValue (CredentialMatcherOutput).+/// Filter credentials for autofill. Input: `CredentialMatcherInput`. Output: `CredentialMatcherOutput`.#[wasm_bindgen(js_name = filterCredentials)]pub fn filter_credentials_js(input: JsValue) -> Result<JsValue, JsValue> {- let input: CredentialMatcherInput = serde_wasm_bindgen::from_value(input)- .map_err(|e| JsValue::from_str(&format!("Failed to parse input: {}", e)))?;+ let input: CredentialMatcherInput = serde_wasm_bindgen::from_value(input).map_err(js_err)?;let output: CredentialMatcherOutput = filter_credentials(input);- serde_wasm_bindgen::to_value(&output)- .map_err(|e| JsValue::from_str(&format!("Failed to serialize output: {}", e)))+ serde_wasm_bindgen::to_value(&output).map_err(js_err)}-/// Filter credentials using JSON strings (alternative API).-///-/// Takes a JSON string and returns a JSON string.+/// JSON sibling of `filterCredentials`.#[wasm_bindgen(js_name = filterCredentialsJson)]pub fn filter_credentials_json_js(input_json: &str) -> Result<String, JsValue> {- crate::credential_matcher::filter_credentials_json(input_json).map_err(|e| JsValue::from_str(&e.to_string()))+ crate::credential_matcher::filter_credentials_json(input_json).map_err(js_err)}-/// Extract domain from URL.-///-/// Handles both full URLs and partial domains, returning normalized domain-/// without protocol, www prefix, path, query, or fragment.+/// The domain of a URL or partial domain: no protocol, `www.` prefix, path, query or fragment.#[wasm_bindgen(js_name = extractDomain)]pub fn extract_domain_js(url: &str) -> String {crate::credential_matcher::extract_domain(url)}-/// Extract root domain from a domain string.-///-/// E.g., "sub.example.com" -> "example.com"-/// E.g., "sub.example.co.uk" -> "example.co.uk"+/// The root domain of a domain: `sub.example.co.uk` gives `example.co.uk`.#[wasm_bindgen(js_name = extractRootDomain)]pub fn extract_root_domain_js(domain: &str) -> String {crate::credential_matcher::extract_root_domain(domain)@@ -319,39 +244,23 @@ pub fn extract_root_domain_js(domain: &str) -> String {// Favicon WASM Bindings// ═══════════════════════════════════════════════════════════════════════════════-/// Pick the favicon target for an item from its URLs, in the order the item lists them.-///-/// Returns the URL to fetch from and the `Logos.Source` key to store the result under, or-/// `null` when no URL qualifies.+/// The favicon target for an item's URLs (in item order): the URL to fetch and the `Logos.Source` key, or null.#[wasm_bindgen(js_name = selectFaviconTarget)]pub fn select_favicon_target_js(urls: Vec<String>) -> Result<JsValue, JsValue> {match crate::favicon::select_favicon_target(&urls) {- Some(target) => serde_wasm_bindgen::to_value(&target)- .map_err(|e| JsValue::from_str(&format!("Serialization error: {}", e))),+ Some(target) => serde_wasm_bindgen::to_value(&target).map_err(js_err),None => Ok(JsValue::NULL),}}-/// Derive the `Logos.Source` key for a URL.-///-/// Returns an empty string when the URL is not something a favicon can be fetched from.-#[wasm_bindgen(js_name = faviconSourceKey)]-pub fn favicon_source_key_js(url: &str) -> String {- crate::favicon::favicon_source_key(url)-}-// ═══════════════════════════════════════════════════════════════════════════════// Password Generator WASM Bindings// ═══════════════════════════════════════════════════════════════════════════════-/// Generate a password or passphrase from JSON-serialized settings.-///-/// Takes a JSON string (PasswordSettings) and returns the generated password string.-/// The `Type` field selects the generator ("basic" or "diceware").+/// Generate a password or passphrase from `PasswordSettings` JSON; `Type` selects "basic" or "diceware".#[wasm_bindgen(js_name = generatePassword)]pub fn generate_password_js(settings_json: &str) -> Result<String, JsValue> {- generate_password(settings_json)- .map_err(|e| JsValue::from_str(&format!("Password generation failed: {}", e)))+ generate_password(settings_json).map_err(js_err)}/// Get the list of bundled Diceware language codes (first is the default, English).@@ -364,31 +273,25 @@ pub fn get_diceware_languages_js() -> Vec<String> {// Identity Generator WASM Bindings// ═══════════════════════════════════════════════════════════════════════════════-/// Generate a random identity from a JSON-serialized request.-///-/// The request accepts `language`, `gender` ("male"/"female"/"random"), `ageRange`-/// (e.g. "21-25" or "random") and/or explicit `birthdateOptions`. Returns the-/// generated identity as a JSON string with camelCase fields.+/// Generate a random identity from `IdentityRequest` JSON (`language`, `gender`, `ageRange`, `birthdateOptions`);+/// returns `Identity` JSON.#[wasm_bindgen(js_name = generateIdentity)]pub fn generate_identity_js(request_json: &str) -> Result<String, JsValue> {- crate::identity_generator::generate_identity(request_json)- .map_err(|e| JsValue::from_str(&format!("Identity generation failed: {}", e)))+ crate::identity_generator::generate_identity(request_json).map_err(js_err)}/// Generate a username from a JSON-serialized name input/// (`firstName`, `lastName`, `birthDate`).#[wasm_bindgen(js_name = generateIdentityUsername)]pub fn generate_identity_username_js(input_json: &str) -> Result<String, JsValue> {- crate::identity_generator::generate_username(input_json)- .map_err(|e| JsValue::from_str(&format!("Username generation failed: {}", e)))+ crate::identity_generator::generate_username(input_json).map_err(js_err)}/// Generate an email prefix from a JSON-serialized name input/// (`firstName`, `lastName`, `birthDate`).#[wasm_bindgen(js_name = generateIdentityEmailPrefix)]pub fn generate_identity_email_prefix_js(input_json: &str) -> Result<String, JsValue> {- crate::identity_generator::generate_email_prefix(input_json)- .map_err(|e| JsValue::from_str(&format!("Email prefix generation failed: {}", e)))+ crate::identity_generator::generate_email_prefix(input_json).map_err(js_err)}/// Generate a random alphanumeric email prefix that is not based on any identity.@@ -412,19 +315,13 @@ pub fn get_identity_languages_js() -> Vec<String> {/// decrypted `MessageSource` of both legacy and source-only emails can be passed as-is.#[wasm_bindgen(js_name = parseEmailSource)]pub fn parse_email_source_js(source: &[u8]) -> Result<JsValue, JsValue> {- let parsed = crate::email_parser::parse_email_source(source)- .map_err(|e| JsValue::from_str(&format!("Email parse failed: {}", e)))?;-- parsed- .serialize(&serde_wasm_bindgen::Serializer::new().serialize_maps_as_objects(true).serialize_missing_as_null(true))- .map_err(|e| JsValue::from_str(&format!("Failed to serialize output: {}", e)))+ to_js(&crate::email_parser::parse_email_source(source).map_err(js_err)?)}/// Turn a stored email source into the raw RFC 822 message bytes for showing the message source without parsing it.#[wasm_bindgen(js_name = decodeEmailSource)]pub fn decode_email_source_js(source: &[u8]) -> Result<Vec<u8>, JsValue> {- crate::email_parser::decode_email_source(source)- .map_err(|e| JsValue::from_str(&format!("Email source decode failed: {}", e)))+ crate::email_parser::decode_email_source(source).map_err(js_err)}/// Extract the decoded bytes of one attachment, identified by its index in the parsed attachment list.@@ -432,8 +329,7 @@ pub fn decode_email_source_js(source: &[u8]) -> Result<Vec<u8>, JsValue> {/// fetched body as `detachedBody`. It is ignored for attachments that are still inline.#[wasm_bindgen(js_name = extractEmailAttachment)]pub fn extract_email_attachment_js(source: &[u8], index: usize, detached_body: Option<Box<[u8]>>) -> Result<Vec<u8>, JsValue> {- crate::email_parser::extract_email_attachment(source, index, detached_body.as_deref())- .map_err(|e| JsValue::from_str(&format!("Email attachment extraction failed: {}", e)))+ crate::email_parser::extract_email_attachment(source, index, detached_body.as_deref()).map_err(js_err)}/// Get the list of age range option values ("random" plus 5-year ranges).@@ -446,172 +342,71 @@ pub fn get_identity_age_ranges_js() -> Vec<String> {// Argon2id Key Derivation WASM Bindings// ═══════════════════════════════════════════════════════════════════════════════-/// Derive a key from a password using Argon2id.-///-/// # Arguments-/// * `password` - The password, hashed as its UTF-8 bytes-/// * `salt` - The salt, hashed as its UTF-8 bytes, at least 8 bytes long-/// * `encryption_settings` - The `EncryptionSettings` JSON, or an empty string for the defaults-///-/// # Returns-/// The derived key as 32 bytes+/// Derive a 32-byte key from a password and salt (UTF-8 bytes) with Argon2id under the `EncryptionSettings`+/// JSON, or the defaults for an empty string.#[wasm_bindgen(js_name = argon2DeriveKey)]pub fn argon2_derive_key_js(password: &str, salt: &str, encryption_settings: &str) -> Result<Vec<u8>, JsValue> {- crate::crypto::argon2::argon2_derive_key_from_settings(password, salt, encryption_settings)- .map_err(|e| JsValue::from_str(&format!("Argon2 error: {}", e)))+ crate::crypto::argon2::argon2_derive_key_from_settings(password, salt, encryption_settings).map_err(js_err)}// ═══════════════════════════════════════════════════════════════════════════════// SRP (Secure Remote Password) WASM Bindings// ═══════════════════════════════════════════════════════════════════════════════-/// Generate a cryptographic salt for SRP.-/// Returns a 32-byte random salt as an uppercase hex string.+/// A random 32-byte SRP salt as an uppercase hex string.#[wasm_bindgen(js_name = srpGenerateSalt)]pub fn srp_generate_salt_js() -> String {crate::crypto::srp::srp_generate_salt()}-/// Derive the SRP private key (x) from credentials.-///-/// # Arguments-/// * `salt` - Salt as uppercase hex string-/// * `identity` - User identity (username or SRP identity GUID)-/// * `password_hash` - Pre-hashed password as uppercase hex string (from Argon2id)-///-/// # Returns-/// Private key as uppercase hex string+/// The SRP private key (x) as uppercase hex from the hex salt, the identity and the hex password hash.#[wasm_bindgen(js_name = srpDerivePrivateKey)]-pub fn srp_derive_private_key_js(- salt: &str,- identity: &str,- password_hash: &str,-) -> Result<String, JsValue> {- crate::crypto::srp::srp_derive_private_key(salt, identity, password_hash)- .map_err(|e| JsValue::from_str(&format!("SRP error: {}", e)))+pub fn srp_derive_private_key_js(salt: &str, identity: &str, password_hash: &str) -> Result<String, JsValue> {+ crate::crypto::srp::srp_derive_private_key(salt, identity, password_hash).map_err(js_err)}-/// Derive the SRP verifier (v) from a private key.-///-/// # Arguments-/// * `private_key` - Private key as uppercase hex string-///-/// # Returns-/// Verifier as uppercase hex string (for registration)+/// The SRP verifier (v) as uppercase hex from the hex private key, for registration.#[wasm_bindgen(js_name = srpDeriveVerifier)]pub fn srp_derive_verifier_js(private_key: &str) -> Result<String, JsValue> {- crate::crypto::srp::srp_derive_verifier(private_key)- .map_err(|e| JsValue::from_str(&format!("SRP error: {}", e)))+ crate::crypto::srp::srp_derive_verifier(private_key).map_err(js_err)}-/// Generate a client ephemeral key pair.-/// Returns a JsValue object with `public` and `secret` properties (uppercase hex strings).+/// A client ephemeral pair as `{ public, secret }` (uppercase hex).#[wasm_bindgen(js_name = srpGenerateEphemeral)]pub fn srp_generate_ephemeral_js() -> Result<JsValue, JsValue> {let ephemeral = crate::crypto::srp::srp_generate_ephemeral();- serde_wasm_bindgen::to_value(&ephemeral)- .map_err(|e| JsValue::from_str(&format!("Failed to serialize ephemeral: {}", e)))+ serde_wasm_bindgen::to_value(&ephemeral).map_err(js_err)}-/// Derive the client session from server response.-///-/// # Arguments-/// * `client_secret` - Client secret ephemeral (a) as hex string-/// * `server_public` - Server public ephemeral (B) as hex string-/// * `salt` - Salt as hex string-/// * `identity` - User identity (username or SRP identity GUID)-/// * `private_key` - Private key (x) as hex string-///-/// # Returns-/// JsValue object with `proof` and `key` properties (uppercase hex strings)+/// The client session as `{ proof, key }` (uppercase hex) from the server's public ephemeral; hex inputs.#[wasm_bindgen(js_name = srpDeriveSession)]-pub fn srp_derive_session_js(- client_secret: &str,- server_public: &str,- salt: &str,- identity: &str,- private_key: &str,-) -> Result<JsValue, JsValue> {- let session = crate::crypto::srp::srp_derive_session(client_secret, server_public, salt, identity, private_key)- .map_err(|e| JsValue::from_str(&format!("SRP error: {}", e)))?;- serde_wasm_bindgen::to_value(&session)- .map_err(|e| JsValue::from_str(&format!("Failed to serialize session: {}", e)))-}--/// Generate a server ephemeral key pair.-///-/// # Arguments-/// * `verifier` - Password verifier (v) as hex string-///-/// # Returns-/// JsValue object with `public` and `secret` properties (uppercase hex strings)+pub fn srp_derive_session_js(client_secret: &str, server_public: &str, salt: &str, identity: &str, private_key: &str) -> Result<JsValue, JsValue> {+ let session = crate::crypto::srp::srp_derive_session(client_secret, server_public, salt, identity, private_key).map_err(js_err)?;+ serde_wasm_bindgen::to_value(&session).map_err(js_err)+}++/// A server ephemeral pair as `{ public, secret }` (uppercase hex) from the hex verifier.#[wasm_bindgen(js_name = srpGenerateEphemeralServer)]pub fn srp_generate_ephemeral_server_js(verifier: &str) -> Result<JsValue, JsValue> {- let ephemeral = crate::crypto::srp::srp_generate_ephemeral_server(verifier)- .map_err(|e| JsValue::from_str(&format!("SRP error: {}", e)))?;- serde_wasm_bindgen::to_value(&ephemeral)- .map_err(|e| JsValue::from_str(&format!("Failed to serialize ephemeral: {}", e)))+ let ephemeral = crate::crypto::srp::srp_generate_ephemeral_server(verifier).map_err(js_err)?;+ serde_wasm_bindgen::to_value(&ephemeral).map_err(js_err)}-/// Derive and verify the server session from client response.-///-/// # Arguments-/// * `server_secret` - Server secret ephemeral (b) as hex string-/// * `client_public` - Client public ephemeral (A) as hex string-/// * `salt` - Salt as hex string-/// * `identity` - User identity (username or SRP identity GUID)-/// * `verifier` - Password verifier (v) as hex string-/// * `client_proof` - Client proof (M1) as hex string-///-/// # Returns-/// JsValue: object with `proof` and `key` if valid, null if client proof is invalid+/// The server session as `{ proof, key }` once the client's proof verifies, null when it does not; hex inputs.#[wasm_bindgen(js_name = srpDeriveSessionServer)]-pub fn srp_derive_session_server_js(- server_secret: &str,- client_public: &str,- salt: &str,- identity: &str,- verifier: &str,- client_proof: &str,-) -> Result<JsValue, JsValue> {- let session = crate::crypto::srp::srp_derive_session_server(- server_secret,- client_public,- salt,- identity,- verifier,- client_proof,- )- .map_err(|e| JsValue::from_str(&format!("SRP error: {}", e)))?;+pub fn srp_derive_session_server_js(server_secret: &str, client_public: &str, salt: &str, identity: &str, verifier: &str, client_proof: &str) -> Result<JsValue, JsValue> {+ let session = crate::crypto::srp::srp_derive_session_server(server_secret, client_public, salt, identity, verifier, client_proof).map_err(js_err)?;match session {- Some(s) => serde_wasm_bindgen::to_value(&s)- .map_err(|e| JsValue::from_str(&format!("Failed to serialize session: {}", e))),+ Some(s) => serde_wasm_bindgen::to_value(&s).map_err(js_err),None => Ok(JsValue::NULL),}}-/// Verify the server's session proof (M2) on the client side.-///-/// This confirms that the server successfully derived the same session key.-///-/// # Arguments-/// * `client_public` - Client public ephemeral (A) as hex string-/// * `client_proof` - Client proof (M1) as hex string-/// * `session_key` - Session key (K) as hex string-/// * `server_proof` - Server proof (M2) as hex string to verify-///-/// # Returns-/// True if verification succeeds, false otherwise+/// Whether the server's proof (M2) matches, which confirms it derived the same session key; hex inputs.#[wasm_bindgen(js_name = srpVerifySession)]-pub fn srp_verify_session_js(- client_public: &str,- client_proof: &str,- session_key: &str,- server_proof: &str,-) -> Result<bool, JsValue> {- crate::crypto::srp::srp_verify_session(client_public, client_proof, session_key, server_proof)- .map_err(|e| JsValue::from_str(&format!("SRP error: {}", e)))+pub fn srp_verify_session_js(client_public: &str, client_proof: &str, session_key: &str, server_proof: &str) -> Result<bool, JsValue> {+ crate::crypto::srp::srp_verify_session(client_public, client_proof, session_key, server_proof).map_err(js_err)}// ═══════════════════════════════════════════════════════════════════════════════@@ -621,33 +416,33 @@ pub fn srp_verify_session_js(/// AES-256-GCM encrypt bytes with a base64 key. Returns base64 of `IV | ciphertext | tag`.#[wasm_bindgen(js_name = aesGcmEncrypt)]pub fn aes_gcm_encrypt_js(plaintext: &[u8], key_base64: &str) -> Result<String, JsValue> {- crate::crypto::symmetric_encrypt_bytes(plaintext, key_base64).map_err(|e| JsValue::from_str(&e.to_string()))+ crate::crypto::symmetric_encrypt_bytes(plaintext, key_base64).map_err(js_err)}/// AES-256-GCM decrypt base64 `IV | ciphertext | tag` with a base64 key.#[wasm_bindgen(js_name = aesGcmDecrypt)]pub fn aes_gcm_decrypt_js(base64_ciphertext: &str, key_base64: &str) -> Result<Vec<u8>, JsValue> {- let bytes = crate::encoding::base64_decode(base64_ciphertext).map_err(|e| JsValue::from_str(&e.to_string()))?;- crate::crypto::symmetric_decrypt_bytes(&bytes, key_base64).map_err(|e| JsValue::from_str(&e.to_string()))+ let bytes = crate::encoding::base64_decode(base64_ciphertext).map_err(js_err)?;+ crate::crypto::symmetric_decrypt_bytes(&bytes, key_base64).map_err(js_err)}/// Generate an RSA-OAEP-256 key pair as `{ publicKey, privateKey }` JWK strings.#[wasm_bindgen(js_name = rsaGenerateKeyPair)]pub fn rsa_generate_key_pair_js() -> Result<JsValue, JsValue> {- let pair = crate::crypto::generate_rsa_key_pair().map_err(|e| JsValue::from_str(&e.to_string()))?;- serde_wasm_bindgen::to_value(&pair).map_err(|e| JsValue::from_str(&e.to_string()))+ let pair = crate::crypto::generate_rsa_key_pair().map_err(js_err)?;+ serde_wasm_bindgen::to_value(&pair).map_err(js_err)}/// RSA-OAEP-256 encrypt bytes for a JWK public key. Returns base64 ciphertext.#[wasm_bindgen(js_name = rsaEncrypt)]pub fn rsa_encrypt_js(plaintext: &[u8], public_key_jwk: &str) -> Result<String, JsValue> {- crate::crypto::encrypt_with_public_key(plaintext, public_key_jwk).map_err(|e| JsValue::from_str(&e.to_string()))+ crate::crypto::encrypt_with_public_key(plaintext, public_key_jwk).map_err(js_err)}/// RSA-OAEP-256 decrypt base64 ciphertext with a JWK private key.#[wasm_bindgen(js_name = rsaDecrypt)]pub fn rsa_decrypt_js(base64_ciphertext: &str, private_key_jwk: &str) -> Result<Vec<u8>, JsValue> {- crate::crypto::decrypt_with_private_key(base64_ciphertext, private_key_jwk).map_err(|e| JsValue::from_str(&e.to_string()))+ crate::crypto::decrypt_with_private_key(base64_ciphertext, private_key_jwk).map_err(js_err)}// ═══════════════════════════════════════════════════════════════════════════════@@ -666,18 +461,18 @@ impl VaultSyncSessionJs {/// Start an operation from its `SyncRequest` JSON.#[wasm_bindgen(constructor)]pub fn new(request_json: &str) -> Result<VaultSyncSessionJs, JsValue> {- Ok(Self { inner: crate::vault_sync::SyncSession::new(request_json).map_err(|e| JsValue::from_str(&e.to_string()))? })+ Ok(Self { inner: crate::vault_sync::SyncSession::new(request_json).map_err(js_err)? })}/// The next command for the host, as JSON.#[wasm_bindgen(js_name = nextCommand)]pub fn next_command(&self) -> Result<String, JsValue> {- self.inner.next_command().map_err(|e| JsValue::from_str(&e.to_string()))+ self.inner.next_command().map_err(js_err)}/// Hand the host's response to the last command back, as JSON.pub fn resume(&self, response_json: &str) -> Result<(), JsValue> {- self.inner.resume(response_json).map_err(|e| JsValue::from_str(&e.to_string()))+ self.inner.resume(response_json).map_err(js_err)}}@@ -689,6 +484,7 @@ mod sqlite_js {use js_sys::{Array, Object, Reflect, Uint8Array};use wasm_bindgen::prelude::*;+ use super::js_err;use crate::sqlite_host::{MemoryDatabase, SqlResult, SqlValue};/// An in-memory SQLite database that can be used by host applications to be have uniform access to the database.@@ -702,38 +498,32 @@ mod sqlite_js {/// Open a database from its SQLite file bytes.#[wasm_bindgen(js_name = fromBytes)]pub fn from_bytes(bytes: &[u8]) -> Result<SqliteMemoryDatabaseJs, JsValue> {- Ok(Self { inner: Some(MemoryDatabase::from_bytes(bytes).map_err(error)?) })+ Ok(Self { inner: Some(MemoryDatabase::from_bytes(bytes).map_err(js_err)?) })}/// Open an empty database.pub fn empty() -> Result<SqliteMemoryDatabaseJs, JsValue> {- Ok(Self { inner: Some(MemoryDatabase::empty().map_err(error)?) })- }-- /// Open an empty database and run a schema script on it.- #[wasm_bindgen(js_name = withSchema)]- pub fn with_schema(schema_sql: &str) -> Result<SqliteMemoryDatabaseJs, JsValue> {- Ok(Self { inner: Some(MemoryDatabase::with_schema(schema_sql).map_err(error)?) })+ Ok(Self { inner: Some(MemoryDatabase::empty().map_err(js_err)?) })}/// Run a statement that returns no rows and report how many rows it changed.pub fn run(&self, sql: &str, params: JsValue) -> Result<f64, JsValue> {- Ok(self.db()?.execute(sql, ¶ms_from_js(params)?).map_err(error)? as f64)+ Ok(self.db()?.execute(sql, ¶ms_from_js(params)?).map_err(js_err)? as f64)}/// Run a statement and return its rows.pub fn query(&self, sql: &str, params: JsValue) -> Result<JsValue, JsValue> {- Ok(rows_to_js(&self.db()?.query_values(sql, ¶ms_from_js(params)?).map_err(error)?))+ Ok(rows_to_js(&self.db()?.query_values(sql, ¶ms_from_js(params)?).map_err(js_err)?))}/// Run one or more statements separated by semicolons, without parameters.pub fn exec(&self, sql: &str) -> Result<(), JsValue> {- self.db()?.execute_batch(sql).map_err(error)+ self.db()?.execute_batch(sql).map_err(js_err)}/// The database as SQLite file bytes.pub fn export(&self) -> Result<Vec<u8>, JsValue> {- self.db()?.export().map_err(error)+ self.db()?.export().map_err(js_err)}/// Close the database and free its memory.@@ -746,10 +536,6 @@ mod sqlite_js {}}- fn error(error: crate::VaultError) -> JsValue {- JsValue::from_str(&error.to_string())- }-fn params_from_js(params: JsValue) -> Result<Vec<SqlValue>, JsValue> {if params.is_undefined() || params.is_null() {return Ok(Vec::new());