Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update Rust codec to strip unnecessary incompressible guids from manifest storage model (#2159)
Leendert de Borstdb4179edce13
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rsindex 9e456c759..83c1b4965 100644--- a/core/rust/src/vault_codec/canonicalize.rs+++ b/core/rust/src/vault_codec/canonicalize.rs@@ -16,6 +16,7 @@ use base64::engine::general_purpose::STANDARD as BASE64;use base64::Engine;use serde_json::{json, Value};+use super::normalize::normalize_row_shapes;use super::hash::salted_blob_hash;use super::scoped_assets::{normalize_logo_scope, reconcile_logo_references};use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedManifest, CanonicalizedVault, CodecOverflow, DataBucket, Manifest, ManifestSpec, CodecRecord};@@ -92,6 +93,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonicnormalize_logo_scope(&mut all_tables, &writing_manifest_id);prune_unreferenced_logos(&mut all_tables);clone_referenced_rows(&mut all_tables, &writing_manifest_id, &snapshots);+ normalize_row_shapes(&mut all_tables);let mut blobs: HashMap<String, BlobEntry> = HashMap::new();let mut manifest_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();@@ -119,7 +121,8 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic});// Each remaining partition becomes its own manifest, its blobs hashed with its own per-manifest salt.- for partition in partitions {+ for mut partition in partitions {+ normalize_row_shapes(&mut partition.tables);let mut partition_blobs: HashMap<String, BlobEntry> = HashMap::new();let partition_tables: HashMap<String, Vec<CodecRecord>> = partition.tablesdiff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rsindex 50aabbda1..463cf69dd 100644--- a/core/rust/src/vault_codec/materialize.rs+++ b/core/rust/src/vault_codec/materialize.rs@@ -44,7 +44,11 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materializelet others: Vec<Manifest> = manifests;let base_manifest_id = base.manifest_id.clone();- let combined = super::sharing::combine_manifest_tables(base.tables, &base_manifest_id, others);+ let mut combined = super::sharing::combine_manifest_tables(base.tables, &base_manifest_id, others);++ // The wire omits derived row ids (single-value FieldValues, FieldHistories); mint them back so the+ // SQLite projection has the primary keys it expects back, identical on every device.+ super::normalize::mint_missing_derived_ids(&mut combined);let mut overflow = CodecOverflow::default();let mut tables: Vec<CodecTableData> = Vec::with_capacity(combined.len() + data_buckets.len());diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rsindex 9dff877b4..e333fc6fb 100644--- a/core/rust/src/vault_codec/mod.rs+++ b/core/rust/src/vault_codec/mod.rs@@ -9,6 +9,7 @@mod compress;mod canonicalize;+pub(crate) mod normalize;mod hash;mod manifest;mod materialize;@@ -33,7 +34,8 @@ pub use scoped_assets::{KIND_BUILTIN as LOGO_KIND_BUILTIN, KIND_CUSTOM as LOGO_Kpub use sharing::{active_encryption_key, extract_encryption_key_for_public_key};pub use types::{bucket_categories, bucket_category_for, is_bucketed_table, is_personal_table, tables_for_category, BLOB_COLUMNS, BUCKET_TABLES,- ENCRYPTION_KEYS_TABLE, MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_TABLE, PERSONAL_TABLES, SCHEMA_VERSION, SKIP_TABLES,+ ENCRYPTION_KEYS_TABLE, MANIFESTS_TABLE, MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS, OVERFLOW_TABLE, PERSONAL_TABLES, SCHEMA_VERSION,+ SKIP_TABLES,};pub use validate::ValidationResult;diff --git a/core/rust/src/vault_codec/normalize.rs b/core/rust/src/vault_codec/normalize.rsnew file mode 100644index 000000000..73711010b--- /dev/null+++ b/core/rust/src/vault_codec/normalize.rs@@ -0,0 +1,283 @@+//! Normalization of row shapes for converting from materialized SQLite to the manifest format to save on filesize.++use std::collections::{HashMap, HashSet};++use serde_json::json;++use super::manifest::CodecRecord;+use super::types::{MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS};++const FIELD_VALUES_TABLE: &str = "FieldValues";+const FIELD_HISTORIES_TABLE: &str = "FieldHistories";+const FIELD_DEFINITIONS_TABLE: &str = "FieldDefinitions";+const ITEM_TAGS_TABLE: &str = "ItemTags";+const ID_COL: &str = "Id";+const ITEM_ID_COL: &str = "ItemId";+const TAG_ID_COL: &str = "TagId";+const FIELD_KEY_COL: &str = "FieldKey";+const FIELD_DEFINITION_ID_COL: &str = "FieldDefinitionId";+const VALUE_INDEX_COL: &str = "ValueIndex";+const CHANGED_AT_COL: &str = "ChangedAt";+const IS_MULTI_VALUE_COL: &str = "IsMultiValue";++/// Domain-separation prefix for derived field value ids.+const FIELD_VALUE_ID_NAMESPACE: &str = "aliasvault:fieldvalue:v1";++/// Domain-separation prefix for derived field history ids.+const FIELD_HISTORY_ID_NAMESPACE: &str = "aliasvault:fieldhistory:v1";++/// The `FieldValues.Id` of the single-value row `(manifest, item, field, position)`: a UUIDv8 whose+/// bytes come from `sha256(namespace | manifest | item | field | position)`.+pub fn field_value_id_for(manifest_id: &str, item_id: &str, field_key: &str, field_definition_id: &str, value_index: i64) -> String {+ let field = if field_key.is_empty() { format!("fd:{}", field_definition_id.to_lowercase()) } else { format!("fk:{}", field_key.to_lowercase()) };+ super::hash::derived_uuid(&format!("{}\n{}\n{}\n{}\n{}", FIELD_VALUE_ID_NAMESPACE, manifest_id.to_lowercase(), item_id.to_lowercase(), field, value_index))+}++/// The `FieldHistories.Id` of the history row `(manifest, item, field, changed at)`: every row derives+/// it, since `ChangedAt` (millisecond precision) is the natural discriminator: two devices changing the+/// same field concurrently snapshot at different times and union. Two snapshots of one field in the very+/// same millisecond collapse to one, which history can afford. `changed_at` is used verbatim.+pub fn field_history_id_for(manifest_id: &str, item_id: &str, field_key: &str, field_definition_id: &str, changed_at: &str) -> String {+ let field = if field_key.is_empty() { format!("fd:{}", field_definition_id.to_lowercase()) } else { format!("fk:{}", field_key.to_lowercase()) };+ super::hash::derived_uuid(&format!("{}\n{}\n{}\n{}\n{}", FIELD_HISTORY_ID_NAMESPACE, manifest_id.to_lowercase(), item_id.to_lowercase(), field, changed_at))+}++/// Normalize the shape of rows for converting from materialized SQLite to the manifest format to save on filesize.+pub(crate) fn normalize_row_shapes(tables: &mut HashMap<String, Vec<CodecRecord>>) {+ let multi_value_defs = multi_value_definition_ids(tables);+ if let Some(rows) = tables.get_mut(FIELD_VALUES_TABLE) {+ normalize_field_values(rows, &multi_value_defs);+ }+ if let Some(rows) = tables.get_mut(FIELD_HISTORIES_TABLE) {+ normalize_field_histories(rows);+ }+ if let Some(rows) = tables.get_mut(ITEM_TAGS_TABLE) {+ normalize_item_tags(rows);+ }+}++/// The materialize direction: a FieldValues or FieldHistories row without an `Id` is a row whose+/// derived id the wire omits; mint it from the row's own natural key so every device materializes+/// the same SQLite row.+pub(crate) fn mint_missing_derived_ids(tables: &mut HashMap<String, Vec<CodecRecord>>) {+ if let Some(rows) = tables.get_mut(FIELD_VALUES_TABLE) {+ for row in rows.iter_mut().filter(|row| !has_id(row)) {+ let id = derive_row_id(row, value_index_of(row).unwrap_or(0));+ row.insert(ID_COL.to_string(), json!(id));+ }+ }+ if let Some(rows) = tables.get_mut(FIELD_HISTORIES_TABLE) {+ for row in rows.iter_mut().filter(|row| !has_id(row)) {+ let id = derive_history_row_id(row);+ row.insert(ID_COL.to_string(), json!(id));+ }+ }+}++/// The `(manifest, definition id)` pairs (lowercased) of every custom field definition marked multi-value.+fn multi_value_definition_ids(tables: &HashMap<String, Vec<CodecRecord>>) -> HashSet<(String, String)> {+ let Some(rows) = tables.get(FIELD_DEFINITIONS_TABLE) else { return HashSet::new() };+ rows.iter()+ .filter(|row| is_truthy(row.get(IS_MULTI_VALUE_COL)))+ .filter_map(|row| Some((lower_str(row, MANIFEST_ID_COL)?, lower_str(row, ID_COL)?)))+ .collect()+}++/// Renumber, collapse and re-id one FieldValues row set, per `(manifest, item, field)` group.+fn normalize_field_values(rows: &mut Vec<CodecRecord>, multi_value_defs: &HashSet<(String, String)>) {+ // Group row positions by natural key, preserving read order within each group.+ let mut groups: HashMap<(String, String, String), Vec<usize>> = HashMap::new();+ for (position, row) in rows.iter().enumerate() {+ let manifest = lower_str(row, MANIFEST_ID_COL).unwrap_or_default();+ let item = lower_str(row, ITEM_ID_COL).unwrap_or_default();+ groups.entry((manifest, item, field_discriminator(row))).or_default().push(position);+ }++ let mut removed: HashSet<usize> = HashSet::new();+ for ((manifest, _, _), mut positions) in groups {+ let multi_value = rows[positions[0]].get(FIELD_KEY_COL).and_then(|v| v.as_str()).is_some_and(|key| MULTI_VALUE_FIELD_KEYS.contains(&key.to_lowercase().as_str()))+ || lower_str(&rows[positions[0]], FIELD_DEFINITION_ID_COL).is_some_and(|def| multi_value_defs.contains(&(manifest.clone(), def)));++ if multi_value {+ // Stable order: declared position first (a row written before the column existed sorts last), read order breaks ties.+ positions.sort_by_key(|p| value_index_of(&rows[*p]).unwrap_or(i64::MAX));+ for (index, position) in positions.iter().enumerate() {+ let row = &mut rows[*position];+ row.insert(VALUE_INDEX_COL.to_string(), json!(index as i64));+ if !has_id(row) {+ let id = derive_row_id(row, index as i64);+ row.insert(ID_COL.to_string(), json!(id));+ }+ }+ } else {+ // A single-value field is one row; duplicates collapse to the newest UpdatedAt instead of+ // materializing into a primary-key violation (or a field the UI renders as an array).+ let winner = *positions.iter().max_by_key(|p| (crate::vault_merge::get_updated_at(&rows[**p]), std::cmp::Reverse(**p))).unwrap();+ removed.extend(positions.iter().filter(|p| **p != winner));+ let row = &mut rows[winner];+ row.insert(VALUE_INDEX_COL.to_string(), json!(0));+ row.remove(ID_COL);+ }+ }++ if !removed.is_empty() {+ let mut position = 0;+ rows.retain(|_| {+ let keep = !removed.contains(&position);+ position += 1;+ keep+ });+ }+}++/// Strip every history row's id (all of them derive it) and collapse rows sharing a natural key,+/// which takes two same-millisecond snapshots of one field, to the newest `UpdatedAt`.+fn normalize_field_histories(rows: &mut Vec<CodecRecord>) {+ for row in rows.iter_mut() {+ row.remove(ID_COL);+ }++ let mut winners: HashMap<(String, String, String, String), usize> = HashMap::new();+ for (position, row) in rows.iter().enumerate() {+ let changed_at = row.get(CHANGED_AT_COL).and_then(|v| v.as_str()).unwrap_or_default().to_string();+ let key = (lower_str(row, MANIFEST_ID_COL).unwrap_or_default(), lower_str(row, ITEM_ID_COL).unwrap_or_default(), field_discriminator(row), changed_at);+ match winners.get(&key) {+ // Ties keep the earlier row, matching the single-value collapse rule.+ Some(current) if crate::vault_merge::get_updated_at(row) <= crate::vault_merge::get_updated_at(&rows[*current]) => {}+ _ => {+ winners.insert(key, position);+ }+ }+ }++ retain_positions(rows, &winners.into_values().collect());+}++/// The derived id for a normalized FieldHistories row.+fn derive_history_row_id(row: &CodecRecord) -> String {+ let manifest = row.get(MANIFEST_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();+ let item = row.get(ITEM_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();+ let field_key = row.get(FIELD_KEY_COL).and_then(|v| v.as_str()).unwrap_or_default();+ let field_def = row.get(FIELD_DEFINITION_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();+ let changed_at = row.get(CHANGED_AT_COL).and_then(|v| v.as_str()).unwrap_or_default();+ field_history_id_for(manifest, item, field_key, field_def, changed_at)+}++/// Drop the legacy surrogate `Id` and collapse duplicates to the newest `UpdatedAt` per natural key.+fn normalize_item_tags(rows: &mut Vec<CodecRecord>) {+ for row in rows.iter_mut() {+ row.remove(ID_COL);+ }++ let mut winners: HashMap<(String, String, String), usize> = HashMap::new();+ for (position, row) in rows.iter().enumerate() {+ let key = (lower_str(row, MANIFEST_ID_COL).unwrap_or_default(), lower_str(row, ITEM_ID_COL).unwrap_or_default(), lower_str(row, TAG_ID_COL).unwrap_or_default());+ match winners.get(&key) {+ // Ties keep the earlier row, matching the single-value collapse rule.+ Some(current) if crate::vault_merge::get_updated_at(row) <= crate::vault_merge::get_updated_at(&rows[*current]) => {}+ _ => {+ winners.insert(key, position);+ }+ }+ }++ retain_positions(rows, &winners.into_values().collect());+}++/// Keep only the rows at `keep` positions, preserving order.+fn retain_positions(rows: &mut Vec<CodecRecord>, keep: &HashSet<usize>) {+ let mut position = 0;+ rows.retain(|_| {+ let kept = keep.contains(&position);+ position += 1;+ kept+ });+}++/// The derived id for a normalized FieldValues row at `value_index`.+fn derive_row_id(row: &CodecRecord, value_index: i64) -> String {+ let manifest = row.get(MANIFEST_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();+ let item = row.get(ITEM_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();+ let field_key = row.get(FIELD_KEY_COL).and_then(|v| v.as_str()).unwrap_or_default();+ let field_def = row.get(FIELD_DEFINITION_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();+ field_value_id_for(manifest, item, field_key, field_def, value_index)+}++/// The field half of a FieldValues row's natural key: `fk:<key>` for a system field, `fd:<id>` for a+/// custom field (exactly one is set; a row carrying neither groups under `fk:` and collapses).+fn field_discriminator(row: &CodecRecord) -> String {+ match row.get(FIELD_KEY_COL).and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {+ Some(key) => format!("fk:{}", key.to_lowercase()),+ None => match row.get(FIELD_DEFINITION_ID_COL).and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {+ Some(def) => format!("fd:{}", def.to_lowercase()),+ None => "fk:".to_string(),+ },+ }+}++fn has_id(row: &CodecRecord) -> bool {+ row.get(ID_COL).and_then(|v| v.as_str()).is_some_and(|s| !s.is_empty())+}++fn value_index_of(row: &CodecRecord) -> Option<i64> {+ match row.get(VALUE_INDEX_COL)? {+ serde_json::Value::Number(n) => n.as_i64(),+ serde_json::Value::String(s) => s.parse().ok(),+ _ => None,+ }+}++fn lower_str(row: &CodecRecord, column: &str) -> Option<String> {+ row.get(column).and_then(|v| v.as_str()).filter(|s| !s.is_empty()).map(str::to_lowercase)+}++fn is_truthy(value: Option<&serde_json::Value>) -> bool {+ match value {+ Some(serde_json::Value::Bool(b)) => *b,+ Some(serde_json::Value::Number(n)) => n.as_i64().unwrap_or(0) != 0,+ Some(serde_json::Value::String(s)) => s == "1" || s.eq_ignore_ascii_case("true"),+ _ => false,+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn derivation_is_stable_and_well_formed() {+ let id = field_value_id_for("m-1", "item-1", "login.username", "", 0);+ assert_eq!(id, field_value_id_for("m-1", "item-1", "login.username", "", 0));+ assert_eq!(id.len(), 36);+ assert_eq!(id.as_bytes()[14], b'8', "UUIDv8 version nibble");+ }++ #[test]+ fn derivation_ignores_guid_casing() {+ // iOS and the extension mint uppercase GUIDs; the natural key must not fork on casing.+ let lower = field_value_id_for("aaaa-bbbb", "item-x", "", "def-1", 2);+ let upper = field_value_id_for("AAAA-BBBB", "ITEM-X", "", "DEF-1", 2);+ assert_eq!(lower, upper);+ }++ #[test]+ fn derivation_separates_system_and_custom_fields() {+ // The same string as a FieldKey and as a FieldDefinitionId are different key spaces.+ assert_ne!(field_value_id_for("m", "i", "x", "", 0), field_value_id_for("m", "i", "", "x", 0));+ }++ #[test]+ fn derivation_separates_positions() {+ assert_ne!(field_value_id_for("m", "i", "login.url", "", 0), field_value_id_for("m", "i", "login.url", "", 1));+ }++ #[test]+ fn history_derivation_is_stable_and_disjoint_from_field_values() {+ let id = field_history_id_for("m-1", "item-1", "login.password", "", "2026-01-01 10:00:00.000");+ assert_eq!(id, field_history_id_for("m-1", "item-1", "login.password", "", "2026-01-01 10:00:00.000"));+ assert_eq!(id.len(), 36);+ // A different millisecond is a different row; the two namespaces never collide.+ assert_ne!(id, field_history_id_for("m-1", "item-1", "login.password", "", "2026-01-01 10:00:00.001"));+ assert_ne!(id, field_value_id_for("m-1", "item-1", "login.password", "", 0));+ }+}diff --git a/core/rust/src/vault_codec/sharing_tests.rs b/core/rust/src/vault_codec/sharing_tests.rsindex a4e100a80..1adeac2ba 100644--- a/core/rust/src/vault_codec/sharing_tests.rs+++ b/core/rust/src/vault_codec/sharing_tests.rs@@ -71,6 +71,23 @@ fn ids(records: &[CodecRecord]) -> Vec<&str> {out}+/// FieldValues rows are identified by their `Value` marker.+fn values(records: &[CodecRecord]) -> Vec<&str> {+ let mut out: Vec<&str> = records.iter().filter_map(|r| r.get("Value").and_then(|v| v.as_str())).collect();+ out.sort();+ out+}++/// ItemTags rows are identified by their natural key.+fn tag_links(records: &[CodecRecord]) -> Vec<(String, String)> {+ let mut out: Vec<(String, String)> = records+ .iter()+ .filter_map(|r| Some((r.get("ItemId")?.as_str()?.to_string(), r.get("TagId")?.as_str()?.to_string())))+ .collect();+ out.sort();+ out+}+/// The scoped logo id the codec derives for `(manifest id, source)`, what tests assert against, since/// a logo's identity is a function of its manifest and domain rather than whatever id the writer minted.fn logo_id(scope: &str, source: &str) -> String {@@ -271,8 +288,8 @@ fn split_routes_item_scoped_tables_generically() {let out = canonicalize_owner();let shared = &out.rest()[0].manifest;- assert_eq!(ids(rows(shared, "FieldValues")), vec!["fv-shared", "fv-sub"]);- assert_eq!(ids(rows(&out.first().manifest, "FieldValues")), vec!["fv-personal"]);+ assert_eq!(values(rows(shared, "FieldValues")), vec!["family", "hunter2"]);+ assert_eq!(values(rows(&out.first().manifest, "FieldValues")), vec!["me"]);assert_eq!(ids(rows(shared, "TotpCodes")), vec!["totp-shared"]);assert!(rows(&out.first().manifest, "TotpCodes").is_empty());@@ -280,8 +297,8 @@ fn split_routes_item_scoped_tables_generically() {assert_eq!(ids(rows(shared, "Attachments")), vec!["att-shared"]);assert_eq!(ids(rows(&out.first().manifest, "Attachments")), vec!["att-personal"]);- assert_eq!(ids(rows(shared, "ItemTags")), vec!["it-2", "it-3"]);- assert_eq!(ids(rows(&out.first().manifest, "ItemTags")), vec!["it-1"]);+ assert_eq!(tag_links(rows(shared, "ItemTags")), vec![("i-shared".into(), "tag-both".into()), ("i-sub".into(), "tag-shared-only".into())]);+ assert_eq!(tag_links(rows(&out.first().manifest, "ItemTags")), vec![("i-personal".into(), "tag-both".into())]);}#[test]@@ -802,7 +819,7 @@ fn split_with_a_single_spec_drops_rows_of_manifests_it_does_not_carry() {assert_eq!(ids(rows(&out.first().manifest, "Items")), vec!["i-nofolder", "i-personal"]);assert_eq!(ids(rows(&out.first().manifest, "Folders")), vec!["f-personal"]);// The rows hanging off a dropped item go with it rather than dangling.- assert_eq!(ids(rows(&out.first().manifest, "FieldValues")), vec!["fv-personal"]);+ assert_eq!(values(rows(&out.first().manifest, "FieldValues")), vec!["me"]);assert!(rows(&out.first().manifest, "TotpCodes").is_empty());// The serialized output carries the one manifest, with nothing flagging it and no folder anchor.@@ -890,10 +907,10 @@ fn owner_split_then_combine_roundtrips_to_original_tables() {assert_eq!(ids(&map["Items"]), vec!["i-nofolder", "i-personal", "i-shared", "i-sub"]);assert_eq!(ids(&map["Folders"]), vec!["f-personal", "f-shared", "f-sub"]);- assert_eq!(ids(&map["FieldValues"]), vec!["fv-personal", "fv-shared", "fv-sub"]);+ assert_eq!(values(&map["FieldValues"]), vec!["family", "hunter2", "me"]);assert_eq!(ids(&map["Attachments"]), vec!["att-personal", "att-shared"]);assert_eq!(ids(&map["TotpCodes"]), vec!["totp-shared"]);- assert_eq!(ids(&map["ItemTags"]), vec!["it-1", "it-2", "it-3"]);+ assert_eq!(tag_links(&map["ItemTags"]), vec![("i-personal".into(), "tag-both".into()), ("i-shared".into(), "tag-both".into()), ("i-sub".into(), "tag-shared-only".into())]);/** Tags and field definitions are manifest-scoped like the rest: the two the shared items carry* exist once in each manifest that uses them, side by side under `(ManifestId, Id)`. The base keeps@@ -971,7 +988,7 @@ fn recipient_combine_materializes_shared_manifest_into_their_vault() {// Recipient sees their own rows plus the shared subtree.assert_eq!(ids(&map["Folders"]), vec!["f-mine", "f-shared", "f-sub"]);assert_eq!(ids(&map["Items"]), vec!["i-mine", "i-shared", "i-sub"]);- assert_eq!(ids(&map["FieldValues"]), vec!["fv-shared", "fv-sub"]);+ assert_eq!(values(&map["FieldValues"]), vec!["family", "hunter2"]);/** The owner and the recipient both hold a netflix.com logo, legitimately different images (fetched* at different times, or one of them uploaded by hand). They must BOTH survive, in their own@@ -1021,7 +1038,7 @@ fn recipient_roundtrip_reproduces_shared_manifest_without_leaking_into_personal(let shared = &pushed.rest()[0].manifest;assert_eq!(ids(rows(shared, "Items")), vec!["i-shared", "i-sub"]);assert_eq!(ids(rows(shared, "Folders")), vec!["f-shared", "f-sub"]);- assert_eq!(ids(rows(shared, "FieldValues")), vec!["fv-shared", "fv-sub"]);+ assert_eq!(values(rows(shared, "FieldValues")), vec!["family", "hunter2"]);assert_eq!(ids(rows(shared, "Tags")), vec!["tag-both", "tag-shared-only"]);// Recipient's personal manifest holds only their own rows, no shared items/folders leaked in.@@ -1234,7 +1251,7 @@ fn combine_drops_child_rows_naming_an_item_the_manifest_does_not_hold() {let re = materialize_as_sqlite(materialize_input(personal, vec![hostile], vec![])).unwrap();let map = materialized_map(&re);- assert_eq!(ids(&map["FieldValues"]), vec!["fv-mine"], "the foreign row never reaches the reader's item");+ assert_eq!(values(&map["FieldValues"]), vec!["me"], "the foreign row never reaches the reader's item");}#[test]@@ -2047,8 +2064,8 @@ fn split_never_follows_a_bare_item_id_into_another_manifest() {assert!(manifest_mentions(personal, "personal_secret"), "and they must still be in the manifest they belong to");// Each manifest keeps exactly its own child rows, and the shared one keeps working.- assert_eq!(ids(rows(personal, "FieldValues")), vec!["fv-personal"]);- assert_eq!(ids(rows(shared, "FieldValues")), vec!["fv-shared"]);+ assert_eq!(values(rows(personal, "FieldValues")), vec!["personal_secret"]);+ assert_eq!(values(rows(shared, "FieldValues")), vec!["shared_secret"]);assert_eq!(ids(rows(personal, "TotpCodes")), vec!["totp-personal"]);assert_eq!(ids(rows(shared, "TotpCodes")), vec!["totp-shared"]);for name in ["Attachments", "Passkeys", "FieldHistories", "ItemTags"] {@@ -2088,7 +2105,7 @@ fn split_keeps_child_rows_when_a_manifest_this_vault_lost_shares_their_item_id()let out = canonicalize_from_sqlite(raw_input_with_shares(tables, vec![])).unwrap();let personal = &out.first().manifest;assert_eq!(ids(rows(personal, "Items")), vec!["i-dup"], "the revoked manifest's item is dropped, the live one kept");- assert_eq!(ids(rows(personal, "FieldValues")), vec!["fv-personal"], "the live item keeps its own child rows");+ assert_eq!(values(rows(personal, "FieldValues")), vec!["personal_secret"], "the live item keeps its own child rows");assert!(!manifest_mentions(personal, "gone"), "the revoked manifest's rows are not adopted into the personal one");}@@ -2105,7 +2122,7 @@ fn split_still_follows_an_item_whose_children_were_left_unstamped() {// `input_with_shares` stamps everything the fixture left unstamped as personal, child rows included.let shared = &out.rest()[0].manifest;- assert_eq!(ids(rows(shared, "FieldValues")), vec!["fv-shared", "fv-sub"], "children followed their item across the boundary");+ assert_eq!(values(rows(shared, "FieldValues")), vec!["family", "hunter2"], "children followed their item across the boundary");assert!(rows(shared, "FieldValues").iter().all(|r| r["ManifestId"] == json!("m-f-shared")), "and were re-stamped to agree with it");}diff --git a/core/rust/src/vault_codec/tests.rs b/core/rust/src/vault_codec/tests.rsindex a95307b1c..a4f2d680e 100644--- a/core/rust/src/vault_codec/tests.rs+++ b/core/rust/src/vault_codec/tests.rs@@ -100,9 +100,11 @@ fn every_bucketed_table_is_manifest_scoped_with_a_composite_identity() {table,category);+ let mut expected = vec![super::types::MANIFEST_ID_COL];+ expected.extend_from_slice(super::types::primary_key_columns_for(table));assert_eq!(super::types::identity_columns_for(table),- vec![super::types::MANIFEST_ID_COL, super::types::primary_key_for(table)],+ expected,"bucketed table {} must be addressed by (ManifestId, primary key)",table);@@ -837,3 +839,150 @@ pub(super) fn materialize_manifests(manifests: Vec<Manifest>, data_buckets: Vec<let schema = fitting_schema(manifests.iter(), &data_buckets);MaterializeInput { manifests, data_buckets, schema_columns: schema }}++// ---------------------------------------------------------------------------+// Derived ids + wire shape (see normalize.rs)+// ---------------------------------------------------------------------------++/// The FieldValues rows of the first canonicalized manifest, keyed by their Value marker.+fn field_values_by_value(out: &CanonicalizedVault) -> HashMap<String, CodecRecord> {+ out.first().manifest.tables["FieldValues"]+ .iter()+ .map(|r| (r["Value"].as_str().unwrap().to_string(), r.clone()))+ .collect()+}++#[test]+fn canonicalize_strips_derived_ids_and_renumbers_multi_value_rows() {+ let out = canonicalize_from_sqlite(basic_input(vec![+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i-1"))])] },+ CodecTableData {+ name: "FieldDefinitions".to_string(),+ records: vec![row(&[("Id", json!("fd-multi")), ("IsMultiValue", json!(1))]), row(&[("Id", json!("fd-single")), ("IsMultiValue", json!(0))])],+ },+ CodecTableData {+ name: "FieldValues".to_string(),+ records: vec![+ row(&[("Id", json!("fv-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("Value", json!("me"))]),+ row(&[("Id", json!("u-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("Value", json!("https://a.example"))]),+ row(&[("Id", json!("u-2")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("Value", json!("https://b.example"))]),+ row(&[("Id", json!("c-1")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-multi")), ("Value", json!("multi-a"))]),+ row(&[("Id", json!("c-2")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-multi")), ("Value", json!("multi-b"))]),+ row(&[("Id", json!("c-3")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-single")), ("Value", json!("single-c"))]),+ ],+ },+ CodecTableData { name: "Tags".to_string(), records: vec![row(&[("Id", json!("t-1"))])] },+ CodecTableData { name: "ItemTags".to_string(), records: vec![row(&[("Id", json!("it-legacy")), ("ItemId", json!("i-1")), ("TagId", json!("t-1"))])] },+ ]))+ .unwrap();++ let fv = field_values_by_value(&out);+ assert!(fv["me"].get("Id").is_none(), "a single-value system field's id never reaches the wire");+ assert_eq!(fv["me"]["ValueIndex"], json!(0));+ assert!(fv["single-c"].get("Id").is_none(), "a single-value custom field's id never reaches the wire");+ assert_eq!(fv["https://a.example"]["Id"], json!("u-1"), "a multi-value system field's value owns its id");+ assert_eq!(fv["https://b.example"]["Id"], json!("u-2"));+ assert_eq!((fv["https://a.example"]["ValueIndex"].as_i64().unwrap(), fv["https://b.example"]["ValueIndex"].as_i64().unwrap()), (0, 1), "values are renumbered in read order");+ assert_eq!(fv["multi-a"]["Id"], json!("c-1"), "a multi-value custom field's value owns its id");+ assert_eq!(fv["multi-b"]["Id"], json!("c-2"));++ let item_tags = &out.first().manifest.tables["ItemTags"];+ assert_eq!(item_tags.len(), 1);+ assert!(item_tags[0].get("Id").is_none(), "ItemTags carries no id at all");+}++#[test]+fn canonicalize_strips_field_history_ids() {+ let out = canonicalize_from_sqlite(basic_input(vec![+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i-1"))])] },+ CodecTableData {+ name: "FieldHistories".to_string(),+ records: vec![+ row(&[("Id", json!("fh-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-01-01 10:00:00.000")), ("ValueSnapshot", json!("old-pass"))]),+ row(&[("Id", json!("fh-2")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-02-01 10:00:00.000")), ("ValueSnapshot", json!("newer-pass"))]),+ ],+ },+ ]))+ .unwrap();++ let rows = &out.first().manifest.tables["FieldHistories"];+ assert_eq!(rows.len(), 2, "distinct ChangedAt values are distinct history rows");+ assert!(rows.iter().all(|r| r.get("Id").is_none()), "every history row derives its id; none reaches the wire");+}++#[test]+fn canonicalize_collapses_duplicate_single_value_rows_to_the_newest() {+ // Two rows for one single-value field (two local writers racing before their next sync) must+ // converge instead of reaching the wire as colliding rows.+ let out = canonicalize_from_sqlite(basic_input(vec![+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i-1"))])] },+ CodecTableData {+ name: "FieldValues".to_string(),+ records: vec![+ row(&[("Id", json!("fv-old")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("Value", json!("stale")), ("UpdatedAt", json!("2024-01-01 00:00:00.000"))]),+ row(&[("Id", json!("fv-new")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("Value", json!("fresh")), ("UpdatedAt", json!("2024-06-01 00:00:00.000"))]),+ ],+ },+ ]))+ .unwrap();++ let rows = &out.first().manifest.tables["FieldValues"];+ assert_eq!(rows.len(), 1);+ assert_eq!(rows[0]["Value"], json!("fresh"));+}++#[test]+fn materialize_mints_missing_field_value_ids() {+ let manifest = Manifest {+ schema_version: SCHEMA_VERSION,+ manifest_salt: "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff".to_string(),+ canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(),+ manifest_id: PERSONAL_MANIFEST.to_string(),+ name: None,+ tables: [+ ("Items".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Id", json!("i-1"))])]),+ (+ "FieldValues".to_string(),+ vec![+ // Single-value wire shape: no id, explicit position; the id is minted here.+ row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("ValueIndex", json!(0)), ("Value", json!("me"))]),+ // Multi-value: the owned id rides along untouched.+ row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Id", json!("u-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("ValueIndex", json!(0)), ("Value", json!("https://a.example"))]),+ ],+ ),+ ("Tags".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Id", json!("t-1"))])]),+ ("ItemTags".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("ItemId", json!("i-1")), ("TagId", json!("t-1"))])]),+ ("FieldHistories".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-01-01 10:00:00.000")), ("ValueSnapshot", json!("old-pass"))])]),+ ]+ .into_iter()+ .collect(),+ extra: HashMap::new(),+ };++ let schema: HashMap<String, Vec<String>> = [+ ("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string()]),+ ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string()]),+ ("Tags".to_string(), vec!["ManifestId".to_string(), "Id".to_string()]),+ ("ItemTags".to_string(), vec!["ManifestId".to_string(), "ItemId".to_string(), "TagId".to_string()]),+ ("FieldHistories".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ChangedAt".to_string(), "ValueSnapshot".to_string()]),+ (MANIFESTS_TABLE.to_string(), vec!["Id".to_string(), "Name".to_string()]),+ ]+ .into_iter()+ .collect();++ let out = materialize_as_sqlite(MaterializeInput { manifests: vec![manifest], data_buckets: vec![], schema_columns: schema }).unwrap();+ let tables: HashMap<&str, &Vec<CodecRecord>> = out.tables.iter().map(|t| (t.name.as_str(), &t.records)).collect();++ let fv: HashMap<&str, &CodecRecord> = tables["FieldValues"].iter().map(|r| (r["Value"].as_str().unwrap(), r)).collect();+ let expected_username_id = super::normalize::field_value_id_for(PERSONAL_MANIFEST, "i-1", "login.username", "", 0);+ assert_eq!(fv["me"]["Id"], json!(expected_username_id), "materialize mints the derived id");+ assert_eq!(fv["https://a.example"]["Id"], json!("u-1"), "an owned multi-value id is kept verbatim");++ assert_eq!(tables["ItemTags"].len(), 1, "the id-less join row inserts as-is");++ let history = &tables["FieldHistories"][0];+ let expected_history_id = super::normalize::field_history_id_for(PERSONAL_MANIFEST, "i-1", "login.password", "", "2026-01-01 10:00:00.000");+ assert_eq!(history["Id"], json!(expected_history_id), "materialize mints the derived history id");++ assert!(out.overflow.is_empty());+}diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rsindex c413fa2da..09a191347 100644--- a/core/rust/src/vault_codec/types.rs+++ b/core/rust/src/vault_codec/types.rs@@ -38,6 +38,10 @@ pub static BUCKET_TABLES: &[(&str, &str)] = &[/// the declaration point for a future personal-only table.pub static PERSONAL_TABLES: &[&str] = &[];+/// System field keys whose field holds multiple values. A value of such a field owns its row id+/// (two devices each adding a value are adding two different things) and are not derived.+pub static MULTI_VALUE_FIELD_KEYS: &[&str] = &["login.url"];+/// The per-manifest delivery-keypair table. Every manifest carries its own asymmetric keypair(s),/// stamped with that manifest's id (`ManifestId`).pub const ENCRYPTION_KEYS_TABLE: &str = "EncryptionKeys";@@ -118,9 +122,9 @@ pub fn is_bucketed_table(table_name: &str) -> bool {bucket_category_for(table_name).is_some()}-/// Get the primary key column for a table.-pub fn primary_key_for(table_name: &str) -> &'static str {- crate::vault_merge::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key).unwrap_or("Id")+/// Get the primary key columns for a table.+pub fn primary_key_columns_for(table_name: &str) -> &'static [&'static str] {+ crate::vault_merge::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key_columns).unwrap_or(&["Id"])}/// True when `table_name`'s rows are namespaced per manifest.@@ -148,8 +152,10 @@ pub fn row_identity(table_name: &str, row: &super::manifest::CodecRecord) -> Optif !columns.contains(&MANIFEST_ID_COL) && row.get(MANIFEST_ID_COL).filter(|value| !value.is_null()).is_some() {columns.insert(0, MANIFEST_ID_COL);}- // A row with no primary key cannot be addressed at all.- row.get(primary_key_for(table_name))?;+ // A row missing any primary key column cannot be addressed at all.+ for column in primary_key_columns_for(table_name) {+ row.get(*column)?;+ }Some(columns.iter()diff --git a/core/rust/src/vault_merge/canonical.rs b/core/rust/src/vault_merge/canonical.rsindex c834ef800..434c7c522 100644--- a/core/rust/src/vault_merge/canonical.rs+++ b/core/rust/src/vault_merge/canonical.rs@@ -142,6 +142,10 @@ fn merge_manifest_pair(}}+ // A union of concurrently added multi-value rows can leave two rows at the same ValueIndex;+ // re-normalizing the output renumbers them, so a merged manifest is normalized like any other.+ crate::vault_codec::normalize::normalize_row_shapes(&mut merged);+let mut buckets = unknown_server_buckets;for category in bucket_categories() {let mut bucket_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();@@ -200,7 +204,15 @@ fn merge_rows(stats: &mut MergeStats,) -> Vec<CodecRecord> {let identity_columns = config.identity_columns();- let match_columns: &[&str] = if config.uses_composite_key() { config.composite_key_columns } else { &identity_columns };+ // Rows arrive normalized to the wire shape here, so a canonical-only key may rely on stripped+ // derived ids (see the FieldValues registry comment).+ let match_columns: &[&str] = if !config.canonical_key_columns.is_empty() {+ config.canonical_key_columns+ } else if config.uses_composite_key() {+ config.composite_key_columns+ } else {+ &identity_columns+ };let known_columns: Option<HashSet<&str>> = schema_columns.get(config.name).map(|cols| cols.iter().map(String::as_str).collect());// Winner per match key among incoming rows; on duplicates the latest UpdatedAt wins.@@ -302,7 +314,8 @@ mod tests {("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Name".to_string(), "UpdatedAt".to_string()]),("Settings".to_string(), vec!["ManifestId".to_string(), "Key".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),("ItemStats".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "LastUsedAt".to_string(), "UpdatedAt".to_string()]),- ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),+ ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),+ ("ItemTags".to_string(), vec!["ManifestId".to_string(), "ItemId".to_string(), "TagId".to_string(), "UpdatedAt".to_string()]),("Attachments".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Blob".to_string(), "UpdatedAt".to_string()]),].into_iter()@@ -387,15 +400,15 @@ mod tests {}#[test]- fn field_values_keep_the_base_identity_on_a_semantic_match() {- // FieldValues match on (ManifestId, ItemId, FieldKey): a winning local row minted under a- // different Id updates the base row, which keeps its own Id.- let field_value = |id: &str, value: &str, updated_at: &str| -> CodecRecord {+ fn field_values_match_semantically_and_carry_no_surrogate_id() {+ // A single-value FieldValue arrives in wire shape (no Id, position 0) and matches on its+ // natural key: both sides converge to one row holding the winning value, still id-less.+ let field_value = |value: &str, updated_at: &str| -> CodecRecord {[("ManifestId".to_string(), json!(PERSONAL)),- ("Id".to_string(), json!(id)),("ItemId".to_string(), json!("item-1")),("FieldKey".to_string(), json!("username")),+ ("ValueIndex".to_string(), json!(0)),("Value".to_string(), json!(value)),("UpdatedAt".to_string(), json!(updated_at)),]@@ -403,20 +416,178 @@ mod tests {.collect()};+ let rows = merge_field_values(vec![field_value("old", "2024-01-01T00:00:00Z")], vec![field_value("new", "2024-01-09T00:00:00Z")]);+ assert_eq!(rows.len(), 1);+ assert!(rows[0].get("Id").is_none(), "a single-value row's id is derived at materialize, never carried on the wire");+ assert_eq!(rows[0]["Value"], json!("new"), "and the row carries the winning value");+ }++ /// A `login.url` FieldValues row: multi-value, so it owns its id on the wire.+ fn url_value(id: &str, index: i64, value: &str, updated_at: &str) -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(PERSONAL)),+ ("Id".to_string(), json!(id)),+ ("ItemId".to_string(), json!("item-1")),+ ("FieldKey".to_string(), json!("login.url")),+ ("ValueIndex".to_string(), json!(index)),+ ("Value".to_string(), json!(value)),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect()+ }++ fn merge_field_values(server: Vec<CodecRecord>, local: Vec<CodecRecord>) -> Vec<CodecRecord> {let output = merge_canonical(CanonicalMergeInput {- server_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), vec![field_value("fv-base", "old", "2024-01-01T00:00:00Z")])].into_iter().collect())],+ server_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), server)].into_iter().collect())],server_buckets: vec![],contentless_server_manifest_ids: vec![],- local_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), vec![field_value("fv-local", "new", "2024-01-09T00:00:00Z")])].into_iter().collect())],+ local_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), local)].into_iter().collect())],local_buckets: vec![],schema_columns: schema(),}).unwrap();+ output.manifests[0].manifest.tables["FieldValues"].clone()+ }- let rows = &output.manifests[0].manifest.tables["FieldValues"];+ #[test]+ fn multi_value_edit_converges_on_the_owned_id() {+ // The same owned id on both sides is the same row edited: LWW, one row out.+ let rows = merge_field_values(+ vec![url_value("url-1", 0, "https://a.example", "2024-01-01T00:00:00Z")],+ vec![url_value("url-1", 0, "https://a-edited.example", "2024-01-09T00:00:00Z")],+ );assert_eq!(rows.len(), 1);- assert_eq!(rows[0]["Id"], json!("fv-base"), "the base row keeps its own Id");- assert_eq!(rows[0]["Value"], json!("new"), "but carries the winning value");+ assert_eq!(rows[0]["Id"], json!("url-1"), "the row keeps its owned id");+ assert_eq!(rows[0]["Value"], json!("https://a-edited.example"));+ }++ #[test]+ fn multi_value_concurrent_adds_both_survive() {+ // Two devices each add a 2nd url while offline: both additions land at position 1 under+ // different owned ids. Position is not identity, so the union keeps both, renumbered.+ let rows = merge_field_values(+ vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-05T00:00:00Z")],+ vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-c", 1, "https://c.example", "2024-01-06T00:00:00Z")],+ );+ let mut values: Vec<&str> = rows.iter().map(|r| r["Value"].as_str().unwrap()).collect();+ values.sort();+ assert_eq!(values, vec!["https://a.example", "https://b.example", "https://c.example"]);+ let mut indexes: Vec<i64> = rows.iter().map(|r| r["ValueIndex"].as_i64().unwrap()).collect();+ indexes.sort();+ assert_eq!(indexes, vec![0, 1, 2], "colliding positions are renumbered in the merged output");+ }++ #[test]+ fn multi_value_reorder_does_not_duplicate() {+ // Reordering changes only ValueIndex; the owned id keeps the rows matched, so no row doubles.+ let rows = merge_field_values(+ vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-01T00:00:00Z")],+ vec![url_value("url-b", 0, "https://b.example", "2024-01-09T00:00:00Z"), url_value("url-a", 1, "https://a.example", "2024-01-09T00:00:00Z")],+ );+ assert_eq!(rows.len(), 2);+ let by_id: HashMap<&str, i64> = rows.iter().map(|r| (r["Id"].as_str().unwrap(), r["ValueIndex"].as_i64().unwrap())).collect();+ assert_eq!(by_id["url-b"], 0, "the reorder won");+ assert_eq!(by_id["url-a"], 1);+ }++ #[test]+ fn both_sides_holding_the_same_two_urls_stay_two_rows() {+ // The historic corruption case: before the id-based key, the 2nd url's value replaced the+ // 1st on an ordinary push. Identical sides must merge to exactly themselves.+ let two = vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-02T00:00:00Z")];+ let rows = merge_field_values(two.clone(), two);+ assert_eq!(rows.len(), 2);+ let by_id: HashMap<&str, &str> = rows.iter().map(|r| (r["Id"].as_str().unwrap(), r["Value"].as_str().unwrap())).collect();+ assert_eq!(by_id["url-a"], "https://a.example");+ assert_eq!(by_id["url-b"], "https://b.example");+ }++ #[test]+ fn custom_field_values_of_one_item_do_not_collapse() {+ // Custom fields carry no FieldKey (only a FieldDefinitionId); before FieldDefinitionId joined+ // the key they all collapsed onto one empty-string key.+ let custom = |def: &str, value: &str, updated_at: &str| -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(PERSONAL)),+ ("Id".to_string(), json!(format!("cv-{}", def))),+ ("ItemId".to_string(), json!("item-1")),+ ("FieldKey".to_string(), serde_json::Value::Null),+ ("FieldDefinitionId".to_string(), json!(def)),+ ("Value".to_string(), json!(value)),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect()+ };+ let rows = merge_field_values(+ vec![custom("fd-pin", "1234", "2024-01-01T00:00:00Z"), custom("fd-member", "M-77", "2024-01-01T00:00:00Z")],+ vec![custom("fd-pin", "9999", "2024-01-09T00:00:00Z"), custom("fd-member", "M-77", "2024-01-01T00:00:00Z")],+ );+ assert_eq!(rows.len(), 2, "each custom field keeps its own row");+ let mut values: Vec<&str> = rows.iter().map(|r| r["Value"].as_str().unwrap()).collect();+ values.sort();+ assert_eq!(values, vec!["9999", "M-77"]);+ }++ #[test]+ fn field_histories_union_by_changed_at_and_converge_on_the_same_snapshot() {+ // History rows derive their id from (item, field, ChangedAt): concurrent changes on two+ // devices carry different timestamps and union; the same snapshot on both sides is one row.+ let history = |changed_at: &str, snapshot: &str| -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(PERSONAL)),+ ("ItemId".to_string(), json!("item-1")),+ ("FieldKey".to_string(), json!("login.password")),+ ("ChangedAt".to_string(), json!(changed_at)),+ ("ValueSnapshot".to_string(), json!(snapshot)),+ ("UpdatedAt".to_string(), json!(changed_at)),+ ]+ .into_iter()+ .collect()+ };+ let shared = history("2024-01-01 10:00:00.000", "both-know-this");+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![manifest(PERSONAL, [("FieldHistories".to_string(), vec![shared.clone(), history("2024-02-01 10:00:00.000", "server-only")])].into_iter().collect())],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![manifest(PERSONAL, [("FieldHistories".to_string(), vec![shared, history("2024-02-01 11:30:00.000", "local-only")])].into_iter().collect())],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();+ let rows = &output.manifests[0].manifest.tables["FieldHistories"];+ let mut snapshots: Vec<&str> = rows.iter().map(|r| r["ValueSnapshot"].as_str().unwrap()).collect();+ snapshots.sort();+ assert_eq!(snapshots, vec!["both-know-this", "local-only", "server-only"]);+ assert!(rows.iter().all(|r| r.get("Id").is_none()), "merged history rows stay id-less on the wire");+ }++ #[test]+ fn item_tags_converge_on_their_natural_key() {+ // Two devices tag the same item with the same tag: the id-less join rows are one row.+ let tag_row = |updated_at: &str| -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(PERSONAL)),+ ("ItemId".to_string(), json!("item-1")),+ ("TagId".to_string(), json!("tag-1")),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect()+ };+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![manifest(PERSONAL, [("ItemTags".to_string(), vec![tag_row("2024-01-01T00:00:00Z")])].into_iter().collect())],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![manifest(PERSONAL, [("ItemTags".to_string(), vec![tag_row("2024-01-09T00:00:00Z")])].into_iter().collect())],+ local_buckets: vec![],+ schema_columns: schema(),+ })+ .unwrap();+ let rows = &output.manifests[0].manifest.tables["ItemTags"];+ assert_eq!(rows.len(), 1, "the same (item, tag) link is one row on both sides");+ assert!(rows[0].get("Id").is_none());}#[test]diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rsindex 01f736ff7..c61012e1e 100644--- a/core/rust/src/vault_merge/mod.rs+++ b/core/rust/src/vault_merge/mod.rs@@ -285,7 +285,7 @@ fn key_part(value: &serde_json::Value) -> String {/// Get the UpdatedAt timestamp from a record./// Handles both RFC3339 format (2025-12-11T06:50:10.674Z) and/// SQLite format (2025-12-11 06:50:10.674).-fn get_updated_at(record: &Record) -> Option<DateTime<Utc>> {+pub(crate) fn get_updated_at(record: &Record) -> Option<DateTime<Utc>> {record.get("UpdatedAt").and_then(|v| v.as_str())@@ -641,11 +641,11 @@ mod tests {#[test]fn settings_and_encryption_keys_are_syncable() {let settings = SYNCABLE_TABLES.iter().find(|t| t.name == "Settings").unwrap();- assert_eq!(settings.primary_key, "Key");+ assert_eq!(settings.primary_key_columns, &["Key"]);assert!(!settings.uses_composite_key(), "a key already names one setting per manifest; no semantic match key is needed");let keys = SYNCABLE_TABLES.iter().find(|t| t.name == "EncryptionKeys").unwrap();- assert_eq!(keys.primary_key, "Id");+ assert_eq!(keys.primary_key_columns, &["Id"]);assert!(SYNCABLE_TABLE_NAMES.contains(&"Settings"));assert!(SYNCABLE_TABLE_NAMES.contains(&"EncryptionKeys"));@@ -705,7 +705,7 @@ mod tests {// never go back to composite-key matching on Source.let cfg = SYNCABLE_TABLES.iter().find(|t| t.name == "Logos").unwrap();assert!(!cfg.uses_composite_key());- assert_eq!(cfg.primary_key, "Id");+ assert_eq!(cfg.primary_key_columns, &["Id"]);}#[test]@@ -744,7 +744,9 @@ mod tests {fn scoped_row(config: &TableConfig, manifest_id: &str, updated_at: &str, payload: &str) -> Record {let mut record: Record = HashMap::new();record.insert(crate::vault_codec::MANIFEST_ID_COL.to_string(), serde_json::json!(manifest_id));- record.insert(config.primary_key.to_string(), serde_json::json!("row-1"));+ for column in config.primary_key_columns {+ record.insert((*column).to_string(), serde_json::json!("row-1"));+ }for column in config.composite_key_columns {record.entry((*column).to_string()).or_insert_with(|| serde_json::json!("k"));}@@ -804,7 +806,8 @@ mod tests {let set_clause = sql.trim_start_matches(&prefix).split(" WHERE ").next().unwrap();assert!(!set_clause.contains(crate::vault_codec::MANIFEST_ID_COL), "{}: an update must never rewrite the row's manifest ({})", config.name, sql);- assert!(sql.ends_with(&format!("WHERE ManifestId = ? AND {} = ?", config.primary_key)), "{}: the base row is addressed by its full identity ({})", config.name, sql);+ let expected_where = format!("WHERE {}", config.identity_columns().iter().map(|c| format!("{} = ?", c)).collect::<Vec<_>>().join(" AND "));+ assert!(sql.ends_with(&expected_where), "{}: the base row is addressed by its full identity ({})", config.name, sql);}}diff --git a/core/rust/src/vault_merge/types.rs b/core/rust/src/vault_merge/types.rsindex 53402c77c..0c995cc6b 100644--- a/core/rust/src/vault_merge/types.rs+++ b/core/rust/src/vault_merge/types.rs@@ -11,9 +11,13 @@ pub struct TableConfig {/// When empty, matching falls back to the table's [`identity_columns`](TableConfig::identity_columns)./// When set, these columns are concatenated to form the composite key.pub composite_key_columns: &'static [&'static str],- /// The column that names a row within its manifest. Defaults to "Id"; tables keyed by a- /// different column (e.g. Settings keyed by "Key") override this.- pub primary_key: &'static str,+ /// Match columns for the canonical merge only, where both sides are first normalized to the manifest+ /// shape (see `vault_codec::normalize`). Empty means the canonical merge uses the same rule as+ /// the statement merge (composite key, else identity).+ pub canonical_key_columns: &'static [&'static str],+ /// The columns that name a row within its manifest. Defaults to `["Id"]`; tables keyed+ /// differently (Settings by "Key", ItemTags by its natural key) override this.+ pub primary_key_columns: &'static [&'static str],/// True when the table's rows are namespaced per manifest.pub manifest_scoped: bool,}@@ -23,7 +27,8 @@ impl TableConfig {Self {name,composite_key_columns: &[],- primary_key: "Id",+ canonical_key_columns: &[],+ primary_key_columns: &["Id"],manifest_scoped: false,}}@@ -33,8 +38,13 @@ impl TableConfig {self}- pub const fn with_primary_key(mut self, column: &'static str) -> Self {- self.primary_key = column;+ pub const fn with_canonical_key(mut self, columns: &'static [&'static str]) -> Self {+ self.canonical_key_columns = columns;+ self+ }++ pub const fn with_primary_key(mut self, columns: &'static [&'static str]) -> Self {+ self.primary_key_columns = columns;self}@@ -48,14 +58,16 @@ impl TableConfig {!self.composite_key_columns.is_empty()}- /// The columns that together identify one row of this table: `(ManifestId, primary_key)` for a- /// manifest-scoped table, the primary key alone otherwise. This is what a generated UPDATE- /// addresses a row by, so it must match the table's declared PRIMARY KEY.+ /// The columns that together identify one row of this table: `(ManifestId, primary key columns)`+ /// for a manifest-scoped table, the primary key columns alone otherwise.pub fn identity_columns(&self) -> Vec<&'static str> {if self.manifest_scoped {- vec![MANIFEST_ID_COL, self.primary_key]+ let mut columns = Vec::with_capacity(1 + self.primary_key_columns.len());+ columns.push(MANIFEST_ID_COL);+ columns.extend_from_slice(self.primary_key_columns);+ columns} else {- vec![self.primary_key]+ self.primary_key_columns.to_vec()}}}@@ -67,18 +79,33 @@ pub static SYNCABLE_TABLES: &[TableConfig] = &[// Keyed by the item it describes: `Id` *is* the item's id, so recording a use is an upsert and two// devices never mint competing rows. Listed after Items so a merge inserts the item first.TableConfig::new("ItemStats").manifest_scoped(),- TableConfig::new("FieldValues").manifest_scoped().with_composite_key(&[MANIFEST_ID_COL, "ItemId", "FieldKey"]),+ // Legacy statement merge: a field value matches on the field it belongs to (FieldKey for system+ // fields, FieldDefinitionId for custom ones; exactly one is set), so independently minted rows of+ // the same field converge. Canonical merge: both sides are normalized first, which strips the+ // derived id of every single-value row, so adding `Id` to the key makes a single-value row match+ // by its field (id part empty on both sides) while a multi-value row matches by its OWNED id:+ // two devices each adding a `login.url` are two different rows that must both survive, and the id,+ // unlike ValueIndex, is stable under reordering. See vault_codec::normalize.+ TableConfig::new("FieldValues")+ .manifest_scoped()+ .with_composite_key(&[MANIFEST_ID_COL, "ItemId", "FieldKey", "FieldDefinitionId"])+ .with_canonical_key(&[MANIFEST_ID_COL, "ItemId", "FieldKey", "FieldDefinitionId", "Id"]),TableConfig::new("Folders").manifest_scoped(),TableConfig::new("Tags").manifest_scoped(),- TableConfig::new("ItemTags").manifest_scoped(),+ // A pure join table keyed by its natural key; it carries no surrogate id (since client schema 2.1.6),+ // so two devices tagging the same item converge on one row.+ TableConfig::new("ItemTags").manifest_scoped().with_primary_key(&["ItemId", "TagId"]),TableConfig::new("Attachments").manifest_scoped(),TableConfig::new("TotpCodes").manifest_scoped(),TableConfig::new("Passkeys").manifest_scoped(),TableConfig::new("FieldDefinitions").manifest_scoped(),- TableConfig::new("FieldHistories").manifest_scoped(),+ // Canonical merge: every history row derives its id from `(item, field, ChangedAt)`, so after+ // normalization the natural key IS the identity: concurrent changes union (distinct ChangedAt),+ // same-millisecond snapshots converge. Legacy statement merge keeps plain (ManifestId, Id).+ TableConfig::new("FieldHistories").manifest_scoped().with_canonical_key(&[MANIFEST_ID_COL, "ItemId", "FieldKey", "FieldDefinitionId", "ChangedAt"]),TableConfig::new("Logos").manifest_scoped(),TableConfig::new("EncryptionKeys").manifest_scoped(),- TableConfig::new("Settings").with_primary_key("Key").manifest_scoped(),+ TableConfig::new("Settings").with_primary_key(&["Key"]).manifest_scoped(),];/// The tables a platform must read into the merge input: the syncable tables.