Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update Rust codec to strip unnecessary incompressible guids from manifest storage model (#2159)

Leendert de Borstdb4179edce13
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rs
index 9e456c759..83c1b4965 100644
--- a/core/rust/src/vault_codec/canonicalize.rs
+++ b/core/rust/src/vault_codec/canonicalize.rs
@@ -16,6 +16,7 @@ use base64::engine::general_purpose::STANDARD as BASE64;
use base64::Engine;
use serde_json::{json, Value};
+use super::normalize::normalize_row_shapes;
use super::hash::salted_blob_hash;
use super::scoped_assets::{normalize_logo_scope, reconcile_logo_references};
use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedManifest, CanonicalizedVault, CodecOverflow, DataBucket, Manifest, ManifestSpec, CodecRecord};
@@ -92,6 +93,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
normalize_logo_scope(&mut all_tables, &writing_manifest_id);
prune_unreferenced_logos(&mut all_tables);
clone_referenced_rows(&mut all_tables, &writing_manifest_id, &snapshots);
+ normalize_row_shapes(&mut all_tables);
let mut blobs: HashMap<String, BlobEntry> = HashMap::new();
let mut manifest_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();
@@ -119,7 +121,8 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
});
// Each remaining partition becomes its own manifest, its blobs hashed with its own per-manifest salt.
- for partition in partitions {
+ for mut partition in partitions {
+ normalize_row_shapes(&mut partition.tables);
let mut partition_blobs: HashMap<String, BlobEntry> = HashMap::new();
let partition_tables: HashMap<String, Vec<CodecRecord>> = partition
.tables
diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rs
index 50aabbda1..463cf69dd 100644
--- a/core/rust/src/vault_codec/materialize.rs
+++ b/core/rust/src/vault_codec/materialize.rs
@@ -44,7 +44,11 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
let others: Vec<Manifest> = manifests;
let base_manifest_id = base.manifest_id.clone();
- let combined = super::sharing::combine_manifest_tables(base.tables, &base_manifest_id, others);
+ let mut combined = super::sharing::combine_manifest_tables(base.tables, &base_manifest_id, others);
+
+ // The wire omits derived row ids (single-value FieldValues, FieldHistories); mint them back so the
+ // SQLite projection has the primary keys it expects back, identical on every device.
+ super::normalize::mint_missing_derived_ids(&mut combined);
let mut overflow = CodecOverflow::default();
let mut tables: Vec<CodecTableData> = Vec::with_capacity(combined.len() + data_buckets.len());
diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rs
index 9dff877b4..e333fc6fb 100644
--- a/core/rust/src/vault_codec/mod.rs
+++ b/core/rust/src/vault_codec/mod.rs
@@ -9,6 +9,7 @@
mod compress;
mod canonicalize;
+pub(crate) mod normalize;
mod hash;
mod manifest;
mod materialize;
@@ -33,7 +34,8 @@ pub use scoped_assets::{KIND_BUILTIN as LOGO_KIND_BUILTIN, KIND_CUSTOM as LOGO_K
pub use sharing::{active_encryption_key, extract_encryption_key_for_public_key};
pub use types::{
bucket_categories, bucket_category_for, is_bucketed_table, is_personal_table, tables_for_category, BLOB_COLUMNS, BUCKET_TABLES,
- ENCRYPTION_KEYS_TABLE, MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_TABLE, PERSONAL_TABLES, SCHEMA_VERSION, SKIP_TABLES,
+ ENCRYPTION_KEYS_TABLE, MANIFESTS_TABLE, MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS, OVERFLOW_TABLE, PERSONAL_TABLES, SCHEMA_VERSION,
+ SKIP_TABLES,
};
pub use validate::ValidationResult;
diff --git a/core/rust/src/vault_codec/normalize.rs b/core/rust/src/vault_codec/normalize.rs
new file mode 100644
index 000000000..73711010b
--- /dev/null
+++ b/core/rust/src/vault_codec/normalize.rs
@@ -0,0 +1,283 @@
+//! Normalization of row shapes for converting from materialized SQLite to the manifest format to save on filesize.
+
+use std::collections::{HashMap, HashSet};
+
+use serde_json::json;
+
+use super::manifest::CodecRecord;
+use super::types::{MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS};
+
+const FIELD_VALUES_TABLE: &str = "FieldValues";
+const FIELD_HISTORIES_TABLE: &str = "FieldHistories";
+const FIELD_DEFINITIONS_TABLE: &str = "FieldDefinitions";
+const ITEM_TAGS_TABLE: &str = "ItemTags";
+const ID_COL: &str = "Id";
+const ITEM_ID_COL: &str = "ItemId";
+const TAG_ID_COL: &str = "TagId";
+const FIELD_KEY_COL: &str = "FieldKey";
+const FIELD_DEFINITION_ID_COL: &str = "FieldDefinitionId";
+const VALUE_INDEX_COL: &str = "ValueIndex";
+const CHANGED_AT_COL: &str = "ChangedAt";
+const IS_MULTI_VALUE_COL: &str = "IsMultiValue";
+
+/// Domain-separation prefix for derived field value ids.
+const FIELD_VALUE_ID_NAMESPACE: &str = "aliasvault:fieldvalue:v1";
+
+/// Domain-separation prefix for derived field history ids.
+const FIELD_HISTORY_ID_NAMESPACE: &str = "aliasvault:fieldhistory:v1";
+
+/// The `FieldValues.Id` of the single-value row `(manifest, item, field, position)`: a UUIDv8 whose
+/// bytes come from `sha256(namespace | manifest | item | field | position)`.
+pub fn field_value_id_for(manifest_id: &str, item_id: &str, field_key: &str, field_definition_id: &str, value_index: i64) -> String {
+ let field = if field_key.is_empty() { format!("fd:{}", field_definition_id.to_lowercase()) } else { format!("fk:{}", field_key.to_lowercase()) };
+ super::hash::derived_uuid(&format!("{}\n{}\n{}\n{}\n{}", FIELD_VALUE_ID_NAMESPACE, manifest_id.to_lowercase(), item_id.to_lowercase(), field, value_index))
+}
+
+/// The `FieldHistories.Id` of the history row `(manifest, item, field, changed at)`: every row derives
+/// it, since `ChangedAt` (millisecond precision) is the natural discriminator: two devices changing the
+/// same field concurrently snapshot at different times and union. Two snapshots of one field in the very
+/// same millisecond collapse to one, which history can afford. `changed_at` is used verbatim.
+pub fn field_history_id_for(manifest_id: &str, item_id: &str, field_key: &str, field_definition_id: &str, changed_at: &str) -> String {
+ let field = if field_key.is_empty() { format!("fd:{}", field_definition_id.to_lowercase()) } else { format!("fk:{}", field_key.to_lowercase()) };
+ super::hash::derived_uuid(&format!("{}\n{}\n{}\n{}\n{}", FIELD_HISTORY_ID_NAMESPACE, manifest_id.to_lowercase(), item_id.to_lowercase(), field, changed_at))
+}
+
+/// Normalize the shape of rows for converting from materialized SQLite to the manifest format to save on filesize.
+pub(crate) fn normalize_row_shapes(tables: &mut HashMap<String, Vec<CodecRecord>>) {
+ let multi_value_defs = multi_value_definition_ids(tables);
+ if let Some(rows) = tables.get_mut(FIELD_VALUES_TABLE) {
+ normalize_field_values(rows, &multi_value_defs);
+ }
+ if let Some(rows) = tables.get_mut(FIELD_HISTORIES_TABLE) {
+ normalize_field_histories(rows);
+ }
+ if let Some(rows) = tables.get_mut(ITEM_TAGS_TABLE) {
+ normalize_item_tags(rows);
+ }
+}
+
+/// The materialize direction: a FieldValues or FieldHistories row without an `Id` is a row whose
+/// derived id the wire omits; mint it from the row's own natural key so every device materializes
+/// the same SQLite row.
+pub(crate) fn mint_missing_derived_ids(tables: &mut HashMap<String, Vec<CodecRecord>>) {
+ if let Some(rows) = tables.get_mut(FIELD_VALUES_TABLE) {
+ for row in rows.iter_mut().filter(|row| !has_id(row)) {
+ let id = derive_row_id(row, value_index_of(row).unwrap_or(0));
+ row.insert(ID_COL.to_string(), json!(id));
+ }
+ }
+ if let Some(rows) = tables.get_mut(FIELD_HISTORIES_TABLE) {
+ for row in rows.iter_mut().filter(|row| !has_id(row)) {
+ let id = derive_history_row_id(row);
+ row.insert(ID_COL.to_string(), json!(id));
+ }
+ }
+}
+
+/// The `(manifest, definition id)` pairs (lowercased) of every custom field definition marked multi-value.
+fn multi_value_definition_ids(tables: &HashMap<String, Vec<CodecRecord>>) -> HashSet<(String, String)> {
+ let Some(rows) = tables.get(FIELD_DEFINITIONS_TABLE) else { return HashSet::new() };
+ rows.iter()
+ .filter(|row| is_truthy(row.get(IS_MULTI_VALUE_COL)))
+ .filter_map(|row| Some((lower_str(row, MANIFEST_ID_COL)?, lower_str(row, ID_COL)?)))
+ .collect()
+}
+
+/// Renumber, collapse and re-id one FieldValues row set, per `(manifest, item, field)` group.
+fn normalize_field_values(rows: &mut Vec<CodecRecord>, multi_value_defs: &HashSet<(String, String)>) {
+ // Group row positions by natural key, preserving read order within each group.
+ let mut groups: HashMap<(String, String, String), Vec<usize>> = HashMap::new();
+ for (position, row) in rows.iter().enumerate() {
+ let manifest = lower_str(row, MANIFEST_ID_COL).unwrap_or_default();
+ let item = lower_str(row, ITEM_ID_COL).unwrap_or_default();
+ groups.entry((manifest, item, field_discriminator(row))).or_default().push(position);
+ }
+
+ let mut removed: HashSet<usize> = HashSet::new();
+ for ((manifest, _, _), mut positions) in groups {
+ let multi_value = rows[positions[0]].get(FIELD_KEY_COL).and_then(|v| v.as_str()).is_some_and(|key| MULTI_VALUE_FIELD_KEYS.contains(&key.to_lowercase().as_str()))
+ || lower_str(&rows[positions[0]], FIELD_DEFINITION_ID_COL).is_some_and(|def| multi_value_defs.contains(&(manifest.clone(), def)));
+
+ if multi_value {
+ // Stable order: declared position first (a row written before the column existed sorts last), read order breaks ties.
+ positions.sort_by_key(|p| value_index_of(&rows[*p]).unwrap_or(i64::MAX));
+ for (index, position) in positions.iter().enumerate() {
+ let row = &mut rows[*position];
+ row.insert(VALUE_INDEX_COL.to_string(), json!(index as i64));
+ if !has_id(row) {
+ let id = derive_row_id(row, index as i64);
+ row.insert(ID_COL.to_string(), json!(id));
+ }
+ }
+ } else {
+ // A single-value field is one row; duplicates collapse to the newest UpdatedAt instead of
+ // materializing into a primary-key violation (or a field the UI renders as an array).
+ let winner = *positions.iter().max_by_key(|p| (crate::vault_merge::get_updated_at(&rows[**p]), std::cmp::Reverse(**p))).unwrap();
+ removed.extend(positions.iter().filter(|p| **p != winner));
+ let row = &mut rows[winner];
+ row.insert(VALUE_INDEX_COL.to_string(), json!(0));
+ row.remove(ID_COL);
+ }
+ }
+
+ if !removed.is_empty() {
+ let mut position = 0;
+ rows.retain(|_| {
+ let keep = !removed.contains(&position);
+ position += 1;
+ keep
+ });
+ }
+}
+
+/// Strip every history row's id (all of them derive it) and collapse rows sharing a natural key,
+/// which takes two same-millisecond snapshots of one field, to the newest `UpdatedAt`.
+fn normalize_field_histories(rows: &mut Vec<CodecRecord>) {
+ for row in rows.iter_mut() {
+ row.remove(ID_COL);
+ }
+
+ let mut winners: HashMap<(String, String, String, String), usize> = HashMap::new();
+ for (position, row) in rows.iter().enumerate() {
+ let changed_at = row.get(CHANGED_AT_COL).and_then(|v| v.as_str()).unwrap_or_default().to_string();
+ let key = (lower_str(row, MANIFEST_ID_COL).unwrap_or_default(), lower_str(row, ITEM_ID_COL).unwrap_or_default(), field_discriminator(row), changed_at);
+ match winners.get(&key) {
+ // Ties keep the earlier row, matching the single-value collapse rule.
+ Some(current) if crate::vault_merge::get_updated_at(row) <= crate::vault_merge::get_updated_at(&rows[*current]) => {}
+ _ => {
+ winners.insert(key, position);
+ }
+ }
+ }
+
+ retain_positions(rows, &winners.into_values().collect());
+}
+
+/// The derived id for a normalized FieldHistories row.
+fn derive_history_row_id(row: &CodecRecord) -> String {
+ let manifest = row.get(MANIFEST_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();
+ let item = row.get(ITEM_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();
+ let field_key = row.get(FIELD_KEY_COL).and_then(|v| v.as_str()).unwrap_or_default();
+ let field_def = row.get(FIELD_DEFINITION_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();
+ let changed_at = row.get(CHANGED_AT_COL).and_then(|v| v.as_str()).unwrap_or_default();
+ field_history_id_for(manifest, item, field_key, field_def, changed_at)
+}
+
+/// Drop the legacy surrogate `Id` and collapse duplicates to the newest `UpdatedAt` per natural key.
+fn normalize_item_tags(rows: &mut Vec<CodecRecord>) {
+ for row in rows.iter_mut() {
+ row.remove(ID_COL);
+ }
+
+ let mut winners: HashMap<(String, String, String), usize> = HashMap::new();
+ for (position, row) in rows.iter().enumerate() {
+ let key = (lower_str(row, MANIFEST_ID_COL).unwrap_or_default(), lower_str(row, ITEM_ID_COL).unwrap_or_default(), lower_str(row, TAG_ID_COL).unwrap_or_default());
+ match winners.get(&key) {
+ // Ties keep the earlier row, matching the single-value collapse rule.
+ Some(current) if crate::vault_merge::get_updated_at(row) <= crate::vault_merge::get_updated_at(&rows[*current]) => {}
+ _ => {
+ winners.insert(key, position);
+ }
+ }
+ }
+
+ retain_positions(rows, &winners.into_values().collect());
+}
+
+/// Keep only the rows at `keep` positions, preserving order.
+fn retain_positions(rows: &mut Vec<CodecRecord>, keep: &HashSet<usize>) {
+ let mut position = 0;
+ rows.retain(|_| {
+ let kept = keep.contains(&position);
+ position += 1;
+ kept
+ });
+}
+
+/// The derived id for a normalized FieldValues row at `value_index`.
+fn derive_row_id(row: &CodecRecord, value_index: i64) -> String {
+ let manifest = row.get(MANIFEST_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();
+ let item = row.get(ITEM_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();
+ let field_key = row.get(FIELD_KEY_COL).and_then(|v| v.as_str()).unwrap_or_default();
+ let field_def = row.get(FIELD_DEFINITION_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();
+ field_value_id_for(manifest, item, field_key, field_def, value_index)
+}
+
+/// The field half of a FieldValues row's natural key: `fk:<key>` for a system field, `fd:<id>` for a
+/// custom field (exactly one is set; a row carrying neither groups under `fk:` and collapses).
+fn field_discriminator(row: &CodecRecord) -> String {
+ match row.get(FIELD_KEY_COL).and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
+ Some(key) => format!("fk:{}", key.to_lowercase()),
+ None => match row.get(FIELD_DEFINITION_ID_COL).and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {
+ Some(def) => format!("fd:{}", def.to_lowercase()),
+ None => "fk:".to_string(),
+ },
+ }
+}
+
+fn has_id(row: &CodecRecord) -> bool {
+ row.get(ID_COL).and_then(|v| v.as_str()).is_some_and(|s| !s.is_empty())
+}
+
+fn value_index_of(row: &CodecRecord) -> Option<i64> {
+ match row.get(VALUE_INDEX_COL)? {
+ serde_json::Value::Number(n) => n.as_i64(),
+ serde_json::Value::String(s) => s.parse().ok(),
+ _ => None,
+ }
+}
+
+fn lower_str(row: &CodecRecord, column: &str) -> Option<String> {
+ row.get(column).and_then(|v| v.as_str()).filter(|s| !s.is_empty()).map(str::to_lowercase)
+}
+
+fn is_truthy(value: Option<&serde_json::Value>) -> bool {
+ match value {
+ Some(serde_json::Value::Bool(b)) => *b,
+ Some(serde_json::Value::Number(n)) => n.as_i64().unwrap_or(0) != 0,
+ Some(serde_json::Value::String(s)) => s == "1" || s.eq_ignore_ascii_case("true"),
+ _ => false,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn derivation_is_stable_and_well_formed() {
+ let id = field_value_id_for("m-1", "item-1", "login.username", "", 0);
+ assert_eq!(id, field_value_id_for("m-1", "item-1", "login.username", "", 0));
+ assert_eq!(id.len(), 36);
+ assert_eq!(id.as_bytes()[14], b'8', "UUIDv8 version nibble");
+ }
+
+ #[test]
+ fn derivation_ignores_guid_casing() {
+ // iOS and the extension mint uppercase GUIDs; the natural key must not fork on casing.
+ let lower = field_value_id_for("aaaa-bbbb", "item-x", "", "def-1", 2);
+ let upper = field_value_id_for("AAAA-BBBB", "ITEM-X", "", "DEF-1", 2);
+ assert_eq!(lower, upper);
+ }
+
+ #[test]
+ fn derivation_separates_system_and_custom_fields() {
+ // The same string as a FieldKey and as a FieldDefinitionId are different key spaces.
+ assert_ne!(field_value_id_for("m", "i", "x", "", 0), field_value_id_for("m", "i", "", "x", 0));
+ }
+
+ #[test]
+ fn derivation_separates_positions() {
+ assert_ne!(field_value_id_for("m", "i", "login.url", "", 0), field_value_id_for("m", "i", "login.url", "", 1));
+ }
+
+ #[test]
+ fn history_derivation_is_stable_and_disjoint_from_field_values() {
+ let id = field_history_id_for("m-1", "item-1", "login.password", "", "2026-01-01 10:00:00.000");
+ assert_eq!(id, field_history_id_for("m-1", "item-1", "login.password", "", "2026-01-01 10:00:00.000"));
+ assert_eq!(id.len(), 36);
+ // A different millisecond is a different row; the two namespaces never collide.
+ assert_ne!(id, field_history_id_for("m-1", "item-1", "login.password", "", "2026-01-01 10:00:00.001"));
+ assert_ne!(id, field_value_id_for("m-1", "item-1", "login.password", "", 0));
+ }
+}
diff --git a/core/rust/src/vault_codec/sharing_tests.rs b/core/rust/src/vault_codec/sharing_tests.rs
index a4e100a80..1adeac2ba 100644
--- a/core/rust/src/vault_codec/sharing_tests.rs
+++ b/core/rust/src/vault_codec/sharing_tests.rs
@@ -71,6 +71,23 @@ fn ids(records: &[CodecRecord]) -> Vec<&str> {
out
}
+/// FieldValues rows are identified by their `Value` marker.
+fn values(records: &[CodecRecord]) -> Vec<&str> {
+ let mut out: Vec<&str> = records.iter().filter_map(|r| r.get("Value").and_then(|v| v.as_str())).collect();
+ out.sort();
+ out
+}
+
+/// ItemTags rows are identified by their natural key.
+fn tag_links(records: &[CodecRecord]) -> Vec<(String, String)> {
+ let mut out: Vec<(String, String)> = records
+ .iter()
+ .filter_map(|r| Some((r.get("ItemId")?.as_str()?.to_string(), r.get("TagId")?.as_str()?.to_string())))
+ .collect();
+ out.sort();
+ out
+}
+
/// The scoped logo id the codec derives for `(manifest id, source)`, what tests assert against, since
/// a logo's identity is a function of its manifest and domain rather than whatever id the writer minted.
fn logo_id(scope: &str, source: &str) -> String {
@@ -271,8 +288,8 @@ fn split_routes_item_scoped_tables_generically() {
let out = canonicalize_owner();
let shared = &out.rest()[0].manifest;
- assert_eq!(ids(rows(shared, "FieldValues")), vec!["fv-shared", "fv-sub"]);
- assert_eq!(ids(rows(&out.first().manifest, "FieldValues")), vec!["fv-personal"]);
+ assert_eq!(values(rows(shared, "FieldValues")), vec!["family", "hunter2"]);
+ assert_eq!(values(rows(&out.first().manifest, "FieldValues")), vec!["me"]);
assert_eq!(ids(rows(shared, "TotpCodes")), vec!["totp-shared"]);
assert!(rows(&out.first().manifest, "TotpCodes").is_empty());
@@ -280,8 +297,8 @@ fn split_routes_item_scoped_tables_generically() {
assert_eq!(ids(rows(shared, "Attachments")), vec!["att-shared"]);
assert_eq!(ids(rows(&out.first().manifest, "Attachments")), vec!["att-personal"]);
- assert_eq!(ids(rows(shared, "ItemTags")), vec!["it-2", "it-3"]);
- assert_eq!(ids(rows(&out.first().manifest, "ItemTags")), vec!["it-1"]);
+ assert_eq!(tag_links(rows(shared, "ItemTags")), vec![("i-shared".into(), "tag-both".into()), ("i-sub".into(), "tag-shared-only".into())]);
+ assert_eq!(tag_links(rows(&out.first().manifest, "ItemTags")), vec![("i-personal".into(), "tag-both".into())]);
}
#[test]
@@ -802,7 +819,7 @@ fn split_with_a_single_spec_drops_rows_of_manifests_it_does_not_carry() {
assert_eq!(ids(rows(&out.first().manifest, "Items")), vec!["i-nofolder", "i-personal"]);
assert_eq!(ids(rows(&out.first().manifest, "Folders")), vec!["f-personal"]);
// The rows hanging off a dropped item go with it rather than dangling.
- assert_eq!(ids(rows(&out.first().manifest, "FieldValues")), vec!["fv-personal"]);
+ assert_eq!(values(rows(&out.first().manifest, "FieldValues")), vec!["me"]);
assert!(rows(&out.first().manifest, "TotpCodes").is_empty());
// The serialized output carries the one manifest, with nothing flagging it and no folder anchor.
@@ -890,10 +907,10 @@ fn owner_split_then_combine_roundtrips_to_original_tables() {
assert_eq!(ids(&map["Items"]), vec!["i-nofolder", "i-personal", "i-shared", "i-sub"]);
assert_eq!(ids(&map["Folders"]), vec!["f-personal", "f-shared", "f-sub"]);
- assert_eq!(ids(&map["FieldValues"]), vec!["fv-personal", "fv-shared", "fv-sub"]);
+ assert_eq!(values(&map["FieldValues"]), vec!["family", "hunter2", "me"]);
assert_eq!(ids(&map["Attachments"]), vec!["att-personal", "att-shared"]);
assert_eq!(ids(&map["TotpCodes"]), vec!["totp-shared"]);
- assert_eq!(ids(&map["ItemTags"]), vec!["it-1", "it-2", "it-3"]);
+ assert_eq!(tag_links(&map["ItemTags"]), vec![("i-personal".into(), "tag-both".into()), ("i-shared".into(), "tag-both".into()), ("i-sub".into(), "tag-shared-only".into())]);
/*
* Tags and field definitions are manifest-scoped like the rest: the two the shared items carry
* exist once in each manifest that uses them, side by side under `(ManifestId, Id)`. The base keeps
@@ -971,7 +988,7 @@ fn recipient_combine_materializes_shared_manifest_into_their_vault() {
// Recipient sees their own rows plus the shared subtree.
assert_eq!(ids(&map["Folders"]), vec!["f-mine", "f-shared", "f-sub"]);
assert_eq!(ids(&map["Items"]), vec!["i-mine", "i-shared", "i-sub"]);
- assert_eq!(ids(&map["FieldValues"]), vec!["fv-shared", "fv-sub"]);
+ assert_eq!(values(&map["FieldValues"]), vec!["family", "hunter2"]);
/*
* The owner and the recipient both hold a netflix.com logo, legitimately different images (fetched
* at different times, or one of them uploaded by hand). They must BOTH survive, in their own
@@ -1021,7 +1038,7 @@ fn recipient_roundtrip_reproduces_shared_manifest_without_leaking_into_personal(
let shared = &pushed.rest()[0].manifest;
assert_eq!(ids(rows(shared, "Items")), vec!["i-shared", "i-sub"]);
assert_eq!(ids(rows(shared, "Folders")), vec!["f-shared", "f-sub"]);
- assert_eq!(ids(rows(shared, "FieldValues")), vec!["fv-shared", "fv-sub"]);
+ assert_eq!(values(rows(shared, "FieldValues")), vec!["family", "hunter2"]);
assert_eq!(ids(rows(shared, "Tags")), vec!["tag-both", "tag-shared-only"]);
// Recipient's personal manifest holds only their own rows, no shared items/folders leaked in.
@@ -1234,7 +1251,7 @@ fn combine_drops_child_rows_naming_an_item_the_manifest_does_not_hold() {
let re = materialize_as_sqlite(materialize_input(personal, vec![hostile], vec![])).unwrap();
let map = materialized_map(&re);
- assert_eq!(ids(&map["FieldValues"]), vec!["fv-mine"], "the foreign row never reaches the reader's item");
+ assert_eq!(values(&map["FieldValues"]), vec!["me"], "the foreign row never reaches the reader's item");
}
#[test]
@@ -2047,8 +2064,8 @@ fn split_never_follows_a_bare_item_id_into_another_manifest() {
assert!(manifest_mentions(personal, "personal_secret"), "and they must still be in the manifest they belong to");
// Each manifest keeps exactly its own child rows, and the shared one keeps working.
- assert_eq!(ids(rows(personal, "FieldValues")), vec!["fv-personal"]);
- assert_eq!(ids(rows(shared, "FieldValues")), vec!["fv-shared"]);
+ assert_eq!(values(rows(personal, "FieldValues")), vec!["personal_secret"]);
+ assert_eq!(values(rows(shared, "FieldValues")), vec!["shared_secret"]);
assert_eq!(ids(rows(personal, "TotpCodes")), vec!["totp-personal"]);
assert_eq!(ids(rows(shared, "TotpCodes")), vec!["totp-shared"]);
for name in ["Attachments", "Passkeys", "FieldHistories", "ItemTags"] {
@@ -2088,7 +2105,7 @@ fn split_keeps_child_rows_when_a_manifest_this_vault_lost_shares_their_item_id()
let out = canonicalize_from_sqlite(raw_input_with_shares(tables, vec![])).unwrap();
let personal = &out.first().manifest;
assert_eq!(ids(rows(personal, "Items")), vec!["i-dup"], "the revoked manifest's item is dropped, the live one kept");
- assert_eq!(ids(rows(personal, "FieldValues")), vec!["fv-personal"], "the live item keeps its own child rows");
+ assert_eq!(values(rows(personal, "FieldValues")), vec!["personal_secret"], "the live item keeps its own child rows");
assert!(!manifest_mentions(personal, "gone"), "the revoked manifest's rows are not adopted into the personal one");
}
@@ -2105,7 +2122,7 @@ fn split_still_follows_an_item_whose_children_were_left_unstamped() {
// `input_with_shares` stamps everything the fixture left unstamped as personal, child rows included.
let shared = &out.rest()[0].manifest;
- assert_eq!(ids(rows(shared, "FieldValues")), vec!["fv-shared", "fv-sub"], "children followed their item across the boundary");
+ assert_eq!(values(rows(shared, "FieldValues")), vec!["family", "hunter2"], "children followed their item across the boundary");
assert!(rows(shared, "FieldValues").iter().all(|r| r["ManifestId"] == json!("m-f-shared")), "and were re-stamped to agree with it");
}
diff --git a/core/rust/src/vault_codec/tests.rs b/core/rust/src/vault_codec/tests.rs
index a95307b1c..a4f2d680e 100644
--- a/core/rust/src/vault_codec/tests.rs
+++ b/core/rust/src/vault_codec/tests.rs
@@ -100,9 +100,11 @@ fn every_bucketed_table_is_manifest_scoped_with_a_composite_identity() {
table,
category
);
+ let mut expected = vec![super::types::MANIFEST_ID_COL];
+ expected.extend_from_slice(super::types::primary_key_columns_for(table));
assert_eq!(
super::types::identity_columns_for(table),
- vec![super::types::MANIFEST_ID_COL, super::types::primary_key_for(table)],
+ expected,
"bucketed table {} must be addressed by (ManifestId, primary key)",
table
);
@@ -837,3 +839,150 @@ pub(super) fn materialize_manifests(manifests: Vec<Manifest>, data_buckets: Vec<
let schema = fitting_schema(manifests.iter(), &data_buckets);
MaterializeInput { manifests, data_buckets, schema_columns: schema }
}
+
+// ---------------------------------------------------------------------------
+// Derived ids + wire shape (see normalize.rs)
+// ---------------------------------------------------------------------------
+
+/// The FieldValues rows of the first canonicalized manifest, keyed by their Value marker.
+fn field_values_by_value(out: &CanonicalizedVault) -> HashMap<String, CodecRecord> {
+ out.first().manifest.tables["FieldValues"]
+ .iter()
+ .map(|r| (r["Value"].as_str().unwrap().to_string(), r.clone()))
+ .collect()
+}
+
+#[test]
+fn canonicalize_strips_derived_ids_and_renumbers_multi_value_rows() {
+ let out = canonicalize_from_sqlite(basic_input(vec![
+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i-1"))])] },
+ CodecTableData {
+ name: "FieldDefinitions".to_string(),
+ records: vec![row(&[("Id", json!("fd-multi")), ("IsMultiValue", json!(1))]), row(&[("Id", json!("fd-single")), ("IsMultiValue", json!(0))])],
+ },
+ CodecTableData {
+ name: "FieldValues".to_string(),
+ records: vec![
+ row(&[("Id", json!("fv-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("Value", json!("me"))]),
+ row(&[("Id", json!("u-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("Value", json!("https://a.example"))]),
+ row(&[("Id", json!("u-2")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("Value", json!("https://b.example"))]),
+ row(&[("Id", json!("c-1")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-multi")), ("Value", json!("multi-a"))]),
+ row(&[("Id", json!("c-2")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-multi")), ("Value", json!("multi-b"))]),
+ row(&[("Id", json!("c-3")), ("ItemId", json!("i-1")), ("FieldDefinitionId", json!("fd-single")), ("Value", json!("single-c"))]),
+ ],
+ },
+ CodecTableData { name: "Tags".to_string(), records: vec![row(&[("Id", json!("t-1"))])] },
+ CodecTableData { name: "ItemTags".to_string(), records: vec![row(&[("Id", json!("it-legacy")), ("ItemId", json!("i-1")), ("TagId", json!("t-1"))])] },
+ ]))
+ .unwrap();
+
+ let fv = field_values_by_value(&out);
+ assert!(fv["me"].get("Id").is_none(), "a single-value system field's id never reaches the wire");
+ assert_eq!(fv["me"]["ValueIndex"], json!(0));
+ assert!(fv["single-c"].get("Id").is_none(), "a single-value custom field's id never reaches the wire");
+ assert_eq!(fv["https://a.example"]["Id"], json!("u-1"), "a multi-value system field's value owns its id");
+ assert_eq!(fv["https://b.example"]["Id"], json!("u-2"));
+ assert_eq!((fv["https://a.example"]["ValueIndex"].as_i64().unwrap(), fv["https://b.example"]["ValueIndex"].as_i64().unwrap()), (0, 1), "values are renumbered in read order");
+ assert_eq!(fv["multi-a"]["Id"], json!("c-1"), "a multi-value custom field's value owns its id");
+ assert_eq!(fv["multi-b"]["Id"], json!("c-2"));
+
+ let item_tags = &out.first().manifest.tables["ItemTags"];
+ assert_eq!(item_tags.len(), 1);
+ assert!(item_tags[0].get("Id").is_none(), "ItemTags carries no id at all");
+}
+
+#[test]
+fn canonicalize_strips_field_history_ids() {
+ let out = canonicalize_from_sqlite(basic_input(vec![
+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i-1"))])] },
+ CodecTableData {
+ name: "FieldHistories".to_string(),
+ records: vec![
+ row(&[("Id", json!("fh-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-01-01 10:00:00.000")), ("ValueSnapshot", json!("old-pass"))]),
+ row(&[("Id", json!("fh-2")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-02-01 10:00:00.000")), ("ValueSnapshot", json!("newer-pass"))]),
+ ],
+ },
+ ]))
+ .unwrap();
+
+ let rows = &out.first().manifest.tables["FieldHistories"];
+ assert_eq!(rows.len(), 2, "distinct ChangedAt values are distinct history rows");
+ assert!(rows.iter().all(|r| r.get("Id").is_none()), "every history row derives its id; none reaches the wire");
+}
+
+#[test]
+fn canonicalize_collapses_duplicate_single_value_rows_to_the_newest() {
+ // Two rows for one single-value field (two local writers racing before their next sync) must
+ // converge instead of reaching the wire as colliding rows.
+ let out = canonicalize_from_sqlite(basic_input(vec![
+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i-1"))])] },
+ CodecTableData {
+ name: "FieldValues".to_string(),
+ records: vec![
+ row(&[("Id", json!("fv-old")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("Value", json!("stale")), ("UpdatedAt", json!("2024-01-01 00:00:00.000"))]),
+ row(&[("Id", json!("fv-new")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("Value", json!("fresh")), ("UpdatedAt", json!("2024-06-01 00:00:00.000"))]),
+ ],
+ },
+ ]))
+ .unwrap();
+
+ let rows = &out.first().manifest.tables["FieldValues"];
+ assert_eq!(rows.len(), 1);
+ assert_eq!(rows[0]["Value"], json!("fresh"));
+}
+
+#[test]
+fn materialize_mints_missing_field_value_ids() {
+ let manifest = Manifest {
+ schema_version: SCHEMA_VERSION,
+ manifest_salt: "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff".to_string(),
+ canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(),
+ manifest_id: PERSONAL_MANIFEST.to_string(),
+ name: None,
+ tables: [
+ ("Items".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Id", json!("i-1"))])]),
+ (
+ "FieldValues".to_string(),
+ vec![
+ // Single-value wire shape: no id, explicit position; the id is minted here.
+ row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("ItemId", json!("i-1")), ("FieldKey", json!("login.username")), ("ValueIndex", json!(0)), ("Value", json!("me"))]),
+ // Multi-value: the owned id rides along untouched.
+ row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Id", json!("u-1")), ("ItemId", json!("i-1")), ("FieldKey", json!("login.url")), ("ValueIndex", json!(0)), ("Value", json!("https://a.example"))]),
+ ],
+ ),
+ ("Tags".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("Id", json!("t-1"))])]),
+ ("ItemTags".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("ItemId", json!("i-1")), ("TagId", json!("t-1"))])]),
+ ("FieldHistories".to_string(), vec![row(&[("ManifestId", json!(PERSONAL_MANIFEST)), ("ItemId", json!("i-1")), ("FieldKey", json!("login.password")), ("ChangedAt", json!("2026-01-01 10:00:00.000")), ("ValueSnapshot", json!("old-pass"))])]),
+ ]
+ .into_iter()
+ .collect(),
+ extra: HashMap::new(),
+ };
+
+ let schema: HashMap<String, Vec<String>> = [
+ ("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string()]),
+ ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string()]),
+ ("Tags".to_string(), vec!["ManifestId".to_string(), "Id".to_string()]),
+ ("ItemTags".to_string(), vec!["ManifestId".to_string(), "ItemId".to_string(), "TagId".to_string()]),
+ ("FieldHistories".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ChangedAt".to_string(), "ValueSnapshot".to_string()]),
+ (MANIFESTS_TABLE.to_string(), vec!["Id".to_string(), "Name".to_string()]),
+ ]
+ .into_iter()
+ .collect();
+
+ let out = materialize_as_sqlite(MaterializeInput { manifests: vec![manifest], data_buckets: vec![], schema_columns: schema }).unwrap();
+ let tables: HashMap<&str, &Vec<CodecRecord>> = out.tables.iter().map(|t| (t.name.as_str(), &t.records)).collect();
+
+ let fv: HashMap<&str, &CodecRecord> = tables["FieldValues"].iter().map(|r| (r["Value"].as_str().unwrap(), r)).collect();
+ let expected_username_id = super::normalize::field_value_id_for(PERSONAL_MANIFEST, "i-1", "login.username", "", 0);
+ assert_eq!(fv["me"]["Id"], json!(expected_username_id), "materialize mints the derived id");
+ assert_eq!(fv["https://a.example"]["Id"], json!("u-1"), "an owned multi-value id is kept verbatim");
+
+ assert_eq!(tables["ItemTags"].len(), 1, "the id-less join row inserts as-is");
+
+ let history = &tables["FieldHistories"][0];
+ let expected_history_id = super::normalize::field_history_id_for(PERSONAL_MANIFEST, "i-1", "login.password", "", "2026-01-01 10:00:00.000");
+ assert_eq!(history["Id"], json!(expected_history_id), "materialize mints the derived history id");
+
+ assert!(out.overflow.is_empty());
+}
diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rs
index c413fa2da..09a191347 100644
--- a/core/rust/src/vault_codec/types.rs
+++ b/core/rust/src/vault_codec/types.rs
@@ -38,6 +38,10 @@ pub static BUCKET_TABLES: &[(&str, &str)] = &[
/// the declaration point for a future personal-only table.
pub static PERSONAL_TABLES: &[&str] = &[];
+/// System field keys whose field holds multiple values. A value of such a field owns its row id
+/// (two devices each adding a value are adding two different things) and are not derived.
+pub static MULTI_VALUE_FIELD_KEYS: &[&str] = &["login.url"];
+
/// The per-manifest delivery-keypair table. Every manifest carries its own asymmetric keypair(s),
/// stamped with that manifest's id (`ManifestId`).
pub const ENCRYPTION_KEYS_TABLE: &str = "EncryptionKeys";
@@ -118,9 +122,9 @@ pub fn is_bucketed_table(table_name: &str) -> bool {
bucket_category_for(table_name).is_some()
}
-/// Get the primary key column for a table.
-pub fn primary_key_for(table_name: &str) -> &'static str {
- crate::vault_merge::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key).unwrap_or("Id")
+/// Get the primary key columns for a table.
+pub fn primary_key_columns_for(table_name: &str) -> &'static [&'static str] {
+ crate::vault_merge::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key_columns).unwrap_or(&["Id"])
}
/// True when `table_name`'s rows are namespaced per manifest.
@@ -148,8 +152,10 @@ pub fn row_identity(table_name: &str, row: &super::manifest::CodecRecord) -> Opt
if !columns.contains(&MANIFEST_ID_COL) && row.get(MANIFEST_ID_COL).filter(|value| !value.is_null()).is_some() {
columns.insert(0, MANIFEST_ID_COL);
}
- // A row with no primary key cannot be addressed at all.
- row.get(primary_key_for(table_name))?;
+ // A row missing any primary key column cannot be addressed at all.
+ for column in primary_key_columns_for(table_name) {
+ row.get(*column)?;
+ }
Some(
columns
.iter()
diff --git a/core/rust/src/vault_merge/canonical.rs b/core/rust/src/vault_merge/canonical.rs
index c834ef800..434c7c522 100644
--- a/core/rust/src/vault_merge/canonical.rs
+++ b/core/rust/src/vault_merge/canonical.rs
@@ -142,6 +142,10 @@ fn merge_manifest_pair(
}
}
+ // A union of concurrently added multi-value rows can leave two rows at the same ValueIndex;
+ // re-normalizing the output renumbers them, so a merged manifest is normalized like any other.
+ crate::vault_codec::normalize::normalize_row_shapes(&mut merged);
+
let mut buckets = unknown_server_buckets;
for category in bucket_categories() {
let mut bucket_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();
@@ -200,7 +204,15 @@ fn merge_rows(
stats: &mut MergeStats,
) -> Vec<CodecRecord> {
let identity_columns = config.identity_columns();
- let match_columns: &[&str] = if config.uses_composite_key() { config.composite_key_columns } else { &identity_columns };
+ // Rows arrive normalized to the wire shape here, so a canonical-only key may rely on stripped
+ // derived ids (see the FieldValues registry comment).
+ let match_columns: &[&str] = if !config.canonical_key_columns.is_empty() {
+ config.canonical_key_columns
+ } else if config.uses_composite_key() {
+ config.composite_key_columns
+ } else {
+ &identity_columns
+ };
let known_columns: Option<HashSet<&str>> = schema_columns.get(config.name).map(|cols| cols.iter().map(String::as_str).collect());
// Winner per match key among incoming rows; on duplicates the latest UpdatedAt wins.
@@ -302,7 +314,8 @@ mod tests {
("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Name".to_string(), "UpdatedAt".to_string()]),
("Settings".to_string(), vec!["ManifestId".to_string(), "Key".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),
("ItemStats".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "LastUsedAt".to_string(), "UpdatedAt".to_string()]),
- ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),
+ ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),
+ ("ItemTags".to_string(), vec!["ManifestId".to_string(), "ItemId".to_string(), "TagId".to_string(), "UpdatedAt".to_string()]),
("Attachments".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Blob".to_string(), "UpdatedAt".to_string()]),
]
.into_iter()
@@ -387,15 +400,15 @@ mod tests {
}
#[test]
- fn field_values_keep_the_base_identity_on_a_semantic_match() {
- // FieldValues match on (ManifestId, ItemId, FieldKey): a winning local row minted under a
- // different Id updates the base row, which keeps its own Id.
- let field_value = |id: &str, value: &str, updated_at: &str| -> CodecRecord {
+ fn field_values_match_semantically_and_carry_no_surrogate_id() {
+ // A single-value FieldValue arrives in wire shape (no Id, position 0) and matches on its
+ // natural key: both sides converge to one row holding the winning value, still id-less.
+ let field_value = |value: &str, updated_at: &str| -> CodecRecord {
[
("ManifestId".to_string(), json!(PERSONAL)),
- ("Id".to_string(), json!(id)),
("ItemId".to_string(), json!("item-1")),
("FieldKey".to_string(), json!("username")),
+ ("ValueIndex".to_string(), json!(0)),
("Value".to_string(), json!(value)),
("UpdatedAt".to_string(), json!(updated_at)),
]
@@ -403,20 +416,178 @@ mod tests {
.collect()
};
+ let rows = merge_field_values(vec![field_value("old", "2024-01-01T00:00:00Z")], vec![field_value("new", "2024-01-09T00:00:00Z")]);
+ assert_eq!(rows.len(), 1);
+ assert!(rows[0].get("Id").is_none(), "a single-value row's id is derived at materialize, never carried on the wire");
+ assert_eq!(rows[0]["Value"], json!("new"), "and the row carries the winning value");
+ }
+
+ /// A `login.url` FieldValues row: multi-value, so it owns its id on the wire.
+ fn url_value(id: &str, index: i64, value: &str, updated_at: &str) -> CodecRecord {
+ [
+ ("ManifestId".to_string(), json!(PERSONAL)),
+ ("Id".to_string(), json!(id)),
+ ("ItemId".to_string(), json!("item-1")),
+ ("FieldKey".to_string(), json!("login.url")),
+ ("ValueIndex".to_string(), json!(index)),
+ ("Value".to_string(), json!(value)),
+ ("UpdatedAt".to_string(), json!(updated_at)),
+ ]
+ .into_iter()
+ .collect()
+ }
+
+ fn merge_field_values(server: Vec<CodecRecord>, local: Vec<CodecRecord>) -> Vec<CodecRecord> {
let output = merge_canonical(CanonicalMergeInput {
- server_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), vec![field_value("fv-base", "old", "2024-01-01T00:00:00Z")])].into_iter().collect())],
+ server_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), server)].into_iter().collect())],
server_buckets: vec![],
contentless_server_manifest_ids: vec![],
- local_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), vec![field_value("fv-local", "new", "2024-01-09T00:00:00Z")])].into_iter().collect())],
+ local_manifests: vec![manifest(PERSONAL, [("FieldValues".to_string(), local)].into_iter().collect())],
local_buckets: vec![],
schema_columns: schema(),
})
.unwrap();
+ output.manifests[0].manifest.tables["FieldValues"].clone()
+ }
- let rows = &output.manifests[0].manifest.tables["FieldValues"];
+ #[test]
+ fn multi_value_edit_converges_on_the_owned_id() {
+ // The same owned id on both sides is the same row edited: LWW, one row out.
+ let rows = merge_field_values(
+ vec![url_value("url-1", 0, "https://a.example", "2024-01-01T00:00:00Z")],
+ vec![url_value("url-1", 0, "https://a-edited.example", "2024-01-09T00:00:00Z")],
+ );
assert_eq!(rows.len(), 1);
- assert_eq!(rows[0]["Id"], json!("fv-base"), "the base row keeps its own Id");
- assert_eq!(rows[0]["Value"], json!("new"), "but carries the winning value");
+ assert_eq!(rows[0]["Id"], json!("url-1"), "the row keeps its owned id");
+ assert_eq!(rows[0]["Value"], json!("https://a-edited.example"));
+ }
+
+ #[test]
+ fn multi_value_concurrent_adds_both_survive() {
+ // Two devices each add a 2nd url while offline: both additions land at position 1 under
+ // different owned ids. Position is not identity, so the union keeps both, renumbered.
+ let rows = merge_field_values(
+ vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-05T00:00:00Z")],
+ vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-c", 1, "https://c.example", "2024-01-06T00:00:00Z")],
+ );
+ let mut values: Vec<&str> = rows.iter().map(|r| r["Value"].as_str().unwrap()).collect();
+ values.sort();
+ assert_eq!(values, vec!["https://a.example", "https://b.example", "https://c.example"]);
+ let mut indexes: Vec<i64> = rows.iter().map(|r| r["ValueIndex"].as_i64().unwrap()).collect();
+ indexes.sort();
+ assert_eq!(indexes, vec![0, 1, 2], "colliding positions are renumbered in the merged output");
+ }
+
+ #[test]
+ fn multi_value_reorder_does_not_duplicate() {
+ // Reordering changes only ValueIndex; the owned id keeps the rows matched, so no row doubles.
+ let rows = merge_field_values(
+ vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-01T00:00:00Z")],
+ vec![url_value("url-b", 0, "https://b.example", "2024-01-09T00:00:00Z"), url_value("url-a", 1, "https://a.example", "2024-01-09T00:00:00Z")],
+ );
+ assert_eq!(rows.len(), 2);
+ let by_id: HashMap<&str, i64> = rows.iter().map(|r| (r["Id"].as_str().unwrap(), r["ValueIndex"].as_i64().unwrap())).collect();
+ assert_eq!(by_id["url-b"], 0, "the reorder won");
+ assert_eq!(by_id["url-a"], 1);
+ }
+
+ #[test]
+ fn both_sides_holding_the_same_two_urls_stay_two_rows() {
+ // The historic corruption case: before the id-based key, the 2nd url's value replaced the
+ // 1st on an ordinary push. Identical sides must merge to exactly themselves.
+ let two = vec![url_value("url-a", 0, "https://a.example", "2024-01-01T00:00:00Z"), url_value("url-b", 1, "https://b.example", "2024-01-02T00:00:00Z")];
+ let rows = merge_field_values(two.clone(), two);
+ assert_eq!(rows.len(), 2);
+ let by_id: HashMap<&str, &str> = rows.iter().map(|r| (r["Id"].as_str().unwrap(), r["Value"].as_str().unwrap())).collect();
+ assert_eq!(by_id["url-a"], "https://a.example");
+ assert_eq!(by_id["url-b"], "https://b.example");
+ }
+
+ #[test]
+ fn custom_field_values_of_one_item_do_not_collapse() {
+ // Custom fields carry no FieldKey (only a FieldDefinitionId); before FieldDefinitionId joined
+ // the key they all collapsed onto one empty-string key.
+ let custom = |def: &str, value: &str, updated_at: &str| -> CodecRecord {
+ [
+ ("ManifestId".to_string(), json!(PERSONAL)),
+ ("Id".to_string(), json!(format!("cv-{}", def))),
+ ("ItemId".to_string(), json!("item-1")),
+ ("FieldKey".to_string(), serde_json::Value::Null),
+ ("FieldDefinitionId".to_string(), json!(def)),
+ ("Value".to_string(), json!(value)),
+ ("UpdatedAt".to_string(), json!(updated_at)),
+ ]
+ .into_iter()
+ .collect()
+ };
+ let rows = merge_field_values(
+ vec![custom("fd-pin", "1234", "2024-01-01T00:00:00Z"), custom("fd-member", "M-77", "2024-01-01T00:00:00Z")],
+ vec![custom("fd-pin", "9999", "2024-01-09T00:00:00Z"), custom("fd-member", "M-77", "2024-01-01T00:00:00Z")],
+ );
+ assert_eq!(rows.len(), 2, "each custom field keeps its own row");
+ let mut values: Vec<&str> = rows.iter().map(|r| r["Value"].as_str().unwrap()).collect();
+ values.sort();
+ assert_eq!(values, vec!["9999", "M-77"]);
+ }
+
+ #[test]
+ fn field_histories_union_by_changed_at_and_converge_on_the_same_snapshot() {
+ // History rows derive their id from (item, field, ChangedAt): concurrent changes on two
+ // devices carry different timestamps and union; the same snapshot on both sides is one row.
+ let history = |changed_at: &str, snapshot: &str| -> CodecRecord {
+ [
+ ("ManifestId".to_string(), json!(PERSONAL)),
+ ("ItemId".to_string(), json!("item-1")),
+ ("FieldKey".to_string(), json!("login.password")),
+ ("ChangedAt".to_string(), json!(changed_at)),
+ ("ValueSnapshot".to_string(), json!(snapshot)),
+ ("UpdatedAt".to_string(), json!(changed_at)),
+ ]
+ .into_iter()
+ .collect()
+ };
+ let shared = history("2024-01-01 10:00:00.000", "both-know-this");
+ let output = merge_canonical(CanonicalMergeInput {
+ server_manifests: vec![manifest(PERSONAL, [("FieldHistories".to_string(), vec![shared.clone(), history("2024-02-01 10:00:00.000", "server-only")])].into_iter().collect())],
+ server_buckets: vec![],
+ contentless_server_manifest_ids: vec![],
+ local_manifests: vec![manifest(PERSONAL, [("FieldHistories".to_string(), vec![shared, history("2024-02-01 11:30:00.000", "local-only")])].into_iter().collect())],
+ local_buckets: vec![],
+ schema_columns: schema(),
+ })
+ .unwrap();
+ let rows = &output.manifests[0].manifest.tables["FieldHistories"];
+ let mut snapshots: Vec<&str> = rows.iter().map(|r| r["ValueSnapshot"].as_str().unwrap()).collect();
+ snapshots.sort();
+ assert_eq!(snapshots, vec!["both-know-this", "local-only", "server-only"]);
+ assert!(rows.iter().all(|r| r.get("Id").is_none()), "merged history rows stay id-less on the wire");
+ }
+
+ #[test]
+ fn item_tags_converge_on_their_natural_key() {
+ // Two devices tag the same item with the same tag: the id-less join rows are one row.
+ let tag_row = |updated_at: &str| -> CodecRecord {
+ [
+ ("ManifestId".to_string(), json!(PERSONAL)),
+ ("ItemId".to_string(), json!("item-1")),
+ ("TagId".to_string(), json!("tag-1")),
+ ("UpdatedAt".to_string(), json!(updated_at)),
+ ]
+ .into_iter()
+ .collect()
+ };
+ let output = merge_canonical(CanonicalMergeInput {
+ server_manifests: vec![manifest(PERSONAL, [("ItemTags".to_string(), vec![tag_row("2024-01-01T00:00:00Z")])].into_iter().collect())],
+ server_buckets: vec![],
+ contentless_server_manifest_ids: vec![],
+ local_manifests: vec![manifest(PERSONAL, [("ItemTags".to_string(), vec![tag_row("2024-01-09T00:00:00Z")])].into_iter().collect())],
+ local_buckets: vec![],
+ schema_columns: schema(),
+ })
+ .unwrap();
+ let rows = &output.manifests[0].manifest.tables["ItemTags"];
+ assert_eq!(rows.len(), 1, "the same (item, tag) link is one row on both sides");
+ assert!(rows[0].get("Id").is_none());
}
#[test]
diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rs
index 01f736ff7..c61012e1e 100644
--- a/core/rust/src/vault_merge/mod.rs
+++ b/core/rust/src/vault_merge/mod.rs
@@ -285,7 +285,7 @@ fn key_part(value: &serde_json::Value) -> String {
/// Get the UpdatedAt timestamp from a record.
/// Handles both RFC3339 format (2025-12-11T06:50:10.674Z) and
/// SQLite format (2025-12-11 06:50:10.674).
-fn get_updated_at(record: &Record) -> Option<DateTime<Utc>> {
+pub(crate) fn get_updated_at(record: &Record) -> Option<DateTime<Utc>> {
record
.get("UpdatedAt")
.and_then(|v| v.as_str())
@@ -641,11 +641,11 @@ mod tests {
#[test]
fn settings_and_encryption_keys_are_syncable() {
let settings = SYNCABLE_TABLES.iter().find(|t| t.name == "Settings").unwrap();
- assert_eq!(settings.primary_key, "Key");
+ assert_eq!(settings.primary_key_columns, &["Key"]);
assert!(!settings.uses_composite_key(), "a key already names one setting per manifest; no semantic match key is needed");
let keys = SYNCABLE_TABLES.iter().find(|t| t.name == "EncryptionKeys").unwrap();
- assert_eq!(keys.primary_key, "Id");
+ assert_eq!(keys.primary_key_columns, &["Id"]);
assert!(SYNCABLE_TABLE_NAMES.contains(&"Settings"));
assert!(SYNCABLE_TABLE_NAMES.contains(&"EncryptionKeys"));
@@ -705,7 +705,7 @@ mod tests {
// never go back to composite-key matching on Source.
let cfg = SYNCABLE_TABLES.iter().find(|t| t.name == "Logos").unwrap();
assert!(!cfg.uses_composite_key());
- assert_eq!(cfg.primary_key, "Id");
+ assert_eq!(cfg.primary_key_columns, &["Id"]);
}
#[test]
@@ -744,7 +744,9 @@ mod tests {
fn scoped_row(config: &TableConfig, manifest_id: &str, updated_at: &str, payload: &str) -> Record {
let mut record: Record = HashMap::new();
record.insert(crate::vault_codec::MANIFEST_ID_COL.to_string(), serde_json::json!(manifest_id));
- record.insert(config.primary_key.to_string(), serde_json::json!("row-1"));
+ for column in config.primary_key_columns {
+ record.insert((*column).to_string(), serde_json::json!("row-1"));
+ }
for column in config.composite_key_columns {
record.entry((*column).to_string()).or_insert_with(|| serde_json::json!("k"));
}
@@ -804,7 +806,8 @@ mod tests {
let set_clause = sql.trim_start_matches(&prefix).split(" WHERE ").next().unwrap();
assert!(!set_clause.contains(crate::vault_codec::MANIFEST_ID_COL), "{}: an update must never rewrite the row's manifest ({})", config.name, sql);
- assert!(sql.ends_with(&format!("WHERE ManifestId = ? AND {} = ?", config.primary_key)), "{}: the base row is addressed by its full identity ({})", config.name, sql);
+ let expected_where = format!("WHERE {}", config.identity_columns().iter().map(|c| format!("{} = ?", c)).collect::<Vec<_>>().join(" AND "));
+ assert!(sql.ends_with(&expected_where), "{}: the base row is addressed by its full identity ({})", config.name, sql);
}
}
diff --git a/core/rust/src/vault_merge/types.rs b/core/rust/src/vault_merge/types.rs
index 53402c77c..0c995cc6b 100644
--- a/core/rust/src/vault_merge/types.rs
+++ b/core/rust/src/vault_merge/types.rs
@@ -11,9 +11,13 @@ pub struct TableConfig {
/// When empty, matching falls back to the table's [`identity_columns`](TableConfig::identity_columns).
/// When set, these columns are concatenated to form the composite key.
pub composite_key_columns: &'static [&'static str],
- /// The column that names a row within its manifest. Defaults to "Id"; tables keyed by a
- /// different column (e.g. Settings keyed by "Key") override this.
- pub primary_key: &'static str,
+ /// Match columns for the canonical merge only, where both sides are first normalized to the manifest
+ /// shape (see `vault_codec::normalize`). Empty means the canonical merge uses the same rule as
+ /// the statement merge (composite key, else identity).
+ pub canonical_key_columns: &'static [&'static str],
+ /// The columns that name a row within its manifest. Defaults to `["Id"]`; tables keyed
+ /// differently (Settings by "Key", ItemTags by its natural key) override this.
+ pub primary_key_columns: &'static [&'static str],
/// True when the table's rows are namespaced per manifest.
pub manifest_scoped: bool,
}
@@ -23,7 +27,8 @@ impl TableConfig {
Self {
name,
composite_key_columns: &[],
- primary_key: "Id",
+ canonical_key_columns: &[],
+ primary_key_columns: &["Id"],
manifest_scoped: false,
}
}
@@ -33,8 +38,13 @@ impl TableConfig {
self
}
- pub const fn with_primary_key(mut self, column: &'static str) -> Self {
- self.primary_key = column;
+ pub const fn with_canonical_key(mut self, columns: &'static [&'static str]) -> Self {
+ self.canonical_key_columns = columns;
+ self
+ }
+
+ pub const fn with_primary_key(mut self, columns: &'static [&'static str]) -> Self {
+ self.primary_key_columns = columns;
self
}
@@ -48,14 +58,16 @@ impl TableConfig {
!self.composite_key_columns.is_empty()
}
- /// The columns that together identify one row of this table: `(ManifestId, primary_key)` for a
- /// manifest-scoped table, the primary key alone otherwise. This is what a generated UPDATE
- /// addresses a row by, so it must match the table's declared PRIMARY KEY.
+ /// The columns that together identify one row of this table: `(ManifestId, primary key columns)`
+ /// for a manifest-scoped table, the primary key columns alone otherwise.
pub fn identity_columns(&self) -> Vec<&'static str> {
if self.manifest_scoped {
- vec![MANIFEST_ID_COL, self.primary_key]
+ let mut columns = Vec::with_capacity(1 + self.primary_key_columns.len());
+ columns.push(MANIFEST_ID_COL);
+ columns.extend_from_slice(self.primary_key_columns);
+ columns
} else {
- vec![self.primary_key]
+ self.primary_key_columns.to_vec()
}
}
}
@@ -67,18 +79,33 @@ pub static SYNCABLE_TABLES: &[TableConfig] = &[
// Keyed by the item it describes: `Id` *is* the item's id, so recording a use is an upsert and two
// devices never mint competing rows. Listed after Items so a merge inserts the item first.
TableConfig::new("ItemStats").manifest_scoped(),
- TableConfig::new("FieldValues").manifest_scoped().with_composite_key(&[MANIFEST_ID_COL, "ItemId", "FieldKey"]),
+ // Legacy statement merge: a field value matches on the field it belongs to (FieldKey for system
+ // fields, FieldDefinitionId for custom ones; exactly one is set), so independently minted rows of
+ // the same field converge. Canonical merge: both sides are normalized first, which strips the
+ // derived id of every single-value row, so adding `Id` to the key makes a single-value row match
+ // by its field (id part empty on both sides) while a multi-value row matches by its OWNED id:
+ // two devices each adding a `login.url` are two different rows that must both survive, and the id,
+ // unlike ValueIndex, is stable under reordering. See vault_codec::normalize.
+ TableConfig::new("FieldValues")
+ .manifest_scoped()
+ .with_composite_key(&[MANIFEST_ID_COL, "ItemId", "FieldKey", "FieldDefinitionId"])
+ .with_canonical_key(&[MANIFEST_ID_COL, "ItemId", "FieldKey", "FieldDefinitionId", "Id"]),
TableConfig::new("Folders").manifest_scoped(),
TableConfig::new("Tags").manifest_scoped(),
- TableConfig::new("ItemTags").manifest_scoped(),
+ // A pure join table keyed by its natural key; it carries no surrogate id (since client schema 2.1.6),
+ // so two devices tagging the same item converge on one row.
+ TableConfig::new("ItemTags").manifest_scoped().with_primary_key(&["ItemId", "TagId"]),
TableConfig::new("Attachments").manifest_scoped(),
TableConfig::new("TotpCodes").manifest_scoped(),
TableConfig::new("Passkeys").manifest_scoped(),
TableConfig::new("FieldDefinitions").manifest_scoped(),
- TableConfig::new("FieldHistories").manifest_scoped(),
+ // Canonical merge: every history row derives its id from `(item, field, ChangedAt)`, so after
+ // normalization the natural key IS the identity: concurrent changes union (distinct ChangedAt),
+ // same-millisecond snapshots converge. Legacy statement merge keeps plain (ManifestId, Id).
+ TableConfig::new("FieldHistories").manifest_scoped().with_canonical_key(&[MANIFEST_ID_COL, "ItemId", "FieldKey", "FieldDefinitionId", "ChangedAt"]),
TableConfig::new("Logos").manifest_scoped(),
TableConfig::new("EncryptionKeys").manifest_scoped(),
- TableConfig::new("Settings").with_primary_key("Key").manifest_scoped(),
+ TableConfig::new("Settings").with_primary_key(&["Key"]).manifest_scoped(),
];
/// The tables a platform must read into the merge input: the syncable tables.