Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Refactor (#2159)

Leendert de Borstbb31d075f953
diff --git a/core/rust/src/vault_merge/canonical.rs b/core/rust/src/vault_merge/canonical.rs
index 6d65f8e3a..3af08090a 100644
--- a/core/rust/src/vault_merge/canonical.rs
+++ b/core/rust/src/vault_merge/canonical.rs
@@ -12,8 +12,10 @@ use std::collections::{BTreeSet, HashMap, HashSet};
use serde::{Deserialize, Serialize};
-use super::{get_key, get_updated_at, MergeStats, TableConfig, SYNCABLE_TABLES};
+use super::{get_key, MergeStats};
use crate::error::VaultResult;
+use crate::timestamp::updated_at;
+use crate::vault_model::{id_key, TableConfig, SYNCABLE_TABLES};
use crate::vault_codec::{bucket_categories, is_bucketed_table, tables_for_category, CodecRecord, DataBucket, Manifest};
/// Input of the canonical merge. The server side is the base (kept on ties); the local side is the
@@ -52,13 +54,13 @@ pub struct CanonicalMergeOutput {
pub fn merge_canonical(input: CanonicalMergeInput) -> VaultResult<CanonicalMergeOutput> {
let CanonicalMergeInput { server_manifests, server_buckets, contentless_server_manifest_ids, local_manifests, local_buckets, schema_columns } = input;
- let mut local_by_id: HashMap<String, Manifest> = local_manifests.into_iter().map(|m| (m.manifest_id.to_lowercase(), m)).collect();
+ let mut local_by_id: HashMap<String, Manifest> = local_manifests.into_iter().map(|m| (id_key(&m.manifest_id), m)).collect();
let mut local_buckets_by_id = group_buckets(local_buckets);
let mut server_buckets_by_id = group_buckets(server_buckets);
let mut manifests: Vec<CanonicalManifestMerge> = Vec::new();
for server_manifest in server_manifests {
- let key = server_manifest.manifest_id.to_lowercase();
+ let key = id_key(&server_manifest.manifest_id);
let local = local_by_id.remove(&key);
let local_buckets = local_buckets_by_id.remove(&key).unwrap_or_default();
let server_buckets = server_buckets_by_id.remove(&key).unwrap_or_default();
@@ -67,7 +69,7 @@ pub fn merge_canonical(input: CanonicalMergeInput) -> VaultResult<CanonicalMerge
// A contentless server manifest has no base; the local counterpart passes through whole.
for manifest_id in contentless_server_manifest_ids {
- let key = manifest_id.to_lowercase();
+ let key = id_key(&manifest_id);
if let Some(local) = local_by_id.remove(&key) {
manifests.push(pass_through(local, local_buckets_by_id.remove(&key).unwrap_or_default()));
}
@@ -79,17 +81,11 @@ pub fn merge_canonical(input: CanonicalMergeInput) -> VaultResult<CanonicalMerge
Ok(CanonicalMergeOutput { manifests, dropped_local_manifest_ids })
}
-/// JSON convenience wrapper for FFI.
-pub fn merge_canonical_json(input_json: &str) -> VaultResult<String> {
- let input: CanonicalMergeInput = serde_json::from_str(input_json)?;
- Ok(serde_json::to_string(&merge_canonical(input)?)?)
-}
-
/// Group buckets by their manifest id (lowercased).
fn group_buckets(buckets: Vec<DataBucket>) -> HashMap<String, Vec<DataBucket>> {
let mut grouped: HashMap<String, Vec<DataBucket>> = HashMap::new();
for bucket in buckets {
- grouped.entry(bucket.manifest_id.to_lowercase()).or_default().push(bucket);
+ grouped.entry(id_key(&bucket.manifest_id)).or_default().push(bucket);
}
grouped
}
@@ -206,13 +202,7 @@ fn merge_rows(
let identity_columns = config.identity_columns();
// Rows arrive normalized to the wire shape here, so a canonical-only key may rely on stripped
// derived ids (see the FieldValues registry comment).
- let match_columns: &[&str] = if !config.canonical_key_columns.is_empty() {
- config.canonical_key_columns
- } else if config.uses_composite_key() {
- config.composite_key_columns
- } else {
- &identity_columns
- };
+ let match_columns: &[&str] = if config.canonical_key_columns.is_empty() { &identity_columns } else { config.canonical_key_columns };
let known_columns: Option<HashSet<&str>> = schema_columns.get(config.name).map(|cols| cols.iter().map(String::as_str).collect());
// Winner per match key among incoming rows; on duplicates the latest UpdatedAt wins.
@@ -220,7 +210,7 @@ fn merge_rows(
for record in &incoming_rows {
let key = get_key(record, match_columns);
match incoming_map.get(&key) {
- Some(existing) if get_updated_at(record) <= get_updated_at(existing) => {}
+ Some(existing) if updated_at(record) <= updated_at(existing) => {}
_ => {
incoming_map.insert(key, record);
}
@@ -232,7 +222,7 @@ fn merge_rows(
let key = get_key(&base_record, match_columns);
match incoming_map.remove(&key) {
Some(incoming) => {
- let (incoming_ts, base_ts) = (get_updated_at(incoming), get_updated_at(&base_record));
+ let (incoming_ts, base_ts) = (updated_at(incoming), updated_at(&base_record));
match (incoming_ts, base_ts) {
(Some(i_ts), Some(b_ts)) if i_ts > b_ts => {
stats.conflicts += 1;
diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rs
index ef9443697..affc136ff 100644
--- a/core/rust/src/vault_merge/mod.rs
+++ b/core/rust/src/vault_merge/mod.rs
@@ -1,40 +1,13 @@
//! Vault merge logic using Last-Write-Wins (LWW) strategy.
-//!
-//! This module provides the core merge functionality that works on JSON table data.
-//! It generates SQL statements that clients can execute directly on their local database.
mod canonical;
-mod types;
-use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-use crate::error::VaultResult;
-pub use canonical::{merge_canonical, merge_canonical_json, CanonicalManifestMerge, CanonicalMergeInput, CanonicalMergeOutput};
-pub use types::SYNCABLE_TABLE_NAMES;
-pub use types::{merge_table_names, SYNCABLE_TABLES, TableConfig};
+pub use canonical::{merge_canonical, CanonicalManifestMerge, CanonicalMergeInput, CanonicalMergeOutput};
/// A record is a map of column names to JSON values.
-pub type Record = HashMap<String, serde_json::Value>;
-
-/// Data for a single table.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct TableData {
- /// Table name
- pub name: String,
- /// All records in this table
- pub records: Vec<Record>,
-}
-
-/// Input for the merge operation.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct MergeInput {
- /// Tables from the local database
- pub local_tables: Vec<TableData>,
- /// Tables from the server database
- pub server_tables: Vec<TableData>,
-}
+pub type Record = crate::vault_codec::CodecRecord;
/// A SQL statement with its parameter values.
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -50,819 +23,43 @@ pub struct SqlStatement {
#[serde(rename_all = "camelCase")]
#[cfg_attr(feature = "uniffi", derive(uniffi::Record))]
pub struct MergeStats {
- /// Number of tables processed
pub tables_processed: u32,
- /// Records where local version was kept
pub records_from_local: u32,
- /// Records where server version was used (updates)
pub records_from_server: u32,
- /// Records that only existed locally (created offline)
pub records_created_locally: u32,
- /// Number of conflicts resolved (both had the record)
pub conflicts: u32,
- /// Records inserted from server (server-only records)
pub records_inserted: u32,
}
-/// Output of the merge operation.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct MergeOutput {
- /// Whether the merge was successful
- pub success: bool,
- /// SQL statements to execute on the local database (in order)
- pub statements: Vec<SqlStatement>,
- /// Overall statistics
- pub stats: MergeStats,
-}
-
-/// Main entry point: merge local and server vault data.
-///
-/// The merge **base is the server vault** (`input.server_tables`): a freshly-materialized SQLite with
-/// the newest schema and the newest codec overflow carrier already in place. The returned statements
-/// bring the local vault's winning changes (`input.local_tables`) onto that base.
-///
-/// # Arguments
-/// * `input` - MergeInput containing local (incoming) and server (base) table data
-///
-/// # Returns
-/// MergeOutput with SQL statements to execute on the server (base) database
-pub fn merge_vaults(input: MergeInput) -> VaultResult<MergeOutput> {
- let mut total_stats = MergeStats::default();
- let mut statements: Vec<SqlStatement> = Vec::new();
-
- // Base = server vault; incoming = local vault. Statements mutate the base.
- let base_map: HashMap<&str, &TableData> = input
- .server_tables
- .iter()
- .map(|t| (t.name.as_str(), t))
- .collect();
-
- let incoming_map: HashMap<&str, &TableData> = input
- .local_tables
- .iter()
- .map(|t| (t.name.as_str(), t))
- .collect();
-
- // Process each syncable table
- for table_config in SYNCABLE_TABLES {
- let table_name = table_config.name;
-
- let base_data = base_map.get(table_name);
- let incoming_data = incoming_map.get(table_name);
-
- // Skip if table doesn't exist in either database
- let (base_records, incoming_records) = match (base_data, incoming_data) {
- (Some(b), Some(i)) => (&b.records, &i.records),
- (Some(_), None) => {
- // Table only in the server base. Already present, nothing to bring over.
- continue;
- }
- (None, Some(i)) => {
- // Table only in local - insert all local rows into the server base.
- for record in &i.records {
- if let Some(stmt) = generate_insert_sql(table_name, record) {
- statements.push(stmt);
- total_stats.records_inserted += 1;
- }
- }
- total_stats.tables_processed += 1;
- continue;
- }
- (None, None) => continue,
- };
-
- // A row is addressed by its identity columns, and matched to its counterpart by those same
- // columns unless the table declares a semantic match key of its own (FieldValues).
- let identity_columns = table_config.identity_columns();
- let match_columns: &[&str] = if table_config.uses_composite_key() {
- table_config.composite_key_columns
- } else {
- &identity_columns
- };
-
- let table_statements = merge_table(
- table_name,
- base_records,
- incoming_records,
- match_columns,
- &identity_columns,
- &mut total_stats,
- );
-
- statements.extend(table_statements);
- total_stats.tables_processed += 1;
- }
-
- Ok(MergeOutput {
- success: true,
- statements,
- stats: total_stats,
- })
-}
-
-/// Merge a JSON string input and return JSON string output.
-/// Convenience function for FFI.
-pub fn merge_vaults_json(input_json: &str) -> VaultResult<String> {
- let input: MergeInput = serde_json::from_str(input_json)?;
- let output = merge_vaults(input)?;
- let output_json = serde_json::to_string(&output)?;
- Ok(output_json)
-}
-
-/// Merge one table's records with last-write-wins.
-///
-/// `match_columns` decide which base row an incoming row *is* (its identity columns, or the table's own
-/// semantic key where it declares one); `identity_columns` are what a winning incoming row is written
-/// back through, so the base row keeps its own identity on conflict. On a manifest-scoped table that
-/// identity is `(ManifestId, Id)`, which keeps two manifests holding the same `Id` apart.
-///
-/// `base_records` are the merge base (kept on tie); `incoming_records` are the changes merged onto it.
-/// Returns SQL statements to apply to the base database.
-fn merge_table(
- table_name: &str,
- base_records: &[Record],
- incoming_records: &[Record],
- match_columns: &[&str],
- identity_columns: &[&str],
- stats: &mut MergeStats,
-) -> Vec<SqlStatement> {
- let mut statements: Vec<SqlStatement> = Vec::new();
-
- // Map of incoming records by match key; on duplicates the latest UpdatedAt wins.
- let mut incoming_map: HashMap<String, &Record> = HashMap::new();
- for record in incoming_records {
- let key = get_key(record, match_columns);
- match incoming_map.get(&key) {
- Some(existing) if get_updated_at(record) <= get_updated_at(existing) => {}
- _ => {
- incoming_map.insert(key, record);
- }
- }
- }
-
- // Process base records
- for base_record in base_records {
- let match_key = get_key(base_record, match_columns);
-
- let base_identity = match get_identity(base_record, identity_columns) {
- Some(identity) => identity,
- None => continue,
- };
-
- if let Some(incoming_record) = incoming_map.get(&match_key) {
- // Record exists in both - compare UpdatedAt for LWW
- let incoming_ts = get_updated_at(incoming_record);
- let base_ts = get_updated_at(base_record);
-
- match (incoming_ts, base_ts) {
- (Some(i_ts), Some(b_ts)) if i_ts > b_ts => {
- // Incoming wins - update the base row in place, keeping its own identity
- stats.conflicts += 1;
- stats.records_from_server += 1;
- if let Some(stmt) = generate_update_sql(table_name, incoming_record, &base_identity) {
- statements.push(stmt);
- }
- }
- _ => {
- // Base wins - no action needed
- stats.records_from_local += 1;
- }
- }
- incoming_map.remove(&match_key);
- } else {
- // Only in base - no action needed
- stats.records_created_locally += 1;
- }
- }
-
- // Incoming-only records - generate INSERTs into the base
- for incoming_record in incoming_map.values() {
- stats.records_inserted += 1;
- if let Some(stmt) = generate_insert_sql(table_name, incoming_record) {
- statements.push(stmt);
- }
- }
-
- statements
-}
-
-/// The `(column, value)` pairs that address `record`, or `None` when it carries no primary key.
-///
-/// `columns` always ends with the table's primary key (see `TableConfig::identity_columns`). A row
-/// missing an earlier column (a manifest-scoped row with no scope, which the schema no longer permits
-/// but an older vault can still carry) is addressed by the columns it does carry, rather than emitting
-/// a `WHERE ManifestId = NULL` that silently matches nothing.
-fn get_identity<'a>(record: &Record, columns: &[&'a str]) -> Option<Vec<(&'a str, serde_json::Value)>> {
- let (primary_key, scope_columns) = columns.split_last()?;
- let primary_value = record.get(*primary_key).filter(|v| !v.is_null())?.clone();
-
- let mut identity: Vec<(&'a str, serde_json::Value)> = scope_columns
- .iter()
- .filter_map(|column| Some((*column, record.get(*column).filter(|v| !v.is_null())?.clone())))
- .collect();
- identity.push((*primary_key, primary_value));
- Some(identity)
-}
-
/// Stable string key over `columns`. A column the record does not carry contributes an empty part,
-/// so a row missing one still matches its counterpart rather than dropping out of the merge.
+/// so a row missing one still matches its counterpart rather than dropping out of the merge. See
+/// [`crate::vault_codec::identity_part`]: ids compare case-insensitively, everything else exactly as spelled.
fn get_key(record: &Record, columns: &[&str]) -> String {
columns
.iter()
- .map(|column| record.get(*column).filter(|v| !v.is_null()).map(key_part).unwrap_or_default())
+ .map(|column| record.get(*column).filter(|v| !v.is_null()).map(crate::vault_codec::identity_part).unwrap_or_default())
.collect::<Vec<_>>()
.join(":")
}
-/// One key component rendered as a string. See [`crate::vault_codec::identity_part`]: ids compare
-/// case-insensitively, everything else exactly as it is spelled.
-fn key_part(value: &serde_json::Value) -> String {
- crate::vault_codec::identity_part(value)
-}
-
-/// Get the UpdatedAt timestamp from a record.
-/// Handles both RFC3339 format (2025-12-11T06:50:10.674Z) and
-/// SQLite format (2025-12-11 06:50:10.674).
-pub(crate) fn get_updated_at(record: &Record) -> Option<DateTime<Utc>> {
- record
- .get("UpdatedAt")
- .and_then(|v| v.as_str())
- .and_then(|s| {
- // Try RFC3339 first
- DateTime::parse_from_rfc3339(s)
- .map(|dt| dt.with_timezone(&Utc))
- .ok()
- .or_else(|| {
- // Try SQLite format: "YYYY-MM-DD HH:MM:SS.mmm"
- chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f")
- .ok()
- .map(|naive| naive.and_utc())
- })
- })
-}
-
-/// Generate an INSERT SQL statement for a record.
-/// Uses INSERT OR REPLACE to handle potential conflicts.
-fn generate_insert_sql(table_name: &str, record: &Record) -> Option<SqlStatement> {
- if record.is_empty() {
- return None;
- }
-
- // Sort column names for consistent ordering
- let mut columns: Vec<&String> = record.keys().collect();
- columns.sort();
-
- let column_list = columns.iter().map(|c| c.as_str()).collect::<Vec<_>>().join(", ");
- let placeholders = columns.iter().map(|_| "?").collect::<Vec<_>>().join(", ");
- let params: Vec<serde_json::Value> = columns.iter().map(|c| record[*c].clone()).collect();
-
- let sql = format!(
- "INSERT OR REPLACE INTO {} ({}) VALUES ({})",
- table_name, column_list, placeholders
- );
-
- Some(SqlStatement { sql, params })
-}
-
-/// Generate an UPDATE SQL statement for a record, addressing the base row by `identity`: the *base*
-/// row's own `(column, value)` pairs, so it keeps its identity when a semantically-matched incoming row
-/// wins. Those columns are excluded from the SET clause for the same reason, since writing the incoming
-/// `ManifestId`/`Id` would move the row out of the namespace it was matched in.
-fn generate_update_sql(table_name: &str, record: &Record, identity: &[(&str, serde_json::Value)]) -> Option<SqlStatement> {
- if record.is_empty() {
- return None;
- }
-
- // Sort column names for consistent ordering, excluding the identity columns
- let mut columns: Vec<&String> = record.keys().filter(|c| !identity.iter().any(|(id_col, _)| *id_col == c.as_str())).collect();
- columns.sort();
-
- if columns.is_empty() {
- return None;
- }
-
- let set_clause = columns.iter().map(|c| format!("{} = ?", c)).collect::<Vec<_>>().join(", ");
- let where_clause = identity.iter().map(|(column, _)| format!("{} = ?", column)).collect::<Vec<_>>().join(" AND ");
-
- let mut params: Vec<serde_json::Value> = columns.iter().map(|c| record[*c].clone()).collect();
- params.extend(identity.iter().map(|(_, value)| value.clone()));
-
- let sql = format!("UPDATE {} SET {} WHERE {}", table_name, set_clause, where_clause);
-
- Some(SqlStatement { sql, params })
-}
-
#[cfg(test)]
mod tests {
use super::*;
-
- fn make_record(id: &str, updated_at: &str) -> Record {
- let mut record = HashMap::new();
- record.insert("Id".to_string(), serde_json::json!(id));
- record.insert("UpdatedAt".to_string(), serde_json::json!(updated_at));
- record.insert("Name".to_string(), serde_json::json!(format!("Record {}", id)));
- record
- }
-
- #[test]
- fn test_local_wins_when_newer() {
- let local = vec![make_record("1", "2024-01-02T00:00:00Z")];
- let server = vec![make_record("1", "2024-01-01T00:00:00Z")];
- let mut stats = MergeStats::default();
-
- let statements = merge_table("Test", &local, &server, &["Id"], &["Id"], &mut stats);
-
- assert_eq!(stats.records_from_local, 1);
- assert_eq!(stats.records_from_server, 0);
- assert!(statements.is_empty()); // No SQL needed when local wins
- }
-
- #[test]
- fn test_server_wins_when_newer() {
- let local = vec![make_record("1", "2024-01-01T00:00:00Z")];
- let server = vec![make_record("1", "2024-01-02T00:00:00Z")];
- let mut stats = MergeStats::default();
-
- let statements = merge_table("Test", &local, &server, &["Id"], &["Id"], &mut stats);
-
- assert_eq!(stats.records_from_server, 1);
- assert_eq!(stats.conflicts, 1);
- assert_eq!(statements.len(), 1);
- assert!(statements[0].sql.starts_with("UPDATE Test SET"));
- }
-
- #[test]
- fn test_server_only_record_inserted() {
- let local: Vec<Record> = vec![];
- let server = vec![make_record("1", "2024-01-01T00:00:00Z")];
- let mut stats = MergeStats::default();
-
- let statements = merge_table("Test", &local, &server, &["Id"], &["Id"], &mut stats);
-
- assert_eq!(stats.records_inserted, 1);
- assert_eq!(statements.len(), 1);
- assert!(statements[0].sql.starts_with("INSERT OR REPLACE INTO Test"));
- }
+ use std::collections::HashMap;
#[test]
- fn test_local_only_record_kept() {
- let local = vec![make_record("1", "2024-01-01T00:00:00Z")];
- let server: Vec<Record> = vec![];
- let mut stats = MergeStats::default();
-
- let statements = merge_table("Test", &local, &server, &["Id"], &["Id"], &mut stats);
-
- assert_eq!(stats.records_created_locally, 1);
- assert!(statements.is_empty()); // No SQL needed
- }
-
- #[test]
- fn test_merge_vaults_json() {
- // Base is the server vault. A newer LOCAL row wins and is written onto the server base as an
- // UPDATE. (A newer server row would win with no statement, since the base already holds it.)
- let input = MergeInput {
- local_tables: vec![TableData {
- name: "Items".to_string(),
- records: vec![make_record("1", "2024-01-02T00:00:00Z")],
- }],
- server_tables: vec![TableData {
- name: "Items".to_string(),
- records: vec![make_record("1", "2024-01-01T00:00:00Z")],
- }],
+ fn keys_compare_guids_case_insensitively_and_survive_a_missing_column() {
+ const MANIFEST: &str = "1dd1a3fd-8e0e-4b3f-9a3a-0f1a2b3c4d5e";
+ const OTHER_MANIFEST: &str = "2ee2b4fe-9f1f-4c40-8b4b-1a2b3c4d5e6f";
+ const ROW: &str = "3ff3c50f-a020-4d51-9c5c-2b3c4d5e6f70";
+ let row = |manifest: &str, id: &str| -> Record {
+ HashMap::from([("ManifestId".to_string(), serde_json::json!(manifest)), ("Id".to_string(), serde_json::json!(id))])
};
+ let columns = ["ManifestId", "Id"];
- let input_json = serde_json::to_string(&input).unwrap();
- let output_json = merge_vaults_json(&input_json).unwrap();
- let output: MergeOutput = serde_json::from_str(&output_json).unwrap();
-
- assert!(output.success);
- assert_eq!(output.stats.conflicts, 1);
- // Should have one UPDATE statement applied to the server base.
- assert_eq!(output.statements.len(), 1);
- assert!(output.statements[0].sql.starts_with("UPDATE Items SET"));
- }
-
- #[test]
- fn merge_vault_server_newer_is_noop_on_base() {
- // The mirror case: a newer SERVER row already sits in the base, so the merge emits nothing.
- let input = MergeInput {
- local_tables: vec![TableData { name: "Items".to_string(), records: vec![make_record("1", "2024-01-01T00:00:00Z")] }],
- server_tables: vec![TableData { name: "Items".to_string(), records: vec![make_record("1", "2024-01-02T00:00:00Z")] }],
- };
- let output = merge_vaults(input).unwrap();
- assert!(output.success);
- assert!(output.statements.is_empty(), "server base already holds the winning row");
- }
-
- #[test]
- fn merge_vault_local_only_row_inserted_into_base() {
- // A row created offline (local only) must be inserted into the server base.
- let input = MergeInput {
- local_tables: vec![TableData { name: "Items".to_string(), records: vec![make_record("local-only", "2024-01-01T00:00:00Z")] }],
- server_tables: vec![TableData { name: "Items".to_string(), records: vec![] }],
- };
- let output = merge_vaults(input).unwrap();
- assert_eq!(output.statements.len(), 1);
- assert!(output.statements[0].sql.starts_with("INSERT OR REPLACE INTO Items"));
- }
-
- fn logo(id: &str, source: &str, updated_at: &str) -> Record {
- let mut r = HashMap::new();
- r.insert("Id".to_string(), serde_json::json!(id));
- r.insert("Source".to_string(), serde_json::json!(source));
- r.insert("UpdatedAt".to_string(), serde_json::json!(updated_at));
- r
- }
-
- #[test]
- fn logos_merge_by_id_so_same_domain_in_two_scopes_survives() {
- // A personal logo and a shared manifest's logo for the same domain are different rows by design
- // (see vault_codec::logos). Merging Logos by Id keeps both.
- let mut personal = logo(&crate::vault_codec::logo_id_for_source("m-personal", "github.com"), "github.com", "2024-01-01T00:00:00Z");
- personal.insert("ManifestId".to_string(), serde_json::json!("m-personal"));
- let mut shared = logo(&crate::vault_codec::logo_id_for_source("m-1", "github.com"), "github.com", "2024-01-02T00:00:00Z");
- shared.insert("ManifestId".to_string(), serde_json::json!("m-1"));
-
- let output = merge_vaults(MergeInput {
- local_tables: vec![TableData { name: "Logos".to_string(), records: vec![personal.clone(), shared.clone()] }],
- server_tables: vec![TableData { name: "Logos".to_string(), records: vec![personal] }],
- })
- .unwrap();
-
- // The shared-scope row is new to the base and is inserted alongside, not merged into, the
- // personal one.
- assert_eq!(output.statements.len(), 1);
- assert!(output.statements[0].sql.starts_with("INSERT OR REPLACE INTO Logos"));
- assert_eq!(output.stats.conflicts, 0, "different scopes are never a conflict");
- }
-
- #[test]
- fn logos_merge_resolves_same_scope_rows_by_lww() {
- // Within one scope, two writers produce the SAME derived Id, so an ordinary LWW settles whose
- // favicon bytes win, and the UPDATE addresses the row by its full (ManifestId, Id) identity.
- let id = crate::vault_codec::logo_id_for_source("m-personal", "github.com");
- let scoped = |updated_at: &str| {
- let mut row = logo(&id, "github.com", updated_at);
- row.insert("ManifestId".to_string(), serde_json::json!("m-personal"));
- row
- };
- let base = vec![scoped("2024-01-01T00:00:00Z")];
- let incoming = vec![scoped("2024-01-02T00:00:00Z")];
- let mut stats = MergeStats::default();
-
- let statements = merge_table("Logos", &base, &incoming, &["ManifestId", "Id"], &["ManifestId", "Id"], &mut stats);
-
- assert_eq!(stats.conflicts, 1);
- assert_eq!(stats.records_inserted, 0, "no second row for the same scope+domain");
- assert_eq!(statements.len(), 1);
- assert!(statements[0].sql.starts_with("UPDATE Logos SET"));
- assert!(statements[0].sql.ends_with("WHERE ManifestId = ? AND Id = ?"));
- assert_eq!(statements[0].params.last().unwrap(), &serde_json::json!(id));
- }
-
- #[test]
- fn same_id_in_two_manifests_never_merges() {
- // The point of the composite key: two manifests may each hold a row with the same Id. Neither
- // may update the other, however much newer it is: they are different rows.
- let scoped = |manifest_id: &str, updated_at: &str| {
- let mut row = make_record("shared-id", updated_at);
- row.insert("ManifestId".to_string(), serde_json::json!(manifest_id));
- row
- };
-
- let output = merge_vaults(MergeInput {
- local_tables: vec![TableData { name: "Items".to_string(), records: vec![scoped("m-1", "2024-01-09T00:00:00Z")] }],
- server_tables: vec![TableData { name: "Items".to_string(), records: vec![scoped("m-personal", "2024-01-01T00:00:00Z")] }],
- })
- .unwrap();
-
- assert_eq!(output.stats.conflicts, 0, "rows in different manifests are never in conflict");
- assert_eq!(output.statements.len(), 1);
- assert!(output.statements[0].sql.starts_with("INSERT OR REPLACE INTO Items"), "the other manifest's row is inserted alongside");
- }
-
- #[test]
- fn scoped_update_never_rewrites_the_rows_own_scope() {
- // FieldValues match on (ManifestId, ItemId, FieldKey), so a winning incoming row may carry a
- // different Id than the base row it updates. Neither half of the identity may leak into SET:
- // writing them would move the base row out of the namespace it was matched in.
- let field_value = |id: &str, updated_at: &str| {
- let mut row: Record = HashMap::new();
- row.insert("Id".to_string(), serde_json::json!(id));
- row.insert("ManifestId".to_string(), serde_json::json!("m-personal"));
- row.insert("ItemId".to_string(), serde_json::json!("item-1"));
- row.insert("FieldKey".to_string(), serde_json::json!("username"));
- row.insert("Value".to_string(), serde_json::json!(id));
- row.insert("UpdatedAt".to_string(), serde_json::json!(updated_at));
- row
- };
-
- let output = merge_vaults(MergeInput {
- local_tables: vec![TableData { name: "FieldValues".to_string(), records: vec![field_value("fv-incoming", "2024-01-09T00:00:00Z")] }],
- server_tables: vec![TableData { name: "FieldValues".to_string(), records: vec![field_value("fv-base", "2024-01-01T00:00:00Z")] }],
- })
- .unwrap();
-
- assert_eq!(output.statements.len(), 1);
- let stmt = &output.statements[0];
- assert!(stmt.sql.ends_with("WHERE ManifestId = ? AND Id = ?"));
-
- let assignments: Vec<&str> = stmt.sql.trim_start_matches("UPDATE FieldValues SET ").split(" WHERE ").next().unwrap().split(", ").collect();
- assert!(!assignments.contains(&"ManifestId = ?"), "scope must not be in the SET clause");
- assert!(!assignments.contains(&"Id = ?"), "identity must not be in the SET clause");
- assert_eq!(stmt.params.last().unwrap(), &serde_json::json!("fv-base"), "the base row keeps its own Id");
- }
-
- fn setting(key: &str, value: &str, updated_at: &str) -> Record {
- let mut r = HashMap::new();
- r.insert("ManifestId".to_string(), serde_json::json!("m-personal"));
- r.insert("Key".to_string(), serde_json::json!(key));
- r.insert("Value".to_string(), serde_json::json!(value));
- r.insert("UpdatedAt".to_string(), serde_json::json!(updated_at));
- r
- }
-
- #[test]
- fn settings_merge_by_key_within_their_manifest() {
- // A setting is identified by (ManifestId, Key), with no generated id: a newer incoming value
- // updates the base row addressed by that pair instead of becoming a second row.
- let base = vec![setting("theme", "light", "2024-01-01T00:00:00Z")];
- let incoming = vec![setting("theme", "dark", "2024-01-02T00:00:00Z")];
- let mut stats = MergeStats::default();
-
- let statements = merge_table("Settings", &base, &incoming, &["ManifestId", "Key"], &["ManifestId", "Key"], &mut stats);
-
- assert_eq!(stats.conflicts, 1);
- assert_eq!(stats.records_inserted, 0, "the same key must not produce a second row");
- assert_eq!(statements.len(), 1);
- assert!(statements[0].sql.starts_with("UPDATE Settings SET"));
- assert!(statements[0].sql.ends_with("WHERE ManifestId = ? AND Key = ?"));
- // The identity columns address the row; they must never be written back through the SET clause.
- assert!(!statements[0].sql.contains("Key = ?,"));
- assert_eq!(statements[0].params.last().unwrap(), &serde_json::json!("theme"));
- }
-
- #[test]
- fn settings_of_two_manifests_never_merge() {
- // The same key may exist in two manifests; they are different settings, as everywhere else.
- let scoped = |manifest_id: &str, value: &str| {
- let mut row = setting("theme", value, "2024-01-01T00:00:00Z");
- row.insert("ManifestId".to_string(), serde_json::json!(manifest_id));
- row
- };
-
- let output = merge_vaults(MergeInput {
- local_tables: vec![TableData { name: "Settings".to_string(), records: vec![scoped("m-shared", "dark")] }],
- server_tables: vec![TableData { name: "Settings".to_string(), records: vec![scoped("m-personal", "light")] }],
- })
- .unwrap();
-
- assert_eq!(output.stats.conflicts, 0, "the same key in two manifests is not a conflict");
- assert_eq!(output.statements.len(), 1);
- assert!(output.statements[0].sql.starts_with("INSERT OR REPLACE INTO Settings"));
- }
-
- #[test]
- fn settings_merge_local_and_server_only_rows() {
- let base = vec![setting("theme", "dark", "2024-01-02T00:00:00Z")];
- let incoming = vec![setting("language", "en", "2024-01-01T00:00:00Z")];
- let mut stats = MergeStats::default();
-
- let statements = merge_table("Settings", &base, &incoming, &["ManifestId", "Key"], &["ManifestId", "Key"], &mut stats);
-
- assert_eq!(stats.records_created_locally, 1);
- assert_eq!(stats.records_inserted, 1);
- assert_eq!(statements.len(), 1);
- assert!(statements[0].sql.starts_with("INSERT OR REPLACE INTO Settings"));
- }
-
- #[test]
- fn settings_and_encryption_keys_are_syncable() {
- let settings = SYNCABLE_TABLES.iter().find(|t| t.name == "Settings").unwrap();
- assert_eq!(settings.primary_key_columns, &["Key"]);
- assert!(!settings.uses_composite_key(), "a key already names one setting per manifest; no semantic match key is needed");
-
- let keys = SYNCABLE_TABLES.iter().find(|t| t.name == "EncryptionKeys").unwrap();
- assert_eq!(keys.primary_key_columns, &["Id"]);
-
- assert!(SYNCABLE_TABLE_NAMES.contains(&"Settings"));
- assert!(SYNCABLE_TABLE_NAMES.contains(&"EncryptionKeys"));
- }
-
- #[test]
- fn item_stats_merge_is_last_use_wins_per_item() {
- // ItemStats.Id *is* the item's id, so two devices recording a use of the same item produce the
- // same row and LWW settles it: recording a use is a plain upsert, no semantic match key needed.
- let stat = |manifest_id: &str, item_id: &str, last_used_at: &str| -> Record {
- let mut r = HashMap::new();
- r.insert("ManifestId".to_string(), serde_json::json!(manifest_id));
- r.insert("Id".to_string(), serde_json::json!(item_id));
- r.insert("LastUsedAt".to_string(), serde_json::json!(last_used_at));
- r.insert("UpdatedAt".to_string(), serde_json::json!(last_used_at));
- r
- };
-
- let output = merge_vaults(MergeInput {
- local_tables: vec![TableData { name: "ItemStats".to_string(), records: vec![stat("m-personal", "i-1", "2024-01-09T00:00:00Z")] }],
- server_tables: vec![TableData { name: "ItemStats".to_string(), records: vec![stat("m-personal", "i-1", "2024-01-01T00:00:00Z")] }],
- })
- .unwrap();
-
- assert_eq!(output.stats.conflicts, 1);
- assert_eq!(output.stats.records_inserted, 0);
- assert_eq!(output.statements.len(), 1);
- assert!(output.statements[0].sql.starts_with("UPDATE ItemStats SET"));
- assert!(output.statements[0].sql.ends_with("WHERE ManifestId = ? AND Id = ?"));
- }
-
- #[test]
- fn item_stats_of_two_manifests_never_merge() {
- // The same item id may exist in two manifests; their stats are different rows, as everywhere else.
- let stat = |manifest_id: &str| -> Record {
- let mut r = HashMap::new();
- r.insert("ManifestId".to_string(), serde_json::json!(manifest_id));
- r.insert("Id".to_string(), serde_json::json!("i-1"));
- r.insert("UpdatedAt".to_string(), serde_json::json!("2024-01-01T00:00:00Z"));
- r
- };
-
- let output = merge_vaults(MergeInput {
- local_tables: vec![TableData { name: "ItemStats".to_string(), records: vec![stat("m-shared")] }],
- server_tables: vec![TableData { name: "ItemStats".to_string(), records: vec![stat("m-personal")] }],
- })
- .unwrap();
-
- assert_eq!(output.stats.conflicts, 0);
- assert_eq!(output.statements.len(), 1);
- assert!(output.statements[0].sql.starts_with("INSERT OR REPLACE INTO ItemStats"));
- }
-
- #[test]
- fn logos_config_merges_by_id_not_source() {
- // Guard on the decision itself: Logos identity is the codec-derived Id, so this table must
- // never go back to composite-key matching on Source.
- let cfg = SYNCABLE_TABLES.iter().find(|t| t.name == "Logos").unwrap();
- assert!(!cfg.uses_composite_key());
- assert_eq!(cfg.primary_key_columns, &["Id"]);
- }
-
- #[test]
- fn merge_never_touches_codec_overflow_carrier() {
- // The merge base is the server vault, so the carrier rides along on it untouched (implicit
- // server-wins). Even if a platform passes it in the merge input, merge must emit no statement
- // referencing it: it is not a syncable table.
- // a syncable table.
- let overflow_table = crate::vault_codec::OVERFLOW_TABLE;
- let overflow_row = |data: &str| -> Record {
- [("Id".to_string(), serde_json::json!("00000000-0000-0000-0000-00000000c0de")), ("Data".to_string(), serde_json::json!(data))].into_iter().collect()
- };
-
- let output = merge_vaults(MergeInput {
- local_tables: vec![TableData { name: overflow_table.to_string(), records: vec![overflow_row("{\"stale\":true}")] }],
- server_tables: vec![TableData { name: overflow_table.to_string(), records: vec![overflow_row("{\"fresh\":true}")] }],
- })
- .unwrap();
- assert!(!output.statements.iter().any(|s| s.sql.contains(overflow_table)), "no carrier statements: the server base owns the overflow");
- }
-
- #[test]
- fn merge_table_names_excludes_overflow_carrier() {
- let names = merge_table_names();
- assert_eq!(names.len(), SYNCABLE_TABLE_NAMES.len());
- assert!(!names.contains(&crate::vault_codec::OVERFLOW_TABLE), "carrier is not part of the merge input");
- }
-
- // Cross-manifest guards. The local vault holds every manifest's rows in one SQLite file, kept apart
- // only by the `(ManifestId, Id)` identity, so a merge can resolve a row against one in its own
- // manifest and nowhere else. Asserted for every syncable table at once, so a table added later is
- // covered the day it is registered.
-
- /// A row of `config` addressed by `manifest_id`, carrying whatever columns the table's own key
- /// needs plus a payload to tell two versions apart.
- fn scoped_row(config: &TableConfig, manifest_id: &str, updated_at: &str, payload: &str) -> Record {
- let mut record: Record = HashMap::new();
- record.insert(crate::vault_codec::MANIFEST_ID_COL.to_string(), serde_json::json!(manifest_id));
- for column in config.primary_key_columns {
- record.insert((*column).to_string(), serde_json::json!("row-1"));
- }
- for column in config.composite_key_columns {
- record.entry((*column).to_string()).or_insert_with(|| serde_json::json!("k"));
- }
- record.insert("UpdatedAt".to_string(), serde_json::json!(updated_at));
- record.insert("Payload".to_string(), serde_json::json!(payload));
- record
- }
-
- fn merge_one_table(config: &TableConfig, base: Record, incoming: Record) -> MergeOutput {
- merge_vaults(MergeInput {
- local_tables: vec![TableData { name: config.name.to_string(), records: vec![incoming] }],
- server_tables: vec![TableData { name: config.name.to_string(), records: vec![base] }],
- })
- .unwrap()
- }
-
- #[test]
- fn every_syncable_table_is_manifest_scoped() {
- // A table registered without `manifest_scoped()` is matched on its primary key alone, so the
- // same id in two manifests becomes one row and the later write silently overwrites the other
- // manifest's. There is no table for which that is right, so the invariant holds for all of them.
- for config in SYNCABLE_TABLES {
- assert!(config.manifest_scoped, "{} must be manifest_scoped or it merges across manifests", config.name);
- assert_eq!(config.identity_columns().first(), Some(&crate::vault_codec::MANIFEST_ID_COL), "{} must be addressed by its manifest first", config.name);
- }
- }
-
- #[test]
- fn no_syncable_table_merges_a_row_into_another_manifest() {
- // The same id in two manifests is two rows, in every table alike: neither may update the
- // other, however much newer it is.
- for config in SYNCABLE_TABLES {
- let base = scoped_row(config, "m-personal", "2024-01-01T00:00:00Z", "personal");
- let incoming = scoped_row(config, "m-shared", "2024-01-09T00:00:00Z", "shared");
- let output = merge_one_table(config, base, incoming);
-
- assert_eq!(output.stats.conflicts, 0, "{}: rows in different manifests are not in conflict", config.name);
- assert_eq!(output.statements.len(), 1, "{}", config.name);
- assert!(output.statements[0].sql.starts_with(&format!("INSERT OR REPLACE INTO {}", config.name)), "{}: the other manifest's row is inserted alongside, never merged in", config.name);
- }
- }
-
- #[test]
- fn no_merge_update_ever_rewrites_a_rows_manifest() {
- // The other half: a winning incoming row is written into the base row it matched, which keeps
- // its own address. `ManifestId` in the SET clause would move a row out of its manifest on an
- // ordinary edit, handing a personal row to whoever holds the manifest it landed in.
- for config in SYNCABLE_TABLES {
- let base = scoped_row(config, "m-personal", "2024-01-01T00:00:00Z", "old");
- let incoming = scoped_row(config, "m-personal", "2024-01-09T00:00:00Z", "new");
- let output = merge_one_table(config, base, incoming);
-
- assert_eq!(output.statements.len(), 1, "{}", config.name);
- let sql = &output.statements[0].sql;
- let prefix = format!("UPDATE {} SET ", config.name);
- assert!(sql.starts_with(&prefix), "{}: expected an in-place update, got {}", config.name, sql);
-
- let set_clause = sql.trim_start_matches(&prefix).split(" WHERE ").next().unwrap();
- assert!(!set_clause.contains(crate::vault_codec::MANIFEST_ID_COL), "{}: an update must never rewrite the row's manifest ({})", config.name, sql);
- let expected_where = format!("WHERE {}", config.identity_columns().iter().map(|c| format!("{} = ?", c)).collect::<Vec<_>>().join(" AND "));
- assert!(sql.ends_with(&expected_where), "{}: the base row is addressed by its full identity ({})", config.name, sql);
- }
- }
-
- #[test]
- fn encryption_keys_of_two_manifests_never_merge() {
- // Spelled out for the one table where a mis-merge is unrecoverable rather than merely wrong: a
- // manifest's keypair decrypts the mail sent to its aliases, so letting another manifest's row
- // overwrite it makes every message already delivered under it unreadable.
- let key = |manifest_id: &str, private_key: &str, updated_at: &str| -> Record {
- [
- ("ManifestId".to_string(), serde_json::json!(manifest_id)),
- ("Id".to_string(), serde_json::json!("ek-1")),
- ("PublicKey".to_string(), serde_json::json!(format!("pub-{manifest_id}"))),
- ("PrivateKey".to_string(), serde_json::json!(private_key)),
- ("IsPrimary".to_string(), serde_json::json!(1)),
- ("UpdatedAt".to_string(), serde_json::json!(updated_at)),
- ]
- .into_iter()
- .collect()
- };
-
- let output = merge_vaults(MergeInput {
- local_tables: vec![TableData { name: "EncryptionKeys".to_string(), records: vec![key("m-shared", "priv-shared", "2024-01-09T00:00:00Z")] }],
- server_tables: vec![TableData { name: "EncryptionKeys".to_string(), records: vec![key("m-personal", "priv-personal", "2024-01-01T00:00:00Z")] }],
- })
- .unwrap();
-
- assert_eq!(output.stats.conflicts, 0);
- assert_eq!(output.statements.len(), 1);
- assert!(output.statements[0].sql.starts_with("INSERT OR REPLACE INTO EncryptionKeys"));
- assert!(!output.statements[0].sql.contains("UPDATE"), "the personal manifest's private key is never overwritten by another manifest's");
- }
-
- #[test]
- fn test_generate_insert_sql() {
- let record = make_record("test-id", "2024-01-01T00:00:00Z");
- let stmt = generate_insert_sql("Items", &record).unwrap();
-
- assert!(stmt.sql.contains("INSERT OR REPLACE INTO Items"));
- assert!(stmt.sql.contains("Id"));
- assert!(stmt.sql.contains("Name"));
- assert!(stmt.sql.contains("UpdatedAt"));
- assert_eq!(stmt.params.len(), 3);
- }
-
- #[test]
- fn test_generate_update_sql() {
- let record = make_record("test-id", "2024-01-01T00:00:00Z");
- let stmt = generate_update_sql("Items", &record, &[("Id", serde_json::json!("test-id"))]).unwrap();
-
- assert!(stmt.sql.starts_with("UPDATE Items SET"));
- assert!(stmt.sql.contains("WHERE Id = ?"));
- // Should not include Id in SET clause
- assert!(!stmt.sql.contains("Id = ?,")); // Id only at end for WHERE
- // Params: Name, UpdatedAt (sorted), then Id for WHERE
- assert_eq!(stmt.params.len(), 3);
- // Last param should be the Id
- assert_eq!(stmt.params[2], serde_json::json!("test-id"));
+ // Two writers spelling the same ids in different cases address one row (see `identity_part`).
+ assert_eq!(get_key(&row(MANIFEST, ROW), &columns), get_key(&row(&MANIFEST.to_uppercase(), &ROW.to_uppercase()), &columns));
+ assert_ne!(get_key(&row(MANIFEST, ROW), &columns), get_key(&row(OTHER_MANIFEST, ROW), &columns));
+ assert_eq!(get_key(&HashMap::from([("Id".to_string(), serde_json::json!(ROW))]), &columns), format!(":{}", ROW));
}
}
diff --git a/core/rust/src/vault_merge/types.rs b/core/rust/src/vault_merge/types.rs
deleted file mode 100644
index beb79e451..000000000
--- a/core/rust/src/vault_merge/types.rs
+++ /dev/null
@@ -1,12 +0,0 @@
-//! Common types used across the AliasVault core library.
-//!
-//! The table registry itself lives in [`crate::vault_model`], generated from the TypeScript source
-//! of truth in core/models/src/vault/VaultTableRegistry.ts; this module re-exports it for the
-//! merge code and its existing call sites.
-
-pub use crate::vault_model::{TableConfig, SYNCABLE_TABLES, SYNCABLE_TABLE_NAMES};
-
-/// The tables a platform must read into the merge input: the syncable tables.
-pub fn merge_table_names() -> Vec<&'static str> {
- SYNCABLE_TABLE_NAMES.to_vec()
-}
diff --git a/core/rust/src/vault_model/generated.rs b/core/rust/src/vault_model/generated.rs
index 0426f8dfe..aa7e52225 100644
--- a/core/rust/src/vault_model/generated.rs
+++ b/core/rust/src/vault_model/generated.rs
@@ -15,7 +15,6 @@ pub static SYNCABLE_TABLES: &[TableConfig] = &[
TableConfig::new("FieldValues")
.manifest_scoped()
.item_child()
- .with_composite_key(&["ManifestId", "ItemId", "FieldKey", "FieldDefinitionId"])
.with_canonical_key(&["ManifestId", "ItemId", "FieldKey", "FieldDefinitionId", "Id"]),
TableConfig::new("Folders").manifest_scoped(),
TableConfig::new("Tags").manifest_scoped(),
diff --git a/core/rust/src/vault_model/mod.rs b/core/rust/src/vault_model/mod.rs
index 35d20d6d5..95f54699b 100644
--- a/core/rust/src/vault_model/mod.rs
+++ b/core/rust/src/vault_model/mod.rs
@@ -8,18 +8,23 @@ mod generated;
pub use generated::*;
+/// The comparison key of an id (GUIDs compare case-insensitively).
+pub fn id_key(id: &str) -> String {
+ id.trim().to_ascii_lowercase()
+}
+
+/// True when two ids name the same row.
+pub fn ids_equal(a: &str, b: &str) -> bool {
+ a.trim().eq_ignore_ascii_case(b.trim())
+}
+
/// Configuration for a syncable table.
#[derive(Debug, Clone)]
pub struct TableConfig {
/// Table name in the database
pub name: &'static str,
- /// Columns to use for composite key matching (if any).
- /// When empty, matching falls back to the table's [`identity_columns`](TableConfig::identity_columns).
- /// When set, these columns are concatenated to form the composite key.
- pub composite_key_columns: &'static [&'static str],
- /// Match columns for the canonical merge only, where both sides are first normalized to the manifest
- /// shape (see `vault_codec::normalize`). Empty means the canonical merge uses the same rule as
- /// the statement merge (composite key, else identity).
+ /// Merge match columns, applied after both sides are normalized to the manifest shape (see
+ /// `vault_codec::normalize`). Empty means rows match on [`identity_columns`](TableConfig::identity_columns).
pub canonical_key_columns: &'static [&'static str],
/// The columns that name a row within its manifest. Defaults to `["Id"]`; tables keyed
/// differently (Settings by "Key", ItemTags by its natural key) override this.
@@ -34,7 +39,6 @@ impl TableConfig {
pub const fn new(name: &'static str) -> Self {
Self {
name,
- composite_key_columns: &[],
canonical_key_columns: &[],
primary_key_columns: &["Id"],
manifest_scoped: false,
@@ -42,11 +46,6 @@ impl TableConfig {
}
}
- pub const fn with_composite_key(mut self, columns: &'static [&'static str]) -> Self {
- self.composite_key_columns = columns;
- self
- }
-
pub const fn with_canonical_key(mut self, columns: &'static [&'static str]) -> Self {
self.canonical_key_columns = columns;
self
@@ -67,11 +66,6 @@ impl TableConfig {
self
}
- /// Returns true if this table uses composite key matching.
- pub const fn uses_composite_key(&self) -> bool {
- !self.composite_key_columns.is_empty()
- }
-
/// The columns that together identify one row of this table: `(ManifestId, primary key columns)`
/// for a manifest-scoped table, the primary key columns alone otherwise.
pub fn identity_columns(&self) -> Vec<&'static str> {
@@ -85,3 +79,23 @@ impl TableConfig {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn every_syncable_table_is_manifest_scoped() {
+ for config in SYNCABLE_TABLES {
+ assert!(config.manifest_scoped, "{} must be manifest_scoped or it merges across manifests", config.name);
+ assert_eq!(config.identity_columns().first(), Some(&MANIFEST_ID_COL), "{} must be addressed by its manifest first", config.name);
+ }
+ }
+
+ #[test]
+ fn id_keys_ignore_case_and_surrounding_whitespace() {
+ assert_eq!(id_key(" 1DD1A3FD-8e0e-4b3f-9a3a-0f1a2b3c4d5e "), "1dd1a3fd-8e0e-4b3f-9a3a-0f1a2b3c4d5e");
+ assert!(ids_equal("ABC", "abc "));
+ assert!(!ids_equal("abc", "abd"));
+ }
+}
diff --git a/core/rust/src/vault_sync/email_routing.rs b/core/rust/src/vault_sync/email_routing.rs
index e0c14a2a5..c6e181d22 100644
--- a/core/rust/src/vault_sync/email_routing.rs
+++ b/core/rust/src/vault_sync/email_routing.rs
@@ -5,7 +5,8 @@ use std::collections::{HashMap, HashSet};
use serde_json::Value;
use super::types::{ClaimedEmailAddress, EmailRoutingPush};
-use super::db::{truthy, value_string};
+use super::db::value_string;
+use crate::vault_codec::row::{rows_of, truthy};
use crate::vault_codec::Manifest;
/// The field key of an item's login email.
@@ -17,13 +18,13 @@ pub fn build_email_routing(manifests: &[Manifest], private_email_domains: &[Stri
let mut order: Vec<String> = Vec::new();
for manifest in manifests {
- let live_item_ids: HashSet<String> = rows_of(manifest, "Items")
+ let live_item_ids: HashSet<String> = rows_of(&manifest.tables, "Items")
.iter()
.filter(|row| !truthy(row.get("IsDeleted")) && row.get("DeletedAt").map_or(true, Value::is_null))
.filter_map(|row| row.get("Id").map(value_string))
.collect();
- for field_value in rows_of(manifest, "FieldValues") {
+ for field_value in rows_of(&manifest.tables, "FieldValues") {
if field_value.get("FieldKey").and_then(Value::as_str) != Some(FIELD_KEY_LOGIN_EMAIL) || truthy(field_value.get("IsDeleted")) {
continue;
}
@@ -60,10 +61,6 @@ pub fn build_email_routing(manifests: &[Manifest], private_email_domains: &[Stri
}
}
-fn rows_of<'a>(manifest: &'a Manifest, table: &str) -> &'a [crate::vault_codec::CodecRecord] {
- manifest.tables.get(table).map(Vec::as_slice).unwrap_or(&[])
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/core/rust/src/vault_sync/engine.rs b/core/rust/src/vault_sync/engine.rs
index 56a3a7659..8010a5732 100644
--- a/core/rust/src/vault_sync/engine.rs
+++ b/core/rust/src/vault_sync/engine.rs
@@ -4,13 +4,14 @@ use std::collections::HashMap;
use serde_json::Value;
-use super::errors::{Failure, LogoutReason, SyncError, SyncResult};
+use super::db::SchemaState;
+use super::errors::{LogoutReason, SyncError, SyncResult};
use super::merge::{self, PullAndMergeOutcome};
-use super::pull::{self, PullOutcome, SnapshotFacts};
+use super::pull::{self, PulledVault};
use super::push::{self, CanonicalizedSet, PushStatus};
use super::session::Host;
use super::state::{self, Ctx};
-use super::types::{Db, EmailRoutingDto, FullSyncResult, LogLevel, MigrateManifestResult, MigrationKind, MigrationStatusResult, OperationResult, SessionOutcome, StatusCheckResult, StatusResponse, SyncOperation, SyncRequest};
+use super::types::{Db, FailureFields, FullSyncResult, LogLevel, MigrateManifestResult, MigrationKind, MigrationStatusResult, OperationResult, SessionOutcome, StatusCheckResult, StatusResponse, SyncOperation, SyncRequest};
use super::{db, http, keys, legacy, version};
use crate::crypto;
@@ -72,14 +73,11 @@ fn success() -> FullSyncResult {
/// The outcome of a failed sync: a forced logout, or a coded error.
fn failure(error: &SyncError) -> FullSyncResult {
- match error.failure() {
- Failure::Logout(reason) => logout(reason),
- Failure::Coded(code) => FullSyncResult { success: false, error: Some(error.to_string()), error_code: Some(code), ..Default::default() },
- }
+ FullSyncResult { failure: error.into(), ..Default::default() }
}
fn logout(reason: LogoutReason) -> FullSyncResult {
- FullSyncResult { success: false, requires_logout: true, error_key: Some(reason), ..Default::default() }
+ FullSyncResult { failure: FailureFields::logout(reason), ..Default::default() }
}
/// Turn an error thrown during a sync into the outcome the host acts on. A transport failure with a local vault
@@ -139,7 +137,11 @@ async fn full_sync_once(ctx: &mut Ctx) -> SyncResult<Flow> {
let flow = if needs_pull {
pull_and_materialize_server_vault(ctx, grant_sync_changed_vault).await?
} else if ctx.is_dirty && !vault_predates_current_schema(ctx).await? {
- push_pending_local_changes(ctx, canonicalize_cache, grant_sync_changed_vault).await?
+ // Push path: server and client agree on every revision, so the pending local changes upload as-is.
+ match push_local_changes(ctx, canonicalize_cache, false, false).await? {
+ Some(vault_changed) => Flow::Done(FullSyncResult { success: true, has_new_vault: vault_changed || grant_sync_changed_vault, ..Default::default() }),
+ None => Flow::Resync { outdated: true },
+ }
} else {
// A vault the codec cannot canonicalize has no way to the server; the upgrade gate takes it from here.
Flow::Done(pending_migration_result(ctx).await?.unwrap_or_else(success))
@@ -262,10 +264,20 @@ async fn enter_offline_mode(ctx: &mut Ctx) -> SyncResult<FullSyncResult> {
Ok(FullSyncResult { success: true, was_offline: true, is_offline_mode: true, ..Default::default() })
}
+/// Where the local vault's schema stands against the current one.
+async fn schema_state(ctx: &mut Ctx) -> SyncResult<SchemaState> {
+ let schema = ctx.schema().await?;
+ db::schema_state(&ctx.host, &schema.migration_id).await
+}
+
+/// Whether the local vault is still on a schema the codec cannot canonicalize.
+async fn vault_predates_current_schema(ctx: &mut Ctx) -> SyncResult<bool> {
+ Ok(schema_state(ctx).await? != SchemaState::Current)
+}
+
/// Whether the vault still has to run the manifest migration: a stale schema, or a missing account key hierarchy.
async fn vault_requires_manifest_migration(ctx: &mut Ctx) -> SyncResult<bool> {
- let schema = ctx.schema().await?;
- Ok(db::requires_schema_migration(&ctx.host, &schema.migration_id).await? || !keys::has_local_vault_key(&ctx.host).await?)
+ Ok(schema_state(ctx).await? == SchemaState::Stale || !keys::has_local_vault_key(&ctx.host).await?)
}
/// Carry out the work the server has addressed to the current client (e.g. shared groups revocation/rotation actions).
@@ -340,132 +352,100 @@ async fn apply_server_directed_changes(ctx: &mut Ctx, status: &StatusResponse) -
Ok(true)
}
-/// Outcome for a sync that stored a freshly materialized vault.
-async fn materialized_vault_result(ctx: &mut Ctx, revision: i64, email_routing: EmailRoutingDto) -> SyncResult<FullSyncResult> {
- let sqlite_blob_upgrade_required = db::requires_legacy_sqlite_blob_migration(&ctx.host, Db::Local).await?;
- let manifest_migration_required = vault_requires_manifest_migration(ctx).await?;
- Ok(FullSyncResult { success: true, has_new_vault: true, sqlite_blob_upgrade_required, manifest_migration_required, pulled_revision: Some(revision), email_routing: Some(email_routing), ..Default::default() })
+/// Outcome for a sync that stored a freshly pulled vault.
+async fn materialized_vault_result(ctx: &mut Ctx, pulled: &PulledVault) -> SyncResult<FullSyncResult> {
+ let schema = schema_state(ctx).await?;
+ let manifest_migration_required = schema == SchemaState::Stale || !keys::has_local_vault_key(&ctx.host).await?;
+ Ok(FullSyncResult { success: true, has_new_vault: true, sqlite_blob_upgrade_required: schema == SchemaState::LegacyChain, manifest_migration_required, pulled_revision: Some(pulled.revision), email_routing: Some(pulled.email_routing.clone()), ..Default::default() })
}
-/// Store a pulled vault as the local vault, refusing (and re-syncing) when a mutation raced the pull.
-async fn store_pulled_vault(ctx: &mut Ctx, encrypted_vault: &str, revision: i64) -> SyncResult<bool> {
- let stored = state::store_vault(&ctx.host, encrypted_vault, false, Some(ctx.mutation_sequence), Some(revision)).await?;
+/// Store a pulled vault as the local one and only then commit its revisions as the local truth. A store the host
+/// refuses (a mutation raced the pull) re-runs the sync instead, which is the returned flow.
+async fn commit_pulled_vault(ctx: &mut Ctx, pulled: &PulledVault) -> SyncResult<Option<Flow>> {
+ let stored = state::store_vault(&ctx.host, &pulled.encrypted_vault, false, Some(ctx.mutation_sequence), Some(pulled.revision)).await?;
if !stored.success {
ctx.log("[VaultSync] Mutation detected during sync, re-syncing...").await;
- return Ok(false);
+ return Ok(Some(Flow::Resync { outdated: false }));
}
ctx.vault_changed = true;
- Ok(true)
+ pull::commit_revisions(ctx, &pulled.manifest_revisions, &pulled.bucket_revisions).await?;
+ Ok(None)
}
-/// Store the server's vault as the local vault, replacing whatever was there.
-async fn adopt_server_vault(ctx: &mut Ctx, encrypted_vault: &str, revision: i64, email_routing: EmailRoutingDto) -> SyncResult<Flow> {
- if !store_pulled_vault(ctx, encrypted_vault, revision).await? {
- return Ok(Flow::Resync { outdated: false });
+/// Store the server's vault as the local vault, replacing whatever was there, and report it.
+async fn adopt_server_vault(ctx: &mut Ctx, pulled: &PulledVault) -> SyncResult<Flow> {
+ if let Some(resync) = commit_pulled_vault(ctx, pulled).await? {
+ return Ok(resync);
}
- Ok(Flow::Done(materialized_vault_result(ctx, revision, email_routing).await?))
+ Ok(Flow::Done(materialized_vault_result(ctx, pulled).await?))
}
/// Pull the server's latest vault and merge it with what is stored locally when needed.
async fn pull_and_materialize_server_vault(ctx: &mut Ctx, grant_sync_changed_vault: bool) -> SyncResult<Flow> {
- let mut facts = SnapshotFacts::new();
-
if !ctx.is_dirty || !ctx.has_local_vault().await? {
- return adopt_pulled_vault(ctx, &mut facts).await;
+ return adopt_pulled_vault(ctx).await;
}
// A dirty vault the codec cannot canonicalize is not merged.
if vault_predates_current_schema(ctx).await? {
ctx.warn("[VaultSync] The local vault predates the current storage model, so its pending changes can be neither merged nor uploaded; taking the server's vault.").await;
- return adopt_pulled_vault(ctx, &mut facts).await;
+ return adopt_pulled_vault(ctx).await;
}
- canonical_pull_and_merge(ctx, &mut facts, grant_sync_changed_vault).await
+ canonical_pull_and_merge(ctx, grant_sync_changed_vault).await
}
/// Pull the server's vault and store it as the local one.
-async fn adopt_pulled_vault(ctx: &mut Ctx, facts: &mut SnapshotFacts) -> SyncResult<Flow> {
- match pull::pull(ctx, facts).await? {
- PullOutcome::Legacy(legacy, routing) => adopt_server_vault(ctx, &legacy.encrypted_blob, legacy.revision, routing).await,
- PullOutcome::Materialized { encrypted_vault, revision, email_routing } => adopt_server_vault(ctx, &encrypted_vault, revision, email_routing).await,
- }
-}
-
-/// Whether the local vault is still on a schema the codec cannot canonicalize.
-async fn vault_predates_current_schema(ctx: &mut Ctx) -> SyncResult<bool> {
- if db::requires_legacy_sqlite_blob_migration(&ctx.host, Db::Local).await? {
- return Ok(true);
- }
- let schema = ctx.schema().await?;
- db::requires_schema_migration(&ctx.host, &schema.migration_id).await
+async fn adopt_pulled_vault(ctx: &mut Ctx) -> SyncResult<Flow> {
+ let pulled = pull::pull(ctx).await?;
+ adopt_server_vault(ctx, &pulled).await
}
/// Canonical-merge path of a dirty pull.
-async fn canonical_pull_and_merge(ctx: &mut Ctx, facts: &mut SnapshotFacts, grant_sync_changed_vault: bool) -> SyncResult<Flow> {
- match merge::pull_and_merge(ctx, facts).await? {
- PullAndMergeOutcome::LegacyServer(legacy) => {
- ctx.warn(format!("[VaultSync] Server vault (revision {}) is still on the legacy storage format while the local vault is migrated; pushing the local vault.", legacy.revision)).await;
- match push_and_report(ctx, None, true, false).await? {
- Some(flow) => Ok(flow),
- None => Ok(Flow::Done(FullSyncResult { success: true, has_new_vault: grant_sync_changed_vault, ..Default::default() })),
+async fn canonical_pull_and_merge(ctx: &mut Ctx, grant_sync_changed_vault: bool) -> SyncResult<Flow> {
+ match merge::pull_and_merge(ctx).await? {
+ PullAndMergeOutcome::LegacyServer { revision } => {
+ ctx.warn(format!("[VaultSync] Server vault (revision {}) is still on the legacy storage format while the local vault is migrated; pushing the local vault.", revision)).await;
+ if push_local_changes(ctx, None, true, false).await?.is_none() {
+ return Ok(Flow::Resync { outdated: true });
}
+ Ok(Flow::Done(FullSyncResult { success: true, has_new_vault: grant_sync_changed_vault, ..Default::default() }))
}
- PullAndMergeOutcome::ServerOnly { encrypted_vault, revision, email_routing, manifest_revisions, bucket_revisions } => {
- if !store_pulled_vault(ctx, &encrypted_vault, revision).await? {
- return Ok(Flow::Resync { outdated: false });
+ PullAndMergeOutcome::ServerOnly(pulled) => adopt_server_vault(ctx, &pulled).await,
+ PullAndMergeOutcome::Merged { pulled, stats, fallback_manifest_ids, dropped_local_manifest_ids, push_canonical } => {
+ if let Some(resync) = commit_pulled_vault(ctx, &pulled).await? {
+ return Ok(resync);
}
- pull::commit_revisions(ctx, &manifest_revisions, &bucket_revisions).await?;
- Ok(Flow::Done(materialized_vault_result(ctx, revision, email_routing).await?))
- }
- PullAndMergeOutcome::Merged { encrypted_vault, revision, email_routing, stats, fallback_manifest_ids, dropped_local_manifest_ids, manifest_revisions, bucket_revisions, push_canonical } => {
- if !store_pulled_vault(ctx, &encrypted_vault, revision).await? {
- return Ok(Flow::Resync { outdated: false });
- }
- // Only now do the pulled revisions become the local truth.
- pull::commit_revisions(ctx, &manifest_revisions, &bucket_revisions).await?;
ctx.log(format!("[VaultSync] Canonical vault merge completed: {:?}; {} validation fallback(s), {} dropped local manifest(s).", stats, fallback_manifest_ids.len(), dropped_local_manifest_ids.len())).await;
-
- if let Some(flow) = push_and_report(ctx, push_canonical.map(|set| (ctx.mutation_sequence, set)), false, false).await? {
- return Ok(flow);
+ let cache = push_canonical.map(|set| (ctx.mutation_sequence, set));
+ if push_local_changes(ctx, cache, false, false).await?.is_none() {
+ return Ok(Flow::Resync { outdated: true });
}
- Ok(Flow::Done(materialized_vault_result(ctx, revision, email_routing).await?))
- }
- }
-}
-
-/// Push path: server and client agree on every revision, so the pending local changes upload as-is.
-async fn push_pending_local_changes(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)>, grant_sync_changed_vault: bool) -> SyncResult<Flow> {
- let upload = push::upload_vault(ctx, cache, false, false).await?;
- match upload.status {
- PushStatus::Ok => {
- state::mark_clean(&ctx.host, upload.mutation_seq_at_start).await?;
- Ok(Flow::Done(FullSyncResult { success: true, has_new_vault: upload.vault_changed || grant_sync_changed_vault, ..Default::default() }))
+ Ok(Flow::Done(materialized_vault_result(ctx, &pulled).await?))
}
- PushStatus::Outdated => Ok(Flow::Resync { outdated: true }),
}
}
-/// Push the pending local changes. None on success, else the flow to return.
-async fn push_and_report(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)>, force_full_write: bool, create_vault_key: bool) -> SyncResult<Option<Flow>> {
+/// Push the pending local changes and clear the dirty flag.
+async fn push_local_changes(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)>, force_full_write: bool, create_vault_key: bool) -> SyncResult<Option<bool>> {
let upload = push::upload_vault(ctx, cache, force_full_write, create_vault_key).await?;
match upload.status {
PushStatus::Ok => {
state::mark_clean(&ctx.host, upload.mutation_seq_at_start).await?;
- Ok(None)
+ Ok(Some(upload.vault_changed))
}
- PushStatus::Outdated => Ok(Some(Flow::Resync { outdated: true })),
+ PushStatus::Outdated => Ok(None),
}
}
/// Check for any pending migrations of the stored vault.
async fn pending_migration_result(ctx: &mut Ctx) -> SyncResult<Option<FullSyncResult>> {
let checked: SyncResult<Option<FullSyncResult>> = async {
- if db::requires_legacy_sqlite_blob_migration(&ctx.host, Db::Local).await? {
- return Ok(Some(FullSyncResult { success: true, sqlite_blob_upgrade_required: true, ..Default::default() }));
- }
- if vault_requires_manifest_migration(ctx).await? {
- return Ok(Some(FullSyncResult { success: true, manifest_migration_required: true, ..Default::default() }));
- }
- Ok(None)
+ Ok(match schema_state(ctx).await? {
+ SchemaState::LegacyChain => Some(FullSyncResult { success: true, sqlite_blob_upgrade_required: true, ..Default::default() }),
+ SchemaState::Stale => Some(FullSyncResult { success: true, manifest_migration_required: true, ..Default::default() }),
+ SchemaState::Current if !keys::has_local_vault_key(&ctx.host).await? => Some(FullSyncResult { success: true, manifest_migration_required: true, ..Default::default() }),
+ SchemaState::Current => None,
+ })
}
.await;
match checked {
@@ -485,25 +465,21 @@ async fn pending_migration_result(ctx: &mut Ctx) -> SyncResult<Option<FullSyncRe
/// executor with the chain outstanding is a caller bug, while asking the classifier about it is not.
async fn migration_status(ctx: &mut Ctx) -> MigrationStatusResult {
let classified: SyncResult<MigrationKind> = async {
- if db::requires_legacy_sqlite_blob_migration(&ctx.host, Db::Local).await? {
+ let schema = schema_state(ctx).await?;
+ if schema == SchemaState::LegacyChain {
return Ok(MigrationKind::None);
}
match keys::fetch_vault_key(&ctx.host).await {
- Ok((_, vault_key)) => {
- if vault_key.is_some() {
- keys::adopt_remote_vault_key_if_needed(ctx).await?;
- }
+ Ok(Some(_)) => {
+ keys::adopt_remote_vault_key_if_needed(ctx).await?;
}
+ Ok(None) => {}
Err(error) => ctx.warn(format!("[ManifestMigration] Vault key probe failed, classifying from local state: {}", error)).await,
}
if !keys::has_local_vault_key(&ctx.host).await? {
return Ok(MigrationKind::StorageFormatUpgrade);
}
- let schema = ctx.schema().await?;
- if db::requires_schema_migration(&ctx.host, &schema.migration_id).await? {
- return Ok(MigrationKind::SchemaRebuild);
- }
- Ok(MigrationKind::None)
+ Ok(if schema == SchemaState::Stale { MigrationKind::SchemaRebuild } else { MigrationKind::None })
}
.await;
let kind = match classified {
@@ -520,14 +496,14 @@ async fn migration_status(ctx: &mut Ctx) -> MigrationStatusResult {
async fn migrate_manifest(ctx: &mut Ctx) -> MigrateManifestResult {
let migrated: SyncResult<bool> = async {
let encryption_key = ctx.encryption_key()?;
- if db::requires_legacy_sqlite_blob_migration(&ctx.host, Db::Local).await? {
+ let schema = schema_state(ctx).await?;
+ if schema == SchemaState::LegacyChain {
return Err(SyncError::LegacyUpgradePending);
}
if !keys::adopt_remote_vault_key_if_needed(ctx).await? {
return Err(SyncError::KeyOutOfSync);
}
- let schema = ctx.schema().await?;
- let needs_schema_migration = db::requires_schema_migration(&ctx.host, &schema.migration_id).await?;
+ let needs_schema_migration = schema == SchemaState::Stale;
let needs_vault_key = !keys::has_local_vault_key(&ctx.host).await?;
if !needs_schema_migration && !needs_vault_key {
return Ok(true);
@@ -559,11 +535,7 @@ async fn migrate_manifest(ctx: &mut Ctx) -> MigrateManifestResult {
Ok(pushed) => MigrateManifestResult { success: true, pushed, ..Default::default() },
Err(error) => {
ctx.warn(format!("[ManifestMigration] Migration failed: {}", error)).await;
- let (error_code, error_key) = match error.failure() {
- Failure::Coded(code) => (Some(code), None),
- Failure::Logout(reason) => (None, Some(reason)),
- };
- MigrateManifestResult { success: false, pushed: false, error: Some(error.to_string()), error_code, error_key, requires_logout: error_key.is_some(), ..Default::default() }
+ MigrateManifestResult { failure: (&error).into(), ..Default::default() }
}
}
}
@@ -583,7 +555,7 @@ async fn status_check(ctx: &mut Ctx) -> StatusCheckResult {
None
};
if let Some(reason) = logout_reason {
- return Ok(StatusCheckResult { requires_logout: true, error_key: Some(reason), server_version: Some(status.server_version), ..Default::default() });
+ return Ok(StatusCheckResult { failure: FailureFields::logout(reason), server_version: Some(status.server_version), ..Default::default() });
}
assert_salt_unchanged(ctx, status.srp_salt.as_deref()).await?;
let has_newer_vault = server_state_needs_pull(ctx, &status).await?;
@@ -594,16 +566,7 @@ async fn status_check(ctx: &mut Ctx) -> StatusCheckResult {
Ok(result) => result,
Err(error) => {
let outcome = map_sync_failure(ctx, error).await;
- StatusCheckResult {
- success: outcome.success,
- has_dirty_changes: ctx.is_dirty,
- is_offline: outcome.was_offline,
- requires_logout: outcome.requires_logout,
- error_key: outcome.error_key,
- error: outcome.error,
- error_code: outcome.error_code,
- ..Default::default()
- }
+ StatusCheckResult { success: outcome.success, has_dirty_changes: ctx.is_dirty, is_offline: outcome.was_offline, failure: outcome.failure, ..Default::default() }
}
}
}
@@ -615,12 +578,12 @@ mod tests {
#[test]
fn failures_carry_a_code_or_a_logout_reason_never_both() {
- let coded = failure(&SyncError::Timeout("slow".to_string()));
+ let coded = failure(&SyncError::Timeout("slow".to_string())).failure;
assert_eq!(coded.error_code, Some(ErrorCode::UploadTimeout));
assert_eq!(coded.error_key, None);
assert!(!coded.requires_logout);
- let logout = failure(&SyncError::VaultVersionIncompatible("3.0.0".to_string()));
+ let logout = failure(&SyncError::VaultVersionIncompatible("3.0.0".to_string())).failure;
assert_eq!(logout.error_key, Some(LogoutReason::VaultVersionIncompatible));
assert_eq!(logout.error_code, None);
assert!(logout.requires_logout);
diff --git a/core/rust/src/vault_sync/errors.rs b/core/rust/src/vault_sync/errors.rs
index 4d8813d6a..66883f3a6 100644
--- a/core/rust/src/vault_sync/errors.rs
+++ b/core/rust/src/vault_sync/errors.rs
@@ -5,61 +5,55 @@ use serde::{Deserialize, Serialize};
use super::types::CommandKind;
use crate::error::VaultError;
-macro_rules! error_codes {
- ($($(#[$doc:meta])* $variant:ident = $code:literal),* $(,)?) => {
- /// Client error codes, shared with the client apps (`AppErrorCodes` in the client core).
- #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
- pub enum ErrorCode {
- $($(#[$doc])* #[serde(rename = $code)] $variant,)*
- }
-
- impl ErrorCode {
- /// The wire form of the code (`E-202`).
- pub const fn as_str(self) -> &'static str {
- match self {
- $(ErrorCode::$variant => $code,)*
- }
- }
- }
- };
-}
-
-error_codes! {
- UnknownError = "E-001",
+/// Client error codes, shared with the client apps (`AppErrorCodes` in the client core).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum ErrorCode {
+ #[serde(rename = "E-001")]
+ UnknownError,
/// No encryption key in the session.
- VaultLocked = "E-202",
+ #[serde(rename = "E-202")]
+ VaultLocked,
/// The stored vault does not decrypt with the session key.
- VaultDecryptFailed = "E-203",
+ #[serde(rename = "E-203")]
+ VaultDecryptFailed,
/// The server's snapshot cannot be assembled into a vault.
- SyncVaultFetchFailed = "E-502",
+ #[serde(rename = "E-502")]
+ SyncVaultFetchFailed,
/// A server manifest or bucket fails its hash check or does not decrypt.
- SyncVaultDecryptFailed = "E-503",
+ #[serde(rename = "E-503")]
+ SyncVaultDecryptFailed,
/// The server cannot be reached and there is no local vault to fall back on.
- SyncServerUnreachable = "E-505",
+ #[serde(rename = "E-505")]
+ SyncServerUnreachable,
/// The server answered a request with an unexpected HTTP failure.
- SyncServerError = "E-506",
+ #[serde(rename = "E-506")]
+ SyncServerError,
/// The host failed a read command (state, database, at-rest blob).
- StorageReadFailed = "E-601",
+ #[serde(rename = "E-601")]
+ StorageReadFailed,
/// The host failed a write command (state, database, at-rest blob, dirty flag).
- StorageWriteFailed = "E-602",
+ #[serde(rename = "E-602")]
+ StorageWriteFailed,
/// The host could not open the staging database.
- DatabaseInitFailed = "E-603",
- MergeFailed = "E-701",
+ #[serde(rename = "E-603")]
+ DatabaseInitFailed,
+ #[serde(rename = "E-701")]
+ MergeFailed,
/// The server kept refusing the write as outdated after the re-sync limit.
- MergeConflict = "E-702",
+ #[serde(rename = "E-702")]
+ MergeConflict,
/// The write was rejected: integrity check, or blobs the server asked for that this client cannot supply.
- UploadFailed = "E-801",
- UploadTooLarge = "E-804",
- UploadTimeout = "E-805",
- /// A migration cannot run yet (the vault still has to walk the legacy upgrade chain).
- MigrationCheckFailed = "E-901",
- ServerUpdateRequired = "E-903",
-}
-
-impl std::fmt::Display for ErrorCode {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.write_str(self.as_str())
- }
+ #[serde(rename = "E-801")]
+ UploadFailed,
+ #[serde(rename = "E-804")]
+ UploadTooLarge,
+ #[serde(rename = "E-805")]
+ UploadTimeout,
+ /// LEGACY: a migration cannot run yet (the vault is not on the latest pre-manifest-v1 schema).
+ #[serde(rename = "E-901")]
+ MigrationCheckFailed,
+ #[serde(rename = "E-903")]
+ ServerUpdateRequired,
}
/// Why the sync has to end and host needs to logout user session.
@@ -135,7 +129,7 @@ pub enum SyncError {
/// The server asked for blobs this client cannot supply.
#[error("Server reported blobs this client cannot supply: {}", .0.join(", "))]
MissingBlobs(Vec<String>),
- /// The manifest migration cannot run before the legacy sqlite-blob upgrade chain.
+ /// LEGACY: the manifest migration cannot run before the sqlite-blob upgrade chain.
#[error("The vault has to walk the legacy upgrade chain first")]
LegacyUpgradePending,
/// The host reported a failure for a command.
@@ -177,21 +171,6 @@ impl SyncError {
}
}
- /// The client error code for this failure; `None` when it is a logout instead.
- pub fn code(&self) -> Option<ErrorCode> {
- match self.failure() {
- Failure::Coded(code) => Some(code),
- Failure::Logout(_) => None,
- }
- }
-
- /// The logout reason for this failure; `None` when it is a coded error instead.
- pub fn logout_reason(&self) -> Option<LogoutReason> {
- match self.failure() {
- Failure::Logout(reason) => Some(reason),
- Failure::Coded(_) => None,
- }
- }
}
impl CommandKind {
@@ -215,11 +194,10 @@ mod tests {
#[test]
fn codes_and_reasons_serialize_to_the_client_vocabulary() {
assert_eq!(serde_json::to_string(&ErrorCode::VaultLocked).unwrap(), "\"E-202\"");
- assert_eq!(ErrorCode::UploadTimeout.as_str(), "E-805");
+ assert_eq!(serde_json::to_string(&ErrorCode::UploadTimeout).unwrap(), "\"E-805\"");
assert_eq!(serde_json::to_string(&LogoutReason::PasswordChanged).unwrap(), "\"passwordChanged\"");
- assert_eq!(SyncError::Auth.logout_reason(), Some(LogoutReason::SessionExpired));
- assert_eq!(SyncError::Auth.code(), None);
- assert_eq!(SyncError::Host { command: CommandKind::DbOpen, message: String::new() }.code(), Some(ErrorCode::DatabaseInitFailed));
- assert_eq!(SyncError::Host { command: CommandKind::DbExec, message: String::new() }.code(), Some(ErrorCode::StorageWriteFailed));
+ assert_eq!(SyncError::Auth.failure(), Failure::Logout(LogoutReason::SessionExpired));
+ assert_eq!(SyncError::Host { command: CommandKind::DbOpen, message: String::new() }.failure(), Failure::Coded(ErrorCode::DatabaseInitFailed));
+ assert_eq!(SyncError::Host { command: CommandKind::DbExec, message: String::new() }.failure(), Failure::Coded(ErrorCode::StorageWriteFailed));
}
}
diff --git a/core/rust/src/vault_sync/http.rs b/core/rust/src/vault_sync/http.rs
index ac659812c..179dd6fe5 100644
--- a/core/rust/src/vault_sync/http.rs
+++ b/core/rust/src/vault_sync/http.rs
@@ -60,6 +60,35 @@ pub(crate) async fn delete(host: &Host, path: &str) -> SyncResult<()> {
Ok(())
}
+/// The v2 vault endpoint (`GET`/`POST v2/Vault`).
+pub(crate) const VAULT_ENDPOINT: &str = "Vault";
+
+/// Max base64 characters in one blob transfer request or response body.
+pub(crate) const BLOB_TRANSFER_BATCH_MAX_CHARS: usize = 4 * 1024 * 1024;
+
+/// Upper bound on the number of blobs in one transfer batch.
+pub(crate) const BLOB_TRANSFER_BATCH_MAX_COUNT: usize = 100;
+
+/// Split items into transfer batches bounded by both blob transfer limits.
+pub(crate) fn batch_by_transfer_cost<T>(items: Vec<T>, cost_of: impl Fn(&T) -> usize) -> Vec<Vec<T>> {
+ let mut batches = Vec::new();
+ let mut batch = Vec::new();
+ let mut chars = 0usize;
+ for item in items {
+ let cost = cost_of(&item);
+ if !batch.is_empty() && (chars + cost > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.len() >= BLOB_TRANSFER_BATCH_MAX_COUNT) {
+ batches.push(std::mem::take(&mut batch));
+ chars = 0;
+ }
+ batch.push(item);
+ chars += cost;
+ }
+ if !batch.is_empty() {
+ batches.push(batch);
+ }
+ batches
+}
+
/// Turn a 404 from a v2 vault endpoint into "the server predates the v2 API".
pub(crate) fn with_outdated_server_guard<T>(result: SyncResult<T>) -> SyncResult<T> {
match result {
diff --git a/core/rust/src/vault_sync/keys.rs b/core/rust/src/vault_sync/keys.rs
index 79ea9a4c8..9fe707c16 100644
--- a/core/rust/src/vault_sync/keys.rs
+++ b/core/rust/src/vault_sync/keys.rs
@@ -14,11 +14,11 @@ pub(crate) async fn has_local_vault_key(host: &Host) -> SyncResult<bool> {
Ok(state::get::<String>(host, state::ENCRYPTED_ACCOUNT_KEY).await?.is_some())
}
-/// `GET v2/VaultKey/Password`.
-pub(crate) async fn fetch_vault_key(host: &Host) -> SyncResult<(bool, Option<VaultKeyResponse>)> {
+/// `GET v2/VaultKey/Password`: the account's vault key, `None` when the server holds none.
+pub(crate) async fn fetch_vault_key(host: &Host) -> SyncResult<Option<VaultKeyResponse>> {
match http::get::<VaultKeyGetResponse>(host, "VaultKey/Password", false).await {
- Ok(response) => Ok((true, response.vault_key)),
- Err(SyncError::Http { status: 404, .. }) => Ok((false, None)),
+ Ok(response) => Ok(response.vault_key),
+ Err(SyncError::Http { status: 404, .. }) => Ok(None),
Err(error) => Err(error),
}
}
@@ -125,7 +125,7 @@ pub(crate) async fn adopt_remote_vault_key_if_needed(ctx: &mut Ctx) -> SyncResul
}
let Some(session_key) = ctx.encryption_key.clone() else { return Ok(true) };
- let (_, vault_key) = match fetch_vault_key(&ctx.host).await {
+ let vault_key = match fetch_vault_key(&ctx.host).await {
Ok(result) => result,
Err(error) => {
ctx.warn(format!("[VaultSync] Vault key probe failed, deferring vault key adoption: {}", error)).await;