Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update Rust core for vault merge codec overflow logic (#2159)

Leendert de Borstd76f5d60a11d
diff --git a/core/rust/src/ffi.rs b/core/rust/src/ffi.rs
index 6233fe245..2ba6b3d99 100644
--- a/core/rust/src/ffi.rs
+++ b/core/rust/src/ffi.rs
@@ -144,7 +144,7 @@ pub unsafe extern "C" fn filter_credentials_ffi(input_json: *const c_char) -> *m
/// A null-terminated C string containing a JSON array of table names.
#[no_mangle]
pub extern "C" fn get_syncable_table_names_ffi() -> *mut c_char {
- let names: Vec<&str> = SYNCABLE_TABLE_NAMES.iter().map(|s| *s).collect();
+ let names = crate::vault_merge::merge_table_names();
match serde_json::to_string(&names) {
Ok(json) => string_to_c_char(json),
Err(_) => ptr::null_mut(),
diff --git a/core/rust/src/uniffi_api.rs b/core/rust/src/uniffi_api.rs
index 69cae50f2..7b5841757 100644
--- a/core/rust/src/uniffi_api.rs
+++ b/core/rust/src/uniffi_api.rs
@@ -12,14 +12,10 @@ pub fn get_core_version() -> String {
crate::get_core_version().to_string()
}
-/// Get the list of syncable table names.
-/// These are the tables that need to be read from the database for merge/prune operations.
+/// Get the list of syncable table names a platform must read into the merge input.
#[uniffi::export]
pub fn get_syncable_table_names() -> Vec<String> {
- SYNCABLE_TABLE_NAMES
- .iter()
- .map(|s| s.to_string())
- .collect()
+ crate::vault_merge::merge_table_names().iter().map(|s| s.to_string()).collect()
}
/// Merge local and server vaults using Last-Write-Wins strategy.
@@ -359,6 +355,7 @@ mod tests {
assert!(names.contains(&"FieldValues".to_string()));
assert!(names.contains(&"Settings".to_string()));
assert!(names.contains(&"EncryptionKeys".to_string()));
+ assert!(!names.contains(&crate::vault_codec::OVERFLOW_TABLE.to_string()), "overflow carrier is not in the merge input; the server base owns it");
assert_eq!(names.len(), 13);
}
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rs
index a05e62457..9172ffc4a 100644
--- a/core/rust/src/vault_codec/canonicalize.rs
+++ b/core/rust/src/vault_codec/canonicalize.rs
@@ -20,7 +20,7 @@ use super::hash::salted_blob_hash;
use super::logos::dedupe_logos_by_source;
use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedVault, CodecOverflow, DataBucket, Manifest, CodecRecord};
use super::materialize::row_key;
-use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_skip_table, primary_key_for, SCHEMA_VERSION};
+use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_skip_table, primary_key_for, OVERFLOW_TABLE, SCHEMA_VERSION};
use crate::error::VaultResult;
/// Canonicalize normalized tables into the split resources: the manifest, one data bucket per declared
@@ -30,16 +30,22 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
// extraction and bucket-splitting happen below; deduping Logos first (before extraction) means a
// dropped duplicate never registers an orphan favicon blob.
let mut all_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();
+ let mut overflow = CodecOverflow::default();
for table in &input.tables {
+ // The OVERFLOW_TABLE row carries a newer writer's tables/columns this client's schema
+ // couldn't hold (written by the last materialize). Consume it here — re-merged below,
+ // never emitted into the manifest itself.
+ if table.name == OVERFLOW_TABLE {
+ overflow = CodecOverflow::from_table_records(&table.records);
+ continue;
+ }
if is_skip_table(&table.name) {
continue;
}
all_tables.entry(table.name.clone()).or_default().extend(table.records.iter().cloned());
}
- // Re-merge overflow from the last materialize (a newer writer's tables/columns this client's
- // local schema couldn't hold) so this push doesn't drop them. See `CodecOverflow`.
- let overflow = input.overflow.unwrap_or_default();
+ // Re-merge the overflow so this push doesn't drop a newer writer's data. See `CodecOverflow`.
remerge_overflow_columns(&mut all_tables, &overflow);
for (name, rows) in &overflow.tables {
// Local rows win if the table somehow exists locally now (e.g. client upgraded since the pull).
@@ -175,17 +181,17 @@ fn remerge_overflow_columns(tables: &mut HashMap<String, Vec<CodecRecord>>, over
/// The bucket-only push path (a bucket changed but the manifest didn't). Input rows are
/// already normalized by the platform read logic.
///
-/// `overflow` (persisted from the last materialize) is re-merged the same way the full
-/// canonicalize does it: unknown columns re-attach to this bucket's rows, and whole unknown tables
-/// recorded under this bucket's category are re-emitted, so a bucket-only push from an older
-/// client never drops a newer writer's data either.
-pub fn extract_bucket(category: String, mut tables: HashMap<String, Vec<CodecRecord>>, overflow: Option<&CodecOverflow>) -> DataBucket {
- if let Some(overflow) = overflow {
- remerge_overflow_columns(&mut tables, overflow);
- if let Some(ov_tables) = overflow.bucket_tables.get(&category) {
- for (name, rows) in ov_tables {
- tables.entry(name.clone()).or_insert_with(|| rows.clone());
- }
+/// When `tables` includes the [`OVERFLOW_TABLE`] row (callers should read it alongside the
+/// bucket's tables), it is consumed and re-merged the same way the full canonicalize does it:
+/// unknown columns re-attach to this bucket's rows, and whole unknown tables recorded under this
+/// bucket's category are re-emitted, so a bucket-only push from an older client never drops a
+/// newer writer's data either.
+pub fn extract_bucket(category: String, mut tables: HashMap<String, Vec<CodecRecord>>) -> DataBucket {
+ let overflow = tables.remove(OVERFLOW_TABLE).map(|records| CodecOverflow::from_table_records(&records)).unwrap_or_default();
+ remerge_overflow_columns(&mut tables, &overflow);
+ if let Some(ov_tables) = overflow.bucket_tables.get(&category) {
+ for (name, rows) in ov_tables {
+ tables.entry(name.clone()).or_insert_with(|| rows.clone());
}
}
DataBucket::new(category, tables)
diff --git a/core/rust/src/vault_codec/manifest.rs b/core/rust/src/vault_codec/manifest.rs
index 206e4fd4d..45c26a2d6 100644
--- a/core/rust/src/vault_codec/manifest.rs
+++ b/core/rust/src/vault_codec/manifest.rs
@@ -115,12 +115,33 @@ impl CodecOverflow {
pub fn is_empty(&self) -> bool {
self.tables.is_empty() && self.bucket_tables.is_empty() && self.columns.is_empty()
}
+
+ /// Render this overflow as the single `OVERFLOW_TABLE` row the platform inserts into the vault DB.
+ pub fn to_table_records(&self) -> Vec<CodecRecord> {
+ let mut row: CodecRecord = HashMap::new();
+ row.insert("Id".to_string(), serde_json::Value::String(super::types::OVERFLOW_ROW_ID.to_string()));
+ row.insert("Data".to_string(), serde_json::Value::String(serde_json::to_string(self).unwrap_or_default()));
+ vec![row]
+ }
+
+ /// Parse an `OVERFLOW_TABLE` row set read back from the vault DB. Tolerant: no rows, a missing
+ /// `Data` column, or unparseable JSON all yield an empty overflow (better to push what we have
+ /// than refuse to push at all).
+ pub fn from_table_records(records: &[CodecRecord]) -> Self {
+ records
+ .first()
+ .and_then(|row| row.get("Data"))
+ .and_then(|v| v.as_str())
+ .and_then(|json| serde_json::from_str(json).ok())
+ .unwrap_or_default()
+ }
}
/// Materialized tables the platform inserts into a fresh schema DB. Blob columns carry
-/// `{ "__blobRef": hash }`; inline byte columns carry `{ "__b64": ... }`. `overflow` holds whatever
-/// the local schema couldn't accept (see [`CodecOverflow`]); the platform must persist it and feed
-/// it back into the next canonicalize.
+/// `{ "__blobRef": hash }`; inline byte columns carry `{ "__b64": ... }`. Any overflow (see
+/// [`CodecOverflow`]) is already included in `tables` as the `OVERFLOW_TABLE` row — the platform
+/// inserts it like any other table and needs no separate persistence. The `overflow` field is a
+/// diagnostics copy of the same data (for logging), not something the platform must store.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MaterializedTables {
@@ -134,16 +155,15 @@ pub struct MaterializedTables {
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CanonicalizeInput {
+ /// All local vault tables (a plain `SELECT *` read). When the read includes the
+ /// [`OVERFLOW_TABLE`](super::types::OVERFLOW_TABLE) row written by the last materialize, its
+ /// overflow (a newer writer's tables/columns this schema can't hold) is re-merged automatically.
pub tables: Vec<CodecTableData>,
pub user_salt: String,
pub migration_id: String,
#[serde(default = "default_version")]
pub version: String,
pub canonicalized_at: String,
- /// Overflow persisted from the last materialize (see [`CodecOverflow`]); re-merged into the
- /// output so a client with an older schema never drops a newer writer's tables/columns.
- #[serde(default)]
- pub overflow: Option<CodecOverflow>,
}
fn default_version() -> String {
diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rs
index 5497b97fa..ce680856a 100644
--- a/core/rust/src/vault_codec/materialize.rs
+++ b/core/rust/src/vault_codec/materialize.rs
@@ -10,13 +10,15 @@
//! Forward compatibility: when the caller supplies its local schema (`schema_columns`), anything a
//! newer writer put in the manifest that this schema cannot hold — whole unknown tables or unknown
//! columns on known tables — is split into [`CodecOverflow`] instead of being emitted (which would
-//! crash the platform insert). The platform persists the overflow and passes it back into
-//! `canonicalize_from_sqlite`, which re-merges it so this client's next push never drops the data.
+//! crash the platform insert). The overflow is emitted as a regular table row (`OVERFLOW_TABLE`),
+//! so it lives inside the vault DB itself and `canonicalize_from_sqlite` re-merges it from the
+//! ordinary table read — this client's next push never drops the data, and no platform has to wire
+//! (or remember) a separate persistence channel.
use std::collections::{HashMap, HashSet};
use super::manifest::{CodecOverflow, CodecRecord, CodecTableData, MaterializeInput, MaterializedTables};
-use super::types::{is_skip_table, primary_key_for};
+use super::types::{is_skip_table, primary_key_for, OVERFLOW_TABLE};
use crate::error::VaultResult;
/// Materialize the manifest + its data buckets into the table set the platform inserts.
@@ -27,7 +29,9 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
let mut tables: Vec<CodecTableData> = Vec::with_capacity(manifest.tables.len() + data_buckets.len());
for (name, records) in manifest.tables {
- if is_skip_table(&name) {
+ // OVERFLOW_TABLE is local-only bookkeeping: it must never occur in a manifest, and passing
+ // one through would collide with the row this function emits below.
+ if is_skip_table(&name) || name == OVERFLOW_TABLE {
continue;
}
match split_for_schema(&name, records, schema_columns.as_ref(), &mut overflow.columns) {
@@ -42,7 +46,7 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
// their category so canonicalize / extract_bucket can re-emit them into the right bucket.
for bucket in data_buckets {
for (name, records) in bucket.tables {
- if is_skip_table(&name) {
+ if is_skip_table(&name) || name == OVERFLOW_TABLE {
continue;
}
match split_for_schema(&name, records, schema_columns.as_ref(), &mut overflow.columns) {
@@ -54,6 +58,11 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
}
}
+ // Carry the overflow inside the vault DB itself: one OVERFLOW_TABLE row, inserted like any table.
+ if !overflow.is_empty() {
+ tables.push(CodecTableData { name: OVERFLOW_TABLE.to_string(), records: overflow.to_table_records() });
+ }
+
Ok(MaterializedTables {
tables,
migration_id: manifest.migration_id,
diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rs
index c3596cc8e..2ba572361 100644
--- a/core/rust/src/vault_codec/mod.rs
+++ b/core/rust/src/vault_codec/mod.rs
@@ -29,7 +29,7 @@ pub use manifest::{
};
pub use types::{
bucket_categories, bucket_category_for, tables_for_category, BLOB_COLUMNS, BUCKET_TABLES,
- SCHEMA_VERSION, SKIP_TABLES,
+ OVERFLOW_ROW_ID, OVERFLOW_TABLE, SCHEMA_VERSION, SKIP_TABLES,
};
pub use validate::ValidationResult;
@@ -47,10 +47,11 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
materialize::materialize_as_sqlite(input)
}
-/// Build a single data bucket for `category` from its tables (bucket-only push path). `overflow`
-/// (persisted from the last materialize) re-merges a newer writer's columns/tables so they survive.
-pub fn extract_bucket(category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>, overflow: Option<&CodecOverflow>) -> DataBucket {
- canonicalize::extract_bucket(category, tables, overflow)
+/// Build a single data bucket for `category` from its tables (bucket-only push path). Include the
+/// [`OVERFLOW_TABLE`] row in `tables` (read it alongside the bucket's tables) so a newer writer's
+/// columns/tables re-merge and survive; it is consumed and never emitted into the bucket.
+pub fn extract_bucket(category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>) -> DataBucket {
+ canonicalize::extract_bucket(category, tables)
}
/// The bucket layout: every category and the tables it owns, in declaration order.
@@ -160,19 +161,16 @@ pub fn materialize_as_sqlite_json(input_json: &str) -> VaultResult<String> {
Ok(serde_json::to_string(&materialize_as_sqlite(input)?)?)
}
-/// JSON-string sibling of [`extract_bucket`]. Input: `{ "category": <str>, "tables": { <name>: [rows] }, "overflow"?: CodecOverflow }`.
+/// JSON-string sibling of [`extract_bucket`]. Input: `{ "category": <str>, "tables": { <name>: [rows] } }`.
pub fn extract_bucket_json(input_json: &str) -> VaultResult<String> {
#[derive(serde::Deserialize)]
- #[serde(rename_all = "camelCase")]
struct Input {
category: String,
#[serde(default)]
tables: std::collections::HashMap<String, Vec<CodecRecord>>,
- #[serde(default)]
- overflow: Option<CodecOverflow>,
}
let input: Input = serde_json::from_str(input_json)?;
- Ok(serde_json::to_string(&extract_bucket(input.category, input.tables, input.overflow.as_ref()))?)
+ Ok(serde_json::to_string(&extract_bucket(input.category, input.tables))?)
}
/// JSON-string sibling of [`bucket_layout`]. Output: `[{ "category": <str>, "tables": [<str>] }]`.
diff --git a/core/rust/src/vault_codec/tests.rs b/core/rust/src/vault_codec/tests.rs
index 560d9ba67..91c9797e4 100644
--- a/core/rust/src/vault_codec/tests.rs
+++ b/core/rust/src/vault_codec/tests.rs
@@ -29,7 +29,6 @@ fn basic_input(tables: Vec<CodecTableData>) -> CanonicalizeInput {
migration_id: "20250101000000_Init".to_string(),
version: "2.0.0".to_string(),
canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(),
- overflow: None,
}
}
@@ -382,21 +381,29 @@ fn validate_manifest_rejects_duplicate_logo_sources() {
assert!(result.failed_rules.iter().any(|r| r == "logo-sources-not-unique"));
}
-/// A minimal local schema map for overflow tests: Items(Id, Name), Settings(Key, Value).
+/// A minimal local schema map for overflow tests: Items(Id, Name), Settings(Key, Value), and the
+/// CodecOverflows carrier table every client schema owns since migration 2.1.0.
fn old_client_schema() -> std::collections::HashMap<String, Vec<String>> {
[
("Items".to_string(), vec!["Id".to_string(), "Name".to_string()]),
("Settings".to_string(), vec!["Key".to_string(), "Value".to_string()]),
+ (OVERFLOW_TABLE.to_string(), vec!["Id".to_string(), "Data".to_string()]),
]
.into_iter()
.collect()
}
+/// The OVERFLOW_TABLE entry from a materialize result (what the platform inserts into the vault DB).
+fn overflow_table_of(re: &MaterializedTables) -> Option<&CodecTableData> {
+ re.tables.iter().find(|t| t.name == OVERFLOW_TABLE)
+}
+
#[test]
-fn materialize_splits_unknown_columns_into_overflow_and_canonicalize_remerges() {
+fn materialize_splits_unknown_columns_into_overflow_table_and_canonicalize_remerges() {
// A newer client wrote Items.AliasEnabled; this client's schema doesn't know it. The column must
- // not reach the insert (it would crash), must land in overflow keyed by row Id, and must
- // reappear on the next canonicalize so the push doesn't drop it.
+ // not reach the Items insert (it would crash), must land in the emitted OVERFLOW_TABLE row, and
+ // must reappear on the next canonicalize (which reads that row back like any table) so the push
+ // doesn't drop it.
let out = canonicalize_from_sqlite(basic_input(vec![CodecTableData {
name: "Items".to_string(),
records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub")), ("AliasEnabled", json!(true))])],
@@ -407,18 +414,23 @@ fn materialize_splits_unknown_columns_into_overflow_and_canonicalize_remerges()
let items = re.tables.iter().find(|t| t.name == "Items").unwrap();
assert!(!items.records[0].contains_key("AliasEnabled"), "unknown column filtered out of the insert set");
assert_eq!(items.records[0]["Name"], json!("GitHub"));
- assert_eq!(re.overflow.columns["Items"]["i1"]["AliasEnabled"], json!(true));
+ assert_eq!(re.overflow.columns["Items"]["i1"]["AliasEnabled"], json!(true), "diagnostics copy populated");
- // The old client edits Name locally, then pushes: the overflow column re-attaches to the row.
- let mut push_input = basic_input(vec![CodecTableData {
- name: "Items".to_string(),
- records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub (renamed)"))])],
- }]);
- push_input.overflow = Some(re.overflow);
- let pushed = canonicalize_from_sqlite(push_input).unwrap();
+ let overflow_table = overflow_table_of(&re).expect("overflow emitted as a regular table row");
+ assert_eq!(overflow_table.records.len(), 1);
+ assert_eq!(overflow_table.records[0]["Id"], json!(OVERFLOW_ROW_ID));
+
+ // The old client edits Name locally, then pushes: canonicalize reads the overflow row back from
+ // the DB (plain SELECT *) and re-attaches the column; the carrier table itself never reaches the manifest.
+ let pushed = canonicalize_from_sqlite(basic_input(vec![
+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub (renamed)"))])] },
+ overflow_table.clone(),
+ ]))
+ .unwrap();
let item = &pushed.manifest.tables["Items"][0];
assert_eq!(item["Name"], json!("GitHub (renamed)"));
assert_eq!(item["AliasEnabled"], json!(true), "newer writer's column survives the old client's push");
+ assert!(!pushed.manifest.tables.contains_key(OVERFLOW_TABLE), "carrier table consumed, never emitted into the manifest");
}
#[test]
@@ -428,16 +440,18 @@ fn overflow_of_locally_deleted_row_is_dropped_on_canonicalize() {
columns: [("Items".to_string(), [("gone".to_string(), row(&[("AliasEnabled", json!(true))]))].into_iter().collect())].into_iter().collect(),
..Default::default()
};
- let mut input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("kept"))])] }]);
- input.overflow = Some(overflow);
- let out = canonicalize_from_sqlite(input).unwrap();
+ let out = canonicalize_from_sqlite(basic_input(vec![
+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("kept"))])] },
+ CodecTableData { name: OVERFLOW_TABLE.to_string(), records: overflow.to_table_records() },
+ ]))
+ .unwrap();
assert!(!out.manifest.tables["Items"][0].contains_key("AliasEnabled"));
}
#[test]
fn materialize_splits_unknown_tables_into_overflow_and_canonicalize_reemits() {
// A newer client added a whole table (manifest-level) and a whole bucket table. Neither exists
- // in this client's schema; both must round-trip through overflow back to their original place.
+ // in this client's schema; both must round-trip through the overflow row back to their original place.
let mut out = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }])).unwrap();
out.manifest.tables.insert("NewTable".to_string(), vec![row(&[("Id", json!("n1")), ("Data", json!("x"))])]);
out.data_buckets[0].tables.insert("Preferences".to_string(), vec![row(&[("Key", json!("p1"))])]);
@@ -446,17 +460,29 @@ fn materialize_splits_unknown_tables_into_overflow_and_canonicalize_reemits() {
assert!(!re.tables.iter().any(|t| t.name == "NewTable" || t.name == "Preferences"), "unknown tables never reach the insert set");
assert_eq!(re.overflow.tables["NewTable"].len(), 1);
assert_eq!(re.overflow.bucket_tables["Settings"]["Preferences"].len(), 1);
+ let overflow_table = overflow_table_of(&re).expect("overflow emitted as a regular table row").clone();
- let mut push_input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }]);
- push_input.overflow = Some(re.overflow.clone());
- let pushed = canonicalize_from_sqlite(push_input).unwrap();
+ let pushed = canonicalize_from_sqlite(basic_input(vec![
+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] },
+ overflow_table.clone(),
+ ]))
+ .unwrap();
assert_eq!(pushed.manifest.tables["NewTable"].len(), 1, "unknown manifest table re-emitted");
assert_eq!(bucket_rows(&pushed, "Settings", "Preferences").len(), 1, "unknown bucket table re-emitted into its category");
- // Bucket-only push path: extract_bucket must re-emit the bucket's overflow table too.
- let bucket = extract_bucket("Settings".to_string(), [("Settings".to_string(), vec![row(&[("Key", json!("k")), ("Value", json!("v"))])])].into_iter().collect(), Some(&re.overflow));
+ // Bucket-only push path: extract_bucket consumes the overflow row read alongside the bucket's tables.
+ let bucket = extract_bucket(
+ "Settings".to_string(),
+ [
+ ("Settings".to_string(), vec![row(&[("Key", json!("k")), ("Value", json!("v"))])]),
+ (OVERFLOW_TABLE.to_string(), overflow_table.records),
+ ]
+ .into_iter()
+ .collect(),
+ );
assert_eq!(bucket.tables["Preferences"].len(), 1);
assert_eq!(bucket.tables["Settings"].len(), 1);
+ assert!(!bucket.tables.contains_key(OVERFLOW_TABLE), "carrier table consumed, never emitted into the bucket");
}
#[test]
@@ -466,14 +492,22 @@ fn extract_bucket_remerges_overflow_columns() {
columns: [("Settings".to_string(), [("theme".to_string(), row(&[("SyncScope", json!("device"))]))].into_iter().collect())].into_iter().collect(),
..Default::default()
};
- let bucket = extract_bucket("Settings".to_string(), [("Settings".to_string(), vec![row(&[("Key", json!("theme")), ("Value", json!("dark"))])])].into_iter().collect(), Some(&overflow));
+ let bucket = extract_bucket(
+ "Settings".to_string(),
+ [
+ ("Settings".to_string(), vec![row(&[("Key", json!("theme")), ("Value", json!("dark"))])]),
+ (OVERFLOW_TABLE.to_string(), overflow.to_table_records()),
+ ]
+ .into_iter()
+ .collect(),
+ );
assert_eq!(bucket.tables["Settings"][0]["SyncScope"], json!("device"));
assert_eq!(bucket.tables["Settings"][0]["Value"], json!("dark"));
}
#[test]
fn materialize_without_schema_columns_passes_rows_through_verbatim() {
- // Legacy callers (no schema map) keep today's behavior exactly: nothing filtered, empty overflow.
+ // Legacy callers (no schema map) keep today's behavior exactly: nothing filtered, no overflow row.
let out = canonicalize_from_sqlite(basic_input(vec![CodecTableData {
name: "Items".to_string(),
records: vec![row(&[("Id", json!("i1")), ("AliasEnabled", json!(true))])],
@@ -483,6 +517,20 @@ fn materialize_without_schema_columns_passes_rows_through_verbatim() {
let items = re.tables.iter().find(|t| t.name == "Items").unwrap();
assert_eq!(items.records[0]["AliasEnabled"], json!(true));
assert!(re.overflow.is_empty());
+ assert!(overflow_table_of(&re).is_none(), "no overflow row emitted when there is nothing to carry");
+}
+
+#[test]
+fn materialize_drops_overflow_table_smuggled_into_a_manifest() {
+ // Defense: OVERFLOW_TABLE is local-only bookkeeping. A manifest that somehow carries one (corrupt
+ // or malicious) must not pass through — it would collide with the row materialize emits itself.
+ let mut out = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("AliasEnabled", json!(true))])] }])).unwrap();
+ out.manifest.tables.insert(OVERFLOW_TABLE.to_string(), vec![row(&[("Id", json!("smuggled")), ("Data", json!("{}"))])]);
+
+ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: Some(old_client_schema()) }).unwrap();
+ let overflow_table = overflow_table_of(&re).expect("legitimate overflow row still emitted");
+ assert_eq!(overflow_table.records.len(), 1);
+ assert_eq!(overflow_table.records[0]["Id"], json!(OVERFLOW_ROW_ID), "smuggled row dropped, only the codec's own row remains");
}
#[test]
diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rs
index 6edbad373..8a207577c 100644
--- a/core/rust/src/vault_codec/types.rs
+++ b/core/rust/src/vault_codec/types.rs
@@ -69,3 +69,12 @@ pub fn is_skip_table(table_name: &str) -> bool {
pub fn primary_key_for(table_name: &str) -> &'static str {
crate::vault_merge::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key).unwrap_or("Id")
}
+
+/// Client-local SQLite table that carries the codec overflow inside the vault database itself (see
+/// `CodecOverflow`): materialize writes a single row `{ Id: OVERFLOW_ROW_ID, Data: <json> }`, and
+/// canonicalize / extract_bucket consume it to build the manifest.
+pub const OVERFLOW_TABLE: &str = "CodecOverflows";
+
+/// Fixed sentinel primary key of the single `OVERFLOW_TABLE` row (deterministic on purpose:
+/// materialize output must not depend on a random source).
+pub const OVERFLOW_ROW_ID: &str = "00000000-0000-0000-0000-00000000c0de";
diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rs
index 599c8ddb0..b26b4ec1d 100644
--- a/core/rust/src/vault_merge/mod.rs
+++ b/core/rust/src/vault_merge/mod.rs
@@ -11,7 +11,7 @@ use std::collections::HashMap;
use crate::error::VaultResult;
pub use types::SYNCABLE_TABLE_NAMES;
-pub use types::{SYNCABLE_TABLES, TableConfig};
+pub use types::{merge_table_names, SYNCABLE_TABLES, TableConfig};
/// A record is a map of column names to JSON values.
pub type Record = HashMap<String, serde_json::Value>;
@@ -74,24 +74,28 @@ pub struct MergeOutput {
/// Main entry point: merge local and server vault data.
///
+/// The merge **base is the server vault** (`input.server_tables`): a freshly-materialized SQLite with
+/// the newest schema and the newest codec overflow carrier already in place. The returned statements
+/// bring the local vault's winning changes (`input.local_tables`) onto that base.
+///
/// # Arguments
-/// * `input` - MergeInput containing local and server table data
+/// * `input` - MergeInput containing local (incoming) and server (base) table data
///
/// # Returns
-/// MergeOutput with SQL statements to execute on local database
+/// MergeOutput with SQL statements to execute on the server (base) database
pub fn merge_vaults(input: MergeInput) -> VaultResult<MergeOutput> {
let mut total_stats = MergeStats::default();
let mut statements: Vec<SqlStatement> = Vec::new();
- // Create lookup maps for quick access
- let local_map: HashMap<&str, &TableData> = input
- .local_tables
+ // Base = server vault; incoming = local vault. Statements mutate the base.
+ let base_map: HashMap<&str, &TableData> = input
+ .server_tables
.iter()
.map(|t| (t.name.as_str(), t))
.collect();
- let server_map: HashMap<&str, &TableData> = input
- .server_tables
+ let incoming_map: HashMap<&str, &TableData> = input
+ .local_tables
.iter()
.map(|t| (t.name.as_str(), t))
.collect();
@@ -100,20 +104,19 @@ pub fn merge_vaults(input: MergeInput) -> VaultResult<MergeOutput> {
for table_config in SYNCABLE_TABLES {
let table_name = table_config.name;
- let local_data = local_map.get(table_name);
- let server_data = server_map.get(table_name);
+ let base_data = base_map.get(table_name);
+ let incoming_data = incoming_map.get(table_name);
// Skip if table doesn't exist in either database
- let (local_records, server_records) = match (local_data, server_data) {
- (Some(l), Some(s)) => (&l.records, &s.records),
- (Some(l), None) => {
- // Table only in local - nothing to merge
- total_stats.records_created_locally += l.records.len() as u32;
+ let (base_records, incoming_records) = match (base_data, incoming_data) {
+ (Some(b), Some(i)) => (&b.records, &i.records),
+ (Some(_), None) => {
+ // Table only in the server base. Already present, nothing to bring over.
continue;
}
- (None, Some(s)) => {
- // Table only in server - insert all
- for record in &s.records {
+ (None, Some(i)) => {
+ // Table only in local - insert all local rows into the server base.
+ for record in &i.records {
if let Some(stmt) = generate_insert_sql(table_name, record) {
statements.push(stmt);
total_stats.records_inserted += 1;
@@ -129,8 +132,8 @@ pub fn merge_vaults(input: MergeInput) -> VaultResult<MergeOutput> {
let table_statements = if table_config.uses_composite_key() {
merge_table_by_composite_key(
table_name,
- local_records,
- server_records,
+ base_records,
+ incoming_records,
table_config.composite_key_columns,
table_config.primary_key,
&mut total_stats,
@@ -138,8 +141,8 @@ pub fn merge_vaults(input: MergeInput) -> VaultResult<MergeOutput> {
} else {
merge_table_by_id(
table_name,
- local_records,
- server_records,
+ base_records,
+ incoming_records,
table_config.primary_key,
&mut total_stats,
)
@@ -167,61 +170,62 @@ pub fn merge_vaults_json(input_json: &str) -> VaultResult<String> {
/// Merge table records by their primary-key column (standard merge).
/// `primary_key` is the column that identifies a row (usually "Id"; "Key" for Settings).
-/// Returns SQL statements to apply to local database.
+/// `base_records` are the merge base (kept on tie / when the base wins); `incoming_records` are the
+/// changes merged onto it. Returns SQL statements to apply to the base database.
fn merge_table_by_id(
table_name: &str,
- local_records: &[Record],
- server_records: &[Record],
+ base_records: &[Record],
+ incoming_records: &[Record],
primary_key: &str,
stats: &mut MergeStats,
) -> Vec<SqlStatement> {
let mut statements: Vec<SqlStatement> = Vec::new();
- // Create map of server records by primary key
- let mut server_map: HashMap<String, &Record> = HashMap::new();
- for record in server_records {
+ // Create map of incoming records by primary key
+ let mut incoming_map: HashMap<String, &Record> = HashMap::new();
+ for record in incoming_records {
if let Some(id) = get_record_pk(record, primary_key) {
- server_map.insert(id, record);
+ incoming_map.insert(id, record);
}
}
- // Process local records
- for local_record in local_records {
- let local_id = match get_record_pk(local_record, primary_key) {
+ // Process base records
+ for base_record in base_records {
+ let base_id = match get_record_pk(base_record, primary_key) {
Some(id) => id,
None => continue,
};
- if let Some(server_record) = server_map.get(&local_id) {
+ if let Some(incoming_record) = incoming_map.get(&base_id) {
// Record exists in both - compare UpdatedAt for LWW
- let local_ts = get_updated_at(local_record);
- let server_ts = get_updated_at(server_record);
+ let incoming_ts = get_updated_at(incoming_record);
+ let base_ts = get_updated_at(base_record);
- match (server_ts, local_ts) {
- (Some(s_ts), Some(l_ts)) if s_ts > l_ts => {
- // Server wins - generate UPDATE
+ match (incoming_ts, base_ts) {
+ (Some(i_ts), Some(b_ts)) if i_ts > b_ts => {
+ // Incoming wins - generate UPDATE on the base row
stats.conflicts += 1;
stats.records_from_server += 1;
- if let Some(stmt) = generate_update_sql(table_name, server_record, &local_id, primary_key) {
+ if let Some(stmt) = generate_update_sql(table_name, incoming_record, &base_id, primary_key) {
statements.push(stmt);
}
}
_ => {
- // Local wins - no action needed
+ // Base wins - no action needed
stats.records_from_local += 1;
}
}
- server_map.remove(&local_id);
+ incoming_map.remove(&base_id);
} else {
- // Only in local (created offline) - no action needed
+ // Only in base - no action needed
stats.records_created_locally += 1;
}
}
- // Server-only records - generate INSERTs
- for server_record in server_map.values() {
+ // Incoming-only records - generate INSERTs into the base
+ for incoming_record in incoming_map.values() {
stats.records_inserted += 1;
- if let Some(stmt) = generate_insert_sql(table_name, server_record) {
+ if let Some(stmt) = generate_insert_sql(table_name, incoming_record) {
statements.push(stmt);
}
}
@@ -230,70 +234,72 @@ fn merge_table_by_id(
}
/// Merge table by composite key.
-/// Returns SQL statements to apply to local database.
+/// `base_records` are the merge base (kept on tie / when the base wins, and their primary key is
+/// preserved on conflict); `incoming_records` are merged onto the base by composite key.
+/// Returns SQL statements to apply to the base database.
fn merge_table_by_composite_key(
table_name: &str,
- local_records: &[Record],
- server_records: &[Record],
+ base_records: &[Record],
+ incoming_records: &[Record],
key_columns: &[&str],
primary_key: &str,
stats: &mut MergeStats,
) -> Vec<SqlStatement> {
let mut statements: Vec<SqlStatement> = Vec::new();
- // Create map of server records by composite key
- let mut server_map: HashMap<String, &Record> = HashMap::new();
- for record in server_records {
+ // Create map of incoming records by composite key
+ let mut incoming_map: HashMap<String, &Record> = HashMap::new();
+ for record in incoming_records {
let key = get_composite_key(record, key_columns);
// Keep the one with latest UpdatedAt if duplicate keys
- if let Some(existing) = server_map.get(&key) {
+ if let Some(existing) = incoming_map.get(&key) {
if get_updated_at(record) > get_updated_at(existing) {
- server_map.insert(key, record);
+ incoming_map.insert(key, record);
}
} else {
- server_map.insert(key, record);
+ incoming_map.insert(key, record);
}
}
- // Process local records
- for local_record in local_records {
- let composite_key = get_composite_key(local_record, key_columns);
+ // Process base records
+ for base_record in base_records {
+ let composite_key = get_composite_key(base_record, key_columns);
- let local_id = match get_record_pk(local_record, primary_key) {
+ let base_id = match get_record_pk(base_record, primary_key) {
Some(id) => id,
None => continue,
};
- if let Some(server_record) = server_map.get(&composite_key) {
+ if let Some(incoming_record) = incoming_map.get(&composite_key) {
// Record exists in both - compare UpdatedAt
- let local_ts = get_updated_at(local_record);
- let server_ts = get_updated_at(server_record);
+ let incoming_ts = get_updated_at(incoming_record);
+ let base_ts = get_updated_at(base_record);
- match (server_ts, local_ts) {
- (Some(s_ts), Some(l_ts)) if s_ts > l_ts => {
- // Server wins - update with server data but keep local Id
+ match (incoming_ts, base_ts) {
+ (Some(i_ts), Some(b_ts)) if i_ts > b_ts => {
+ // Incoming wins - update with incoming data but keep the base Id
stats.conflicts += 1;
stats.records_from_server += 1;
- if let Some(stmt) = generate_update_sql(table_name, server_record, &local_id, primary_key) {
+ if let Some(stmt) = generate_update_sql(table_name, incoming_record, &base_id, primary_key) {
statements.push(stmt);
}
}
_ => {
- // Local wins - no action needed
+ // Base wins - no action needed
stats.records_from_local += 1;
}
}
- server_map.remove(&composite_key);
+ incoming_map.remove(&composite_key);
} else {
- // Only in local - no action needed
+ // Only in base - no action needed
stats.records_created_locally += 1;
}
}
- // Server-only records (by composite key) - generate INSERTs
- for (_key, server_record) in &server_map {
+ // Incoming-only records (by composite key) - generate INSERTs into the base
+ for (_key, incoming_record) in &incoming_map {
stats.records_inserted += 1;
- if let Some(stmt) = generate_insert_sql(table_name, server_record) {
+ if let Some(stmt) = generate_insert_sql(table_name, incoming_record) {
statements.push(stmt);
}
}
@@ -460,14 +466,16 @@ mod tests {
#[test]
fn test_merge_vaults_json() {
+ // Base is the server vault. A newer LOCAL row wins and is written onto the server base as an
+ // UPDATE. (A newer server row would win with no statement, since the base already holds it.)
let input = MergeInput {
local_tables: vec![TableData {
name: "Items".to_string(),
- records: vec![make_record("1", "2024-01-01T00:00:00Z")],
+ records: vec![make_record("1", "2024-01-02T00:00:00Z")],
}],
server_tables: vec![TableData {
name: "Items".to_string(),
- records: vec![make_record("1", "2024-01-02T00:00:00Z")],
+ records: vec![make_record("1", "2024-01-01T00:00:00Z")],
}],
};
@@ -477,11 +485,35 @@ mod tests {
assert!(output.success);
assert_eq!(output.stats.conflicts, 1);
- // Should have one UPDATE statement
+ // Should have one UPDATE statement applied to the server base.
assert_eq!(output.statements.len(), 1);
assert!(output.statements[0].sql.starts_with("UPDATE Items SET"));
}
+ #[test]
+ fn merge_vault_server_newer_is_noop_on_base() {
+ // The mirror case: a newer SERVER row already sits in the base, so the merge emits nothing.
+ let input = MergeInput {
+ local_tables: vec![TableData { name: "Items".to_string(), records: vec![make_record("1", "2024-01-01T00:00:00Z")] }],
+ server_tables: vec![TableData { name: "Items".to_string(), records: vec![make_record("1", "2024-01-02T00:00:00Z")] }],
+ };
+ let output = merge_vaults(input).unwrap();
+ assert!(output.success);
+ assert!(output.statements.is_empty(), "server base already holds the winning row");
+ }
+
+ #[test]
+ fn merge_vault_local_only_row_inserted_into_base() {
+ // A row created offline (local only) must be inserted into the server base.
+ let input = MergeInput {
+ local_tables: vec![TableData { name: "Items".to_string(), records: vec![make_record("local-only", "2024-01-01T00:00:00Z")] }],
+ server_tables: vec![TableData { name: "Items".to_string(), records: vec![] }],
+ };
+ let output = merge_vaults(input).unwrap();
+ assert_eq!(output.statements.len(), 1);
+ assert!(output.statements[0].sql.starts_with("INSERT OR REPLACE INTO Items"));
+ }
+
fn logo(id: &str, source: &str, updated_at: &str) -> Record {
let mut r = HashMap::new();
r.insert("Id".to_string(), serde_json::json!(id));
@@ -572,6 +604,32 @@ mod tests {
assert_eq!(cfg.composite_key_columns, &["Source"]);
}
+ #[test]
+ fn merge_never_touches_codec_overflow_carrier() {
+ // The merge base is the server vault, so the codec overflow carrier rides along on the server
+ // base untouched (implicit server-wins) and needs no special case. Even if a platform were to
+ // pass the carrier in the merge input, merge must emit NO statement referencing it — it is not
+ // a syncable table.
+ let overflow_table = crate::vault_codec::OVERFLOW_TABLE;
+ let overflow_row = |data: &str| -> Record {
+ [("Id".to_string(), serde_json::json!("00000000-0000-0000-0000-00000000c0de")), ("Data".to_string(), serde_json::json!(data))].into_iter().collect()
+ };
+
+ let output = merge_vaults(MergeInput {
+ local_tables: vec![TableData { name: overflow_table.to_string(), records: vec![overflow_row("{\"stale\":true}")] }],
+ server_tables: vec![TableData { name: overflow_table.to_string(), records: vec![overflow_row("{\"fresh\":true}")] }],
+ })
+ .unwrap();
+ assert!(!output.statements.iter().any(|s| s.sql.contains(overflow_table)), "no carrier statements: the server base owns the overflow");
+ }
+
+ #[test]
+ fn merge_table_names_excludes_overflow_carrier() {
+ let names = merge_table_names();
+ assert_eq!(names.len(), SYNCABLE_TABLE_NAMES.len());
+ assert!(!names.contains(&crate::vault_codec::OVERFLOW_TABLE), "carrier is not part of the merge input");
+ }
+
#[test]
fn test_generate_insert_sql() {
let record = make_record("test-id", "2024-01-01T00:00:00Z");
diff --git a/core/rust/src/vault_merge/types.rs b/core/rust/src/vault_merge/types.rs
index 43ffc633d..60db8fba3 100644
--- a/core/rust/src/vault_merge/types.rs
+++ b/core/rust/src/vault_merge/types.rs
@@ -58,6 +58,11 @@ pub static SYNCABLE_TABLES: &[TableConfig] = &[
TableConfig::new("Settings").with_primary_key("Key"),
];
+/// The tables a platform must read into the merge input: the syncable tables.
+pub fn merge_table_names() -> Vec<&'static str> {
+ SYNCABLE_TABLE_NAMES.to_vec()
+}
+
/// List of syncable table names (for clients to know which tables to read).
pub const SYNCABLE_TABLE_NAMES: &[&str] = &[
"Items",
diff --git a/core/rust/src/wasm.rs b/core/rust/src/wasm.rs
index 6c4ea9c3f..794fe324e 100644
--- a/core/rust/src/wasm.rs
+++ b/core/rust/src/wasm.rs
@@ -38,13 +38,10 @@ pub fn get_core_version_js() -> String {
// Vault Merge WASM Bindings
// ═══════════════════════════════════════════════════════════════════════════════
-/// Get the list of table names that need to be synced.
+/// Get the list of syncable table names a platform must read into the merge input.
#[wasm_bindgen(js_name = getSyncableTableNames)]
pub fn get_syncable_table_names() -> Vec<String> {
- crate::vault_merge::SYNCABLE_TABLE_NAMES
- .iter()
- .map(|s| s.to_string())
- .collect()
+ crate::vault_merge::merge_table_names().iter().map(|s| s.to_string()).collect()
}
/// Merge vaults using LWW strategy.
@@ -145,21 +142,18 @@ pub fn vault_codec_materialize_as_sqlite_js(input: JsValue) -> Result<JsValue, J
codec_to_js(&output)
}
-/// Build a single data bucket. Input: `{ category, tables: { <name>: [rows] }, overflow? }`. Output: `DataBucket`.
+/// Build a single data bucket. Input: `{ category, tables: { <name>: [rows] } }`.
#[wasm_bindgen(js_name = vaultCodecExtractBucket)]
pub fn vault_codec_extract_bucket_js(input: JsValue) -> Result<JsValue, JsValue> {
#[derive(serde::Deserialize)]
- #[serde(rename_all = "camelCase")]
struct Input {
category: String,
#[serde(default)]
tables: std::collections::HashMap<String, Vec<CodecRecord>>,
- #[serde(default)]
- overflow: Option<vault_codec::CodecOverflow>,
}
let input: Input = serde_wasm_bindgen::from_value(input)
.map_err(|e| JsValue::from_str(&format!("Failed to parse extract-bucket input: {}", e)))?;
- codec_to_js(&vault_codec::extract_bucket(input.category, input.tables, input.overflow.as_ref()))
+ codec_to_js(&vault_codec::extract_bucket(input.category, input.tables))
}
/// The bucket layout: `[{ category, tables: [<name>] }]`. Source of truth for platform bucket-only sync.