Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update Rust core for vault merge codec overflow logic (#2159)
Leendert de Borstd76f5d60a11d
diff --git a/core/rust/src/ffi.rs b/core/rust/src/ffi.rsindex 6233fe245..2ba6b3d99 100644--- a/core/rust/src/ffi.rs+++ b/core/rust/src/ffi.rs@@ -144,7 +144,7 @@ pub unsafe extern "C" fn filter_credentials_ffi(input_json: *const c_char) -> *m/// A null-terminated C string containing a JSON array of table names.#[no_mangle]pub extern "C" fn get_syncable_table_names_ffi() -> *mut c_char {- let names: Vec<&str> = SYNCABLE_TABLE_NAMES.iter().map(|s| *s).collect();+ let names = crate::vault_merge::merge_table_names();match serde_json::to_string(&names) {Ok(json) => string_to_c_char(json),Err(_) => ptr::null_mut(),diff --git a/core/rust/src/uniffi_api.rs b/core/rust/src/uniffi_api.rsindex 69cae50f2..7b5841757 100644--- a/core/rust/src/uniffi_api.rs+++ b/core/rust/src/uniffi_api.rs@@ -12,14 +12,10 @@ pub fn get_core_version() -> String {crate::get_core_version().to_string()}-/// Get the list of syncable table names.-/// These are the tables that need to be read from the database for merge/prune operations.+/// Get the list of syncable table names a platform must read into the merge input.#[uniffi::export]pub fn get_syncable_table_names() -> Vec<String> {- SYNCABLE_TABLE_NAMES- .iter()- .map(|s| s.to_string())- .collect()+ crate::vault_merge::merge_table_names().iter().map(|s| s.to_string()).collect()}/// Merge local and server vaults using Last-Write-Wins strategy.@@ -359,6 +355,7 @@ mod tests {assert!(names.contains(&"FieldValues".to_string()));assert!(names.contains(&"Settings".to_string()));assert!(names.contains(&"EncryptionKeys".to_string()));+ assert!(!names.contains(&crate::vault_codec::OVERFLOW_TABLE.to_string()), "overflow carrier is not in the merge input; the server base owns it");assert_eq!(names.len(), 13);}diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rsindex a05e62457..9172ffc4a 100644--- a/core/rust/src/vault_codec/canonicalize.rs+++ b/core/rust/src/vault_codec/canonicalize.rs@@ -20,7 +20,7 @@ use super::hash::salted_blob_hash;use super::logos::dedupe_logos_by_source;use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedVault, CodecOverflow, DataBucket, Manifest, CodecRecord};use super::materialize::row_key;-use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_skip_table, primary_key_for, SCHEMA_VERSION};+use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_skip_table, primary_key_for, OVERFLOW_TABLE, SCHEMA_VERSION};use crate::error::VaultResult;/// Canonicalize normalized tables into the split resources: the manifest, one data bucket per declared@@ -30,16 +30,22 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic// extraction and bucket-splitting happen below; deduping Logos first (before extraction) means a// dropped duplicate never registers an orphan favicon blob.let mut all_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();+ let mut overflow = CodecOverflow::default();for table in &input.tables {+ // The OVERFLOW_TABLE row carries a newer writer's tables/columns this client's schema+ // couldn't hold (written by the last materialize). Consume it here — re-merged below,+ // never emitted into the manifest itself.+ if table.name == OVERFLOW_TABLE {+ overflow = CodecOverflow::from_table_records(&table.records);+ continue;+ }if is_skip_table(&table.name) {continue;}all_tables.entry(table.name.clone()).or_default().extend(table.records.iter().cloned());}- // Re-merge overflow from the last materialize (a newer writer's tables/columns this client's- // local schema couldn't hold) so this push doesn't drop them. See `CodecOverflow`.- let overflow = input.overflow.unwrap_or_default();+ // Re-merge the overflow so this push doesn't drop a newer writer's data. See `CodecOverflow`.remerge_overflow_columns(&mut all_tables, &overflow);for (name, rows) in &overflow.tables {// Local rows win if the table somehow exists locally now (e.g. client upgraded since the pull).@@ -175,17 +181,17 @@ fn remerge_overflow_columns(tables: &mut HashMap<String, Vec<CodecRecord>>, over/// The bucket-only push path (a bucket changed but the manifest didn't). Input rows are/// already normalized by the platform read logic.///-/// `overflow` (persisted from the last materialize) is re-merged the same way the full-/// canonicalize does it: unknown columns re-attach to this bucket's rows, and whole unknown tables-/// recorded under this bucket's category are re-emitted, so a bucket-only push from an older-/// client never drops a newer writer's data either.-pub fn extract_bucket(category: String, mut tables: HashMap<String, Vec<CodecRecord>>, overflow: Option<&CodecOverflow>) -> DataBucket {- if let Some(overflow) = overflow {- remerge_overflow_columns(&mut tables, overflow);- if let Some(ov_tables) = overflow.bucket_tables.get(&category) {- for (name, rows) in ov_tables {- tables.entry(name.clone()).or_insert_with(|| rows.clone());- }+/// When `tables` includes the [`OVERFLOW_TABLE`] row (callers should read it alongside the+/// bucket's tables), it is consumed and re-merged the same way the full canonicalize does it:+/// unknown columns re-attach to this bucket's rows, and whole unknown tables recorded under this+/// bucket's category are re-emitted, so a bucket-only push from an older client never drops a+/// newer writer's data either.+pub fn extract_bucket(category: String, mut tables: HashMap<String, Vec<CodecRecord>>) -> DataBucket {+ let overflow = tables.remove(OVERFLOW_TABLE).map(|records| CodecOverflow::from_table_records(&records)).unwrap_or_default();+ remerge_overflow_columns(&mut tables, &overflow);+ if let Some(ov_tables) = overflow.bucket_tables.get(&category) {+ for (name, rows) in ov_tables {+ tables.entry(name.clone()).or_insert_with(|| rows.clone());}}DataBucket::new(category, tables)diff --git a/core/rust/src/vault_codec/manifest.rs b/core/rust/src/vault_codec/manifest.rsindex 206e4fd4d..45c26a2d6 100644--- a/core/rust/src/vault_codec/manifest.rs+++ b/core/rust/src/vault_codec/manifest.rs@@ -115,12 +115,33 @@ impl CodecOverflow {pub fn is_empty(&self) -> bool {self.tables.is_empty() && self.bucket_tables.is_empty() && self.columns.is_empty()}++ /// Render this overflow as the single `OVERFLOW_TABLE` row the platform inserts into the vault DB.+ pub fn to_table_records(&self) -> Vec<CodecRecord> {+ let mut row: CodecRecord = HashMap::new();+ row.insert("Id".to_string(), serde_json::Value::String(super::types::OVERFLOW_ROW_ID.to_string()));+ row.insert("Data".to_string(), serde_json::Value::String(serde_json::to_string(self).unwrap_or_default()));+ vec![row]+ }++ /// Parse an `OVERFLOW_TABLE` row set read back from the vault DB. Tolerant: no rows, a missing+ /// `Data` column, or unparseable JSON all yield an empty overflow (better to push what we have+ /// than refuse to push at all).+ pub fn from_table_records(records: &[CodecRecord]) -> Self {+ records+ .first()+ .and_then(|row| row.get("Data"))+ .and_then(|v| v.as_str())+ .and_then(|json| serde_json::from_str(json).ok())+ .unwrap_or_default()+ }}/// Materialized tables the platform inserts into a fresh schema DB. Blob columns carry-/// `{ "__blobRef": hash }`; inline byte columns carry `{ "__b64": ... }`. `overflow` holds whatever-/// the local schema couldn't accept (see [`CodecOverflow`]); the platform must persist it and feed-/// it back into the next canonicalize.+/// `{ "__blobRef": hash }`; inline byte columns carry `{ "__b64": ... }`. Any overflow (see+/// [`CodecOverflow`]) is already included in `tables` as the `OVERFLOW_TABLE` row — the platform+/// inserts it like any other table and needs no separate persistence. The `overflow` field is a+/// diagnostics copy of the same data (for logging), not something the platform must store.#[derive(Debug, Clone, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]pub struct MaterializedTables {@@ -134,16 +155,15 @@ pub struct MaterializedTables {#[derive(Debug, Clone, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]pub struct CanonicalizeInput {+ /// All local vault tables (a plain `SELECT *` read). When the read includes the+ /// [`OVERFLOW_TABLE`](super::types::OVERFLOW_TABLE) row written by the last materialize, its+ /// overflow (a newer writer's tables/columns this schema can't hold) is re-merged automatically.pub tables: Vec<CodecTableData>,pub user_salt: String,pub migration_id: String,#[serde(default = "default_version")]pub version: String,pub canonicalized_at: String,- /// Overflow persisted from the last materialize (see [`CodecOverflow`]); re-merged into the- /// output so a client with an older schema never drops a newer writer's tables/columns.- #[serde(default)]- pub overflow: Option<CodecOverflow>,}fn default_version() -> String {diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rsindex 5497b97fa..ce680856a 100644--- a/core/rust/src/vault_codec/materialize.rs+++ b/core/rust/src/vault_codec/materialize.rs@@ -10,13 +10,15 @@//! Forward compatibility: when the caller supplies its local schema (`schema_columns`), anything a//! newer writer put in the manifest that this schema cannot hold — whole unknown tables or unknown//! columns on known tables — is split into [`CodecOverflow`] instead of being emitted (which would-//! crash the platform insert). The platform persists the overflow and passes it back into-//! `canonicalize_from_sqlite`, which re-merges it so this client's next push never drops the data.+//! crash the platform insert). The overflow is emitted as a regular table row (`OVERFLOW_TABLE`),+//! so it lives inside the vault DB itself and `canonicalize_from_sqlite` re-merges it from the+//! ordinary table read — this client's next push never drops the data, and no platform has to wire+//! (or remember) a separate persistence channel.use std::collections::{HashMap, HashSet};use super::manifest::{CodecOverflow, CodecRecord, CodecTableData, MaterializeInput, MaterializedTables};-use super::types::{is_skip_table, primary_key_for};+use super::types::{is_skip_table, primary_key_for, OVERFLOW_TABLE};use crate::error::VaultResult;/// Materialize the manifest + its data buckets into the table set the platform inserts.@@ -27,7 +29,9 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materializelet mut tables: Vec<CodecTableData> = Vec::with_capacity(manifest.tables.len() + data_buckets.len());for (name, records) in manifest.tables {- if is_skip_table(&name) {+ // OVERFLOW_TABLE is local-only bookkeeping: it must never occur in a manifest, and passing+ // one through would collide with the row this function emits below.+ if is_skip_table(&name) || name == OVERFLOW_TABLE {continue;}match split_for_schema(&name, records, schema_columns.as_ref(), &mut overflow.columns) {@@ -42,7 +46,7 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize// their category so canonicalize / extract_bucket can re-emit them into the right bucket.for bucket in data_buckets {for (name, records) in bucket.tables {- if is_skip_table(&name) {+ if is_skip_table(&name) || name == OVERFLOW_TABLE {continue;}match split_for_schema(&name, records, schema_columns.as_ref(), &mut overflow.columns) {@@ -54,6 +58,11 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize}}+ // Carry the overflow inside the vault DB itself: one OVERFLOW_TABLE row, inserted like any table.+ if !overflow.is_empty() {+ tables.push(CodecTableData { name: OVERFLOW_TABLE.to_string(), records: overflow.to_table_records() });+ }+Ok(MaterializedTables {tables,migration_id: manifest.migration_id,diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rsindex c3596cc8e..2ba572361 100644--- a/core/rust/src/vault_codec/mod.rs+++ b/core/rust/src/vault_codec/mod.rs@@ -29,7 +29,7 @@ pub use manifest::{};pub use types::{bucket_categories, bucket_category_for, tables_for_category, BLOB_COLUMNS, BUCKET_TABLES,- SCHEMA_VERSION, SKIP_TABLES,+ OVERFLOW_ROW_ID, OVERFLOW_TABLE, SCHEMA_VERSION, SKIP_TABLES,};pub use validate::ValidationResult;@@ -47,10 +47,11 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materializematerialize::materialize_as_sqlite(input)}-/// Build a single data bucket for `category` from its tables (bucket-only push path). `overflow`-/// (persisted from the last materialize) re-merges a newer writer's columns/tables so they survive.-pub fn extract_bucket(category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>, overflow: Option<&CodecOverflow>) -> DataBucket {- canonicalize::extract_bucket(category, tables, overflow)+/// Build a single data bucket for `category` from its tables (bucket-only push path). Include the+/// [`OVERFLOW_TABLE`] row in `tables` (read it alongside the bucket's tables) so a newer writer's+/// columns/tables re-merge and survive; it is consumed and never emitted into the bucket.+pub fn extract_bucket(category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>) -> DataBucket {+ canonicalize::extract_bucket(category, tables)}/// The bucket layout: every category and the tables it owns, in declaration order.@@ -160,19 +161,16 @@ pub fn materialize_as_sqlite_json(input_json: &str) -> VaultResult<String> {Ok(serde_json::to_string(&materialize_as_sqlite(input)?)?)}-/// JSON-string sibling of [`extract_bucket`]. Input: `{ "category": <str>, "tables": { <name>: [rows] }, "overflow"?: CodecOverflow }`.+/// JSON-string sibling of [`extract_bucket`]. Input: `{ "category": <str>, "tables": { <name>: [rows] } }`.pub fn extract_bucket_json(input_json: &str) -> VaultResult<String> {#[derive(serde::Deserialize)]- #[serde(rename_all = "camelCase")]struct Input {category: String,#[serde(default)]tables: std::collections::HashMap<String, Vec<CodecRecord>>,- #[serde(default)]- overflow: Option<CodecOverflow>,}let input: Input = serde_json::from_str(input_json)?;- Ok(serde_json::to_string(&extract_bucket(input.category, input.tables, input.overflow.as_ref()))?)+ Ok(serde_json::to_string(&extract_bucket(input.category, input.tables))?)}/// JSON-string sibling of [`bucket_layout`]. Output: `[{ "category": <str>, "tables": [<str>] }]`.diff --git a/core/rust/src/vault_codec/tests.rs b/core/rust/src/vault_codec/tests.rsindex 560d9ba67..91c9797e4 100644--- a/core/rust/src/vault_codec/tests.rs+++ b/core/rust/src/vault_codec/tests.rs@@ -29,7 +29,6 @@ fn basic_input(tables: Vec<CodecTableData>) -> CanonicalizeInput {migration_id: "20250101000000_Init".to_string(),version: "2.0.0".to_string(),canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(),- overflow: None,}}@@ -382,21 +381,29 @@ fn validate_manifest_rejects_duplicate_logo_sources() {assert!(result.failed_rules.iter().any(|r| r == "logo-sources-not-unique"));}-/// A minimal local schema map for overflow tests: Items(Id, Name), Settings(Key, Value).+/// A minimal local schema map for overflow tests: Items(Id, Name), Settings(Key, Value), and the+/// CodecOverflows carrier table every client schema owns since migration 2.1.0.fn old_client_schema() -> std::collections::HashMap<String, Vec<String>> {[("Items".to_string(), vec!["Id".to_string(), "Name".to_string()]),("Settings".to_string(), vec!["Key".to_string(), "Value".to_string()]),+ (OVERFLOW_TABLE.to_string(), vec!["Id".to_string(), "Data".to_string()]),].into_iter().collect()}+/// The OVERFLOW_TABLE entry from a materialize result (what the platform inserts into the vault DB).+fn overflow_table_of(re: &MaterializedTables) -> Option<&CodecTableData> {+ re.tables.iter().find(|t| t.name == OVERFLOW_TABLE)+}+#[test]-fn materialize_splits_unknown_columns_into_overflow_and_canonicalize_remerges() {+fn materialize_splits_unknown_columns_into_overflow_table_and_canonicalize_remerges() {// A newer client wrote Items.AliasEnabled; this client's schema doesn't know it. The column must- // not reach the insert (it would crash), must land in overflow keyed by row Id, and must- // reappear on the next canonicalize so the push doesn't drop it.+ // not reach the Items insert (it would crash), must land in the emitted OVERFLOW_TABLE row, and+ // must reappear on the next canonicalize (which reads that row back like any table) so the push+ // doesn't drop it.let out = canonicalize_from_sqlite(basic_input(vec![CodecTableData {name: "Items".to_string(),records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub")), ("AliasEnabled", json!(true))])],@@ -407,18 +414,23 @@ fn materialize_splits_unknown_columns_into_overflow_and_canonicalize_remerges()let items = re.tables.iter().find(|t| t.name == "Items").unwrap();assert!(!items.records[0].contains_key("AliasEnabled"), "unknown column filtered out of the insert set");assert_eq!(items.records[0]["Name"], json!("GitHub"));- assert_eq!(re.overflow.columns["Items"]["i1"]["AliasEnabled"], json!(true));+ assert_eq!(re.overflow.columns["Items"]["i1"]["AliasEnabled"], json!(true), "diagnostics copy populated");- // The old client edits Name locally, then pushes: the overflow column re-attaches to the row.- let mut push_input = basic_input(vec![CodecTableData {- name: "Items".to_string(),- records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub (renamed)"))])],- }]);- push_input.overflow = Some(re.overflow);- let pushed = canonicalize_from_sqlite(push_input).unwrap();+ let overflow_table = overflow_table_of(&re).expect("overflow emitted as a regular table row");+ assert_eq!(overflow_table.records.len(), 1);+ assert_eq!(overflow_table.records[0]["Id"], json!(OVERFLOW_ROW_ID));++ // The old client edits Name locally, then pushes: canonicalize reads the overflow row back from+ // the DB (plain SELECT *) and re-attaches the column; the carrier table itself never reaches the manifest.+ let pushed = canonicalize_from_sqlite(basic_input(vec![+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub (renamed)"))])] },+ overflow_table.clone(),+ ]))+ .unwrap();let item = &pushed.manifest.tables["Items"][0];assert_eq!(item["Name"], json!("GitHub (renamed)"));assert_eq!(item["AliasEnabled"], json!(true), "newer writer's column survives the old client's push");+ assert!(!pushed.manifest.tables.contains_key(OVERFLOW_TABLE), "carrier table consumed, never emitted into the manifest");}#[test]@@ -428,16 +440,18 @@ fn overflow_of_locally_deleted_row_is_dropped_on_canonicalize() {columns: [("Items".to_string(), [("gone".to_string(), row(&[("AliasEnabled", json!(true))]))].into_iter().collect())].into_iter().collect(),..Default::default()};- let mut input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("kept"))])] }]);- input.overflow = Some(overflow);- let out = canonicalize_from_sqlite(input).unwrap();+ let out = canonicalize_from_sqlite(basic_input(vec![+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("kept"))])] },+ CodecTableData { name: OVERFLOW_TABLE.to_string(), records: overflow.to_table_records() },+ ]))+ .unwrap();assert!(!out.manifest.tables["Items"][0].contains_key("AliasEnabled"));}#[test]fn materialize_splits_unknown_tables_into_overflow_and_canonicalize_reemits() {// A newer client added a whole table (manifest-level) and a whole bucket table. Neither exists- // in this client's schema; both must round-trip through overflow back to their original place.+ // in this client's schema; both must round-trip through the overflow row back to their original place.let mut out = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }])).unwrap();out.manifest.tables.insert("NewTable".to_string(), vec![row(&[("Id", json!("n1")), ("Data", json!("x"))])]);out.data_buckets[0].tables.insert("Preferences".to_string(), vec![row(&[("Key", json!("p1"))])]);@@ -446,17 +460,29 @@ fn materialize_splits_unknown_tables_into_overflow_and_canonicalize_reemits() {assert!(!re.tables.iter().any(|t| t.name == "NewTable" || t.name == "Preferences"), "unknown tables never reach the insert set");assert_eq!(re.overflow.tables["NewTable"].len(), 1);assert_eq!(re.overflow.bucket_tables["Settings"]["Preferences"].len(), 1);+ let overflow_table = overflow_table_of(&re).expect("overflow emitted as a regular table row").clone();- let mut push_input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }]);- push_input.overflow = Some(re.overflow.clone());- let pushed = canonicalize_from_sqlite(push_input).unwrap();+ let pushed = canonicalize_from_sqlite(basic_input(vec![+ CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] },+ overflow_table.clone(),+ ]))+ .unwrap();assert_eq!(pushed.manifest.tables["NewTable"].len(), 1, "unknown manifest table re-emitted");assert_eq!(bucket_rows(&pushed, "Settings", "Preferences").len(), 1, "unknown bucket table re-emitted into its category");- // Bucket-only push path: extract_bucket must re-emit the bucket's overflow table too.- let bucket = extract_bucket("Settings".to_string(), [("Settings".to_string(), vec![row(&[("Key", json!("k")), ("Value", json!("v"))])])].into_iter().collect(), Some(&re.overflow));+ // Bucket-only push path: extract_bucket consumes the overflow row read alongside the bucket's tables.+ let bucket = extract_bucket(+ "Settings".to_string(),+ [+ ("Settings".to_string(), vec![row(&[("Key", json!("k")), ("Value", json!("v"))])]),+ (OVERFLOW_TABLE.to_string(), overflow_table.records),+ ]+ .into_iter()+ .collect(),+ );assert_eq!(bucket.tables["Preferences"].len(), 1);assert_eq!(bucket.tables["Settings"].len(), 1);+ assert!(!bucket.tables.contains_key(OVERFLOW_TABLE), "carrier table consumed, never emitted into the bucket");}#[test]@@ -466,14 +492,22 @@ fn extract_bucket_remerges_overflow_columns() {columns: [("Settings".to_string(), [("theme".to_string(), row(&[("SyncScope", json!("device"))]))].into_iter().collect())].into_iter().collect(),..Default::default()};- let bucket = extract_bucket("Settings".to_string(), [("Settings".to_string(), vec![row(&[("Key", json!("theme")), ("Value", json!("dark"))])])].into_iter().collect(), Some(&overflow));+ let bucket = extract_bucket(+ "Settings".to_string(),+ [+ ("Settings".to_string(), vec![row(&[("Key", json!("theme")), ("Value", json!("dark"))])]),+ (OVERFLOW_TABLE.to_string(), overflow.to_table_records()),+ ]+ .into_iter()+ .collect(),+ );assert_eq!(bucket.tables["Settings"][0]["SyncScope"], json!("device"));assert_eq!(bucket.tables["Settings"][0]["Value"], json!("dark"));}#[test]fn materialize_without_schema_columns_passes_rows_through_verbatim() {- // Legacy callers (no schema map) keep today's behavior exactly: nothing filtered, empty overflow.+ // Legacy callers (no schema map) keep today's behavior exactly: nothing filtered, no overflow row.let out = canonicalize_from_sqlite(basic_input(vec![CodecTableData {name: "Items".to_string(),records: vec![row(&[("Id", json!("i1")), ("AliasEnabled", json!(true))])],@@ -483,6 +517,20 @@ fn materialize_without_schema_columns_passes_rows_through_verbatim() {let items = re.tables.iter().find(|t| t.name == "Items").unwrap();assert_eq!(items.records[0]["AliasEnabled"], json!(true));assert!(re.overflow.is_empty());+ assert!(overflow_table_of(&re).is_none(), "no overflow row emitted when there is nothing to carry");+}++#[test]+fn materialize_drops_overflow_table_smuggled_into_a_manifest() {+ // Defense: OVERFLOW_TABLE is local-only bookkeeping. A manifest that somehow carries one (corrupt+ // or malicious) must not pass through — it would collide with the row materialize emits itself.+ let mut out = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("AliasEnabled", json!(true))])] }])).unwrap();+ out.manifest.tables.insert(OVERFLOW_TABLE.to_string(), vec![row(&[("Id", json!("smuggled")), ("Data", json!("{}"))])]);++ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: Some(old_client_schema()) }).unwrap();+ let overflow_table = overflow_table_of(&re).expect("legitimate overflow row still emitted");+ assert_eq!(overflow_table.records.len(), 1);+ assert_eq!(overflow_table.records[0]["Id"], json!(OVERFLOW_ROW_ID), "smuggled row dropped, only the codec's own row remains");}#[test]diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rsindex 6edbad373..8a207577c 100644--- a/core/rust/src/vault_codec/types.rs+++ b/core/rust/src/vault_codec/types.rs@@ -69,3 +69,12 @@ pub fn is_skip_table(table_name: &str) -> bool {pub fn primary_key_for(table_name: &str) -> &'static str {crate::vault_merge::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key).unwrap_or("Id")}++/// Client-local SQLite table that carries the codec overflow inside the vault database itself (see+/// `CodecOverflow`): materialize writes a single row `{ Id: OVERFLOW_ROW_ID, Data: <json> }`, and+/// canonicalize / extract_bucket consume it to build the manifest.+pub const OVERFLOW_TABLE: &str = "CodecOverflows";++/// Fixed sentinel primary key of the single `OVERFLOW_TABLE` row (deterministic on purpose:+/// materialize output must not depend on a random source).+pub const OVERFLOW_ROW_ID: &str = "00000000-0000-0000-0000-00000000c0de";diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rsindex 599c8ddb0..b26b4ec1d 100644--- a/core/rust/src/vault_merge/mod.rs+++ b/core/rust/src/vault_merge/mod.rs@@ -11,7 +11,7 @@ use std::collections::HashMap;use crate::error::VaultResult;pub use types::SYNCABLE_TABLE_NAMES;-pub use types::{SYNCABLE_TABLES, TableConfig};+pub use types::{merge_table_names, SYNCABLE_TABLES, TableConfig};/// A record is a map of column names to JSON values.pub type Record = HashMap<String, serde_json::Value>;@@ -74,24 +74,28 @@ pub struct MergeOutput {/// Main entry point: merge local and server vault data.///+/// The merge **base is the server vault** (`input.server_tables`): a freshly-materialized SQLite with+/// the newest schema and the newest codec overflow carrier already in place. The returned statements+/// bring the local vault's winning changes (`input.local_tables`) onto that base.+////// # Arguments-/// * `input` - MergeInput containing local and server table data+/// * `input` - MergeInput containing local (incoming) and server (base) table data////// # Returns-/// MergeOutput with SQL statements to execute on local database+/// MergeOutput with SQL statements to execute on the server (base) databasepub fn merge_vaults(input: MergeInput) -> VaultResult<MergeOutput> {let mut total_stats = MergeStats::default();let mut statements: Vec<SqlStatement> = Vec::new();- // Create lookup maps for quick access- let local_map: HashMap<&str, &TableData> = input- .local_tables+ // Base = server vault; incoming = local vault. Statements mutate the base.+ let base_map: HashMap<&str, &TableData> = input+ .server_tables.iter().map(|t| (t.name.as_str(), t)).collect();- let server_map: HashMap<&str, &TableData> = input- .server_tables+ let incoming_map: HashMap<&str, &TableData> = input+ .local_tables.iter().map(|t| (t.name.as_str(), t)).collect();@@ -100,20 +104,19 @@ pub fn merge_vaults(input: MergeInput) -> VaultResult<MergeOutput> {for table_config in SYNCABLE_TABLES {let table_name = table_config.name;- let local_data = local_map.get(table_name);- let server_data = server_map.get(table_name);+ let base_data = base_map.get(table_name);+ let incoming_data = incoming_map.get(table_name);// Skip if table doesn't exist in either database- let (local_records, server_records) = match (local_data, server_data) {- (Some(l), Some(s)) => (&l.records, &s.records),- (Some(l), None) => {- // Table only in local - nothing to merge- total_stats.records_created_locally += l.records.len() as u32;+ let (base_records, incoming_records) = match (base_data, incoming_data) {+ (Some(b), Some(i)) => (&b.records, &i.records),+ (Some(_), None) => {+ // Table only in the server base. Already present, nothing to bring over.continue;}- (None, Some(s)) => {- // Table only in server - insert all- for record in &s.records {+ (None, Some(i)) => {+ // Table only in local - insert all local rows into the server base.+ for record in &i.records {if let Some(stmt) = generate_insert_sql(table_name, record) {statements.push(stmt);total_stats.records_inserted += 1;@@ -129,8 +132,8 @@ pub fn merge_vaults(input: MergeInput) -> VaultResult<MergeOutput> {let table_statements = if table_config.uses_composite_key() {merge_table_by_composite_key(table_name,- local_records,- server_records,+ base_records,+ incoming_records,table_config.composite_key_columns,table_config.primary_key,&mut total_stats,@@ -138,8 +141,8 @@ pub fn merge_vaults(input: MergeInput) -> VaultResult<MergeOutput> {} else {merge_table_by_id(table_name,- local_records,- server_records,+ base_records,+ incoming_records,table_config.primary_key,&mut total_stats,)@@ -167,61 +170,62 @@ pub fn merge_vaults_json(input_json: &str) -> VaultResult<String> {/// Merge table records by their primary-key column (standard merge)./// `primary_key` is the column that identifies a row (usually "Id"; "Key" for Settings).-/// Returns SQL statements to apply to local database.+/// `base_records` are the merge base (kept on tie / when the base wins); `incoming_records` are the+/// changes merged onto it. Returns SQL statements to apply to the base database.fn merge_table_by_id(table_name: &str,- local_records: &[Record],- server_records: &[Record],+ base_records: &[Record],+ incoming_records: &[Record],primary_key: &str,stats: &mut MergeStats,) -> Vec<SqlStatement> {let mut statements: Vec<SqlStatement> = Vec::new();- // Create map of server records by primary key- let mut server_map: HashMap<String, &Record> = HashMap::new();- for record in server_records {+ // Create map of incoming records by primary key+ let mut incoming_map: HashMap<String, &Record> = HashMap::new();+ for record in incoming_records {if let Some(id) = get_record_pk(record, primary_key) {- server_map.insert(id, record);+ incoming_map.insert(id, record);}}- // Process local records- for local_record in local_records {- let local_id = match get_record_pk(local_record, primary_key) {+ // Process base records+ for base_record in base_records {+ let base_id = match get_record_pk(base_record, primary_key) {Some(id) => id,None => continue,};- if let Some(server_record) = server_map.get(&local_id) {+ if let Some(incoming_record) = incoming_map.get(&base_id) {// Record exists in both - compare UpdatedAt for LWW- let local_ts = get_updated_at(local_record);- let server_ts = get_updated_at(server_record);+ let incoming_ts = get_updated_at(incoming_record);+ let base_ts = get_updated_at(base_record);- match (server_ts, local_ts) {- (Some(s_ts), Some(l_ts)) if s_ts > l_ts => {- // Server wins - generate UPDATE+ match (incoming_ts, base_ts) {+ (Some(i_ts), Some(b_ts)) if i_ts > b_ts => {+ // Incoming wins - generate UPDATE on the base rowstats.conflicts += 1;stats.records_from_server += 1;- if let Some(stmt) = generate_update_sql(table_name, server_record, &local_id, primary_key) {+ if let Some(stmt) = generate_update_sql(table_name, incoming_record, &base_id, primary_key) {statements.push(stmt);}}_ => {- // Local wins - no action needed+ // Base wins - no action neededstats.records_from_local += 1;}}- server_map.remove(&local_id);+ incoming_map.remove(&base_id);} else {- // Only in local (created offline) - no action needed+ // Only in base - no action neededstats.records_created_locally += 1;}}- // Server-only records - generate INSERTs- for server_record in server_map.values() {+ // Incoming-only records - generate INSERTs into the base+ for incoming_record in incoming_map.values() {stats.records_inserted += 1;- if let Some(stmt) = generate_insert_sql(table_name, server_record) {+ if let Some(stmt) = generate_insert_sql(table_name, incoming_record) {statements.push(stmt);}}@@ -230,70 +234,72 @@ fn merge_table_by_id(}/// Merge table by composite key.-/// Returns SQL statements to apply to local database.+/// `base_records` are the merge base (kept on tie / when the base wins, and their primary key is+/// preserved on conflict); `incoming_records` are merged onto the base by composite key.+/// Returns SQL statements to apply to the base database.fn merge_table_by_composite_key(table_name: &str,- local_records: &[Record],- server_records: &[Record],+ base_records: &[Record],+ incoming_records: &[Record],key_columns: &[&str],primary_key: &str,stats: &mut MergeStats,) -> Vec<SqlStatement> {let mut statements: Vec<SqlStatement> = Vec::new();- // Create map of server records by composite key- let mut server_map: HashMap<String, &Record> = HashMap::new();- for record in server_records {+ // Create map of incoming records by composite key+ let mut incoming_map: HashMap<String, &Record> = HashMap::new();+ for record in incoming_records {let key = get_composite_key(record, key_columns);// Keep the one with latest UpdatedAt if duplicate keys- if let Some(existing) = server_map.get(&key) {+ if let Some(existing) = incoming_map.get(&key) {if get_updated_at(record) > get_updated_at(existing) {- server_map.insert(key, record);+ incoming_map.insert(key, record);}} else {- server_map.insert(key, record);+ incoming_map.insert(key, record);}}- // Process local records- for local_record in local_records {- let composite_key = get_composite_key(local_record, key_columns);+ // Process base records+ for base_record in base_records {+ let composite_key = get_composite_key(base_record, key_columns);- let local_id = match get_record_pk(local_record, primary_key) {+ let base_id = match get_record_pk(base_record, primary_key) {Some(id) => id,None => continue,};- if let Some(server_record) = server_map.get(&composite_key) {+ if let Some(incoming_record) = incoming_map.get(&composite_key) {// Record exists in both - compare UpdatedAt- let local_ts = get_updated_at(local_record);- let server_ts = get_updated_at(server_record);+ let incoming_ts = get_updated_at(incoming_record);+ let base_ts = get_updated_at(base_record);- match (server_ts, local_ts) {- (Some(s_ts), Some(l_ts)) if s_ts > l_ts => {- // Server wins - update with server data but keep local Id+ match (incoming_ts, base_ts) {+ (Some(i_ts), Some(b_ts)) if i_ts > b_ts => {+ // Incoming wins - update with incoming data but keep the base Idstats.conflicts += 1;stats.records_from_server += 1;- if let Some(stmt) = generate_update_sql(table_name, server_record, &local_id, primary_key) {+ if let Some(stmt) = generate_update_sql(table_name, incoming_record, &base_id, primary_key) {statements.push(stmt);}}_ => {- // Local wins - no action needed+ // Base wins - no action neededstats.records_from_local += 1;}}- server_map.remove(&composite_key);+ incoming_map.remove(&composite_key);} else {- // Only in local - no action needed+ // Only in base - no action neededstats.records_created_locally += 1;}}- // Server-only records (by composite key) - generate INSERTs- for (_key, server_record) in &server_map {+ // Incoming-only records (by composite key) - generate INSERTs into the base+ for (_key, incoming_record) in &incoming_map {stats.records_inserted += 1;- if let Some(stmt) = generate_insert_sql(table_name, server_record) {+ if let Some(stmt) = generate_insert_sql(table_name, incoming_record) {statements.push(stmt);}}@@ -460,14 +466,16 @@ mod tests {#[test]fn test_merge_vaults_json() {+ // Base is the server vault. A newer LOCAL row wins and is written onto the server base as an+ // UPDATE. (A newer server row would win with no statement, since the base already holds it.)let input = MergeInput {local_tables: vec![TableData {name: "Items".to_string(),- records: vec![make_record("1", "2024-01-01T00:00:00Z")],+ records: vec![make_record("1", "2024-01-02T00:00:00Z")],}],server_tables: vec![TableData {name: "Items".to_string(),- records: vec![make_record("1", "2024-01-02T00:00:00Z")],+ records: vec![make_record("1", "2024-01-01T00:00:00Z")],}],};@@ -477,11 +485,35 @@ mod tests {assert!(output.success);assert_eq!(output.stats.conflicts, 1);- // Should have one UPDATE statement+ // Should have one UPDATE statement applied to the server base.assert_eq!(output.statements.len(), 1);assert!(output.statements[0].sql.starts_with("UPDATE Items SET"));}+ #[test]+ fn merge_vault_server_newer_is_noop_on_base() {+ // The mirror case: a newer SERVER row already sits in the base, so the merge emits nothing.+ let input = MergeInput {+ local_tables: vec![TableData { name: "Items".to_string(), records: vec![make_record("1", "2024-01-01T00:00:00Z")] }],+ server_tables: vec![TableData { name: "Items".to_string(), records: vec![make_record("1", "2024-01-02T00:00:00Z")] }],+ };+ let output = merge_vaults(input).unwrap();+ assert!(output.success);+ assert!(output.statements.is_empty(), "server base already holds the winning row");+ }++ #[test]+ fn merge_vault_local_only_row_inserted_into_base() {+ // A row created offline (local only) must be inserted into the server base.+ let input = MergeInput {+ local_tables: vec![TableData { name: "Items".to_string(), records: vec![make_record("local-only", "2024-01-01T00:00:00Z")] }],+ server_tables: vec![TableData { name: "Items".to_string(), records: vec![] }],+ };+ let output = merge_vaults(input).unwrap();+ assert_eq!(output.statements.len(), 1);+ assert!(output.statements[0].sql.starts_with("INSERT OR REPLACE INTO Items"));+ }+fn logo(id: &str, source: &str, updated_at: &str) -> Record {let mut r = HashMap::new();r.insert("Id".to_string(), serde_json::json!(id));@@ -572,6 +604,32 @@ mod tests {assert_eq!(cfg.composite_key_columns, &["Source"]);}+ #[test]+ fn merge_never_touches_codec_overflow_carrier() {+ // The merge base is the server vault, so the codec overflow carrier rides along on the server+ // base untouched (implicit server-wins) and needs no special case. Even if a platform were to+ // pass the carrier in the merge input, merge must emit NO statement referencing it — it is not+ // a syncable table.+ let overflow_table = crate::vault_codec::OVERFLOW_TABLE;+ let overflow_row = |data: &str| -> Record {+ [("Id".to_string(), serde_json::json!("00000000-0000-0000-0000-00000000c0de")), ("Data".to_string(), serde_json::json!(data))].into_iter().collect()+ };++ let output = merge_vaults(MergeInput {+ local_tables: vec![TableData { name: overflow_table.to_string(), records: vec![overflow_row("{\"stale\":true}")] }],+ server_tables: vec![TableData { name: overflow_table.to_string(), records: vec![overflow_row("{\"fresh\":true}")] }],+ })+ .unwrap();+ assert!(!output.statements.iter().any(|s| s.sql.contains(overflow_table)), "no carrier statements: the server base owns the overflow");+ }++ #[test]+ fn merge_table_names_excludes_overflow_carrier() {+ let names = merge_table_names();+ assert_eq!(names.len(), SYNCABLE_TABLE_NAMES.len());+ assert!(!names.contains(&crate::vault_codec::OVERFLOW_TABLE), "carrier is not part of the merge input");+ }+#[test]fn test_generate_insert_sql() {let record = make_record("test-id", "2024-01-01T00:00:00Z");diff --git a/core/rust/src/vault_merge/types.rs b/core/rust/src/vault_merge/types.rsindex 43ffc633d..60db8fba3 100644--- a/core/rust/src/vault_merge/types.rs+++ b/core/rust/src/vault_merge/types.rs@@ -58,6 +58,11 @@ pub static SYNCABLE_TABLES: &[TableConfig] = &[TableConfig::new("Settings").with_primary_key("Key"),];+/// The tables a platform must read into the merge input: the syncable tables.+pub fn merge_table_names() -> Vec<&'static str> {+ SYNCABLE_TABLE_NAMES.to_vec()+}+/// List of syncable table names (for clients to know which tables to read).pub const SYNCABLE_TABLE_NAMES: &[&str] = &["Items",diff --git a/core/rust/src/wasm.rs b/core/rust/src/wasm.rsindex 6c4ea9c3f..794fe324e 100644--- a/core/rust/src/wasm.rs+++ b/core/rust/src/wasm.rs@@ -38,13 +38,10 @@ pub fn get_core_version_js() -> String {// Vault Merge WASM Bindings// ═══════════════════════════════════════════════════════════════════════════════-/// Get the list of table names that need to be synced.+/// Get the list of syncable table names a platform must read into the merge input.#[wasm_bindgen(js_name = getSyncableTableNames)]pub fn get_syncable_table_names() -> Vec<String> {- crate::vault_merge::SYNCABLE_TABLE_NAMES- .iter()- .map(|s| s.to_string())- .collect()+ crate::vault_merge::merge_table_names().iter().map(|s| s.to_string()).collect()}/// Merge vaults using LWW strategy.@@ -145,21 +142,18 @@ pub fn vault_codec_materialize_as_sqlite_js(input: JsValue) -> Result<JsValue, Jcodec_to_js(&output)}-/// Build a single data bucket. Input: `{ category, tables: { <name>: [rows] }, overflow? }`. Output: `DataBucket`.+/// Build a single data bucket. Input: `{ category, tables: { <name>: [rows] } }`.#[wasm_bindgen(js_name = vaultCodecExtractBucket)]pub fn vault_codec_extract_bucket_js(input: JsValue) -> Result<JsValue, JsValue> {#[derive(serde::Deserialize)]- #[serde(rename_all = "camelCase")]struct Input {category: String,#[serde(default)]tables: std::collections::HashMap<String, Vec<CodecRecord>>,- #[serde(default)]- overflow: Option<vault_codec::CodecOverflow>,}let input: Input = serde_wasm_bindgen::from_value(input).map_err(|e| JsValue::from_str(&format!("Failed to parse extract-bucket input: {}", e)))?;- codec_to_js(&vault_codec::extract_bucket(input.category, input.tables, input.overflow.as_ref()))+ codec_to_js(&vault_codec::extract_bucket(input.category, input.tables))}/// The bucket layout: `[{ category, tables: [<name>] }]`. Source of truth for platform bucket-only sync.