Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Improve backwards and forwards compatbility in codec (#2159)
Leendert de Borst2523d1a888f4
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rsindex c6d0cfb08..b85940a39 100644--- a/core/rust/src/vault_codec/canonicalize.rs+++ b/core/rust/src/vault_codec/canonicalize.rs@@ -113,6 +113,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canoniclet data_buckets = build_data_buckets(bucketed_rows, &overflow, &manifest_ids);// Start with the manifest the caller wrote from, which the input lists first.+ let writing_extra = overflow.manifest_extra(&writing_manifest_id);let mut manifests: Vec<CanonicalizedManifest> = Vec::with_capacity(1 + partitions.len());manifests.push(CanonicalizedManifest {manifest: Manifest {@@ -122,7 +123,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonicmanifest_id: writing_manifest_id,name: writing_spec.name.clone(),tables: manifest_tables,- extra: HashMap::new(),+ extra: writing_extra,},blobs,});@@ -139,6 +140,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic(name, out_rows)}).collect();+ let partition_extra = overflow.manifest_extra(&partition.manifest_id);manifests.push(CanonicalizedManifest {manifest: Manifest {schema_version: SCHEMA_VERSION,@@ -147,7 +149,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonicmanifest_id: partition.manifest_id,name: partition.name,tables: partition_tables,- extra: HashMap::new(),+ extra: partition_extra,},blobs: partition_blobs,});@@ -161,12 +163,18 @@ fn build_data_buckets(bucketed_rows: HashMap<String, Vec<CodecRecord>>, overflowlet mut data_buckets: Vec<DataBucket> = Vec::new();for category in categories_present(overflow) {let grouped = group_category_rows(category_tables(&category, &bucketed_rows, overflow), manifest_ids);- data_buckets.extend(grouped.into_iter().map(|(manifest_id, tables)| DataBucket::new(manifest_id, category.clone(), tables)));+ data_buckets.extend(grouped.into_iter().map(|(manifest_id, tables)| bucket_with_extra(manifest_id, &category, tables, overflow)));}data_buckets.sort_by(|a, b| (&a.manifest_id, &a.category).cmp(&(&b.manifest_id, &b.category)));data_buckets}+/// A data bucket carrying the unknown top-level keys the overflow last saw on it.+fn bucket_with_extra(manifest_id: String, category: &str, tables: HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow) -> DataBucket {+ let extra = overflow.bucket_extra(&manifest_id, category);+ DataBucket { extra, ..DataBucket::new(manifest_id, category, tables) }+}+/// Every bucket category to emit: the declared ones plus any a newer writer put in the overflow (a category/// this client's schema does not know yet still has to be carried forward). Sorted, so output stays stable.fn categories_present(overflow: &CodecOverflow) -> Vec<String> {@@ -363,7 +371,7 @@ pub fn extract_buckets(category: String, manifest_ids: Vec<String>, mut tables:let mut buckets: Vec<DataBucket> = group_category_rows(tables, &manifest_ids).into_iter()- .map(|(manifest_id, tables)| DataBucket::new(manifest_id, category.clone(), tables))+ .map(|(manifest_id, tables)| bucket_with_extra(manifest_id, &category, tables, &overflow)).collect();buckets.sort_by(|a, b| a.manifest_id.cmp(&b.manifest_id));Ok(buckets)diff --git a/core/rust/src/vault_codec/manifest.rs b/core/rust/src/vault_codec/manifest.rsindex f950c2738..73220ae6a 100644--- a/core/rust/src/vault_codec/manifest.rs+++ b/core/rust/src/vault_codec/manifest.rs@@ -30,7 +30,7 @@ pub struct Manifest {pub name: Option<String>,/// Tables mapped to arrays of row objects. Blob columns replaced with `{ "__blobRef", "__blobKind" }`.pub tables: HashMap<String, Vec<CodecRecord>>,- /// Forward-compat: unknown top-level keys preserved on round-trip.+ /// Unknown top-level keys from a newer writer, carried through the local vault in [`CodecOverflow`].#[serde(flatten)]pub extra: HashMap<String, serde_json::Value>,}@@ -147,12 +147,28 @@ pub struct CodecOverflow {/// Unknown columns split off rows of known tables: table > row primary-key value > {column: value}.#[serde(default)]pub columns: HashMap<String, HashMap<String, CodecRecord>>,+ /// Unknown top-level manifest keys: lowercased manifest id > {key: value}.+ #[serde(default)]+ pub manifest_extras: HashMap<String, HashMap<String, serde_json::Value>>,+ /// Unknown top-level data bucket keys: lowercased manifest id > category > {key: value}.+ #[serde(default)]+ pub bucket_extras: HashMap<String, HashMap<String, HashMap<String, serde_json::Value>>>,}impl CodecOverflow {/// True when nothing was split off (the common case: reader and writer share a schema).pub fn is_empty(&self) -> bool {- self.tables.is_empty() && self.bucket_tables.is_empty() && self.columns.is_empty()+ self.tables.is_empty() && self.bucket_tables.is_empty() && self.columns.is_empty() && self.manifest_extras.is_empty() && self.bucket_extras.is_empty()+ }++ /// The unknown top-level keys last seen on `manifest_id`, to write back into it.+ pub fn manifest_extra(&self, manifest_id: &str) -> HashMap<String, serde_json::Value> {+ self.manifest_extras.get(&crate::vault_model::id_key(manifest_id)).cloned().unwrap_or_default()+ }++ /// The unknown top-level keys last seen on the `category` bucket of `manifest_id`, to write back into it.+ pub fn bucket_extra(&self, manifest_id: &str, category: &str) -> HashMap<String, serde_json::Value> {+ self.bucket_extras.get(&crate::vault_model::id_key(manifest_id)).and_then(|by_category| by_category.get(category)).cloned().unwrap_or_default()}/// Render this overflow as the single `OVERFLOW_TABLE` row the platform inserts into the vault DB.diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rsindex a19784568..432c2ed23 100644--- a/core/rust/src/vault_codec/materialize.rs+++ b/core/rust/src/vault_codec/materialize.rs@@ -10,7 +10,9 @@//! crash the platform insert). The overflow is emitted as a regular table row (`OVERFLOW_TABLE`),//! so it lives inside the vault DB itself and `canonicalize_from_sqlite` re-merges it from the//! ordinary table read, this client's next push never drops the data, and no platform has to wire-//! (or remember) a separate persistence channel.+//! (or remember) a separate persistence channel. Unknown top-level keys of a manifest or bucket ride in+//! the same overflow. A manifest or bucket written at a newer major format version is refused and reqiures+/// updating the app to read it.use std::collections::{HashMap, HashSet};@@ -18,10 +20,10 @@ use serde_json::json;use super::manifest::{CodecOverflow, CodecRecord, CodecTableData, Manifest, MaterializeInput, MaterializedTables};use super::row::blob_ref_of;-use super::types::{blob_spec_for, is_local_only_table, row_identity};+use super::types::{blob_spec_for, ensure_readable_schema_version, is_local_only_table, row_identity};use crate::error::{VaultError, VaultResult};use crate::vault_model::names::ID_COL;-use crate::vault_model::{MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_TABLE};+use crate::vault_model::{id_key, MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_TABLE};/// Materialize the vault's manifests into the table set the platform inserts. Every manifest arrives/// in one list, each carrying its own data buckets; they are combined into a single table set with@@ -37,6 +39,22 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materializereturn Err(VaultError::General("materialize input carries no manifests".to_string()));}+ for manifest in &manifests {+ ensure_readable_schema_version(manifest.schema_version, &format!("manifest {}", manifest.manifest_id))?;+ }+ for bucket in &data_buckets {+ ensure_readable_schema_version(bucket.schema_version, &format!("\"{}\" bucket of manifest {}", bucket.category, bucket.manifest_id))?;+ }++ let mut overflow = CodecOverflow::default();+ // Top-level keys a newer writer added to a manifest or bucket; the next canonicalize writes them back.+ for manifest in manifests.iter().filter(|manifest| !manifest.extra.is_empty()) {+ overflow.manifest_extras.insert(id_key(&manifest.manifest_id), manifest.extra.clone());+ }+ for bucket in data_buckets.iter().filter(|bucket| !bucket.extra.is_empty()) {+ overflow.bucket_extras.entry(id_key(&bucket.manifest_id)).or_default().insert(bucket.category.clone(), bucket.extra.clone());+ }+let manifest_records = manifest_bookkeeping_records(&manifests);// The first manifest is the caller's own (see `MaterializeInput::manifests`).@@ -53,7 +71,6 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize// Normalize all id columns.super::normalize::normalize_id_spelling(&mut combined);- let mut overflow = CodecOverflow::default();let mut tables: Vec<CodecTableData> = Vec::with_capacity(combined.len() + data_buckets.len());for (name, mut records) in combined {diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rsindex 4f785b98f..98531ef74 100644--- a/core/rust/src/vault_codec/mod.rs+++ b/core/rust/src/vault_codec/mod.rs@@ -26,7 +26,7 @@ use serde_json::json;use crate::encoding::{base64_decode, hex_encode_lower};use crate::error::{VaultError, VaultResult};use crate::vault_model::names::LOGO_KIND_FAVICON;-use types::SCHEMA_VERSION;+pub use types::SCHEMA_VERSION;pub use canonicalize::{canonicalize_from_sqlite, extract_buckets};pub use manifest::{@@ -36,7 +36,8 @@ pub use manifest::{pub use materialize::materialize_as_sqlite;pub use scoped_assets::logo_id_for;pub use sharing::extract_encryption_key_for_public_key;-pub use types::{bucket_categories, identity_part, is_bucketed_table, is_skip_table, manifest_scoped_tables, tables_for_category};+pub(crate) use types::ensure_readable_schema_version;+pub use types::{bucket_categories, identity_part, is_bucketed_table, is_readable_schema_version, is_skip_table, manifest_scoped_tables, tables_for_category};pub use validate::{validate_data_bucket, validate_manifest, ValidationResult};/// The bucket layout: every category and the tables it owns, in declaration order.@@ -83,13 +84,34 @@ pub fn pack_payload(payload_json: &str) -> VaultResult<Vec<u8>> {compress::gzip(envelope_json.as_bytes())}+/// A decompressed payload envelope: its content, or the format version of one this build cannot read.+pub enum UnpackedPayload {+ Readable(String),+ NewerFormat(u32),+}+/// Unpack a payload: decompress (gzip or plain JSON) > parse envelope > verify the embedded content hash.+/// Refuses an envelope written at a format version this build cannot read.pub fn unpack_payload(plain_bytes: &[u8]) -> VaultResult<String> {+ match unpack_versioned_payload(plain_bytes)? {+ UnpackedPayload::Readable(payload_json) => Ok(payload_json),+ UnpackedPayload::NewerFormat(version) => Err(VaultError::General(format!("payload has format version {}, this build supports reading up to {}", version, SCHEMA_VERSION))),+ }+}++/// [`unpack_payload`], reporting a newer format version as a value instead of an error. The version is read+/// before the content hash, since a newer format may hash differently and must not read as corruption.+pub fn unpack_versioned_payload(plain_bytes: &[u8]) -> VaultResult<UnpackedPayload> {let envelope_json = compress::decompress_to_string(plain_bytes)?;let envelope: serde_json::Value = serde_json::from_str(&envelope_json)?;- if !envelope.get("schemaVersion").map(|v| v.is_number()).unwrap_or(false) {- return Err(VaultError::General("envelope missing schemaVersion".to_string()));+ let version = envelope.get("schemaVersion").and_then(|v| v.as_u64()).ok_or_else(|| VaultError::General("envelope missing schemaVersion".to_string()))?;+ let version = u32::try_from(version).unwrap_or(u32::MAX);+ if !types::is_readable_schema_version(version) {+ if version > SCHEMA_VERSION {+ return Ok(UnpackedPayload::NewerFormat(version));+ }+ return Err(VaultError::General(format!("envelope has invalid format version {}", version)));}let content_hash = envelope@@ -110,7 +132,7 @@ pub fn unpack_payload(plain_bytes: &[u8]) -> VaultResult<String> {)));}- Ok(serde_json::to_string(payload)?)+ Ok(UnpackedPayload::Readable(serde_json::to_string(payload)?))}/// SHA-256 (lowercase hex) of a base64 ciphertext string: storage-layer integrity.diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rsindex 444d745f8..b89a2b9eb 100644--- a/core/rust/src/vault_codec/types.rs+++ b/core/rust/src/vault_codec/types.rs@@ -7,9 +7,22 @@use crate::vault_model::names::ID_COL;use crate::vault_model::{ids_equal, BlobColumn, BLOB_COLUMNS, BUCKET_TABLES, MANIFEST_ID_COL, OVERFLOW_TABLE, PERSONAL_TABLES, SKIP_TABLES, SYNCABLE_TABLES, UNSTAMPED_SCOPE_SENTINEL};-/// Manifest / metadata schema version.+/// Manifest / data bucket format version; bump only for a change older clients cannot carry without the built-in overflow.pub const SCHEMA_VERSION: u32 = 1;+/// True when this build can read a manifest or data bucket written at `schema_version`.+pub fn is_readable_schema_version(schema_version: u32) -> bool {+ (1..=SCHEMA_VERSION).contains(&schema_version)+}++/// Refuse a manifest or data bucket written at a format version this build cannot read.+pub(crate) fn ensure_readable_schema_version(schema_version: u32, label: &str) -> crate::error::VaultResult<()> {+ if is_readable_schema_version(schema_version) {+ return Ok(());+ }+ Err(crate::error::VaultError::General(format!("{} has format version {}, this build reads up to {}", label, schema_version, SCHEMA_VERSION)))+}+/// One identity component of a row, as a string: a GUID lowercased, any other string as-is,/// anything else canonical JSON.pub fn identity_part(value: &serde_json::Value) -> String {diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rsindex f08bf461f..793bc6db6 100644--- a/core/rust/src/vault_merge/mod.rs+++ b/core/rust/src/vault_merge/mod.rs@@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize};use crate::error::VaultResult;use crate::timestamp::updated_at;use crate::vault_model::{id_key, TableConfig, SYNCABLE_TABLES};-use crate::vault_codec::{bucket_categories, identity_part, is_bucketed_table, tables_for_category, CodecRecord, DataBucket, Manifest};+use crate::vault_codec::{bucket_categories, ensure_readable_schema_version, identity_part, is_bucketed_table, tables_for_category, CodecRecord, DataBucket, Manifest};mod item_deletes;#[cfg(test)]@@ -82,6 +82,13 @@ pub struct CanonicalMergeOutput {pub fn merge_canonical(input: CanonicalMergeInput) -> VaultResult<CanonicalMergeOutput> {let CanonicalMergeInput { server_manifests, server_buckets, contentless_server_manifest_ids, local_manifests, local_buckets, schema_columns } = input;+ for manifest in &server_manifests {+ ensure_readable_schema_version(manifest.schema_version, &format!("server manifest {}", manifest.manifest_id))?;+ }+ for bucket in &server_buckets {+ ensure_readable_schema_version(bucket.schema_version, &format!("\"{}\" server bucket of manifest {}", bucket.category, bucket.manifest_id))?;+ }+let mut local_by_id: HashMap<String, Manifest> = local_manifests.into_iter().map(|m| (id_key(&m.manifest_id), m)).collect();let mut local_buckets_by_id = group_buckets(local_buckets);let mut server_buckets_by_id = group_buckets(server_buckets);@@ -133,6 +140,7 @@ fn merge_manifest_pair(// A bucket of a category this build does not know cannot be flattened without losing its// category; the server's ride through as-is and the local ones are dropped with the rest of// the local carrier, exactly as the base-wins rule treats every unknown table.+ let server_bucket_extras: HashMap<String, HashMap<String, serde_json::Value>> = server_buckets.iter().map(|bucket| (bucket.category.clone(), bucket.extra.clone())).collect();let (server_bucket_tables, unknown_server_buckets) = split_known_buckets(server_buckets);let (local_bucket_tables, _) = split_known_buckets(local_buckets);@@ -183,7 +191,9 @@ fn merge_manifest_pair(}}if !bucket_tables.is_empty() {- buckets.push(DataBucket::new(manifest_id.clone(), category.to_string(), bucket_tables));+ // A merged bucket keeps the server's extra top-level keys.+ let extra = server_bucket_extras.get(category).cloned().unwrap_or_default();+ buckets.push(DataBucket { extra, ..DataBucket::new(manifest_id.clone(), category.to_string(), bucket_tables) });}}diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rsindex 35b763f7e..e6d0f8604 100644--- a/core/rust/src/vault_sync/pull.rs+++ b/core/rust/src/vault_sync/pull.rs@@ -10,7 +10,7 @@ use super::types::{self, BlobDto, BlobHashesRequest, Db, EmailRoutingDto, GetResuse super::blob_keys::{self, EncryptedBlob};use super::{db, http, keys, legacy};use crate::crypto;-use crate::vault_codec::{self, DataBucket, Manifest, MaterializeInput};+use crate::vault_codec::{self, DataBucket, Manifest, MaterializeInput, UnpackedPayload};const BLOBS_DOWNLOAD_ENDPOINT: &str = "Vault/blobs/download";@@ -80,7 +80,23 @@ fn verify_decrypt_unpack(base64_ciphertext: &str, vek: &str, expected_ciphertextlet unreadable = |e: crate::error::VaultError| SyncError::ServerVaultUnreadable(format!("{}: {}", label, e));let encrypted = crate::encoding::base64_decode(base64_ciphertext).map_err(unreadable)?;let plain = crypto::symmetric_decrypt_bytes(&encrypted, vek).map_err(unreadable)?;- vault_codec::unpack_payload(&plain).map_err(unreadable)+ match vault_codec::unpack_versioned_payload(&plain).map_err(unreadable)? {+ UnpackedPayload::Readable(payload_json) => Ok(payload_json),+ UnpackedPayload::NewerFormat(version) => Err(newer_format(label, version)),+ }+}++/// Refuse a manifest or data bucket written at a format version this build cannot read: the app is outdated.+fn ensure_readable(schema_version: u32, label: &str) -> SyncResult<()> {+ if vault_codec::is_readable_schema_version(schema_version) {+ return Ok(());+ }+ Err(newer_format(label, schema_version))+}++/// The error for data written by a newer app than this one.+fn newer_format(label: &str, version: u32) -> SyncError {+ SyncError::VaultVersionIncompatible(format!("{} has format version {}, this app reads up to {}; update the app", label, version, vault_codec::SCHEMA_VERSION))}/// The grant a shared manifest is remembered by, when the snapshot carries one.@@ -226,6 +242,7 @@ async fn open_data_buckets(ctx: &Ctx, snapshot: &GetResponse, resolved: &[Resolvlet label = format!("\"{}\" bucket of manifest {}", dto.category, dto.manifest_id);let bucket_json = verify_decrypt_unpack(blob, key, dto.ciphertext_hash.as_deref(), &label)?;let bucket: DataBucket = serde_json::from_str(&bucket_json)?;+ ensure_readable(bucket.schema_version, &label)?;if !ids_equal(&bucket.manifest_id, &dto.manifest_id) || bucket.category != dto.category {return Err(SyncError::Snapshot(format!("{} declares a different address (manifest {}, category \"{}\") inside its encrypted payload, refusing to assemble", label, bucket.manifest_id, bucket.category)));}@@ -322,6 +339,7 @@ fn open_manifest(dto: &ManifestDto, vek: &str, is_personal: bool) -> SyncResult<let label = if is_personal { "manifest".to_string() } else { format!("shared manifest {}", dto.manifest_id) };let manifest_json = verify_decrypt_unpack(dto.blob.as_deref().unwrap_or(""), vek, dto.ciphertext_hash.as_deref(), &label)?;let manifest: Manifest = serde_json::from_str(&manifest_json)?;+ ensure_readable(manifest.schema_version, &label)?;if !ids_equal(&manifest.manifest_id, &dto.manifest_id) {return Err(SyncError::Snapshot(format!("manifest {} declares a different id ({}) inside its encrypted payload, refusing to open it", dto.manifest_id, manifest.manifest_id)));}