Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Improve backwards and forwards compatbility in codec (#2159)

Leendert de Borst2523d1a888f4
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rs
index c6d0cfb08..b85940a39 100644
--- a/core/rust/src/vault_codec/canonicalize.rs
+++ b/core/rust/src/vault_codec/canonicalize.rs
@@ -113,6 +113,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
let data_buckets = build_data_buckets(bucketed_rows, &overflow, &manifest_ids);
// Start with the manifest the caller wrote from, which the input lists first.
+ let writing_extra = overflow.manifest_extra(&writing_manifest_id);
let mut manifests: Vec<CanonicalizedManifest> = Vec::with_capacity(1 + partitions.len());
manifests.push(CanonicalizedManifest {
manifest: Manifest {
@@ -122,7 +123,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
manifest_id: writing_manifest_id,
name: writing_spec.name.clone(),
tables: manifest_tables,
- extra: HashMap::new(),
+ extra: writing_extra,
},
blobs,
});
@@ -139,6 +140,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
(name, out_rows)
})
.collect();
+ let partition_extra = overflow.manifest_extra(&partition.manifest_id);
manifests.push(CanonicalizedManifest {
manifest: Manifest {
schema_version: SCHEMA_VERSION,
@@ -147,7 +149,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
manifest_id: partition.manifest_id,
name: partition.name,
tables: partition_tables,
- extra: HashMap::new(),
+ extra: partition_extra,
},
blobs: partition_blobs,
});
@@ -161,12 +163,18 @@ fn build_data_buckets(bucketed_rows: HashMap<String, Vec<CodecRecord>>, overflow
let mut data_buckets: Vec<DataBucket> = Vec::new();
for category in categories_present(overflow) {
let grouped = group_category_rows(category_tables(&category, &bucketed_rows, overflow), manifest_ids);
- data_buckets.extend(grouped.into_iter().map(|(manifest_id, tables)| DataBucket::new(manifest_id, category.clone(), tables)));
+ data_buckets.extend(grouped.into_iter().map(|(manifest_id, tables)| bucket_with_extra(manifest_id, &category, tables, overflow)));
}
data_buckets.sort_by(|a, b| (&a.manifest_id, &a.category).cmp(&(&b.manifest_id, &b.category)));
data_buckets
}
+/// A data bucket carrying the unknown top-level keys the overflow last saw on it.
+fn bucket_with_extra(manifest_id: String, category: &str, tables: HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow) -> DataBucket {
+ let extra = overflow.bucket_extra(&manifest_id, category);
+ DataBucket { extra, ..DataBucket::new(manifest_id, category, tables) }
+}
+
/// Every bucket category to emit: the declared ones plus any a newer writer put in the overflow (a category
/// this client's schema does not know yet still has to be carried forward). Sorted, so output stays stable.
fn categories_present(overflow: &CodecOverflow) -> Vec<String> {
@@ -363,7 +371,7 @@ pub fn extract_buckets(category: String, manifest_ids: Vec<String>, mut tables:
let mut buckets: Vec<DataBucket> = group_category_rows(tables, &manifest_ids)
.into_iter()
- .map(|(manifest_id, tables)| DataBucket::new(manifest_id, category.clone(), tables))
+ .map(|(manifest_id, tables)| bucket_with_extra(manifest_id, &category, tables, &overflow))
.collect();
buckets.sort_by(|a, b| a.manifest_id.cmp(&b.manifest_id));
Ok(buckets)
diff --git a/core/rust/src/vault_codec/manifest.rs b/core/rust/src/vault_codec/manifest.rs
index f950c2738..73220ae6a 100644
--- a/core/rust/src/vault_codec/manifest.rs
+++ b/core/rust/src/vault_codec/manifest.rs
@@ -30,7 +30,7 @@ pub struct Manifest {
pub name: Option<String>,
/// Tables mapped to arrays of row objects. Blob columns replaced with `{ "__blobRef", "__blobKind" }`.
pub tables: HashMap<String, Vec<CodecRecord>>,
- /// Forward-compat: unknown top-level keys preserved on round-trip.
+ /// Unknown top-level keys from a newer writer, carried through the local vault in [`CodecOverflow`].
#[serde(flatten)]
pub extra: HashMap<String, serde_json::Value>,
}
@@ -147,12 +147,28 @@ pub struct CodecOverflow {
/// Unknown columns split off rows of known tables: table > row primary-key value > {column: value}.
#[serde(default)]
pub columns: HashMap<String, HashMap<String, CodecRecord>>,
+ /// Unknown top-level manifest keys: lowercased manifest id > {key: value}.
+ #[serde(default)]
+ pub manifest_extras: HashMap<String, HashMap<String, serde_json::Value>>,
+ /// Unknown top-level data bucket keys: lowercased manifest id > category > {key: value}.
+ #[serde(default)]
+ pub bucket_extras: HashMap<String, HashMap<String, HashMap<String, serde_json::Value>>>,
}
impl CodecOverflow {
/// True when nothing was split off (the common case: reader and writer share a schema).
pub fn is_empty(&self) -> bool {
- self.tables.is_empty() && self.bucket_tables.is_empty() && self.columns.is_empty()
+ self.tables.is_empty() && self.bucket_tables.is_empty() && self.columns.is_empty() && self.manifest_extras.is_empty() && self.bucket_extras.is_empty()
+ }
+
+ /// The unknown top-level keys last seen on `manifest_id`, to write back into it.
+ pub fn manifest_extra(&self, manifest_id: &str) -> HashMap<String, serde_json::Value> {
+ self.manifest_extras.get(&crate::vault_model::id_key(manifest_id)).cloned().unwrap_or_default()
+ }
+
+ /// The unknown top-level keys last seen on the `category` bucket of `manifest_id`, to write back into it.
+ pub fn bucket_extra(&self, manifest_id: &str, category: &str) -> HashMap<String, serde_json::Value> {
+ self.bucket_extras.get(&crate::vault_model::id_key(manifest_id)).and_then(|by_category| by_category.get(category)).cloned().unwrap_or_default()
}
/// Render this overflow as the single `OVERFLOW_TABLE` row the platform inserts into the vault DB.
diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rs
index a19784568..432c2ed23 100644
--- a/core/rust/src/vault_codec/materialize.rs
+++ b/core/rust/src/vault_codec/materialize.rs
@@ -10,7 +10,9 @@
//! crash the platform insert). The overflow is emitted as a regular table row (`OVERFLOW_TABLE`),
//! so it lives inside the vault DB itself and `canonicalize_from_sqlite` re-merges it from the
//! ordinary table read, this client's next push never drops the data, and no platform has to wire
-//! (or remember) a separate persistence channel.
+//! (or remember) a separate persistence channel. Unknown top-level keys of a manifest or bucket ride in
+//! the same overflow. A manifest or bucket written at a newer major format version is refused and reqiures
+/// updating the app to read it.
use std::collections::{HashMap, HashSet};
@@ -18,10 +20,10 @@ use serde_json::json;
use super::manifest::{CodecOverflow, CodecRecord, CodecTableData, Manifest, MaterializeInput, MaterializedTables};
use super::row::blob_ref_of;
-use super::types::{blob_spec_for, is_local_only_table, row_identity};
+use super::types::{blob_spec_for, ensure_readable_schema_version, is_local_only_table, row_identity};
use crate::error::{VaultError, VaultResult};
use crate::vault_model::names::ID_COL;
-use crate::vault_model::{MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_TABLE};
+use crate::vault_model::{id_key, MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_TABLE};
/// Materialize the vault's manifests into the table set the platform inserts. Every manifest arrives
/// in one list, each carrying its own data buckets; they are combined into a single table set with
@@ -37,6 +39,22 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
return Err(VaultError::General("materialize input carries no manifests".to_string()));
}
+ for manifest in &manifests {
+ ensure_readable_schema_version(manifest.schema_version, &format!("manifest {}", manifest.manifest_id))?;
+ }
+ for bucket in &data_buckets {
+ ensure_readable_schema_version(bucket.schema_version, &format!("\"{}\" bucket of manifest {}", bucket.category, bucket.manifest_id))?;
+ }
+
+ let mut overflow = CodecOverflow::default();
+ // Top-level keys a newer writer added to a manifest or bucket; the next canonicalize writes them back.
+ for manifest in manifests.iter().filter(|manifest| !manifest.extra.is_empty()) {
+ overflow.manifest_extras.insert(id_key(&manifest.manifest_id), manifest.extra.clone());
+ }
+ for bucket in data_buckets.iter().filter(|bucket| !bucket.extra.is_empty()) {
+ overflow.bucket_extras.entry(id_key(&bucket.manifest_id)).or_default().insert(bucket.category.clone(), bucket.extra.clone());
+ }
+
let manifest_records = manifest_bookkeeping_records(&manifests);
// The first manifest is the caller's own (see `MaterializeInput::manifests`).
@@ -53,7 +71,6 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
// Normalize all id columns.
super::normalize::normalize_id_spelling(&mut combined);
- let mut overflow = CodecOverflow::default();
let mut tables: Vec<CodecTableData> = Vec::with_capacity(combined.len() + data_buckets.len());
for (name, mut records) in combined {
diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rs
index 4f785b98f..98531ef74 100644
--- a/core/rust/src/vault_codec/mod.rs
+++ b/core/rust/src/vault_codec/mod.rs
@@ -26,7 +26,7 @@ use serde_json::json;
use crate::encoding::{base64_decode, hex_encode_lower};
use crate::error::{VaultError, VaultResult};
use crate::vault_model::names::LOGO_KIND_FAVICON;
-use types::SCHEMA_VERSION;
+pub use types::SCHEMA_VERSION;
pub use canonicalize::{canonicalize_from_sqlite, extract_buckets};
pub use manifest::{
@@ -36,7 +36,8 @@ pub use manifest::{
pub use materialize::materialize_as_sqlite;
pub use scoped_assets::logo_id_for;
pub use sharing::extract_encryption_key_for_public_key;
-pub use types::{bucket_categories, identity_part, is_bucketed_table, is_skip_table, manifest_scoped_tables, tables_for_category};
+pub(crate) use types::ensure_readable_schema_version;
+pub use types::{bucket_categories, identity_part, is_bucketed_table, is_readable_schema_version, is_skip_table, manifest_scoped_tables, tables_for_category};
pub use validate::{validate_data_bucket, validate_manifest, ValidationResult};
/// The bucket layout: every category and the tables it owns, in declaration order.
@@ -83,13 +84,34 @@ pub fn pack_payload(payload_json: &str) -> VaultResult<Vec<u8>> {
compress::gzip(envelope_json.as_bytes())
}
+/// A decompressed payload envelope: its content, or the format version of one this build cannot read.
+pub enum UnpackedPayload {
+ Readable(String),
+ NewerFormat(u32),
+}
+
/// Unpack a payload: decompress (gzip or plain JSON) > parse envelope > verify the embedded content hash.
+/// Refuses an envelope written at a format version this build cannot read.
pub fn unpack_payload(plain_bytes: &[u8]) -> VaultResult<String> {
+ match unpack_versioned_payload(plain_bytes)? {
+ UnpackedPayload::Readable(payload_json) => Ok(payload_json),
+ UnpackedPayload::NewerFormat(version) => Err(VaultError::General(format!("payload has format version {}, this build supports reading up to {}", version, SCHEMA_VERSION))),
+ }
+}
+
+/// [`unpack_payload`], reporting a newer format version as a value instead of an error. The version is read
+/// before the content hash, since a newer format may hash differently and must not read as corruption.
+pub fn unpack_versioned_payload(plain_bytes: &[u8]) -> VaultResult<UnpackedPayload> {
let envelope_json = compress::decompress_to_string(plain_bytes)?;
let envelope: serde_json::Value = serde_json::from_str(&envelope_json)?;
- if !envelope.get("schemaVersion").map(|v| v.is_number()).unwrap_or(false) {
- return Err(VaultError::General("envelope missing schemaVersion".to_string()));
+ let version = envelope.get("schemaVersion").and_then(|v| v.as_u64()).ok_or_else(|| VaultError::General("envelope missing schemaVersion".to_string()))?;
+ let version = u32::try_from(version).unwrap_or(u32::MAX);
+ if !types::is_readable_schema_version(version) {
+ if version > SCHEMA_VERSION {
+ return Ok(UnpackedPayload::NewerFormat(version));
+ }
+ return Err(VaultError::General(format!("envelope has invalid format version {}", version)));
}
let content_hash = envelope
@@ -110,7 +132,7 @@ pub fn unpack_payload(plain_bytes: &[u8]) -> VaultResult<String> {
)));
}
- Ok(serde_json::to_string(payload)?)
+ Ok(UnpackedPayload::Readable(serde_json::to_string(payload)?))
}
/// SHA-256 (lowercase hex) of a base64 ciphertext string: storage-layer integrity.
diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rs
index 444d745f8..b89a2b9eb 100644
--- a/core/rust/src/vault_codec/types.rs
+++ b/core/rust/src/vault_codec/types.rs
@@ -7,9 +7,22 @@
use crate::vault_model::names::ID_COL;
use crate::vault_model::{ids_equal, BlobColumn, BLOB_COLUMNS, BUCKET_TABLES, MANIFEST_ID_COL, OVERFLOW_TABLE, PERSONAL_TABLES, SKIP_TABLES, SYNCABLE_TABLES, UNSTAMPED_SCOPE_SENTINEL};
-/// Manifest / metadata schema version.
+/// Manifest / data bucket format version; bump only for a change older clients cannot carry without the built-in overflow.
pub const SCHEMA_VERSION: u32 = 1;
+/// True when this build can read a manifest or data bucket written at `schema_version`.
+pub fn is_readable_schema_version(schema_version: u32) -> bool {
+ (1..=SCHEMA_VERSION).contains(&schema_version)
+}
+
+/// Refuse a manifest or data bucket written at a format version this build cannot read.
+pub(crate) fn ensure_readable_schema_version(schema_version: u32, label: &str) -> crate::error::VaultResult<()> {
+ if is_readable_schema_version(schema_version) {
+ return Ok(());
+ }
+ Err(crate::error::VaultError::General(format!("{} has format version {}, this build reads up to {}", label, schema_version, SCHEMA_VERSION)))
+}
+
/// One identity component of a row, as a string: a GUID lowercased, any other string as-is,
/// anything else canonical JSON.
pub fn identity_part(value: &serde_json::Value) -> String {
diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rs
index f08bf461f..793bc6db6 100644
--- a/core/rust/src/vault_merge/mod.rs
+++ b/core/rust/src/vault_merge/mod.rs
@@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize};
use crate::error::VaultResult;
use crate::timestamp::updated_at;
use crate::vault_model::{id_key, TableConfig, SYNCABLE_TABLES};
-use crate::vault_codec::{bucket_categories, identity_part, is_bucketed_table, tables_for_category, CodecRecord, DataBucket, Manifest};
+use crate::vault_codec::{bucket_categories, ensure_readable_schema_version, identity_part, is_bucketed_table, tables_for_category, CodecRecord, DataBucket, Manifest};
mod item_deletes;
#[cfg(test)]
@@ -82,6 +82,13 @@ pub struct CanonicalMergeOutput {
pub fn merge_canonical(input: CanonicalMergeInput) -> VaultResult<CanonicalMergeOutput> {
let CanonicalMergeInput { server_manifests, server_buckets, contentless_server_manifest_ids, local_manifests, local_buckets, schema_columns } = input;
+ for manifest in &server_manifests {
+ ensure_readable_schema_version(manifest.schema_version, &format!("server manifest {}", manifest.manifest_id))?;
+ }
+ for bucket in &server_buckets {
+ ensure_readable_schema_version(bucket.schema_version, &format!("\"{}\" server bucket of manifest {}", bucket.category, bucket.manifest_id))?;
+ }
+
let mut local_by_id: HashMap<String, Manifest> = local_manifests.into_iter().map(|m| (id_key(&m.manifest_id), m)).collect();
let mut local_buckets_by_id = group_buckets(local_buckets);
let mut server_buckets_by_id = group_buckets(server_buckets);
@@ -133,6 +140,7 @@ fn merge_manifest_pair(
// A bucket of a category this build does not know cannot be flattened without losing its
// category; the server's ride through as-is and the local ones are dropped with the rest of
// the local carrier, exactly as the base-wins rule treats every unknown table.
+ let server_bucket_extras: HashMap<String, HashMap<String, serde_json::Value>> = server_buckets.iter().map(|bucket| (bucket.category.clone(), bucket.extra.clone())).collect();
let (server_bucket_tables, unknown_server_buckets) = split_known_buckets(server_buckets);
let (local_bucket_tables, _) = split_known_buckets(local_buckets);
@@ -183,7 +191,9 @@ fn merge_manifest_pair(
}
}
if !bucket_tables.is_empty() {
- buckets.push(DataBucket::new(manifest_id.clone(), category.to_string(), bucket_tables));
+ // A merged bucket keeps the server's extra top-level keys.
+ let extra = server_bucket_extras.get(category).cloned().unwrap_or_default();
+ buckets.push(DataBucket { extra, ..DataBucket::new(manifest_id.clone(), category.to_string(), bucket_tables) });
}
}
diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rs
index 35b763f7e..e6d0f8604 100644
--- a/core/rust/src/vault_sync/pull.rs
+++ b/core/rust/src/vault_sync/pull.rs
@@ -10,7 +10,7 @@ use super::types::{self, BlobDto, BlobHashesRequest, Db, EmailRoutingDto, GetRes
use super::blob_keys::{self, EncryptedBlob};
use super::{db, http, keys, legacy};
use crate::crypto;
-use crate::vault_codec::{self, DataBucket, Manifest, MaterializeInput};
+use crate::vault_codec::{self, DataBucket, Manifest, MaterializeInput, UnpackedPayload};
const BLOBS_DOWNLOAD_ENDPOINT: &str = "Vault/blobs/download";
@@ -80,7 +80,23 @@ fn verify_decrypt_unpack(base64_ciphertext: &str, vek: &str, expected_ciphertext
let unreadable = |e: crate::error::VaultError| SyncError::ServerVaultUnreadable(format!("{}: {}", label, e));
let encrypted = crate::encoding::base64_decode(base64_ciphertext).map_err(unreadable)?;
let plain = crypto::symmetric_decrypt_bytes(&encrypted, vek).map_err(unreadable)?;
- vault_codec::unpack_payload(&plain).map_err(unreadable)
+ match vault_codec::unpack_versioned_payload(&plain).map_err(unreadable)? {
+ UnpackedPayload::Readable(payload_json) => Ok(payload_json),
+ UnpackedPayload::NewerFormat(version) => Err(newer_format(label, version)),
+ }
+}
+
+/// Refuse a manifest or data bucket written at a format version this build cannot read: the app is outdated.
+fn ensure_readable(schema_version: u32, label: &str) -> SyncResult<()> {
+ if vault_codec::is_readable_schema_version(schema_version) {
+ return Ok(());
+ }
+ Err(newer_format(label, schema_version))
+}
+
+/// The error for data written by a newer app than this one.
+fn newer_format(label: &str, version: u32) -> SyncError {
+ SyncError::VaultVersionIncompatible(format!("{} has format version {}, this app reads up to {}; update the app", label, version, vault_codec::SCHEMA_VERSION))
}
/// The grant a shared manifest is remembered by, when the snapshot carries one.
@@ -226,6 +242,7 @@ async fn open_data_buckets(ctx: &Ctx, snapshot: &GetResponse, resolved: &[Resolv
let label = format!("\"{}\" bucket of manifest {}", dto.category, dto.manifest_id);
let bucket_json = verify_decrypt_unpack(blob, key, dto.ciphertext_hash.as_deref(), &label)?;
let bucket: DataBucket = serde_json::from_str(&bucket_json)?;
+ ensure_readable(bucket.schema_version, &label)?;
if !ids_equal(&bucket.manifest_id, &dto.manifest_id) || bucket.category != dto.category {
return Err(SyncError::Snapshot(format!("{} declares a different address (manifest {}, category \"{}\") inside its encrypted payload, refusing to assemble", label, bucket.manifest_id, bucket.category)));
}
@@ -322,6 +339,7 @@ fn open_manifest(dto: &ManifestDto, vek: &str, is_personal: bool) -> SyncResult<
let label = if is_personal { "manifest".to_string() } else { format!("shared manifest {}", dto.manifest_id) };
let manifest_json = verify_decrypt_unpack(dto.blob.as_deref().unwrap_or(""), vek, dto.ciphertext_hash.as_deref(), &label)?;
let manifest: Manifest = serde_json::from_str(&manifest_json)?;
+ ensure_readable(manifest.schema_version, &label)?;
if !ids_equal(&manifest.manifest_id, &dto.manifest_id) {
return Err(SyncError::Snapshot(format!("manifest {} declares a different id ({}) inside its encrypted payload, refusing to open it", dto.manifest_id, manifest.manifest_id)));
}