Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Move legacy sync logic to separate legacy.rs file (#2159)
Leendert de Borst577d5025bf47
diff --git a/core/rust/src/vault_sharing/mod.rs b/core/rust/src/vault_sharing/mod.rsindex 4617b884f..a83df51a8 100644--- a/core/rust/src/vault_sharing/mod.rs+++ b/core/rust/src/vault_sharing/mod.rs@@ -6,14 +6,12 @@ mod tests;use std::collections::HashMap;-use crate::error::VaultResult;-pub use types::{ManifestAccessPartition, ManifestAccessRequest, ManifestWriteRecord, ManifestWriteSet,ManifestWriteSetRequest, SharedManifestRecord, SkippedManifest, WriteSkipReason,};-use types::id_key;+use crate::vault_model::id_key;/// Resolve which manifests the next push writes, personal manifest first.///@@ -87,13 +85,3 @@ fn resolve_name(display_names: &HashMap<String, String>, manifest_id: &str, reco.or_else(|| record_name.map(|name| name.to_string())).filter(|name| !name.is_empty())}--/// Resolve a push's write set. Input: `ManifestWriteSetRequest` JSON. Output: `ManifestWriteSet` JSON.-pub fn resolve_manifest_write_set_json(input_json: &str) -> VaultResult<String> {- Ok(serde_json::to_string(&resolve_manifest_write_set(serde_json::from_str(input_json)?))?)-}--/// Partition manifest access. Input: `ManifestAccessRequest` JSON. Output: `ManifestAccessPartition` JSON.-pub fn partition_manifest_access_json(input_json: &str) -> VaultResult<String> {- Ok(serde_json::to_string(&partition_manifest_access(serde_json::from_str(input_json)?))?)-}diff --git a/core/rust/src/vault_sharing/tests.rs b/core/rust/src/vault_sharing/tests.rsindex a401cec9f..2e710f356 100644--- a/core/rust/src/vault_sharing/tests.rs+++ b/core/rust/src/vault_sharing/tests.rs@@ -177,7 +177,7 @@ fn the_json_boundary_round_trips() {"displayNames": { "man-1": "Family" }}"#;- let output: serde_json::Value = serde_json::from_str(&resolve_manifest_write_set_json(input).unwrap()).unwrap();+ let output: serde_json::Value = serde_json::from_str(&crate::error::json_call(input, |request| Ok(resolve_manifest_write_set(request))).unwrap()).unwrap();assert_eq!(output["records"][1]["manifestId"], "MAN-1");assert_eq!(output["records"][1]["name"], "Family");assert_eq!(output["records"][1]["isPersonal"], false);diff --git a/core/rust/src/vault_sharing/types.rs b/core/rust/src/vault_sharing/types.rsindex 9de8a6e3e..45a8d845c 100644--- a/core/rust/src/vault_sharing/types.rs+++ b/core/rust/src/vault_sharing/types.rs@@ -3,11 +3,6 @@ use std::collections::HashMap;use serde::{Deserialize, Serialize};-/// Lower-cased comparison key for a manifest id.-pub(crate) fn id_key(id: &str) -> String {- id.trim().to_lowercase()-}-/// A shared manifest's key record.#[derive(Debug, Clone, Deserialize, Serialize)]#[serde(rename_all = "camelCase")]diff --git a/core/rust/src/vault_sync/legacy.rs b/core/rust/src/vault_sync/legacy.rsindex 0a7132b70..2428bc925 100644--- a/core/rust/src/vault_sync/legacy.rs+++ b/core/rust/src/vault_sync/legacy.rs@@ -1,15 +1,89 @@-//! LEGACY: what only a vault still on the sqlite-blob storage format needs, namely its local migration onto-//! the current schema. TODO: remove this module once all users have migrated to the manifest-v1 storage model.+//! LEGACY: everything only a vault still on the sqlite-blob storage format needs. TODO: remove this module and all callers once+//! all users have migrated to the manifest-v1 storage model.use std::collections::HashMap;-use super::errors::SyncResult;-use super::pull::materialize_to_sqlite;+use super::errors::{SyncError, SyncResult};+use super::pull::{email_routing_of, materialize_to_sqlite, PulledVault};use super::push::{canonicalize_vault, resolve_personal_manifest_id};-use super::state::Ctx;-use crate::crypto;+use super::state::{self, Ctx};+use super::types::GetResponse;use crate::vault_codec::Manifest;+/// The `storageFormat` a manifest-v1 snapshot declares; 0 (or absent) is a sqlite blob.+const STORAGE_FORMAT_MANIFEST: i32 = 1;++/// Whether a snapshot is still on the legacy sqlite-blob format.+pub(crate) fn is_legacy_sqlite_blob_snapshot(snapshot: &GetResponse) -> bool {+ snapshot.storage_format != Some(STORAGE_FORMAT_MANIFEST)+}++/// Take a legacy snapshot apart for local storage: the blob passes through untouched, the manifest-v1 fingerprints+/// are reset, and the personal manifest id is recorded for the migration push.+pub(crate) async fn open_legacy_snapshot(ctx: &Ctx, snapshot: &GetResponse) -> SyncResult<PulledVault> {+ state::remove(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?;+ let revision = snapshot.legacy_revision.unwrap_or(0);+ let mut manifest_revisions = HashMap::new();+ if let Some(personal) = &snapshot.personal_manifest_id {+ state::set(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID, personal).await?;+ manifest_revisions.insert(personal.clone(), revision);+ }+ ctx.log("[V2Pull] Legacy sqlite-blob pass-through (user not yet migrated), returning the blob as-is.").await;+ Ok(PulledVault { encrypted_vault: snapshot.legacy_vault_blob.clone().unwrap_or_default(), revision, email_routing: email_routing_of(snapshot), manifest_revisions, bucket_revisions: HashMap::new() })+}++/// The frozen sqlite-blob upgrade chain: (revision, data version). It ends at 2.0.0, the first schema compatible+/// with manifest-v1; later schema changes ship through the full schema only, which every materialization uses directly.+const LEGACY_VAULT_VERSIONS: &[(u32, &str)] = &[+ (1, "1.0.0"),+ (2, "1.0.1"),+ (3, "1.0.2"),+ (4, "1.1.0"),+ (5, "1.2.0"),+ (6, "1.3.0"),+ (7, "1.3.1"),+ (8, "1.4.0"),+ (9, "1.4.1"),+ (10, "1.5.0"),+ (11, "1.6.0"),+ (12, "1.7.0"),+ (13, "2.0.0"),+];++/// The revision of the last entry in the legacy chain.+fn latest_legacy_revision() -> u32 {+ LEGACY_VAULT_VERSIONS.last().map(|(revision, _)| *revision).unwrap_or(0)+}++/// The data version embedded in an EF migration id (`20250101000000_2.0.0-Name`).+fn extract_version_from_migration_id(migration_id: &str) -> Option<String> {+ let start = migration_id.find('_')? + 1;+ let rest = &migration_id[start..];+ let end = rest.find('-')?;+ let candidate = &rest[..end];+ if is_well_formed_version(candidate) { Some(candidate.to_string()) } else { None }+}++fn is_well_formed_version(version: &str) -> bool {+ version.split('.').count() == 3 && version.split('.').all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit()))+}++/// The legacy-chain revision a database version maps to. A known version maps to its entry; an unknown but+/// well-formed version is treated as the latest (backwards compatible); a malformed version is incompatible.+fn legacy_revision_for(database_version: &str) -> Result<u32, String> {+ if let Some((revision, _)) = LEGACY_VAULT_VERSIONS.iter().find(|(_, version)| *version == database_version) {+ return Ok(*revision);+ }+ if is_well_formed_version(database_version) { Ok(latest_legacy_revision()) } else { Err(format!("Vault version {} is not compatible with this client", database_version)) }+}++/// Whether a vault's latest migration stamp still has to walk the frozen sqlite-blob chain (a pre-2.0.0 vault).+pub(crate) fn stamp_predates_manifest_schema(migration_id: &str) -> SyncResult<bool> {+ let database_version = extract_version_from_migration_id(migration_id).ok_or_else(|| SyncError::Other("Could not extract version from migration ID".to_string()))?;+ let revision = legacy_revision_for(&database_version).map_err(SyncError::VaultVersionIncompatible)?;+ Ok(revision < latest_legacy_revision())+}+/// Migrate the local vault onto the current schema, entirely locally.pub(crate) async fn migrate_vault_to_current_schema(ctx: &mut Ctx) -> SyncResult<Vec<u8>> {ctx.log("[ManifestMigration] Migrating local vault onto the current schema (local round-trip, no server involved)...").await;@@ -19,7 +93,7 @@ pub(crate) async fn migrate_vault_to_current_schema(ctx: &mut Ctx) -> SyncResultlet mut blob_map = HashMap::new();for entry in &set.canonicalized.manifests {for (hash, blob) in &entry.blobs {- blob_map.insert(hash.clone(), crypto::aes_gcm::decode_base64(&blob.bytes_base64)?);+ blob_map.insert(hash.clone(), crate::encoding::base64_decode(&blob.bytes_base64)?);}}let manifests: Vec<Manifest> = set.canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect();@@ -27,3 +101,26 @@ pub(crate) async fn migrate_vault_to_current_schema(ctx: &mut Ctx) -> SyncResultctx.log(format!("[ManifestMigration] Migration complete: {} blobs re-embedded, {} bytes.", blob_map.len(), bytes.len())).await;Ok(bytes)}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn migration_ids_yield_their_version() {+ assert_eq!(extract_version_from_migration_id("20250101000000_2.0.0-Squashed").as_deref(), Some("2.0.0"));+ assert_eq!(extract_version_from_migration_id("20250101000000_Initial"), None);+ assert_eq!(legacy_revision_for("1.7.0"), Ok(12));+ assert_eq!(legacy_revision_for("2.0.0"), Ok(13));+ assert_eq!(legacy_revision_for("2.1.0"), Ok(13));+ assert!(legacy_revision_for("nope").is_err());+ }++ #[test]+ fn stamps_before_2_0_0_are_on_the_chain() {+ assert_eq!(stamp_predates_manifest_schema("20250101000000_1.7.0-Old").unwrap(), true);+ assert_eq!(stamp_predates_manifest_schema("20250101000000_2.0.0-Squashed").unwrap(), false);+ assert_eq!(stamp_predates_manifest_schema("20260101000000_2.1.0-Later").unwrap(), false);+ assert!(matches!(stamp_predates_manifest_schema("20250101000000_Initial"), Err(SyncError::Other(_))));+ }+}diff --git a/core/rust/src/vault_sync/merge.rs b/core/rust/src/vault_sync/merge.rsindex 163dc7a95..9f09729db 100644--- a/core/rust/src/vault_sync/merge.rs+++ b/core/rust/src/vault_sync/merge.rs@@ -3,17 +3,15 @@use std::collections::{HashMap, HashSet};-use base64::engine::general_purpose::STANDARD as BASE64;-use base64::Engine;use serde::{Deserialize, Serialize};-use super::db::{id_key, ids_equal};+use crate::vault_model::{id_key, ids_equal};use super::errors::{SyncError, SyncResult};-use super::pull::{self, LegacyPassthrough, OpenedManifestSet, SnapshotFacts};+use super::pull::{self, OpenedManifestSet, PulledVault};use super::push::{canonicalize_vault, CanonicalizedSet, ManifestRecord};use super::state::{self, Ctx};use super::types::EmailRoutingDto;-use crate::crypto;+use super::legacy;use crate::vault_codec::{self, BlobEntry, CanonicalizedVault, DataBucket, Manifest};use crate::vault_merge::{merge_canonical, CanonicalManifestMerge, CanonicalMergeInput, MergeStats};@@ -42,38 +40,32 @@ impl MergeSummary {/// What a dirty pull (pull and merge) produced.pub(crate) enum PullAndMergeOutcome {- LegacyServer(LegacyPassthrough),+ /// LEGACY: the server's vault is still a sqlite blob at this revision; the caller pushes the local vault over it.+ LegacyServer { revision: i64 },Merged {- encrypted_vault: String,- revision: i64,- email_routing: EmailRoutingDto,+ pulled: PulledVault,stats: MergeSummary,fallback_manifest_ids: Vec<String>,dropped_local_manifest_ids: Vec<String>,- manifest_revisions: HashMap<String, i64>,- bucket_revisions: HashMap<String, i64>,push_canonical: Option<CanonicalizedSet>,},- ServerOnly {- encrypted_vault: String,- revision: i64,- email_routing: EmailRoutingDto,- manifest_revisions: HashMap<String, i64>,- bucket_revisions: HashMap<String, i64>,- },+ /// The merge failed, so the server's vault stands as pulled and the local changes are dropped.+ ServerOnly(PulledVault),}/// Pull the latest snapshot and merge the local vault onto it at canonical level, one manifest at a time.-pub(crate) async fn pull_and_merge(ctx: &mut Ctx, facts: &mut SnapshotFacts) -> SyncResult<PullAndMergeOutcome> {+pub(crate) async fn pull_and_merge(ctx: &mut Ctx) -> SyncResult<PullAndMergeOutcome> {let vek = ctx.encryption_key()?;ctx.log("[V2Merge] Fetching vault snapshot for canonical merge (GET /v2/Vault)...").await;- let snapshot = pull::fetch_snapshot(ctx, facts).await?;- let email_routing = pull::email_routing_of(&snapshot);+ let snapshot = pull::fetch_snapshot(ctx).await?;// LEGACY: a server still on the sqlite-blob format cannot merge with a manifest-v1 vault; the caller pushes over it.- if facts.legacy {- return Ok(PullAndMergeOutcome::LegacyServer(pull::open_legacy_snapshot(ctx, &snapshot).await?));+ if legacy::is_legacy_sqlite_blob_snapshot(&snapshot) {+ let legacy = legacy::open_legacy_snapshot(ctx, &snapshot).await?;+ pull::commit_revisions(ctx, &legacy.manifest_revisions, &legacy.bucket_revisions).await?;+ return Ok(PullAndMergeOutcome::LegacyServer { revision: legacy.revision });}+ let email_routing = pull::email_routing_of(&snapshot);let opened = pull::open_manifests_and_record_sync_state(ctx, &snapshot, &vek).await?;let local_side = canonicalize_vault(ctx, None).await?;@@ -91,13 +83,7 @@ pub(crate) async fn pull_and_merge(ctx: &mut Ctx, facts: &mut SnapshotFacts) ->// The merge-failure fallback, from the same snapshot: the server vault stands, local changes are dropped.ctx.warn(format!("[V2Merge] Canonical merge failed, falling back to the server vault: {}", merge_error)).await;let sqlite_bytes = pull::materialize_to_sqlite(ctx, &opened.manifests(), &opened.data_buckets, &opened.blob_map).await?;- Ok(PullAndMergeOutcome::ServerOnly {- encrypted_vault: state::encrypt_vault_blob(&sqlite_bytes, &vek)?,- revision: opened.personal_revision,- email_routing,- manifest_revisions: opened.manifest_revisions,- bucket_revisions: opened.bucket_revisions,- })+ Ok(PullAndMergeOutcome::ServerOnly(opened.pulled_vault(state::encrypt_vault_blob(&sqlite_bytes, &vek)?, email_routing)))}}}@@ -174,7 +160,7 @@ async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet,for (hash, blob) in &canonicalized.blobs {local_blobs.insert(hash.clone(), blob.clone());if !blob_map.contains_key(hash) {- blob_map.insert(hash.clone(), crypto::aes_gcm::decode_base64(&blob.bytes_base64)?);+ blob_map.insert(hash.clone(), crate::encoding::base64_decode(&blob.bytes_base64)?);}}}@@ -186,17 +172,7 @@ async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet,ctx.log(format!("[V2Merge] Canonical merge complete: {} conflict(s), {} offline row(s) kept, {} validation fallback(s), {} dropped local manifest(s).", stats.conflicts, stats.records_inserted, fallback_manifest_ids.len(), merge_output.dropped_local_manifest_ids.len())).await;let push_canonical = merge_output_for_push(&manifests, &data_buckets, merged_blobs, &local_side.manifest_records, !fallback_manifest_ids.is_empty());- Ok(PullAndMergeOutcome::Merged {- encrypted_vault,- revision: opened.personal_revision,- email_routing,- stats,- fallback_manifest_ids,- dropped_local_manifest_ids: merge_output.dropped_local_manifest_ids,- manifest_revisions: opened.manifest_revisions.clone(),- bucket_revisions: opened.bucket_revisions.clone(),- push_canonical,- })+ Ok(PullAndMergeOutcome::Merged { pulled: opened.pulled_vault(encrypted_vault, email_routing), stats, fallback_manifest_ids, dropped_local_manifest_ids: merge_output.dropped_local_manifest_ids, push_canonical })}/// The merged vault in the shape the push writes from.@@ -257,7 +233,7 @@ async fn resolve_merged_blob_refs(ctx: &Ctx, manifests: &[Manifest], blob_map: &if let Some(local) = local_blobs.get(reference) {blobs.insert(reference.to_string(), local.clone());} else if let Some(kind) = kind {- blobs.insert(reference.to_string(), BlobEntry { kind: kind.to_string(), bytes_base64: BASE64.encode(bytes) });+ blobs.insert(reference.to_string(), BlobEntry { kind: kind.to_string(), bytes_base64: crate::encoding::base64_encode(bytes) });} else {complete = false;}diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rsindex 791fd52cc..760de4c6c 100644--- a/core/rust/src/vault_sync/pull.rs+++ b/core/rust/src/vault_sync/pull.rs@@ -3,22 +3,16 @@use std::collections::HashMap;-use super::db::ids_equal;+use crate::vault_model::ids_equal;use super::errors::{SyncError, SyncResult};use super::state::{self, Ctx};use super::types::{self, BlobDto, BlobHashesRequest, Db, EmailRoutingDto, GetResponse, ManifestDto, SharedManifestRecord, StoredBlobRef};-use super::{db, http, keys};+use super::{db, http, keys, legacy};use crate::crypto;use crate::vault_codec::{self, DataBucket, Manifest, MaterializeInput};-const VAULT_ENDPOINT: &str = "Vault";const BLOBS_DOWNLOAD_ENDPOINT: &str = "Vault/blobs/download";-/// Max base64 characters in one blob transfer request or response body.-pub(crate) const BLOB_TRANSFER_BATCH_MAX_CHARS: usize = 4 * 1024 * 1024;-/// Upper bound on the number of blobs in one transfer batch.-pub(crate) const BLOB_TRANSFER_BATCH_MAX_COUNT: usize = 100;-/// One manifest of a pull, opened.pub(crate) struct ResolvedManifest {pub manifest_id: String,@@ -45,74 +39,31 @@ impl OpenedManifestSet {pub fn manifests(&self) -> Vec<Manifest> {self.resolved.iter().map(|m| m.manifest.clone()).collect()}-}--/// A not-yet-migrated server's vault, passed through as served.-/// TODO: remove all legacy related code once all users have migrated to the manifest-v1 storage model.-pub(crate) struct LegacyPassthrough {- pub encrypted_blob: String,- pub revision: i64,-}--/// What a pull produced.-pub(crate) enum PullOutcome {- Legacy(LegacyPassthrough, EmailRoutingDto),- Materialized { encrypted_vault: String, revision: i64, email_routing: EmailRoutingDto },-}-/// What the last snapshot this run fetched looked like.-pub(crate) struct SnapshotFacts {- pub served_manifest_ids: Vec<String>,- /// Whether the snapshot is still on the legacy sqlite-blob format.- pub legacy: bool,-}--impl SnapshotFacts {- pub fn new() -> Self {- Self { served_manifest_ids: Vec::new(), legacy: false }- }-- fn record(&mut self, snapshot: &GetResponse) {- self.served_manifest_ids = snapshot.manifests.iter().map(|m| m.manifest_id.clone()).collect();- self.legacy = is_legacy_sqlite_blob_snapshot(snapshot);+ /// The opened set as a vault ready to become the local one.+ pub fn pulled_vault(&self, encrypted_vault: String, email_routing: EmailRoutingDto) -> PulledVault {+ PulledVault { encrypted_vault, revision: self.personal_revision, email_routing, manifest_revisions: self.manifest_revisions.clone(), bucket_revisions: self.bucket_revisions.clone() }}}-/// `GET v2/Vault`.-pub(crate) async fn fetch_snapshot(ctx: &Ctx, facts: &mut SnapshotFacts) -> SyncResult<GetResponse> {- let snapshot = http::with_outdated_server_guard(http::get::<GetResponse>(&ctx.host, VAULT_ENDPOINT, true).await)?;- facts.record(&snapshot);- Ok(snapshot)+/// A pulled vault, encrypted for local storage, with the revisions that become the local truth once it is stored.+pub(crate) struct PulledVault {+ pub encrypted_vault: String,+ pub revision: i64,+ pub email_routing: EmailRoutingDto,+ pub manifest_revisions: HashMap<String, i64>,+ pub bucket_revisions: HashMap<String, i64>,}-pub(crate) fn is_legacy_sqlite_blob_snapshot(snapshot: &GetResponse) -> bool {- snapshot.storage_format != Some(types::STORAGE_FORMAT_MANIFEST)+/// `GET v2/Vault`.+pub(crate) async fn fetch_snapshot(ctx: &Ctx) -> SyncResult<GetResponse> {+ http::with_outdated_server_guard(http::get::<GetResponse>(&ctx.host, http::VAULT_ENDPOINT, true).await)}pub(crate) fn email_routing_of(snapshot: &GetResponse) -> EmailRoutingDto {snapshot.email_routing.clone().unwrap_or_default()}-/// Split items into request batches bounded by both transfer limits.-pub(crate) fn batch_by_transfer_cost<T>(items: Vec<T>, cost_of: impl Fn(&T) -> usize) -> Vec<Vec<T>> {- let mut batches = Vec::new();- let mut batch = Vec::new();- let mut chars = 0usize;- for item in items {- let cost = cost_of(&item);- if !batch.is_empty() && (chars + cost > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.len() >= BLOB_TRANSFER_BATCH_MAX_COUNT) {- batches.push(std::mem::take(&mut batch));- chars = 0;- }- batch.push(item);- chars += cost;- }- if !batch.is_empty() {- batches.push(batch);- }- batches-}-fn base64_chars(size_bytes: i64) -> usize {((size_bytes.max(0) as usize).div_ceil(3)) * 4}@@ -125,7 +76,7 @@ fn verify_decrypt_unpack(base64_ciphertext: &str, vek: &str, expected_ciphertext}}let unreadable = |e: crate::error::VaultError| SyncError::ServerVaultUnreadable(format!("{}: {}", label, e));- let encrypted = crypto::aes_gcm::decode_base64(base64_ciphertext).map_err(unreadable)?;+ let encrypted = crate::encoding::base64_decode(base64_ciphertext).map_err(unreadable)?;let plain = crypto::symmetric_decrypt_bytes(&encrypted, vek).map_err(unreadable)?;vault_codec::unpack_payload(&plain).map_err(unreadable)}@@ -143,43 +94,24 @@ fn select_personal_manifest(snapshot: &GetResponse) -> Option<&ManifestDto> {snapshot.manifests.iter().find(|m| m.manifest_id == personal_id)}-/// Take a legacy snapshot apart for local storage: the blob passes through untouched, the manifest-v1 state-/// is reset, and the personal manifest id is recorded for the migration push.-pub(crate) async fn open_legacy_snapshot(ctx: &Ctx, snapshot: &GetResponse) -> SyncResult<LegacyPassthrough> {- state::remove(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?;- let revision = snapshot.legacy_revision.unwrap_or(0);- if let Some(personal) = &snapshot.personal_manifest_id {- state::set(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID, personal).await?;- let mut revisions = HashMap::new();- revisions.insert(personal.clone(), revision);- state::set(&ctx.host, state::SERVER_MANIFEST_REVISIONS, &revisions).await?;- }- ctx.log("[V2Pull] Legacy sqlite-blob pass-through (user not yet migrated), returning the blob as-is.").await;- Ok(LegacyPassthrough { encrypted_blob: snapshot.legacy_vault_blob.clone().unwrap_or_default(), revision })-}-/// Pull the latest vault: fetch, open, materialize and re-encrypt for local storage.-pub(crate) async fn pull(ctx: &mut Ctx, facts: &mut SnapshotFacts) -> SyncResult<PullOutcome> {+pub(crate) async fn pull(ctx: &mut Ctx) -> SyncResult<PulledVault> {let vek = ctx.encryption_key()?;ctx.log("[V2Pull] Step 1/4: fetching vault snapshot (GET /v2/Vault)...").await;- let snapshot = fetch_snapshot(ctx, facts).await?;- let email_routing = email_routing_of(&snapshot);-- if facts.legacy {- return Ok(PullOutcome::Legacy(open_legacy_snapshot(ctx, &snapshot).await?, email_routing));+ let snapshot = fetch_snapshot(ctx).await?;+ if legacy::is_legacy_sqlite_blob_snapshot(&snapshot) {+ return legacy::open_legacy_snapshot(ctx, &snapshot).await;}ctx.log("[V2Pull] Step 2/4: manifest format: decrypting and reassembling local SQLite...").await;let opened = open_manifests_and_record_sync_state(ctx, &snapshot, &vek).await?;- commit_revisions(ctx, &opened.manifest_revisions, &opened.bucket_revisions).await?;let sqlite_bytes = materialize_to_sqlite(ctx, &opened.manifests(), &opened.data_buckets, &opened.blob_map).await?;ctx.log(format!("[V2Pull] Step 3/4: materialized SQLite ({} bytes); re-encrypting for local storage...", sqlite_bytes.len())).await;- let encrypted_vault = state::encrypt_vault_blob(&sqlite_bytes, &vek)?;- Ok(PullOutcome::Materialized { encrypted_vault, revision: opened.personal_revision, email_routing })+ Ok(opened.pulled_vault(state::encrypt_vault_blob(&sqlite_bytes, &vek)?, email_routing_of(&snapshot)))}/// Open every manifest a snapshot carries, personal first, and record the snapshot as this device's sync state.-/// The revision maps are returned for the caller to commit once the pulled state is local truth.+/// The revision maps travel on the result for the caller to commit once the pulled vault is stored.pub(crate) async fn open_manifests_and_record_sync_state(ctx: &mut Ctx, snapshot: &GetResponse, vek: &str) -> SyncResult<OpenedManifestSet> {let personal_dto = select_personal_manifest(snapshot).ok_or_else(|| SyncError::Snapshot("server returned no personal manifest, refusing to assemble".to_string()))?;if personal_dto.blob.as_deref().unwrap_or("").is_empty() {@@ -300,7 +232,7 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], fallet missing: Vec<StoredBlobRef> = refs.iter().filter(|r| !cache.contains_key(&r.hash)).cloned().collect();ctx.log(format!("[V2Pull] Blob refs: {} referenced, {} cached locally, {} to download.", refs.len(), refs.len() - missing.len(), missing.len())).await;- let batches = batch_by_transfer_cost(missing, |r| base64_chars(r.size_bytes));+ let batches = http::batch_by_transfer_cost(missing, |r| base64_chars(r.size_bytes));let batch_count = batches.len();for (index, chunk) in batches.into_iter().enumerate() {let blobs: Vec<BlobDto> = http::post(&ctx.host, BLOBS_DOWNLOAD_ENDPOINT, &BlobHashesRequest { hashes: chunk.iter().map(|r| r.hash.clone()).collect() }, true).await?;@@ -318,7 +250,7 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], falcontinue;};let key = owners.get(&reference.hash).map(|o| o.vek.as_str()).unwrap_or(fallback_vek);- match crypto::aes_gcm::decode_base64(ciphertext).and_then(|bytes| crypto::symmetric_decrypt_bytes(&bytes, key)) {+ match crate::encoding::base64_decode(ciphertext).and_then(|bytes| crypto::symmetric_decrypt_bytes(&bytes, key)) {Ok(bytes) => {blob_map.insert(reference.hash.clone(), bytes);pruned_cache.insert(reference.hash.clone(), ciphertext.clone());diff --git a/core/rust/src/vault_sync/push.rs b/core/rust/src/vault_sync/push.rsindex ef5b95640..1414b3f38 100644--- a/core/rust/src/vault_sync/push.rs+++ b/core/rust/src/vault_sync/push.rs@@ -3,18 +3,16 @@use std::collections::{HashMap, HashSet};-use super::db::{id_key, ids_equal};+use crate::vault_model::{id_key, ids_equal};use super::email_routing::build_email_routing;use super::errors::{SyncError, SyncResult};-use super::pull::{BLOB_TRANSFER_BATCH_MAX_CHARS, BLOB_TRANSFER_BATCH_MAX_COUNT};use super::state::{self, Ctx};-use super::types::{BlobDto, BlobHashesRequest, BlobRef, BlobUploadRequest, BucketWrite, Db, ManifestWrite, MissingBlobsResponse, VaultWriteRequest, VaultWriteResponse};+use super::types::{BlobDto, BlobHashesRequest, BlobRef, BlobUploadRequest, BucketRevision, BucketWrite, Db, ManifestRevision, ManifestWrite, MissingBlobsResponse, VaultWriteRequest, VaultWriteResponse};use super::{db, http, keys};use crate::crypto;use crate::vault_codec::{self, BlobEntry, CanonicalizeInput, CanonicalizedVault, DataBucket, Manifest, ManifestSpec};use crate::vault_sharing::{self, ManifestAccessRequest, ManifestWriteSetRequest, SharedManifestRecord as SharingRecord};-const VAULT_ENDPOINT: &str = "Vault";const BLOBS_ENDPOINT: &str = "Vault/blobs";const BLOBS_MISSING_ENDPOINT: &str = "Vault/blobs/missing";@@ -120,17 +118,17 @@ pub(crate) async fn resolve_personal_manifest_id(ctx: &Ctx) -> SyncResult<String/// result too, so a push right after does not canonicalize a second time.pub(crate) async fn detect_no_op_mutation(ctx: &Ctx) -> SyncResult<(bool, CanonicalizedSet)> {let set = canonicalize_vault(ctx, None).await?;- let fingerprints: HashMap<String, String> = state::get(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?.unwrap_or_default();+ let baselines = PushBaselines::load(ctx).await?;let writable: HashSet<String> = set.manifest_records.iter().map(|r| id_key(&r.manifest_id)).collect();for entry in set.canonicalized.manifests.iter().filter(|m| writable.contains(&id_key(&m.manifest.manifest_id))) {- let fingerprint = vault_codec::compute_content_fingerprint(&serde_json::to_string(&entry.manifest)?);- if fingerprints.get(&state::fingerprint_manifest_key(&entry.manifest.manifest_id)) != Some(&fingerprint) {+ let (_, fingerprint) = fingerprinted(&entry.manifest)?;+ if !baselines.unchanged(&state::fingerprint_manifest_key(&entry.manifest.manifest_id), &fingerprint) {return Ok((false, set));}}for bucket in &set.canonicalized.data_buckets {- let fingerprint = vault_codec::compute_content_fingerprint(&serde_json::to_string(bucket)?);- if fingerprints.get(&state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category)) != Some(&fingerprint) {+ let (_, fingerprint) = fingerprinted(bucket)?;+ if !baselines.unchanged(&state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category), &fingerprint) {return Ok((false, set));}}@@ -290,6 +288,15 @@ impl PushBaselines {known_server_hashes: state::get::<Vec<String>>(&ctx.host, state::VAULT_SERVER_BLOB_HASHES).await?.unwrap_or_default().into_iter().collect(),})}++ /// Whether the content behind a fingerprint key still matches the last-known server state.+ fn unchanged(&self, fingerprint_key: &str, fingerprint: &str) -> bool {+ self.fingerprints.get(fingerprint_key).map(String::as_str) == Some(fingerprint)+ }++ fn bucket_revision(&self, bucket: &DataBucket) -> i64 {+ self.bucket_revisions.get(&state::bucket_revision_key(&bucket.manifest_id, &bucket.category)).copied().unwrap_or(0)+ }}/// What decides whether an unchanged element still goes into the write.@@ -350,12 +357,8 @@ struct LegacyAccountKeyMigration {account_private_key: String,}-/// The change fingerprints of everything one write carried, to become the new baselines on success.-#[derive(Default)]-struct WrittenFingerprints {- manifests: HashMap<String, String>,- buckets: HashMap<String, String>,-}+/// The content fingerprints one write carried, keyed like the baselines, to become the new baselines on success.+type WrittenFingerprints = HashMap<String, String>;/// A byte or character count for log lines.fn format_kb(chars: usize) -> String {@@ -366,10 +369,26 @@ fn format_kb(chars: usize) -> String {}}-/// Pack-then-encrypt a JSON payload. Returns the base64 ciphertext plus the packed size, for logging.-fn pack_encrypt(payload_json: &str, vek: &str) -> SyncResult<(String, usize)> {- let packed = vault_codec::pack_payload(payload_json)?;- Ok((crypto::symmetric_encrypt_bytes(&packed, vek)?, packed.len()))+/// A payload as JSON text plus its content fingerprint.+fn fingerprinted<T: serde::Serialize>(payload: &T) -> SyncResult<(String, String)> {+ let plaintext = serde_json::to_string(payload)?;+ let fingerprint = vault_codec::compute_content_fingerprint(&plaintext);+ Ok((plaintext, fingerprint))+}++/// A payload packed and encrypted for the write, with the hash the server verifies it by.+struct Sealed {+ ciphertext: String,+ hash: String,+}++/// Pack and encrypt a JSON payload under `key`, logging its size at every stage.+async fn seal(ctx: &Ctx, label: &str, plaintext: &str, key: &str) -> SyncResult<Sealed> {+ let packed = vault_codec::pack_payload(plaintext)?;+ let ciphertext = crypto::symmetric_encrypt_bytes(&packed, key)?;+ ctx.log(format!("[V2Push] {}: raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(packed.len()), format_kb(ciphertext.len()))).await;+ let hash = vault_codec::compute_ciphertext_hash(&ciphertext);+ Ok(Sealed { ciphertext, hash })}/// Canonicalize, gate by content fingerprint, encrypt and `POST v2/Vault`. Returns the outcome and, on a KEK/VEK@@ -426,7 +445,8 @@ async fn push_internal(ctx: &mut Ctx, cached: Option<CanonicalizedSet>, create_vreturn Ok((PushStatus::Outdated, None));}- commit_push_baselines(ctx, baselines, &response, written, &blobs, &uploaded).await?;+ commit_push_baselines(ctx, baselines, &response.manifest_revisions, &response.bucket_revisions, written).await?;+ commit_blob_baselines(ctx, &blobs, &uploaded).await?;if let Some(migration) = &migration {complete_account_key_migration(ctx, migration).await?;ctx.log("[V2Push] Account-key migration complete: hierarchy created server-side, blob chain cached locally.").await;@@ -470,7 +490,7 @@ fn collect_upload_blobs(candidates: &[Candidate]) -> SyncResult<UploadBlobs> {continue;}blobs.order.push(hash.clone());- blobs.entries.insert(hash.clone(), UploadBlobEntry { bytes: crypto::aes_gcm::decode_base64(&blob.bytes_base64)?, kind: blob.kind.clone(), vek: candidate.vek.clone(), from_personal: candidate.record.is_personal });+ blobs.entries.insert(hash.clone(), UploadBlobEntry { bytes: crate::encoding::base64_decode(&blob.bytes_base64)?, kind: blob.kind.clone(), vek: candidate.vek.clone(), from_personal: candidate.record.is_personal });}}Ok(blobs)@@ -483,9 +503,8 @@ async fn encrypt_changed_buckets(ctx: &Ctx, buckets: &[DataBucket], candidates:for bucket in buckets {let label = format!("Data bucket \"{}\" of manifest {}", bucket.category, bucket.manifest_id);let fingerprint_key = state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category);- let plaintext = serde_json::to_string(bucket)?;- let fingerprint = vault_codec::compute_content_fingerprint(&plaintext);- if !gate.force_full_write && !gate.migrating && baselines.fingerprints.get(&fingerprint_key) == Some(&fingerprint) {+ let (plaintext, fingerprint) = fingerprinted(bucket)?;+ if !gate.force_full_write && !gate.migrating && baselines.unchanged(&fingerprint_key, &fingerprint) {ctx.log(format!("[V2Push] {} unchanged versus server baseline, leaving it out of this write.", label)).await;continue;}@@ -497,12 +516,9 @@ async fn encrypt_changed_buckets(ctx: &Ctx, buckets: &[DataBucket], candidates:if !validation.ok {return Err(SyncError::UploadRejected(vec![format!("{} validation failed: {}. {}", label, validation.failed_rules.join(", "), validation.message).trim().to_string()]));}- let (ciphertext, compressed) = pack_encrypt(&plaintext, bucket_key)?;- let ciphertext_hash = vault_codec::compute_ciphertext_hash(&ciphertext);- let current_revision = baselines.bucket_revisions.get(&state::bucket_revision_key(&bucket.manifest_id, &bucket.category)).copied().unwrap_or(0);- ctx.log(format!("[V2Push] {}: raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(compressed), format_kb(ciphertext.len()))).await;- writes.push(BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: ciphertext, ciphertext_hash, current_revision });- written.buckets.insert(fingerprint_key, fingerprint);+ let sealed = seal(ctx, &label, &plaintext, bucket_key).await?;+ writes.push(BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: sealed.ciphertext, ciphertext_hash: sealed.hash, current_revision: baselines.bucket_revision(bucket) });+ written.insert(fingerprint_key, fingerprint);}Ok(writes)}@@ -512,10 +528,10 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], baselet mut writes = Vec::new();for candidate in candidates {let label = candidate.label();- let plaintext = serde_json::to_string(candidate.manifest)?;- let fingerprint = vault_codec::compute_content_fingerprint(&plaintext);+ let fingerprint_key = state::fingerprint_manifest_key(&candidate.record.manifest_id);+ let (plaintext, fingerprint) = fingerprinted(candidate.manifest)?;let rekeyed = gate.migrating && candidate.record.is_personal;- if !gate.force_full_write && !rekeyed && baselines.fingerprints.get(&state::fingerprint_manifest_key(&candidate.record.manifest_id)) == Some(&fingerprint) {+ if !gate.force_full_write && !rekeyed && baselines.unchanged(&fingerprint_key, &fingerprint) {ctx.log(format!("[V2Push] {} unchanged versus server baseline, leaving it out of this write.", label)).await;continue;}@@ -527,9 +543,7 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], basectx.warn(format!("[V2Push] {} failed validation ({}), dropping it from this write.", label, validation.failed_rules.join(", "))).await;continue;}- let (ciphertext, compressed) = pack_encrypt(&plaintext, &candidate.vek)?;- let ciphertext_hash = vault_codec::compute_ciphertext_hash(&ciphertext);- ctx.log(format!("[V2Push] {}: raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(compressed), format_kb(ciphertext.len()))).await;+ let sealed = seal(ctx, &label, &plaintext, &candidate.vek).await?;// Publish the public half of this manifest's mail delivery keypair; only admins may publish a shared one.let may_publish = candidate.record.is_personal || candidate.record.can_administer;@@ -542,14 +556,14 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], baseblob_refs.sort_by(|a, b| a.hash.cmp(&b.hash));writes.push(ManifestWrite {manifest_id: candidate.record.manifest_id.clone(),- manifest_blob: ciphertext,- manifest_ciphertext_hash: ciphertext_hash,+ manifest_blob: sealed.ciphertext,+ manifest_ciphertext_hash: sealed.hash,current_revision: candidate.current_revision,credentials_count: candidate.manifest.tables.get("Items").map(Vec::len).unwrap_or(0),blob_references: blob_refs,encryption_public_key: manifest_key.and_then(|row| row.get("PublicKey").and_then(serde_json::Value::as_str).map(str::to_string)),});- written.manifests.insert(candidate.record.manifest_id.clone(), fingerprint);+ written.insert(fingerprint_key, fingerprint);}Ok(writes)}@@ -585,7 +599,7 @@ async fn missing_on_server(ctx: &Ctx, hashes: Vec<String>) -> SyncResult<Vec<Str/// `POST v2/Vault`. When the server reports blobs it lacks (stale local knowledge of its blob set), upload/// them and retry the identical write once; blobs this client cannot supply fail the push.async fn write_vault(ctx: &Ctx, payload: &VaultWriteRequest, blobs: &UploadBlobs, gate: WriteGate, uploaded: &mut HashMap<String, String>) -> SyncResult<VaultWriteResponse> {- let response: VaultWriteResponse = http::post(&ctx.host, VAULT_ENDPOINT, payload, true).await?;+ let response: VaultWriteResponse = http::post(&ctx.host, http::VAULT_ENDPOINT, payload, true).await?;if response.missing_blob_hashes.is_empty() {return Ok(response);}@@ -597,35 +611,35 @@ async fn write_vault(ctx: &Ctx, payload: &VaultWriteRequest, blobs: &UploadBlobslet (missing_personal, missing_shared): (Vec<String>, Vec<String>) = response.missing_blob_hashes.iter().cloned().partition(|h| blobs.entries[h].from_personal);uploaded.extend(upload_blobs(ctx, &blobs.entries, &missing_personal, gate.migrating).await?);uploaded.extend(upload_blobs(ctx, &blobs.entries, &missing_shared, false).await?);- let response: VaultWriteResponse = http::post(&ctx.host, VAULT_ENDPOINT, payload, true).await?;+ let response: VaultWriteResponse = http::post(&ctx.host, http::VAULT_ENDPOINT, payload, true).await?;if !response.missing_blob_hashes.is_empty() {return Err(SyncError::MissingBlobs(response.missing_blob_hashes));}Ok(response)}-/// Advance the local baselines of everything a successful write carried, and refresh the encrypted blob cache.-async fn commit_push_baselines(ctx: &Ctx, baselines: PushBaselines, response: &VaultWriteResponse, written: WrittenFingerprints, blobs: &UploadBlobs, uploaded: &HashMap<String, String>) -> SyncResult<()> {- let PushBaselines { mut fingerprints, mut manifest_revisions, mut bucket_revisions, .. } = baselines;- if !response.bucket_revisions.is_empty() {- for br in &response.bucket_revisions {- bucket_revisions.insert(state::bucket_revision_key(&br.manifest_id, &br.category), br.revision);+/// Advance the revision and fingerprint baselines of everything a successful write carried.+async fn commit_push_baselines(ctx: &Ctx, baselines: PushBaselines, manifest_revisions: &[ManifestRevision], bucket_revisions: &[BucketRevision], written: WrittenFingerprints) -> SyncResult<()> {+ let PushBaselines { mut fingerprints, manifest_revisions: mut known_manifests, bucket_revisions: mut known_buckets, .. } = baselines;+ if !bucket_revisions.is_empty() {+ for br in bucket_revisions {+ known_buckets.insert(state::bucket_revision_key(&br.manifest_id, &br.category), br.revision);}- state::set(&ctx.host, state::VAULT_BUCKET_REVISIONS, &bucket_revisions).await?;+ state::set(&ctx.host, state::VAULT_BUCKET_REVISIONS, &known_buckets).await?;}- for mr in &response.manifest_revisions {- manifest_revisions.insert(mr.manifest_id.clone(), mr.revision);+ if !manifest_revisions.is_empty() {+ for mr in manifest_revisions {+ known_manifests.insert(mr.manifest_id.clone(), mr.revision);+ }+ state::set(&ctx.host, state::SERVER_MANIFEST_REVISIONS, &known_manifests).await?;}- state::set(&ctx.host, state::SERVER_MANIFEST_REVISIONS, &manifest_revisions).await?;+ fingerprints.extend(written);+ state::set(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS, &fingerprints).await+}- fingerprints.extend(written.buckets);- for (manifest_id, fingerprint) in written.manifests {- fingerprints.insert(state::fingerprint_manifest_key(&manifest_id), fingerprint);- }- state::set(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS, &fingerprints).await?;+/// Record the server's blob set as this write left it, and keep the encrypted blob cache to what is still referenced.+async fn commit_blob_baselines(ctx: &Ctx, blobs: &UploadBlobs, uploaded: &HashMap<String, String>) -> SyncResult<()> {state::set(&ctx.host, state::VAULT_SERVER_BLOB_HASHES, &blobs.order).await?;-- // The encrypted blob cache: entries still referenced, plus the ciphertexts just uploaded.let cache: HashMap<String, String> = state::get(&ctx.host, state::VAULT_BLOB_CIPHER_CACHE).await?.unwrap_or_default();let mut new_cache: HashMap<String, String> = HashMap::new();for hash in &blobs.order {@@ -644,7 +658,7 @@ async fn complete_account_key_migration(ctx: &mut Ctx, migration: &LegacyAccountstate::set(&ctx.host, state::ACCOUNT_PUBLIC_KEY, &blobs.account_public_key).await?;state::set(&ctx.host, state::ENCRYPTED_ACCOUNT_PRIVATE_KEY, &blobs.encrypted_account_private_key).await?;ctx.account_public_key = Some(blobs.account_public_key.clone());- ctx.set_account_private_key(Some(migration.account_private_key.clone()));+ ctx.set_account_private_key(migration.account_private_key.clone());Ok(())}@@ -652,25 +666,16 @@ async fn complete_account_key_migration(ctx: &mut Ctx, migration: &LegacyAccount/// ciphertext for the local encrypted blob cache.async fn upload_blobs(ctx: &Ctx, entries: &HashMap<String, UploadBlobEntry>, hashes: &[String], overwrite: bool) -> SyncResult<HashMap<String, String>> {let mut ciphertexts = HashMap::new();- if hashes.is_empty() {- return Ok(ciphertexts);- }- let mut batch: Vec<BlobDto> = Vec::new();- let mut batch_chars = 0usize;+ let mut dtos = Vec::new();for hash in hashes {let Some(entry) = entries.get(hash) else { continue };let ciphertext = crypto::symmetric_encrypt_bytes(&entry.bytes, &entry.vek)?;ciphertexts.insert(hash.clone(), ciphertext.clone());- if !batch.is_empty() && (batch_chars + ciphertext.len() > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.len() >= BLOB_TRANSFER_BATCH_MAX_COUNT) {- ctx.log(format!("[V2Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(batch_chars))).await;- http::post_no_content(&ctx.host, BLOBS_ENDPOINT, &BlobUploadRequest { blobs: std::mem::take(&mut batch), overwrite }).await?;- batch_chars = 0;- }- batch_chars += ciphertext.len();- batch.push(BlobDto { hash: hash.clone(), category: entry.kind.clone(), encrypted_data_base64: ciphertext });+ dtos.push(BlobDto { hash: hash.clone(), category: entry.kind.clone(), encrypted_data_base64: ciphertext });}- if !batch.is_empty() {- ctx.log(format!("[V2Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(batch_chars))).await;+ for batch in http::batch_by_transfer_cost(dtos, |dto| dto.encrypted_data_base64.len()) {+ let chars: usize = batch.iter().map(|dto| dto.encrypted_data_base64.len()).sum();+ ctx.log(format!("[V2Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(chars))).await;http::post_no_content(&ctx.host, BLOBS_ENDPOINT, &BlobUploadRequest { blobs: batch, overwrite }).await?;}Ok(ciphertexts)@@ -686,37 +691,30 @@ async fn push_data_bucket_only(ctx: &Ctx, bucket: &DataBucket, vek: &str) -> Syn}async fn push_data_bucket_only_internal(ctx: &Ctx, bucket: &DataBucket, vek: &str) -> SyncResult<(PushStatus, i64)> {- let label = format!("Bucket \"{}\" of manifest {}", bucket.category, bucket.manifest_id);- let revision_key = state::bucket_revision_key(&bucket.manifest_id, &bucket.category);- let plaintext = serde_json::to_string(bucket)?;-- let mut fingerprints: HashMap<String, String> = state::get(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?.unwrap_or_default();- let mut bucket_revisions: HashMap<String, i64> = state::get(&ctx.host, state::VAULT_BUCKET_REVISIONS).await?.unwrap_or_default();- let fingerprint = vault_codec::compute_content_fingerprint(&plaintext);+ let label = format!("Bucket \"{}\" of manifest {} (bucket-only)", bucket.category, bucket.manifest_id);let fingerprint_key = state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category);- if fingerprints.get(&fingerprint_key) == Some(&fingerprint) {- ctx.log(format!("[V2Push] {} (bucket-only) unchanged versus server baseline, skipping upload.", label)).await;- return Ok((PushStatus::Ok, bucket_revisions.get(&revision_key).copied().unwrap_or(0)));+ let baselines = PushBaselines::load(ctx).await?;+ let (plaintext, fingerprint) = fingerprinted(bucket)?;+ if baselines.unchanged(&fingerprint_key, &fingerprint) {+ ctx.log(format!("[V2Push] {} unchanged versus server baseline, skipping upload.", label)).await;+ return Ok((PushStatus::Ok, baselines.bucket_revision(bucket)));}-- let (ciphertext, compressed) = pack_encrypt(&plaintext, vek)?;- let ciphertext_hash = vault_codec::compute_ciphertext_hash(&ciphertext);- ctx.log(format!("[V2Push] {} (bucket-only): raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(compressed), format_kb(ciphertext.len()))).await;+ let sealed = seal(ctx, &label, &plaintext, vek).await?;let post = |current_revision: i64| {let payload = VaultWriteRequest {username: ctx.request.username.clone(),manifests: Vec::new(),- buckets: vec![BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: ciphertext.clone(), ciphertext_hash: ciphertext_hash.clone(), current_revision }],+ buckets: vec![BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: sealed.ciphertext.clone(), ciphertext_hash: sealed.hash.clone(), current_revision }],new_blobs: Vec::new(),email_routing: None,account_keys: None,};- async move { http::post::<_, VaultWriteResponse>(&ctx.host, VAULT_ENDPOINT, &payload, true).await }+ async move { http::post::<_, VaultWriteResponse>(&ctx.host, http::VAULT_ENDPOINT, &payload, true).await }};let reported = |response: &VaultWriteResponse| response.bucket_revisions.iter().find(|b| ids_equal(&b.manifest_id, &bucket.manifest_id) && b.category == bucket.category).map(|b| b.revision);- let mut current_revision = bucket_revisions.get(&revision_key).copied().unwrap_or(0);+ let mut current_revision = baselines.bucket_revision(bucket);let mut response = post(current_revision).await?;if response.status != 0 {let server_revision = reported(&response).unwrap_or(current_revision);@@ -729,9 +727,7 @@ async fn push_data_bucket_only_internal(ctx: &Ctx, bucket: &DataBucket, vek: &st}let new_revision = reported(&response).unwrap_or(current_revision + 1);- bucket_revisions.insert(revision_key, new_revision);- state::set(&ctx.host, state::VAULT_BUCKET_REVISIONS, &bucket_revisions).await?;- fingerprints.insert(fingerprint_key, fingerprint);- state::set(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS, &fingerprints).await?;+ let advanced = BucketRevision { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), revision: new_revision };+ commit_push_baselines(ctx, baselines, &[], &[advanced], HashMap::from([(fingerprint_key, fingerprint)])).await?;Ok((PushStatus::Ok, new_revision))}diff --git a/core/rust/src/vault_sync/state.rs b/core/rust/src/vault_sync/state.rsindex 7ae649dac..da5fa5c04 100644--- a/core/rust/src/vault_sync/state.rs+++ b/core/rust/src/vault_sync/state.rs@@ -70,12 +70,10 @@ impl Ctx {self.encryption_key = Some(key);}- pub fn set_account_private_key(&mut self, key: Option<String>) {- match &key {- Some(value) => self.updates.account_private_key = Some(value.clone()),- None => self.updates.clear_account_private_key = true,- }- self.account_private_key = key;+ /// Adopt the account private key a migration push minted, reporting it to the host.+ pub fn set_account_private_key(&mut self, key: String) {+ self.updates.account_private_key = Some(key.clone());+ self.account_private_key = Some(key);}/// The current schema.@@ -177,14 +175,14 @@ const SQLITE_HEADER: &[u8] = b"SQLite format 3\0";/// Decrypt a stored vault blob into the plaintext SQLite database.pub(crate) fn decrypt_vault_blob(encrypted_blob: &str, key: &str) -> SyncResult<Vec<u8>> {- let ciphertext = crypto::aes_gcm::decode_base64(encrypted_blob)?;+ let ciphertext = crate::encoding::base64_decode(encrypted_blob)?;let plaintext = crypto::symmetric_decrypt_bytes(&ciphertext, key).map_err(|e| SyncError::VaultDecryptFailed(e.to_string()))?;if plaintext.starts_with(SQLITE_HEADER) {return Ok(plaintext);}let not_a_database = || SyncError::VaultDecryptFailed("plaintext is neither a database nor base64 text".to_string());let text = String::from_utf8(plaintext).map_err(|_| not_a_database())?;- crypto::aes_gcm::decode_base64(text.trim()).map_err(|_| not_a_database())+ crate::encoding::base64_decode(text.trim()).map_err(|_| not_a_database())}/// Encrypt a plaintext SQLite database for local storage.diff --git a/core/rust/src/vault_sync/types.rs b/core/rust/src/vault_sync/types.rsindex 82c5a8acc..4ce3a4262 100644--- a/core/rust/src/vault_sync/types.rs+++ b/core/rust/src/vault_sync/types.rs@@ -6,7 +6,7 @@ use std::fmt;use serde::{Deserialize, Serialize};use serde_json::{Map, Value};-use super::errors::{ErrorCode, LogoutReason};+use super::errors::{ErrorCode, Failure, LogoutReason, SyncError};/// The operation a session runs.#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]@@ -60,9 +60,6 @@ pub struct SessionUpdates {pub encryption_key: Option<String>,#[serde(skip_serializing_if = "Option::is_none")]pub account_private_key: Option<String>,- /// Set when the account private key must be dropped from the session.- #[serde(default, skip_serializing_if = "std::ops::Not::not")]- pub clear_account_private_key: bool,}/// What every operation reports on top of its own outcome.@@ -81,6 +78,35 @@ pub(crate) trait OperationResult: Serialize {fn session_mut(&mut self) -> &mut SessionOutcome;}+/// How a failed operation reports itself: a coded error the host translates, or a forced logout.+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct FailureFields {+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error_code: Option<ErrorCode>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error_key: Option<LogoutReason>,+ pub requires_logout: bool,+}++impl FailureFields {+ /// A forced logout for the given reason.+ pub fn logout(reason: LogoutReason) -> Self {+ Self { error_key: Some(reason), requires_logout: true, ..Default::default() }+ }+}++impl From<&SyncError> for FailureFields {+ fn from(error: &SyncError) -> Self {+ match error.failure() {+ Failure::Logout(reason) => Self::logout(reason),+ Failure::Coded(code) => Self { error: Some(error.to_string()), error_code: Some(code), ..Default::default() },+ }+ }+}+/// Outcome of a full sync.#[derive(Debug, Clone, Default, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]@@ -90,13 +116,8 @@ pub struct FullSyncResult {pub was_offline: bool,pub sqlite_blob_upgrade_required: bool,pub manifest_migration_required: bool,- #[serde(skip_serializing_if = "Option::is_none")]- pub error: Option<String>,- #[serde(skip_serializing_if = "Option::is_none")]- pub error_code: Option<ErrorCode>,- #[serde(skip_serializing_if = "Option::is_none")]- pub error_key: Option<LogoutReason>,- pub requires_logout: bool,+ #[serde(flatten)]+ pub failure: FailureFields,#[serde(skip_serializing_if = "Option::is_none")]pub server_version: Option<String>,#[serde(skip_serializing_if = "Option::is_none")]@@ -151,13 +172,8 @@ pub struct MigrateManifestResult {pub success: bool,/// Whether the migrated vault reached the server; false leaves it dirty for the next sync.pub pushed: bool,- #[serde(skip_serializing_if = "Option::is_none")]- pub error: Option<String>,- #[serde(skip_serializing_if = "Option::is_none")]- pub error_code: Option<ErrorCode>,- #[serde(skip_serializing_if = "Option::is_none")]- pub error_key: Option<LogoutReason>,- pub requires_logout: bool,+ #[serde(flatten)]+ pub failure: FailureFields,#[serde(flatten)]pub session: SessionOutcome,}@@ -176,13 +192,8 @@ pub struct StatusCheckResult {pub has_newer_vault: bool,pub has_dirty_changes: bool,pub is_offline: bool,- pub requires_logout: bool,- #[serde(skip_serializing_if = "Option::is_none")]- pub error_key: Option<LogoutReason>,- #[serde(skip_serializing_if = "Option::is_none")]- pub error: Option<String>,- #[serde(skip_serializing_if = "Option::is_none")]- pub error_code: Option<ErrorCode>,+ #[serde(flatten)]+ pub failure: FailureFields,#[serde(skip_serializing_if = "Option::is_none")]pub server_version: Option<String>,#[serde(skip_serializing_if = "Option::is_none")]@@ -211,16 +222,6 @@ pub enum HttpMethod {Delete,}-impl fmt::Display for HttpMethod {- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- f.write_str(match self {- HttpMethod::Get => "GET",- HttpMethod::Post => "POST",- HttpMethod::Delete => "DELETE",- })- }-}-/// The databases a command may address.#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]#[serde(rename_all = "lowercase")]@@ -492,7 +493,7 @@ pub struct PendingAction {pub struct GetResponse {#[serde(default)]pub status: i32,- /// 0 = sqlite-blob, 1 = manifest-v1; absent on servers predating the field.+ /// LEGACY: 0 = sqlite-blob, 1 = manifest-v1; absent on servers predating the field.#[serde(default)]pub storage_format: Option<i32>,#[serde(default)]@@ -511,8 +512,6 @@ pub struct GetResponse {pub email_routing: Option<EmailRoutingDto>,}-pub const STORAGE_FORMAT_MANIFEST: i32 = 1;-#[derive(Debug, Clone, Default, Deserialize)]#[serde(rename_all = "camelCase")]pub struct ManifestDto {@@ -716,3 +715,31 @@ pub struct SharedManifestRecord {#[serde(default)]pub can_administer: bool,}++#[cfg(test)]+mod tests {+ use super::*;++ /// The failure fields flatten into every result exactly where the four loose fields used to sit.+ #[test]+ fn failure_fields_flatten_into_the_result_wire_shape() {+ let coded = FailureFields::from(&SyncError::VaultLocked);+ let full = serde_json::to_string(&FullSyncResult { failure: coded, ..Default::default() }).unwrap();+ assert_eq!(full, r#"{"success":false,"hasNewVault":false,"wasOffline":false,"sqliteBlobUpgradeRequired":false,"manifestMigrationRequired":false,"error":"No encryption key available","errorCode":"E-202","requiresLogout":false,"isOfflineMode":false,"sessionUpdates":{},"vaultChanged":false}"#);+ let round_trip: FullSyncResult = serde_json::from_str(&full).unwrap();+ assert_eq!(serde_json::to_string(&round_trip).unwrap(), full);++ let logout = FailureFields::logout(LogoutReason::SessionExpired);+ let migrate = serde_json::to_string(&MigrateManifestResult { failure: logout.clone(), ..Default::default() }).unwrap();+ assert_eq!(migrate, r#"{"success":false,"pushed":false,"errorKey":"sessionExpired","requiresLogout":true,"sessionUpdates":{},"vaultChanged":false}"#);+ let round_trip: MigrateManifestResult = serde_json::from_str(&migrate).unwrap();+ assert_eq!(round_trip.failure, logout);++ let status = serde_json::to_value(StatusCheckResult { failure: logout.clone(), ..Default::default() }).unwrap();+ assert_eq!(status["errorKey"], "sessionExpired");+ assert_eq!(status["requiresLogout"], true);+ assert!(status.get("error").is_none() && status.get("errorCode").is_none() && status.get("failure").is_none());+ let round_trip: StatusCheckResult = serde_json::from_value(status.clone()).unwrap();+ assert_eq!(serde_json::to_value(&round_trip).unwrap(), status);+ }+}diff --git a/core/rust/src/vault_sync/version.rs b/core/rust/src/vault_sync/version.rsindex 15e4c67f8..79b355af6 100644--- a/core/rust/src/vault_sync/version.rs+++ b/core/rust/src/vault_sync/version.rs@@ -1,4 +1,4 @@-//! Version rules: server support, and the frozen legacy sqlite-blob vault version chain.+//! Version rules: which server versions this client supports./// Whether `version1 >= version2` under the client's SemVer rules: a pre-release sorts below its release, and/// two pre-releases compare lexically.@@ -35,57 +35,6 @@ fn split(version: &str) -> (&str, Option<&str>) {}}-/// The frozen sqlite-blob upgrade chain: (revision, data version). It ends at 2.0.0, the first schema compatible-/// with manifest-v1; later schema changes ship through the full schema only which is used by every materilization directly.-const LEGACY_VAULT_VERSIONS: &[(u32, &str)] = &[- (1, "1.0.0"),- (2, "1.0.1"),- (3, "1.0.2"),- (4, "1.1.0"),- (5, "1.2.0"),- (6, "1.3.0"),- (7, "1.3.1"),- (8, "1.4.0"),- (9, "1.4.1"),- (10, "1.5.0"),- (11, "1.6.0"),- (12, "1.7.0"),- (13, "2.0.0"),-];--/// The revision of the last entry in the legacy chain.-pub fn latest_legacy_revision() -> u32 {- LEGACY_VAULT_VERSIONS.last().map(|(revision, _)| *revision).unwrap_or(0)-}--/// The data version embedded in an EF migration id (`20250101000000_2.0.0-Name`).-pub fn extract_version_from_migration_id(migration_id: &str) -> Option<String> {- let start = migration_id.find('_')? + 1;- let rest = &migration_id[start..];- let end = rest.find('-')?;- let candidate = &rest[..end];- let parts: Vec<&str> = candidate.split('.').collect();- if parts.len() == 3 && parts.iter().all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit())) {- Some(candidate.to_string())- } else {- None- }-}--/// The legacy-chain revision a database version maps to. A known version maps to its entry; an unknown but-/// well-formed version is treated as the latest (backwards compatible); a malformed version is incompatible.-pub fn legacy_revision_for(database_version: &str) -> Result<u32, String> {- if let Some((revision, _)) = LEGACY_VAULT_VERSIONS.iter().find(|(_, version)| *version == database_version) {- return Ok(*revision);- }- let well_formed = database_version.split('.').count() == 3 && database_version.split('.').all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit()));- if well_formed {- Ok(latest_legacy_revision())- } else {- Err(format!("Vault version {} is not compatible with this client", database_version))- }-}-#[cfg(test)]mod tests {use super::*;@@ -99,14 +48,4 @@ mod tests {assert!(version_gte("1.0", "1.0.0"));assert!(version_gte("0.12.0-beta", "0.12.0-alpha"));}-- #[test]- fn migration_ids_yield_their_version() {- assert_eq!(extract_version_from_migration_id("20250101000000_2.0.0-Squashed").as_deref(), Some("2.0.0"));- assert_eq!(extract_version_from_migration_id("20250101000000_Initial"), None);- assert_eq!(legacy_revision_for("1.7.0"), Ok(12));- assert_eq!(legacy_revision_for("2.0.0"), Ok(13));- assert_eq!(legacy_revision_for("2.1.0"), Ok(13));- assert!(legacy_revision_for("nope").is_err());- }}