Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Move legacy sync logic to separate legacy.rs file (#2159)

Leendert de Borst577d5025bf47
diff --git a/core/rust/src/vault_sharing/mod.rs b/core/rust/src/vault_sharing/mod.rs
index 4617b884f..a83df51a8 100644
--- a/core/rust/src/vault_sharing/mod.rs
+++ b/core/rust/src/vault_sharing/mod.rs
@@ -6,14 +6,12 @@ mod tests;
use std::collections::HashMap;
-use crate::error::VaultResult;
-
pub use types::{
ManifestAccessPartition, ManifestAccessRequest, ManifestWriteRecord, ManifestWriteSet,
ManifestWriteSetRequest, SharedManifestRecord, SkippedManifest, WriteSkipReason,
};
-use types::id_key;
+use crate::vault_model::id_key;
/// Resolve which manifests the next push writes, personal manifest first.
///
@@ -87,13 +85,3 @@ fn resolve_name(display_names: &HashMap<String, String>, manifest_id: &str, reco
.or_else(|| record_name.map(|name| name.to_string()))
.filter(|name| !name.is_empty())
}
-
-/// Resolve a push's write set. Input: `ManifestWriteSetRequest` JSON. Output: `ManifestWriteSet` JSON.
-pub fn resolve_manifest_write_set_json(input_json: &str) -> VaultResult<String> {
- Ok(serde_json::to_string(&resolve_manifest_write_set(serde_json::from_str(input_json)?))?)
-}
-
-/// Partition manifest access. Input: `ManifestAccessRequest` JSON. Output: `ManifestAccessPartition` JSON.
-pub fn partition_manifest_access_json(input_json: &str) -> VaultResult<String> {
- Ok(serde_json::to_string(&partition_manifest_access(serde_json::from_str(input_json)?))?)
-}
diff --git a/core/rust/src/vault_sharing/tests.rs b/core/rust/src/vault_sharing/tests.rs
index a401cec9f..2e710f356 100644
--- a/core/rust/src/vault_sharing/tests.rs
+++ b/core/rust/src/vault_sharing/tests.rs
@@ -177,7 +177,7 @@ fn the_json_boundary_round_trips() {
"displayNames": { "man-1": "Family" }
}"#;
- let output: serde_json::Value = serde_json::from_str(&resolve_manifest_write_set_json(input).unwrap()).unwrap();
+ let output: serde_json::Value = serde_json::from_str(&crate::error::json_call(input, |request| Ok(resolve_manifest_write_set(request))).unwrap()).unwrap();
assert_eq!(output["records"][1]["manifestId"], "MAN-1");
assert_eq!(output["records"][1]["name"], "Family");
assert_eq!(output["records"][1]["isPersonal"], false);
diff --git a/core/rust/src/vault_sharing/types.rs b/core/rust/src/vault_sharing/types.rs
index 9de8a6e3e..45a8d845c 100644
--- a/core/rust/src/vault_sharing/types.rs
+++ b/core/rust/src/vault_sharing/types.rs
@@ -3,11 +3,6 @@ use std::collections::HashMap;
use serde::{Deserialize, Serialize};
-/// Lower-cased comparison key for a manifest id.
-pub(crate) fn id_key(id: &str) -> String {
- id.trim().to_lowercase()
-}
-
/// A shared manifest's key record.
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
diff --git a/core/rust/src/vault_sync/legacy.rs b/core/rust/src/vault_sync/legacy.rs
index 0a7132b70..2428bc925 100644
--- a/core/rust/src/vault_sync/legacy.rs
+++ b/core/rust/src/vault_sync/legacy.rs
@@ -1,15 +1,89 @@
-//! LEGACY: what only a vault still on the sqlite-blob storage format needs, namely its local migration onto
-//! the current schema. TODO: remove this module once all users have migrated to the manifest-v1 storage model.
+//! LEGACY: everything only a vault still on the sqlite-blob storage format needs. TODO: remove this module and all callers once
+//! all users have migrated to the manifest-v1 storage model.
use std::collections::HashMap;
-use super::errors::SyncResult;
-use super::pull::materialize_to_sqlite;
+use super::errors::{SyncError, SyncResult};
+use super::pull::{email_routing_of, materialize_to_sqlite, PulledVault};
use super::push::{canonicalize_vault, resolve_personal_manifest_id};
-use super::state::Ctx;
-use crate::crypto;
+use super::state::{self, Ctx};
+use super::types::GetResponse;
use crate::vault_codec::Manifest;
+/// The `storageFormat` a manifest-v1 snapshot declares; 0 (or absent) is a sqlite blob.
+const STORAGE_FORMAT_MANIFEST: i32 = 1;
+
+/// Whether a snapshot is still on the legacy sqlite-blob format.
+pub(crate) fn is_legacy_sqlite_blob_snapshot(snapshot: &GetResponse) -> bool {
+ snapshot.storage_format != Some(STORAGE_FORMAT_MANIFEST)
+}
+
+/// Take a legacy snapshot apart for local storage: the blob passes through untouched, the manifest-v1 fingerprints
+/// are reset, and the personal manifest id is recorded for the migration push.
+pub(crate) async fn open_legacy_snapshot(ctx: &Ctx, snapshot: &GetResponse) -> SyncResult<PulledVault> {
+ state::remove(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?;
+ let revision = snapshot.legacy_revision.unwrap_or(0);
+ let mut manifest_revisions = HashMap::new();
+ if let Some(personal) = &snapshot.personal_manifest_id {
+ state::set(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID, personal).await?;
+ manifest_revisions.insert(personal.clone(), revision);
+ }
+ ctx.log("[V2Pull] Legacy sqlite-blob pass-through (user not yet migrated), returning the blob as-is.").await;
+ Ok(PulledVault { encrypted_vault: snapshot.legacy_vault_blob.clone().unwrap_or_default(), revision, email_routing: email_routing_of(snapshot), manifest_revisions, bucket_revisions: HashMap::new() })
+}
+
+/// The frozen sqlite-blob upgrade chain: (revision, data version). It ends at 2.0.0, the first schema compatible
+/// with manifest-v1; later schema changes ship through the full schema only, which every materialization uses directly.
+const LEGACY_VAULT_VERSIONS: &[(u32, &str)] = &[
+ (1, "1.0.0"),
+ (2, "1.0.1"),
+ (3, "1.0.2"),
+ (4, "1.1.0"),
+ (5, "1.2.0"),
+ (6, "1.3.0"),
+ (7, "1.3.1"),
+ (8, "1.4.0"),
+ (9, "1.4.1"),
+ (10, "1.5.0"),
+ (11, "1.6.0"),
+ (12, "1.7.0"),
+ (13, "2.0.0"),
+];
+
+/// The revision of the last entry in the legacy chain.
+fn latest_legacy_revision() -> u32 {
+ LEGACY_VAULT_VERSIONS.last().map(|(revision, _)| *revision).unwrap_or(0)
+}
+
+/// The data version embedded in an EF migration id (`20250101000000_2.0.0-Name`).
+fn extract_version_from_migration_id(migration_id: &str) -> Option<String> {
+ let start = migration_id.find('_')? + 1;
+ let rest = &migration_id[start..];
+ let end = rest.find('-')?;
+ let candidate = &rest[..end];
+ if is_well_formed_version(candidate) { Some(candidate.to_string()) } else { None }
+}
+
+fn is_well_formed_version(version: &str) -> bool {
+ version.split('.').count() == 3 && version.split('.').all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit()))
+}
+
+/// The legacy-chain revision a database version maps to. A known version maps to its entry; an unknown but
+/// well-formed version is treated as the latest (backwards compatible); a malformed version is incompatible.
+fn legacy_revision_for(database_version: &str) -> Result<u32, String> {
+ if let Some((revision, _)) = LEGACY_VAULT_VERSIONS.iter().find(|(_, version)| *version == database_version) {
+ return Ok(*revision);
+ }
+ if is_well_formed_version(database_version) { Ok(latest_legacy_revision()) } else { Err(format!("Vault version {} is not compatible with this client", database_version)) }
+}
+
+/// Whether a vault's latest migration stamp still has to walk the frozen sqlite-blob chain (a pre-2.0.0 vault).
+pub(crate) fn stamp_predates_manifest_schema(migration_id: &str) -> SyncResult<bool> {
+ let database_version = extract_version_from_migration_id(migration_id).ok_or_else(|| SyncError::Other("Could not extract version from migration ID".to_string()))?;
+ let revision = legacy_revision_for(&database_version).map_err(SyncError::VaultVersionIncompatible)?;
+ Ok(revision < latest_legacy_revision())
+}
+
/// Migrate the local vault onto the current schema, entirely locally.
pub(crate) async fn migrate_vault_to_current_schema(ctx: &mut Ctx) -> SyncResult<Vec<u8>> {
ctx.log("[ManifestMigration] Migrating local vault onto the current schema (local round-trip, no server involved)...").await;
@@ -19,7 +93,7 @@ pub(crate) async fn migrate_vault_to_current_schema(ctx: &mut Ctx) -> SyncResult
let mut blob_map = HashMap::new();
for entry in &set.canonicalized.manifests {
for (hash, blob) in &entry.blobs {
- blob_map.insert(hash.clone(), crypto::aes_gcm::decode_base64(&blob.bytes_base64)?);
+ blob_map.insert(hash.clone(), crate::encoding::base64_decode(&blob.bytes_base64)?);
}
}
let manifests: Vec<Manifest> = set.canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect();
@@ -27,3 +101,26 @@ pub(crate) async fn migrate_vault_to_current_schema(ctx: &mut Ctx) -> SyncResult
ctx.log(format!("[ManifestMigration] Migration complete: {} blobs re-embedded, {} bytes.", blob_map.len(), bytes.len())).await;
Ok(bytes)
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn migration_ids_yield_their_version() {
+ assert_eq!(extract_version_from_migration_id("20250101000000_2.0.0-Squashed").as_deref(), Some("2.0.0"));
+ assert_eq!(extract_version_from_migration_id("20250101000000_Initial"), None);
+ assert_eq!(legacy_revision_for("1.7.0"), Ok(12));
+ assert_eq!(legacy_revision_for("2.0.0"), Ok(13));
+ assert_eq!(legacy_revision_for("2.1.0"), Ok(13));
+ assert!(legacy_revision_for("nope").is_err());
+ }
+
+ #[test]
+ fn stamps_before_2_0_0_are_on_the_chain() {
+ assert_eq!(stamp_predates_manifest_schema("20250101000000_1.7.0-Old").unwrap(), true);
+ assert_eq!(stamp_predates_manifest_schema("20250101000000_2.0.0-Squashed").unwrap(), false);
+ assert_eq!(stamp_predates_manifest_schema("20260101000000_2.1.0-Later").unwrap(), false);
+ assert!(matches!(stamp_predates_manifest_schema("20250101000000_Initial"), Err(SyncError::Other(_))));
+ }
+}
diff --git a/core/rust/src/vault_sync/merge.rs b/core/rust/src/vault_sync/merge.rs
index 163dc7a95..9f09729db 100644
--- a/core/rust/src/vault_sync/merge.rs
+++ b/core/rust/src/vault_sync/merge.rs
@@ -3,17 +3,15 @@
use std::collections::{HashMap, HashSet};
-use base64::engine::general_purpose::STANDARD as BASE64;
-use base64::Engine;
use serde::{Deserialize, Serialize};
-use super::db::{id_key, ids_equal};
+use crate::vault_model::{id_key, ids_equal};
use super::errors::{SyncError, SyncResult};
-use super::pull::{self, LegacyPassthrough, OpenedManifestSet, SnapshotFacts};
+use super::pull::{self, OpenedManifestSet, PulledVault};
use super::push::{canonicalize_vault, CanonicalizedSet, ManifestRecord};
use super::state::{self, Ctx};
use super::types::EmailRoutingDto;
-use crate::crypto;
+use super::legacy;
use crate::vault_codec::{self, BlobEntry, CanonicalizedVault, DataBucket, Manifest};
use crate::vault_merge::{merge_canonical, CanonicalManifestMerge, CanonicalMergeInput, MergeStats};
@@ -42,38 +40,32 @@ impl MergeSummary {
/// What a dirty pull (pull and merge) produced.
pub(crate) enum PullAndMergeOutcome {
- LegacyServer(LegacyPassthrough),
+ /// LEGACY: the server's vault is still a sqlite blob at this revision; the caller pushes the local vault over it.
+ LegacyServer { revision: i64 },
Merged {
- encrypted_vault: String,
- revision: i64,
- email_routing: EmailRoutingDto,
+ pulled: PulledVault,
stats: MergeSummary,
fallback_manifest_ids: Vec<String>,
dropped_local_manifest_ids: Vec<String>,
- manifest_revisions: HashMap<String, i64>,
- bucket_revisions: HashMap<String, i64>,
push_canonical: Option<CanonicalizedSet>,
},
- ServerOnly {
- encrypted_vault: String,
- revision: i64,
- email_routing: EmailRoutingDto,
- manifest_revisions: HashMap<String, i64>,
- bucket_revisions: HashMap<String, i64>,
- },
+ /// The merge failed, so the server's vault stands as pulled and the local changes are dropped.
+ ServerOnly(PulledVault),
}
/// Pull the latest snapshot and merge the local vault onto it at canonical level, one manifest at a time.
-pub(crate) async fn pull_and_merge(ctx: &mut Ctx, facts: &mut SnapshotFacts) -> SyncResult<PullAndMergeOutcome> {
+pub(crate) async fn pull_and_merge(ctx: &mut Ctx) -> SyncResult<PullAndMergeOutcome> {
let vek = ctx.encryption_key()?;
ctx.log("[V2Merge] Fetching vault snapshot for canonical merge (GET /v2/Vault)...").await;
- let snapshot = pull::fetch_snapshot(ctx, facts).await?;
- let email_routing = pull::email_routing_of(&snapshot);
+ let snapshot = pull::fetch_snapshot(ctx).await?;
// LEGACY: a server still on the sqlite-blob format cannot merge with a manifest-v1 vault; the caller pushes over it.
- if facts.legacy {
- return Ok(PullAndMergeOutcome::LegacyServer(pull::open_legacy_snapshot(ctx, &snapshot).await?));
+ if legacy::is_legacy_sqlite_blob_snapshot(&snapshot) {
+ let legacy = legacy::open_legacy_snapshot(ctx, &snapshot).await?;
+ pull::commit_revisions(ctx, &legacy.manifest_revisions, &legacy.bucket_revisions).await?;
+ return Ok(PullAndMergeOutcome::LegacyServer { revision: legacy.revision });
}
+ let email_routing = pull::email_routing_of(&snapshot);
let opened = pull::open_manifests_and_record_sync_state(ctx, &snapshot, &vek).await?;
let local_side = canonicalize_vault(ctx, None).await?;
@@ -91,13 +83,7 @@ pub(crate) async fn pull_and_merge(ctx: &mut Ctx, facts: &mut SnapshotFacts) ->
// The merge-failure fallback, from the same snapshot: the server vault stands, local changes are dropped.
ctx.warn(format!("[V2Merge] Canonical merge failed, falling back to the server vault: {}", merge_error)).await;
let sqlite_bytes = pull::materialize_to_sqlite(ctx, &opened.manifests(), &opened.data_buckets, &opened.blob_map).await?;
- Ok(PullAndMergeOutcome::ServerOnly {
- encrypted_vault: state::encrypt_vault_blob(&sqlite_bytes, &vek)?,
- revision: opened.personal_revision,
- email_routing,
- manifest_revisions: opened.manifest_revisions,
- bucket_revisions: opened.bucket_revisions,
- })
+ Ok(PullAndMergeOutcome::ServerOnly(opened.pulled_vault(state::encrypt_vault_blob(&sqlite_bytes, &vek)?, email_routing)))
}
}
}
@@ -174,7 +160,7 @@ async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet,
for (hash, blob) in &canonicalized.blobs {
local_blobs.insert(hash.clone(), blob.clone());
if !blob_map.contains_key(hash) {
- blob_map.insert(hash.clone(), crypto::aes_gcm::decode_base64(&blob.bytes_base64)?);
+ blob_map.insert(hash.clone(), crate::encoding::base64_decode(&blob.bytes_base64)?);
}
}
}
@@ -186,17 +172,7 @@ async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet,
ctx.log(format!("[V2Merge] Canonical merge complete: {} conflict(s), {} offline row(s) kept, {} validation fallback(s), {} dropped local manifest(s).", stats.conflicts, stats.records_inserted, fallback_manifest_ids.len(), merge_output.dropped_local_manifest_ids.len())).await;
let push_canonical = merge_output_for_push(&manifests, &data_buckets, merged_blobs, &local_side.manifest_records, !fallback_manifest_ids.is_empty());
- Ok(PullAndMergeOutcome::Merged {
- encrypted_vault,
- revision: opened.personal_revision,
- email_routing,
- stats,
- fallback_manifest_ids,
- dropped_local_manifest_ids: merge_output.dropped_local_manifest_ids,
- manifest_revisions: opened.manifest_revisions.clone(),
- bucket_revisions: opened.bucket_revisions.clone(),
- push_canonical,
- })
+ Ok(PullAndMergeOutcome::Merged { pulled: opened.pulled_vault(encrypted_vault, email_routing), stats, fallback_manifest_ids, dropped_local_manifest_ids: merge_output.dropped_local_manifest_ids, push_canonical })
}
/// The merged vault in the shape the push writes from.
@@ -257,7 +233,7 @@ async fn resolve_merged_blob_refs(ctx: &Ctx, manifests: &[Manifest], blob_map: &
if let Some(local) = local_blobs.get(reference) {
blobs.insert(reference.to_string(), local.clone());
} else if let Some(kind) = kind {
- blobs.insert(reference.to_string(), BlobEntry { kind: kind.to_string(), bytes_base64: BASE64.encode(bytes) });
+ blobs.insert(reference.to_string(), BlobEntry { kind: kind.to_string(), bytes_base64: crate::encoding::base64_encode(bytes) });
} else {
complete = false;
}
diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rs
index 791fd52cc..760de4c6c 100644
--- a/core/rust/src/vault_sync/pull.rs
+++ b/core/rust/src/vault_sync/pull.rs
@@ -3,22 +3,16 @@
use std::collections::HashMap;
-use super::db::ids_equal;
+use crate::vault_model::ids_equal;
use super::errors::{SyncError, SyncResult};
use super::state::{self, Ctx};
use super::types::{self, BlobDto, BlobHashesRequest, Db, EmailRoutingDto, GetResponse, ManifestDto, SharedManifestRecord, StoredBlobRef};
-use super::{db, http, keys};
+use super::{db, http, keys, legacy};
use crate::crypto;
use crate::vault_codec::{self, DataBucket, Manifest, MaterializeInput};
-const VAULT_ENDPOINT: &str = "Vault";
const BLOBS_DOWNLOAD_ENDPOINT: &str = "Vault/blobs/download";
-/// Max base64 characters in one blob transfer request or response body.
-pub(crate) const BLOB_TRANSFER_BATCH_MAX_CHARS: usize = 4 * 1024 * 1024;
-/// Upper bound on the number of blobs in one transfer batch.
-pub(crate) const BLOB_TRANSFER_BATCH_MAX_COUNT: usize = 100;
-
/// One manifest of a pull, opened.
pub(crate) struct ResolvedManifest {
pub manifest_id: String,
@@ -45,74 +39,31 @@ impl OpenedManifestSet {
pub fn manifests(&self) -> Vec<Manifest> {
self.resolved.iter().map(|m| m.manifest.clone()).collect()
}
-}
-
-/// A not-yet-migrated server's vault, passed through as served.
-/// TODO: remove all legacy related code once all users have migrated to the manifest-v1 storage model.
-pub(crate) struct LegacyPassthrough {
- pub encrypted_blob: String,
- pub revision: i64,
-}
-
-/// What a pull produced.
-pub(crate) enum PullOutcome {
- Legacy(LegacyPassthrough, EmailRoutingDto),
- Materialized { encrypted_vault: String, revision: i64, email_routing: EmailRoutingDto },
-}
-/// What the last snapshot this run fetched looked like.
-pub(crate) struct SnapshotFacts {
- pub served_manifest_ids: Vec<String>,
- /// Whether the snapshot is still on the legacy sqlite-blob format.
- pub legacy: bool,
-}
-
-impl SnapshotFacts {
- pub fn new() -> Self {
- Self { served_manifest_ids: Vec::new(), legacy: false }
- }
-
- fn record(&mut self, snapshot: &GetResponse) {
- self.served_manifest_ids = snapshot.manifests.iter().map(|m| m.manifest_id.clone()).collect();
- self.legacy = is_legacy_sqlite_blob_snapshot(snapshot);
+ /// The opened set as a vault ready to become the local one.
+ pub fn pulled_vault(&self, encrypted_vault: String, email_routing: EmailRoutingDto) -> PulledVault {
+ PulledVault { encrypted_vault, revision: self.personal_revision, email_routing, manifest_revisions: self.manifest_revisions.clone(), bucket_revisions: self.bucket_revisions.clone() }
}
}
-/// `GET v2/Vault`.
-pub(crate) async fn fetch_snapshot(ctx: &Ctx, facts: &mut SnapshotFacts) -> SyncResult<GetResponse> {
- let snapshot = http::with_outdated_server_guard(http::get::<GetResponse>(&ctx.host, VAULT_ENDPOINT, true).await)?;
- facts.record(&snapshot);
- Ok(snapshot)
+/// A pulled vault, encrypted for local storage, with the revisions that become the local truth once it is stored.
+pub(crate) struct PulledVault {
+ pub encrypted_vault: String,
+ pub revision: i64,
+ pub email_routing: EmailRoutingDto,
+ pub manifest_revisions: HashMap<String, i64>,
+ pub bucket_revisions: HashMap<String, i64>,
}
-pub(crate) fn is_legacy_sqlite_blob_snapshot(snapshot: &GetResponse) -> bool {
- snapshot.storage_format != Some(types::STORAGE_FORMAT_MANIFEST)
+/// `GET v2/Vault`.
+pub(crate) async fn fetch_snapshot(ctx: &Ctx) -> SyncResult<GetResponse> {
+ http::with_outdated_server_guard(http::get::<GetResponse>(&ctx.host, http::VAULT_ENDPOINT, true).await)
}
pub(crate) fn email_routing_of(snapshot: &GetResponse) -> EmailRoutingDto {
snapshot.email_routing.clone().unwrap_or_default()
}
-/// Split items into request batches bounded by both transfer limits.
-pub(crate) fn batch_by_transfer_cost<T>(items: Vec<T>, cost_of: impl Fn(&T) -> usize) -> Vec<Vec<T>> {
- let mut batches = Vec::new();
- let mut batch = Vec::new();
- let mut chars = 0usize;
- for item in items {
- let cost = cost_of(&item);
- if !batch.is_empty() && (chars + cost > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.len() >= BLOB_TRANSFER_BATCH_MAX_COUNT) {
- batches.push(std::mem::take(&mut batch));
- chars = 0;
- }
- batch.push(item);
- chars += cost;
- }
- if !batch.is_empty() {
- batches.push(batch);
- }
- batches
-}
-
fn base64_chars(size_bytes: i64) -> usize {
((size_bytes.max(0) as usize).div_ceil(3)) * 4
}
@@ -125,7 +76,7 @@ fn verify_decrypt_unpack(base64_ciphertext: &str, vek: &str, expected_ciphertext
}
}
let unreadable = |e: crate::error::VaultError| SyncError::ServerVaultUnreadable(format!("{}: {}", label, e));
- let encrypted = crypto::aes_gcm::decode_base64(base64_ciphertext).map_err(unreadable)?;
+ let encrypted = crate::encoding::base64_decode(base64_ciphertext).map_err(unreadable)?;
let plain = crypto::symmetric_decrypt_bytes(&encrypted, vek).map_err(unreadable)?;
vault_codec::unpack_payload(&plain).map_err(unreadable)
}
@@ -143,43 +94,24 @@ fn select_personal_manifest(snapshot: &GetResponse) -> Option<&ManifestDto> {
snapshot.manifests.iter().find(|m| m.manifest_id == personal_id)
}
-/// Take a legacy snapshot apart for local storage: the blob passes through untouched, the manifest-v1 state
-/// is reset, and the personal manifest id is recorded for the migration push.
-pub(crate) async fn open_legacy_snapshot(ctx: &Ctx, snapshot: &GetResponse) -> SyncResult<LegacyPassthrough> {
- state::remove(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?;
- let revision = snapshot.legacy_revision.unwrap_or(0);
- if let Some(personal) = &snapshot.personal_manifest_id {
- state::set(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID, personal).await?;
- let mut revisions = HashMap::new();
- revisions.insert(personal.clone(), revision);
- state::set(&ctx.host, state::SERVER_MANIFEST_REVISIONS, &revisions).await?;
- }
- ctx.log("[V2Pull] Legacy sqlite-blob pass-through (user not yet migrated), returning the blob as-is.").await;
- Ok(LegacyPassthrough { encrypted_blob: snapshot.legacy_vault_blob.clone().unwrap_or_default(), revision })
-}
-
/// Pull the latest vault: fetch, open, materialize and re-encrypt for local storage.
-pub(crate) async fn pull(ctx: &mut Ctx, facts: &mut SnapshotFacts) -> SyncResult<PullOutcome> {
+pub(crate) async fn pull(ctx: &mut Ctx) -> SyncResult<PulledVault> {
let vek = ctx.encryption_key()?;
ctx.log("[V2Pull] Step 1/4: fetching vault snapshot (GET /v2/Vault)...").await;
- let snapshot = fetch_snapshot(ctx, facts).await?;
- let email_routing = email_routing_of(&snapshot);
-
- if facts.legacy {
- return Ok(PullOutcome::Legacy(open_legacy_snapshot(ctx, &snapshot).await?, email_routing));
+ let snapshot = fetch_snapshot(ctx).await?;
+ if legacy::is_legacy_sqlite_blob_snapshot(&snapshot) {
+ return legacy::open_legacy_snapshot(ctx, &snapshot).await;
}
ctx.log("[V2Pull] Step 2/4: manifest format: decrypting and reassembling local SQLite...").await;
let opened = open_manifests_and_record_sync_state(ctx, &snapshot, &vek).await?;
- commit_revisions(ctx, &opened.manifest_revisions, &opened.bucket_revisions).await?;
let sqlite_bytes = materialize_to_sqlite(ctx, &opened.manifests(), &opened.data_buckets, &opened.blob_map).await?;
ctx.log(format!("[V2Pull] Step 3/4: materialized SQLite ({} bytes); re-encrypting for local storage...", sqlite_bytes.len())).await;
- let encrypted_vault = state::encrypt_vault_blob(&sqlite_bytes, &vek)?;
- Ok(PullOutcome::Materialized { encrypted_vault, revision: opened.personal_revision, email_routing })
+ Ok(opened.pulled_vault(state::encrypt_vault_blob(&sqlite_bytes, &vek)?, email_routing_of(&snapshot)))
}
/// Open every manifest a snapshot carries, personal first, and record the snapshot as this device's sync state.
-/// The revision maps are returned for the caller to commit once the pulled state is local truth.
+/// The revision maps travel on the result for the caller to commit once the pulled vault is stored.
pub(crate) async fn open_manifests_and_record_sync_state(ctx: &mut Ctx, snapshot: &GetResponse, vek: &str) -> SyncResult<OpenedManifestSet> {
let personal_dto = select_personal_manifest(snapshot).ok_or_else(|| SyncError::Snapshot("server returned no personal manifest, refusing to assemble".to_string()))?;
if personal_dto.blob.as_deref().unwrap_or("").is_empty() {
@@ -300,7 +232,7 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], fal
let missing: Vec<StoredBlobRef> = refs.iter().filter(|r| !cache.contains_key(&r.hash)).cloned().collect();
ctx.log(format!("[V2Pull] Blob refs: {} referenced, {} cached locally, {} to download.", refs.len(), refs.len() - missing.len(), missing.len())).await;
- let batches = batch_by_transfer_cost(missing, |r| base64_chars(r.size_bytes));
+ let batches = http::batch_by_transfer_cost(missing, |r| base64_chars(r.size_bytes));
let batch_count = batches.len();
for (index, chunk) in batches.into_iter().enumerate() {
let blobs: Vec<BlobDto> = http::post(&ctx.host, BLOBS_DOWNLOAD_ENDPOINT, &BlobHashesRequest { hashes: chunk.iter().map(|r| r.hash.clone()).collect() }, true).await?;
@@ -318,7 +250,7 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], fal
continue;
};
let key = owners.get(&reference.hash).map(|o| o.vek.as_str()).unwrap_or(fallback_vek);
- match crypto::aes_gcm::decode_base64(ciphertext).and_then(|bytes| crypto::symmetric_decrypt_bytes(&bytes, key)) {
+ match crate::encoding::base64_decode(ciphertext).and_then(|bytes| crypto::symmetric_decrypt_bytes(&bytes, key)) {
Ok(bytes) => {
blob_map.insert(reference.hash.clone(), bytes);
pruned_cache.insert(reference.hash.clone(), ciphertext.clone());
diff --git a/core/rust/src/vault_sync/push.rs b/core/rust/src/vault_sync/push.rs
index ef5b95640..1414b3f38 100644
--- a/core/rust/src/vault_sync/push.rs
+++ b/core/rust/src/vault_sync/push.rs
@@ -3,18 +3,16 @@
use std::collections::{HashMap, HashSet};
-use super::db::{id_key, ids_equal};
+use crate::vault_model::{id_key, ids_equal};
use super::email_routing::build_email_routing;
use super::errors::{SyncError, SyncResult};
-use super::pull::{BLOB_TRANSFER_BATCH_MAX_CHARS, BLOB_TRANSFER_BATCH_MAX_COUNT};
use super::state::{self, Ctx};
-use super::types::{BlobDto, BlobHashesRequest, BlobRef, BlobUploadRequest, BucketWrite, Db, ManifestWrite, MissingBlobsResponse, VaultWriteRequest, VaultWriteResponse};
+use super::types::{BlobDto, BlobHashesRequest, BlobRef, BlobUploadRequest, BucketRevision, BucketWrite, Db, ManifestRevision, ManifestWrite, MissingBlobsResponse, VaultWriteRequest, VaultWriteResponse};
use super::{db, http, keys};
use crate::crypto;
use crate::vault_codec::{self, BlobEntry, CanonicalizeInput, CanonicalizedVault, DataBucket, Manifest, ManifestSpec};
use crate::vault_sharing::{self, ManifestAccessRequest, ManifestWriteSetRequest, SharedManifestRecord as SharingRecord};
-const VAULT_ENDPOINT: &str = "Vault";
const BLOBS_ENDPOINT: &str = "Vault/blobs";
const BLOBS_MISSING_ENDPOINT: &str = "Vault/blobs/missing";
@@ -120,17 +118,17 @@ pub(crate) async fn resolve_personal_manifest_id(ctx: &Ctx) -> SyncResult<String
/// result too, so a push right after does not canonicalize a second time.
pub(crate) async fn detect_no_op_mutation(ctx: &Ctx) -> SyncResult<(bool, CanonicalizedSet)> {
let set = canonicalize_vault(ctx, None).await?;
- let fingerprints: HashMap<String, String> = state::get(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?.unwrap_or_default();
+ let baselines = PushBaselines::load(ctx).await?;
let writable: HashSet<String> = set.manifest_records.iter().map(|r| id_key(&r.manifest_id)).collect();
for entry in set.canonicalized.manifests.iter().filter(|m| writable.contains(&id_key(&m.manifest.manifest_id))) {
- let fingerprint = vault_codec::compute_content_fingerprint(&serde_json::to_string(&entry.manifest)?);
- if fingerprints.get(&state::fingerprint_manifest_key(&entry.manifest.manifest_id)) != Some(&fingerprint) {
+ let (_, fingerprint) = fingerprinted(&entry.manifest)?;
+ if !baselines.unchanged(&state::fingerprint_manifest_key(&entry.manifest.manifest_id), &fingerprint) {
return Ok((false, set));
}
}
for bucket in &set.canonicalized.data_buckets {
- let fingerprint = vault_codec::compute_content_fingerprint(&serde_json::to_string(bucket)?);
- if fingerprints.get(&state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category)) != Some(&fingerprint) {
+ let (_, fingerprint) = fingerprinted(bucket)?;
+ if !baselines.unchanged(&state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category), &fingerprint) {
return Ok((false, set));
}
}
@@ -290,6 +288,15 @@ impl PushBaselines {
known_server_hashes: state::get::<Vec<String>>(&ctx.host, state::VAULT_SERVER_BLOB_HASHES).await?.unwrap_or_default().into_iter().collect(),
})
}
+
+ /// Whether the content behind a fingerprint key still matches the last-known server state.
+ fn unchanged(&self, fingerprint_key: &str, fingerprint: &str) -> bool {
+ self.fingerprints.get(fingerprint_key).map(String::as_str) == Some(fingerprint)
+ }
+
+ fn bucket_revision(&self, bucket: &DataBucket) -> i64 {
+ self.bucket_revisions.get(&state::bucket_revision_key(&bucket.manifest_id, &bucket.category)).copied().unwrap_or(0)
+ }
}
/// What decides whether an unchanged element still goes into the write.
@@ -350,12 +357,8 @@ struct LegacyAccountKeyMigration {
account_private_key: String,
}
-/// The change fingerprints of everything one write carried, to become the new baselines on success.
-#[derive(Default)]
-struct WrittenFingerprints {
- manifests: HashMap<String, String>,
- buckets: HashMap<String, String>,
-}
+/// The content fingerprints one write carried, keyed like the baselines, to become the new baselines on success.
+type WrittenFingerprints = HashMap<String, String>;
/// A byte or character count for log lines.
fn format_kb(chars: usize) -> String {
@@ -366,10 +369,26 @@ fn format_kb(chars: usize) -> String {
}
}
-/// Pack-then-encrypt a JSON payload. Returns the base64 ciphertext plus the packed size, for logging.
-fn pack_encrypt(payload_json: &str, vek: &str) -> SyncResult<(String, usize)> {
- let packed = vault_codec::pack_payload(payload_json)?;
- Ok((crypto::symmetric_encrypt_bytes(&packed, vek)?, packed.len()))
+/// A payload as JSON text plus its content fingerprint.
+fn fingerprinted<T: serde::Serialize>(payload: &T) -> SyncResult<(String, String)> {
+ let plaintext = serde_json::to_string(payload)?;
+ let fingerprint = vault_codec::compute_content_fingerprint(&plaintext);
+ Ok((plaintext, fingerprint))
+}
+
+/// A payload packed and encrypted for the write, with the hash the server verifies it by.
+struct Sealed {
+ ciphertext: String,
+ hash: String,
+}
+
+/// Pack and encrypt a JSON payload under `key`, logging its size at every stage.
+async fn seal(ctx: &Ctx, label: &str, plaintext: &str, key: &str) -> SyncResult<Sealed> {
+ let packed = vault_codec::pack_payload(plaintext)?;
+ let ciphertext = crypto::symmetric_encrypt_bytes(&packed, key)?;
+ ctx.log(format!("[V2Push] {}: raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(packed.len()), format_kb(ciphertext.len()))).await;
+ let hash = vault_codec::compute_ciphertext_hash(&ciphertext);
+ Ok(Sealed { ciphertext, hash })
}
/// Canonicalize, gate by content fingerprint, encrypt and `POST v2/Vault`. Returns the outcome and, on a KEK/VEK
@@ -426,7 +445,8 @@ async fn push_internal(ctx: &mut Ctx, cached: Option<CanonicalizedSet>, create_v
return Ok((PushStatus::Outdated, None));
}
- commit_push_baselines(ctx, baselines, &response, written, &blobs, &uploaded).await?;
+ commit_push_baselines(ctx, baselines, &response.manifest_revisions, &response.bucket_revisions, written).await?;
+ commit_blob_baselines(ctx, &blobs, &uploaded).await?;
if let Some(migration) = &migration {
complete_account_key_migration(ctx, migration).await?;
ctx.log("[V2Push] Account-key migration complete: hierarchy created server-side, blob chain cached locally.").await;
@@ -470,7 +490,7 @@ fn collect_upload_blobs(candidates: &[Candidate]) -> SyncResult<UploadBlobs> {
continue;
}
blobs.order.push(hash.clone());
- blobs.entries.insert(hash.clone(), UploadBlobEntry { bytes: crypto::aes_gcm::decode_base64(&blob.bytes_base64)?, kind: blob.kind.clone(), vek: candidate.vek.clone(), from_personal: candidate.record.is_personal });
+ blobs.entries.insert(hash.clone(), UploadBlobEntry { bytes: crate::encoding::base64_decode(&blob.bytes_base64)?, kind: blob.kind.clone(), vek: candidate.vek.clone(), from_personal: candidate.record.is_personal });
}
}
Ok(blobs)
@@ -483,9 +503,8 @@ async fn encrypt_changed_buckets(ctx: &Ctx, buckets: &[DataBucket], candidates:
for bucket in buckets {
let label = format!("Data bucket \"{}\" of manifest {}", bucket.category, bucket.manifest_id);
let fingerprint_key = state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category);
- let plaintext = serde_json::to_string(bucket)?;
- let fingerprint = vault_codec::compute_content_fingerprint(&plaintext);
- if !gate.force_full_write && !gate.migrating && baselines.fingerprints.get(&fingerprint_key) == Some(&fingerprint) {
+ let (plaintext, fingerprint) = fingerprinted(bucket)?;
+ if !gate.force_full_write && !gate.migrating && baselines.unchanged(&fingerprint_key, &fingerprint) {
ctx.log(format!("[V2Push] {} unchanged versus server baseline, leaving it out of this write.", label)).await;
continue;
}
@@ -497,12 +516,9 @@ async fn encrypt_changed_buckets(ctx: &Ctx, buckets: &[DataBucket], candidates:
if !validation.ok {
return Err(SyncError::UploadRejected(vec![format!("{} validation failed: {}. {}", label, validation.failed_rules.join(", "), validation.message).trim().to_string()]));
}
- let (ciphertext, compressed) = pack_encrypt(&plaintext, bucket_key)?;
- let ciphertext_hash = vault_codec::compute_ciphertext_hash(&ciphertext);
- let current_revision = baselines.bucket_revisions.get(&state::bucket_revision_key(&bucket.manifest_id, &bucket.category)).copied().unwrap_or(0);
- ctx.log(format!("[V2Push] {}: raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(compressed), format_kb(ciphertext.len()))).await;
- writes.push(BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: ciphertext, ciphertext_hash, current_revision });
- written.buckets.insert(fingerprint_key, fingerprint);
+ let sealed = seal(ctx, &label, &plaintext, bucket_key).await?;
+ writes.push(BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: sealed.ciphertext, ciphertext_hash: sealed.hash, current_revision: baselines.bucket_revision(bucket) });
+ written.insert(fingerprint_key, fingerprint);
}
Ok(writes)
}
@@ -512,10 +528,10 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], base
let mut writes = Vec::new();
for candidate in candidates {
let label = candidate.label();
- let plaintext = serde_json::to_string(candidate.manifest)?;
- let fingerprint = vault_codec::compute_content_fingerprint(&plaintext);
+ let fingerprint_key = state::fingerprint_manifest_key(&candidate.record.manifest_id);
+ let (plaintext, fingerprint) = fingerprinted(candidate.manifest)?;
let rekeyed = gate.migrating && candidate.record.is_personal;
- if !gate.force_full_write && !rekeyed && baselines.fingerprints.get(&state::fingerprint_manifest_key(&candidate.record.manifest_id)) == Some(&fingerprint) {
+ if !gate.force_full_write && !rekeyed && baselines.unchanged(&fingerprint_key, &fingerprint) {
ctx.log(format!("[V2Push] {} unchanged versus server baseline, leaving it out of this write.", label)).await;
continue;
}
@@ -527,9 +543,7 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], base
ctx.warn(format!("[V2Push] {} failed validation ({}), dropping it from this write.", label, validation.failed_rules.join(", "))).await;
continue;
}
- let (ciphertext, compressed) = pack_encrypt(&plaintext, &candidate.vek)?;
- let ciphertext_hash = vault_codec::compute_ciphertext_hash(&ciphertext);
- ctx.log(format!("[V2Push] {}: raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(compressed), format_kb(ciphertext.len()))).await;
+ let sealed = seal(ctx, &label, &plaintext, &candidate.vek).await?;
// Publish the public half of this manifest's mail delivery keypair; only admins may publish a shared one.
let may_publish = candidate.record.is_personal || candidate.record.can_administer;
@@ -542,14 +556,14 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], base
blob_refs.sort_by(|a, b| a.hash.cmp(&b.hash));
writes.push(ManifestWrite {
manifest_id: candidate.record.manifest_id.clone(),
- manifest_blob: ciphertext,
- manifest_ciphertext_hash: ciphertext_hash,
+ manifest_blob: sealed.ciphertext,
+ manifest_ciphertext_hash: sealed.hash,
current_revision: candidate.current_revision,
credentials_count: candidate.manifest.tables.get("Items").map(Vec::len).unwrap_or(0),
blob_references: blob_refs,
encryption_public_key: manifest_key.and_then(|row| row.get("PublicKey").and_then(serde_json::Value::as_str).map(str::to_string)),
});
- written.manifests.insert(candidate.record.manifest_id.clone(), fingerprint);
+ written.insert(fingerprint_key, fingerprint);
}
Ok(writes)
}
@@ -585,7 +599,7 @@ async fn missing_on_server(ctx: &Ctx, hashes: Vec<String>) -> SyncResult<Vec<Str
/// `POST v2/Vault`. When the server reports blobs it lacks (stale local knowledge of its blob set), upload
/// them and retry the identical write once; blobs this client cannot supply fail the push.
async fn write_vault(ctx: &Ctx, payload: &VaultWriteRequest, blobs: &UploadBlobs, gate: WriteGate, uploaded: &mut HashMap<String, String>) -> SyncResult<VaultWriteResponse> {
- let response: VaultWriteResponse = http::post(&ctx.host, VAULT_ENDPOINT, payload, true).await?;
+ let response: VaultWriteResponse = http::post(&ctx.host, http::VAULT_ENDPOINT, payload, true).await?;
if response.missing_blob_hashes.is_empty() {
return Ok(response);
}
@@ -597,35 +611,35 @@ async fn write_vault(ctx: &Ctx, payload: &VaultWriteRequest, blobs: &UploadBlobs
let (missing_personal, missing_shared): (Vec<String>, Vec<String>) = response.missing_blob_hashes.iter().cloned().partition(|h| blobs.entries[h].from_personal);
uploaded.extend(upload_blobs(ctx, &blobs.entries, &missing_personal, gate.migrating).await?);
uploaded.extend(upload_blobs(ctx, &blobs.entries, &missing_shared, false).await?);
- let response: VaultWriteResponse = http::post(&ctx.host, VAULT_ENDPOINT, payload, true).await?;
+ let response: VaultWriteResponse = http::post(&ctx.host, http::VAULT_ENDPOINT, payload, true).await?;
if !response.missing_blob_hashes.is_empty() {
return Err(SyncError::MissingBlobs(response.missing_blob_hashes));
}
Ok(response)
}
-/// Advance the local baselines of everything a successful write carried, and refresh the encrypted blob cache.
-async fn commit_push_baselines(ctx: &Ctx, baselines: PushBaselines, response: &VaultWriteResponse, written: WrittenFingerprints, blobs: &UploadBlobs, uploaded: &HashMap<String, String>) -> SyncResult<()> {
- let PushBaselines { mut fingerprints, mut manifest_revisions, mut bucket_revisions, .. } = baselines;
- if !response.bucket_revisions.is_empty() {
- for br in &response.bucket_revisions {
- bucket_revisions.insert(state::bucket_revision_key(&br.manifest_id, &br.category), br.revision);
+/// Advance the revision and fingerprint baselines of everything a successful write carried.
+async fn commit_push_baselines(ctx: &Ctx, baselines: PushBaselines, manifest_revisions: &[ManifestRevision], bucket_revisions: &[BucketRevision], written: WrittenFingerprints) -> SyncResult<()> {
+ let PushBaselines { mut fingerprints, manifest_revisions: mut known_manifests, bucket_revisions: mut known_buckets, .. } = baselines;
+ if !bucket_revisions.is_empty() {
+ for br in bucket_revisions {
+ known_buckets.insert(state::bucket_revision_key(&br.manifest_id, &br.category), br.revision);
}
- state::set(&ctx.host, state::VAULT_BUCKET_REVISIONS, &bucket_revisions).await?;
+ state::set(&ctx.host, state::VAULT_BUCKET_REVISIONS, &known_buckets).await?;
}
- for mr in &response.manifest_revisions {
- manifest_revisions.insert(mr.manifest_id.clone(), mr.revision);
+ if !manifest_revisions.is_empty() {
+ for mr in manifest_revisions {
+ known_manifests.insert(mr.manifest_id.clone(), mr.revision);
+ }
+ state::set(&ctx.host, state::SERVER_MANIFEST_REVISIONS, &known_manifests).await?;
}
- state::set(&ctx.host, state::SERVER_MANIFEST_REVISIONS, &manifest_revisions).await?;
+ fingerprints.extend(written);
+ state::set(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS, &fingerprints).await
+}
- fingerprints.extend(written.buckets);
- for (manifest_id, fingerprint) in written.manifests {
- fingerprints.insert(state::fingerprint_manifest_key(&manifest_id), fingerprint);
- }
- state::set(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS, &fingerprints).await?;
+/// Record the server's blob set as this write left it, and keep the encrypted blob cache to what is still referenced.
+async fn commit_blob_baselines(ctx: &Ctx, blobs: &UploadBlobs, uploaded: &HashMap<String, String>) -> SyncResult<()> {
state::set(&ctx.host, state::VAULT_SERVER_BLOB_HASHES, &blobs.order).await?;
-
- // The encrypted blob cache: entries still referenced, plus the ciphertexts just uploaded.
let cache: HashMap<String, String> = state::get(&ctx.host, state::VAULT_BLOB_CIPHER_CACHE).await?.unwrap_or_default();
let mut new_cache: HashMap<String, String> = HashMap::new();
for hash in &blobs.order {
@@ -644,7 +658,7 @@ async fn complete_account_key_migration(ctx: &mut Ctx, migration: &LegacyAccount
state::set(&ctx.host, state::ACCOUNT_PUBLIC_KEY, &blobs.account_public_key).await?;
state::set(&ctx.host, state::ENCRYPTED_ACCOUNT_PRIVATE_KEY, &blobs.encrypted_account_private_key).await?;
ctx.account_public_key = Some(blobs.account_public_key.clone());
- ctx.set_account_private_key(Some(migration.account_private_key.clone()));
+ ctx.set_account_private_key(migration.account_private_key.clone());
Ok(())
}
@@ -652,25 +666,16 @@ async fn complete_account_key_migration(ctx: &mut Ctx, migration: &LegacyAccount
/// ciphertext for the local encrypted blob cache.
async fn upload_blobs(ctx: &Ctx, entries: &HashMap<String, UploadBlobEntry>, hashes: &[String], overwrite: bool) -> SyncResult<HashMap<String, String>> {
let mut ciphertexts = HashMap::new();
- if hashes.is_empty() {
- return Ok(ciphertexts);
- }
- let mut batch: Vec<BlobDto> = Vec::new();
- let mut batch_chars = 0usize;
+ let mut dtos = Vec::new();
for hash in hashes {
let Some(entry) = entries.get(hash) else { continue };
let ciphertext = crypto::symmetric_encrypt_bytes(&entry.bytes, &entry.vek)?;
ciphertexts.insert(hash.clone(), ciphertext.clone());
- if !batch.is_empty() && (batch_chars + ciphertext.len() > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.len() >= BLOB_TRANSFER_BATCH_MAX_COUNT) {
- ctx.log(format!("[V2Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(batch_chars))).await;
- http::post_no_content(&ctx.host, BLOBS_ENDPOINT, &BlobUploadRequest { blobs: std::mem::take(&mut batch), overwrite }).await?;
- batch_chars = 0;
- }
- batch_chars += ciphertext.len();
- batch.push(BlobDto { hash: hash.clone(), category: entry.kind.clone(), encrypted_data_base64: ciphertext });
+ dtos.push(BlobDto { hash: hash.clone(), category: entry.kind.clone(), encrypted_data_base64: ciphertext });
}
- if !batch.is_empty() {
- ctx.log(format!("[V2Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(batch_chars))).await;
+ for batch in http::batch_by_transfer_cost(dtos, |dto| dto.encrypted_data_base64.len()) {
+ let chars: usize = batch.iter().map(|dto| dto.encrypted_data_base64.len()).sum();
+ ctx.log(format!("[V2Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(chars))).await;
http::post_no_content(&ctx.host, BLOBS_ENDPOINT, &BlobUploadRequest { blobs: batch, overwrite }).await?;
}
Ok(ciphertexts)
@@ -686,37 +691,30 @@ async fn push_data_bucket_only(ctx: &Ctx, bucket: &DataBucket, vek: &str) -> Syn
}
async fn push_data_bucket_only_internal(ctx: &Ctx, bucket: &DataBucket, vek: &str) -> SyncResult<(PushStatus, i64)> {
- let label = format!("Bucket \"{}\" of manifest {}", bucket.category, bucket.manifest_id);
- let revision_key = state::bucket_revision_key(&bucket.manifest_id, &bucket.category);
- let plaintext = serde_json::to_string(bucket)?;
-
- let mut fingerprints: HashMap<String, String> = state::get(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?.unwrap_or_default();
- let mut bucket_revisions: HashMap<String, i64> = state::get(&ctx.host, state::VAULT_BUCKET_REVISIONS).await?.unwrap_or_default();
- let fingerprint = vault_codec::compute_content_fingerprint(&plaintext);
+ let label = format!("Bucket \"{}\" of manifest {} (bucket-only)", bucket.category, bucket.manifest_id);
let fingerprint_key = state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category);
- if fingerprints.get(&fingerprint_key) == Some(&fingerprint) {
- ctx.log(format!("[V2Push] {} (bucket-only) unchanged versus server baseline, skipping upload.", label)).await;
- return Ok((PushStatus::Ok, bucket_revisions.get(&revision_key).copied().unwrap_or(0)));
+ let baselines = PushBaselines::load(ctx).await?;
+ let (plaintext, fingerprint) = fingerprinted(bucket)?;
+ if baselines.unchanged(&fingerprint_key, &fingerprint) {
+ ctx.log(format!("[V2Push] {} unchanged versus server baseline, skipping upload.", label)).await;
+ return Ok((PushStatus::Ok, baselines.bucket_revision(bucket)));
}
-
- let (ciphertext, compressed) = pack_encrypt(&plaintext, vek)?;
- let ciphertext_hash = vault_codec::compute_ciphertext_hash(&ciphertext);
- ctx.log(format!("[V2Push] {} (bucket-only): raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(compressed), format_kb(ciphertext.len()))).await;
+ let sealed = seal(ctx, &label, &plaintext, vek).await?;
let post = |current_revision: i64| {
let payload = VaultWriteRequest {
username: ctx.request.username.clone(),
manifests: Vec::new(),
- buckets: vec![BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: ciphertext.clone(), ciphertext_hash: ciphertext_hash.clone(), current_revision }],
+ buckets: vec![BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: sealed.ciphertext.clone(), ciphertext_hash: sealed.hash.clone(), current_revision }],
new_blobs: Vec::new(),
email_routing: None,
account_keys: None,
};
- async move { http::post::<_, VaultWriteResponse>(&ctx.host, VAULT_ENDPOINT, &payload, true).await }
+ async move { http::post::<_, VaultWriteResponse>(&ctx.host, http::VAULT_ENDPOINT, &payload, true).await }
};
let reported = |response: &VaultWriteResponse| response.bucket_revisions.iter().find(|b| ids_equal(&b.manifest_id, &bucket.manifest_id) && b.category == bucket.category).map(|b| b.revision);
- let mut current_revision = bucket_revisions.get(&revision_key).copied().unwrap_or(0);
+ let mut current_revision = baselines.bucket_revision(bucket);
let mut response = post(current_revision).await?;
if response.status != 0 {
let server_revision = reported(&response).unwrap_or(current_revision);
@@ -729,9 +727,7 @@ async fn push_data_bucket_only_internal(ctx: &Ctx, bucket: &DataBucket, vek: &st
}
let new_revision = reported(&response).unwrap_or(current_revision + 1);
- bucket_revisions.insert(revision_key, new_revision);
- state::set(&ctx.host, state::VAULT_BUCKET_REVISIONS, &bucket_revisions).await?;
- fingerprints.insert(fingerprint_key, fingerprint);
- state::set(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS, &fingerprints).await?;
+ let advanced = BucketRevision { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), revision: new_revision };
+ commit_push_baselines(ctx, baselines, &[], &[advanced], HashMap::from([(fingerprint_key, fingerprint)])).await?;
Ok((PushStatus::Ok, new_revision))
}
diff --git a/core/rust/src/vault_sync/state.rs b/core/rust/src/vault_sync/state.rs
index 7ae649dac..da5fa5c04 100644
--- a/core/rust/src/vault_sync/state.rs
+++ b/core/rust/src/vault_sync/state.rs
@@ -70,12 +70,10 @@ impl Ctx {
self.encryption_key = Some(key);
}
- pub fn set_account_private_key(&mut self, key: Option<String>) {
- match &key {
- Some(value) => self.updates.account_private_key = Some(value.clone()),
- None => self.updates.clear_account_private_key = true,
- }
- self.account_private_key = key;
+ /// Adopt the account private key a migration push minted, reporting it to the host.
+ pub fn set_account_private_key(&mut self, key: String) {
+ self.updates.account_private_key = Some(key.clone());
+ self.account_private_key = Some(key);
}
/// The current schema.
@@ -177,14 +175,14 @@ const SQLITE_HEADER: &[u8] = b"SQLite format 3\0";
/// Decrypt a stored vault blob into the plaintext SQLite database.
pub(crate) fn decrypt_vault_blob(encrypted_blob: &str, key: &str) -> SyncResult<Vec<u8>> {
- let ciphertext = crypto::aes_gcm::decode_base64(encrypted_blob)?;
+ let ciphertext = crate::encoding::base64_decode(encrypted_blob)?;
let plaintext = crypto::symmetric_decrypt_bytes(&ciphertext, key).map_err(|e| SyncError::VaultDecryptFailed(e.to_string()))?;
if plaintext.starts_with(SQLITE_HEADER) {
return Ok(plaintext);
}
let not_a_database = || SyncError::VaultDecryptFailed("plaintext is neither a database nor base64 text".to_string());
let text = String::from_utf8(plaintext).map_err(|_| not_a_database())?;
- crypto::aes_gcm::decode_base64(text.trim()).map_err(|_| not_a_database())
+ crate::encoding::base64_decode(text.trim()).map_err(|_| not_a_database())
}
/// Encrypt a plaintext SQLite database for local storage.
diff --git a/core/rust/src/vault_sync/types.rs b/core/rust/src/vault_sync/types.rs
index 82c5a8acc..4ce3a4262 100644
--- a/core/rust/src/vault_sync/types.rs
+++ b/core/rust/src/vault_sync/types.rs
@@ -6,7 +6,7 @@ use std::fmt;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
-use super::errors::{ErrorCode, LogoutReason};
+use super::errors::{ErrorCode, Failure, LogoutReason, SyncError};
/// The operation a session runs.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -60,9 +60,6 @@ pub struct SessionUpdates {
pub encryption_key: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub account_private_key: Option<String>,
- /// Set when the account private key must be dropped from the session.
- #[serde(default, skip_serializing_if = "std::ops::Not::not")]
- pub clear_account_private_key: bool,
}
/// What every operation reports on top of its own outcome.
@@ -81,6 +78,35 @@ pub(crate) trait OperationResult: Serialize {
fn session_mut(&mut self) -> &mut SessionOutcome;
}
+/// How a failed operation reports itself: a coded error the host translates, or a forced logout.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FailureFields {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub error: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub error_code: Option<ErrorCode>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub error_key: Option<LogoutReason>,
+ pub requires_logout: bool,
+}
+
+impl FailureFields {
+ /// A forced logout for the given reason.
+ pub fn logout(reason: LogoutReason) -> Self {
+ Self { error_key: Some(reason), requires_logout: true, ..Default::default() }
+ }
+}
+
+impl From<&SyncError> for FailureFields {
+ fn from(error: &SyncError) -> Self {
+ match error.failure() {
+ Failure::Logout(reason) => Self::logout(reason),
+ Failure::Coded(code) => Self { error: Some(error.to_string()), error_code: Some(code), ..Default::default() },
+ }
+ }
+}
+
/// Outcome of a full sync.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -90,13 +116,8 @@ pub struct FullSyncResult {
pub was_offline: bool,
pub sqlite_blob_upgrade_required: bool,
pub manifest_migration_required: bool,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub error: Option<String>,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub error_code: Option<ErrorCode>,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub error_key: Option<LogoutReason>,
- pub requires_logout: bool,
+ #[serde(flatten)]
+ pub failure: FailureFields,
#[serde(skip_serializing_if = "Option::is_none")]
pub server_version: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -151,13 +172,8 @@ pub struct MigrateManifestResult {
pub success: bool,
/// Whether the migrated vault reached the server; false leaves it dirty for the next sync.
pub pushed: bool,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub error: Option<String>,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub error_code: Option<ErrorCode>,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub error_key: Option<LogoutReason>,
- pub requires_logout: bool,
+ #[serde(flatten)]
+ pub failure: FailureFields,
#[serde(flatten)]
pub session: SessionOutcome,
}
@@ -176,13 +192,8 @@ pub struct StatusCheckResult {
pub has_newer_vault: bool,
pub has_dirty_changes: bool,
pub is_offline: bool,
- pub requires_logout: bool,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub error_key: Option<LogoutReason>,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub error: Option<String>,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub error_code: Option<ErrorCode>,
+ #[serde(flatten)]
+ pub failure: FailureFields,
#[serde(skip_serializing_if = "Option::is_none")]
pub server_version: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -211,16 +222,6 @@ pub enum HttpMethod {
Delete,
}
-impl fmt::Display for HttpMethod {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.write_str(match self {
- HttpMethod::Get => "GET",
- HttpMethod::Post => "POST",
- HttpMethod::Delete => "DELETE",
- })
- }
-}
-
/// The databases a command may address.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
@@ -492,7 +493,7 @@ pub struct PendingAction {
pub struct GetResponse {
#[serde(default)]
pub status: i32,
- /// 0 = sqlite-blob, 1 = manifest-v1; absent on servers predating the field.
+ /// LEGACY: 0 = sqlite-blob, 1 = manifest-v1; absent on servers predating the field.
#[serde(default)]
pub storage_format: Option<i32>,
#[serde(default)]
@@ -511,8 +512,6 @@ pub struct GetResponse {
pub email_routing: Option<EmailRoutingDto>,
}
-pub const STORAGE_FORMAT_MANIFEST: i32 = 1;
-
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ManifestDto {
@@ -716,3 +715,31 @@ pub struct SharedManifestRecord {
#[serde(default)]
pub can_administer: bool,
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// The failure fields flatten into every result exactly where the four loose fields used to sit.
+ #[test]
+ fn failure_fields_flatten_into_the_result_wire_shape() {
+ let coded = FailureFields::from(&SyncError::VaultLocked);
+ let full = serde_json::to_string(&FullSyncResult { failure: coded, ..Default::default() }).unwrap();
+ assert_eq!(full, r#"{"success":false,"hasNewVault":false,"wasOffline":false,"sqliteBlobUpgradeRequired":false,"manifestMigrationRequired":false,"error":"No encryption key available","errorCode":"E-202","requiresLogout":false,"isOfflineMode":false,"sessionUpdates":{},"vaultChanged":false}"#);
+ let round_trip: FullSyncResult = serde_json::from_str(&full).unwrap();
+ assert_eq!(serde_json::to_string(&round_trip).unwrap(), full);
+
+ let logout = FailureFields::logout(LogoutReason::SessionExpired);
+ let migrate = serde_json::to_string(&MigrateManifestResult { failure: logout.clone(), ..Default::default() }).unwrap();
+ assert_eq!(migrate, r#"{"success":false,"pushed":false,"errorKey":"sessionExpired","requiresLogout":true,"sessionUpdates":{},"vaultChanged":false}"#);
+ let round_trip: MigrateManifestResult = serde_json::from_str(&migrate).unwrap();
+ assert_eq!(round_trip.failure, logout);
+
+ let status = serde_json::to_value(StatusCheckResult { failure: logout.clone(), ..Default::default() }).unwrap();
+ assert_eq!(status["errorKey"], "sessionExpired");
+ assert_eq!(status["requiresLogout"], true);
+ assert!(status.get("error").is_none() && status.get("errorCode").is_none() && status.get("failure").is_none());
+ let round_trip: StatusCheckResult = serde_json::from_value(status.clone()).unwrap();
+ assert_eq!(serde_json::to_value(&round_trip).unwrap(), status);
+ }
+}
diff --git a/core/rust/src/vault_sync/version.rs b/core/rust/src/vault_sync/version.rs
index 15e4c67f8..79b355af6 100644
--- a/core/rust/src/vault_sync/version.rs
+++ b/core/rust/src/vault_sync/version.rs
@@ -1,4 +1,4 @@
-//! Version rules: server support, and the frozen legacy sqlite-blob vault version chain.
+//! Version rules: which server versions this client supports.
/// Whether `version1 >= version2` under the client's SemVer rules: a pre-release sorts below its release, and
/// two pre-releases compare lexically.
@@ -35,57 +35,6 @@ fn split(version: &str) -> (&str, Option<&str>) {
}
}
-/// The frozen sqlite-blob upgrade chain: (revision, data version). It ends at 2.0.0, the first schema compatible
-/// with manifest-v1; later schema changes ship through the full schema only which is used by every materilization directly.
-const LEGACY_VAULT_VERSIONS: &[(u32, &str)] = &[
- (1, "1.0.0"),
- (2, "1.0.1"),
- (3, "1.0.2"),
- (4, "1.1.0"),
- (5, "1.2.0"),
- (6, "1.3.0"),
- (7, "1.3.1"),
- (8, "1.4.0"),
- (9, "1.4.1"),
- (10, "1.5.0"),
- (11, "1.6.0"),
- (12, "1.7.0"),
- (13, "2.0.0"),
-];
-
-/// The revision of the last entry in the legacy chain.
-pub fn latest_legacy_revision() -> u32 {
- LEGACY_VAULT_VERSIONS.last().map(|(revision, _)| *revision).unwrap_or(0)
-}
-
-/// The data version embedded in an EF migration id (`20250101000000_2.0.0-Name`).
-pub fn extract_version_from_migration_id(migration_id: &str) -> Option<String> {
- let start = migration_id.find('_')? + 1;
- let rest = &migration_id[start..];
- let end = rest.find('-')?;
- let candidate = &rest[..end];
- let parts: Vec<&str> = candidate.split('.').collect();
- if parts.len() == 3 && parts.iter().all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit())) {
- Some(candidate.to_string())
- } else {
- None
- }
-}
-
-/// The legacy-chain revision a database version maps to. A known version maps to its entry; an unknown but
-/// well-formed version is treated as the latest (backwards compatible); a malformed version is incompatible.
-pub fn legacy_revision_for(database_version: &str) -> Result<u32, String> {
- if let Some((revision, _)) = LEGACY_VAULT_VERSIONS.iter().find(|(_, version)| *version == database_version) {
- return Ok(*revision);
- }
- let well_formed = database_version.split('.').count() == 3 && database_version.split('.').all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit()));
- if well_formed {
- Ok(latest_legacy_revision())
- } else {
- Err(format!("Vault version {} is not compatible with this client", database_version))
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
@@ -99,14 +48,4 @@ mod tests {
assert!(version_gte("1.0", "1.0.0"));
assert!(version_gte("0.12.0-beta", "0.12.0-alpha"));
}
-
- #[test]
- fn migration_ids_yield_their_version() {
- assert_eq!(extract_version_from_migration_id("20250101000000_2.0.0-Squashed").as_deref(), Some("2.0.0"));
- assert_eq!(extract_version_from_migration_id("20250101000000_Initial"), None);
- assert_eq!(legacy_revision_for("1.7.0"), Ok(12));
- assert_eq!(legacy_revision_for("2.0.0"), Ok(13));
- assert_eq!(legacy_revision_for("2.1.0"), Ok(13));
- assert!(legacy_revision_for("nope").is_err());
- }
}