Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
//! Merge: a dirty vault's pull, which merges the local vault onto the server snapshot at canonical level, one
//! manifest at a time, and materializes the result.
use std::collections::{HashMap, HashSet};
use secrecy::ExposeSecret;
use crate::vault_model::{id_key, ids_equal};
use super::errors::SyncResult;
use super::pull::{self, OpenedManifestSet, PulledVault, Snapshot};
use super::push::{canonicalize_vault, CanonicalizedSet};
use super::state::{self, Ctx};
use super::write_set::ManifestRecord;
use super::legacy;
use crate::vault_codec::{self, BlobEntry, CanonicalizedVault, DataBucket, Manifest};
use crate::vault_merge::{merge_canonical, CanonicalManifestMerge, CanonicalMergeInput, MergeStats};
/// What a dirty pull (pull and merge) produced.
pub(crate) enum PullAndMergeOutcome {
/// LEGACY: the server's vault is still a sqlite blob at this revision; the caller pushes the local vault over it.
/// TODO: remove once every account has upgraded to manifest-v1.
LegacyServer { revision: i64 },
/// The merged vault, with the canonical set the push writes from unless a manifest fell back to the server's version.
Merged { pulled: PulledVault, push_canonical: Option<CanonicalizedSet> },
/// The merge failed, so the server's vault stands as pulled and the local changes are dropped.
ServerOnly(PulledVault),
}
/// Pull the latest snapshot and merge the local vault onto it at canonical level, one manifest at a time.
pub(crate) async fn pull_and_merge(ctx: &mut Ctx) -> SyncResult<PullAndMergeOutcome> {
let vek = ctx.encryption_key()?;
ctx.log("[Merge] Fetching vault snapshot for canonical merge (GET /v2/Vault)...").await;
let snapshot = match pull::fetch_snapshot(ctx).await? {
// LEGACY: a server still on the sqlite-blob format cannot merge with a manifest-v1 vault, so the caller pushes
// over it. TODO: remove once every account has upgraded to manifest-v1.
Snapshot::LegacySqliteBlob(snapshot) => {
let legacy = legacy::open_legacy_snapshot(ctx, &snapshot).await?;
pull::commit_revisions(ctx, &legacy).await?;
return Ok(PullAndMergeOutcome::LegacyServer { revision: legacy.personal_revision });
}
Snapshot::Manifest(snapshot) => snapshot,
};
let opened = pull::open_manifests_and_record_sync_state(ctx, &snapshot, vek.expose_secret()).await?;
let local_side = canonicalize_vault(ctx, None).await?;
let local_side = match ground_moved_under_canonicalize(&local_side.manifest_records, &opened) {
None => local_side,
Some(moved) => {
ctx.warn(format!("[Merge] This snapshot changed {}; canonicalizing the local vault again so its blob hashes match the server's.", moved)).await;
canonicalize_vault(ctx, None).await?
}
};
match merge_onto_opened_manifests(ctx, &opened, local_side, vek.expose_secret()).await {
Ok(outcome) => Ok(outcome),
Err(merge_error) => {
ctx.warn(format!("[Merge] Canonical merge failed, falling back to the server vault: {}", merge_error)).await;
let sqlite_bytes = pull::materialize_to_sqlite(ctx, &opened.manifests(), &opened.data_buckets, &opened.blob_map, &opened.manifest_names).await?;
Ok(PullAndMergeOutcome::ServerOnly(opened.pulled_vault(state::encrypt_vault_blob(&sqlite_bytes, vek.expose_secret())?)))
}
}
}
/// What a snapshot changed under a canonicalize that ran against the previous local vault.
fn ground_moved_under_canonicalize(records: &[ManifestRecord], opened: &OpenedManifestSet) -> Option<String> {
let personal_record = records.iter().find(|r| r.is_personal);
let server_personal = opened.resolved.iter().find(|m| m.is_personal);
if let Some(server) = server_personal {
if !personal_record.map(|r| ids_equal(&r.manifest_id, &server.manifest_id)).unwrap_or(false) {
return Some(format!("the personal manifest id ({} became {})", personal_record.map(|r| r.manifest_id.as_str()).unwrap_or("none"), server.manifest_id));
}
}
for record in records {
if let Some(server) = opened.resolved.iter().find(|m| ids_equal(&m.manifest_id, &record.manifest_id)) {
if server.manifest.manifest_salt != record.salt {
return Some(format!("the blob salt of manifest {}", record.manifest_id));
}
}
}
None
}
/// Merge the local side against the opened manifests, validate per manifest, and materialize.
async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet, local_side: CanonicalizedSet, vek: &str) -> SyncResult<PullAndMergeOutcome> {
let schema = ctx.schema().await?;
ctx.log(format!("[Merge] Merging {} local manifest(s) onto {} server manifest(s)...", local_side.canonicalized.manifests.len(), opened.resolved.len())).await;
let merge_output = merge_canonical(CanonicalMergeInput {
server_manifests: opened.manifests(),
server_buckets: opened.data_buckets.clone(),
contentless_server_manifest_ids: opened.contentless_manifest_ids.clone(),
local_manifests: local_side.canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect(),
local_buckets: local_side.canonicalized.data_buckets.clone(),
schema_columns: schema.columns.clone(),
})?;
let server_manifest_by_id: HashMap<String, &Manifest> = opened.resolved.iter().map(|m| (id_key(&m.manifest_id), &m.manifest)).collect();
let mut server_buckets_by_id: HashMap<String, Vec<DataBucket>> = HashMap::new();
for bucket in &opened.data_buckets {
server_buckets_by_id.entry(id_key(&bucket.manifest_id)).or_default().push(bucket.clone());
}
let contentless: HashSet<String> = opened.contentless_manifest_ids.iter().map(|id| id_key(id)).collect();
let mut manifests: Vec<Manifest> = Vec::new();
let mut data_buckets: Vec<DataBucket> = Vec::new();
let mut fallbacks = 0usize;
let mut stats = MergeStats::default();
for entry in merge_output.manifests {
if let Some(failure) = validate_merged_manifest(&entry) {
if !contentless.contains(&id_key(&entry.manifest_id)) {
ctx.warn(format!("[Merge] Merged manifest {} failed validation ({}); the server's version stands and local changes to it are dropped.", entry.manifest_id, failure)).await;
fallbacks += 1;
if let Some(server) = server_manifest_by_id.get(&id_key(&entry.manifest_id)) {
manifests.push((*server).clone());
data_buckets.extend(server_buckets_by_id.get(&id_key(&entry.manifest_id)).cloned().unwrap_or_default());
}
continue;
}
ctx.warn(format!("[Merge] Pass-through manifest {} failed validation ({}); keeping its local rows.", entry.manifest_id, failure)).await;
}
manifests.push(entry.manifest);
data_buckets.extend(entry.buckets);
stats.add(&entry.stats);
}
for dropped in &merge_output.dropped_local_manifest_ids {
ctx.warn(format!("[Merge] Local manifest {} is no longer served; its rows are dropped from the merged vault.", dropped)).await;
}
// Blob bytes for materialize: the server download plus everything the local canonicalize extracted.
let mut blob_map = opened.blob_map.clone();
let mut local_blobs: HashMap<String, BlobEntry> = HashMap::new();
for canonicalized in &local_side.canonicalized.manifests {
for (hash, blob) in &canonicalized.blobs {
local_blobs.insert(hash.clone(), blob.clone());
if !blob_map.contains_key(hash) {
blob_map.insert(hash.clone(), blob.bytes.clone());
}
}
}
let merged_blobs = collect_merged_blobs(&manifests, &blob_map, &local_blobs);
let sqlite_bytes = pull::materialize_to_sqlite(ctx, &manifests, &data_buckets, &blob_map, &opened.manifest_names).await?;
let encrypted_vault = state::encrypt_vault_blob(&sqlite_bytes, vek)?;
ctx.log(format!("[Merge] Canonical merge complete: {} local row(s) won, {} local-only row(s) kept, {} validation fallback(s), {} dropped local manifest(s).", stats.incoming_won, stats.incoming_only, fallbacks, merge_output.dropped_local_manifest_ids.len())).await;
let push_canonical = if fallbacks == 0 { merge_output_for_push(&manifests, &data_buckets, merged_blobs, &local_side.manifest_records) } else { None };
Ok(PullAndMergeOutcome::Merged { pulled: opened.pulled_vault(encrypted_vault), push_canonical })
}
/// The merged vault in the shape the push writes from, when it covers exactly the manifests this session writes.
fn merge_output_for_push(manifests: &[Manifest], data_buckets: &[DataBucket], blobs_by_manifest: HashMap<String, HashMap<String, BlobEntry>>, manifest_records: &[ManifestRecord]) -> Option<CanonicalizedSet> {
let merged_ids: HashSet<String> = manifests.iter().map(|m| id_key(&m.manifest_id)).collect();
let record_ids: Vec<String> = manifest_records.iter().map(|r| id_key(&r.manifest_id)).collect();
if merged_ids.len() != record_ids.len() || record_ids.iter().any(|id| !merged_ids.contains(id)) {
return None;
}
Some(CanonicalizedSet {
canonicalized: CanonicalizedVault {
manifests: manifests.iter().map(|m| vault_codec::CanonicalizedManifest { manifest: m.clone(), blobs: blobs_by_manifest.get(&id_key(&m.manifest_id)).cloned().unwrap_or_default() }).collect(),
data_buckets: data_buckets.to_vec(),
},
manifest_records: manifest_records.to_vec(),
})
}
/// Validate one merged manifest and its buckets.
fn validate_merged_manifest(entry: &CanonicalManifestMerge) -> Option<String> {
let validation = vault_codec::validate_manifest(&entry.manifest);
if !validation.ok {
return Some(validation.failed_rules.join(", "));
}
for bucket in &entry.buckets {
let bucket_validation = vault_codec::validate_data_bucket(bucket);
if !bucket_validation.ok {
return Some(format!("bucket \"{}\": {}", bucket.category, bucket_validation.failed_rules.join(", ")));
}
}
None
}
/// The bytes at hand for the blobs the merged manifests reference, per manifest; a blob without bytes keeps its reference.
fn collect_merged_blobs(manifests: &[Manifest], blob_map: &HashMap<String, Vec<u8>>, local_blobs: &HashMap<String, BlobEntry>) -> HashMap<String, HashMap<String, BlobEntry>> {
let mut by_manifest: HashMap<String, HashMap<String, BlobEntry>> = HashMap::new();
for manifest in manifests {
let blobs = by_manifest.entry(id_key(&manifest.manifest_id)).or_default();
for (hash, kind) in manifest.referenced_blobs() {
if let Some(local) = local_blobs.get(&hash) {
blobs.insert(hash, local.clone());
} else if let Some(bytes) = blob_map.get(&hash) {
blobs.insert(hash, BlobEntry { kind, bytes: bytes.clone() });
}
}
}
by_manifest
}