Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

10.8 KB
Raw Blame
//! Merge: a dirty vault's pull, which merges the local vault onto the server snapshot at canonical level, one
//! manifest at a time, and materializes the result.

use std::collections::{HashMap, HashSet};
use secrecy::ExposeSecret;

use crate::vault_model::{id_key, ids_equal};
use super::errors::SyncResult;
use super::pull::{self, OpenedManifestSet, PulledVault, Snapshot};
use super::push::{canonicalize_vault, CanonicalizedSet};
use super::state::{self, Ctx};
use super::write_set::ManifestRecord;
use super::legacy;
use crate::vault_codec::{self, BlobEntry, CanonicalizedVault, DataBucket, Manifest};
use crate::vault_merge::{merge_canonical, CanonicalManifestMerge, CanonicalMergeInput, MergeStats};

/// What a dirty pull (pull and merge) produced.
pub(crate) enum PullAndMergeOutcome {
    /// LEGACY: the server's vault is still a sqlite blob at this revision; the caller pushes the local vault over it.
    /// TODO: remove once every account has upgraded to manifest-v1.
    LegacyServer { revision: i64 },
    /// The merged vault, with the canonical set the push writes from unless a manifest fell back to the server's version.
    Merged { pulled: PulledVault, push_canonical: Option<CanonicalizedSet> },
    /// The merge failed, so the server's vault stands as pulled and the local changes are dropped.
    ServerOnly(PulledVault),
}

/// Pull the latest snapshot and merge the local vault onto it at canonical level, one manifest at a time.
pub(crate) async fn pull_and_merge(ctx: &mut Ctx) -> SyncResult<PullAndMergeOutcome> {
    let vek = ctx.encryption_key()?;
    ctx.log("[Merge] Fetching vault snapshot for canonical merge (GET /v2/Vault)...").await;
    let snapshot = match pull::fetch_snapshot(ctx).await? {
        // LEGACY: a server still on the sqlite-blob format cannot merge with a manifest-v1 vault, so the caller pushes
        // over it. TODO: remove once every account has upgraded to manifest-v1.
        Snapshot::LegacySqliteBlob(snapshot) => {
            let legacy = legacy::open_legacy_snapshot(ctx, &snapshot).await?;
            pull::commit_revisions(ctx, &legacy).await?;
            return Ok(PullAndMergeOutcome::LegacyServer { revision: legacy.personal_revision });
        }
        Snapshot::Manifest(snapshot) => snapshot,
    };

    let opened = pull::open_manifests_and_record_sync_state(ctx, &snapshot, vek.expose_secret()).await?;
    let local_side = canonicalize_vault(ctx, None).await?;
    let local_side = match ground_moved_under_canonicalize(&local_side.manifest_records, &opened) {
        None => local_side,
        Some(moved) => {
            ctx.warn(format!("[Merge] This snapshot changed {}; canonicalizing the local vault again so its blob hashes match the server's.", moved)).await;
            canonicalize_vault(ctx, None).await?
        }
    };

    match merge_onto_opened_manifests(ctx, &opened, local_side, vek.expose_secret()).await {
        Ok(outcome) => Ok(outcome),
        Err(merge_error) => {
            ctx.warn(format!("[Merge] Canonical merge failed, falling back to the server vault: {}", merge_error)).await;
            let sqlite_bytes = pull::materialize_to_sqlite(ctx, &opened.manifests(), &opened.data_buckets, &opened.blob_map, &opened.manifest_names).await?;
            Ok(PullAndMergeOutcome::ServerOnly(opened.pulled_vault(state::encrypt_vault_blob(&sqlite_bytes, vek.expose_secret())?)))
        }
    }
}

/// What a snapshot changed under a canonicalize that ran against the previous local vault.
fn ground_moved_under_canonicalize(records: &[ManifestRecord], opened: &OpenedManifestSet) -> Option<String> {
    let personal_record = records.iter().find(|r| r.is_personal);
    let server_personal = opened.resolved.iter().find(|m| m.is_personal);
    if let Some(server) = server_personal {
        if !personal_record.map(|r| ids_equal(&r.manifest_id, &server.manifest_id)).unwrap_or(false) {
            return Some(format!("the personal manifest id ({} became {})", personal_record.map(|r| r.manifest_id.as_str()).unwrap_or("none"), server.manifest_id));
        }
    }
    for record in records {
        if let Some(server) = opened.resolved.iter().find(|m| ids_equal(&m.manifest_id, &record.manifest_id)) {
            if server.manifest.manifest_salt != record.salt {
                return Some(format!("the blob salt of manifest {}", record.manifest_id));
            }
        }
    }
    None
}

/// Merge the local side against the opened manifests, validate per manifest, and materialize.
async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet, local_side: CanonicalizedSet, vek: &str) -> SyncResult<PullAndMergeOutcome> {
    let schema = ctx.schema().await?;
    ctx.log(format!("[Merge] Merging {} local manifest(s) onto {} server manifest(s)...", local_side.canonicalized.manifests.len(), opened.resolved.len())).await;
    let merge_output = merge_canonical(CanonicalMergeInput {
        server_manifests: opened.manifests(),
        server_buckets: opened.data_buckets.clone(),
        contentless_server_manifest_ids: opened.contentless_manifest_ids.clone(),
        local_manifests: local_side.canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect(),
        local_buckets: local_side.canonicalized.data_buckets.clone(),
        schema_columns: schema.columns.clone(),
    })?;

    let server_manifest_by_id: HashMap<String, &Manifest> = opened.resolved.iter().map(|m| (id_key(&m.manifest_id), &m.manifest)).collect();
    let mut server_buckets_by_id: HashMap<String, Vec<DataBucket>> = HashMap::new();
    for bucket in &opened.data_buckets {
        server_buckets_by_id.entry(id_key(&bucket.manifest_id)).or_default().push(bucket.clone());
    }
    let contentless: HashSet<String> = opened.contentless_manifest_ids.iter().map(|id| id_key(id)).collect();

    let mut manifests: Vec<Manifest> = Vec::new();
    let mut data_buckets: Vec<DataBucket> = Vec::new();
    let mut fallbacks = 0usize;
    let mut stats = MergeStats::default();
    for entry in merge_output.manifests {
        if let Some(failure) = validate_merged_manifest(&entry) {
            if !contentless.contains(&id_key(&entry.manifest_id)) {
                ctx.warn(format!("[Merge] Merged manifest {} failed validation ({}); the server's version stands and local changes to it are dropped.", entry.manifest_id, failure)).await;
                fallbacks += 1;
                if let Some(server) = server_manifest_by_id.get(&id_key(&entry.manifest_id)) {
                    manifests.push((*server).clone());
                    data_buckets.extend(server_buckets_by_id.get(&id_key(&entry.manifest_id)).cloned().unwrap_or_default());
                }
                continue;
            }
            ctx.warn(format!("[Merge] Pass-through manifest {} failed validation ({}); keeping its local rows.", entry.manifest_id, failure)).await;
        }
        manifests.push(entry.manifest);
        data_buckets.extend(entry.buckets);
        stats.add(&entry.stats);
    }
    for dropped in &merge_output.dropped_local_manifest_ids {
        ctx.warn(format!("[Merge] Local manifest {} is no longer served; its rows are dropped from the merged vault.", dropped)).await;
    }

    // Blob bytes for materialize: the server download plus everything the local canonicalize extracted.
    let mut blob_map = opened.blob_map.clone();
    let mut local_blobs: HashMap<String, BlobEntry> = HashMap::new();
    for canonicalized in &local_side.canonicalized.manifests {
        for (hash, blob) in &canonicalized.blobs {
            local_blobs.insert(hash.clone(), blob.clone());
            if !blob_map.contains_key(hash) {
                blob_map.insert(hash.clone(), blob.bytes.clone());
            }
        }
    }
    let merged_blobs = collect_merged_blobs(&manifests, &blob_map, &local_blobs);

    let sqlite_bytes = pull::materialize_to_sqlite(ctx, &manifests, &data_buckets, &blob_map, &opened.manifest_names).await?;
    let encrypted_vault = state::encrypt_vault_blob(&sqlite_bytes, vek)?;
    ctx.log(format!("[Merge] Canonical merge complete: {} local row(s) won, {} local-only row(s) kept, {} validation fallback(s), {} dropped local manifest(s).", stats.incoming_won, stats.incoming_only, fallbacks, merge_output.dropped_local_manifest_ids.len())).await;

    let push_canonical = if fallbacks == 0 { merge_output_for_push(&manifests, &data_buckets, merged_blobs, &local_side.manifest_records) } else { None };
    Ok(PullAndMergeOutcome::Merged { pulled: opened.pulled_vault(encrypted_vault), push_canonical })
}

/// The merged vault in the shape the push writes from, when it covers exactly the manifests this session writes.
fn merge_output_for_push(manifests: &[Manifest], data_buckets: &[DataBucket], blobs_by_manifest: HashMap<String, HashMap<String, BlobEntry>>, manifest_records: &[ManifestRecord]) -> Option<CanonicalizedSet> {
    let merged_ids: HashSet<String> = manifests.iter().map(|m| id_key(&m.manifest_id)).collect();
    let record_ids: Vec<String> = manifest_records.iter().map(|r| id_key(&r.manifest_id)).collect();
    if merged_ids.len() != record_ids.len() || record_ids.iter().any(|id| !merged_ids.contains(id)) {
        return None;
    }
    Some(CanonicalizedSet {
        canonicalized: CanonicalizedVault {
            manifests: manifests.iter().map(|m| vault_codec::CanonicalizedManifest { manifest: m.clone(), blobs: blobs_by_manifest.get(&id_key(&m.manifest_id)).cloned().unwrap_or_default() }).collect(),
            data_buckets: data_buckets.to_vec(),
        },
        manifest_records: manifest_records.to_vec(),
    })
}

/// Validate one merged manifest and its buckets.
fn validate_merged_manifest(entry: &CanonicalManifestMerge) -> Option<String> {
    let validation = vault_codec::validate_manifest(&entry.manifest);
    if !validation.ok {
        return Some(validation.failed_rules.join(", "));
    }
    for bucket in &entry.buckets {
        let bucket_validation = vault_codec::validate_data_bucket(bucket);
        if !bucket_validation.ok {
            return Some(format!("bucket \"{}\": {}", bucket.category, bucket_validation.failed_rules.join(", ")));
        }
    }
    None
}

/// The bytes at hand for the blobs the merged manifests reference, per manifest; a blob without bytes keeps its reference.
fn collect_merged_blobs(manifests: &[Manifest], blob_map: &HashMap<String, Vec<u8>>, local_blobs: &HashMap<String, BlobEntry>) -> HashMap<String, HashMap<String, BlobEntry>> {
    let mut by_manifest: HashMap<String, HashMap<String, BlobEntry>> = HashMap::new();
    for manifest in manifests {
        let blobs = by_manifest.entry(id_key(&manifest.manifest_id)).or_default();
        for (hash, kind) in manifest.referenced_blobs() {
            if let Some(local) = local_blobs.get(&hash) {
                blobs.insert(hash, local.clone());
            } else if let Some(bytes) = blob_map.get(&hash) {
                blobs.insert(hash, BlobEntry { kind, bytes: bytes.clone() });
            }
        }
    }
    by_manifest
}