Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
//! The sync flow: status check, server-directed changes, then pull (and merge) or push as the revisions decide.
use std::collections::HashMap;
use secrecy::ExposeSecret;
use serde_json::Value;
use super::errors::{LogoutReason, SyncError, SyncResult};
use super::merge::{self, PullAndMergeOutcome};
use super::pull::{self, PulledVault};
use super::push::{self, CanonicalizedSet, PushStatus, WriteKind};
use super::session::Host;
use super::state::{self, Ctx};
use super::types::{FailureFields, FullSyncResult, LogLevel, ResolveVaultKeyResult, StatusCheckResult, StatusResponse, SyncOperation, SyncRequest};
use super::{db, http, keys, migration, rotation, sharing};
use crate::vault_model::id_key;
/// How many times a sync may pull, merge and push again after an outdated push before giving up.
const MAX_OUTDATED_RESYNCS: u32 = 3;
/// Run one operation to completion; the value is the `done` command's result.
pub(crate) async fn run(host: Host, request: SyncRequest) -> Value {
let mut ctx = Ctx::new(host, request);
let result = match ctx.request.operation {
SyncOperation::FullSync => serde_json::to_value(full_sync_operation(&mut ctx).await),
SyncOperation::MigrationStatus => serde_json::to_value(migration::migration_status(&mut ctx).await),
SyncOperation::MigrateManifest => serde_json::to_value(migration::migrate_manifest(&mut ctx).await),
SyncOperation::StatusCheck => serde_json::to_value(status_check(&mut ctx).await),
SyncOperation::ResolveVaultKey => serde_json::to_value(resolve_vault_key(&mut ctx).await),
SyncOperation::CreateSharedManifest => serde_json::to_value(sharing::create_shared_manifest_operation(&mut ctx).await),
SyncOperation::InviteToSharedManifest => serde_json::to_value(sharing::invite_to_shared_manifest_operation(&mut ctx).await),
SyncOperation::UpdateSharedManifest => serde_json::to_value(sharing::update_shared_manifest_operation(&mut ctx).await),
SyncOperation::LookupAccountFingerprint => serde_json::to_value(sharing::lookup_account_fingerprint_operation(&mut ctx).await),
SyncOperation::AcceptSharedManifestInvitation => serde_json::to_value(sharing::accept_shared_manifest_invitation_operation(&mut ctx).await),
SyncOperation::RotateAccountKeys => serde_json::to_value(rotation::rotate_account_keys_operation(&mut ctx).await),
SyncOperation::TransferEmailClaim => serde_json::to_value(sharing::transfer_email_claim_operation(&mut ctx).await),
SyncOperation::ApproveMobileLogin => serde_json::to_value(sharing::approve_mobile_login_operation(&mut ctx).await),
};
finish(&ctx, result.unwrap_or(Value::Null))
}
/// Add `vaultChanged` to a serialized result.
fn finish(ctx: &Ctx, mut result: Value) -> Value {
if let Some(object) = result.as_object_mut() {
object.insert("vaultChanged".to_string(), Value::Bool(ctx.vault_changed));
}
result
}
/// What one pass of the sync decided: a final outcome, or that the whole sync has to re-run.
#[allow(clippy::large_enum_variant)]
enum Flow {
Done(FullSyncResult),
/// Re-run the sync; `outdated` counts against the resync limit.
Resync { outdated: bool },
}
fn success() -> FullSyncResult {
FullSyncResult { success: true, ..Default::default() }
}
/// The outcome of a failed sync: a forced logout, or a coded error.
pub(super) fn failure(error: &SyncError) -> FullSyncResult {
FullSyncResult { failure: error.into(), ..Default::default() }
}
/// The outcome of a sync that fell back on the local vault because the server could not be reached.
fn offline() -> FullSyncResult {
FullSyncResult { success: true, was_offline: true, is_offline_mode: true, ..Default::default() }
}
fn logout(reason: LogoutReason) -> FullSyncResult {
FullSyncResult { failure: FailureFields::logout(reason), ..Default::default() }
}
/// The outcome of an error thrown during a sync: offline mode when the server is unreachable and a local vault exists.
async fn map_sync_failure(ctx: &mut Ctx, error: SyncError) -> FullSyncResult {
ctx.warn(format!("[Sync] Sync failed: {}", error)).await;
if matches!(error, SyncError::Network(_)) && ctx.has_local_vault().await.is_ok_and(|has_vault| has_vault) {
return offline();
}
failure(&error)
}
async fn full_sync_operation(ctx: &mut Ctx) -> FullSyncResult {
match full_sync(ctx).await {
Ok(result) => result,
Err(error) => map_sync_failure(ctx, error).await,
}
}
/// Full vault sync, re-running itself when a mutation raced a store or the server refused a push as outdated.
pub(crate) async fn full_sync(ctx: &mut Ctx) -> SyncResult<FullSyncResult> {
let mut outdated_resyncs = 0u32;
let mut dropped_local_vault = false;
loop {
let flow = match full_sync_once(ctx).await {
Ok(flow) => flow,
Err(error) => {
let error = ctx.diagnose_local_vault_failure(error).await;
if dropped_local_vault || !matches!(error, SyncError::VaultDecryptFailed(_)) {
return Err(error);
}
drop_local_vault_for_pull(ctx, &error).await?;
dropped_local_vault = true;
continue;
}
};
match flow {
Flow::Done(result) => return Ok(result),
Flow::Resync { outdated } => {
if outdated {
if outdated_resyncs >= MAX_OUTDATED_RESYNCS {
ctx.warn(format!("[Sync] The server still refuses the write after {} re-syncs; giving up on this chain.", MAX_OUTDATED_RESYNCS)).await;
return Ok(failure(&SyncError::ResyncLimitReached));
}
outdated_resyncs += 1;
}
ctx.log("[Sync] Re-running the sync.").await;
}
}
}
}
/// Treat a stored vault the session key does not open as absent and force the retry to pull the server's vault in its place.
async fn drop_local_vault_for_pull(ctx: &mut Ctx, error: &SyncError) -> SyncResult<()> {
ctx.warn(format!("[Sync] The stored vault does not decrypt with the session key; dropping it and pulling the server's vault. {}", error)).await;
ctx.discard_local_vault();
if ctx.is_dirty && state::mark_clean(&ctx.host, ctx.mutation_sequence).await? {
ctx.is_dirty = false;
}
ctx.request.force_pull = true;
Ok(())
}
async fn full_sync_once(ctx: &mut Ctx) -> SyncResult<Flow> {
ctx.log("[Sync] Sync started").await;
let status = match check_session(ctx).await? {
Ok(status) => status,
Err(result) => return Ok(Flow::Done(result)),
};
if let Some(domains) = &status.email_domains {
ctx.request.private_email_domains = domains.private_email_domain_list.clone();
}
let mut needs_pull = ctx.request.force_pull || server_state_needs_pull(ctx, &status).await?;
ctx.log(format!("[Sync] Status received (needsPull {}, isDirty {})", needs_pull, ctx.is_dirty)).await;
// A key hierarchy another device created shows up as a revision change, so it is checked here.
if needs_pull || ctx.is_dirty {
keys::ensure_key_chain_accepted(ctx).await?;
}
announce_phase(ctx, needs_pull, ctx.is_dirty).await;
let server_directed_changes = refresh_manifest_names(ctx, &status, needs_pull).await?;
// A dirty vault: drop the dirty flag when nothing changed, and pull first when it holds rows this session cannot write.
let mut canonicalize_cache = None;
if !needs_pull && ctx.is_dirty && !migration::vault_predates_current_schema(ctx).await? {
canonicalize_cache = clear_no_op_mutation(ctx).await;
if ctx.is_dirty && push::vault_holds_unwritable_manifests(ctx).await {
needs_pull = true;
announce_phase(ctx, true, true).await;
}
}
let flow = if needs_pull {
pull_and_materialize_server_vault(ctx, server_directed_changes).await?
} else if ctx.is_dirty && !migration::vault_predates_current_schema(ctx).await? {
match push_local_changes(ctx, canonicalize_cache, WriteKind::Changes { buckets_only: true }).await? {
Some(vault_changed) => Flow::Done(FullSyncResult { success: true, has_new_vault: vault_changed || server_directed_changes, ..Default::default() }),
None => Flow::Resync { outdated: true },
}
} else {
Flow::Done(migration::pending_migration_result(ctx).await?.unwrap_or_else(success))
};
Ok(match flow {
Flow::Done(mut result) => {
result.server_version = Some(status.server_version.clone());
result.capabilities = status.capabilities.clone();
result.email_domains = status.email_domains.clone();
result.pending_sharing_invitations = status.pending_sharing_invitations;
ctx.log("[Sync] Sync finished").await;
Flow::Done(result)
}
other => other,
})
}
/// The checks before the sync touches the vault; `Err` carries the outcome of a sync that stops here.
async fn check_session(ctx: &mut Ctx) -> SyncResult<Result<StatusResponse, FullSyncResult>> {
if ctx.request.username.is_empty() {
return Ok(Err(failure(&SyncError::Other("no username in the sync request".to_string()))));
}
if ctx.encryption_key.is_none() {
return Ok(Err(failure(&SyncError::VaultLocked)));
}
let status = http::get_status(&ctx.host).await?;
if status.server_version == http::SERVER_UNREACHABLE_VERSION {
return Ok(Err(enter_offline_mode(ctx).await?));
}
if let Some(reason) = version_logout_reason(ctx, &status) {
return Ok(Err(logout(reason)));
}
assert_salt_unchanged(ctx, status.srp_salt.as_deref()).await?;
Ok(Ok(status))
}
/// Tell the host which phase the sync enters.
async fn announce_phase(ctx: &Ctx, needs_pull: bool, is_dirty: bool) {
if needs_pull {
ctx.host.log(LogLevel::Phase, "pull").await;
} else if is_dirty {
ctx.host.log(LogLevel::Phase, "push").await;
}
}
/// Clear the dirty flag of a vault that equals the last known server state, else return the canonicalize result for the push.
async fn clear_no_op_mutation(ctx: &mut Ctx) -> Option<(u64, CanonicalizedSet)> {
match push::detect_no_op_mutation(ctx).await {
Ok((true, _)) => {
if let Ok(true) = state::mark_clean(&ctx.host, ctx.mutation_sequence).await {
ctx.is_dirty = false;
}
None
}
Ok((false, set)) => Some((ctx.mutation_sequence, set)),
Err(error) => {
ctx.warn(format!("[Sync] No-op mutation pre-check failed, proceeding with a normal push: {}", error)).await;
None
}
}
}
/// Why this client and server cannot work together, if they cannot.
fn version_logout_reason(ctx: &Ctx, status: &StatusResponse) -> Option<LogoutReason> {
if !status.client_version_supported {
return Some(LogoutReason::ClientVersionNotSupported);
}
let server_too_old = ctx.request.min_server_version.as_deref().is_some_and(|min| !version_gte(&status.server_version, min));
server_too_old.then_some(LogoutReason::ServerVersionNotSupported)
}
/// Whether `version1 >= version2`, a pre-release sorting below its release.
pub(super) fn version_gte(version1: &str, version2: &str) -> bool {
let (core1, pre1) = split_version(version1);
let (core2, pre2) = split_version(version2);
let parts1: Vec<u64> = core1.split('.').map(|part| part.parse().unwrap_or(0)).collect();
let parts2: Vec<u64> = core2.split('.').map(|part| part.parse().unwrap_or(0)).collect();
for index in 0..parts1.len().max(parts2.len()) {
let part1 = parts1.get(index).copied().unwrap_or(0);
let part2 = parts2.get(index).copied().unwrap_or(0);
if part1 != part2 {
return part1 > part2;
}
}
match (pre1, pre2) {
(None, Some(_)) => true,
(Some(_), None) => false,
(None, None) => true,
(Some(a), Some(b)) => a >= b,
}
}
/// A version's core and optional pre-release tag.
fn split_version(version: &str) -> (&str, Option<&str>) {
match version.split_once('-') {
Some((core, pre)) if !pre.is_empty() => (core, Some(pre)),
Some((core, _)) => (core, None),
None => (version, None),
}
}
/// Whether any manifest or data bucket changed on the server.
pub(crate) async fn server_state_needs_pull(ctx: &Ctx, status: &StatusResponse) -> SyncResult<bool> {
let server_manifests: HashMap<String, i64> = status.manifest_revisions.iter().map(|m| (m.manifest_id.clone(), m.revision)).collect();
let server_buckets: HashMap<String, i64> = status.bucket_revisions.iter().map(|b| (state::bucket_revision_key(&b.manifest_id, &b.category), b.revision)).collect();
let local_manifests: HashMap<String, i64> = state::get(&ctx.host, state::SERVER_MANIFEST_REVISIONS).await?.unwrap_or_default();
let local_buckets: HashMap<String, i64> = state::get(&ctx.host, state::VAULT_BUCKET_REVISIONS).await?.unwrap_or_default();
let manifests_to_pull = requiring_pull(&server_manifests, &local_manifests);
let buckets_to_pull = requiring_pull(&server_buckets, &local_buckets);
if manifests_to_pull == 0 && buckets_to_pull == 0 {
ctx.log("[Sync] No pull needed: every manifest and data bucket matches the local revisions.").await;
return Ok(false);
}
ctx.log(format!("[Sync] Pull needed for {} manifest(s) and {} data bucket(s).", manifests_to_pull, buckets_to_pull)).await;
Ok(true)
}
/// The entries whose local revision no longer matches the server's, plus the ones the server no longer lists.
fn requiring_pull(server: &HashMap<String, i64>, local: &HashMap<String, i64>) -> usize {
server.iter().filter(|(id, revision)| local.get(*id) != Some(revision)).count() + local.keys().filter(|id| !server.contains_key(*id)).count()
}
/// Fail with `PasswordChangedElsewhere` when the server's salt differs from the one this device derived its keys from.
pub(crate) async fn assert_salt_unchanged(ctx: &Ctx, server_salt: Option<&str>) -> SyncResult<()> {
let stored: Option<Value> = state::get(&ctx.host, state::UNLOCK_KEY_DERIVATION_PARAMS).await?;
let stored_salt = stored.as_ref().and_then(|params| params.get("salt")).and_then(Value::as_str).map(str::to_string);
if let (Some(stored), Some(server)) = (stored_salt, server_salt) {
if !server.is_empty() && server != stored {
return Err(SyncError::PasswordChangedElsewhere);
}
}
Ok(())
}
/// Fall back to offline mode when the server cannot be reached; without a local vault that is a failure.
async fn enter_offline_mode(ctx: &mut Ctx) -> SyncResult<FullSyncResult> {
if !ctx.has_local_vault().await? {
let unreachable = SyncError::Network("server unreachable and no local vault to fall back on".to_string());
return Ok(FullSyncResult { was_offline: true, is_offline_mode: true, ..failure(&unreachable) });
}
Ok(offline())
}
/// Fetch the shared manifest names the status served and store them in the local vault.
async fn refresh_manifest_names(ctx: &mut Ctx, status: &StatusResponse, needs_pull: bool) -> SyncResult<bool> {
ctx.served_manifest_names = status.manifest_revisions.iter().filter_map(|m| Some((id_key(&m.manifest_id), m.encrypted_name.clone()?))).collect();
if needs_pull || ctx.served_manifest_names.is_empty() || !ctx.has_local_vault().await? || migration::vault_requires_manifest_migration(ctx).await? {
return Ok(false);
}
let mut records = keys::shared_manifest_records(ctx).await?;
let mut opened: HashMap<String, String> = HashMap::new();
for record in records.values_mut() {
let Some(served) = ctx.served_manifest_names.get(&id_key(&record.manifest_id)).filter(|served| record.encrypted_name.as_ref() != Some(*served)).cloned() else { continue };
let Some(vek) = keys::open_shared_manifest_vek(ctx, record).await? else { continue };
match pull::open_manifest_name(&served, &record.manifest_id, vek.expose_secret()) {
Some(name) => drop(opened.insert(id_key(&record.manifest_id), name)),
None => ctx.warn(format!("[Sharing] The name of shared manifest {} did not open with its key.", record.manifest_id)).await,
}
record.encrypted_name = Some(served);
}
if opened.is_empty() {
return Ok(false);
}
let key = ctx.encryption_key()?;
db::apply_manifest_names(&ctx.host, super::types::Db::Local, &opened).await?;
ctx.persist_local_vault(false).await?;
keys::set_shared_manifest_records(ctx, &records, key.expose_secret()).await?;
ctx.log(format!("[Sharing] Refreshed the name of {} shared manifest(s).", opened.len())).await;
Ok(true)
}
/// Outcome for a sync that stored a freshly pulled vault.
async fn materialized_vault_result(ctx: &mut Ctx, pulled: &PulledVault) -> SyncResult<FullSyncResult> {
let migration_required = migration::vault_requires_manifest_migration(ctx).await?;
// LEGACY: a pulled sqlite-blob vault can still be on the frozen upgrade chain.
let legacy_chain = migration::schema_state(ctx).await? == db::SchemaState::LegacyChain;
Ok(FullSyncResult { success: true, has_new_vault: true, sqlite_blob_upgrade_required: legacy_chain, manifest_migration_required: migration_required, pulled_revision: Some(pulled.personal_revision), ..Default::default() })
}
/// Store a pulled vault and commit its revisions; a store refused by a raced mutation returns a resync.
async fn commit_pulled_vault(ctx: &mut Ctx, pulled: &PulledVault) -> SyncResult<Option<Flow>> {
let stored = ctx.store_vault(&pulled.encrypted_vault, pulled.needs_first_write, Some(ctx.mutation_sequence), Some(pulled.personal_revision)).await?;
if !stored.success {
ctx.log("[Sync] Mutation detected during sync, re-syncing...").await;
ctx.is_dirty = true;
ctx.mutation_sequence = stored.mutation_sequence;
ctx.request.dirty_scopes.clear();
return Ok(Some(Flow::Resync { outdated: false }));
}
if pulled.needs_first_write {
ctx.is_dirty = true;
ctx.mutation_sequence = stored.mutation_sequence;
}
ctx.vault_changed = true;
pull::commit_revisions(ctx, pulled).await?;
Ok(None)
}
/// Store the server's vault as the local vault, replacing whatever was there, and report it.
async fn store_server_vault(ctx: &mut Ctx, pulled: &PulledVault) -> SyncResult<Flow> {
if let Some(resync) = commit_pulled_vault(ctx, pulled).await? {
return Ok(resync);
}
if pulled.needs_first_write {
ctx.log("[Sync] Writing the first revision of the new vault.").await;
if push_local_changes(ctx, None, WriteKind::Everything).await?.is_none() {
return Ok(Flow::Resync { outdated: true });
}
}
Ok(Flow::Done(materialized_vault_result(ctx, pulled).await?))
}
/// Pull the server's latest vault and merge it with what is stored locally when needed.
async fn pull_and_materialize_server_vault(ctx: &mut Ctx, server_directed_changes: bool) -> SyncResult<Flow> {
if !ctx.is_dirty || !ctx.has_local_vault().await? {
let pulled = pull::pull(ctx).await?;
return store_server_vault(ctx, &pulled).await;
}
if migration::vault_predates_current_schema(ctx).await? {
ctx.warn("[Sync] The local vault predates the current storage model, so its pending changes can be neither merged nor uploaded; taking the server's vault.").await;
let pulled = pull::pull(ctx).await?;
return store_server_vault(ctx, &pulled).await;
}
match merge::pull_and_merge(ctx).await? {
PullAndMergeOutcome::LegacyServer { revision } => {
ctx.warn(format!("[Sync] Server vault (revision {}) is still on the legacy storage format while the local vault is migrated; pushing the local vault.", revision)).await;
if push_local_changes(ctx, None, WriteKind::Everything).await?.is_none() {
return Ok(Flow::Resync { outdated: true });
}
Ok(Flow::Done(FullSyncResult { success: true, has_new_vault: server_directed_changes, ..Default::default() }))
}
PullAndMergeOutcome::ServerOnly(pulled) => store_server_vault(ctx, &pulled).await,
PullAndMergeOutcome::Merged { pulled, push_canonical } => {
if let Some(resync) = commit_pulled_vault(ctx, &pulled).await? {
return Ok(resync);
}
// The merge may have changed any manifest, so the push after it is never bucket-only.
let cache = push_canonical.map(|set| (ctx.mutation_sequence, set));
if push_local_changes(ctx, cache, WriteKind::Changes { buckets_only: false }).await?.is_none() {
return Ok(Flow::Resync { outdated: true });
}
Ok(Flow::Done(materialized_vault_result(ctx, &pulled).await?))
}
}
}
/// Push the pending local changes and clear the dirty flag. `None` when the server refused the write as outdated.
async fn push_local_changes(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)>, kind: WriteKind) -> SyncResult<Option<bool>> {
let upload = push::upload_vault(ctx, cache, kind).await?;
match upload.status {
PushStatus::Ok => {
state::mark_clean(&ctx.host, upload.mutation_seq_at_start).await?;
Ok(Some(upload.vault_changed))
}
PushStatus::Outdated => Ok(None),
}
}
/// The login-time key resolution.
async fn resolve_vault_key(ctx: &mut Ctx) -> ResolveVaultKeyResult {
match keys::resolve_vault_key(ctx).await {
Ok(has_vault_key) => ResolveVaultKeyResult { success: true, has_vault_key, encryption_key: ctx.encryption_key.clone(), ..Default::default() },
Err(error) => {
ctx.warn(format!("[Keys] Key resolution failed: {}", error)).await;
ResolveVaultKeyResult { failure: (&error).into(), ..Default::default() }
}
}
}
/// A lightweight status check: does the server hold newer state, and does the session still stand.
async fn status_check(ctx: &mut Ctx) -> StatusCheckResult {
let checked: SyncResult<StatusCheckResult> = async {
let status = http::get_status(&ctx.host).await?;
if status.server_version == http::SERVER_UNREACHABLE_VERSION {
return Ok(StatusCheckResult { success: true, has_dirty_changes: ctx.is_dirty, is_offline: true, ..Default::default() });
}
if let Some(reason) = version_logout_reason(ctx, &status) {
return Ok(StatusCheckResult { failure: FailureFields::logout(reason), server_version: Some(status.server_version), ..Default::default() });
}
assert_salt_unchanged(ctx, status.srp_salt.as_deref()).await?;
let has_newer_vault = server_state_needs_pull(ctx, &status).await?;
Ok(StatusCheckResult { success: true, has_newer_vault, has_dirty_changes: ctx.is_dirty, server_version: Some(status.server_version), capabilities: status.capabilities, email_domains: status.email_domains, pending_sharing_invitations: status.pending_sharing_invitations, ..Default::default() })
}
.await;
match checked {
Ok(result) => result,
Err(error) => {
let outcome = map_sync_failure(ctx, error).await;
StatusCheckResult { success: outcome.success, has_dirty_changes: ctx.is_dirty, is_offline: outcome.was_offline, failure: outcome.failure, ..Default::default() }
}
}
}