Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update vault sync pull compare to include databuckets (#2159)
Leendert de Borst7bae04eb0860
diff --git a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.tsindex 3ae320622..6bdf8db4e 100644--- a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts+++ b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts@@ -13,11 +13,11 @@ import type { DraftItem } from '@/utils/db/ItemRef';import { devError, devLog, devWarn } from '@/utils/devLogger/DevLogger';import type { EncryptionKeyDerivationParams } from '@/utils/dist/core/models/metadata';import { FieldKey, ItemTypes, VaultDataBucketCategory, createSystemField, type Item, type PasswordSettings } from '@/utils/dist/core/models/vault';-import { VaultKeyAlgorithm, type VaultResponse, type ManifestRevision, type StatusResponseV2 } from '@/utils/dist/core/models/webapi';+import { VaultKeyAlgorithm, type VaultResponse, type StatusResponseV2 } from '@/utils/dist/core/models/webapi';import { EncryptionUtility } from '@/utils/EncryptionUtility';import { requiresLegacyAccountKeyMigration } from '@/utils/legacy/LegacyStorageModelMigration';import { LocalPreferencesService } from '@/utils/LocalPreferencesService';-import { getManifestRevisions, manifestsRequiringPull, recordManifestRevisions, toManifestRevisionMap } from '@/utils/ManifestRevisions';+import { bucketsRequiringPull, getBucketRevisions, getManifestRevisions, manifestsRequiringPull, recordManifestRevisions, toBucketRevisionMap, toManifestRevisionMap } from '@/utils/ManifestRevisions';import { sendMessage, type TotpSecret } from '@/utils/messaging/ExtensionMessaging';import { multiManifestRendering } from '@/utils/MultiManifestRendering';import { PendingActionProcessor } from '@/utils/PendingActionProcessor';@@ -94,23 +94,31 @@ function clearInMemoryVaultState(): void {}/**- * Whether the client has to pull and re-materialize, i.e. whether any manifest's local state no longer matches what- * the server reports.- * @param serverManifests - the per-manifest revisions from the status response- * @param localRevisions - the client's last-known revision per manifest+ * Whether the client has to pull and re-materialize, i.e. whether any manifest, or any data bucket has been updated on the server.+ * @param statusResponse - the status response carrying the server's manifest and bucket revisions*/-function serverManifestsNeedPull(serverManifests: ManifestRevision[], localRevisions: Record<string, number>): boolean {- const serverRevisions = toManifestRevisionMap(serverManifests);- const requiringPull = manifestsRequiringPull(serverRevisions, localRevisions);+async function serverStateNeedsPull(statusResponse: StatusResponseV2): Promise<boolean> {+ const serverManifestRevisions = toManifestRevisionMap(statusResponse.manifestRevisions);+ const localManifestRevisions = await getManifestRevisions();+ const manifestsToPull = manifestsRequiringPull(serverManifestRevisions, localManifestRevisions);- if (requiringPull.length === 0) {- devLog(`[VaultSync] No pull needed: ${Object.keys(serverRevisions).length} manifest(s) all match local revisions.`);+ const serverBucketRevisions = toBucketRevisionMap(statusResponse.bucketRevisions);+ const localBucketRevisions = await getBucketRevisions();+ const bucketsToPull = bucketsRequiringPull(serverBucketRevisions, localBucketRevisions);++ if (manifestsToPull.length === 0 && bucketsToPull.length === 0) {+ devLog(`[VaultSync] No pull needed: ${Object.keys(serverManifestRevisions).length} manifest(s) and ${Object.keys(serverBucketRevisions).length} data bucket(s) all match local revisions.`);return false;}- /** One manifest as `id (local rev X, server rev Y)`, with "untracked"/"unlisted" for a one-sided manifest. */- const describe = (manifestId: string): string => `${manifestId} (local ${localRevisions[manifestId] ?? 'untracked'}, server ${serverRevisions[manifestId] ?? 'unlisted'})`;- devLog(`[VaultSync] Pull needed for ${requiringPull.length} manifest(s): ${requiringPull.map(describe).join(', ')}.`);+ // Log the manifests and buckets that need to be pulled.+ const describe = (server: Record<string, number>, local: Record<string, number>) => (key: string): string => `${key} (local ${local[key] ?? 'untracked'}, server ${server[key] ?? 'unlisted'})`;+ if (manifestsToPull.length > 0) {+ devLog(`[VaultSync] Pull needed for ${manifestsToPull.length} manifest(s): ${manifestsToPull.map(describe(serverManifestRevisions, localManifestRevisions)).join(', ')}.`);+ }+ if (bucketsToPull.length > 0) {+ devLog(`[VaultSync] Pull needed for ${bucketsToPull.length} data bucket(s): ${bucketsToPull.map(describe(serverBucketRevisions, localBucketRevisions)).join(', ')}.`);+ }return true;}@@ -121,11 +129,7 @@ function serverManifestsNeedPull(serverManifests: ManifestRevision[], localRevislet isSyncInProgress = false;let hasPendingSync = false;-/**- * How many times one chain of syncs may restart itself over a push the server called outdated. A pull settles a- * genuine conflict on the first retry; a chain that keeps going is one where the pull cannot repair what makes the- * push refuse itself, and repeating it forever would pull and merge the whole vault on every turn.- */+/** How many times one chain of syncs may restart itself over a push the server called outdated. */const MAX_OUTDATED_RESYNCS = 3;/** How many times the running chain of syncs has already restarted over an outdated push. */@@ -327,7 +331,7 @@ export async function handleSyncVault() : Promise<messageBoolResponse> {return { success: false, error: await t('common.errors.' + statusError) };}- if (serverManifestsNeedPull(statusResponse.manifestRevisions, await getManifestRevisions())) {+ if (await serverStateNeedsPull(statusResponse)) {/** Retrieve the latest vault from the server.*/@@ -1611,7 +1615,7 @@ async function runSyncPreflight(webApi: WebApiService, options?: FullVaultSyncOp// Get current sync stateconst syncState = await handleGetSyncState();- let needsPull = options?.forcePull === true || serverManifestsNeedPull(statusResponse.manifestRevisions, await getManifestRevisions());+ let needsPull = options?.forcePull === true || await serverStateNeedsPull(statusResponse);devLog(`[VaultSync] Status received (needsPull ${needsPull}, isDirty ${syncState.isDirty})`);diff --git a/apps/browser-extension/src/utils/ManifestRevisions.ts b/apps/browser-extension/src/utils/ManifestRevisions.tsindex 325d5ffb8..41f923ac7 100644--- a/apps/browser-extension/src/utils/ManifestRevisions.ts+++ b/apps/browser-extension/src/utils/ManifestRevisions.ts@@ -1,13 +1,14 @@/*** ManifestRevisions.*- * The client's last-known server revision per manifest.+ * The client's last-known server revisions: one per manifest, plus one per data bucket split out of a manifest.+ * The two counters advance independently, so both are compared before deciding a pull.*/import { storage } from 'wxt/utils/storage';-import { StorageKeys } from '@/utils/constants/storageKeys';-import type { ManifestRevision } from '@/utils/dist/core/models/webapi';+import { bucketRevisionKey, StorageKeys } from '@/utils/constants/storageKeys';+import type { BucketRevision, ManifestRevision } from '@/utils/dist/core/models/webapi';/*** Index a revision list as reported by the server (status call) by manifest id.@@ -17,6 +18,14 @@ export function toManifestRevisionMap(revisions: ManifestRevision[] | null | undreturn Object.fromEntries((revisions ?? []).map(m => [m.manifestId, m.revision]));}+/**+ * Index a data-bucket revision list as reported by the server (status call) by {@link bucketRevisionKey}.+ * @param revisions - the per-bucket revisions from the server+ */+export function toBucketRevisionMap(revisions: BucketRevision[] | null | undefined): Record<string, number> {+ return Object.fromEntries((revisions ?? []).map(b => [bucketRevisionKey(b.manifestId, b.category), b.revision]));+}+/*** The manifests whose local state no longer matches what the server reports, so the client has to pull and* re-materialize. Every manifest is judged on its own, the personal one included: the server reports it by id like@@ -35,6 +44,22 @@ export function manifestsRequiringPull(serverRevisions: Record<string, number>,return requiringPull;}+/**+ * The data buckets whose local state no longer matches what the server reports.+ *+ * Same shape and idempotence as the manifest comparison; keys are {@link bucketRevisionKey}.+ * @param serverRevisions - the server's current revision per data bucket+ * @param localRevisions - the client's last-known revision per data bucket+ */+export function bucketsRequiringPull(serverRevisions: Record<string, number>, localRevisions: Record<string, number>): string[] {+ const requiringPull = Object.keys(serverRevisions).filter(key => localRevisions[key] !== serverRevisions[key]);++ // A bucket the client still tracks but the server no longer lists was removed or revoked; re-materializing drops it.+ requiringPull.push(...Object.keys(localRevisions).filter(key => !(key in serverRevisions)));++ return requiringPull;+}+/*** Every manifest's last-known server revision, keyed by manifest id; empty when nothing has been pulled yet.*/@@ -58,6 +83,29 @@ export async function recordManifestRevisions(revisions: Record<string, number>)await replaceManifestRevisions({ ...await getManifestRevisions(), ...revisions });}+/**+ * Every data bucket's last-known server revision, keyed by {@link bucketRevisionKey}; empty when nothing has been pulled yet.+ */+export async function getBucketRevisions(): Promise<Record<string, number>> {+ return ((await storage.getItem(StorageKeys.VAULT_BUCKET_REVISIONS)) as Record<string, number> | null) ?? {};+}++/**+ * Replace with new data (from a pull snapshot), so the buckets of a manifest that is gone drop out with it.+ * @param revisions - the full set of bucket revisions from the snapshot+ */+export async function replaceBucketRevisions(revisions: Record<string, number>): Promise<void> {+ await storage.setItem(StorageKeys.VAULT_BUCKET_REVISIONS, revisions);+}++/**+ * Merge new revisions into the stored map, used after a write where only the buckets actually written advanced.+ * @param revisions - the bucket revisions to record+ */+export async function recordBucketRevisions(revisions: Record<string, number>): Promise<void> {+ await replaceBucketRevisions({ ...await getBucketRevisions(), ...revisions });+}+/*** The id of this vault's own (personal) manifest, as reported by the server on the last pull.*/diff --git a/apps/browser-extension/src/utils/VaultSyncService.ts b/apps/browser-extension/src/utils/VaultSyncService.tsindex 4d72dfa64..dbd39fed8 100644--- a/apps/browser-extension/src/utils/VaultSyncService.ts+++ b/apps/browser-extension/src/utils/VaultSyncService.ts@@ -14,7 +14,7 @@ import { VaultSqlGenerator } from '@/utils/dist/core/vault';import { buildEmailRouting } from '@/utils/EmailRouting';import { EncryptionUtility } from '@/utils/EncryptionUtility';import { completeLegacyAccountKeyMigration, isLegacySqliteBlobSnapshot, legacyUnstampedRowAdoption, openLegacySqliteBlobSnapshot, prepareLegacyAccountKeyMigration, withOutdatedServerGuard, type LegacyAccountKeyMigration, type LegacySqliteBlobSnapshot } from '@/utils/legacy/LegacyStorageModelMigration';-import { getManifestRevisions, getPersonalManifestId, recordManifestRevisions, replaceManifestRevisions, toManifestRevisionMap } from '@/utils/ManifestRevisions';+import { getBucketRevisions, getManifestRevisions, getPersonalManifestId, recordBucketRevisions, recordManifestRevisions, replaceBucketRevisions, replaceManifestRevisions, toManifestRevisionMap } from '@/utils/ManifestRevisions';import { multiManifestRendering } from '@/utils/MultiManifestRendering';import {vaultCodecComputeCiphertextHash, vaultCodecComputeContentFingerprint, vaultCodecCanonicalizeFromSqlite, vaultCodecExtractEncryptionKeyForPublicKey, vaultCodecGenerateManifestSalt, vaultCodecMergeCanonical, vaultCodecUnpackPayload, vaultCodecMaterializeAsSqlite, vaultCodecPackPayload, vaultCodecValidateManifest, vaultCodecValidateDataBucket, type CodecBlobEntry, type CodecCanonicalized, type CodecCanonicalManifestMerge, type CodecManifest, type CodecManifestSpec, vaultSharingPartitionManifestAccess, vaultSharingResolveManifestWriteSet, type SharingAccessPartition} from '@/utils/RustCore';import { SharingService, type ManifestVekGrant, type SharedManifestRecord } from '@/utils/SharingService';@@ -172,7 +172,7 @@ type OpenedManifestSet = {blobMap: Map<string, Uint8Array>;contentlessManifestIds: string[];personalRevision: number;- commitManifestRevisions: () => Promise<void>;+ commitRevisions: () => Promise<void>;};/** Aggregate canonical-merge statistics across all manifests. */@@ -472,7 +472,7 @@ export class VaultSyncService {devError('[V2Merge] Canonical merge failed, falling back to the server vault:', mergeError);const sqliteBytes = await this.materializeToSqlite(opened.resolved.map(m => m.manifest), opened.dataBuckets, opened.blobMap);const encryptedVault = await encryptVaultBlob(sqliteBytes, encryptionKey);- return { kind: 'server-only', response: this.buildResponse(encryptedVault, '2.0.0', opened.personalRevision, snapshot), commitRevisions: opened.commitManifestRevisions };+ return { kind: 'server-only', response: this.buildResponse(encryptedVault, '2.0.0', opened.personalRevision, snapshot), commitRevisions: opened.commitRevisions };}} catch (error) {devError('[V2Merge] FAILED: the last logged step above is where it broke:', error);@@ -607,7 +607,7 @@ export class VaultSyncService {stats,fallbackManifestIds,droppedLocalManifestIds: mergeOutput.droppedLocalManifestIds,- commitRevisions: opened.commitManifestRevisions,+ commitRevisions: opened.commitRevisions,mergedSqliteBytes: sqliteBytes,pushCanonical: this.mergeOutputForPush(manifests, dataBuckets, mergedBlobs, manifestRecords, fallbackManifestIds.length > 0),};@@ -787,7 +787,7 @@ export class VaultSyncService {* this device's sync state.* @param snapshot - the raw GET /v2/Vault response* @param vek - the personal manifest's symmetric key (from the unlock chain); every other manifest key resolves from it- * @param options - set deferRevisionCommit to hand the revision write back to the caller as `commitManifestRevisions`,+ * @param options - set deferRevisionCommit to hand the revision write back to the caller as `commitRevisions`,* for when the pulled revisions may only become local truth once a later step has succeeded*/private async openManifestsAndAdoptSyncState(snapshot: GetResponseDto, vek: string, options?: { deferRevisionCommit?: boolean }): Promise<OpenedManifestSet> {@@ -861,20 +861,25 @@ export class VaultSyncService {const personal = resolved[0];// 2) Open the data buckets belonging to those manifests.- const { dataBuckets, bucketFingerprints } = await this.openDataBuckets(snapshot, resolved);+ const { dataBuckets, bucketFingerprints, bucketRevisions } = await this.openDataBuckets(snapshot, resolved);Object.assign(pulledFingerprints, bucketFingerprints);// 3) Revision baselines: what the next status check compares this device against.const manifestRevisions = { ...Object.fromEntries(resolved.map(m => [m.manifestId, m.revision])), ...contentlessRevisions };/**- * Commit the snapshot's revision map as the local believed-current revisions.+ * Commit the snapshot's revision maps, manifests and data buckets alike, as the local believed-current+ * revisions. Both are replaced rather than merged, so a manifest that is gone takes its buckets with it.+ * The two move independently and the status check compares both, so they have to be committed together:+ * committing the buckets earlier would clear a pending bucket pull that the manifests cannot re-trigger.*/- const commitManifestRevisions = async (): Promise<void> => {+ const commitRevisions = async (): Promise<void> => {await replaceManifestRevisions(manifestRevisions);+ await replaceBucketRevisions(bucketRevisions);devLog(`[V2Pull] Stored local manifest revisions from snapshot: ${Object.entries(manifestRevisions).map(([id, rev]) => `${id}=${rev}`).join(', ')}. Next status check compares against these.`);+ devLog(`[V2Pull] Stored local data bucket revisions from snapshot: ${Object.entries(bucketRevisions).map(([key, rev]) => `${key}=${rev}`).join(', ') || 'none'}.`);};if (options?.deferRevisionCommit !== true) {- await commitManifestRevisions();+ await commitRevisions();}// 4) Fetch and decrypt every blob the opened manifests reference.@@ -897,7 +902,7 @@ export class VaultSyncService {blobMap,contentlessManifestIds: Object.keys(contentlessRevisions),personalRevision: personal.revision,- commitManifestRevisions,+ commitRevisions,};}@@ -905,9 +910,10 @@ export class VaultSyncService {* Open the data buckets a snapshot carries and record their revisions as the local baseline.* @param snapshot - the raw GET /v2/Vault response* @param resolved - the manifests already opened; a bucket addressed to any other manifest is refused- * @returns The decrypted buckets plus one content fingerprint per bucket, for the push-side change detection+ * @returns The decrypted buckets, one content fingerprint per bucket for the push-side change detection, and the+ * snapshot's bucket revisions, which the caller commits together with the manifest revisions*/- private async openDataBuckets(snapshot: GetResponseDto, resolved: ResolvedManifest[]): Promise<{ dataBuckets: VaultDataBucket[]; bucketFingerprints: Record<string, string> }> {+ private async openDataBuckets(snapshot: GetResponseDto, resolved: ResolvedManifest[]): Promise<{ dataBuckets: VaultDataBucket[]; bucketFingerprints: Record<string, string>; bucketRevisions: Record<string, number> }> {const bucketFingerprints: Record<string, string> = {};const keyByManifestId = new Map(resolved.map(entry => [entry.manifestId, entry.vek]));const dataBuckets: VaultDataBucket[] = [];@@ -937,13 +943,11 @@ export class VaultSyncService {pulledBucketRevisions[bucketRevisionKey(bucketDto.manifestId, bucketDto.category)] = bucketDto.revision;}}- // Replace rather than merge, so the buckets of a manifest that is gone drop out with it.- await storage.setItem(StorageKeys.VAULT_BUCKET_REVISIONS, pulledBucketRevisions);if (dataBuckets.length === 0) {devLog('[V2Pull] No data buckets in snapshot.');}- return { dataBuckets, bucketFingerprints };+ return { dataBuckets, bucketFingerprints, bucketRevisions: pulledBucketRevisions };}/**@@ -1295,7 +1299,7 @@ export class VaultSyncService {*/const bucketDtos: Array<{ manifestId: string; category: string; blob: string; ciphertextHash: string; currentRevision: number }> = [];const writtenBucketFingerprints: Record<string, string> = {};- const storedBucketRevisions = await this.loadBucketRevisions();+ const storedBucketRevisions = await getBucketRevisions();const keyByManifestId = new Map(candidates.map(candidate => [candidate.manifestId, candidate.vek]));for (const bucket of canonicalizedBuckets) {const label = `Data bucket "${bucket.category}" of manifest ${bucket.manifestId}`;@@ -1486,11 +1490,7 @@ export class VaultSyncService {// 5) Update local persisted state on success.if ((resp.bucketRevisions ?? []).length > 0) {- const bucketRevisions = await this.loadBucketRevisions();- for (const br of resp.bucketRevisions) {- bucketRevisions[bucketRevisionKey(br.manifestId, br.category)] = br.revision;- }- await storage.setItem(StorageKeys.VAULT_BUCKET_REVISIONS, bucketRevisions);+ await recordBucketRevisions(Object.fromEntries(resp.bucketRevisions.map(br => [bucketRevisionKey(br.manifestId, br.category), br.revision])));}// Advance the baseline of the manifests this write included.@@ -1631,7 +1631,7 @@ export class VaultSyncService {// Same content-fingerprint gate as the full push: a mutation that ended up changing nothing skips the write.const fingerprints = await this.loadContentFingerprints();- const bucketRevisions = await this.loadBucketRevisions();+ const bucketRevisions = await getBucketRevisions();const bucketFingerprint = await vaultCodecComputeContentFingerprint(plaintext);if (fingerprints[fingerprintBucketKey(manifestId, category)] === bucketFingerprint) {devLog(`[V2Push] ${label} (bucket-only) unchanged versus server baseline, skipping upload.`);@@ -1667,8 +1667,7 @@ export class VaultSyncService {}const newRevision = reportedRevision(resp) ?? currentRevision + 1;- bucketRevisions[revisionKey] = newRevision;- await storage.setItem(StorageKeys.VAULT_BUCKET_REVISIONS, bucketRevisions);+ await recordBucketRevisions({ [revisionKey]: newRevision });// New server baseline for this bucket, so an unchanged follow-up push (bucket-only or full) can skip it.fingerprints[fingerprintBucketKey(manifestId, category)] = bucketFingerprint;@@ -1897,14 +1896,6 @@ export class VaultSyncService {await storage.setItem(StorageKeys.VAULT_CONTENT_FINGERPRINTS, fingerprints);}- /**- * Load the known server revision of every data bucket, keyed by {@link bucketRevisionKey}. An absent entry- * reads as revision 0, which the server answers with its current one so the next attempt rebases.- */- private async loadBucketRevisions(): Promise<Record<string, number>> {- return ((await storage.getItem(StorageKeys.VAULT_BUCKET_REVISIONS)) as Record<string, number> | null) ?? {};- }-/*** Load the local encrypted blob cache (hash → base64 ciphertext) used to skip re-downloading known blobs.* Entries are stored as served to/from the server, so nothing in the cache is plaintext at rest.