Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update vault sync pull compare to include databuckets (#2159)

Leendert de Borst7bae04eb0860
diff --git a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
index 3ae320622..6bdf8db4e 100644
--- a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
+++ b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
@@ -13,11 +13,11 @@ import type { DraftItem } from '@/utils/db/ItemRef';
import { devError, devLog, devWarn } from '@/utils/devLogger/DevLogger';
import type { EncryptionKeyDerivationParams } from '@/utils/dist/core/models/metadata';
import { FieldKey, ItemTypes, VaultDataBucketCategory, createSystemField, type Item, type PasswordSettings } from '@/utils/dist/core/models/vault';
-import { VaultKeyAlgorithm, type VaultResponse, type ManifestRevision, type StatusResponseV2 } from '@/utils/dist/core/models/webapi';
+import { VaultKeyAlgorithm, type VaultResponse, type StatusResponseV2 } from '@/utils/dist/core/models/webapi';
import { EncryptionUtility } from '@/utils/EncryptionUtility';
import { requiresLegacyAccountKeyMigration } from '@/utils/legacy/LegacyStorageModelMigration';
import { LocalPreferencesService } from '@/utils/LocalPreferencesService';
-import { getManifestRevisions, manifestsRequiringPull, recordManifestRevisions, toManifestRevisionMap } from '@/utils/ManifestRevisions';
+import { bucketsRequiringPull, getBucketRevisions, getManifestRevisions, manifestsRequiringPull, recordManifestRevisions, toBucketRevisionMap, toManifestRevisionMap } from '@/utils/ManifestRevisions';
import { sendMessage, type TotpSecret } from '@/utils/messaging/ExtensionMessaging';
import { multiManifestRendering } from '@/utils/MultiManifestRendering';
import { PendingActionProcessor } from '@/utils/PendingActionProcessor';
@@ -94,23 +94,31 @@ function clearInMemoryVaultState(): void {
}
/**
- * Whether the client has to pull and re-materialize, i.e. whether any manifest's local state no longer matches what
- * the server reports.
- * @param serverManifests - the per-manifest revisions from the status response
- * @param localRevisions - the client's last-known revision per manifest
+ * Whether the client has to pull and re-materialize, i.e. whether any manifest, or any data bucket has been updated on the server.
+ * @param statusResponse - the status response carrying the server's manifest and bucket revisions
*/
-function serverManifestsNeedPull(serverManifests: ManifestRevision[], localRevisions: Record<string, number>): boolean {
- const serverRevisions = toManifestRevisionMap(serverManifests);
- const requiringPull = manifestsRequiringPull(serverRevisions, localRevisions);
+async function serverStateNeedsPull(statusResponse: StatusResponseV2): Promise<boolean> {
+ const serverManifestRevisions = toManifestRevisionMap(statusResponse.manifestRevisions);
+ const localManifestRevisions = await getManifestRevisions();
+ const manifestsToPull = manifestsRequiringPull(serverManifestRevisions, localManifestRevisions);
- if (requiringPull.length === 0) {
- devLog(`[VaultSync] No pull needed: ${Object.keys(serverRevisions).length} manifest(s) all match local revisions.`);
+ const serverBucketRevisions = toBucketRevisionMap(statusResponse.bucketRevisions);
+ const localBucketRevisions = await getBucketRevisions();
+ const bucketsToPull = bucketsRequiringPull(serverBucketRevisions, localBucketRevisions);
+
+ if (manifestsToPull.length === 0 && bucketsToPull.length === 0) {
+ devLog(`[VaultSync] No pull needed: ${Object.keys(serverManifestRevisions).length} manifest(s) and ${Object.keys(serverBucketRevisions).length} data bucket(s) all match local revisions.`);
return false;
}
- /** One manifest as `id (local rev X, server rev Y)`, with "untracked"/"unlisted" for a one-sided manifest. */
- const describe = (manifestId: string): string => `${manifestId} (local ${localRevisions[manifestId] ?? 'untracked'}, server ${serverRevisions[manifestId] ?? 'unlisted'})`;
- devLog(`[VaultSync] Pull needed for ${requiringPull.length} manifest(s): ${requiringPull.map(describe).join(', ')}.`);
+ // Log the manifests and buckets that need to be pulled.
+ const describe = (server: Record<string, number>, local: Record<string, number>) => (key: string): string => `${key} (local ${local[key] ?? 'untracked'}, server ${server[key] ?? 'unlisted'})`;
+ if (manifestsToPull.length > 0) {
+ devLog(`[VaultSync] Pull needed for ${manifestsToPull.length} manifest(s): ${manifestsToPull.map(describe(serverManifestRevisions, localManifestRevisions)).join(', ')}.`);
+ }
+ if (bucketsToPull.length > 0) {
+ devLog(`[VaultSync] Pull needed for ${bucketsToPull.length} data bucket(s): ${bucketsToPull.map(describe(serverBucketRevisions, localBucketRevisions)).join(', ')}.`);
+ }
return true;
}
@@ -121,11 +129,7 @@ function serverManifestsNeedPull(serverManifests: ManifestRevision[], localRevis
let isSyncInProgress = false;
let hasPendingSync = false;
-/**
- * How many times one chain of syncs may restart itself over a push the server called outdated. A pull settles a
- * genuine conflict on the first retry; a chain that keeps going is one where the pull cannot repair what makes the
- * push refuse itself, and repeating it forever would pull and merge the whole vault on every turn.
- */
+/** How many times one chain of syncs may restart itself over a push the server called outdated. */
const MAX_OUTDATED_RESYNCS = 3;
/** How many times the running chain of syncs has already restarted over an outdated push. */
@@ -327,7 +331,7 @@ export async function handleSyncVault() : Promise<messageBoolResponse> {
return { success: false, error: await t('common.errors.' + statusError) };
}
- if (serverManifestsNeedPull(statusResponse.manifestRevisions, await getManifestRevisions())) {
+ if (await serverStateNeedsPull(statusResponse)) {
/*
* Retrieve the latest vault from the server.
*/
@@ -1611,7 +1615,7 @@ async function runSyncPreflight(webApi: WebApiService, options?: FullVaultSyncOp
// Get current sync state
const syncState = await handleGetSyncState();
- let needsPull = options?.forcePull === true || serverManifestsNeedPull(statusResponse.manifestRevisions, await getManifestRevisions());
+ let needsPull = options?.forcePull === true || await serverStateNeedsPull(statusResponse);
devLog(`[VaultSync] Status received (needsPull ${needsPull}, isDirty ${syncState.isDirty})`);
diff --git a/apps/browser-extension/src/utils/ManifestRevisions.ts b/apps/browser-extension/src/utils/ManifestRevisions.ts
index 325d5ffb8..41f923ac7 100644
--- a/apps/browser-extension/src/utils/ManifestRevisions.ts
+++ b/apps/browser-extension/src/utils/ManifestRevisions.ts
@@ -1,13 +1,14 @@
/**
* ManifestRevisions.
*
- * The client's last-known server revision per manifest.
+ * The client's last-known server revisions: one per manifest, plus one per data bucket split out of a manifest.
+ * The two counters advance independently, so both are compared before deciding a pull.
*/
import { storage } from 'wxt/utils/storage';
-import { StorageKeys } from '@/utils/constants/storageKeys';
-import type { ManifestRevision } from '@/utils/dist/core/models/webapi';
+import { bucketRevisionKey, StorageKeys } from '@/utils/constants/storageKeys';
+import type { BucketRevision, ManifestRevision } from '@/utils/dist/core/models/webapi';
/**
* Index a revision list as reported by the server (status call) by manifest id.
@@ -17,6 +18,14 @@ export function toManifestRevisionMap(revisions: ManifestRevision[] | null | und
return Object.fromEntries((revisions ?? []).map(m => [m.manifestId, m.revision]));
}
+/**
+ * Index a data-bucket revision list as reported by the server (status call) by {@link bucketRevisionKey}.
+ * @param revisions - the per-bucket revisions from the server
+ */
+export function toBucketRevisionMap(revisions: BucketRevision[] | null | undefined): Record<string, number> {
+ return Object.fromEntries((revisions ?? []).map(b => [bucketRevisionKey(b.manifestId, b.category), b.revision]));
+}
+
/**
* The manifests whose local state no longer matches what the server reports, so the client has to pull and
* re-materialize. Every manifest is judged on its own, the personal one included: the server reports it by id like
@@ -35,6 +44,22 @@ export function manifestsRequiringPull(serverRevisions: Record<string, number>,
return requiringPull;
}
+/**
+ * The data buckets whose local state no longer matches what the server reports.
+ *
+ * Same shape and idempotence as the manifest comparison; keys are {@link bucketRevisionKey}.
+ * @param serverRevisions - the server's current revision per data bucket
+ * @param localRevisions - the client's last-known revision per data bucket
+ */
+export function bucketsRequiringPull(serverRevisions: Record<string, number>, localRevisions: Record<string, number>): string[] {
+ const requiringPull = Object.keys(serverRevisions).filter(key => localRevisions[key] !== serverRevisions[key]);
+
+ // A bucket the client still tracks but the server no longer lists was removed or revoked; re-materializing drops it.
+ requiringPull.push(...Object.keys(localRevisions).filter(key => !(key in serverRevisions)));
+
+ return requiringPull;
+}
+
/**
* Every manifest's last-known server revision, keyed by manifest id; empty when nothing has been pulled yet.
*/
@@ -58,6 +83,29 @@ export async function recordManifestRevisions(revisions: Record<string, number>)
await replaceManifestRevisions({ ...await getManifestRevisions(), ...revisions });
}
+/**
+ * Every data bucket's last-known server revision, keyed by {@link bucketRevisionKey}; empty when nothing has been pulled yet.
+ */
+export async function getBucketRevisions(): Promise<Record<string, number>> {
+ return ((await storage.getItem(StorageKeys.VAULT_BUCKET_REVISIONS)) as Record<string, number> | null) ?? {};
+}
+
+/**
+ * Replace with new data (from a pull snapshot), so the buckets of a manifest that is gone drop out with it.
+ * @param revisions - the full set of bucket revisions from the snapshot
+ */
+export async function replaceBucketRevisions(revisions: Record<string, number>): Promise<void> {
+ await storage.setItem(StorageKeys.VAULT_BUCKET_REVISIONS, revisions);
+}
+
+/**
+ * Merge new revisions into the stored map, used after a write where only the buckets actually written advanced.
+ * @param revisions - the bucket revisions to record
+ */
+export async function recordBucketRevisions(revisions: Record<string, number>): Promise<void> {
+ await replaceBucketRevisions({ ...await getBucketRevisions(), ...revisions });
+}
+
/**
* The id of this vault's own (personal) manifest, as reported by the server on the last pull.
*/
diff --git a/apps/browser-extension/src/utils/VaultSyncService.ts b/apps/browser-extension/src/utils/VaultSyncService.ts
index 4d72dfa64..dbd39fed8 100644
--- a/apps/browser-extension/src/utils/VaultSyncService.ts
+++ b/apps/browser-extension/src/utils/VaultSyncService.ts
@@ -14,7 +14,7 @@ import { VaultSqlGenerator } from '@/utils/dist/core/vault';
import { buildEmailRouting } from '@/utils/EmailRouting';
import { EncryptionUtility } from '@/utils/EncryptionUtility';
import { completeLegacyAccountKeyMigration, isLegacySqliteBlobSnapshot, legacyUnstampedRowAdoption, openLegacySqliteBlobSnapshot, prepareLegacyAccountKeyMigration, withOutdatedServerGuard, type LegacyAccountKeyMigration, type LegacySqliteBlobSnapshot } from '@/utils/legacy/LegacyStorageModelMigration';
-import { getManifestRevisions, getPersonalManifestId, recordManifestRevisions, replaceManifestRevisions, toManifestRevisionMap } from '@/utils/ManifestRevisions';
+import { getBucketRevisions, getManifestRevisions, getPersonalManifestId, recordBucketRevisions, recordManifestRevisions, replaceBucketRevisions, replaceManifestRevisions, toManifestRevisionMap } from '@/utils/ManifestRevisions';
import { multiManifestRendering } from '@/utils/MultiManifestRendering';
import {vaultCodecComputeCiphertextHash, vaultCodecComputeContentFingerprint, vaultCodecCanonicalizeFromSqlite, vaultCodecExtractEncryptionKeyForPublicKey, vaultCodecGenerateManifestSalt, vaultCodecMergeCanonical, vaultCodecUnpackPayload, vaultCodecMaterializeAsSqlite, vaultCodecPackPayload, vaultCodecValidateManifest, vaultCodecValidateDataBucket, type CodecBlobEntry, type CodecCanonicalized, type CodecCanonicalManifestMerge, type CodecManifest, type CodecManifestSpec, vaultSharingPartitionManifestAccess, vaultSharingResolveManifestWriteSet, type SharingAccessPartition} from '@/utils/RustCore';
import { SharingService, type ManifestVekGrant, type SharedManifestRecord } from '@/utils/SharingService';
@@ -172,7 +172,7 @@ type OpenedManifestSet = {
blobMap: Map<string, Uint8Array>;
contentlessManifestIds: string[];
personalRevision: number;
- commitManifestRevisions: () => Promise<void>;
+ commitRevisions: () => Promise<void>;
};
/** Aggregate canonical-merge statistics across all manifests. */
@@ -472,7 +472,7 @@ export class VaultSyncService {
devError('[V2Merge] Canonical merge failed, falling back to the server vault:', mergeError);
const sqliteBytes = await this.materializeToSqlite(opened.resolved.map(m => m.manifest), opened.dataBuckets, opened.blobMap);
const encryptedVault = await encryptVaultBlob(sqliteBytes, encryptionKey);
- return { kind: 'server-only', response: this.buildResponse(encryptedVault, '2.0.0', opened.personalRevision, snapshot), commitRevisions: opened.commitManifestRevisions };
+ return { kind: 'server-only', response: this.buildResponse(encryptedVault, '2.0.0', opened.personalRevision, snapshot), commitRevisions: opened.commitRevisions };
}
} catch (error) {
devError('[V2Merge] FAILED: the last logged step above is where it broke:', error);
@@ -607,7 +607,7 @@ export class VaultSyncService {
stats,
fallbackManifestIds,
droppedLocalManifestIds: mergeOutput.droppedLocalManifestIds,
- commitRevisions: opened.commitManifestRevisions,
+ commitRevisions: opened.commitRevisions,
mergedSqliteBytes: sqliteBytes,
pushCanonical: this.mergeOutputForPush(manifests, dataBuckets, mergedBlobs, manifestRecords, fallbackManifestIds.length > 0),
};
@@ -787,7 +787,7 @@ export class VaultSyncService {
* this device's sync state.
* @param snapshot - the raw GET /v2/Vault response
* @param vek - the personal manifest's symmetric key (from the unlock chain); every other manifest key resolves from it
- * @param options - set deferRevisionCommit to hand the revision write back to the caller as `commitManifestRevisions`,
+ * @param options - set deferRevisionCommit to hand the revision write back to the caller as `commitRevisions`,
* for when the pulled revisions may only become local truth once a later step has succeeded
*/
private async openManifestsAndAdoptSyncState(snapshot: GetResponseDto, vek: string, options?: { deferRevisionCommit?: boolean }): Promise<OpenedManifestSet> {
@@ -861,20 +861,25 @@ export class VaultSyncService {
const personal = resolved[0];
// 2) Open the data buckets belonging to those manifests.
- const { dataBuckets, bucketFingerprints } = await this.openDataBuckets(snapshot, resolved);
+ const { dataBuckets, bucketFingerprints, bucketRevisions } = await this.openDataBuckets(snapshot, resolved);
Object.assign(pulledFingerprints, bucketFingerprints);
// 3) Revision baselines: what the next status check compares this device against.
const manifestRevisions = { ...Object.fromEntries(resolved.map(m => [m.manifestId, m.revision])), ...contentlessRevisions };
/**
- * Commit the snapshot's revision map as the local believed-current revisions.
+ * Commit the snapshot's revision maps, manifests and data buckets alike, as the local believed-current
+ * revisions. Both are replaced rather than merged, so a manifest that is gone takes its buckets with it.
+ * The two move independently and the status check compares both, so they have to be committed together:
+ * committing the buckets earlier would clear a pending bucket pull that the manifests cannot re-trigger.
*/
- const commitManifestRevisions = async (): Promise<void> => {
+ const commitRevisions = async (): Promise<void> => {
await replaceManifestRevisions(manifestRevisions);
+ await replaceBucketRevisions(bucketRevisions);
devLog(`[V2Pull] Stored local manifest revisions from snapshot: ${Object.entries(manifestRevisions).map(([id, rev]) => `${id}=${rev}`).join(', ')}. Next status check compares against these.`);
+ devLog(`[V2Pull] Stored local data bucket revisions from snapshot: ${Object.entries(bucketRevisions).map(([key, rev]) => `${key}=${rev}`).join(', ') || 'none'}.`);
};
if (options?.deferRevisionCommit !== true) {
- await commitManifestRevisions();
+ await commitRevisions();
}
// 4) Fetch and decrypt every blob the opened manifests reference.
@@ -897,7 +902,7 @@ export class VaultSyncService {
blobMap,
contentlessManifestIds: Object.keys(contentlessRevisions),
personalRevision: personal.revision,
- commitManifestRevisions,
+ commitRevisions,
};
}
@@ -905,9 +910,10 @@ export class VaultSyncService {
* Open the data buckets a snapshot carries and record their revisions as the local baseline.
* @param snapshot - the raw GET /v2/Vault response
* @param resolved - the manifests already opened; a bucket addressed to any other manifest is refused
- * @returns The decrypted buckets plus one content fingerprint per bucket, for the push-side change detection
+ * @returns The decrypted buckets, one content fingerprint per bucket for the push-side change detection, and the
+ * snapshot's bucket revisions, which the caller commits together with the manifest revisions
*/
- private async openDataBuckets(snapshot: GetResponseDto, resolved: ResolvedManifest[]): Promise<{ dataBuckets: VaultDataBucket[]; bucketFingerprints: Record<string, string> }> {
+ private async openDataBuckets(snapshot: GetResponseDto, resolved: ResolvedManifest[]): Promise<{ dataBuckets: VaultDataBucket[]; bucketFingerprints: Record<string, string>; bucketRevisions: Record<string, number> }> {
const bucketFingerprints: Record<string, string> = {};
const keyByManifestId = new Map(resolved.map(entry => [entry.manifestId, entry.vek]));
const dataBuckets: VaultDataBucket[] = [];
@@ -937,13 +943,11 @@ export class VaultSyncService {
pulledBucketRevisions[bucketRevisionKey(bucketDto.manifestId, bucketDto.category)] = bucketDto.revision;
}
}
- // Replace rather than merge, so the buckets of a manifest that is gone drop out with it.
- await storage.setItem(StorageKeys.VAULT_BUCKET_REVISIONS, pulledBucketRevisions);
if (dataBuckets.length === 0) {
devLog('[V2Pull] No data buckets in snapshot.');
}
- return { dataBuckets, bucketFingerprints };
+ return { dataBuckets, bucketFingerprints, bucketRevisions: pulledBucketRevisions };
}
/**
@@ -1295,7 +1299,7 @@ export class VaultSyncService {
*/
const bucketDtos: Array<{ manifestId: string; category: string; blob: string; ciphertextHash: string; currentRevision: number }> = [];
const writtenBucketFingerprints: Record<string, string> = {};
- const storedBucketRevisions = await this.loadBucketRevisions();
+ const storedBucketRevisions = await getBucketRevisions();
const keyByManifestId = new Map(candidates.map(candidate => [candidate.manifestId, candidate.vek]));
for (const bucket of canonicalizedBuckets) {
const label = `Data bucket "${bucket.category}" of manifest ${bucket.manifestId}`;
@@ -1486,11 +1490,7 @@ export class VaultSyncService {
// 5) Update local persisted state on success.
if ((resp.bucketRevisions ?? []).length > 0) {
- const bucketRevisions = await this.loadBucketRevisions();
- for (const br of resp.bucketRevisions) {
- bucketRevisions[bucketRevisionKey(br.manifestId, br.category)] = br.revision;
- }
- await storage.setItem(StorageKeys.VAULT_BUCKET_REVISIONS, bucketRevisions);
+ await recordBucketRevisions(Object.fromEntries(resp.bucketRevisions.map(br => [bucketRevisionKey(br.manifestId, br.category), br.revision])));
}
// Advance the baseline of the manifests this write included.
@@ -1631,7 +1631,7 @@ export class VaultSyncService {
// Same content-fingerprint gate as the full push: a mutation that ended up changing nothing skips the write.
const fingerprints = await this.loadContentFingerprints();
- const bucketRevisions = await this.loadBucketRevisions();
+ const bucketRevisions = await getBucketRevisions();
const bucketFingerprint = await vaultCodecComputeContentFingerprint(plaintext);
if (fingerprints[fingerprintBucketKey(manifestId, category)] === bucketFingerprint) {
devLog(`[V2Push] ${label} (bucket-only) unchanged versus server baseline, skipping upload.`);
@@ -1667,8 +1667,7 @@ export class VaultSyncService {
}
const newRevision = reportedRevision(resp) ?? currentRevision + 1;
- bucketRevisions[revisionKey] = newRevision;
- await storage.setItem(StorageKeys.VAULT_BUCKET_REVISIONS, bucketRevisions);
+ await recordBucketRevisions({ [revisionKey]: newRevision });
// New server baseline for this bucket, so an unchanged follow-up push (bucket-only or full) can skip it.
fingerprints[fingerprintBucketKey(manifestId, category)] = bucketFingerprint;
@@ -1897,14 +1896,6 @@ export class VaultSyncService {
await storage.setItem(StorageKeys.VAULT_CONTENT_FINGERPRINTS, fingerprints);
}
- /**
- * Load the known server revision of every data bucket, keyed by {@link bucketRevisionKey}. An absent entry
- * reads as revision 0, which the server answers with its current one so the next attempt rebases.
- */
- private async loadBucketRevisions(): Promise<Record<string, number>> {
- return ((await storage.getItem(StorageKeys.VAULT_BUCKET_REVISIONS)) as Record<string, number> | null) ?? {};
- }
-
/**
* Load the local encrypted blob cache (hash → base64 ciphertext) used to skip re-downloading known blobs.
* Entries are stored as served to/from the server, so nothing in the cache is plaintext at rest.