Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update bucket update flow (#2159)
Leendert de Borst1a16f74a76a7
diff --git a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.tsindex da3c39821..c1ce8987d 100644--- a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts+++ b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts@@ -4,20 +4,23 @@ import { storage } from 'wxt/utils/storage';import { allVaultDataStorageKeys, AUTH_STORAGE_KEYS, dirtyScopeStorageKey, SESSION_STORAGE_KEYS, StorageKeys, VAULT_LOCK_STORAGE_KEYS } from '@/utils/constants/storageKeys';import { TRASH_RETENTION_DAYS } from '@/utils/constants/vault';+import type { ItemUsageAction } from '@/utils/db';import type { DraftItem } from '@/utils/db/ItemRef';import { devError, devLog, devWarn } from '@/utils/devLogger/DevLogger';import type { EncryptionKeyDerivationParams } from '@/utils/dist/core/models/metadata';-import { FieldKey, ItemTypes, createSystemField, type Item, type PasswordSettings } from '@/utils/dist/core/models/vault';-import type { VaultResponse, VaultPostResponse, StatusResponseV2, ManifestRevision } from '@/utils/dist/core/models/webapi';+import { FieldKey, ItemTypes, VaultDataBucketCategory, createSystemField, type Item, type PasswordSettings } from '@/utils/dist/core/models/vault';+import type { VaultResponse, StatusResponseV2, ManifestRevision } from '@/utils/dist/core/models/webapi';import { EncryptionUtility } from '@/utils/EncryptionUtility';-import { requiresLegacyVaultKeyMigration } from '@/utils/LegacyVaultKeyMigration';+import { readLegacySessionEncryptionKey } from '@/utils/legacy/LegacyStorageKeyFallbacks';+import { requiresLegacyAccountKeyMigration } from '@/utils/legacy/LegacyStorageModelMigration';import { LocalPreferencesService } from '@/utils/LocalPreferencesService';+import { getManifestRevisions, getPersonalManifestId } from '@/utils/ManifestRevisions';import { sendMessage } from '@/utils/messaging/ExtensionMessaging';import { RecentlySelectedItemService } from '@/utils/RecentlySelectedItemService';-import { filterItems, AutofillMatchingMode, extractRootDomain, isUrlAlreadyLinked, generatePassword, vaultCodecExtractBucket, vaultCodecBucketLayout, vaultCodecOverflowTable } from '@/utils/RustCore';+import { filterItems, AutofillMatchingMode, extractRootDomain, isUrlAlreadyLinked, generatePassword, vaultCodecExtractBuckets, vaultCodecBucketLayout, vaultCodecOverflowTable } from '@/utils/RustCore';import { ServiceDetectionUtility } from '@/utils/serviceDetection/ServiceDetectionUtility';import { SqliteClient } from '@/utils/SqliteClient';-import { getItemWithFallback } from '@/utils/StorageUtility';+import { getStorageItem } from '@/utils/StorageUtility';import { ApiAuthError } from '@/utils/types/errors/ApiAuthError';import { ApiRequestError } from '@/utils/types/errors/ApiRequestError';import { AppErrorCode, formatErrorWithCode } from '@/utils/types/errors/AppErrorCodes';@@ -69,59 +72,36 @@ function personalManifestRevision(status: StatusResponseV2): number {return personal?.revision ?? 0;}-/** The client's last-known revision per shared manifest (manifestId → revision); empty when no shared manifests. */-async function getLocalSharedManifestRevisions(): Promise<Record<string, number>> {- return (await storage.getItem(StorageKeys.SERVER_MANIFEST_REVISIONS)) as Record<string, number> | null ?? {};-}-/*** Whether the client should pull and re-materialize because the server's manifest set no longer matches what the- * client has locally materialized. Compares the full server list against the client's per-manifest last-known- * revisions and is idempotent: after a successful materialize the local record matches+ * client has locally materialized. One map-versus-map comparison covers every manifest, the personal one included:+ * the server reports it by id like any other. Idempotent: after a successful materialize the local record matches* the server, so this returns false until the next server-side change.- *+ ** @param serverManifests - the per-manifest revisions from the status response- * @param personalManifestId - the id the server names as the caller's own; every other entry is a shared manifest- * @param localMainRevision - the client's last-known personal manifest revision (local:serverRevision)- * @param localSharedRevisions - the client's last-known revision per shared manifest- */-function serverManifestsNeedPull(serverManifests: ManifestRevision[], personalManifestId: string | null, localMainRevision: number, localSharedRevisions: Record<string, number>): boolean {- const serverShared = new Map<string, number>();- let serverPersonalRevision: number | null = null;- for (const m of (serverManifests ?? [])) {- if (personalManifestId !== null && m.manifestId === personalManifestId) {- serverPersonalRevision = m.revision;- } else {- serverShared.set(m.manifestId, m.revision);- }- }-- /*- * Decide whether to pull and log the concrete reason.- */- if (serverPersonalRevision !== null && serverPersonalRevision > localMainRevision) {- devLog(`[VaultSync] Pull needed: personal manifest server rev ${serverPersonalRevision} > local rev ${localMainRevision}.`);- return true;- }+ * @param localRevisions - the client's last-known revision per manifest+ */+function serverManifestsNeedPull(serverManifests: ManifestRevision[], localRevisions: Record<string, number>): boolean {+ const serverRevisions = new Map((serverManifests ?? []).map(m => [m.manifestId, m.revision]));- // Any shared manifest added or with a changed revision on the server.- for (const [manifestId, revision] of serverShared) {- const local = localSharedRevisions[manifestId];+ // Any manifest added or with a changed revision on the server.+ for (const [manifestId, revision] of serverRevisions) {+ const local = localRevisions[manifestId];if (local !== revision) {const reason = local === undefined ? `not tracked locally (server rev ${revision})` : `server rev ${revision} != local rev ${local}`;- devLog(`[VaultSync] Pull needed: shared manifest ${manifestId} ${reason}.`);+ devLog(`[VaultSync] Pull needed: manifest ${manifestId} ${reason}.`);return true;}}- // Any shared manifest the client still tracks but the server no longer lists (removed / revoked).- const removed = Object.keys(localSharedRevisions).find(manifestId => !serverShared.has(manifestId));+ // Any manifest the client still tracks but the server no longer lists (removed / revoked).+ const removed = Object.keys(localRevisions).find(manifestId => !serverRevisions.has(manifestId));if (removed) {- devLog(`[VaultSync] Pull needed: shared manifest ${removed} (local rev ${localSharedRevisions[removed]}) no longer listed by the server (revoked/removed).`);+ devLog(`[VaultSync] Pull needed: manifest ${removed} (local rev ${localRevisions[removed]}) no longer listed by the server (revoked/removed).`);return true;}- devLog(`[VaultSync] No pull needed: personal rev ${serverPersonalRevision ?? 'n/a'} (local ${localMainRevision}), ${serverShared.size} shared manifest(s) all match local revisions.`);+ devLog(`[VaultSync] No pull needed: ${serverRevisions.size} manifest(s) all match local revisions.`);return false;}@@ -258,6 +238,13 @@ async function fetchLatestVaultFromServer(): Promise<VaultResponse> {}}+/**+ * Outcome of a push attempt: 0 = ok, 2 = the server had newer state and the caller must re-sync (pull/merge/retry).+ * Revision bookkeeping happens inside VaultSyncService, which records the new revision of every manifest a+ * successful write actually carried; no revision travels back through this result.+ */+type VaultPushOutcome = { status: number };+/*** Push the current SQLite vault to the server.*@@ -265,7 +252,7 @@ async function fetchLatestVaultFromServer(): Promise<VaultResponse> {* @param options - forceFullWrite bypasses the content-fingerprint gating and rewrites every manifest and bucket* (server rollback recovery); createVaultKey mints the VEK as part of this push (KEK/VEK migration)*/-async function pushVaultToServer(sqliteClient: SqliteClient, options: { forceFullWrite: boolean; createVaultKey: boolean }): Promise<VaultPostResponse> {+async function pushVaultToServer(sqliteClient: SqliteClient, options: { forceFullWrite: boolean; createVaultKey: boolean }): Promise<VaultPushOutcome> {const encryptionKey = await handleGetEncryptionKey();if (!encryptionKey) {throw new Error(formatErrorWithCode(await t('common.errors.vaultIsLocked'), AppErrorCode.VAULT_LOCKED));@@ -284,15 +271,15 @@ async function pushVaultToServer(sqliteClient: SqliteClient, options: { forceFulawait handleStoreEncryptionKey(result.newEncryptionKey);}- return { status: 0, newRevisionNumber: result.newManifestRevision ?? 0 };+ return { status: 0 };}if (result.status === 'outdated') {- return { status: 2, newRevisionNumber: result.newManifestRevision ?? 0 };+ return { status: 2 };}if (result.status === 'rejected') {- // Structural validation tripped — surface the reasons rather than silently uploading garbage.+ // Structural validation errorconst reason = (result.reasons ?? ['Integrity check failed']).join('; ');devError('[V2Sync] Refusing to upload corrupt vault:', reason);throw new Error(formatErrorWithCode(`Vault integrity check failed: ${reason}`, AppErrorCode.UPLOAD_FAILED));@@ -308,8 +295,7 @@ async function pushVaultToServer(sqliteClient: SqliteClient, options: { forceFul/*** Sync the vault with the server to check if a newer vault is available. If so, the vault will be updated.*/-export async function handleSyncVault(-) : Promise<messageBoolResponse> {+export async function handleSyncVault() : Promise<messageBoolResponse> {const webApi = new WebApiService();const statusResponse = await webApi.getStatus();const statusError = webApi.validateStatusResponse(statusResponse);@@ -317,22 +303,18 @@ export async function handleSyncVault(return { success: false, error: await t('common.errors.' + statusError) };}- const localServerRevision = await storage.getItem(StorageKeys.SERVER_REVISION) as number | null ?? 0;- const localSharedRevisions = await getLocalSharedManifestRevisions();-- if (serverManifestsNeedPull(statusResponse.manifestRevisions, statusResponse.personalManifestId, localServerRevision, localSharedRevisions)) {+ if (serverManifestsNeedPull(statusResponse.manifestRevisions, await getManifestRevisions())) {/** Retrieve the latest vault from the server.*/const vaultResponse = await fetchLatestVaultFromServer();- // Store in local: storage for persistence (fresh from server, not dirty)+ // Store in local: storage for persistence (fresh from server, not dirty). The pull recorded the per-manifest revisions itself.await storage.setItems([{ key: StorageKeys.ENCRYPTED_VAULT, value: vaultResponse.vault.blob },{ key: StorageKeys.PUBLIC_EMAIL_DOMAINS, value: vaultResponse.vault.publicEmailDomainList },{ key: StorageKeys.PRIVATE_EMAIL_DOMAINS, value: vaultResponse.vault.privateEmailDomainList },{ key: StorageKeys.HIDDEN_PRIVATE_EMAIL_DOMAINS, value: vaultResponse.vault.hiddenPrivateEmailDomainList },- { key: StorageKeys.SERVER_REVISION, value: vaultResponse.vault.currentRevisionNumber },{ key: StorageKeys.IS_DIRTY, value: false }]);@@ -354,10 +336,9 @@ export async function handleGetVault(const encryptedVault = await storage.getItem(StorageKeys.ENCRYPTED_VAULT) as string;// TODO: the fallback mechanism can be removed some period of time after 0.27.0 is released.- const publicEmailDomains = await getItemWithFallback<string[]>(StorageKeys.PUBLIC_EMAIL_DOMAINS);- const privateEmailDomains = await getItemWithFallback<string[]>(StorageKeys.PRIVATE_EMAIL_DOMAINS);- const hiddenPrivateEmailDomains = await getItemWithFallback<string[]>(StorageKeys.HIDDEN_PRIVATE_EMAIL_DOMAINS) ?? [];- const serverRevision = await storage.getItem(StorageKeys.SERVER_REVISION) as number | null;+ const publicEmailDomains = await getStorageItem<string[]>(StorageKeys.PUBLIC_EMAIL_DOMAINS);+ const privateEmailDomains = await getStorageItem<string[]>(StorageKeys.PRIVATE_EMAIL_DOMAINS);+ const hiddenPrivateEmailDomains = await getStorageItem<string[]>(StorageKeys.HIDDEN_PRIVATE_EMAIL_DOMAINS) ?? [];if (!encryptedVault) {console.error('Vault not available');@@ -381,8 +362,7 @@ export async function handleGetVault(vault: decryptedVault,publicEmailDomains: publicEmailDomains ?? [],privateEmailDomains: privateEmailDomains ?? [],- hiddenPrivateEmailDomains: hiddenPrivateEmailDomains ?? [],- serverRevision: serverRevision ?? 0+ hiddenPrivateEmailDomains: hiddenPrivateEmailDomains ?? []};} catch (error) {console.error('Failed to get vault:', error);@@ -430,9 +410,6 @@ export async function handleClearVaultData(): Promise<messageBoolResponse> {// Clear vault data and every piece of state derived from it (sync bookkeeping, dirty flags, bucket revisions)await storage.removeItems(allVaultDataStorageKeys());- // Clear the cached vault key and encrypted VEK.- VaultKeyService.clearCache();-// Clear all local preferences (site settings, login save settings, etc.)await LocalPreferencesService.clearAll();@@ -642,8 +619,8 @@ export function handleGetDefaultEmailDomain(): Promise<stringResponse> {// If no default domain is configured, fall back to first private or public domainif (!domain) {- const privateEmailDomains = await getItemWithFallback<string[]>(StorageKeys.PRIVATE_EMAIL_DOMAINS) ?? [];- const publicEmailDomains = await getItemWithFallback<string[]>(StorageKeys.PUBLIC_EMAIL_DOMAINS) ?? [];+ const privateEmailDomains = await getStorageItem<string[]>(StorageKeys.PRIVATE_EMAIL_DOMAINS) ?? [];+ const publicEmailDomains = await getStorageItem<string[]>(StorageKeys.PUBLIC_EMAIL_DOMAINS) ?? [];domain = privateEmailDomains[0] || publicEmailDomains[0] || '';}@@ -719,15 +696,10 @@ export async function handleGeneratePassword(export async function handleGetEncryptionKey() : Promise<string | null> {// Try the current key name first (since 0.22.0)- let encryptionKey = await storage.getItem(StorageKeys.ENCRYPTION_KEY) as string | null;+ const encryptionKey = await storage.getItem(StorageKeys.ENCRYPTION_KEY) as string | null;- // Fall back to the legacy key name if not found- if (!encryptionKey) {- // TODO: this check can be removed some period of time after 0.22.0 is released.- encryptionKey = await storage.getItem(StorageKeys.LEGACY_DERIVED_KEY) as string | null;- }-- return encryptionKey;+ // LEGACY: fall back to the pre-0.22.0 key name.+ return encryptionKey ?? await readLegacySessionEncryptionKey();}/**@@ -737,7 +709,7 @@ export async function handleGetEncryptionKey(export async function handleGetEncryptionKeyDerivationParams() : Promise<EncryptionKeyDerivationParams | null> {// Get metadata from storage- return await getItemWithFallback<EncryptionKeyDerivationParams>(StorageKeys.ENCRYPTION_KEY_DERIVATION_PARAMS);+ return await getStorageItem<EncryptionKeyDerivationParams>(StorageKeys.ENCRYPTION_KEY_DERIVATION_PARAMS);}/**@@ -770,18 +742,32 @@ async function clearDirtyScopes(): Promise<void> {* @param sqliteClient - the in-memory SQLite client to read bucket data from* @param scopes - the pending dirty scopes (bucket category names, deduplicated here)*/-async function uploadDirtyBucketsOnly(sqliteClient: SqliteClient, scopes: VaultMutationScope[]): Promise<VaultPostResponse> {+async function uploadDirtyBucketsOnly(sqliteClient: SqliteClient, scopes: VaultMutationScope[]): Promise<VaultPushOutcome> {const encryptionKey = await handleGetEncryptionKey();if (!encryptionKey) {throw new Error(formatErrorWithCode(await t('common.errors.vaultIsLocked'), AppErrorCode.VAULT_LOCKED));}/*- * Which tables make up each bucket category is owned by the Rust layer.+ * Which tables make up each bucket category is owned by the Rust layer, and so is splitting their rows+ * across the manifests that own them: this loop reads a category whole and hands it over per category.*/const layout = await vaultCodecBucketLayout();const username = (await storage.getItem(StorageKeys.USERNAME)) as string;+ if (!sqliteClient.getActiveManifestId() && !sqliteClient.getPersonalManifestId()) {+ devWarn('[V2Push] No manifest to address the bucket write to, falling back to a full vault upload.');+ return (await uploadNewVaultToServer(sqliteClient, { forceFullWrite: false, createVaultKey: false })).response;+ }++ /*+ * Every manifest this vault can write. Each gets a bucket back, empty where it holds no rows of the+ * category, so deleting a manifest's last row still reaches the server; a row stamped for anything else+ * (a manifest whose key this session never opened) is dropped and left to the next full push.+ */+ const writeKeys = await vaultSyncService.resolveBucketWriteKeys(encryptionKey);+ const overflowTable = await vaultCodecOverflowTable();+for (const category of new Set(scopes)) {const spec = layout.find(entry => entry.category === category);if (!spec) {@@ -789,22 +775,21 @@ async function uploadDirtyBucketsOnly(sqliteClient: SqliteClient, scopes: VaultMreturn (await uploadNewVaultToServer(sqliteClient, { forceFullWrite: false, createVaultKey: false })).response;}- /*- * Read the CodecOverflows carrier row alongside the bucket's tables: extract_bucket consumes it- * to re-merge a newer client's bucket tables/columns, so this partial push doesn't drop them.- */- const tables = VaultCodec.readNamedTables(sqliteClient, [...spec.tables, await vaultCodecOverflowTable()]);- const bucket = await vaultCodecExtractBucket(category, tables);- const result = await vaultSyncService.pushDataBucketOnly(bucket, encryptionKey, username);- if (result.status !== 'ok') {- // Conflict persisted even after the rebase-retry, let the caller run a full re-sync (status 2).- return { status: 2, newRevisionNumber: (await storage.getItem(StorageKeys.SERVER_REVISION)) as number | null ?? 0 };+ const tables = VaultCodec.readNamedTables(sqliteClient, [...spec.tables, overflowTable]);+ const buckets = await vaultCodecExtractBuckets(category, [...writeKeys.keys()], tables);++ for (const bucket of buckets) {+ const result = await vaultSyncService.pushDataBucketOnly(bucket, writeKeys.get(bucket.manifestId)!, username);+ if (result.status !== 'ok') {+ // Conflict persisted even after the rebase-retry, let the caller run a full re-sync (status 2).+ return { status: 2 };+ }+ devLog(`[V2Push] Bucket-only push for "${category}" of manifest ${bucket.manifestId} done (bucket revision ${result.revision}); manifest untouched.`);}- devLog(`[V2Push] Bucket-only push for "${category}" done (bucket revision ${result.revision}); manifest untouched.`);}- // Manifest unchanged, so the content revision stays where it was.- return { status: 0, newRevisionNumber: (await storage.getItem(StorageKeys.SERVER_REVISION)) as number | null ?? 0 };+ // Manifests untouched, so every manifest revision baseline stays where it was.+ return { status: 0 };}/**@@ -846,7 +831,7 @@ export async function handleUploadVault(if (bucketOnly) {devLog(`[V2Push] All pending mutations are bucket-scoped (${dirtyScopes.join(', ')}), skipping manifest upload.`);}- let response: VaultPostResponse;+ let response: VaultPushOutcome;let vaultPruned = false;if (bucketOnly) {// Bucket-only pushes never prune (settings buckets carry no trash items).@@ -858,7 +843,6 @@ export async function handleUploadVault(return {success: true,status: response.status,- newRevisionNumber: response.newRevisionNumber,mutationSeqAtStart,vaultPruned};@@ -960,7 +944,7 @@ export async function handleClearPersistedFormValues(): Promise<void> {* @param sqliteClient - the in-memory SQLite client to upload* @param options - forceFullWrite: whether to force a full write of the vault, createVaultKey: whether to create a vault key*/-async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { forceFullWrite: boolean; createVaultKey: boolean }) : Promise<{ response: VaultPostResponse; vaultPruned: boolean }> {+async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { forceFullWrite: boolean; createVaultKey: boolean }) : Promise<{ response: VaultPushOutcome; vaultPruned: boolean }> {devLog('[VaultSync] Upload started');let vaultPruned = false;const encryptionKey = await handleGetEncryptionKey();@@ -988,7 +972,7 @@ async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { forconsole.warn('[VaultSync] Failed to prune vault, continuing with upload:', pruneError);}- let pushResponse: VaultPostResponse;+ let pushResponse: VaultPushOutcome;try {pushResponse = await pushVaultToServer(sqliteClient, options);} catch (err) {@@ -1011,9 +995,6 @@ async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { forcachedSqliteClient = sqliteClient;cachedVaultBlob = reEncrypted;}- if (pushResponse.status === 0) {- await storage.setItem(StorageKeys.SERVER_REVISION, pushResponse.newRevisionNumber);- }return { response: pushResponse, vaultPruned };}@@ -1022,11 +1003,15 @@ async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { for* Persist a locally-mutated vault and attempt to sync it to the server in the background.** This is tolerant to server being offline (in which case the vault state will be stored locally for next sync).+ * @param sqliteClient - the mutated vault+ * @param encryptionKey - the key the local blob is stored under+ * @param scope - what the mutation touched; bucket-scoped mutations (e.g. 'Stats') let the sync push just+ * that data bucket instead of the whole manifest. Defaults to a full manifest push.*/-async function persistLocalVaultMutation(sqliteClient: SqliteClient, encryptionKey: string) : Promise<void> {+async function persistLocalVaultMutation(sqliteClient: SqliteClient, encryptionKey: string, scope?: VaultMutationScope) : Promise<void> {const updatedVaultData = sqliteClient.exportToBase64();const encryptedVault = await EncryptionUtility.symmetricEncrypt(updatedVaultData, encryptionKey);- await handleStoreEncryptedVault({ vaultBlob: encryptedVault, markDirty: true });+ await handleStoreEncryptedVault({ vaultBlob: encryptedVault, markDirty: true, scope });/** The stored blob is exactly this client's content, so re-adopt the pair as the cache (the store just cleared@@ -1117,14 +1102,12 @@ export async function handleGetEncryptedVault(): Promise<string | null> {* @param request Object with:* - vaultBlob: The encrypted vault data* - markDirty: If true, marks vault as dirty and increments mutation sequence (for local mutations)- * - serverRevision: Optional explicit server revision (for sync operations)* - expectedMutationSeq: If provided, only store if current sequence matches (for sync operations)* @returns { success, mutationSequence } - success=false if expectedMutationSeq didn't match*/export async function handleStoreEncryptedVault(request: {vaultBlob: string;markDirty?: boolean;- serverRevision?: number;expectedMutationSeq?: number;scope?: VaultMutationScope;}): Promise<{ success: boolean; mutationSequence: number }> {@@ -1147,26 +1130,13 @@ export async function handleStoreEncryptedVault(request: {const dirtyScopeKey = dirtyScopeStorageKey(request.scope ?? DEFAULT_VAULT_MUTATION_SCOPE);// Build items to store.- if (request.markDirty && request.serverRevision !== undefined) {- await storage.setItems([- { key: StorageKeys.ENCRYPTED_VAULT, value: request.vaultBlob },- { key: StorageKeys.MUTATION_SEQUENCE, value: mutationSequence },- { key: StorageKeys.IS_DIRTY, value: true },- { key: StorageKeys.SERVER_REVISION, value: request.serverRevision },- { key: dirtyScopeKey, value: true }- ]);- } else if (request.markDirty) {+ if (request.markDirty) {await storage.setItems([{ key: StorageKeys.ENCRYPTED_VAULT, value: request.vaultBlob },{ key: StorageKeys.MUTATION_SEQUENCE, value: mutationSequence },{ key: StorageKeys.IS_DIRTY, value: true },{ key: dirtyScopeKey, value: true }]);- } else if (request.serverRevision !== undefined) {- await storage.setItems([- { key: StorageKeys.ENCRYPTED_VAULT, value: request.vaultBlob },- { key: StorageKeys.SERVER_REVISION, value: request.serverRevision }- ]);} else {await storage.setItem(StorageKeys.ENCRYPTED_VAULT, request.vaultBlob);}@@ -1211,7 +1181,7 @@ export async function handleMigrateVaultManifest(): Promise<VaultManifestMigrati}const needsSchemaMigration = await sqliteClient.requiresSchemaMigration();- const needsVaultKey = await requiresLegacyVaultKeyMigration();+ const needsVaultKey = await requiresLegacyAccountKeyMigration();if (!needsSchemaMigration && !needsVaultKey) {devLog('[ManifestMigration] Vault is already on the current storage model, nothing to migrate.');return { success: true, pushed: true };@@ -1236,11 +1206,8 @@ export async function handleMigrateVaultManifest(): Promise<VaultManifestMigratitry {const uploadResponse = await handleUploadVault({ createVaultKey: needsVaultKey });if (uploadResponse.success && uploadResponse.status === 0) {- await handleMarkVaultClean({- mutationSeqAtStart: uploadResponse.mutationSeqAtStart!,- newServerRevision: uploadResponse.newRevisionNumber!- });- devLog(`[ManifestMigration] Migration pushed, server now at revision ${uploadResponse.newRevisionNumber}.`);+ await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart! });+ devLog('[ManifestMigration] Migration pushed to the server.');return { success: true, pushed: true };}devWarn('[ManifestMigration] Migration push did not succeed, vault stays dirty for the next sync:', uploadResponse.error);@@ -1260,27 +1227,20 @@ export async function handleMigrateVaultManifest(): Promise<VaultManifestMigrati* Only clears dirty flag if no mutations happened during sync.** @param mutationSeqAtStart - The mutation sequence when sync started- * @param newServerRevision - The new server revision after successful upload* @returns Whether the dirty flag was cleared*/export async function handleMarkVaultClean(request: {mutationSeqAtStart: number;- newServerRevision: number;}): Promise<{ cleared: boolean; currentMutationSeq: number }> {const currentMutationSeq = await storage.getItem(StorageKeys.MUTATION_SEQUENCE) as number | null ?? 0;if (currentMutationSeq === request.mutationSeqAtStart) {// No mutations during sync - safe to mark as clean- await storage.setItems([- { key: StorageKeys.IS_DIRTY, value: false },- { key: StorageKeys.SERVER_REVISION, value: request.newServerRevision }- ]);+ await storage.setItem(StorageKeys.IS_DIRTY, false);await clearDirtyScopes();return { cleared: true, currentMutationSeq };}- // Mutations happened during sync - keep dirty, but still update server revision- await storage.setItem(StorageKeys.SERVER_REVISION, request.newServerRevision);return { cleared: false, currentMutationSeq };}@@ -1290,49 +1250,20 @@ export async function handleMarkVaultClean(request: {export async function handleGetSyncState(): Promise<{isDirty: boolean;mutationSequence: number;- serverRevision: number;isSyncInProgress: boolean;}> {- const [isDirty, mutationSequence, serverRevision] = await Promise.all([+ const [isDirty, mutationSequence] = await Promise.all([storage.getItem(StorageKeys.IS_DIRTY) as Promise<boolean | null>,- storage.getItem(StorageKeys.MUTATION_SEQUENCE) as Promise<number | null>,- storage.getItem(StorageKeys.SERVER_REVISION) as Promise<number | null>+ storage.getItem(StorageKeys.MUTATION_SEQUENCE) as Promise<number | null>]);return {isDirty: isDirty ?? false,mutationSequence: mutationSequence ?? 0,- serverRevision: serverRevision ?? 0,isSyncInProgress};}-/**- * Get the current server revision.- */-export async function handleGetServerRevision(): Promise<number> {- // First try new key, then fall back to legacy key- let revision = await storage.getItem(StorageKeys.SERVER_REVISION) as number | null;-- if (revision === null) {- // Try legacy key - parse string format "250" or "250+1" to get server part- const legacyRevision = await storage.getItem(StorageKeys.LEGACY_VAULT_REVISION_NUMBER) as string | number | null;- if (legacyRevision !== null) {- if (typeof legacyRevision === 'number') {- revision = legacyRevision;- } else {- // Handle legacy "250+1" format - extract just the server part- const parts = legacyRevision.split('+');- revision = parseInt(parts[0], 10) || 0;- }- // Migrate to new key- await storage.setItem(StorageKeys.SERVER_REVISION, revision);- }- }-- return revision ?? 0;-}-/*** Adopt a server-side vault key this device does not know about yet.*@@ -1385,7 +1316,7 @@ async function adoptRemoteVaultKeyIfNeeded(): Promise<boolean> {await storage.setItem(StorageKeys.ENCRYPTED_VAULT, await EncryptionUtility.symmetricEncrypt(decrypted, vek));}- await VaultKeyService.cacheEncryptedVekFromServer();+ await VaultKeyService.cacheVaultKeyBlobs(fetchResult.vaultKey);await handleStoreEncryptionKey(vek);cachedSqliteClient = null;cachedVaultBlob = null;@@ -1505,11 +1436,16 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {// Get current sync stateconst syncState = await handleGetSyncState();+ const localRevisions = await getManifestRevisions();+ const personalManifestId = await getPersonalManifestId();const serverManifestRevision = personalManifestRevision(statusResponse);- const localSharedRevisions = await getLocalSharedManifestRevisions();- const needsPull = serverManifestsNeedPull(statusResponse.manifestRevisions, statusResponse.personalManifestId, syncState.serverRevision, localSharedRevisions);+ const localManifestRevision = personalManifestId ? localRevisions[personalManifestId] ?? 0 : 0;++ // A personal manifest that went backwards is a rolled-back server (restored backup), which we want to re-push.+ const serverRolledBack = serverManifestRevision < localManifestRevision;+ const needsPull = !serverRolledBack && serverManifestsNeedPull(statusResponse.manifestRevisions, localRevisions);- devLog(`[VaultSync] Status received (server main rev ${serverManifestRevision}, local rev ${syncState.serverRevision}, needsPull ${needsPull}, isDirty ${syncState.isDirty})`);+ devLog(`[VaultSync] Status received (server main rev ${serverManifestRevision}, local rev ${localManifestRevision}, needsPull ${needsPull}, isDirty ${syncState.isDirty})`);// Check if server is actually available (0.0.0 indicates connection error)if (statusResponse.serverVersion === '0.0.0') {@@ -1561,7 +1497,7 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {try {const sqliteClient = await createVaultSqliteClient();if (await vaultSyncService.detectNoOpMutation(sqliteClient, syncState.mutationSequence)) {- const cleanResult = await handleMarkVaultClean({ mutationSeqAtStart: syncState.mutationSequence, newServerRevision: syncState.serverRevision });+ const cleanResult = await handleMarkVaultClean({ mutationSeqAtStart: syncState.mutationSequence });if (cleanResult.cleared) {devLog('[VaultSync] Local changes are canonically identical to the server baselines (no-op mutation); nothing to push.');syncState.isDirty = false;@@ -1615,7 +1551,7 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {devWarn('[VaultSync] Server vault is still on the legacy storage format while the local vault is migrated; skipping the merge and pushing the local vault.');const uploadResponse = await handleUploadVault({ forceFullWrite: true });if (uploadResponse.success && uploadResponse.status === 0) {- await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart!, newServerRevision: uploadResponse.newRevisionNumber! });+ await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart! });return { success: true, hasNewVault: false, wasOffline: false, sqliteBlobUpgradeRequired: false, requiresLogout: false };}if (uploadResponse.status === 2) {@@ -1640,7 +1576,6 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {*/const storeResult = await handleStoreEncryptedVault({vaultBlob: mergedEncryptedVault,- serverRevision: vaultResponseJson.vault.currentRevisionNumber,expectedMutationSeq: syncState.mutationSequence});@@ -1653,10 +1588,7 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {const uploadResponse = await handleUploadVault();if (uploadResponse.success && uploadResponse.status === 0) {- await handleMarkVaultClean({- mutationSeqAtStart: uploadResponse.mutationSeqAtStart!,- newServerRevision: uploadResponse.newRevisionNumber!- });+ await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart! });} else if (uploadResponse.status === 2) {// Server returned Outdated - another device uploaded. Re-sync.return handleFullVaultSync();@@ -1691,7 +1623,6 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {*/const storeResult = await handleStoreEncryptedVault({vaultBlob: vaultResponseJson.vault.blob,- serverRevision: vaultResponseJson.vault.currentRevisionNumber,expectedMutationSeq: syncState.mutationSequence});@@ -1722,7 +1653,7 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {AppErrorCode.VAULT_DECRYPT_FAILED));}- } else if (serverManifestRevision === syncState.serverRevision) {+ } else if (serverManifestRevision === localManifestRevision) {/*** Server and local vault are at the same revision.* If we have pending local changes, upload them now.@@ -1730,10 +1661,7 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {if (syncState.isDirty) {const uploadResponse = await handleUploadVault();if (uploadResponse.success && uploadResponse.status === 0) {- await handleMarkVaultClean({- mutationSeqAtStart: uploadResponse.mutationSeqAtStart!,- newServerRevision: uploadResponse.newRevisionNumber!- });+ await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart! });/** If expired trash items were pruned during upload, report the vault as new@@ -1754,14 +1682,14 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {}return { success: true, hasNewVault: false, wasOffline: false, sqliteBlobUpgradeRequired: false, requiresLogout: false };- } else if (serverManifestRevision < syncState.serverRevision) {+ } else if (serverRolledBack) {/*** Server revision DECREASED - server data loss/rollback detected.* Client has more advanced revision - upload to recover server state.*/console.warn(`Server data loss detected! Server at rev ${serverManifestRevision}, ` +- `client at rev ${syncState.serverRevision}. Uploading to recover server state.`+ `client at rev ${localManifestRevision}. Uploading to recover server state.`);/*@@ -1771,14 +1699,9 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {const uploadResponse = await handleUploadVault({ forceFullWrite: true });if (uploadResponse.success && uploadResponse.status === 0) {- await handleMarkVaultClean({- mutationSeqAtStart: uploadResponse.mutationSeqAtStart!,- newServerRevision: uploadResponse.newRevisionNumber!- });+ await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart! });- console.info(- `Server recovery complete: rev ${serverManifestRevision} → ${uploadResponse.newRevisionNumber}`- );+ console.info(`Server recovery complete: server restored from client state (was at personal rev ${serverManifestRevision}, client at ${localManifestRevision}).`);return { success: true, hasNewVault: uploadResponse.vaultPruned === true, wasOffline: false, sqliteBlobUpgradeRequired: false, requiresLogout: false };} else if (uploadResponse.status === 2) {@@ -2318,7 +2241,7 @@ export async function handleGetTotpSecrets(const secrets: Record<string, string> = {};for (const itemId of message.itemIds) {- const totpCodes = sqliteClient.settings.getTotpCodesForItem(itemId);+ const totpCodes = sqliteClient.items.getTotpCodesForItem(itemId);if (totpCodes.length > 0) {secrets[itemId] = totpCodes[0].SecretKey;}@@ -2348,7 +2271,7 @@ export async function handleGenerateTotpCode(try {const sqliteClient = await createVaultSqliteClient();- const totpCodes = sqliteClient.settings.getTotpCodesForItem(message.itemId);+ const totpCodes = sqliteClient.items.getTotpCodesForItem(message.itemId);if (totpCodes.length === 0) {return { success: false, error: 'No TOTP codes found for this item' };@@ -2368,6 +2291,33 @@ export async function handleGenerateTotpCode(}}+/**+ * Record one use of an item: when it was last used, and how often.+ * @param message - The item that was used and what was done with it.+ */+export async function handleRecordItemUsage(+ message: { itemId: string; action: ItemUsageAction }+): Promise<{ success: boolean }> {+ try {+ const encryptionKey = await handleGetEncryptionKey();+ if (!encryptionKey) {+ return { success: false };+ }++ const sqliteClient = await createVaultSqliteClient();+ if (!sqliteClient.itemStats.recordUsage(message.itemId, message.action)) {+ // No such item (deleted between use and record); nothing to attribute the use to.+ return { success: false };+ }++ await persistLocalVaultMutation(sqliteClient, encryptionKey, VaultDataBucketCategory.Stats);+ return { success: true };+ } catch (error) {+ console.error('Failed to record item usage:', error);+ return { success: false };+ }+}+/*** Set recently selected item for smart autofill.*/diff --git a/apps/browser-extension/src/utils/BucketRowGrouping.ts b/apps/browser-extension/src/utils/BucketRowGrouping.tsdeleted file mode 100644index 597e9e388..000000000--- a/apps/browser-extension/src/utils/BucketRowGrouping.ts+++ /dev/null@@ -1,98 +0,0 @@-/**- * Splitting a data bucket's rows across the manifests that own them.- */--/** A bucket row as read out of SQLite. */-export type BucketRow = Record<string, unknown>;--/** A bucket category's tables and their rows. */-export type BucketTables = Record<string, BucketRow[]>;--/**- * The `ManifestId` a row carries when it names no manifest.- */-export const UNSTAMPED_MANIFEST_ID = '00000000-0000-0000-0000-000000000000';--/** True when a `ManifestId` value names no manifest (absent, empty, or the all-zero sentinel). */-export function isUnstampedManifestId(manifestId: string): boolean {- return manifestId.length === 0 || manifestId.toLowerCase() === UNSTAMPED_MANIFEST_ID;-}--/**- * Find `manifestId` among `candidates` case-insensitively, returning the candidate as it is spelled there.- * @param candidates - The ids to match against- * @param manifestId - The id to find- * @returns The matching candidate, or undefined- */-export function matchManifestId(candidates: Iterable<string>, manifestId: string): string | undefined {- const lower = manifestId.toLowerCase();- for (const candidate of candidates) {- if (candidate.toLowerCase() === lower) {- return candidate;- }- }- return undefined;-}--/**- * Look `manifestId` up in `map`, tolerating a casing difference the way {@link matchManifestId} does.- * @param map - The map to search- * @param manifestId - The id to find- * @returns The matching entry's value, or undefined- */-export function findByManifestId<T>(map: Map<string, T>, manifestId: string): T | undefined {- const match = matchManifestId(map.keys(), manifestId);- return match === undefined ? undefined : map.get(match);-}--/**- * Split a bucket category's rows into one table set per owning manifest. Ownership is each row's own- * `ManifestId` stamp — the writing manifest is no fallback: a row that names no manifest is a bug in- * whatever wrote it, refused loudly here rather than silently homed somewhere plausible.- * @param tableNames - The tables making up the category- * @param tables - The rows read for those tables- * @param writingManifestId - The manifest this vault writes from; its bucket is always declared, even empty- * @param alsoInclude - Manifests that must be written even when they hold no rows- * @returns Manifest id → that manifest's rows, per table- */-export function groupBucketRowsByManifest(- tableNames: string[],- tables: BucketTables,- writingManifestId: string,- alsoInclude: string[] = []-): Map<string, BucketTables> {- const grouped = new Map<string, BucketTables>([[writingManifestId, {}]]);-- for (const tableName of tableNames) {- for (const row of tables[tableName] ?? []) {- const stamp = typeof row.ManifestId === 'string' ? row.ManifestId : '';- if (isUnstampedManifestId(stamp)) {- throw new Error(`Bucket table ${tableName} holds a row that names no manifest; every row must carry the manifest it belongs to`);- }- const owner = matchManifestId(grouped.keys(), stamp) ?? stamp;-- const manifestTables = grouped.get(owner) ?? {};- manifestTables[tableName] = [...(manifestTables[tableName] ?? []), row];- grouped.set(owner, manifestTables);- }- }-- for (const manifestId of alsoInclude) {- if (!matchManifestId(grouped.keys(), manifestId)) {- grouped.set(manifestId, {});- }- }-- /*- * Every manifest declares every table of the category, empty where it holds nothing. A table left out- * entirely reads as "unchanged" rather than "emptied", so dropping the last row of one would otherwise- * never reach the server.- */- for (const manifestTables of grouped.values()) {- for (const tableName of tableNames) {- manifestTables[tableName] ??= [];- }- }-- return grouped;-}diff --git a/apps/browser-extension/src/utils/RustCore.ts b/apps/browser-extension/src/utils/RustCore.tsindex 45d667525..d68dcba8f 100644--- a/apps/browser-extension/src/utils/RustCore.ts+++ b/apps/browser-extension/src/utils/RustCore.ts@@ -364,13 +364,12 @@ export async function vaultCodecExtractEncryptionKeyForPublicKey(manifest: Codec}/**- * Build a single data bucket for `(manifestId, category)` from its tables (bucket-only push path). Include the- * `CodecOverflows` table (see {@link vaultCodecOverflowTable}) in `tables` so a newer writer's- * columns/tables re-merge and survive; it is consumed and never emitted into the bucket.+ * Build a bucket category's data buckets from its tables, one per manifest in `manifestIds` (bucket-only push+ * path).*/-export async function vaultCodecExtractBucket(manifestId: string, category: string, tables: Record<string, Array<Record<string, unknown>>>): Promise<CodecDataBucket> {+export async function vaultCodecExtractBuckets(category: string, manifestIds: string[], tables: Record<string, Array<Record<string, unknown>>>): Promise<CodecDataBucket[]> {await initRustCore();- return core.vaultCodecExtractBucket({ manifestId, category, tables }) as CodecDataBucket;+ return core.vaultCodecExtractBuckets({ category, manifestIds, tables }) as CodecDataBucket[];}/**diff --git a/core/rust/src/ffi.rs b/core/rust/src/ffi.rsindex 1fe06653e..e1bc8fd45 100644--- a/core/rust/src/ffi.rs+++ b/core/rust/src/ffi.rs@@ -244,13 +244,14 @@ pub unsafe extern "C" fn vault_codec_materialize_as_sqlite_ffi(input_json: *conscodec_json_ffi(input_json, "input_json", crate::vault_codec::materialize_as_sqlite_json)}-/// Build a single data bucket. Input: `{ manifestId, category, tables }` JSON. Output: `DataBucket` JSON.+/// Build a bucket category's data buckets, one per manifest. Input: `{ category, manifestIds, tables }` JSON.+/// Output: `DataBucket[]` JSON.////// # Safety/// `input_json` must be a valid null-terminated C string; free the result with `free_string`.#[no_mangle]-pub unsafe extern "C" fn vault_codec_extract_bucket_ffi(input_json: *const c_char) -> *mut c_char {- codec_json_ffi(input_json, "input_json", crate::vault_codec::extract_bucket_json)+pub unsafe extern "C" fn vault_codec_extract_buckets_ffi(input_json: *const c_char) -> *mut c_char {+ codec_json_ffi(input_json, "input_json", crate::vault_codec::extract_buckets_json)}/// Validate a manifest. Input: `Manifest` JSON. Output: `ValidationResult` JSON.diff --git a/core/rust/src/lib.rs b/core/rust/src/lib.rsindex c7d3fea0c..3de537c5c 100644--- a/core/rust/src/lib.rs+++ b/core/rust/src/lib.rs@@ -32,7 +32,7 @@ pub use vault_merge::{};pub use vault_codec::{compute_ciphertext_hash, compute_content_fingerprint, canonicalize_from_sqlite,- extract_bucket, generate_manifest_salt, unpack_payload, materialize_as_sqlite, pack_payload,+ extract_buckets, generate_manifest_salt, unpack_payload, materialize_as_sqlite, pack_payload,validate_manifest, validate_data_bucket, BlobEntry,CanonicalizeInput, CanonicalizedManifest, CanonicalizedVault, CodecRecord, CodecTableData, DataBucket, Manifest,MaterializeInput, MaterializedTables, ManifestSpec, ValidationResult,diff --git a/core/rust/src/uniffi_api.rs b/core/rust/src/uniffi_api.rsindex e94796d17..22431a62f 100644--- a/core/rust/src/uniffi_api.rs+++ b/core/rust/src/uniffi_api.rs@@ -213,10 +213,11 @@ pub fn vault_codec_materialize_as_sqlite(input_json: String) -> Result<String, Vcrate::vault_codec::materialize_as_sqlite_json(&input_json)}-/// Build a single data bucket. Input: `{ manifestId, category, tables }` JSON. Output: `DataBucket` JSON.+/// Build a bucket category's data buckets, one per manifest this vault writes.+/// Input: `{ category, manifestIds, tables }` JSON. Output: `DataBucket[]` JSON.#[uniffi::export]-pub fn vault_codec_extract_bucket(input_json: String) -> Result<String, VaultError> {- crate::vault_codec::extract_bucket_json(&input_json)+pub fn vault_codec_extract_buckets(input_json: String) -> Result<String, VaultError> {+ crate::vault_codec::extract_buckets_json(&input_json)}/// The bucket layout: `[{ category, tables: [<name>] }]` JSON. Source of truth for bucket-only sync.diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rsindex 982ba7660..9e456c759 100644--- a/core/rust/src/vault_codec/canonicalize.rs+++ b/core/rust/src/vault_codec/canonicalize.rs@@ -10,7 +10,7 @@//! `{ "__blobRef": hash, "__blobKind": kind }`;//! - every other column (including non-blob `{ "__b64" }` inline bytes) is copied verbatim.-use std::collections::{HashMap, HashSet};+use std::collections::HashMap;use base64::engine::general_purpose::STANDARD as BASE64;use base64::Engine;@@ -22,7 +22,7 @@ use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedManifest, Canonuse super::sharing::{clone_referenced_rows, partition_by_manifest, prune_unreferenced_logos, referenced_tables};use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_bucketed_table, is_skip_table, is_unstamped_scope,- manifest_scoped_tables, row_identity, tables_for_category, MANIFEST_ID_COL, OVERFLOW_TABLE, SCHEMA_VERSION,+ manifest_scoped_tables, row_identity, MANIFEST_ID_COL, OVERFLOW_TABLE, SCHEMA_VERSION,};use crate::error::VaultResult;@@ -83,6 +83,8 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canoniclet all_logos = snapshots.get("Logos").unwrap_or(&no_rows);let other_specs: Vec<ManifestSpec> = input.manifests.iter().skip(1).cloned().collect();+ let other_ids = other_specs.iter().map(|spec| spec.manifest_id.clone());+ let manifest_ids: Vec<String> = std::iter::once(writing_manifest_id.clone()).chain(other_ids).collect();let partitions = partition_by_manifest(&mut all_tables, &other_specs, &snapshots, &writing_manifest_id)?;// The writing manifest is finished exactly like every partition.@@ -99,7 +101,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonicmanifest_tables.insert(name, out_rows);}- let data_buckets = build_data_buckets(bucketed_rows, &overflow, &writing_manifest_id, &other_specs);+ let data_buckets = build_data_buckets(bucketed_rows, &overflow, &manifest_ids);// Start with the manifest the caller wrote from, which the input lists first.let mut manifests: Vec<CanonicalizedManifest> = Vec::with_capacity(1 + partitions.len());@@ -145,72 +147,69 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic}/// Route every bucketed row into the bucket of the manifest that owns it: `(ManifestId, category)`.-fn build_data_buckets(- bucketed_rows: HashMap<String, Vec<CodecRecord>>,- overflow: &CodecOverflow,- writing_manifest_id: &str,- other_specs: &[ManifestSpec],-) -> Vec<DataBucket> {- let known: HashSet<&str> = std::iter::once(writing_manifest_id).chain(other_specs.iter().map(|spec| spec.manifest_id.as_str())).collect();-- // (manifest id, category) > (table > rows).- let mut buckets: HashMap<(String, String), HashMap<String, Vec<CodecRecord>>> = HashMap::new();- for category in bucket_categories() {- let tables = buckets.entry((writing_manifest_id.to_string(), category.to_string())).or_default();- for table in tables_for_category(category) {- if bucketed_rows.contains_key(table) {- tables.insert(table.to_string(), Vec::new());- }- }+fn build_data_buckets(bucketed_rows: HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow, manifest_ids: &[String]) -> Vec<DataBucket> {+ let mut data_buckets: Vec<DataBucket> = Vec::new();+ for category in categories_present(overflow) {+ let grouped = group_category_rows(category_tables(&category, &bucketed_rows, overflow), manifest_ids);+ data_buckets.extend(grouped.into_iter().map(|(manifest_id, tables)| DataBucket::new(manifest_id, category.clone(), tables)));}+ data_buckets.sort_by(|a, b| (&a.manifest_id, &a.category).cmp(&(&b.manifest_id, &b.category)));+ data_buckets+}- /// The manifest a bucketed row belongs to: the one its stamp names. A stamp naming a manifest this- /// vault no longer carries drops the row with that manifest (the partition split's `Gone` rule).- fn owner<'a>(row: &'a CodecRecord, known: &HashSet<&str>) -> Option<&'a str> {- match row.get(MANIFEST_ID_COL).and_then(|value| value.as_str()) {- Some(id) if !is_unstamped_scope(Some(id)) => known.contains(id).then_some(id),- _ => None,- }- }+/// Every bucket category to emit: the declared ones plus any a newer writer put in the overflow (a category+/// this client's schema does not know yet still has to be carried forward). Sorted, so output stays stable.+fn categories_present(overflow: &CodecOverflow) -> Vec<String> {+ let mut categories: Vec<String> = bucket_categories().into_iter().map(str::to_string).collect();+ let mut extra: Vec<String> = overflow.bucket_tables.keys().filter(|category| !categories.contains(category)).cloned().collect();+ extra.sort();+ categories.extend(extra);+ categories+}- for (name, rows) in &bucketed_rows {- let category = bucket_category_for(name).expect("only bucketed tables are collected into bucketed_rows");- for row in rows {- let Some(manifest_id) = owner(row, &known) else { continue };- buckets- .entry((manifest_id.to_string(), category.to_string()))- .or_default()- .entry(name.clone())- .or_default()- .push(row.clone());+/// The tables read for one bucket category: the ones the local schema holds, plus whole tables a newer writer+/// put in that bucket which this client's schema cannot hold (carried in the overflow). Local rows win.+fn category_tables(category: &str, bucketed_rows: &HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow) -> HashMap<String, Vec<CodecRecord>> {+ let mut tables: HashMap<String, Vec<CodecRecord>> = bucketed_rows+ .iter()+ .filter(|(name, _)| bucket_category_for(name) == Some(category))+ .map(|(name, rows)| (name.clone(), rows.clone()))+ .collect();+ if let Some(ov_tables) = overflow.bucket_tables.get(category) {+ for (name, rows) in ov_tables {+ tables.entry(name.clone()).or_insert_with(|| rows.clone());}}+ tables+}- // Re-emit the overflow's bucket tables: whole tables a newer writer put in a bucket that this client's schema cannot hold.- for (category, ov_tables) in &overflow.bucket_tables {- for (name, rows) in ov_tables {- if bucketed_rows.contains_key(name) {- continue;- }- for row in rows {- let Some(manifest_id) = owner(row, &known) else { continue };- buckets- .entry((manifest_id.to_string(), category.clone()))- .or_default()- .entry(name.clone())- .or_default()- .push(row.clone());- }+/// Split one bucket category's tables into the table set each manifest owns. This is the single grouping+/// rule behind both write paths: the full push (via [`build_data_buckets`]) and the bucket-only push+/// (via [`extract_buckets`]).+fn group_category_rows(tables: HashMap<String, Vec<CodecRecord>>, manifest_ids: &[String]) -> HashMap<String, HashMap<String, Vec<CodecRecord>>> {+ let empty: HashMap<String, Vec<CodecRecord>> = tables.keys().map(|name| (name.clone(), Vec::new())).collect();+ let mut grouped: HashMap<String, HashMap<String, Vec<CodecRecord>>> = manifest_ids.iter().map(|id| (id.clone(), empty.clone())).collect();++ for (name, rows) in tables {+ for mut row in rows {+ let Some(owner) = owning_manifest(&row, manifest_ids) else { continue };+ // Normalize the stamp to the spelling the bucket declares the id with.+ row.insert(MANIFEST_ID_COL.to_string(), json!(owner));+ grouped.entry(owner).or_default().entry(name.clone()).or_default().push(row);}}- let mut data_buckets: Vec<DataBucket> = buckets- .into_iter()- .map(|((manifest_id, category), tables)| DataBucket::new(manifest_id, category, tables))- .collect();- // Deterministic bucket order (HashMap iteration is unordered) so canonicalize is reproducible.- data_buckets.sort_by(|a, b| (&a.manifest_id, &a.category).cmp(&(&b.manifest_id, &b.category)));- data_buckets+ grouped+}++/// The manifest a bucketed row belongs to, spelled the way `manifest_ids` spells it, or `None` when the row+/// names no manifest or one this vault does not carry.+fn owning_manifest(row: &CodecRecord, manifest_ids: &[String]) -> Option<String> {+ let stamp = row.get(MANIFEST_ID_COL).and_then(|value| value.as_str())?;+ if is_unstamped_scope(Some(stamp)) {+ return None;+ }+ manifest_ids.iter().find(|id| id.eq_ignore_ascii_case(stamp)).cloned()}/// For legacy sqlite-blob migration: the manifest that unstamped rows are adopted into.@@ -237,7 +236,7 @@ fn reject_unstamped_rows(tables: &HashMap<String, Vec<CodecRecord>>) -> VaultResif unstamped > 0 {let first = rows.iter().find(|row| is_unstamped(row)).and_then(|row| row.get("Id")).cloned().unwrap_or(Value::Null);return Err(crate::error::VaultError::General(format!(- "canonicalize refuses to write {} row(s) of {} that name no manifest (first: Id {}); every row must carry the manifest it belongs to",+ "the codec refuses to write {} row(s) of {} that name no manifest (first: Id {}); every row must carry the manifest it belongs to",unstamped, name, first)));}@@ -327,13 +326,17 @@ fn primary_key_of(identity: &str) -> &str {identity.rsplit('\u{1f}').next().unwrap_or(identity)}-/// Build a single data bucket for `(manifest_id, category)` from its already-normalized tables-/// (name > rows). The bucket-only push path (a bucket changed but the manifest didn't).+/// Build `category`'s data buckets from its already-normalized tables (name > rows): the bucket-only push+/// path, taken when a bucket changed but the manifest did not.///-/// Every row must be stamped for the bucket's own manifest: the caller groups rows by their stamps-/// before calling, so a row naming no manifest or another one is a grouping bug, refused loudly-/// rather than repaired by re-homing the row into a scope it never claimed.-pub fn extract_bucket(manifest_id: String, category: String, mut tables: HashMap<String, Vec<CodecRecord>>) -> VaultResult<DataBucket> {+/// Include the [`OVERFLOW_TABLE`] row in `tables` (read it alongside the category's tables) so a newer+/// writer's columns/tables re-merge and survive; it is consumed and never emitted into a bucket.+pub fn extract_buckets(category: String, manifest_ids: Vec<String>, mut tables: HashMap<String, Vec<CodecRecord>>) -> VaultResult<Vec<DataBucket>> {+ if manifest_ids.is_empty() {+ let message = "extract_buckets declares no manifests; a bucket write needs the manifests it may be addressed to";+ return Err(crate::error::VaultError::General(message.to_string()));+ }+let overflow = tables.remove(OVERFLOW_TABLE).map(|records| CodecOverflow::from_table_records(&records)).unwrap_or_default();remerge_overflow_columns(&mut tables, &overflow);if let Some(ov_tables) = overflow.bucket_tables.get(&category) {@@ -341,29 +344,12 @@ pub fn extract_bucket(manifest_id: String, category: String, mut tables: HashMaptables.entry(name.clone()).or_insert_with(|| rows.clone());}}+ reject_unstamped_rows(&tables)?;- for (name, rows) in tables.iter_mut() {- for row in rows.iter_mut() {- match row.get(MANIFEST_ID_COL).and_then(|value| value.as_str()) {- Some(id) if !is_unstamped_scope(Some(id)) => {- if !id.eq_ignore_ascii_case(&manifest_id) {- return Err(crate::error::VaultError::General(format!(- "extract_bucket for manifest {} got a {} row stamped for {}; the caller must group rows by the manifest they name",- manifest_id, name, id- )));- }- // Normalize the stamp to the bucket's declared spelling of the id.- row.insert(MANIFEST_ID_COL.to_string(), json!(manifest_id));- }- _ => {- return Err(crate::error::VaultError::General(format!(- "extract_bucket refuses a {} row that names no manifest; every row must carry the manifest it belongs to",- name- )));- }- }- }- }-- Ok(DataBucket::new(manifest_id, category, tables))+ let mut buckets: Vec<DataBucket> = group_category_rows(tables, &manifest_ids)+ .into_iter()+ .map(|(manifest_id, tables)| DataBucket::new(manifest_id, category.clone(), tables))+ .collect();+ buckets.sort_by(|a, b| a.manifest_id.cmp(&b.manifest_id));+ Ok(buckets)}diff --git a/core/rust/src/vault_codec/manifest.rs b/core/rust/src/vault_codec/manifest.rsindex d21e8242d..9d340f364 100644--- a/core/rust/src/vault_codec/manifest.rs+++ b/core/rust/src/vault_codec/manifest.rs@@ -200,6 +200,18 @@ pub struct CanonicalizeInput {pub adopt_unstamped_into: Option<String>,}+/// Input for [`crate::vault_codec::extract_buckets`].+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct ExtractBucketsInput {+ pub category: String,+ /// Every manifest this vault can write. Each gets a bucket back; rows naming anything else are dropped.+ pub manifest_ids: Vec<String>,+ /// The category's tables as read from the local vault, plus the+ /// [`OVERFLOW_TABLE`](super::types::OVERFLOW_TABLE) row so a newer writer's columns/tables re-merge.+ pub tables: HashMap<String, Vec<CodecRecord>>,+}+/// Input for [`crate::vault_codec::materialize_as_sqlite`].#[derive(Debug, Clone, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rsindex 6577d5be3..9dff877b4 100644--- a/core/rust/src/vault_codec/mod.rs+++ b/core/rust/src/vault_codec/mod.rs@@ -17,6 +17,8 @@ mod sharing;mod types;mod validate;+use std::collections::HashMap;+use base64::engine::general_purpose::STANDARD as BASE64;use base64::Engine;use serde_json::json;@@ -25,7 +27,7 @@ use crate::error::{VaultError, VaultResult};pub use manifest::{BlobEntry, BucketLayoutEntry, CanonicalizeInput, CanonicalizedManifest, CanonicalizedVault, CodecOverflow, DataBucket,- Manifest, MaterializeInput, MaterializedTables, CodecRecord, CodecTableData, ManifestSpec,+ ExtractBucketsInput, Manifest, MaterializeInput, MaterializedTables, CodecRecord, CodecTableData, ManifestSpec,};pub use scoped_assets::{KIND_BUILTIN as LOGO_KIND_BUILTIN, KIND_CUSTOM as LOGO_KIND_CUSTOM, KIND_FAVICON as LOGO_KIND_FAVICON};pub use sharing::{active_encryption_key, extract_encryption_key_for_public_key};@@ -49,11 +51,14 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materializematerialize::materialize_as_sqlite(input)}-/// Build the `(manifest_id, category)` data bucket from its tables (bucket-only push path).-/// Include the [`OVERFLOW_TABLE`] row in `tables` (read it alongside the bucket's tables) so a newer-/// writer's columns/tables re-merge and survive; it is consumed and never emitted into the bucket.-pub fn extract_bucket(manifest_id: String, category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>) -> VaultResult<DataBucket> {- canonicalize::extract_bucket(manifest_id, category, tables)+/// Build `category`'s data buckets, one per manifest in `manifest_ids`, from the category's tables as the+/// platform reads them out of its local vault (bucket-only push path). Rows route by the manifest each one+/// names, exactly as the full push routes them.+///+/// Include the [`OVERFLOW_TABLE`] row in `tables` (read it alongside the category's tables) so a newer+/// writer's columns/tables re-merge and survive; it is consumed and never emitted into a bucket.+pub fn extract_buckets(category: String, manifest_ids: Vec<String>, tables: HashMap<String, Vec<CodecRecord>>) -> VaultResult<Vec<DataBucket>> {+ canonicalize::extract_buckets(category, manifest_ids, tables)}/// The bucket layout: every category and the tables it owns, in declaration order.@@ -205,18 +210,11 @@ pub fn materialize_as_sqlite_json(input_json: &str) -> VaultResult<String> {Ok(serde_json::to_string(&materialize_as_sqlite(input)?)?)}-/// JSON-string sibling of [`extract_bucket`].-/// Input: `{ "manifestId": <str>, "category": <str>, "tables": { <name>: [rows] } }`.-pub fn extract_bucket_json(input_json: &str) -> VaultResult<String> {- #[derive(serde::Deserialize)]- #[serde(rename_all = "camelCase")]- struct Input {- manifest_id: String,- category: String,- tables: std::collections::HashMap<String, Vec<CodecRecord>>,- }- let input: Input = serde_json::from_str(input_json)?;- Ok(serde_json::to_string(&extract_bucket(input.manifest_id, input.category, input.tables)?)?)+/// JSON-string sibling of [`extract_buckets`].+/// Input: `{ "category": <str>, "manifestIds": [<str>], "tables": { <name>: [rows] } }`.+pub fn extract_buckets_json(input_json: &str) -> VaultResult<String> {+ let input: ExtractBucketsInput = serde_json::from_str(input_json)?;+ Ok(serde_json::to_string(&extract_buckets(input.category, input.manifest_ids, input.tables)?)?)}/// JSON-string sibling of [`bucket_layout`]. Output: `[{ "category": <str>, "tables": [<str>] }]`.diff --git a/core/rust/src/wasm.rs b/core/rust/src/wasm.rsindex 4eaeef80a..c0b259424 100644--- a/core/rust/src/wasm.rs+++ b/core/rust/src/wasm.rs@@ -7,7 +7,7 @@ use crate::credential_matcher::{};use crate::password_generator::{available_languages, generate_password};use crate::vault_codec::{- self, CanonicalizeInput, CodecRecord, DataBucket, Manifest, MaterializeInput,+ self, CanonicalizeInput, DataBucket, ExtractBucketsInput, Manifest, MaterializeInput,};use crate::vault_merge::{merge_vaults, MergeInput, MergeOutput};use crate::vault_pruner::{prune_vault, PruneInput, PruneOutput};@@ -164,21 +164,15 @@ pub fn vault_codec_materialize_as_sqlite_js(input: JsValue) -> Result<JsValue, Jcodec_to_js(&output)}-/// Build a single data bucket. Input: `{ manifestId, category, tables: { <name>: [rows] } }`.-#[wasm_bindgen(js_name = vaultCodecExtractBucket)]-pub fn vault_codec_extract_bucket_js(input: JsValue) -> Result<JsValue, JsValue> {- #[derive(serde::Deserialize)]- #[serde(rename_all = "camelCase")]- struct Input {- manifest_id: String,- category: String,- tables: std::collections::HashMap<String, Vec<CodecRecord>>,- }- let input: Input = serde_wasm_bindgen::from_value(input)- .map_err(|e| JsValue::from_str(&format!("Failed to parse extract-bucket input: {}", e)))?;- let bucket = vault_codec::extract_bucket(input.manifest_id, input.category, input.tables)- .map_err(|e| JsValue::from_str(&format!("extract_bucket failed: {}", e)))?;- codec_to_js(&bucket)+/// Build a bucket category's data buckets, one per manifest this vault writes: rows route by the manifest+/// each one names. Input: `{ category, manifestIds, tables: { <name>: [rows] } }`. Output: `DataBucket[]`.+#[wasm_bindgen(js_name = vaultCodecExtractBuckets)]+pub fn vault_codec_extract_buckets_js(input: JsValue) -> Result<JsValue, JsValue> {+ let input: ExtractBucketsInput = serde_wasm_bindgen::from_value(input)+ .map_err(|e| JsValue::from_str(&format!("Failed to parse extract-buckets input: {}", e)))?;+ let buckets = vault_codec::extract_buckets(input.category, input.manifest_ids, input.tables)+ .map_err(|e| JsValue::from_str(&format!("extract_buckets failed: {}", e)))?;+ codec_to_js(&buckets)}/// The bucket layout: `[{ category, tables: [<name>] }]`. Source of truth for platform bucket-only sync.