Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update bucket update flow (#2159)

Leendert de Borst1a16f74a76a7
diff --git a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
index da3c39821..c1ce8987d 100644
--- a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
+++ b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
@@ -4,20 +4,23 @@ import { storage } from 'wxt/utils/storage';
import { allVaultDataStorageKeys, AUTH_STORAGE_KEYS, dirtyScopeStorageKey, SESSION_STORAGE_KEYS, StorageKeys, VAULT_LOCK_STORAGE_KEYS } from '@/utils/constants/storageKeys';
import { TRASH_RETENTION_DAYS } from '@/utils/constants/vault';
+import type { ItemUsageAction } from '@/utils/db';
import type { DraftItem } from '@/utils/db/ItemRef';
import { devError, devLog, devWarn } from '@/utils/devLogger/DevLogger';
import type { EncryptionKeyDerivationParams } from '@/utils/dist/core/models/metadata';
-import { FieldKey, ItemTypes, createSystemField, type Item, type PasswordSettings } from '@/utils/dist/core/models/vault';
-import type { VaultResponse, VaultPostResponse, StatusResponseV2, ManifestRevision } from '@/utils/dist/core/models/webapi';
+import { FieldKey, ItemTypes, VaultDataBucketCategory, createSystemField, type Item, type PasswordSettings } from '@/utils/dist/core/models/vault';
+import type { VaultResponse, StatusResponseV2, ManifestRevision } from '@/utils/dist/core/models/webapi';
import { EncryptionUtility } from '@/utils/EncryptionUtility';
-import { requiresLegacyVaultKeyMigration } from '@/utils/LegacyVaultKeyMigration';
+import { readLegacySessionEncryptionKey } from '@/utils/legacy/LegacyStorageKeyFallbacks';
+import { requiresLegacyAccountKeyMigration } from '@/utils/legacy/LegacyStorageModelMigration';
import { LocalPreferencesService } from '@/utils/LocalPreferencesService';
+import { getManifestRevisions, getPersonalManifestId } from '@/utils/ManifestRevisions';
import { sendMessage } from '@/utils/messaging/ExtensionMessaging';
import { RecentlySelectedItemService } from '@/utils/RecentlySelectedItemService';
-import { filterItems, AutofillMatchingMode, extractRootDomain, isUrlAlreadyLinked, generatePassword, vaultCodecExtractBucket, vaultCodecBucketLayout, vaultCodecOverflowTable } from '@/utils/RustCore';
+import { filterItems, AutofillMatchingMode, extractRootDomain, isUrlAlreadyLinked, generatePassword, vaultCodecExtractBuckets, vaultCodecBucketLayout, vaultCodecOverflowTable } from '@/utils/RustCore';
import { ServiceDetectionUtility } from '@/utils/serviceDetection/ServiceDetectionUtility';
import { SqliteClient } from '@/utils/SqliteClient';
-import { getItemWithFallback } from '@/utils/StorageUtility';
+import { getStorageItem } from '@/utils/StorageUtility';
import { ApiAuthError } from '@/utils/types/errors/ApiAuthError';
import { ApiRequestError } from '@/utils/types/errors/ApiRequestError';
import { AppErrorCode, formatErrorWithCode } from '@/utils/types/errors/AppErrorCodes';
@@ -69,59 +72,36 @@ function personalManifestRevision(status: StatusResponseV2): number {
return personal?.revision ?? 0;
}
-/** The client's last-known revision per shared manifest (manifestId → revision); empty when no shared manifests. */
-async function getLocalSharedManifestRevisions(): Promise<Record<string, number>> {
- return (await storage.getItem(StorageKeys.SERVER_MANIFEST_REVISIONS)) as Record<string, number> | null ?? {};
-}
-
/**
* Whether the client should pull and re-materialize because the server's manifest set no longer matches what the
- * client has locally materialized. Compares the full server list against the client's per-manifest last-known
- * revisions and is idempotent: after a successful materialize the local record matches
+ * client has locally materialized. One map-versus-map comparison covers every manifest, the personal one included:
+ * the server reports it by id like any other. Idempotent: after a successful materialize the local record matches
* the server, so this returns false until the next server-side change.
- *
+ *
* @param serverManifests - the per-manifest revisions from the status response
- * @param personalManifestId - the id the server names as the caller's own; every other entry is a shared manifest
- * @param localMainRevision - the client's last-known personal manifest revision (local:serverRevision)
- * @param localSharedRevisions - the client's last-known revision per shared manifest
- */
-function serverManifestsNeedPull(serverManifests: ManifestRevision[], personalManifestId: string | null, localMainRevision: number, localSharedRevisions: Record<string, number>): boolean {
- const serverShared = new Map<string, number>();
- let serverPersonalRevision: number | null = null;
- for (const m of (serverManifests ?? [])) {
- if (personalManifestId !== null && m.manifestId === personalManifestId) {
- serverPersonalRevision = m.revision;
- } else {
- serverShared.set(m.manifestId, m.revision);
- }
- }
-
- /*
- * Decide whether to pull and log the concrete reason.
- */
- if (serverPersonalRevision !== null && serverPersonalRevision > localMainRevision) {
- devLog(`[VaultSync] Pull needed: personal manifest server rev ${serverPersonalRevision} > local rev ${localMainRevision}.`);
- return true;
- }
+ * @param localRevisions - the client's last-known revision per manifest
+ */
+function serverManifestsNeedPull(serverManifests: ManifestRevision[], localRevisions: Record<string, number>): boolean {
+ const serverRevisions = new Map((serverManifests ?? []).map(m => [m.manifestId, m.revision]));
- // Any shared manifest added or with a changed revision on the server.
- for (const [manifestId, revision] of serverShared) {
- const local = localSharedRevisions[manifestId];
+ // Any manifest added or with a changed revision on the server.
+ for (const [manifestId, revision] of serverRevisions) {
+ const local = localRevisions[manifestId];
if (local !== revision) {
const reason = local === undefined ? `not tracked locally (server rev ${revision})` : `server rev ${revision} != local rev ${local}`;
- devLog(`[VaultSync] Pull needed: shared manifest ${manifestId} ${reason}.`);
+ devLog(`[VaultSync] Pull needed: manifest ${manifestId} ${reason}.`);
return true;
}
}
- // Any shared manifest the client still tracks but the server no longer lists (removed / revoked).
- const removed = Object.keys(localSharedRevisions).find(manifestId => !serverShared.has(manifestId));
+ // Any manifest the client still tracks but the server no longer lists (removed / revoked).
+ const removed = Object.keys(localRevisions).find(manifestId => !serverRevisions.has(manifestId));
if (removed) {
- devLog(`[VaultSync] Pull needed: shared manifest ${removed} (local rev ${localSharedRevisions[removed]}) no longer listed by the server (revoked/removed).`);
+ devLog(`[VaultSync] Pull needed: manifest ${removed} (local rev ${localRevisions[removed]}) no longer listed by the server (revoked/removed).`);
return true;
}
- devLog(`[VaultSync] No pull needed: personal rev ${serverPersonalRevision ?? 'n/a'} (local ${localMainRevision}), ${serverShared.size} shared manifest(s) all match local revisions.`);
+ devLog(`[VaultSync] No pull needed: ${serverRevisions.size} manifest(s) all match local revisions.`);
return false;
}
@@ -258,6 +238,13 @@ async function fetchLatestVaultFromServer(): Promise<VaultResponse> {
}
}
+/**
+ * Outcome of a push attempt: 0 = ok, 2 = the server had newer state and the caller must re-sync (pull/merge/retry).
+ * Revision bookkeeping happens inside VaultSyncService, which records the new revision of every manifest a
+ * successful write actually carried; no revision travels back through this result.
+ */
+type VaultPushOutcome = { status: number };
+
/**
* Push the current SQLite vault to the server.
*
@@ -265,7 +252,7 @@ async function fetchLatestVaultFromServer(): Promise<VaultResponse> {
* @param options - forceFullWrite bypasses the content-fingerprint gating and rewrites every manifest and bucket
* (server rollback recovery); createVaultKey mints the VEK as part of this push (KEK/VEK migration)
*/
-async function pushVaultToServer(sqliteClient: SqliteClient, options: { forceFullWrite: boolean; createVaultKey: boolean }): Promise<VaultPostResponse> {
+async function pushVaultToServer(sqliteClient: SqliteClient, options: { forceFullWrite: boolean; createVaultKey: boolean }): Promise<VaultPushOutcome> {
const encryptionKey = await handleGetEncryptionKey();
if (!encryptionKey) {
throw new Error(formatErrorWithCode(await t('common.errors.vaultIsLocked'), AppErrorCode.VAULT_LOCKED));
@@ -284,15 +271,15 @@ async function pushVaultToServer(sqliteClient: SqliteClient, options: { forceFul
await handleStoreEncryptionKey(result.newEncryptionKey);
}
- return { status: 0, newRevisionNumber: result.newManifestRevision ?? 0 };
+ return { status: 0 };
}
if (result.status === 'outdated') {
- return { status: 2, newRevisionNumber: result.newManifestRevision ?? 0 };
+ return { status: 2 };
}
if (result.status === 'rejected') {
- // Structural validation tripped — surface the reasons rather than silently uploading garbage.
+ // Structural validation error
const reason = (result.reasons ?? ['Integrity check failed']).join('; ');
devError('[V2Sync] Refusing to upload corrupt vault:', reason);
throw new Error(formatErrorWithCode(`Vault integrity check failed: ${reason}`, AppErrorCode.UPLOAD_FAILED));
@@ -308,8 +295,7 @@ async function pushVaultToServer(sqliteClient: SqliteClient, options: { forceFul
/**
* Sync the vault with the server to check if a newer vault is available. If so, the vault will be updated.
*/
-export async function handleSyncVault(
-) : Promise<messageBoolResponse> {
+export async function handleSyncVault() : Promise<messageBoolResponse> {
const webApi = new WebApiService();
const statusResponse = await webApi.getStatus();
const statusError = webApi.validateStatusResponse(statusResponse);
@@ -317,22 +303,18 @@ export async function handleSyncVault(
return { success: false, error: await t('common.errors.' + statusError) };
}
- const localServerRevision = await storage.getItem(StorageKeys.SERVER_REVISION) as number | null ?? 0;
- const localSharedRevisions = await getLocalSharedManifestRevisions();
-
- if (serverManifestsNeedPull(statusResponse.manifestRevisions, statusResponse.personalManifestId, localServerRevision, localSharedRevisions)) {
+ if (serverManifestsNeedPull(statusResponse.manifestRevisions, await getManifestRevisions())) {
/*
* Retrieve the latest vault from the server.
*/
const vaultResponse = await fetchLatestVaultFromServer();
- // Store in local: storage for persistence (fresh from server, not dirty)
+ // Store in local: storage for persistence (fresh from server, not dirty). The pull recorded the per-manifest revisions itself.
await storage.setItems([
{ key: StorageKeys.ENCRYPTED_VAULT, value: vaultResponse.vault.blob },
{ key: StorageKeys.PUBLIC_EMAIL_DOMAINS, value: vaultResponse.vault.publicEmailDomainList },
{ key: StorageKeys.PRIVATE_EMAIL_DOMAINS, value: vaultResponse.vault.privateEmailDomainList },
{ key: StorageKeys.HIDDEN_PRIVATE_EMAIL_DOMAINS, value: vaultResponse.vault.hiddenPrivateEmailDomainList },
- { key: StorageKeys.SERVER_REVISION, value: vaultResponse.vault.currentRevisionNumber },
{ key: StorageKeys.IS_DIRTY, value: false }
]);
@@ -354,10 +336,9 @@ export async function handleGetVault(
const encryptedVault = await storage.getItem(StorageKeys.ENCRYPTED_VAULT) as string;
// TODO: the fallback mechanism can be removed some period of time after 0.27.0 is released.
- const publicEmailDomains = await getItemWithFallback<string[]>(StorageKeys.PUBLIC_EMAIL_DOMAINS);
- const privateEmailDomains = await getItemWithFallback<string[]>(StorageKeys.PRIVATE_EMAIL_DOMAINS);
- const hiddenPrivateEmailDomains = await getItemWithFallback<string[]>(StorageKeys.HIDDEN_PRIVATE_EMAIL_DOMAINS) ?? [];
- const serverRevision = await storage.getItem(StorageKeys.SERVER_REVISION) as number | null;
+ const publicEmailDomains = await getStorageItem<string[]>(StorageKeys.PUBLIC_EMAIL_DOMAINS);
+ const privateEmailDomains = await getStorageItem<string[]>(StorageKeys.PRIVATE_EMAIL_DOMAINS);
+ const hiddenPrivateEmailDomains = await getStorageItem<string[]>(StorageKeys.HIDDEN_PRIVATE_EMAIL_DOMAINS) ?? [];
if (!encryptedVault) {
console.error('Vault not available');
@@ -381,8 +362,7 @@ export async function handleGetVault(
vault: decryptedVault,
publicEmailDomains: publicEmailDomains ?? [],
privateEmailDomains: privateEmailDomains ?? [],
- hiddenPrivateEmailDomains: hiddenPrivateEmailDomains ?? [],
- serverRevision: serverRevision ?? 0
+ hiddenPrivateEmailDomains: hiddenPrivateEmailDomains ?? []
};
} catch (error) {
console.error('Failed to get vault:', error);
@@ -430,9 +410,6 @@ export async function handleClearVaultData(): Promise<messageBoolResponse> {
// Clear vault data and every piece of state derived from it (sync bookkeeping, dirty flags, bucket revisions)
await storage.removeItems(allVaultDataStorageKeys());
- // Clear the cached vault key and encrypted VEK.
- VaultKeyService.clearCache();
-
// Clear all local preferences (site settings, login save settings, etc.)
await LocalPreferencesService.clearAll();
@@ -642,8 +619,8 @@ export function handleGetDefaultEmailDomain(): Promise<stringResponse> {
// If no default domain is configured, fall back to first private or public domain
if (!domain) {
- const privateEmailDomains = await getItemWithFallback<string[]>(StorageKeys.PRIVATE_EMAIL_DOMAINS) ?? [];
- const publicEmailDomains = await getItemWithFallback<string[]>(StorageKeys.PUBLIC_EMAIL_DOMAINS) ?? [];
+ const privateEmailDomains = await getStorageItem<string[]>(StorageKeys.PRIVATE_EMAIL_DOMAINS) ?? [];
+ const publicEmailDomains = await getStorageItem<string[]>(StorageKeys.PUBLIC_EMAIL_DOMAINS) ?? [];
domain = privateEmailDomains[0] || publicEmailDomains[0] || '';
}
@@ -719,15 +696,10 @@ export async function handleGeneratePassword(
export async function handleGetEncryptionKey(
) : Promise<string | null> {
// Try the current key name first (since 0.22.0)
- let encryptionKey = await storage.getItem(StorageKeys.ENCRYPTION_KEY) as string | null;
+ const encryptionKey = await storage.getItem(StorageKeys.ENCRYPTION_KEY) as string | null;
- // Fall back to the legacy key name if not found
- if (!encryptionKey) {
- // TODO: this check can be removed some period of time after 0.22.0 is released.
- encryptionKey = await storage.getItem(StorageKeys.LEGACY_DERIVED_KEY) as string | null;
- }
-
- return encryptionKey;
+ // LEGACY: fall back to the pre-0.22.0 key name.
+ return encryptionKey ?? await readLegacySessionEncryptionKey();
}
/**
@@ -737,7 +709,7 @@ export async function handleGetEncryptionKey(
export async function handleGetEncryptionKeyDerivationParams(
) : Promise<EncryptionKeyDerivationParams | null> {
// Get metadata from storage
- return await getItemWithFallback<EncryptionKeyDerivationParams>(StorageKeys.ENCRYPTION_KEY_DERIVATION_PARAMS);
+ return await getStorageItem<EncryptionKeyDerivationParams>(StorageKeys.ENCRYPTION_KEY_DERIVATION_PARAMS);
}
/**
@@ -770,18 +742,32 @@ async function clearDirtyScopes(): Promise<void> {
* @param sqliteClient - the in-memory SQLite client to read bucket data from
* @param scopes - the pending dirty scopes (bucket category names, deduplicated here)
*/
-async function uploadDirtyBucketsOnly(sqliteClient: SqliteClient, scopes: VaultMutationScope[]): Promise<VaultPostResponse> {
+async function uploadDirtyBucketsOnly(sqliteClient: SqliteClient, scopes: VaultMutationScope[]): Promise<VaultPushOutcome> {
const encryptionKey = await handleGetEncryptionKey();
if (!encryptionKey) {
throw new Error(formatErrorWithCode(await t('common.errors.vaultIsLocked'), AppErrorCode.VAULT_LOCKED));
}
/*
- * Which tables make up each bucket category is owned by the Rust layer.
+ * Which tables make up each bucket category is owned by the Rust layer, and so is splitting their rows
+ * across the manifests that own them: this loop reads a category whole and hands it over per category.
*/
const layout = await vaultCodecBucketLayout();
const username = (await storage.getItem(StorageKeys.USERNAME)) as string;
+ if (!sqliteClient.getActiveManifestId() && !sqliteClient.getPersonalManifestId()) {
+ devWarn('[V2Push] No manifest to address the bucket write to, falling back to a full vault upload.');
+ return (await uploadNewVaultToServer(sqliteClient, { forceFullWrite: false, createVaultKey: false })).response;
+ }
+
+ /*
+ * Every manifest this vault can write. Each gets a bucket back, empty where it holds no rows of the
+ * category, so deleting a manifest's last row still reaches the server; a row stamped for anything else
+ * (a manifest whose key this session never opened) is dropped and left to the next full push.
+ */
+ const writeKeys = await vaultSyncService.resolveBucketWriteKeys(encryptionKey);
+ const overflowTable = await vaultCodecOverflowTable();
+
for (const category of new Set(scopes)) {
const spec = layout.find(entry => entry.category === category);
if (!spec) {
@@ -789,22 +775,21 @@ async function uploadDirtyBucketsOnly(sqliteClient: SqliteClient, scopes: VaultM
return (await uploadNewVaultToServer(sqliteClient, { forceFullWrite: false, createVaultKey: false })).response;
}
- /*
- * Read the CodecOverflows carrier row alongside the bucket's tables: extract_bucket consumes it
- * to re-merge a newer client's bucket tables/columns, so this partial push doesn't drop them.
- */
- const tables = VaultCodec.readNamedTables(sqliteClient, [...spec.tables, await vaultCodecOverflowTable()]);
- const bucket = await vaultCodecExtractBucket(category, tables);
- const result = await vaultSyncService.pushDataBucketOnly(bucket, encryptionKey, username);
- if (result.status !== 'ok') {
- // Conflict persisted even after the rebase-retry, let the caller run a full re-sync (status 2).
- return { status: 2, newRevisionNumber: (await storage.getItem(StorageKeys.SERVER_REVISION)) as number | null ?? 0 };
+ const tables = VaultCodec.readNamedTables(sqliteClient, [...spec.tables, overflowTable]);
+ const buckets = await vaultCodecExtractBuckets(category, [...writeKeys.keys()], tables);
+
+ for (const bucket of buckets) {
+ const result = await vaultSyncService.pushDataBucketOnly(bucket, writeKeys.get(bucket.manifestId)!, username);
+ if (result.status !== 'ok') {
+ // Conflict persisted even after the rebase-retry, let the caller run a full re-sync (status 2).
+ return { status: 2 };
+ }
+ devLog(`[V2Push] Bucket-only push for "${category}" of manifest ${bucket.manifestId} done (bucket revision ${result.revision}); manifest untouched.`);
}
- devLog(`[V2Push] Bucket-only push for "${category}" done (bucket revision ${result.revision}); manifest untouched.`);
}
- // Manifest unchanged, so the content revision stays where it was.
- return { status: 0, newRevisionNumber: (await storage.getItem(StorageKeys.SERVER_REVISION)) as number | null ?? 0 };
+ // Manifests untouched, so every manifest revision baseline stays where it was.
+ return { status: 0 };
}
/**
@@ -846,7 +831,7 @@ export async function handleUploadVault(
if (bucketOnly) {
devLog(`[V2Push] All pending mutations are bucket-scoped (${dirtyScopes.join(', ')}), skipping manifest upload.`);
}
- let response: VaultPostResponse;
+ let response: VaultPushOutcome;
let vaultPruned = false;
if (bucketOnly) {
// Bucket-only pushes never prune (settings buckets carry no trash items).
@@ -858,7 +843,6 @@ export async function handleUploadVault(
return {
success: true,
status: response.status,
- newRevisionNumber: response.newRevisionNumber,
mutationSeqAtStart,
vaultPruned
};
@@ -960,7 +944,7 @@ export async function handleClearPersistedFormValues(): Promise<void> {
* @param sqliteClient - the in-memory SQLite client to upload
* @param options - forceFullWrite: whether to force a full write of the vault, createVaultKey: whether to create a vault key
*/
-async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { forceFullWrite: boolean; createVaultKey: boolean }) : Promise<{ response: VaultPostResponse; vaultPruned: boolean }> {
+async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { forceFullWrite: boolean; createVaultKey: boolean }) : Promise<{ response: VaultPushOutcome; vaultPruned: boolean }> {
devLog('[VaultSync] Upload started');
let vaultPruned = false;
const encryptionKey = await handleGetEncryptionKey();
@@ -988,7 +972,7 @@ async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { for
console.warn('[VaultSync] Failed to prune vault, continuing with upload:', pruneError);
}
- let pushResponse: VaultPostResponse;
+ let pushResponse: VaultPushOutcome;
try {
pushResponse = await pushVaultToServer(sqliteClient, options);
} catch (err) {
@@ -1011,9 +995,6 @@ async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { for
cachedSqliteClient = sqliteClient;
cachedVaultBlob = reEncrypted;
}
- if (pushResponse.status === 0) {
- await storage.setItem(StorageKeys.SERVER_REVISION, pushResponse.newRevisionNumber);
- }
return { response: pushResponse, vaultPruned };
}
@@ -1022,11 +1003,15 @@ async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { for
* Persist a locally-mutated vault and attempt to sync it to the server in the background.
*
* This is tolerant to server being offline (in which case the vault state will be stored locally for next sync).
+ * @param sqliteClient - the mutated vault
+ * @param encryptionKey - the key the local blob is stored under
+ * @param scope - what the mutation touched; bucket-scoped mutations (e.g. 'Stats') let the sync push just
+ * that data bucket instead of the whole manifest. Defaults to a full manifest push.
*/
-async function persistLocalVaultMutation(sqliteClient: SqliteClient, encryptionKey: string) : Promise<void> {
+async function persistLocalVaultMutation(sqliteClient: SqliteClient, encryptionKey: string, scope?: VaultMutationScope) : Promise<void> {
const updatedVaultData = sqliteClient.exportToBase64();
const encryptedVault = await EncryptionUtility.symmetricEncrypt(updatedVaultData, encryptionKey);
- await handleStoreEncryptedVault({ vaultBlob: encryptedVault, markDirty: true });
+ await handleStoreEncryptedVault({ vaultBlob: encryptedVault, markDirty: true, scope });
/*
* The stored blob is exactly this client's content, so re-adopt the pair as the cache (the store just cleared
@@ -1117,14 +1102,12 @@ export async function handleGetEncryptedVault(): Promise<string | null> {
* @param request Object with:
* - vaultBlob: The encrypted vault data
* - markDirty: If true, marks vault as dirty and increments mutation sequence (for local mutations)
- * - serverRevision: Optional explicit server revision (for sync operations)
* - expectedMutationSeq: If provided, only store if current sequence matches (for sync operations)
* @returns { success, mutationSequence } - success=false if expectedMutationSeq didn't match
*/
export async function handleStoreEncryptedVault(request: {
vaultBlob: string;
markDirty?: boolean;
- serverRevision?: number;
expectedMutationSeq?: number;
scope?: VaultMutationScope;
}): Promise<{ success: boolean; mutationSequence: number }> {
@@ -1147,26 +1130,13 @@ export async function handleStoreEncryptedVault(request: {
const dirtyScopeKey = dirtyScopeStorageKey(request.scope ?? DEFAULT_VAULT_MUTATION_SCOPE);
// Build items to store.
- if (request.markDirty && request.serverRevision !== undefined) {
- await storage.setItems([
- { key: StorageKeys.ENCRYPTED_VAULT, value: request.vaultBlob },
- { key: StorageKeys.MUTATION_SEQUENCE, value: mutationSequence },
- { key: StorageKeys.IS_DIRTY, value: true },
- { key: StorageKeys.SERVER_REVISION, value: request.serverRevision },
- { key: dirtyScopeKey, value: true }
- ]);
- } else if (request.markDirty) {
+ if (request.markDirty) {
await storage.setItems([
{ key: StorageKeys.ENCRYPTED_VAULT, value: request.vaultBlob },
{ key: StorageKeys.MUTATION_SEQUENCE, value: mutationSequence },
{ key: StorageKeys.IS_DIRTY, value: true },
{ key: dirtyScopeKey, value: true }
]);
- } else if (request.serverRevision !== undefined) {
- await storage.setItems([
- { key: StorageKeys.ENCRYPTED_VAULT, value: request.vaultBlob },
- { key: StorageKeys.SERVER_REVISION, value: request.serverRevision }
- ]);
} else {
await storage.setItem(StorageKeys.ENCRYPTED_VAULT, request.vaultBlob);
}
@@ -1211,7 +1181,7 @@ export async function handleMigrateVaultManifest(): Promise<VaultManifestMigrati
}
const needsSchemaMigration = await sqliteClient.requiresSchemaMigration();
- const needsVaultKey = await requiresLegacyVaultKeyMigration();
+ const needsVaultKey = await requiresLegacyAccountKeyMigration();
if (!needsSchemaMigration && !needsVaultKey) {
devLog('[ManifestMigration] Vault is already on the current storage model, nothing to migrate.');
return { success: true, pushed: true };
@@ -1236,11 +1206,8 @@ export async function handleMigrateVaultManifest(): Promise<VaultManifestMigrati
try {
const uploadResponse = await handleUploadVault({ createVaultKey: needsVaultKey });
if (uploadResponse.success && uploadResponse.status === 0) {
- await handleMarkVaultClean({
- mutationSeqAtStart: uploadResponse.mutationSeqAtStart!,
- newServerRevision: uploadResponse.newRevisionNumber!
- });
- devLog(`[ManifestMigration] Migration pushed, server now at revision ${uploadResponse.newRevisionNumber}.`);
+ await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart! });
+ devLog('[ManifestMigration] Migration pushed to the server.');
return { success: true, pushed: true };
}
devWarn('[ManifestMigration] Migration push did not succeed, vault stays dirty for the next sync:', uploadResponse.error);
@@ -1260,27 +1227,20 @@ export async function handleMigrateVaultManifest(): Promise<VaultManifestMigrati
* Only clears dirty flag if no mutations happened during sync.
*
* @param mutationSeqAtStart - The mutation sequence when sync started
- * @param newServerRevision - The new server revision after successful upload
* @returns Whether the dirty flag was cleared
*/
export async function handleMarkVaultClean(request: {
mutationSeqAtStart: number;
- newServerRevision: number;
}): Promise<{ cleared: boolean; currentMutationSeq: number }> {
const currentMutationSeq = await storage.getItem(StorageKeys.MUTATION_SEQUENCE) as number | null ?? 0;
if (currentMutationSeq === request.mutationSeqAtStart) {
// No mutations during sync - safe to mark as clean
- await storage.setItems([
- { key: StorageKeys.IS_DIRTY, value: false },
- { key: StorageKeys.SERVER_REVISION, value: request.newServerRevision }
- ]);
+ await storage.setItem(StorageKeys.IS_DIRTY, false);
await clearDirtyScopes();
return { cleared: true, currentMutationSeq };
}
- // Mutations happened during sync - keep dirty, but still update server revision
- await storage.setItem(StorageKeys.SERVER_REVISION, request.newServerRevision);
return { cleared: false, currentMutationSeq };
}
@@ -1290,49 +1250,20 @@ export async function handleMarkVaultClean(request: {
export async function handleGetSyncState(): Promise<{
isDirty: boolean;
mutationSequence: number;
- serverRevision: number;
isSyncInProgress: boolean;
}> {
- const [isDirty, mutationSequence, serverRevision] = await Promise.all([
+ const [isDirty, mutationSequence] = await Promise.all([
storage.getItem(StorageKeys.IS_DIRTY) as Promise<boolean | null>,
- storage.getItem(StorageKeys.MUTATION_SEQUENCE) as Promise<number | null>,
- storage.getItem(StorageKeys.SERVER_REVISION) as Promise<number | null>
+ storage.getItem(StorageKeys.MUTATION_SEQUENCE) as Promise<number | null>
]);
return {
isDirty: isDirty ?? false,
mutationSequence: mutationSequence ?? 0,
- serverRevision: serverRevision ?? 0,
isSyncInProgress
};
}
-/**
- * Get the current server revision.
- */
-export async function handleGetServerRevision(): Promise<number> {
- // First try new key, then fall back to legacy key
- let revision = await storage.getItem(StorageKeys.SERVER_REVISION) as number | null;
-
- if (revision === null) {
- // Try legacy key - parse string format "250" or "250+1" to get server part
- const legacyRevision = await storage.getItem(StorageKeys.LEGACY_VAULT_REVISION_NUMBER) as string | number | null;
- if (legacyRevision !== null) {
- if (typeof legacyRevision === 'number') {
- revision = legacyRevision;
- } else {
- // Handle legacy "250+1" format - extract just the server part
- const parts = legacyRevision.split('+');
- revision = parseInt(parts[0], 10) || 0;
- }
- // Migrate to new key
- await storage.setItem(StorageKeys.SERVER_REVISION, revision);
- }
- }
-
- return revision ?? 0;
-}
-
/**
* Adopt a server-side vault key this device does not know about yet.
*
@@ -1385,7 +1316,7 @@ async function adoptRemoteVaultKeyIfNeeded(): Promise<boolean> {
await storage.setItem(StorageKeys.ENCRYPTED_VAULT, await EncryptionUtility.symmetricEncrypt(decrypted, vek));
}
- await VaultKeyService.cacheEncryptedVekFromServer();
+ await VaultKeyService.cacheVaultKeyBlobs(fetchResult.vaultKey);
await handleStoreEncryptionKey(vek);
cachedSqliteClient = null;
cachedVaultBlob = null;
@@ -1505,11 +1436,16 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {
// Get current sync state
const syncState = await handleGetSyncState();
+ const localRevisions = await getManifestRevisions();
+ const personalManifestId = await getPersonalManifestId();
const serverManifestRevision = personalManifestRevision(statusResponse);
- const localSharedRevisions = await getLocalSharedManifestRevisions();
- const needsPull = serverManifestsNeedPull(statusResponse.manifestRevisions, statusResponse.personalManifestId, syncState.serverRevision, localSharedRevisions);
+ const localManifestRevision = personalManifestId ? localRevisions[personalManifestId] ?? 0 : 0;
+
+ // A personal manifest that went backwards is a rolled-back server (restored backup), which we want to re-push.
+ const serverRolledBack = serverManifestRevision < localManifestRevision;
+ const needsPull = !serverRolledBack && serverManifestsNeedPull(statusResponse.manifestRevisions, localRevisions);
- devLog(`[VaultSync] Status received (server main rev ${serverManifestRevision}, local rev ${syncState.serverRevision}, needsPull ${needsPull}, isDirty ${syncState.isDirty})`);
+ devLog(`[VaultSync] Status received (server main rev ${serverManifestRevision}, local rev ${localManifestRevision}, needsPull ${needsPull}, isDirty ${syncState.isDirty})`);
// Check if server is actually available (0.0.0 indicates connection error)
if (statusResponse.serverVersion === '0.0.0') {
@@ -1561,7 +1497,7 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {
try {
const sqliteClient = await createVaultSqliteClient();
if (await vaultSyncService.detectNoOpMutation(sqliteClient, syncState.mutationSequence)) {
- const cleanResult = await handleMarkVaultClean({ mutationSeqAtStart: syncState.mutationSequence, newServerRevision: syncState.serverRevision });
+ const cleanResult = await handleMarkVaultClean({ mutationSeqAtStart: syncState.mutationSequence });
if (cleanResult.cleared) {
devLog('[VaultSync] Local changes are canonically identical to the server baselines (no-op mutation); nothing to push.');
syncState.isDirty = false;
@@ -1615,7 +1551,7 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {
devWarn('[VaultSync] Server vault is still on the legacy storage format while the local vault is migrated; skipping the merge and pushing the local vault.');
const uploadResponse = await handleUploadVault({ forceFullWrite: true });
if (uploadResponse.success && uploadResponse.status === 0) {
- await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart!, newServerRevision: uploadResponse.newRevisionNumber! });
+ await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart! });
return { success: true, hasNewVault: false, wasOffline: false, sqliteBlobUpgradeRequired: false, requiresLogout: false };
}
if (uploadResponse.status === 2) {
@@ -1640,7 +1576,6 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {
*/
const storeResult = await handleStoreEncryptedVault({
vaultBlob: mergedEncryptedVault,
- serverRevision: vaultResponseJson.vault.currentRevisionNumber,
expectedMutationSeq: syncState.mutationSequence
});
@@ -1653,10 +1588,7 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {
const uploadResponse = await handleUploadVault();
if (uploadResponse.success && uploadResponse.status === 0) {
- await handleMarkVaultClean({
- mutationSeqAtStart: uploadResponse.mutationSeqAtStart!,
- newServerRevision: uploadResponse.newRevisionNumber!
- });
+ await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart! });
} else if (uploadResponse.status === 2) {
// Server returned Outdated - another device uploaded. Re-sync.
return handleFullVaultSync();
@@ -1691,7 +1623,6 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {
*/
const storeResult = await handleStoreEncryptedVault({
vaultBlob: vaultResponseJson.vault.blob,
- serverRevision: vaultResponseJson.vault.currentRevisionNumber,
expectedMutationSeq: syncState.mutationSequence
});
@@ -1722,7 +1653,7 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {
AppErrorCode.VAULT_DECRYPT_FAILED
));
}
- } else if (serverManifestRevision === syncState.serverRevision) {
+ } else if (serverManifestRevision === localManifestRevision) {
/**
* Server and local vault are at the same revision.
* If we have pending local changes, upload them now.
@@ -1730,10 +1661,7 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {
if (syncState.isDirty) {
const uploadResponse = await handleUploadVault();
if (uploadResponse.success && uploadResponse.status === 0) {
- await handleMarkVaultClean({
- mutationSeqAtStart: uploadResponse.mutationSeqAtStart!,
- newServerRevision: uploadResponse.newRevisionNumber!
- });
+ await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart! });
/*
* If expired trash items were pruned during upload, report the vault as new
@@ -1754,14 +1682,14 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {
}
return { success: true, hasNewVault: false, wasOffline: false, sqliteBlobUpgradeRequired: false, requiresLogout: false };
- } else if (serverManifestRevision < syncState.serverRevision) {
+ } else if (serverRolledBack) {
/**
* Server revision DECREASED - server data loss/rollback detected.
* Client has more advanced revision - upload to recover server state.
*/
console.warn(
`Server data loss detected! Server at rev ${serverManifestRevision}, ` +
- `client at rev ${syncState.serverRevision}. Uploading to recover server state.`
+ `client at rev ${localManifestRevision}. Uploading to recover server state.`
);
/*
@@ -1771,14 +1699,9 @@ async function handleFullVaultSyncInternal(): Promise<FullVaultSyncResult> {
const uploadResponse = await handleUploadVault({ forceFullWrite: true });
if (uploadResponse.success && uploadResponse.status === 0) {
- await handleMarkVaultClean({
- mutationSeqAtStart: uploadResponse.mutationSeqAtStart!,
- newServerRevision: uploadResponse.newRevisionNumber!
- });
+ await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart! });
- console.info(
- `Server recovery complete: rev ${serverManifestRevision} → ${uploadResponse.newRevisionNumber}`
- );
+ console.info(`Server recovery complete: server restored from client state (was at personal rev ${serverManifestRevision}, client at ${localManifestRevision}).`);
return { success: true, hasNewVault: uploadResponse.vaultPruned === true, wasOffline: false, sqliteBlobUpgradeRequired: false, requiresLogout: false };
} else if (uploadResponse.status === 2) {
@@ -2318,7 +2241,7 @@ export async function handleGetTotpSecrets(
const secrets: Record<string, string> = {};
for (const itemId of message.itemIds) {
- const totpCodes = sqliteClient.settings.getTotpCodesForItem(itemId);
+ const totpCodes = sqliteClient.items.getTotpCodesForItem(itemId);
if (totpCodes.length > 0) {
secrets[itemId] = totpCodes[0].SecretKey;
}
@@ -2348,7 +2271,7 @@ export async function handleGenerateTotpCode(
try {
const sqliteClient = await createVaultSqliteClient();
- const totpCodes = sqliteClient.settings.getTotpCodesForItem(message.itemId);
+ const totpCodes = sqliteClient.items.getTotpCodesForItem(message.itemId);
if (totpCodes.length === 0) {
return { success: false, error: 'No TOTP codes found for this item' };
@@ -2368,6 +2291,33 @@ export async function handleGenerateTotpCode(
}
}
+/**
+ * Record one use of an item: when it was last used, and how often.
+ * @param message - The item that was used and what was done with it.
+ */
+export async function handleRecordItemUsage(
+ message: { itemId: string; action: ItemUsageAction }
+): Promise<{ success: boolean }> {
+ try {
+ const encryptionKey = await handleGetEncryptionKey();
+ if (!encryptionKey) {
+ return { success: false };
+ }
+
+ const sqliteClient = await createVaultSqliteClient();
+ if (!sqliteClient.itemStats.recordUsage(message.itemId, message.action)) {
+ // No such item (deleted between use and record); nothing to attribute the use to.
+ return { success: false };
+ }
+
+ await persistLocalVaultMutation(sqliteClient, encryptionKey, VaultDataBucketCategory.Stats);
+ return { success: true };
+ } catch (error) {
+ console.error('Failed to record item usage:', error);
+ return { success: false };
+ }
+}
+
/**
* Set recently selected item for smart autofill.
*/
diff --git a/apps/browser-extension/src/utils/BucketRowGrouping.ts b/apps/browser-extension/src/utils/BucketRowGrouping.ts
deleted file mode 100644
index 597e9e388..000000000
--- a/apps/browser-extension/src/utils/BucketRowGrouping.ts
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Splitting a data bucket's rows across the manifests that own them.
- */
-
-/** A bucket row as read out of SQLite. */
-export type BucketRow = Record<string, unknown>;
-
-/** A bucket category's tables and their rows. */
-export type BucketTables = Record<string, BucketRow[]>;
-
-/**
- * The `ManifestId` a row carries when it names no manifest.
- */
-export const UNSTAMPED_MANIFEST_ID = '00000000-0000-0000-0000-000000000000';
-
-/** True when a `ManifestId` value names no manifest (absent, empty, or the all-zero sentinel). */
-export function isUnstampedManifestId(manifestId: string): boolean {
- return manifestId.length === 0 || manifestId.toLowerCase() === UNSTAMPED_MANIFEST_ID;
-}
-
-/**
- * Find `manifestId` among `candidates` case-insensitively, returning the candidate as it is spelled there.
- * @param candidates - The ids to match against
- * @param manifestId - The id to find
- * @returns The matching candidate, or undefined
- */
-export function matchManifestId(candidates: Iterable<string>, manifestId: string): string | undefined {
- const lower = manifestId.toLowerCase();
- for (const candidate of candidates) {
- if (candidate.toLowerCase() === lower) {
- return candidate;
- }
- }
- return undefined;
-}
-
-/**
- * Look `manifestId` up in `map`, tolerating a casing difference the way {@link matchManifestId} does.
- * @param map - The map to search
- * @param manifestId - The id to find
- * @returns The matching entry's value, or undefined
- */
-export function findByManifestId<T>(map: Map<string, T>, manifestId: string): T | undefined {
- const match = matchManifestId(map.keys(), manifestId);
- return match === undefined ? undefined : map.get(match);
-}
-
-/**
- * Split a bucket category's rows into one table set per owning manifest. Ownership is each row's own
- * `ManifestId` stamp — the writing manifest is no fallback: a row that names no manifest is a bug in
- * whatever wrote it, refused loudly here rather than silently homed somewhere plausible.
- * @param tableNames - The tables making up the category
- * @param tables - The rows read for those tables
- * @param writingManifestId - The manifest this vault writes from; its bucket is always declared, even empty
- * @param alsoInclude - Manifests that must be written even when they hold no rows
- * @returns Manifest id → that manifest's rows, per table
- */
-export function groupBucketRowsByManifest(
- tableNames: string[],
- tables: BucketTables,
- writingManifestId: string,
- alsoInclude: string[] = []
-): Map<string, BucketTables> {
- const grouped = new Map<string, BucketTables>([[writingManifestId, {}]]);
-
- for (const tableName of tableNames) {
- for (const row of tables[tableName] ?? []) {
- const stamp = typeof row.ManifestId === 'string' ? row.ManifestId : '';
- if (isUnstampedManifestId(stamp)) {
- throw new Error(`Bucket table ${tableName} holds a row that names no manifest; every row must carry the manifest it belongs to`);
- }
- const owner = matchManifestId(grouped.keys(), stamp) ?? stamp;
-
- const manifestTables = grouped.get(owner) ?? {};
- manifestTables[tableName] = [...(manifestTables[tableName] ?? []), row];
- grouped.set(owner, manifestTables);
- }
- }
-
- for (const manifestId of alsoInclude) {
- if (!matchManifestId(grouped.keys(), manifestId)) {
- grouped.set(manifestId, {});
- }
- }
-
- /*
- * Every manifest declares every table of the category, empty where it holds nothing. A table left out
- * entirely reads as "unchanged" rather than "emptied", so dropping the last row of one would otherwise
- * never reach the server.
- */
- for (const manifestTables of grouped.values()) {
- for (const tableName of tableNames) {
- manifestTables[tableName] ??= [];
- }
- }
-
- return grouped;
-}
diff --git a/apps/browser-extension/src/utils/RustCore.ts b/apps/browser-extension/src/utils/RustCore.ts
index 45d667525..d68dcba8f 100644
--- a/apps/browser-extension/src/utils/RustCore.ts
+++ b/apps/browser-extension/src/utils/RustCore.ts
@@ -364,13 +364,12 @@ export async function vaultCodecExtractEncryptionKeyForPublicKey(manifest: Codec
}
/**
- * Build a single data bucket for `(manifestId, category)` from its tables (bucket-only push path). Include the
- * `CodecOverflows` table (see {@link vaultCodecOverflowTable}) in `tables` so a newer writer's
- * columns/tables re-merge and survive; it is consumed and never emitted into the bucket.
+ * Build a bucket category's data buckets from its tables, one per manifest in `manifestIds` (bucket-only push
+ * path).
*/
-export async function vaultCodecExtractBucket(manifestId: string, category: string, tables: Record<string, Array<Record<string, unknown>>>): Promise<CodecDataBucket> {
+export async function vaultCodecExtractBuckets(category: string, manifestIds: string[], tables: Record<string, Array<Record<string, unknown>>>): Promise<CodecDataBucket[]> {
await initRustCore();
- return core.vaultCodecExtractBucket({ manifestId, category, tables }) as CodecDataBucket;
+ return core.vaultCodecExtractBuckets({ category, manifestIds, tables }) as CodecDataBucket[];
}
/**
diff --git a/core/rust/src/ffi.rs b/core/rust/src/ffi.rs
index 1fe06653e..e1bc8fd45 100644
--- a/core/rust/src/ffi.rs
+++ b/core/rust/src/ffi.rs
@@ -244,13 +244,14 @@ pub unsafe extern "C" fn vault_codec_materialize_as_sqlite_ffi(input_json: *cons
codec_json_ffi(input_json, "input_json", crate::vault_codec::materialize_as_sqlite_json)
}
-/// Build a single data bucket. Input: `{ manifestId, category, tables }` JSON. Output: `DataBucket` JSON.
+/// Build a bucket category's data buckets, one per manifest. Input: `{ category, manifestIds, tables }` JSON.
+/// Output: `DataBucket[]` JSON.
///
/// # Safety
/// `input_json` must be a valid null-terminated C string; free the result with `free_string`.
#[no_mangle]
-pub unsafe extern "C" fn vault_codec_extract_bucket_ffi(input_json: *const c_char) -> *mut c_char {
- codec_json_ffi(input_json, "input_json", crate::vault_codec::extract_bucket_json)
+pub unsafe extern "C" fn vault_codec_extract_buckets_ffi(input_json: *const c_char) -> *mut c_char {
+ codec_json_ffi(input_json, "input_json", crate::vault_codec::extract_buckets_json)
}
/// Validate a manifest. Input: `Manifest` JSON. Output: `ValidationResult` JSON.
diff --git a/core/rust/src/lib.rs b/core/rust/src/lib.rs
index c7d3fea0c..3de537c5c 100644
--- a/core/rust/src/lib.rs
+++ b/core/rust/src/lib.rs
@@ -32,7 +32,7 @@ pub use vault_merge::{
};
pub use vault_codec::{
compute_ciphertext_hash, compute_content_fingerprint, canonicalize_from_sqlite,
- extract_bucket, generate_manifest_salt, unpack_payload, materialize_as_sqlite, pack_payload,
+ extract_buckets, generate_manifest_salt, unpack_payload, materialize_as_sqlite, pack_payload,
validate_manifest, validate_data_bucket, BlobEntry,
CanonicalizeInput, CanonicalizedManifest, CanonicalizedVault, CodecRecord, CodecTableData, DataBucket, Manifest,
MaterializeInput, MaterializedTables, ManifestSpec, ValidationResult,
diff --git a/core/rust/src/uniffi_api.rs b/core/rust/src/uniffi_api.rs
index e94796d17..22431a62f 100644
--- a/core/rust/src/uniffi_api.rs
+++ b/core/rust/src/uniffi_api.rs
@@ -213,10 +213,11 @@ pub fn vault_codec_materialize_as_sqlite(input_json: String) -> Result<String, V
crate::vault_codec::materialize_as_sqlite_json(&input_json)
}
-/// Build a single data bucket. Input: `{ manifestId, category, tables }` JSON. Output: `DataBucket` JSON.
+/// Build a bucket category's data buckets, one per manifest this vault writes.
+/// Input: `{ category, manifestIds, tables }` JSON. Output: `DataBucket[]` JSON.
#[uniffi::export]
-pub fn vault_codec_extract_bucket(input_json: String) -> Result<String, VaultError> {
- crate::vault_codec::extract_bucket_json(&input_json)
+pub fn vault_codec_extract_buckets(input_json: String) -> Result<String, VaultError> {
+ crate::vault_codec::extract_buckets_json(&input_json)
}
/// The bucket layout: `[{ category, tables: [<name>] }]` JSON. Source of truth for bucket-only sync.
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rs
index 982ba7660..9e456c759 100644
--- a/core/rust/src/vault_codec/canonicalize.rs
+++ b/core/rust/src/vault_codec/canonicalize.rs
@@ -10,7 +10,7 @@
//! `{ "__blobRef": hash, "__blobKind": kind }`;
//! - every other column (including non-blob `{ "__b64" }` inline bytes) is copied verbatim.
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64::Engine;
@@ -22,7 +22,7 @@ use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedManifest, Canon
use super::sharing::{clone_referenced_rows, partition_by_manifest, prune_unreferenced_logos, referenced_tables};
use super::types::{
blob_spec_for, bucket_categories, bucket_category_for, is_bucketed_table, is_skip_table, is_unstamped_scope,
- manifest_scoped_tables, row_identity, tables_for_category, MANIFEST_ID_COL, OVERFLOW_TABLE, SCHEMA_VERSION,
+ manifest_scoped_tables, row_identity, MANIFEST_ID_COL, OVERFLOW_TABLE, SCHEMA_VERSION,
};
use crate::error::VaultResult;
@@ -83,6 +83,8 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
let all_logos = snapshots.get("Logos").unwrap_or(&no_rows);
let other_specs: Vec<ManifestSpec> = input.manifests.iter().skip(1).cloned().collect();
+ let other_ids = other_specs.iter().map(|spec| spec.manifest_id.clone());
+ let manifest_ids: Vec<String> = std::iter::once(writing_manifest_id.clone()).chain(other_ids).collect();
let partitions = partition_by_manifest(&mut all_tables, &other_specs, &snapshots, &writing_manifest_id)?;
// The writing manifest is finished exactly like every partition.
@@ -99,7 +101,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
manifest_tables.insert(name, out_rows);
}
- let data_buckets = build_data_buckets(bucketed_rows, &overflow, &writing_manifest_id, &other_specs);
+ let data_buckets = build_data_buckets(bucketed_rows, &overflow, &manifest_ids);
// Start with the manifest the caller wrote from, which the input lists first.
let mut manifests: Vec<CanonicalizedManifest> = Vec::with_capacity(1 + partitions.len());
@@ -145,72 +147,69 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
}
/// Route every bucketed row into the bucket of the manifest that owns it: `(ManifestId, category)`.
-fn build_data_buckets(
- bucketed_rows: HashMap<String, Vec<CodecRecord>>,
- overflow: &CodecOverflow,
- writing_manifest_id: &str,
- other_specs: &[ManifestSpec],
-) -> Vec<DataBucket> {
- let known: HashSet<&str> = std::iter::once(writing_manifest_id).chain(other_specs.iter().map(|spec| spec.manifest_id.as_str())).collect();
-
- // (manifest id, category) > (table > rows).
- let mut buckets: HashMap<(String, String), HashMap<String, Vec<CodecRecord>>> = HashMap::new();
- for category in bucket_categories() {
- let tables = buckets.entry((writing_manifest_id.to_string(), category.to_string())).or_default();
- for table in tables_for_category(category) {
- if bucketed_rows.contains_key(table) {
- tables.insert(table.to_string(), Vec::new());
- }
- }
+fn build_data_buckets(bucketed_rows: HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow, manifest_ids: &[String]) -> Vec<DataBucket> {
+ let mut data_buckets: Vec<DataBucket> = Vec::new();
+ for category in categories_present(overflow) {
+ let grouped = group_category_rows(category_tables(&category, &bucketed_rows, overflow), manifest_ids);
+ data_buckets.extend(grouped.into_iter().map(|(manifest_id, tables)| DataBucket::new(manifest_id, category.clone(), tables)));
}
+ data_buckets.sort_by(|a, b| (&a.manifest_id, &a.category).cmp(&(&b.manifest_id, &b.category)));
+ data_buckets
+}
- /// The manifest a bucketed row belongs to: the one its stamp names. A stamp naming a manifest this
- /// vault no longer carries drops the row with that manifest (the partition split's `Gone` rule).
- fn owner<'a>(row: &'a CodecRecord, known: &HashSet<&str>) -> Option<&'a str> {
- match row.get(MANIFEST_ID_COL).and_then(|value| value.as_str()) {
- Some(id) if !is_unstamped_scope(Some(id)) => known.contains(id).then_some(id),
- _ => None,
- }
- }
+/// Every bucket category to emit: the declared ones plus any a newer writer put in the overflow (a category
+/// this client's schema does not know yet still has to be carried forward). Sorted, so output stays stable.
+fn categories_present(overflow: &CodecOverflow) -> Vec<String> {
+ let mut categories: Vec<String> = bucket_categories().into_iter().map(str::to_string).collect();
+ let mut extra: Vec<String> = overflow.bucket_tables.keys().filter(|category| !categories.contains(category)).cloned().collect();
+ extra.sort();
+ categories.extend(extra);
+ categories
+}
- for (name, rows) in &bucketed_rows {
- let category = bucket_category_for(name).expect("only bucketed tables are collected into bucketed_rows");
- for row in rows {
- let Some(manifest_id) = owner(row, &known) else { continue };
- buckets
- .entry((manifest_id.to_string(), category.to_string()))
- .or_default()
- .entry(name.clone())
- .or_default()
- .push(row.clone());
+/// The tables read for one bucket category: the ones the local schema holds, plus whole tables a newer writer
+/// put in that bucket which this client's schema cannot hold (carried in the overflow). Local rows win.
+fn category_tables(category: &str, bucketed_rows: &HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow) -> HashMap<String, Vec<CodecRecord>> {
+ let mut tables: HashMap<String, Vec<CodecRecord>> = bucketed_rows
+ .iter()
+ .filter(|(name, _)| bucket_category_for(name) == Some(category))
+ .map(|(name, rows)| (name.clone(), rows.clone()))
+ .collect();
+ if let Some(ov_tables) = overflow.bucket_tables.get(category) {
+ for (name, rows) in ov_tables {
+ tables.entry(name.clone()).or_insert_with(|| rows.clone());
}
}
+ tables
+}
- // Re-emit the overflow's bucket tables: whole tables a newer writer put in a bucket that this client's schema cannot hold.
- for (category, ov_tables) in &overflow.bucket_tables {
- for (name, rows) in ov_tables {
- if bucketed_rows.contains_key(name) {
- continue;
- }
- for row in rows {
- let Some(manifest_id) = owner(row, &known) else { continue };
- buckets
- .entry((manifest_id.to_string(), category.clone()))
- .or_default()
- .entry(name.clone())
- .or_default()
- .push(row.clone());
- }
+/// Split one bucket category's tables into the table set each manifest owns. This is the single grouping
+/// rule behind both write paths: the full push (via [`build_data_buckets`]) and the bucket-only push
+/// (via [`extract_buckets`]).
+fn group_category_rows(tables: HashMap<String, Vec<CodecRecord>>, manifest_ids: &[String]) -> HashMap<String, HashMap<String, Vec<CodecRecord>>> {
+ let empty: HashMap<String, Vec<CodecRecord>> = tables.keys().map(|name| (name.clone(), Vec::new())).collect();
+ let mut grouped: HashMap<String, HashMap<String, Vec<CodecRecord>>> = manifest_ids.iter().map(|id| (id.clone(), empty.clone())).collect();
+
+ for (name, rows) in tables {
+ for mut row in rows {
+ let Some(owner) = owning_manifest(&row, manifest_ids) else { continue };
+ // Normalize the stamp to the spelling the bucket declares the id with.
+ row.insert(MANIFEST_ID_COL.to_string(), json!(owner));
+ grouped.entry(owner).or_default().entry(name.clone()).or_default().push(row);
}
}
- let mut data_buckets: Vec<DataBucket> = buckets
- .into_iter()
- .map(|((manifest_id, category), tables)| DataBucket::new(manifest_id, category, tables))
- .collect();
- // Deterministic bucket order (HashMap iteration is unordered) so canonicalize is reproducible.
- data_buckets.sort_by(|a, b| (&a.manifest_id, &a.category).cmp(&(&b.manifest_id, &b.category)));
- data_buckets
+ grouped
+}
+
+/// The manifest a bucketed row belongs to, spelled the way `manifest_ids` spells it, or `None` when the row
+/// names no manifest or one this vault does not carry.
+fn owning_manifest(row: &CodecRecord, manifest_ids: &[String]) -> Option<String> {
+ let stamp = row.get(MANIFEST_ID_COL).and_then(|value| value.as_str())?;
+ if is_unstamped_scope(Some(stamp)) {
+ return None;
+ }
+ manifest_ids.iter().find(|id| id.eq_ignore_ascii_case(stamp)).cloned()
}
/// For legacy sqlite-blob migration: the manifest that unstamped rows are adopted into.
@@ -237,7 +236,7 @@ fn reject_unstamped_rows(tables: &HashMap<String, Vec<CodecRecord>>) -> VaultRes
if unstamped > 0 {
let first = rows.iter().find(|row| is_unstamped(row)).and_then(|row| row.get("Id")).cloned().unwrap_or(Value::Null);
return Err(crate::error::VaultError::General(format!(
- "canonicalize refuses to write {} row(s) of {} that name no manifest (first: Id {}); every row must carry the manifest it belongs to",
+ "the codec refuses to write {} row(s) of {} that name no manifest (first: Id {}); every row must carry the manifest it belongs to",
unstamped, name, first
)));
}
@@ -327,13 +326,17 @@ fn primary_key_of(identity: &str) -> &str {
identity.rsplit('\u{1f}').next().unwrap_or(identity)
}
-/// Build a single data bucket for `(manifest_id, category)` from its already-normalized tables
-/// (name > rows). The bucket-only push path (a bucket changed but the manifest didn't).
+/// Build `category`'s data buckets from its already-normalized tables (name > rows): the bucket-only push
+/// path, taken when a bucket changed but the manifest did not.
///
-/// Every row must be stamped for the bucket's own manifest: the caller groups rows by their stamps
-/// before calling, so a row naming no manifest or another one is a grouping bug, refused loudly
-/// rather than repaired by re-homing the row into a scope it never claimed.
-pub fn extract_bucket(manifest_id: String, category: String, mut tables: HashMap<String, Vec<CodecRecord>>) -> VaultResult<DataBucket> {
+/// Include the [`OVERFLOW_TABLE`] row in `tables` (read it alongside the category's tables) so a newer
+/// writer's columns/tables re-merge and survive; it is consumed and never emitted into a bucket.
+pub fn extract_buckets(category: String, manifest_ids: Vec<String>, mut tables: HashMap<String, Vec<CodecRecord>>) -> VaultResult<Vec<DataBucket>> {
+ if manifest_ids.is_empty() {
+ let message = "extract_buckets declares no manifests; a bucket write needs the manifests it may be addressed to";
+ return Err(crate::error::VaultError::General(message.to_string()));
+ }
+
let overflow = tables.remove(OVERFLOW_TABLE).map(|records| CodecOverflow::from_table_records(&records)).unwrap_or_default();
remerge_overflow_columns(&mut tables, &overflow);
if let Some(ov_tables) = overflow.bucket_tables.get(&category) {
@@ -341,29 +344,12 @@ pub fn extract_bucket(manifest_id: String, category: String, mut tables: HashMap
tables.entry(name.clone()).or_insert_with(|| rows.clone());
}
}
+ reject_unstamped_rows(&tables)?;
- for (name, rows) in tables.iter_mut() {
- for row in rows.iter_mut() {
- match row.get(MANIFEST_ID_COL).and_then(|value| value.as_str()) {
- Some(id) if !is_unstamped_scope(Some(id)) => {
- if !id.eq_ignore_ascii_case(&manifest_id) {
- return Err(crate::error::VaultError::General(format!(
- "extract_bucket for manifest {} got a {} row stamped for {}; the caller must group rows by the manifest they name",
- manifest_id, name, id
- )));
- }
- // Normalize the stamp to the bucket's declared spelling of the id.
- row.insert(MANIFEST_ID_COL.to_string(), json!(manifest_id));
- }
- _ => {
- return Err(crate::error::VaultError::General(format!(
- "extract_bucket refuses a {} row that names no manifest; every row must carry the manifest it belongs to",
- name
- )));
- }
- }
- }
- }
-
- Ok(DataBucket::new(manifest_id, category, tables))
+ let mut buckets: Vec<DataBucket> = group_category_rows(tables, &manifest_ids)
+ .into_iter()
+ .map(|(manifest_id, tables)| DataBucket::new(manifest_id, category.clone(), tables))
+ .collect();
+ buckets.sort_by(|a, b| a.manifest_id.cmp(&b.manifest_id));
+ Ok(buckets)
}
diff --git a/core/rust/src/vault_codec/manifest.rs b/core/rust/src/vault_codec/manifest.rs
index d21e8242d..9d340f364 100644
--- a/core/rust/src/vault_codec/manifest.rs
+++ b/core/rust/src/vault_codec/manifest.rs
@@ -200,6 +200,18 @@ pub struct CanonicalizeInput {
pub adopt_unstamped_into: Option<String>,
}
+/// Input for [`crate::vault_codec::extract_buckets`].
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ExtractBucketsInput {
+ pub category: String,
+ /// Every manifest this vault can write. Each gets a bucket back; rows naming anything else are dropped.
+ pub manifest_ids: Vec<String>,
+ /// The category's tables as read from the local vault, plus the
+ /// [`OVERFLOW_TABLE`](super::types::OVERFLOW_TABLE) row so a newer writer's columns/tables re-merge.
+ pub tables: HashMap<String, Vec<CodecRecord>>,
+}
+
/// Input for [`crate::vault_codec::materialize_as_sqlite`].
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rs
index 6577d5be3..9dff877b4 100644
--- a/core/rust/src/vault_codec/mod.rs
+++ b/core/rust/src/vault_codec/mod.rs
@@ -17,6 +17,8 @@ mod sharing;
mod types;
mod validate;
+use std::collections::HashMap;
+
use base64::engine::general_purpose::STANDARD as BASE64;
use base64::Engine;
use serde_json::json;
@@ -25,7 +27,7 @@ use crate::error::{VaultError, VaultResult};
pub use manifest::{
BlobEntry, BucketLayoutEntry, CanonicalizeInput, CanonicalizedManifest, CanonicalizedVault, CodecOverflow, DataBucket,
- Manifest, MaterializeInput, MaterializedTables, CodecRecord, CodecTableData, ManifestSpec,
+ ExtractBucketsInput, Manifest, MaterializeInput, MaterializedTables, CodecRecord, CodecTableData, ManifestSpec,
};
pub use scoped_assets::{KIND_BUILTIN as LOGO_KIND_BUILTIN, KIND_CUSTOM as LOGO_KIND_CUSTOM, KIND_FAVICON as LOGO_KIND_FAVICON};
pub use sharing::{active_encryption_key, extract_encryption_key_for_public_key};
@@ -49,11 +51,14 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
materialize::materialize_as_sqlite(input)
}
-/// Build the `(manifest_id, category)` data bucket from its tables (bucket-only push path).
-/// Include the [`OVERFLOW_TABLE`] row in `tables` (read it alongside the bucket's tables) so a newer
-/// writer's columns/tables re-merge and survive; it is consumed and never emitted into the bucket.
-pub fn extract_bucket(manifest_id: String, category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>) -> VaultResult<DataBucket> {
- canonicalize::extract_bucket(manifest_id, category, tables)
+/// Build `category`'s data buckets, one per manifest in `manifest_ids`, from the category's tables as the
+/// platform reads them out of its local vault (bucket-only push path). Rows route by the manifest each one
+/// names, exactly as the full push routes them.
+///
+/// Include the [`OVERFLOW_TABLE`] row in `tables` (read it alongside the category's tables) so a newer
+/// writer's columns/tables re-merge and survive; it is consumed and never emitted into a bucket.
+pub fn extract_buckets(category: String, manifest_ids: Vec<String>, tables: HashMap<String, Vec<CodecRecord>>) -> VaultResult<Vec<DataBucket>> {
+ canonicalize::extract_buckets(category, manifest_ids, tables)
}
/// The bucket layout: every category and the tables it owns, in declaration order.
@@ -205,18 +210,11 @@ pub fn materialize_as_sqlite_json(input_json: &str) -> VaultResult<String> {
Ok(serde_json::to_string(&materialize_as_sqlite(input)?)?)
}
-/// JSON-string sibling of [`extract_bucket`].
-/// Input: `{ "manifestId": <str>, "category": <str>, "tables": { <name>: [rows] } }`.
-pub fn extract_bucket_json(input_json: &str) -> VaultResult<String> {
- #[derive(serde::Deserialize)]
- #[serde(rename_all = "camelCase")]
- struct Input {
- manifest_id: String,
- category: String,
- tables: std::collections::HashMap<String, Vec<CodecRecord>>,
- }
- let input: Input = serde_json::from_str(input_json)?;
- Ok(serde_json::to_string(&extract_bucket(input.manifest_id, input.category, input.tables)?)?)
+/// JSON-string sibling of [`extract_buckets`].
+/// Input: `{ "category": <str>, "manifestIds": [<str>], "tables": { <name>: [rows] } }`.
+pub fn extract_buckets_json(input_json: &str) -> VaultResult<String> {
+ let input: ExtractBucketsInput = serde_json::from_str(input_json)?;
+ Ok(serde_json::to_string(&extract_buckets(input.category, input.manifest_ids, input.tables)?)?)
}
/// JSON-string sibling of [`bucket_layout`]. Output: `[{ "category": <str>, "tables": [<str>] }]`.
diff --git a/core/rust/src/wasm.rs b/core/rust/src/wasm.rs
index 4eaeef80a..c0b259424 100644
--- a/core/rust/src/wasm.rs
+++ b/core/rust/src/wasm.rs
@@ -7,7 +7,7 @@ use crate::credential_matcher::{
};
use crate::password_generator::{available_languages, generate_password};
use crate::vault_codec::{
- self, CanonicalizeInput, CodecRecord, DataBucket, Manifest, MaterializeInput,
+ self, CanonicalizeInput, DataBucket, ExtractBucketsInput, Manifest, MaterializeInput,
};
use crate::vault_merge::{merge_vaults, MergeInput, MergeOutput};
use crate::vault_pruner::{prune_vault, PruneInput, PruneOutput};
@@ -164,21 +164,15 @@ pub fn vault_codec_materialize_as_sqlite_js(input: JsValue) -> Result<JsValue, J
codec_to_js(&output)
}
-/// Build a single data bucket. Input: `{ manifestId, category, tables: { <name>: [rows] } }`.
-#[wasm_bindgen(js_name = vaultCodecExtractBucket)]
-pub fn vault_codec_extract_bucket_js(input: JsValue) -> Result<JsValue, JsValue> {
- #[derive(serde::Deserialize)]
- #[serde(rename_all = "camelCase")]
- struct Input {
- manifest_id: String,
- category: String,
- tables: std::collections::HashMap<String, Vec<CodecRecord>>,
- }
- let input: Input = serde_wasm_bindgen::from_value(input)
- .map_err(|e| JsValue::from_str(&format!("Failed to parse extract-bucket input: {}", e)))?;
- let bucket = vault_codec::extract_bucket(input.manifest_id, input.category, input.tables)
- .map_err(|e| JsValue::from_str(&format!("extract_bucket failed: {}", e)))?;
- codec_to_js(&bucket)
+/// Build a bucket category's data buckets, one per manifest this vault writes: rows route by the manifest
+/// each one names. Input: `{ category, manifestIds, tables: { <name>: [rows] } }`. Output: `DataBucket[]`.
+#[wasm_bindgen(js_name = vaultCodecExtractBuckets)]
+pub fn vault_codec_extract_buckets_js(input: JsValue) -> Result<JsValue, JsValue> {
+ let input: ExtractBucketsInput = serde_wasm_bindgen::from_value(input)
+ .map_err(|e| JsValue::from_str(&format!("Failed to parse extract-buckets input: {}", e)))?;
+ let buckets = vault_codec::extract_buckets(input.category, input.manifest_ids, input.tables)
+ .map_err(|e| JsValue::from_str(&format!("extract_buckets failed: {}", e)))?;
+ codec_to_js(&buckets)
}
/// The bucket layout: `[{ category, tables: [<name>] }]`. Source of truth for platform bucket-only sync.