Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Improve browser extension sqlite local storage efficiency (#2159)

Leendert de Borstbb26c8298b5c
diff --git a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
index 7a0322c65..a15a8e236 100644
--- a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
+++ b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
@@ -4,7 +4,7 @@ import { storage } from 'wxt/utils/storage';
import { clearAllSavePromptState } from '@/entrypoints/background/SavePromptStateHandler';
import { handleClearTwoFactorState } from '@/entrypoints/background/TwoFactorStateHandler';
-import { base64ToBytes } from '@/utils/Base64';
+import { base64ToBytes, bytesToBase64 } from '@/utils/Base64';
import { AUTH_STORAGE_KEYS, dirtyScopeStorageKey, SESSION_STORAGE_KEYS, StorageKeys, vaultDataStorageKeys, VAULT_LOCK_STORAGE_KEYS } from '@/utils/constants/storageKeys';
import { TRASH_RETENTION_DAYS } from '@/utils/constants/vault';
import type { ItemUsageAction } from '@/utils/db';
@@ -46,12 +46,13 @@ import type { StringResponse as stringResponse } from '@/utils/types/messaging/S
import type { VaultResponse as messageVaultResponse } from '@/utils/types/messaging/VaultResponse';
import type { VaultUploadResponse as messageVaultUploadResponse } from '@/utils/types/messaging/VaultUploadResponse';
import { type VaultMutationScope, DEFAULT_VAULT_MUTATION_SCOPE, hasUserVisibleScope, isManifestScope } from '@/utils/types/VaultMutationScope';
+import { decryptVaultBlob, encryptVaultBlob } from '@/utils/VaultBlob';
import { VaultCodec } from '@/utils/VaultCodec';
import { clearDirtyScopes, getDirtyScopes } from '@/utils/VaultDirtyState';
import { VaultKeyService } from '@/utils/VaultKeyService';
import { vaultRequiresManifestMigration, VaultMigrationKind, type VaultMigrationStatus } from '@/utils/VaultManifestMigration';
import { vaultMergeService } from '@/utils/VaultMergeService';
-import { type PullAndMergeResult, vaultSyncService, invalidateCanonicalizeCache } from '@/utils/VaultSyncService';
+import { type PullAndMergeResult, vaultSyncService, invalidateCanonicalizeCache, primeCanonicalizeCache } from '@/utils/VaultSyncService';
import { WebApiService } from '@/utils/WebApiService';
import { t } from '@/i18n/StandaloneI18n';
@@ -369,10 +370,8 @@ export async function handleGetVault(
return { success: false, error: formatErrorWithCode(await t('common.errors.vaultIsLocked'), AppErrorCode.VAULT_LOCKED) };
}
- const decryptedVault = await EncryptionUtility.symmetricDecrypt(
- encryptedVault,
- encryptionKey
- );
+ // The popup receives the database as base64: messages carry strings, not bytes.
+ const decryptedVault = bytesToBase64(await decryptVaultBlob(encryptedVault, encryptionKey));
return {
success: true,
@@ -995,7 +994,7 @@ async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { for
*/
const currentKey = await handleGetEncryptionKey() ?? encryptionKey;
if (vaultPruned || currentKey !== encryptionKey) {
- const reEncrypted = await EncryptionUtility.symmetricEncrypt(sqliteClient.exportToBase64(), currentKey);
+ const reEncrypted = await encryptVaultBlob(sqliteClient.exportToBytes(), currentKey);
await storage.setItem(StorageKeys.ENCRYPTED_VAULT, reEncrypted);
cachedSqliteClient = sqliteClient;
cachedVaultBlob = reEncrypted;
@@ -1014,8 +1013,7 @@ async function uploadNewVaultToServer(sqliteClient: SqliteClient, options: { for
* that data bucket instead of the whole manifest. Defaults to a full manifest push.
*/
async function persistLocalVaultMutation(sqliteClient: SqliteClient, encryptionKey: string, scope?: VaultMutationScope) : Promise<void> {
- const updatedVaultData = sqliteClient.exportToBase64();
- const encryptedVault = await EncryptionUtility.symmetricEncrypt(updatedVaultData, encryptionKey);
+ const encryptedVault = await encryptVaultBlob(sqliteClient.exportToBytes(), encryptionKey);
await handleStoreEncryptedVault({ vaultBlob: encryptedVault, markDirty: true, scope });
/*
@@ -1073,14 +1071,11 @@ export async function createVaultSqliteClient() : Promise<SqliteClient> {
}
// Decrypt the vault
- const decryptedVault = await EncryptionUtility.symmetricDecrypt(
- encryptedVault,
- encryptionKey
- );
+ const decryptedVault = await decryptVaultBlob(encryptedVault, encryptionKey);
// Initialize the SQLite client with the decrypted vault
const sqliteClient = new SqliteClient();
- await sqliteClient.initializeFromBase64(decryptedVault);
+ await sqliteClient.initializeFromBytes(decryptedVault);
// Cache the client and vault blob
cachedSqliteClient = sqliteClient;
@@ -1233,8 +1228,8 @@ export async function handleMigrateVaultManifest(): Promise<VaultManifestMigrati
// Step 1: local migration. Store it dirty so the vault is usable immediately, with or without a server.
if (needsSchemaMigration) {
- const migratedBase64 = await vaultSyncService.migrateVaultToCurrentSchema(sqliteClient);
- const migratedEncrypted = await EncryptionUtility.symmetricEncrypt(migratedBase64, encryptionKey);
+ const migratedBytes = await vaultSyncService.migrateVaultToCurrentSchema(sqliteClient);
+ const migratedEncrypted = await encryptVaultBlob(migratedBytes, encryptionKey);
await handleStoreEncryptedVault({ vaultBlob: migratedEncrypted, markDirty: true });
// Invalidate any local cached results that are now stale.
@@ -1380,8 +1375,8 @@ async function adoptRemoteVaultKeyIfNeeded(): Promise<boolean> {
// Re-encrypt the locally persisted vault with the VEK before swapping the session key.
if (encryptedVault) {
- const decrypted = await EncryptionUtility.symmetricDecrypt(encryptedVault, sessionKey);
- await storage.setItem(StorageKeys.ENCRYPTED_VAULT, await EncryptionUtility.symmetricEncrypt(decrypted, vek));
+ const decrypted = await decryptVaultBlob(encryptedVault, sessionKey);
+ await storage.setItem(StorageKeys.ENCRYPTED_VAULT, await encryptVaultBlob(decrypted, vek));
}
await resealSharedManifestRecords(vek);
@@ -1780,7 +1775,7 @@ async function applyServerDirectedChanges(webApi: WebApiService, statusResponse:
}
// A rotated delivery key or a re-rendered shared manifest is a normal local change, so it rides out on this very sync.
- const reconciledVault = await EncryptionUtility.symmetricEncrypt(sqliteClient.exportToBase64(), encryptionKey);
+ const reconciledVault = await encryptVaultBlob(sqliteClient.exportToBytes(), encryptionKey);
const stored = await handleStoreEncryptedVault({ vaultBlob: reconciledVault, markDirty: true });
syncState.isDirty = true;
syncState.mutationSequence = stored.mutationSequence;
@@ -1848,7 +1843,7 @@ async function choosePullStrategy(syncState: VaultSyncState, encryptionKey: stri
// Create a fresh client decrypted from the stored blob for the merge input.
const localClient = new SqliteClient();
try {
- await localClient.initializeFromBase64(await EncryptionUtility.symmetricDecrypt(localEncryptedVault, encryptionKey));
+ await localClient.initializeFromBytes(await decryptVaultBlob(localEncryptedVault, encryptionKey));
/*
* Both checks matter: requiresSchemaMigration deliberately answers false for a vault still on the
@@ -1955,6 +1950,12 @@ async function canonicalPullAndMerge(localClient: SqliteClient, syncState: Vault
return await materializedVaultResult();
}
+ /**
+ * The blob just stored is this merge's own output, so the push is handed the database and the canonical form
+ * the merge already produced.
+ */
+ await adoptMergedVault(result, storeResult.mutationSequence);
+
const uploadResponse = await handleUploadVault();
if (uploadResponse.success && uploadResponse.status === 0) {
await handleMarkVaultClean({ mutationSeqAtStart: uploadResponse.mutationSeqAtStart! });
@@ -1970,6 +1971,32 @@ async function canonicalPullAndMerge(localClient: SqliteClient, syncState: Vault
return await materializedVaultResult();
}
+/**
+ * Adopt a just-stored merge output as this context's live vault: its SQLite becomes the cached client, and the
+ * canonical form the merge produced is handed to the push that follows.
+ *
+ * @param result - the merge whose vault was just stored
+ * @param mutationSequence - the mutation sequence the stored vault is at
+ */
+async function adoptMergedVault(result: Extract<PullAndMergeResult, { kind: 'merged' }>, mutationSequence: number): Promise<void> {
+ try {
+ const mergedClient = new SqliteClient();
+ await mergedClient.initializeFromBytes(result.mergedSqliteBytes);
+
+ // The store above cleared the cache without closing what was there, so there is nothing to close here.
+ cachedSqliteClient = mergedClient;
+ cachedVaultBlob = result.response.vault.blob;
+
+ if (result.pushCanonical) {
+ primeCanonicalizeCache(mergedClient, mutationSequence, result.pushCanonical);
+ } else {
+ devLog('[VaultSync] Merge output cannot stand in for the push canonicalize; the push derives it from the merged vault itself.');
+ }
+ } catch (error) {
+ devWarn('[VaultSync] Could not adopt the merged vault as the live one; the push reads it back from storage.', error);
+ }
+}
+
/**
* LEGACY: pull that merges the local vault onto the server's at the SQLite statement level and uploads the
* result. A merge that cannot run or fails degrades to taking the server vault as-is.
@@ -1987,8 +2014,8 @@ async function legacyStatementPullAndMerge(syncState: VaultSyncState, encryption
return await adoptServerVault(vaultResponse, syncState);
}
- const localDecrypted = await EncryptionUtility.symmetricDecrypt(localEncryptedVault, encryptionKey);
- const serverDecrypted = await EncryptionUtility.symmetricDecrypt(vaultResponse.vault.blob, encryptionKey);
+ const localDecrypted = bytesToBase64(await decryptVaultBlob(localEncryptedVault, encryptionKey));
+ const serverDecrypted = bytesToBase64(await decryptVaultBlob(vaultResponse.vault.blob, encryptionKey));
/*
* Guard for the sqlite-blob to manifest-v1 migration window: this device has already migrated its vault onto the current
@@ -2010,7 +2037,7 @@ async function legacyStatementPullAndMerge(syncState: VaultSyncState, encryption
devLog('[VaultSync] Vault merge during sync completed:', mergeResult.stats);
// Store merged vault. Use expectedMutationSeq to detect if a local mutation happened during merge.
- const mergedEncryptedVault = await EncryptionUtility.symmetricEncrypt(mergeResult.mergedVaultBase64, encryptionKey);
+ const mergedEncryptedVault = await encryptVaultBlob(base64ToBytes(mergeResult.mergedVaultBase64), encryptionKey);
const storeResult = await handleStoreEncryptedVault({ vaultBlob: mergedEncryptedVault, expectedMutationSeq: syncState.mutationSequence });
if (!storeResult.success) {
devLog('[VaultSync] Mutation detected during merge, re-syncing...');
diff --git a/apps/browser-extension/src/entrypoints/popup/context/DbContext.tsx b/apps/browser-extension/src/entrypoints/popup/context/DbContext.tsx
index 5862a2a1a..71d509050 100644
--- a/apps/browser-extension/src/entrypoints/popup/context/DbContext.tsx
+++ b/apps/browser-extension/src/entrypoints/popup/context/DbContext.tsx
@@ -71,7 +71,7 @@ type DbContextType = {
/**
* Load a decrypted vault into memory (SQLite client).
*/
- loadDatabase: (decryptedVaultBase64: string) => Promise<SqliteClient>;
+ loadDatabase: (sqliteBytes: Uint8Array) => Promise<SqliteClient>;
/**
* Load the stored (encrypted) vault from background storage into memory.
* Returns the SqliteClient if vault was loaded successfully, null otherwise.
@@ -232,9 +232,9 @@ export const DbProvider: React.FC<{ children: React.ReactNode }> = ({ children }
/**
* Load a decrypted vault into memory (SQLite client).
*/
- const loadDatabase = useCallback(async (decryptedVaultBase64: string) => {
+ const loadDatabase = useCallback(async (sqliteBytes: Uint8Array) => {
const client = new SqliteClient();
- await client.initializeFromBase64(decryptedVaultBase64);
+ await client.initializeFromBytes(sqliteBytes);
setSqliteClient(client);
setDbInitialized(true);
diff --git a/apps/browser-extension/src/entrypoints/popup/hooks/useVaultMutate.ts b/apps/browser-extension/src/entrypoints/popup/hooks/useVaultMutate.ts
index 6ca732db9..94a37c477 100644
--- a/apps/browser-extension/src/entrypoints/popup/hooks/useVaultMutate.ts
+++ b/apps/browser-extension/src/entrypoints/popup/hooks/useVaultMutate.ts
@@ -3,9 +3,9 @@ import { useCallback, useRef } from 'react';
import { useDb } from '@/entrypoints/popup/context/DbContext';
import { devLog } from '@/utils/devLogger/DevLogger';
-import { EncryptionUtility } from '@/utils/EncryptionUtility';
import { sendMessage } from '@/utils/messaging/ExtensionMessaging';
import { isSilentScope, type VaultMutationOptions, type VaultMutationScope } from '@/utils/types/VaultMutationScope';
+import { encryptVaultBlob } from '@/utils/VaultBlob';
/**
* Hook to execute a vault mutation.
@@ -38,12 +38,8 @@ export function useVaultMutate(): {
await operation();
// Export and encrypt the updated vault
- const base64Vault = dbContext.sqliteClient!.exportToBase64();
const encryptionKey = await sendMessage('GET_ENCRYPTION_KEY') as string;
- const encryptedVaultBlob = await EncryptionUtility.symmetricEncrypt(
- base64Vault,
- encryptionKey
- );
+ const encryptedVaultBlob = await encryptVaultBlob(dbContext.sqliteClient!.exportToBytes(), encryptionKey);
/*
* Store the updated vault locally, mark dirty, increment mutation sequence. The scope records what
diff --git a/apps/browser-extension/src/entrypoints/popup/pages/auth/Login.tsx b/apps/browser-extension/src/entrypoints/popup/pages/auth/Login.tsx
index 828560f8b..03f4988b3 100644
--- a/apps/browser-extension/src/entrypoints/popup/pages/auth/Login.tsx
+++ b/apps/browser-extension/src/entrypoints/popup/pages/auth/Login.tsx
@@ -20,7 +20,6 @@ import { AppInfo } from '@/utils/AppInfo';
import { SrpAuthService } from '@/utils/auth/SrpAuthService';
import { StorageKeys } from '@/utils/constants/storageKeys';
import type { VaultResponse, LoginResponse } from '@/utils/dist/core/models/webapi';
-import { EncryptionUtility } from '@/utils/EncryptionUtility';
import { sendMessage } from '@/utils/messaging/ExtensionMessaging';
import { ApiAuthError } from '@/utils/types/errors/ApiAuthError';
import { hasErrorCode, getErrorMessage } from '@/utils/types/errors/AppErrorCodes';
@@ -28,6 +27,7 @@ import { ClientUpgradeRequiredError } from '@/utils/types/errors/ClientUpgradeRe
import { ServerUpdateRequiredError } from '@/utils/types/errors/ServerUpdateRequiredError';
import { VaultProcessingError } from '@/utils/types/errors/VaultProcessingError';
import type { MobileLoginResult } from '@/utils/types/messaging/MobileLoginResult';
+import { decryptVaultBlob } from '@/utils/VaultBlob';
import { VaultKeyService } from '@/utils/VaultKeyService';
import { vaultSyncService } from '@/utils/VaultSyncService';
@@ -83,7 +83,7 @@ const Login: React.FC = () => {
});
// Decrypt and load the vault into memory
- await dbContext.loadDatabase(await EncryptionUtility.symmetricDecrypt(vaultResponse.vault.blob, encryptionKey));
+ await dbContext.loadDatabase(await decryptVaultBlob(vaultResponse.vault.blob, encryptionKey));
};
/**
diff --git a/apps/browser-extension/src/utils/SqliteClient.ts b/apps/browser-extension/src/utils/SqliteClient.ts
index 03fa1ac15..f8c13d4ee 100644
--- a/apps/browser-extension/src/utils/SqliteClient.ts
+++ b/apps/browser-extension/src/utils/SqliteClient.ts
@@ -170,9 +170,15 @@ export class SqliteClient implements IDatabaseClient {
* @param base64String - Base64 encoded SQLite database
*/
public async initializeFromBase64(base64String: string): Promise<void> {
- try {
- const bytes = base64ToBytes(base64String);
+ return this.initializeFromBytes(base64ToBytes(base64String));
+ }
+ /**
+ * Initialize the SQLite database from the raw database bytes.
+ * @param bytes - the SQLite database
+ */
+ public async initializeFromBytes(bytes: Uint8Array): Promise<void> {
+ try {
// Initialize SQL.js with the WASM file
const SQL = await initSqlJs({
/**
@@ -269,13 +275,21 @@ export class SqliteClient implements IDatabaseClient {
* @returns Base64 encoded string of the database
*/
public exportToBase64(): string {
+ return bytesToBase64(this.exportToBytes());
+ }
+
+ /**
+ * Export the SQLite database as raw bytes.
+ * @returns The database bytes
+ */
+ public exportToBytes(): Uint8Array {
if (!this.db) {
throw new Error('Database not initialized');
}
try {
this.vacuumIfFragmented();
- return bytesToBase64(this.db.export());
+ return this.db.export();
} catch (error) {
console.error('Error exporting SQLite database:', error);
throw error;
diff --git a/apps/browser-extension/src/utils/VaultBlob.ts b/apps/browser-extension/src/utils/VaultBlob.ts
new file mode 100644
index 000000000..8ef99a2d9
--- /dev/null
+++ b/apps/browser-extension/src/utils/VaultBlob.ts
@@ -0,0 +1,50 @@
+/**
+ * The locally stored vault blob: AES-GCM over the raw bytes of the SQLite database.
+ */
+
+import { base64ToBytes } from '@/utils/Base64';
+import { EncryptionUtility } from '@/utils/EncryptionUtility';
+
+/**
+ * The bytes every SQLite database file begins with. They tell the two plaintext shapes apart with no version
+ * marker: base64 text cannot start with them, since the space at index 6 is not in the base64 alphabet.
+ */
+const SQLITE_HEADER = new TextEncoder().encode('SQLite format 3\u0000');
+
+/**
+ * Encrypt a plaintext SQLite database for local storage.
+ * @param sqliteBytes - the database as exported by sql.js
+ * @param key - the symmetric key the vault is stored under
+ * @returns Base64 of (IV | ciphertext | tag).
+ */
+export async function encryptVaultBlob(sqliteBytes: Uint8Array, key: string): Promise<string> {
+ return EncryptionUtility.symmetricEncryptBytes(sqliteBytes, key);
+}
+
+/**
+ * Decrypt a stored vault blob into the plaintext SQLite database.
+ *
+ * Reads both shapes: the database bytes this build writes, and the base64 text older builds wrote (which is
+ * also what a server still on the sqlite-blob storage format serves). A vault in the older shape is converted
+ * by the next write, so nothing has to migrate it up front. TODO: the base64 fallback can be removed
+ * once all users have migrated to 0.31.0+.
+ * @param encryptedBlob - base64 of (IV | ciphertext | tag) as stored
+ * @param key - the symmetric key the vault is stored under
+ */
+export async function decryptVaultBlob(encryptedBlob: string, key: string): Promise<Uint8Array> {
+ const plaintext = await EncryptionUtility.symmetricDecryptBytes(base64ToBytes(encryptedBlob), key);
+ if (isSqliteDatabase(plaintext)) {
+ return plaintext;
+ }
+
+ // LEGACY: the plaintext is base64 text of the database.
+ return base64ToBytes(new TextDecoder().decode(plaintext));
+}
+
+/**
+ * Whether these plaintext bytes are a SQLite database rather than base64 text of one.
+ * @param bytes - the decrypted plaintext
+ */
+function isSqliteDatabase(bytes: Uint8Array): boolean {
+ return bytes.length >= SQLITE_HEADER.length && SQLITE_HEADER.every((byte, index) => bytes[index] === byte);
+}
diff --git a/apps/browser-extension/src/utils/VaultCodec.ts b/apps/browser-extension/src/utils/VaultCodec.ts
index cd2c5073f..73932ce3f 100644
--- a/apps/browser-extension/src/utils/VaultCodec.ts
+++ b/apps/browser-extension/src/utils/VaultCodec.ts
@@ -3,6 +3,7 @@
*/
import { base64ToBytes as decodeBase64, bytesToBase64 as encodeBase64 } from '@/utils/Base64';
+import { devLog } from '@/utils/devLogger/DevLogger';
import type {
CodecManifest,
@@ -42,6 +43,11 @@ type BlobRef = {
*/
export const UNSTAMPED_MANIFEST_ID = '00000000-0000-0000-0000-000000000000';
+/**
+ * Column sets already derived from a schema, keyed by the schema SQL that produced them.
+ */
+const schemaColumnsCache = new Map<string, Promise<Record<string, string[]>>>();
+
/**
* Whether a stamp names no manifest, so it must never be mistaken for one the vault holds rows for.
* @param manifestId - the stamp as the row carries it
@@ -124,6 +130,24 @@ export class VaultCodec {
* @param schemaSql - the COMPLETE_SCHEMA_SQL string for the target client version
*/
public static async getSchemaColumns(schemaSql: string): Promise<Record<string, string[]>> {
+ let columns = schemaColumnsCache.get(schemaSql);
+ if (!columns) {
+ columns = this.readSchemaColumns(schemaSql).catch((error: unknown) => {
+ // A failed probe must not stay cached, or every later call inherits the failure.
+ schemaColumnsCache.delete(schemaSql);
+ throw error;
+ });
+ schemaColumnsCache.set(schemaSql, columns);
+ }
+
+ return Object.fromEntries(Object.entries(await columns).map(([table, cols]) => [table, [...cols]]));
+ }
+
+ /**
+ * Derive the column set of a schema by applying it to a throwaway database.
+ * @param schemaSql - the COMPLETE_SCHEMA_SQL string for the target client version
+ */
+ private static async readSchemaColumns(schemaSql: string): Promise<Record<string, string[]>> {
const db = await this.createDatabase();
try {
db.run(schemaSql);
@@ -139,19 +163,19 @@ export class VaultCodec {
}
/**
- * Insert the materialized tables into a fresh SQLite database and export it (base64).
+ * Insert the materialized tables into a fresh SQLite database and export it.
* @param materialized - tables produced by the Rust `materialize_as_sqlite`
* @param blobs - map of `hash -> plaintext bytes` (caller fetched + decrypted these)
* @param schemaSql - the COMPLETE_SCHEMA_SQL string for the target client version
- * @returns A base64-encoded SQLite database identical (in row content) to the original.
+ * @returns The SQLite database bytes, identical (in row content) to the original.
*/
- public static async insertTables(materialized: CodecMaterialized, blobs: Map<string, Uint8Array>, schemaSql: string): Promise<string> {
+ public static async insertTables(materialized: CodecMaterialized, blobs: Map<string, Uint8Array>, schemaSql: string): Promise<Uint8Array> {
const db = await this.createDatabase();
try {
// 1) Apply the schema.
db.run(schemaSql);
- console.info('[VaultCodec] Schema applied.');
+ devLog('[VaultCodec] Schema applied.');
/*
* SQLite enforces foreign keys immediately (no deferral), and tables are inserted in the order Rust
@@ -179,7 +203,7 @@ export class VaultCodec {
continue;
}
- console.info(`[VaultCodec] Inserting ${rows.length} rows into "${tableName}"...`);
+ devLog(`[VaultCodec] Inserting ${rows.length} rows into "${tableName}"...`);
for (const row of rows) {
const cols = Object.keys(row);
const values: unknown[] = cols.map(c => {
@@ -224,12 +248,12 @@ export class VaultCodec {
const sample = fkViolations[0].values.slice(0, 5).map(v => `${v[0]} row ${v[1]} → missing parent in ${v[2]}`).join('; ');
throw new Error(`VaultCodec: materialized database fails foreign key check (${fkViolations[0].values.length} violations): ${sample}`);
}
- console.info('[VaultCodec] Foreign key check passed.');
+ devLog('[VaultCodec] Foreign key check passed.');
// 4) Export.
const bytes = db.export();
- console.info(`[VaultCodec] Reassembly complete: exported SQLite of ${bytes.length} bytes.`);
- return encodeBase64(bytes);
+ devLog(`[VaultCodec] Reassembly complete: exported SQLite of ${bytes.length} bytes.`);
+ return bytes;
} finally {
db.close();
}
diff --git a/apps/browser-extension/src/utils/VaultSyncService.ts b/apps/browser-extension/src/utils/VaultSyncService.ts
index b9879ebae..12e13cc68 100644
--- a/apps/browser-extension/src/utils/VaultSyncService.ts
+++ b/apps/browser-extension/src/utils/VaultSyncService.ts
@@ -6,7 +6,7 @@
import { storage } from 'wxt/utils/storage';
-import { base64ToBytes } from '@/utils/Base64';
+import { base64ToBytes, bytesToBase64 } from '@/utils/Base64';
import { bucketRevisionKey, StorageKeys } from '@/utils/constants/storageKeys';
import { devError, devLog, devWarn } from '@/utils/devLogger/DevLogger';
import { ManifestKeyType, VaultKeyAlgorithm, type BucketRevision, type VaultResponse } from '@/utils/dist/core/models/webapi';
@@ -21,6 +21,7 @@ import { SharingService, type ManifestVekGrant, type SharedManifestRecord } from
import { SqliteClient } from '@/utils/SqliteClient';
import { getStorageItem } from '@/utils/StorageUtility';
import { VaultProcessingError } from '@/utils/types/errors/VaultProcessingError';
+import { encryptVaultBlob } from '@/utils/VaultBlob';
import { type VaultManifest, type VaultDataBucket, VaultCodec, manifestIdKey } from '@/utils/VaultCodec';
import { VaultKeyService } from '@/utils/VaultKeyService';
import { WebApiService } from '@/utils/WebApiService';
@@ -157,7 +158,7 @@ function batchByTransferCost<T>(items: T[], costOf: (item: T) => number): T[][]
* Result of a pull: the materialized SQLite plus its manifest revision.
*/
type PullResult = {
- sqliteBase64: string;
+ sqliteBytes: Uint8Array;
manifestRevision: number;
};
@@ -197,6 +198,8 @@ export type PullAndMergeResult =
fallbackManifestIds: string[];
droppedLocalManifestIds: string[];
commitRevisions: () => Promise<void>;
+ mergedSqliteBytes: Uint8Array;
+ pushCanonical: CanonicalizedVaultSet | null;
}
| { kind: 'server-only'; response: VaultResponse; commitRevisions: () => Promise<void> };
@@ -339,7 +342,7 @@ type PushManifest = {
* purpose: the codec treats every manifest alike and the push drives one loop over all of them. `isPersonal` marks the
* few places where the user's own manifest genuinely differs (see {@link resolveManifestRecords}).
*/
-type ManifestRecord = {
+export type ManifestRecord = {
manifestId: string;
isPersonal: boolean;
/** Salt this manifest's blob hashes are derived with. */
@@ -356,7 +359,7 @@ type ManifestRecord = {
* The canonicalized vault plus the manifest records it was split against, personal manifest first (the order canonicalize
* requires: the first spec is the manifest being written from).
*/
-type CanonicalizedVaultSet = {
+export type CanonicalizedVaultSet = {
canonicalized: CodecCanonicalized;
manifestRecords: ManifestRecord[];
};
@@ -370,6 +373,16 @@ export function invalidateCanonicalizeCache(): void {
canonicalizeCache = null;
}
+/**
+ * Hand the push a canonicalize result to prevent double-canonicalization where possible.
+ * @param client - the client the result describes; reused for this instance only
+ * @param mutationSequence - the sequence the vault is at; a mutation past it invalidates the result
+ * @param set - the canonicalized vault
+ */
+export function primeCanonicalizeCache(client: SqliteClient, mutationSequence: number, set: CanonicalizedVaultSet): void {
+ canonicalizeCache = { client, mutationSequence, ...set };
+}
+
/**
* Service entry point.
*/
@@ -412,8 +425,8 @@ export class VaultSyncService {
// Manifest-v1 user: materialize the manifest + metadata + blobs into a SQLite blob, then encrypt it.
devLog('[V2Pull] Step 2/4: manifest format: decrypting and reassembling local SQLite...');
const pull = await this.materializeFromSnapshot(snapshot, encryptionKey);
- devLog(`[V2Pull] Step 3/4: materialized SQLite (${pull.sqliteBase64.length} base64 chars); re-encrypting for local storage...`);
- const encryptedVault = await EncryptionUtility.symmetricEncrypt(pull.sqliteBase64, encryptionKey);
+ devLog(`[V2Pull] Step 3/4: materialized SQLite (${pull.sqliteBytes.length} bytes); re-encrypting for local storage...`);
+ const encryptedVault = await encryptVaultBlob(pull.sqliteBytes, encryptionKey);
devLog('[V2Pull] Step 4/4: re-encryption done, returning VaultResponse.');
return this.buildResponse(encryptedVault, '2.0.0', pull.manifestRevision, snapshot);
} catch (error) {
@@ -430,7 +443,12 @@ export class VaultSyncService {
*/
public async pullAndMerge(encryptionKey: string, localClient: SqliteClient): Promise<PullAndMergeResult> {
devLog('[V2Merge] Fetching vault snapshot for canonical merge (GET /v2/Vault)...');
- const snapshot = await this.fetchSnapshot();
+
+ const snapshotPromise = this.fetchSnapshot();
+ const localCanonicalized = this.canonicalizeVault(localClient);
+ void localCanonicalized.catch(() => null);
+
+ const snapshot = await snapshotPromise;
this.lastServedManifestIds = (snapshot.manifests ?? []).map(m => m.manifestId);
// LEGACY: a server still on the sqlite-blob format cannot merge with a manifest-v1 vault; the caller pushes over it.
@@ -443,12 +461,13 @@ export class VaultSyncService {
const opened = await this.openManifestsAndAdoptSyncState(snapshot, encryptionKey, { deferRevisionCommit: true });
try {
- return await this.mergeOntoOpenedManifests(opened, snapshot, encryptionKey, localClient);
+ const localSide = await this.localSideForMerge(localCanonicalized, localClient, opened);
+ return await this.mergeOntoOpenedManifests(opened, snapshot, encryptionKey, localSide);
} catch (mergeError) {
// Today's merge-failure fallback, from the same snapshot: the server vault stands, local changes are dropped.
devError('[V2Merge] Canonical merge failed, falling back to the server vault:', mergeError);
- const sqliteBase64 = await this.materializeToSqlite(opened.resolved.map(m => m.manifest), opened.dataBuckets, opened.blobMap);
- const encryptedVault = await EncryptionUtility.symmetricEncrypt(sqliteBase64, encryptionKey);
+ const sqliteBytes = await this.materializeToSqlite(opened.resolved.map(m => m.manifest), opened.dataBuckets, opened.blobMap);
+ const encryptedVault = await encryptVaultBlob(sqliteBytes, encryptionKey);
return { kind: 'server-only', response: this.buildResponse(encryptedVault, '2.0.0', opened.personalRevision, snapshot), commitRevisions: opened.commitManifestRevisions };
}
} catch (error) {
@@ -458,15 +477,55 @@ export class VaultSyncService {
}
/**
- * The merge core of {@link pullAndMerge}: canonicalize the local vault, run the Rust canonical merge
- * against the manifests just opened, validate the result per manifest, and materialize it.
+ * The local side of the merge.
+ * @param parallelRun - the canonicalize started before the fetch
+ * @param localClient - the local vault, for the re-run
+ * @param opened - the opened server manifests, which decide whether the parallel result still holds
+ */
+ private async localSideForMerge(parallelRun: Promise<CanonicalizedVaultSet>, localClient: SqliteClient, opened: OpenedManifestSet): Promise<CanonicalizedVaultSet> {
+ const set = await parallelRun;
+ const moved = this.groundMovedUnderCanonicalize(set.manifestRecords, opened);
+ if (moved === null) {
+ return set;
+ }
+
+ devWarn(`[V2Merge] This snapshot re-minted ${moved}; canonicalizing the local vault again so its blob hashes match the server's.`);
+ return this.canonicalizeVault(localClient);
+ }
+
+ /**
+ * What this snapshot changed under a canonicalize that ran before it was opened.
+ * @param records - the manifest records the parallel canonicalize was split against
+ * @param opened - the opened server manifests
+ */
+ private groundMovedUnderCanonicalize(records: ManifestRecord[], opened: OpenedManifestSet): string | null {
+ const personalRecord = records.find(record => record.isPersonal);
+ const serverPersonal = opened.resolved.find(entry => entry.isPersonal);
+ if (serverPersonal && !manifestIdsEqual(personalRecord?.manifestId, serverPersonal.manifestId)) {
+ return `the personal manifest id (${personalRecord?.manifestId ?? 'none'} became ${serverPersonal.manifestId})`;
+ }
+
+ const saltByManifestId = new Map(opened.resolved.map(entry => [manifestIdKey(entry.manifestId), entry.manifest.manifestSalt]));
+ for (const record of records) {
+ const serverSalt = saltByManifestId.get(manifestIdKey(record.manifestId));
+ if (serverSalt !== undefined && serverSalt !== record.salt) {
+ return `the blob salt of manifest ${record.manifestId}`;
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * The merge core of {@link pullAndMerge}: run the Rust canonical merge of the local side against the manifests
+ * just opened, validate the result per manifest, and materialize it.
* @param opened - the opened server manifests (the merge base)
* @param snapshot - the raw snapshot (for the email-routing block of the response)
* @param encryptionKey - the personal manifest's symmetric key
- * @param localClient - the local vault, freshly decrypted from storage
+ * @param localSide - the canonicalized local vault, the incoming side of the merge
*/
- private async mergeOntoOpenedManifests(opened: OpenedManifestSet, snapshot: GetResponseDto, encryptionKey: string, localClient: SqliteClient): Promise<PullAndMergeResult> {
- const { canonicalized } = await this.canonicalizeVault(localClient);
+ private async mergeOntoOpenedManifests(opened: OpenedManifestSet, snapshot: GetResponseDto, encryptionKey: string, localSide: CanonicalizedVaultSet): Promise<PullAndMergeResult> {
+ const { canonicalized, manifestRecords } = localSide;
const schemaColumns = await VaultCodec.getSchemaColumns(new VaultSqlGenerator().getCompleteSchemaSql());
devLog(`[V2Merge] Merging ${canonicalized.manifests.length} local manifest(s) onto ${opened.resolved.length} server manifest(s)...`);
@@ -523,17 +582,19 @@ export class VaultSyncService {
// Blob bytes for materialize: the server download plus everything the local canonicalize extracted.
const blobMap = opened.blobMap;
+ const localBlobs = new Map<string, CodecBlobEntry>();
for (const { blobs } of canonicalized.manifests) {
for (const [hash, blob] of Object.entries(blobs)) {
+ localBlobs.set(hash, blob);
if (!blobMap.has(hash)) {
blobMap.set(hash, VaultCodec.base64ToBytes(blob.bytesBase64));
}
}
}
- this.assertMergedBlobRefsResolve(manifests, blobMap, opened.resolved[0]?.manifestId ?? '');
+ const mergedBlobs = this.resolveMergedBlobRefs(manifests, blobMap, opened.resolved[0]?.manifestId ?? '', localBlobs);
- const sqliteBase64 = await this.materializeToSqlite(manifests, dataBuckets, blobMap);
- const encryptedVault = await EncryptionUtility.symmetricEncrypt(sqliteBase64, encryptionKey);
+ const sqliteBytes = await this.materializeToSqlite(manifests, dataBuckets, blobMap);
+ const encryptedVault = await encryptVaultBlob(sqliteBytes, encryptionKey);
devLog(`[V2Merge] Canonical merge complete: ${stats.conflicts} conflict(s), ${stats.recordsInserted} offline row(s) kept, ${fallbackManifestIds.length} validation fallback(s), ${mergeOutput.droppedLocalManifestIds.length} dropped local manifest(s).`);
return {
@@ -543,6 +604,42 @@ export class VaultSyncService {
fallbackManifestIds,
droppedLocalManifestIds: mergeOutput.droppedLocalManifestIds,
commitRevisions: opened.commitManifestRevisions,
+ mergedSqliteBytes: sqliteBytes,
+ pushCanonical: this.mergeOutputForPush(manifests, dataBuckets, mergedBlobs, manifestRecords, fallbackManifestIds.length > 0),
+ };
+ }
+
+ /**
+ * The merged vault in the shape the push writes from.
+ * @param manifests - the merged manifests, as materialized
+ * @param dataBuckets - the merged data buckets, as materialized
+ * @param blobsByManifest - the blob entries per manifest
+ * @param manifestRecords - the records the local side was split against
+ * @param hadFallbacks - whether any manifest fell back to the server's copy
+ */
+ private mergeOutputForPush(
+ manifests: VaultManifest[],
+ dataBuckets: VaultDataBucket[],
+ blobsByManifest: Map<string, Record<string, CodecBlobEntry>> | null,
+ manifestRecords: ManifestRecord[],
+ hadFallbacks: boolean
+ ): CanonicalizedVaultSet | null {
+ if (!blobsByManifest || hadFallbacks) {
+ return null;
+ }
+
+ const mergedIds = new Set(manifests.map(manifest => manifestIdKey(manifest.manifestId)));
+ const recordIds = manifestRecords.map(record => manifestIdKey(record.manifestId));
+ if (mergedIds.size !== recordIds.length || recordIds.some(id => !mergedIds.has(id))) {
+ return null;
+ }
+
+ return {
+ canonicalized: {
+ manifests: manifests.map(manifest => ({ manifest, blobs: blobsByManifest.get(manifestIdKey(manifest.manifestId)) ?? {} })),
+ dataBuckets,
+ },
+ manifestRecords,
};
}
@@ -565,15 +662,27 @@ export class VaultSyncService {
}
/**
- * Every blob marker in the merged manifests must resolve to bytes, or materialize would silently insert
- * NULL.
+ * Every blob marker in the merged manifests must resolve to bytes.
* @param manifests - the manifests about to materialize
* @param blobMap - the bytes available by hash
* @param personalManifestId - the personal manifest's id
+ * @param localBlobs - the blob entries the local canonicalize produced, by hash
+ * @returns The blob entries per manifest id, or null when a marker resolved to no bytes or no kind
*/
- private assertMergedBlobRefsResolve(manifests: VaultManifest[], blobMap: Map<string, Uint8Array>, personalManifestId: string): void {
+ private resolveMergedBlobRefs(
+ manifests: VaultManifest[],
+ blobMap: Map<string, Uint8Array>,
+ personalManifestId: string,
+ localBlobs: Map<string, CodecBlobEntry>
+ ): Map<string, Record<string, CodecBlobEntry>> | null {
+ const byManifestId = new Map<string, Record<string, CodecBlobEntry>>();
+ let complete = true;
+
for (const manifest of manifests) {
const isPersonal = manifestIdsEqual(manifest.manifestId, personalManifestId);
+ const blobs: Record<string, CodecBlobEntry> = {};
+ byManifestId.set(manifestIdKey(manifest.manifestId), blobs);
+
for (const rows of Object.values(manifest.tables)) {
for (const row of rows) {
for (const value of Object.values(row)) {
@@ -581,17 +690,31 @@ export class VaultSyncService {
continue;
}
const ref = value as { __blobRef: string; __blobKind?: string };
- if (blobMap.has(ref.__blobRef)) {
+ const bytes = blobMap.get(ref.__blobRef);
+ if (!bytes) {
+ if (ref.__blobKind === 'attachment' && isPersonal) {
+ throw new Error(`VaultSyncService: merged vault references attachment blob ${ref.__blobRef} with no bytes available, refusing to materialize an incomplete vault.`);
+ }
+ devWarn(`[V2Merge] Merged ${ref.__blobKind ?? 'blob'} ${ref.__blobRef} has no bytes available; it will materialize as empty.`);
+ complete = false;
continue;
}
- if (ref.__blobKind === 'attachment' && isPersonal) {
- throw new Error(`VaultSyncService: merged vault references attachment blob ${ref.__blobRef} with no bytes available, refusing to materialize an incomplete vault.`);
+
+ const local = localBlobs.get(ref.__blobRef);
+ if (local) {
+ blobs[ref.__blobRef] = local;
+ } else if (ref.__blobKind) {
+ blobs[ref.__blobRef] = { kind: ref.__blobKind, bytesBase64: bytesToBase64(bytes) };
+ } else {
+ // A marker without a kind cannot be staged for upload; the push derives its own blob set instead.
+ complete = false;
}
- devWarn(`[V2Merge] Merged ${ref.__blobKind ?? 'blob'} ${ref.__blobRef} has no bytes available; it will materialize as empty.`);
}
}
}
}
+
+ return complete ? byManifestId : null;
}
/**
@@ -602,9 +725,9 @@ export class VaultSyncService {
* It runs for a legacy `sqlite-blob` vault whose schema stops at the frozen sqlite-blob upgrade chain, and equally for a
* manifest-v1 vault materialized by an older client whenever a newer COMPLETE_SCHEMA_SQL ships.
* @param sqliteClient - the local vault to migrate
- * @returns Base64 of the migrated SQLite database (plaintext; the caller encrypts and stores it)
+ * @returns The migrated SQLite database (plaintext bytes; the caller encrypts and stores it)
*/
- public async migrateVaultToCurrentSchema(sqliteClient: SqliteClient): Promise<string> {
+ public async migrateVaultToCurrentSchema(sqliteClient: SqliteClient): Promise<Uint8Array> {
devLog('[ManifestMigration] Migrating local vault onto the current schema (local round-trip, no server involved)...');
try {
@@ -626,10 +749,10 @@ export class VaultSyncService {
const schemaColumns = await VaultCodec.getSchemaColumns(schemaSql);
const materialized = await vaultCodecMaterializeAsSqlite(canonicalized.manifests.map(m => m.manifest), canonicalized.dataBuckets, schemaColumns);
- const sqliteBase64 = await VaultCodec.insertTables(materialized, blobMap, schemaSql);
+ const sqliteBytes = await VaultCodec.insertTables(materialized, blobMap, schemaSql);
- devLog(`[ManifestMigration] Migration complete: ${blobMap.size} blobs re-embedded, ${sqliteBase64.length} base64 chars.`);
- return sqliteBase64;
+ devLog(`[ManifestMigration] Migration complete: ${blobMap.size} blobs re-embedded, ${sqliteBytes.length} bytes.`);
+ return sqliteBytes;
} catch (error) {
devError('[ManifestMigration] FAILED to migrate the local vault:', error);
throw new VaultProcessingError('vault-storage-migration', error);
@@ -651,8 +774,8 @@ export class VaultSyncService {
*/
private async materializeFromSnapshot(snapshot: GetResponseDto, vek: string): Promise<PullResult> {
const opened = await this.openManifestsAndAdoptSyncState(snapshot, vek);
- const sqliteBase64 = await this.materializeToSqlite(opened.resolved.map(m => m.manifest), opened.dataBuckets, opened.blobMap);
- return { sqliteBase64, manifestRevision: opened.personalRevision };
+ const sqliteBytes = await this.materializeToSqlite(opened.resolved.map(m => m.manifest), opened.dataBuckets, opened.blobMap);
+ return { sqliteBytes, manifestRevision: opened.personalRevision };
}
/**
@@ -868,7 +991,7 @@ export class VaultSyncService {
try {
blobMap.set(r.hash, await this.decryptBlobToBytes(ciphertext, blobKey));
prunedCache[r.hash] = ciphertext;
- } catch (e) {
+ } catch {
devWarn(`[V2Sync] Referenced ${r.category} blob ${r.hash} failed to decrypt with the current key, continuing without it.`);
}
}
@@ -885,9 +1008,9 @@ export class VaultSyncService {
* @param manifests - the manifests to combine, the caller's own first
* @param dataBuckets - the data buckets belonging to those manifests
* @param blobMap - plaintext bytes for every blob hash the manifests reference
- * @returns Base64 of the materialized SQLite database (plaintext)
+ * @returns The materialized SQLite database (plaintext bytes)
*/
- private async materializeToSqlite(manifests: VaultManifest[], dataBuckets: VaultDataBucket[], blobMap: Map<string, Uint8Array>): Promise<string> {
+ private async materializeToSqlite(manifests: VaultManifest[], dataBuckets: VaultDataBucket[], blobMap: Map<string, Uint8Array>): Promise<Uint8Array> {
devLog(`[V2Pull] ${blobMap.size} blobs decrypted; running codec reassembly into a fresh SQLite (${manifests.length} manifest(s) combined)...`);
const sqlGen = new VaultSqlGenerator();
const schemaSql = sqlGen.getCompleteSchemaSql();
@@ -900,10 +1023,10 @@ export class VaultSyncService {
devWarn(`[V2Pull] Newer-schema data preserved as overflow: ${overflowTableCount} unknown table(s), unknown columns on [${overflowColumnTables.join(', ')}]. It will round-trip on push but is not usable locally until the app is updated.`);
}
- const sqliteBase64 = await VaultCodec.insertTables(materialized, blobMap, schemaSql);
+ const sqliteBytes = await VaultCodec.insertTables(materialized, blobMap, schemaSql);
devLog('[V2Pull] Codec reassembly complete.');
- return sqliteBase64;
+ return sqliteBytes;
}
/**
@@ -1098,11 +1221,13 @@ export class VaultSyncService {
devLog('[V2Push] Account-key migration: generated new VEK, AK and account keypair; vault content and all blobs will be re-encrypted and re-uploaded.');
}
- // 1) Canonicalize, reusing the pre-push no-op check's result when it is still current (same client instance, no mutations since).
+ /**
+ * Canonicalize.
+ */
const currentMutationSequence = ((await storage.getItem(StorageKeys.MUTATION_SEQUENCE)) as number | null) ?? 0;
const cachedSet = (canonicalizeCache && canonicalizeCache.client === sqliteClient && canonicalizeCache.mutationSequence === currentMutationSequence) ? canonicalizeCache : null;
if (cachedSet) {
- devLog('[V2Push] Reusing the canonicalize result from the pre-push no-op check.');
+ devLog('[V2Push] Reusing the canonicalize result already produced for this vault.');
}
const { canonicalized, manifestRecords } = cachedSet ?? await this.canonicalizeVault(sqliteClient);
// Personal manifest first by construction (see `resolveManifestRecords`), which is also the order canonicalize requires.
@@ -1111,15 +1236,13 @@ export class VaultSyncService {
const emailRouting = buildEmailRouting(canonicalized.manifests.map(m => m.manifest), privateEmailDomains);
/*
- * Debug: manifest-set summary + full unencrypted manifests + data buckets, inspectable in the console.
- * TODO: delete the unencrypted-content logs below before release: they print plaintext vault data.
+ * Debug: manifest-set summary, shapes only.
*/
const canonicalizedBuckets = canonicalized.dataBuckets;
devLog(`[V2Push] Canonicalize produced ${canonicalized.manifests.length} manifest(s) + ${canonicalizedBuckets.length} data bucket(s).`);
- devLog(`[V2Push] Unencrypted data buckets (${canonicalizedBuckets.length}):`, canonicalizedBuckets);
for (const { manifest } of canonicalized.manifests) {
const isPersonal = manifest.manifestId === personalRecord.manifestId;
- devLog(`[V2Push] Unencrypted ${isPersonal ? 'personal manifest' : `manifest "${manifest.name ?? manifest.manifestId}"`}: tables: ${Object.entries(manifest.tables).map(([t, rows]) => `${t}=${rows.length}`).join(', ')}`, manifest);
+ devLog(`[V2Push] ${isPersonal ? 'Personal manifest' : `Shared manifest ${manifest.manifestId}`}: tables: ${Object.entries(manifest.tables).map(([t, rows]) => `${t}=${rows.length}`).join(', ')}`);
}
/*
@@ -1472,7 +1595,7 @@ export class VaultSyncService {
*/
public async detectNoOpMutation(sqliteClient: SqliteClient, mutationSequence: number): Promise<boolean> {
const canonicalizedSet = await this.canonicalizeVault(sqliteClient);
- canonicalizeCache = { client: sqliteClient, mutationSequence, ...canonicalizedSet };
+ primeCanonicalizeCache(sqliteClient, mutationSequence, canonicalizedSet);
const { canonicalized, manifestRecords } = canonicalizedSet;
const fingerprints = await this.loadContentFingerprints();
diff --git a/core/rust/src/vault_merge/canonical.rs b/core/rust/src/vault_merge/canonical.rs
index 877a75e8c..c834ef800 100644
--- a/core/rust/src/vault_merge/canonical.rs
+++ b/core/rust/src/vault_merge/canonical.rs
@@ -121,7 +121,9 @@ fn merge_manifest_pair(
let mut merged: HashMap<String, Vec<CodecRecord>> = HashMap::new();
for name in table_names {
- let base_rows = base_tables.remove(&name).unwrap_or_default();
+ let base_entry = base_tables.remove(&name);
+ let base_carried_table = base_entry.is_some();
+ let base_rows = base_entry.unwrap_or_default();
let incoming_rows = incoming_tables.remove(&name).unwrap_or_default();
let merged_rows = match SYNCABLE_TABLES.iter().find(|t| t.name == name) {
Some(config) => {
@@ -131,7 +133,11 @@ fn merge_manifest_pair(
// Not a syncable table (a skip table, or one from a newer writer): the base wins as-is.
None => base_rows,
};
- if !merged_rows.is_empty() {
+ /*
+ * A table the base carried stays in the output even when it merged to nothing, so the merged
+ * manifest keeps the base's shape.
+ */
+ if base_carried_table || !merged_rows.is_empty() {
merged.insert(name, merged_rows);
}
}
@@ -444,6 +450,30 @@ mod tests {
assert!(crate::vault_codec::validate_data_bucket(&merged.buckets[0]).ok);
}
+ #[test]
+ fn an_empty_table_the_server_carried_stays_in_the_merged_manifest() {
+ /*
+ * The merged manifest is pushed as-is and compared against a baseline canonicalize produced, and
+ * canonicalize keeps an empty table. Dropping it here rewrites the whole manifest on every merge.
+ */
+ let merged = merge_single(vec![], vec![]);
+ assert_eq!(merged.manifest.tables.get("Items").map(Vec::len), Some(0), "the server's empty table keeps its place in the merged manifest");
+
+ // The empty tables of a bucket the server served come back as that bucket, not as a missing one.
+ let output = merge_canonical(CanonicalMergeInput {
+ server_manifests: vec![items_manifest(PERSONAL, vec![])],
+ server_buckets: vec![DataBucket::new(PERSONAL, "Settings", [("Settings".to_string(), vec![])].into_iter().collect())],
+ contentless_server_manifest_ids: vec![],
+ local_manifests: vec![items_manifest(PERSONAL, vec![])],
+ local_buckets: vec![],
+ schema_columns: schema(),
+ })
+ .unwrap();
+ let buckets = &output.manifests[0].buckets;
+ assert_eq!(buckets.len(), 1, "the served bucket is still emitted when it holds no rows");
+ assert_eq!(buckets[0].tables.get("Settings").map(Vec::len), Some(0));
+ }
+
#[test]
fn unknown_columns_keep_the_server_value_when_local_wins() {
// A newer client added Items.NewColumn; this client's schema does not know it. The local