Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Add overflow mechanism to codec for forward-compat (#2159)
Leendert de Borstbe50a380e1a2
diff --git a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.tsindex c7ece257a..d59747ff0 100644--- a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts+++ b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts@@ -34,7 +34,7 @@ import { VaultUploadResponse as messageVaultUploadResponse } from '@/utils/typesimport { type VaultMutationScope, ALL_VAULT_MUTATION_SCOPES, DEFAULT_VAULT_MUTATION_SCOPE, dirtyScopeStorageKey, isManifestScope } from '@/utils/types/VaultMutationScope';import { VaultCodec } from '@/utils/VaultCodec';import { vaultMergeService } from '@/utils/VaultMergeService';-import { vaultSyncService, SERVER_MANIFEST_REVISIONS_STORAGE_KEY } from '@/utils/VaultSyncService';+import { vaultSyncService, SERVER_MANIFEST_REVISIONS_STORAGE_KEY, CODEC_OVERFLOW_STORAGE_KEY } from '@/utils/VaultSyncService';import { WebApiService } from '@/utils/WebApiService';import { t } from '@/i18n/StandaloneI18n';@@ -440,6 +440,7 @@ export async function handleClearVaultData(): Promise<messageBoolResponse> {'local:hiddenPrivateEmailDomains','local:serverRevision',SERVER_MANIFEST_REVISIONS_STORAGE_KEY,+ CODEC_OVERFLOW_STORAGE_KEY,'local:isDirty',...ALL_VAULT_MUTATION_SCOPES.map(scope => dirtyScopeStorageKey(scope)),'local:mutationSequence',@@ -821,7 +822,8 @@ async function uploadDirtyBucketsOnly(sqliteClient: SqliteClient, scopes: VaultM}const tables = VaultCodec.readNamedTables(sqliteClient, spec.tables);- const bucket = await vaultCodecExtractBucket(category, tables);+ // Overflow re-merges a newer client's bucket tables/columns so this partial push doesn't drop them.+ const bucket = await vaultCodecExtractBucket(category, tables, await vaultSyncService.loadOverflow(encryptionKey));const result = await vaultSyncService.pushDataBucketOnly(bucket, encryptionKey);if (result.status !== 'ok') {// Conflict persisted even after the rebase-retry, let the caller run a full re-sync (status 2).diff --git a/apps/browser-extension/src/utils/RustCore.ts b/apps/browser-extension/src/utils/RustCore.tsindex 5a335dd0f..0de7369b9 100644--- a/apps/browser-extension/src/utils/RustCore.ts+++ b/apps/browser-extension/src/utils/RustCore.ts@@ -212,6 +212,18 @@ export type CodecCanonicalized = {blobs: Record<string, CodecBlobEntry>;};+/**+ * Data a newer writer put in the manifest that this client's local SQLite schema cannot hold:+ * whole unknown manifest tables, whole unknown bucket tables (per category), and unknown columns+ * keyed by table > row primary-key value. Produced by materialize, persisted opaquely by the+ * platform, and fed back into canonicalize / extractBucket so a push never drops the data.+ */+export type CodecOverflow = {+ tables: Record<string, Array<Record<string, unknown>>>;+ bucketTables: Record<string, Record<string, Array<Record<string, unknown>>>>;+ columns: Record<string, Record<string, Record<string, unknown>>>;+};+/** Input for canonicalize. */export type CodecCanonicalizeInput = {tables: CodecTableData[];@@ -219,10 +231,11 @@ export type CodecCanonicalizeInput = {migrationId: string;version: string;canonicalizedAt: string;+ overflow?: CodecOverflow;};/** Materialized tables the platform inserts into a fresh SQLite DB. */-export type CodecMaterialized = { tables: CodecTableData[]; migrationId: string };+export type CodecMaterialized = { tables: CodecTableData[]; migrationId: string; overflow: CodecOverflow };/** One entry in the bucket layout: a category and the tables it owns. */export type CodecBucketLayoutEntry = { category: string; tables: string[] };@@ -240,18 +253,22 @@ export async function vaultCodecCanonicalizeFromSqlite(input: CodecCanonicalizeI/*** Materialize the manifest + its data buckets into the table set the platform inserts.+ * `schemaColumns` (table > column names of the local schema) makes Rust split anything the schema+ * can't hold into `overflow` instead of emitting it; the caller must persist that overflow and pass+ * it back into the next canonicalize so the data survives the round trip.*/-export async function vaultCodecMaterializeAsSqlite(manifest: CodecManifest, dataBuckets: CodecDataBucket[]): Promise<CodecMaterialized> {+export async function vaultCodecMaterializeAsSqlite(manifest: CodecManifest, dataBuckets: CodecDataBucket[], schemaColumns?: Record<string, string[]>): Promise<CodecMaterialized> {await initRustCore();- return core.vaultCodecMaterializeAsSqlite({ manifest, dataBuckets }) as CodecMaterialized;+ return core.vaultCodecMaterializeAsSqlite({ manifest, dataBuckets, schemaColumns }) as CodecMaterialized;}/**- * Build a single data bucket for `category` from its tables (bucket-only push path).+ * Build a single data bucket for `category` from its tables (bucket-only push path). `overflow`+ * (persisted from the last materialize) re-merges a newer writer's columns/tables so they survive.*/-export async function vaultCodecExtractBucket(category: string, tables: Record<string, Array<Record<string, unknown>>>): Promise<CodecDataBucket> {+export async function vaultCodecExtractBucket(category: string, tables: Record<string, Array<Record<string, unknown>>>, overflow?: CodecOverflow): Promise<CodecDataBucket> {await initRustCore();- return core.vaultCodecExtractBucket({ category, tables }) as CodecDataBucket;+ return core.vaultCodecExtractBucket({ category, tables, overflow }) as CodecDataBucket;}/**diff --git a/apps/browser-extension/src/utils/VaultCodec.ts b/apps/browser-extension/src/utils/VaultCodec.tsindex 32b3dce5a..39c865b5d 100644--- a/apps/browser-extension/src/utils/VaultCodec.ts+++ b/apps/browser-extension/src/utils/VaultCodec.ts@@ -86,6 +86,26 @@ export class VaultCodec {}}+ /**+ * The column set of the local client schema, per table: the input Rust's materialize uses to+ * split off anything a newer writer stored that this schema cannot hold (see `CodecOverflow`).+ * @param schemaSql - the COMPLETE_SCHEMA_SQL string for the target client version+ */+ public static async getSchemaColumns(schemaSql: string): Promise<Record<string, string[]>> {+ const db = await this.createDatabase();+ try {+ db.run(schemaSql);+ const tables = db.exec("SELECT name FROM sqlite_master WHERE type='table'")[0]?.values.map(v => String(v[0])) ?? [];+ const out: Record<string, string[]> = {};+ for (const table of tables) {+ out[table] = db.exec(`PRAGMA table_info("${table}")`)[0]?.values.map(v => String(v[1])) ?? [];+ }+ return out;+ } finally {+ db.close();+ }+ }+/*** Insert the materialized tables into a fresh SQLite database and export it (base64).* @param materialized - tables + migration id produced by the Rust `materialize_as_sqlite`@@ -94,9 +114,7 @@ export class VaultCodec {* @returns A base64-encoded SQLite database identical (in row content) to the original.*/public static async insertTables(materialized: CodecMaterialized, blobs: Map<string, Uint8Array>, schemaSql: string): Promise<string> {- const initSqlJs = (await import('sql.js')).default;- const SQL = await initSqlJs({ locateFile: (file: string): string => `src/${file}` });- const db = new SQL.Database();+ const db = await this.createDatabase();try {// 1) Apply the schema.@@ -184,6 +202,22 @@ export class VaultCodec {}}+ /**+ * Open a fresh in-memory sql.js database.+ */+ private static async createDatabase(): Promise<import('sql.js').Database> {+ const initSqlJs = (await import('sql.js')).default;+ const SQL = await initSqlJs({+ /**+ * Locates SQL.js files from the local file system.+ * @param file - The name of the file to locate+ * @returns The complete URL path to the file+ */+ locateFile: (file: string): string => `src/${file}`+ });+ return new SQL.Database();+ }+/*** Enumerate the names of all user tables in the database (excluding SQLite internals).* @param sqliteClient - opened clientdiff --git a/apps/browser-extension/src/utils/VaultSyncService.ts b/apps/browser-extension/src/utils/VaultSyncService.tsindex 51844c579..f4847b49c 100644--- a/apps/browser-extension/src/utils/VaultSyncService.ts+++ b/apps/browser-extension/src/utils/VaultSyncService.ts@@ -11,7 +11,7 @@ import { VaultDataBucketCategory } from '@/utils/dist/core/models/vault';import type { VaultResponse } from '@/utils/dist/core/models/webapi';import { VaultSqlGenerator } from '@/utils/dist/core/vault';import { EncryptionUtility } from '@/utils/EncryptionUtility';-import {vaultCodecComputeCiphertextHash, vaultCodecCanonicalizeFromSqlite, vaultCodecGenerateUserSalt, vaultCodecUnpackPayload, vaultCodecMaterializeAsSqlite, vaultCodecPackPayload, vaultCodecValidateManifest, vaultCodecValidateDataBucket} from '@/utils/RustCore';+import {type CodecOverflow, vaultCodecComputeCiphertextHash, vaultCodecCanonicalizeFromSqlite, vaultCodecGenerateUserSalt, vaultCodecUnpackPayload, vaultCodecMaterializeAsSqlite, vaultCodecPackPayload, vaultCodecValidateManifest, vaultCodecValidateDataBucket} from '@/utils/RustCore';import { SqliteClient } from '@/utils/SqliteClient';import { ServerUpdateRequiredError } from '@/utils/types/errors/ServerUpdateRequiredError';import { VaultProcessingError } from '@/utils/types/errors/VaultProcessingError';@@ -96,6 +96,11 @@ const bucketRevKey = (category: string): `local:${string}` => `local:vaultV2Buckconst BLOB_CACHE_STORAGE_KEY = 'local:vaultV2BlobCipherCache';/** Hashes the server has stored (refreshed on every pull/push). */const SERVER_HASHES_STORAGE_KEY = 'local:vaultV2ServerBlobHashes';+/**+ * Codec overflow persisted between pull and push: a newer client's tables/columns that this client's+ * local SQLite schema can't hold.+ */+export const CODEC_OVERFLOW_STORAGE_KEY = 'local:vaultV2CodecOverflow';/*** The client's last-known revision per non-Main manifest (manifestId > revision), refreshed on every materialize.*/@@ -237,7 +242,7 @@ export class VaultSyncService {*/public async pull(encryptionKey: string): Promise<VaultResponse> {/*- * Step 1 — network fetch. Failures here (server unreachable, HTTP error, or ServerUpdateRequiredError for an+ * Step 1: network fetch. Failures here (server unreachable, HTTP error, or ServerUpdateRequiredError for an* outdated server) are "can't reach / must update the server" conditions, NOT vault-processing problems, so* they propagate unchanged and the caller maps them to the appropriate "server" message.*/@@ -247,14 +252,14 @@ export class VaultSyncService {devLog(`[V2Pull] Step 1/4 done: storageFormat=${snapshot.storageFormat}, manifests=${snapshot.manifests?.length ?? 0}, rootRevision=${rootManifest?.revision}, manifestBlob=${rootManifest?.blob?.length ?? 0} chars, buckets=${snapshot.buckets?.length ?? 0}, blobRefs=${rootManifest?.blobReferences?.length ?? 0}`);/*- * Steps 2–4 — decrypt, materialize, and re-encrypt. Any failure here is a client-side vault-processing error+ * Steps 2–4: decrypt, materialize, and re-encrypt. Any failure here is a client-side vault-processing error* (codec/format mismatch, integrity failure, corrupt blob, …). We wrap it in a VaultProcessingError so the UI* can surface the real technical detail in a copyable report instead of a misleading "server unreachable".*/try {if (snapshot.storageFormat === STORAGE_FORMAT_MANIFEST) {// Manifest-v1 user: materialize the manifest + metadata + blobs into a SQLite blob, then encrypt it.- devLog('[V2Pull] Step 2/4: manifest format — decrypting and reassembling local SQLite...');+ devLog('[V2Pull] Step 2/4: manifest format: decrypting and reassembling local SQLite...');const pull = await this.materializeFromSnapshot(snapshot, encryptionKey);devLog(`[V2Pull] Step 3/4: materialized SQLite (${pull.sqliteBase64.length} base64 chars); re-encrypting for local storage...`);const encryptedVault = await EncryptionUtility.symmetricEncrypt(pull.sqliteBase64, encryptionKey);@@ -264,7 +269,7 @@ export class VaultSyncService {/** Not-yet-migrated (sqlite-blob fallback) user: the server returned the legacy encrypted SQLite blob. It's already in the stored- * format (encrypted SQLite), so we pass it through unchanged — the on-open schema upgrade handles the rest.+ * format (encrypted SQLite), so we pass it through unchanged: the on-open schema upgrade handles the rest.*/devLog('[V2Pull] Step 2/4: legacy blob pass-through (user not yet migrated), returning as-is.');return this.buildResponse(@@ -274,7 +279,7 @@ export class VaultSyncService {snapshot);} catch (error) {- devError('[V2Pull] FAILED — the last logged step above is where it broke:', error);+ devError('[V2Pull] FAILED: the last logged step above is where it broke:', error);throw new VaultProcessingError('vault-pull', error);}}@@ -406,8 +411,8 @@ export class VaultSyncService {/** Decrypt referenced blobs for reassembly and prune the persisted cache to exactly the referenced set, so- * the cache stays bounded by the current vault size. A blob that can't be reconstituted — the server couldn't- * serve it (missing) or its ciphertext won't decrypt with the current key (stale key / corruption) — is fatal+ * the cache stays bounded by the current vault size. A blob that can't be reconstituted: the server couldn't+ * serve it (missing) or its ciphertext won't decrypt with the current key (stale key / corruption): is fatal* for attachments (silently dropping bytes would propagate permanent data loss on the next push); a favicon* only degrades cosmetically, so it's logged and skipped. The codec inserts NULL for any skipped blob ref.*/@@ -441,7 +446,21 @@ export class VaultSyncService {devLog(`[V2Pull] ${blobMap.size} blobs decrypted; running codec reassembly into a fresh SQLite...`);const sqlGen = new VaultSqlGenerator();const schemaSql = sqlGen.getCompleteSchemaSql();- const materialized = await vaultCodecMaterializeAsSqlite(manifest, dataBuckets);+ const schemaColumns = await VaultCodec.getSchemaColumns(schemaSql);+ const materialized = await vaultCodecMaterializeAsSqlite(manifest, dataBuckets, schemaColumns);++ /*+ * Anything a newer client wrote that our schema can't hold was split into overflow instead of the+ * insert set. Persist it so push/extract re-merge it into the manifest: without this, our next+ * push would silently delete those tables/columns for every other client.+ */+ await this.saveOverflow(materialized.overflow, vek);+ const overflowTableCount = Object.keys(materialized.overflow.tables).length + Object.values(materialized.overflow.bucketTables).reduce((n, t) => n + Object.keys(t).length, 0);+ const overflowColumnTables = Object.keys(materialized.overflow.columns);+ if (overflowTableCount > 0 || overflowColumnTables.length > 0) {+ devWarn(`[V2Pull] Newer-schema data preserved as overflow: ${overflowTableCount} unknown table(s), unknown columns on [${overflowColumnTables.join(', ')}]. It will round-trip on push but is not usable locally until the app is updated.`);+ }+const sqliteBase64 = await VaultCodec.insertTables(materialized, blobMap, schemaSql);devLog('[V2Pull] Codec reassembly complete.');@@ -513,12 +532,13 @@ export class VaultSyncService {// Read tables from the SQLite database and apply the manifest-v1 format rules.const tables = VaultCodec.readTables(sqliteClient);const migrationId = VaultCodec.getLatestMigrationId(sqliteClient);- const canonicalized = await timedStage('canonicalize (incl. Rust→JS conversion)', () => vaultCodecCanonicalizeFromSqlite({+ const canonicalized = await timedStage('canonicalize (incl. Rust→JS conversion)', async () => vaultCodecCanonicalizeFromSqlite({tables,userSalt,migrationId,version: '2.0.0',canonicalizedAt: new Date().toISOString(),+ overflow: await this.loadOverflow(vek),}));// Plaintext blob bytes held platform-side for encryption/upload.@@ -800,6 +820,38 @@ export class VaultSyncService {private async saveBlobCache(cache: Record<string, string>): Promise<void> {await storage.setItem(BLOB_CACHE_STORAGE_KEY, cache);}++ /**+ * Load the codec overflow persisted by the last pull (undefined when none): a newer client's+ * tables/columns our local schema can't hold. Callers pass it into canonicalize/extractBucket so a+ * push from this client re-emits that data instead of dropping it. Stored AES-GCM encrypted; an+ * entry that no longer decrypts (key rotated, account switched) is discarded: the next pull+ * rebuilds it from the server manifest.+ * @param vek - the user's symmetric key+ */+ public async loadOverflow(vek: string): Promise<CodecOverflow | undefined> {+ const ciphertext = (await storage.getItem(CODEC_OVERFLOW_STORAGE_KEY)) as string | null;+ if (!ciphertext) {+ return undefined;+ }+ try {+ return JSON.parse(await EncryptionUtility.symmetricDecrypt(ciphertext, vek)) as CodecOverflow;+ } catch {+ devWarn('[V2Sync] Persisted codec overflow could not be decrypted with the current key, discarding it. It will be rebuilt on the next pull.');+ await storage.removeItem(CODEC_OVERFLOW_STORAGE_KEY);+ return undefined;+ }+ }++ /**+ * Persist the codec overflow produced by materialize (rebuilt wholesale on every pull),+ * AES-GCM encrypted so no vault row data sits in extension storage as plaintext.+ * @param overflow - the overflow split off by the last materialize+ * @param vek - the user's symmetric key+ */+ private async saveOverflow(overflow: CodecOverflow, vek: string): Promise<void> {+ await storage.setItem(CODEC_OVERFLOW_STORAGE_KEY, await EncryptionUtility.symmetricEncrypt(JSON.stringify(overflow), vek));+ }}export const vaultSyncService = new VaultSyncService();diff --git a/apps/browser-extension/tests/e2e/13-forward-compat-overflow.spec.ts b/apps/browser-extension/tests/e2e/13-forward-compat-overflow.spec.tsnew file mode 100644index 000000000..dbc9b9c01--- /dev/null+++ b/apps/browser-extension/tests/e2e/13-forward-compat-overflow.spec.ts@@ -0,0 +1,125 @@+/**+ * Category 13: Forward-compat overflow — no data loss (Requires API)+ *+ * These tests enforce the manifest-v1 forward-compatibility guarantee: when a NEWER client writes+ * vault data this client's schema doesn't know (a new row column, a whole new table), the current+ * extension must (a) still load the vault, and (b) carry that unknown data through its own pushes+ * untouched instead of silently deleting it (see `CodecOverflow` in the Rust vault codec).+ *+ * Scenario:+ * 1. Extension client creates a credential and syncs (migrating the account to manifest-v1)+ * 2. A simulated "newer client" (node-side, real Rust WASM codec) pulls the manifest, injects an+ * unknown column (`Items.AliasEnabled`) and an unknown table (`FutureFeatures`), and pushes it+ * back as a new revision+ * 3. The extension pulls that manifest — the vault must load and stay fully usable+ * 4. The extension edits the credential (rename) and pushes+ * 5. The newer client pulls again — the rename is applied AND the unknown column/table are intact+ */+import { test, expect, TestClient, FieldSelectors } from '../fixtures';+import { getVaultSnapshot, openManifest, pushManifest, pollUntil, requireRootManifest, type DecryptedManifest } from '../helpers/manifest-v2-api';+import type { TestUser } from '../helpers/test-api';++test.describe.serial('13. Forward-compat overflow', () => {+ let client: TestClient;+ /** The account under test — the testUser fixture is per-test, so 13.1 pins it for later tests. */+ let user: TestUser;+ let baseApiUrl: string;++ const credentialName = `Overflow Credential ${Date.now()}`;+ const renamedCredentialName = `${credentialName} renamed`;+ const futureTableRow = { Id: 'f1e2e000-0000-4000-8000-000000000001', Payload: 'from-the-future' };++ /** Manifest revision created by the simulated newer client in 13.2. */+ let injectedRevision: number;++ test.afterAll(async () => {+ await client?.cleanup();+ });++ test('13.1 extension client should create a credential and sync to manifest-v1', async ({ testUser, apiUrl }) => {+ user = testUser;+ baseApiUrl = apiUrl;+ client = await TestClient.create();+ await client.login(apiUrl, testUser.username, testUser.password);++ await client+ .goToVault()+ .then((c) => c.createCredential(credentialName, '[email protected]', 'OverflowPass123!'))+ .then((c) => c.verifyCredentialExists(credentialName))+ .then((c) => c.triggerSync())+ .then((c) => c.screenshot('13.1-credential-created.png'));++ // The save's push runs in the background; wait until the server actually holds a manifest-v1+ // snapshot whose Items table contains the credential.+ const { manifest } = await openLatestManifest(apiUrl, user.token!.token, user.encryptionKey!);+ const item = (manifest.tables.Items ?? []).find((row) => row.Name === credentialName);+ expect(item, 'credential row should be present in the server manifest').toBeTruthy();+ });++ test('13.2 a newer client should inject an unknown column and table into the server manifest', async () => {+ const token = user.token!.token;+ const { manifest, root } = await openLatestManifest(baseApiUrl, token, user.encryptionKey!);++ // Simulate a newer client's schema additions: a column this extension build doesn't know on an+ // existing row, and a whole table it doesn't know at all.+ const item = (manifest.tables.Items ?? []).find((row) => row.Name === credentialName);+ expect(item).toBeTruthy();+ item!.AliasEnabled = true;+ manifest.tables.FutureFeatures = [futureTableRow];++ injectedRevision = await pushManifest(baseApiUrl, token, user.username, manifest, root.revision, root.blobReferences, user.encryptionKey!);+ expect(injectedRevision).toBeGreaterThan(root.revision);+ });++ test('13.3 extension should load the newer manifest without crashing', async () => {+ await client+ .triggerSync()+ .then((c) => c.goToVault())+ .then((c) => c.verifyCredentialExists(credentialName))+ .then((c) => c.screenshot('13.3-newer-manifest-loaded.png'));+ });++ test('13.4 extension should edit the credential and push', async () => {+ await client+ .clickCredential(credentialName)+ .then((c) => c.openEditForm())+ .then((c) => c.fillField(FieldSelectors.ITEM_NAME, renamedCredentialName))+ .then((c) => c.saveCredential())+ .then((c) => c.verifyCredentialExists(renamedCredentialName))+ .then((c) => c.triggerSync())+ .then((c) => c.screenshot('13.4-credential-renamed.png'));+ });++ test('13.5 the newer client data should survive the extension push (no data loss)', async () => {+ // Wait for the extension's push to land: a revision beyond the injected one, carrying the rename.+ const manifest = await pollUntil(async (): Promise<DecryptedManifest | undefined> => {+ const { manifest: m, root } = await openLatestManifest(baseApiUrl, user.token!.token, user.encryptionKey!);+ const renamed = (m.tables.Items ?? []).some((row) => row.Name === renamedCredentialName);+ return root.revision > injectedRevision && renamed ? m : undefined;+ });++ // The unknown column re-attached to the row the extension itself rewrote (rename bumps the row).+ const item = (manifest.tables.Items ?? []).find((row) => row.Name === renamedCredentialName);+ expect(item, 'renamed credential row should be present').toBeTruthy();+ expect(item!.AliasEnabled, 'unknown column must survive the old-client push').toBe(true);++ // The unknown table re-emitted verbatim.+ expect(manifest.tables.FutureFeatures, 'unknown table must survive the old-client push').toEqual([futureTableRow]);+ });+});++/**+ * Polls until the server snapshot is manifest-v1, then decrypts and returns its root manifest.+ *+ * @param apiUrl - The base URL of the API+ * @param token - Bearer token+ * @param encryptionKey - The user's derived vault encryption key+ * @returns The decrypted manifest and its snapshot entry+ */+async function openLatestManifest(apiUrl: string, token: string, encryptionKey: Uint8Array): Promise<{ manifest: DecryptedManifest; root: ReturnType<typeof requireRootManifest> }> {+ return pollUntil(async () => {+ const snapshot = await getVaultSnapshot(apiUrl, token);+ const root = requireRootManifest(snapshot);+ return { manifest: await openManifest(root.blob, encryptionKey), root };+ });+}diff --git a/apps/browser-extension/tests/helpers/manifest-v2-api.ts b/apps/browser-extension/tests/helpers/manifest-v2-api.tsnew file mode 100644index 000000000..7d3cb2215--- /dev/null+++ b/apps/browser-extension/tests/helpers/manifest-v2-api.ts@@ -0,0 +1,186 @@+/**+ * Node-side manifest-v1 vault client for E2E tests.+ *+ * Lets a test act as a second, "newer" client against the v2 Vault API: pull the snapshot,+ * decrypt + unpack the root manifest, modify it (e.g. inject columns/tables an older client's+ * schema doesn't know), and push it back as a new revision. Pack/unpack goes through the real+ * Rust WASM codec so the integrity envelope's canonical content hash matches exactly what the+ * extension verifies on pull.+ */++import { createHash } from 'crypto';+import { readFileSync } from 'fs';+import { join } from 'path';++import { initSync, vaultCodecPackPayload, vaultCodecUnpackPayload } from '../../src/utils/dist/core/rust/aliasvault_core.js';++import { normalizeUsername, symmetricDecryptBytes, symmetricEncryptBytes } from './test-api';++/** One manifest entry in the v2 GET snapshot. */+export type SnapshotManifest = {+ manifestId: string;+ isRoot: boolean;+ blob: string;+ ciphertextHash: string;+ revision: number;+ blobReferences: Array<{ hash: string; category: string }>;+};++/** The v2 GET /Vault snapshot (fields relevant to these tests). */+export type VaultSnapshot = {+ status: number;+ /** Server StorageFormat enum: 0 = legacy sqlite-blob, 1 = manifest-v1. */+ storageFormat: number;+ manifests?: SnapshotManifest[];+ buckets?: Array<{ category: string; blob: string; ciphertextHash: string; revision: number }>;+};++/** A decrypted manifest-v1 manifest: known top-level fields plus free-form tables. */+export type DecryptedManifest = {+ schemaVersion: number;+ migrationId: string;+ version: string;+ userSalt: string;+ canonicalizedAt: string;+ tables: Record<string, Array<Record<string, unknown>>>;+ [key: string]: unknown;+};++let wasmInitialized = false;++/**+ * Initializes the Rust core WASM module from the extension's dist (idempotent).+ */+function ensureWasm(): void {+ if (!wasmInitialized) {+ const wasmPath = join(process.cwd(), 'src/utils/dist/core/rust/aliasvault_core_bg.wasm');+ initSync({ module: readFileSync(wasmPath) });+ wasmInitialized = true;+ }+}++/**+ * Fetches the v2 vault snapshot for the authenticated user.+ *+ * @param apiBaseUrl - The base URL of the API+ * @param token - Bearer token+ * @returns The snapshot response+ */+export async function getVaultSnapshot(apiBaseUrl: string, token: string): Promise<VaultSnapshot> {+ const response = await fetch(`${apiBaseUrl.replace(/\/$/, '')}/v2/Vault`, {+ headers: { Authorization: `Bearer ${token}` },+ });+ if (!response.ok) {+ throw new Error(`GET /v2/Vault failed with status ${response.status}: ${await response.text()}`);+ }+ return (await response.json()) as VaultSnapshot;+}++/**+ * Returns the root manifest entry of a snapshot, or throws if the user isn't on manifest-v1 yet.+ *+ * @param snapshot - The v2 snapshot+ * @returns The root manifest entry+ */+export function requireRootManifest(snapshot: VaultSnapshot): SnapshotManifest {+ const root = (snapshot.manifests ?? []).find((m) => m.isRoot);+ if (snapshot.storageFormat !== 1 || !root?.blob) {+ throw new Error(`Snapshot is not manifest-v1 yet (storageFormat=${snapshot.storageFormat}, manifests=${snapshot.manifests?.length ?? 0}).`);+ }+ return root;+}++/**+ * Decrypts and unpacks a manifest blob into its JSON object (content hash verified by the codec).+ *+ * @param blobBase64 - The encrypted manifest blob from the snapshot+ * @param encryptionKey - The user's derived vault encryption key+ * @returns The decrypted manifest object+ */+export async function openManifest(blobBase64: string, encryptionKey: Uint8Array): Promise<DecryptedManifest> {+ ensureWasm();+ const packedBytes = await symmetricDecryptBytes(blobBase64, encryptionKey);+ return JSON.parse(vaultCodecUnpackPayload(packedBytes)) as DecryptedManifest;+}++/**+ * Packs (envelope + canonical content hash + gzip via the Rust codec), encrypts, and uploads a+ * manifest as a new revision, exactly like the extension's full push does.+ *+ * @param apiBaseUrl - The base URL of the API+ * @param token - Bearer token+ * @param username - The vault owner's username+ * @param manifest - The (modified) manifest object to upload+ * @param currentRevision - The revision this upload is based on (server assigns currentRevision + 1)+ * @param blobReferences - Blob references to carry over to the new revision+ * @param encryptionKey - The user's derived vault encryption key+ * @returns The new manifest revision number assigned by the server+ */+export async function pushManifest(+ apiBaseUrl: string,+ token: string,+ username: string,+ manifest: DecryptedManifest,+ currentRevision: number,+ blobReferences: Array<{ hash: string; category: string }>,+ encryptionKey: Uint8Array+): Promise<number> {+ ensureWasm();+ const packedBytes = vaultCodecPackPayload(JSON.stringify(manifest));+ const manifestBlob = await symmetricEncryptBytes(packedBytes, encryptionKey);+ const manifestCiphertextHash = createHash('sha256').update(Buffer.from(manifestBlob, 'base64')).digest('hex');++ const payload = {+ username: normalizeUsername(username),+ version: manifest.version,+ manifestBlob,+ manifestCiphertextHash,+ currentManifestRevision: currentRevision,+ credentialsCount: (manifest.tables.Items ?? []).length,+ buckets: [],+ newBlobs: [],+ blobReferences,+ emailRouting: { emailAddressList: [] },+ encryptionPublicKey: '',+ };++ const response = await fetch(`${apiBaseUrl.replace(/\/$/, '')}/v2/Vault`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` },+ body: JSON.stringify(payload),+ });+ if (!response.ok) {+ throw new Error(`POST /v2/Vault failed with status ${response.status}: ${await response.text()}`);+ }++ const result = (await response.json()) as { status: number; newManifestRevision: number; missingBlobHashes?: string[] };+ if (result.status !== 0 || (result.missingBlobHashes?.length ?? 0) > 0) {+ throw new Error(`Manifest push rejected: status=${result.status}, missingBlobs=${result.missingBlobHashes?.join(',') ?? 'none'}`);+ }+ return result.newManifestRevision;+}++/**+ * Polls until `predicate` returns a truthy value or the timeout elapses.+ *+ * @param predicate - Async condition; return a truthy value to stop polling+ * @param timeoutMs - Total time budget in milliseconds+ * @param intervalMs - Delay between attempts in milliseconds+ * @returns The first truthy value the predicate returned+ */+export async function pollUntil<T>(predicate: () => Promise<T | undefined | false>, timeoutMs = 20000, intervalMs = 1000): Promise<T> {+ const deadline = Date.now() + timeoutMs;+ let lastError: unknown;+ while (Date.now() < deadline) {+ try {+ const result = await predicate();+ if (result) {+ return result;+ }+ } catch (e) {+ lastError = e;+ }+ await new Promise((resolve) => setTimeout(resolve, intervalMs));+ }+ throw new Error(`pollUntil timed out after ${timeoutMs}ms${lastError ? `; last error: ${lastError instanceof Error ? lastError.message : String(lastError)}` : ''}`);+}diff --git a/apps/browser-extension/tests/helpers/test-api.ts b/apps/browser-extension/tests/helpers/test-api.tsindex 8ae54d3ee..6174d3163 100644--- a/apps/browser-extension/tests/helpers/test-api.ts+++ b/apps/browser-extension/tests/helpers/test-api.ts@@ -40,6 +40,8 @@ export type TestUser = {token?: TokenModel;/** TOTP secret if 2FA is enabled */totpSecret?: string;+ /** Argon2Id-derived vault encryption key, for tests that decrypt/re-encrypt vault blobs API-side. */+ encryptionKey?: Uint8Array;};/**@@ -190,6 +192,38 @@ async function symmetricEncrypt(plaintext: string, keyBytes: Uint8Array): Promisreturn Buffer.from(combined).toString('base64');}+/**+ * Encrypts raw bytes using AES-GCM (matching the browser extension's `symmetricEncryptBytes`).+ *+ * @param plaintextBytes - The plaintext bytes to encrypt+ * @param keyBytes - The 256-bit encryption key as Uint8Array+ * @returns Base64-encoded ciphertext (IV prepended to ciphertext)+ */+export async function symmetricEncryptBytes(plaintextBytes: Uint8Array, keyBytes: Uint8Array): Promise<string> {+ const key = await webcrypto.subtle.importKey('raw', keyBytes, { name: 'AES-GCM', length: 256 }, false, ['encrypt']);+ const iv = webcrypto.getRandomValues(new Uint8Array(12));+ const ciphertext = await webcrypto.subtle.encrypt({ name: 'AES-GCM', iv }, key, plaintextBytes);++ const combined = new Uint8Array(iv.length + ciphertext.byteLength);+ combined.set(iv);+ combined.set(new Uint8Array(ciphertext), iv.length);+ return Buffer.from(combined).toString('base64');+}++/**+ * Decrypts an AES-GCM ciphertext produced by the extension's `symmetricEncrypt(Bytes)` (IV-prefixed).+ *+ * @param base64Ciphertext - Base64-encoded ciphertext (12-byte IV prepended)+ * @param keyBytes - The 256-bit encryption key as Uint8Array+ * @returns The decrypted plaintext bytes+ */+export async function symmetricDecryptBytes(base64Ciphertext: string, keyBytes: Uint8Array): Promise<Uint8Array> {+ const combined = new Uint8Array(Buffer.from(base64Ciphertext, 'base64'));+ const key = await webcrypto.subtle.importKey('raw', keyBytes, { name: 'AES-GCM', length: 256 }, false, ['decrypt']);+ const plaintext = await webcrypto.subtle.decrypt({ name: 'AES-GCM', iv: combined.slice(0, 12) }, key, combined.slice(12));+ return new Uint8Array(plaintext);+}+/*** Creates an empty vault database with the latest schema.*@@ -302,7 +336,9 @@ async function uploadInitialVault(username: string,encryptionKey: Uint8Array): Promise<void> {- const baseUrl = apiBaseUrl.replace(/\/$/, '') + '/v2/';+ // Intentionally the v1 endpoint: the initial vault is a legacy sqlite-blob (the v2 endpoint is+ // manifest-only). This mirrors a pre-manifest account; the extension migrates it on first save.+ const baseUrl = apiBaseUrl.replace(/\/$/, '') + '/v1/';// Create an empty vault databaseconst vaultBase64 = createEmptyVaultDatabase();@@ -359,7 +395,7 @@ export async function registerTestUser(apiBaseUrl: string,username: string,password: string-): Promise<TokenModel> {+): Promise<{ tokenModel: TokenModel; encryptionKey: Uint8Array }> {// Normalize the API URLconst baseUrl = apiBaseUrl.replace(/\/$/, '') + '/v2/';@@ -392,7 +428,7 @@ export async function registerTestUser(// Upload initial empty vaultawait uploadInitialVault(apiBaseUrl, tokenModel.token, username, encryptionKey);- return tokenModel;+ return { tokenModel, encryptionKey };}/**@@ -405,12 +441,13 @@ export async function createTestUser(apiBaseUrl: string): Promise<TestUser> {const username = generateTestUsername();const password = generateTestPassword();- const token = await registerTestUser(apiBaseUrl, username, password);+ const { tokenModel, encryptionKey } = await registerTestUser(apiBaseUrl, username, password);return {username,password,- token,+ token: tokenModel,+ encryptionKey,};}@@ -519,15 +556,16 @@ export async function createTestUserWith2FA(apiBaseUrl: string): Promise<TestUseconst username = generateTestUsername();const password = generateTestPassword();- const token = await registerTestUser(apiBaseUrl, username, password);+ const { tokenModel, encryptionKey } = await registerTestUser(apiBaseUrl, username, password);// Enable 2FA for the user- const totpSecret = await enableTwoFactor(apiBaseUrl, token.token);+ const totpSecret = await enableTwoFactor(apiBaseUrl, tokenModel.token);return {username,password,- token,+ token: tokenModel,totpSecret,+ encryptionKey,};}diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rsindex 93930a316..a05e62457 100644--- a/core/rust/src/vault_codec/canonicalize.rs+++ b/core/rust/src/vault_codec/canonicalize.rs@@ -18,8 +18,9 @@ use serde_json::json;use super::hash::salted_blob_hash;use super::logos::dedupe_logos_by_source;-use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedVault, DataBucket, Manifest, CodecRecord};-use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_skip_table, SCHEMA_VERSION};+use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedVault, CodecOverflow, DataBucket, Manifest, CodecRecord};+use super::materialize::row_key;+use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_skip_table, primary_key_for, SCHEMA_VERSION};use crate::error::VaultResult;/// Canonicalize normalized tables into the split resources: the manifest, one data bucket per declared@@ -36,6 +37,15 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonicall_tables.entry(table.name.clone()).or_default().extend(table.records.iter().cloned());}+ // Re-merge overflow from the last materialize (a newer writer's tables/columns this client's+ // local schema couldn't hold) so this push doesn't drop them. See `CodecOverflow`.+ let overflow = input.overflow.unwrap_or_default();+ remerge_overflow_columns(&mut all_tables, &overflow);+ for (name, rows) in &overflow.tables {+ // Local rows win if the table somehow exists locally now (e.g. client upgraded since the pull).+ all_tables.entry(name.clone()).or_insert_with(|| rows.clone());+ }+// Collapse duplicate Logos rows sharing a Source (see `logos` module) and remap Items.LogoId so// the manifest we write is free of any UNIQUE(Source) collision that would break materialize.dedupe_logos_by_source(&mut all_tables);@@ -73,6 +83,18 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonicmanifest_tables.insert(name, out_rows);}+ /*+ * Re-emit overflow bucket tables into their original categories. A category this client's+ * BUCKET_TABLES doesn't declare yet is added as its own bucket, preserving the newer writer's+ * bucket structure verbatim.+ */+ for (category, ov_tables) in &overflow.bucket_tables {+ let bucket = bucketed.entry(category.clone()).or_default();+ for (name, rows) in ov_tables {+ bucket.entry(name.clone()).or_insert_with(|| rows.clone());+ }+ }+let manifest = Manifest {schema_version: SCHEMA_VERSION,migration_id: input.migration_id,@@ -124,9 +146,47 @@ fn extract_blob_cell(json!({ "__blobRef": hash, "__blobKind": kind })}-/// Build a single data bucket for `category` from its already-normalized tables (name > rows).-/// The bucket-only push path (a bucket changed but the manifest didn't). Input rows are+/// Re-attach overflow columns (split off at materialize time because the local schema didn't know+/// them) to their rows, matched by the table's primary-key value. A row deleted locally simply has+/// no match and its overflow is dropped with it, which is the correct outcome. Locally-known+/// columns always win on a (theoretical) name collision.+fn remerge_overflow_columns(tables: &mut HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow) {+ for (table_name, by_pk) in &overflow.columns {+ let rows = match tables.get_mut(table_name) {+ Some(rows) => rows,+ None => continue,+ };+ let pk_column = primary_key_for(table_name);+ for row in rows {+ let pk_value = match row.get(pk_column).map(row_key) {+ Some(v) => v,+ None => continue,+ };+ if let Some(extra_columns) = by_pk.get(&pk_value) {+ for (column, value) in extra_columns {+ row.entry(column.clone()).or_insert_with(|| value.clone());+ }+ }+ }+ }+}++/// Build a single data bucket for `category` from its already-normalized tables (name > rows).+/// The bucket-only push path (a bucket changed but the manifest didn't). Input rows are/// already normalized by the platform read logic.-pub fn extract_bucket(category: String, tables: HashMap<String, Vec<CodecRecord>>) -> DataBucket {+///+/// `overflow` (persisted from the last materialize) is re-merged the same way the full+/// canonicalize does it: unknown columns re-attach to this bucket's rows, and whole unknown tables+/// recorded under this bucket's category are re-emitted, so a bucket-only push from an older+/// client never drops a newer writer's data either.+pub fn extract_bucket(category: String, mut tables: HashMap<String, Vec<CodecRecord>>, overflow: Option<&CodecOverflow>) -> DataBucket {+ if let Some(overflow) = overflow {+ remerge_overflow_columns(&mut tables, overflow);+ if let Some(ov_tables) = overflow.bucket_tables.get(&category) {+ for (name, rows) in ov_tables {+ tables.entry(name.clone()).or_insert_with(|| rows.clone());+ }+ }+ }DataBucket::new(category, tables)}diff --git a/core/rust/src/vault_codec/manifest.rs b/core/rust/src/vault_codec/manifest.rsindex ed3ebc7f6..206e4fd4d 100644--- a/core/rust/src/vault_codec/manifest.rs+++ b/core/rust/src/vault_codec/manifest.rs@@ -89,13 +89,45 @@ pub struct CodecTableData {pub records: Vec<CodecRecord>,}+/// Data a newer writer put in the manifest that this client's local SQLite schema cannot hold.+///+/// Materialize splits it off (so inserts don't crash on unknown tables/columns) and canonicalize+/// re-merges it (so this client's next push never drops it). The platform persists this value+/// opaquely between pull and push; it is rebuilt from scratch on every pull, so it tracks the same+/// staleness/LWW semantics as the rest of the row data.+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct CodecOverflow {+ /// Whole manifest tables the local schema doesn't know: name > rows, re-emitted verbatim.+ #[serde(default)]+ pub tables: HashMap<String, Vec<CodecRecord>>,+ /// Whole bucket tables the local schema doesn't know: category > (name > rows). Kept per+ /// category so both full pushes and bucket-only pushes re-emit them into the right bucket.+ #[serde(default)]+ pub bucket_tables: HashMap<String, HashMap<String, Vec<CodecRecord>>>,+ /// Unknown columns split off rows of known tables: table > row primary-key value > {column: value}.+ #[serde(default)]+ pub columns: HashMap<String, HashMap<String, CodecRecord>>,+}++impl CodecOverflow {+ /// True when nothing was split off (the common case: reader and writer share a schema).+ pub fn is_empty(&self) -> bool {+ self.tables.is_empty() && self.bucket_tables.is_empty() && self.columns.is_empty()+ }+}+/// Materialized tables the platform inserts into a fresh schema DB. Blob columns carry-/// `{ "__blobRef": hash }`; inline byte columns carry `{ "__b64": ... }`.+/// `{ "__blobRef": hash }`; inline byte columns carry `{ "__b64": ... }`. `overflow` holds whatever+/// the local schema couldn't accept (see [`CodecOverflow`]); the platform must persist it and feed+/// it back into the next canonicalize.#[derive(Debug, Clone, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]pub struct MaterializedTables {pub tables: Vec<CodecTableData>,pub migration_id: String,+ #[serde(default)]+ pub overflow: CodecOverflow,}/// Input for [`crate::vault_codec::canonicalize_from_sqlite`].@@ -108,6 +140,10 @@ pub struct CanonicalizeInput {#[serde(default = "default_version")]pub version: String,pub canonicalized_at: String,+ /// Overflow persisted from the last materialize (see [`CodecOverflow`]); re-merged into the+ /// output so a client with an older schema never drops a newer writer's tables/columns.+ #[serde(default)]+ pub overflow: Option<CodecOverflow>,}fn default_version() -> String {@@ -121,4 +157,9 @@ pub struct MaterializeInput {pub manifest: Manifest,#[serde(default)]pub data_buckets: Vec<DataBucket>,+ /// The caller's local SQLite schema: table > column names. When present, rows are filtered down+ /// to what the schema can hold and the remainder lands in [`MaterializedTables::overflow`];+ /// when absent, rows pass through verbatim (legacy behavior — unknown columns crash the insert).+ #[serde(default)]+ pub schema_columns: Option<HashMap<String, Vec<String>>>,}diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rsindex fc5866108..5497b97fa 100644--- a/core/rust/src/vault_codec/materialize.rs+++ b/core/rust/src/vault_codec/materialize.rs@@ -6,36 +6,108 @@//!//! Rust does not handle actual SQLite operations. It only emits the destination table rows as-is//! to the caller and let the caller handle the actual SQLite database creation and data insertion.+//!+//! Forward compatibility: when the caller supplies its local schema (`schema_columns`), anything a+//! newer writer put in the manifest that this schema cannot hold — whole unknown tables or unknown+//! columns on known tables — is split into [`CodecOverflow`] instead of being emitted (which would+//! crash the platform insert). The platform persists the overflow and passes it back into+//! `canonicalize_from_sqlite`, which re-merges it so this client's next push never drops the data.++use std::collections::{HashMap, HashSet};-use super::manifest::{MaterializeInput, MaterializedTables, CodecTableData};-use super::types::is_skip_table;+use super::manifest::{CodecOverflow, CodecRecord, CodecTableData, MaterializeInput, MaterializedTables};+use super::types::{is_skip_table, primary_key_for};use crate::error::VaultResult;/// Materialize the manifest + its data buckets into the table set the platform inserts.pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<MaterializedTables> {- let MaterializeInput { manifest, data_buckets } = input;+ let MaterializeInput { manifest, data_buckets, schema_columns } = input;+ let mut overflow = CodecOverflow::default();let mut tables: Vec<CodecTableData> = Vec::with_capacity(manifest.tables.len() + data_buckets.len());for (name, records) in manifest.tables {if is_skip_table(&name) {continue;}- tables.push(CodecTableData { name, records });+ match split_for_schema(&name, records, schema_columns.as_ref(), &mut overflow.columns) {+ SplitResult::Fits(records) => tables.push(CodecTableData { name, records }),+ SplitResult::UnknownTable(records) => {+ overflow.tables.insert(name, records);+ }+ }}- // Reconstitute every data bucket's tables back into the flat set.+ // Reconstitute every data bucket's tables back into the flat set. Unknown bucket tables keep+ // their category so canonicalize / extract_bucket can re-emit them into the right bucket.for bucket in data_buckets {for (name, records) in bucket.tables {if is_skip_table(&name) {continue;}- tables.push(CodecTableData { name, records });+ match split_for_schema(&name, records, schema_columns.as_ref(), &mut overflow.columns) {+ SplitResult::Fits(records) => tables.push(CodecTableData { name, records }),+ SplitResult::UnknownTable(records) => {+ overflow.bucket_tables.entry(bucket.category.clone()).or_default().insert(name, records);+ }+ }}}Ok(MaterializedTables {tables,migration_id: manifest.migration_id,+ overflow,})}++/// Outcome of fitting one table's rows to the caller's schema.+enum SplitResult {+ /// Rows the schema can insert (unknown columns already split off into overflow).+ Fits(Vec<CodecRecord>),+ /// The schema has no such table at all; the whole table belongs in overflow.+ UnknownTable(Vec<CodecRecord>),+}++/// Fit `records` to the caller's schema. Without a schema, rows pass through verbatim. With one,+/// unknown columns are stashed in `column_overflow` keyed by the row's primary-key value; a row+/// whose primary key the schema doesn't know cannot be re-merged later, so its unknown columns are+/// unavoidably dropped (real tables always keep their PK across schema versions).+fn split_for_schema(+ table_name: &str,+ records: Vec<CodecRecord>,+ schema_columns: Option<&HashMap<String, Vec<String>>>,+ column_overflow: &mut HashMap<String, HashMap<String, CodecRecord>>,+) -> SplitResult {+ let known_columns: HashSet<&str> = match schema_columns {+ None => return SplitResult::Fits(records),+ Some(schema) => match schema.get(table_name) {+ None => return SplitResult::UnknownTable(records),+ Some(columns) => columns.iter().map(String::as_str).collect(),+ },+ };++ let pk_column = primary_key_for(table_name);+ let mut fitted: Vec<CodecRecord> = Vec::with_capacity(records.len());+ for row in records {+ let (known, unknown): (CodecRecord, CodecRecord) = row.into_iter().partition(|(column, _)| known_columns.contains(column.as_str()));+ if !unknown.is_empty() {+ if let Some(pk_value) = known.get(pk_column).map(row_key) {+ column_overflow.entry(table_name.to_string()).or_default().insert(pk_value, unknown);+ }+ }+ // A row with no insertable columns would produce invalid SQL (`INSERT INTO t () VALUES ()`); skip it.+ if !known.is_empty() {+ fitted.push(known);+ }+ }+ SplitResult::Fits(fitted)+}++/// Stable string key for a primary-key JSON value (strings unquoted, everything else canonical JSON).+pub(super) fn row_key(value: &serde_json::Value) -> String {+ match value.as_str() {+ Some(s) => s.to_string(),+ None => value.to_string(),+ }+}diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rsindex 40215c134..c3596cc8e 100644--- a/core/rust/src/vault_codec/mod.rs+++ b/core/rust/src/vault_codec/mod.rs@@ -24,8 +24,8 @@ use crate::error::{VaultError, VaultResult};pub use hash::{canonical_json, content_hash};pub use manifest::{- BlobEntry, BucketLayoutEntry, CanonicalizeInput, CanonicalizedVault, DataBucket, Manifest,- MaterializeInput, MaterializedTables, CodecRecord, CodecTableData,+ BlobEntry, BucketLayoutEntry, CanonicalizeInput, CanonicalizedVault, CodecOverflow, DataBucket,+ Manifest, MaterializeInput, MaterializedTables, CodecRecord, CodecTableData,};pub use types::{bucket_categories, bucket_category_for, tables_for_category, BLOB_COLUMNS, BUCKET_TABLES,@@ -47,9 +47,10 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materializematerialize::materialize_as_sqlite(input)}-/// Build a single data bucket for `category` from its tables (bucket-only push path).-pub fn extract_bucket(category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>) -> DataBucket {- canonicalize::extract_bucket(category, tables)+/// Build a single data bucket for `category` from its tables (bucket-only push path). `overflow`+/// (persisted from the last materialize) re-merges a newer writer's columns/tables so they survive.+pub fn extract_bucket(category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>, overflow: Option<&CodecOverflow>) -> DataBucket {+ canonicalize::extract_bucket(category, tables, overflow)}/// The bucket layout: every category and the tables it owns, in declaration order.@@ -159,16 +160,19 @@ pub fn materialize_as_sqlite_json(input_json: &str) -> VaultResult<String> {Ok(serde_json::to_string(&materialize_as_sqlite(input)?)?)}-/// JSON-string sibling of [`extract_bucket`]. Input: `{ "category": <str>, "tables": { <name>: [rows] } }`.+/// JSON-string sibling of [`extract_bucket`]. Input: `{ "category": <str>, "tables": { <name>: [rows] }, "overflow"?: CodecOverflow }`.pub fn extract_bucket_json(input_json: &str) -> VaultResult<String> {#[derive(serde::Deserialize)]+ #[serde(rename_all = "camelCase")]struct Input {category: String,#[serde(default)]tables: std::collections::HashMap<String, Vec<CodecRecord>>,+ #[serde(default)]+ overflow: Option<CodecOverflow>,}let input: Input = serde_json::from_str(input_json)?;- Ok(serde_json::to_string(&extract_bucket(input.category, input.tables))?)+ Ok(serde_json::to_string(&extract_bucket(input.category, input.tables, input.overflow.as_ref()))?)}/// JSON-string sibling of [`bucket_layout`]. Output: `[{ "category": <str>, "tables": [<str>] }]`.diff --git a/core/rust/src/vault_codec/tests.rs b/core/rust/src/vault_codec/tests.rsindex 22003df08..560d9ba67 100644--- a/core/rust/src/vault_codec/tests.rs+++ b/core/rust/src/vault_codec/tests.rs@@ -29,6 +29,7 @@ fn basic_input(tables: Vec<CodecTableData>) -> CanonicalizeInput {migration_id: "20250101000000_Init".to_string(),version: "2.0.0".to_string(),canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(),+ overflow: None,}}@@ -122,7 +123,7 @@ fn inline_b64_columns_survive_roundtrip() {let cell = &out.manifest.tables["Items"][0]["Secret"];assert_eq!(cell["__b64"], json!(b64(&secret)));- let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets }).unwrap();+ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: None }).unwrap();let items = re.tables.iter().find(|t| t.name == "Items").unwrap();assert_eq!(items.records[0]["Secret"]["__b64"], json!(b64(&secret)));}@@ -135,7 +136,7 @@ fn materialize_as_sqlite_emits_settings_table_and_migration_id() {CodecTableData { name: "Settings".to_string(), records: vec![row(&[("Key", json!("k"))])] },]);let out = canonicalize_from_sqlite(input).unwrap();- let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets }).unwrap();+ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: None }).unwrap();assert_eq!(re.migration_id, "20250101000000_Init");assert!(re.tables.iter().any(|t| t.name == "Settings" && t.records.len() == 1));assert!(re.tables.iter().any(|t| t.name == "Items"));@@ -150,7 +151,7 @@ fn materialize_as_sqlite_drops_skip_tables() {manifest.tables.insert("android_metadata".to_string(), vec![row(&[("locale", json!("en_US"))])]);- let re = materialize_as_sqlite(MaterializeInput { manifest, data_buckets: vec![] }).unwrap();+ let re = materialize_as_sqlite(MaterializeInput { manifest, data_buckets: vec![], schema_columns: None }).unwrap();assert!(!re.tables.iter().any(|t| t.name == "android_metadata"));}@@ -167,7 +168,7 @@ fn full_roundtrip_with_blobs_is_semantically_equal() {let out = canonicalize_from_sqlite(basic_input(tables)).unwrap();assert_eq!(out.blobs.len(), 2);- let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest.clone(), data_buckets: out.data_buckets.clone() }).unwrap();+ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest.clone(), data_buckets: out.data_buckets.clone(), schema_columns: None }).unwrap();// Items/Logos/Attachments/Settings all present (skip tables aside).for name in ["Items", "Logos", "Attachments", "Settings"] {assert!(re.tables.iter().any(|t| t.name == name), "missing table {name}");@@ -381,6 +382,109 @@ fn validate_manifest_rejects_duplicate_logo_sources() {assert!(result.failed_rules.iter().any(|r| r == "logo-sources-not-unique"));}+/// A minimal local schema map for overflow tests: Items(Id, Name), Settings(Key, Value).+fn old_client_schema() -> std::collections::HashMap<String, Vec<String>> {+ [+ ("Items".to_string(), vec!["Id".to_string(), "Name".to_string()]),+ ("Settings".to_string(), vec!["Key".to_string(), "Value".to_string()]),+ ]+ .into_iter()+ .collect()+}++#[test]+fn materialize_splits_unknown_columns_into_overflow_and_canonicalize_remerges() {+ // A newer client wrote Items.AliasEnabled; this client's schema doesn't know it. The column must+ // not reach the insert (it would crash), must land in overflow keyed by row Id, and must+ // reappear on the next canonicalize so the push doesn't drop it.+ let out = canonicalize_from_sqlite(basic_input(vec![CodecTableData {+ name: "Items".to_string(),+ records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub")), ("AliasEnabled", json!(true))])],+ }]))+ .unwrap();++ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: Some(old_client_schema()) }).unwrap();+ let items = re.tables.iter().find(|t| t.name == "Items").unwrap();+ assert!(!items.records[0].contains_key("AliasEnabled"), "unknown column filtered out of the insert set");+ assert_eq!(items.records[0]["Name"], json!("GitHub"));+ assert_eq!(re.overflow.columns["Items"]["i1"]["AliasEnabled"], json!(true));++ // The old client edits Name locally, then pushes: the overflow column re-attaches to the row.+ let mut push_input = basic_input(vec![CodecTableData {+ name: "Items".to_string(),+ records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub (renamed)"))])],+ }]);+ push_input.overflow = Some(re.overflow);+ let pushed = canonicalize_from_sqlite(push_input).unwrap();+ let item = &pushed.manifest.tables["Items"][0];+ assert_eq!(item["Name"], json!("GitHub (renamed)"));+ assert_eq!(item["AliasEnabled"], json!(true), "newer writer's column survives the old client's push");+}++#[test]+fn overflow_of_locally_deleted_row_is_dropped_on_canonicalize() {+ // The row carrying the unknown column was deleted locally: its overflow must vanish with it.+ let overflow = CodecOverflow {+ columns: [("Items".to_string(), [("gone".to_string(), row(&[("AliasEnabled", json!(true))]))].into_iter().collect())].into_iter().collect(),+ ..Default::default()+ };+ let mut input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("kept"))])] }]);+ input.overflow = Some(overflow);+ let out = canonicalize_from_sqlite(input).unwrap();+ assert!(!out.manifest.tables["Items"][0].contains_key("AliasEnabled"));+}++#[test]+fn materialize_splits_unknown_tables_into_overflow_and_canonicalize_reemits() {+ // A newer client added a whole table (manifest-level) and a whole bucket table. Neither exists+ // in this client's schema; both must round-trip through overflow back to their original place.+ let mut out = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }])).unwrap();+ out.manifest.tables.insert("NewTable".to_string(), vec![row(&[("Id", json!("n1")), ("Data", json!("x"))])]);+ out.data_buckets[0].tables.insert("Preferences".to_string(), vec![row(&[("Key", json!("p1"))])]);++ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: Some(old_client_schema()) }).unwrap();+ assert!(!re.tables.iter().any(|t| t.name == "NewTable" || t.name == "Preferences"), "unknown tables never reach the insert set");+ assert_eq!(re.overflow.tables["NewTable"].len(), 1);+ assert_eq!(re.overflow.bucket_tables["Settings"]["Preferences"].len(), 1);++ let mut push_input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }]);+ push_input.overflow = Some(re.overflow.clone());+ let pushed = canonicalize_from_sqlite(push_input).unwrap();+ assert_eq!(pushed.manifest.tables["NewTable"].len(), 1, "unknown manifest table re-emitted");+ assert_eq!(bucket_rows(&pushed, "Settings", "Preferences").len(), 1, "unknown bucket table re-emitted into its category");++ // Bucket-only push path: extract_bucket must re-emit the bucket's overflow table too.+ let bucket = extract_bucket("Settings".to_string(), [("Settings".to_string(), vec![row(&[("Key", json!("k")), ("Value", json!("v"))])])].into_iter().collect(), Some(&re.overflow));+ assert_eq!(bucket.tables["Preferences"].len(), 1);+ assert_eq!(bucket.tables["Settings"].len(), 1);+}++#[test]+fn extract_bucket_remerges_overflow_columns() {+ // A newer client added a column to Settings; a bucket-only push from this client must keep it.+ let overflow = CodecOverflow {+ columns: [("Settings".to_string(), [("theme".to_string(), row(&[("SyncScope", json!("device"))]))].into_iter().collect())].into_iter().collect(),+ ..Default::default()+ };+ let bucket = extract_bucket("Settings".to_string(), [("Settings".to_string(), vec![row(&[("Key", json!("theme")), ("Value", json!("dark"))])])].into_iter().collect(), Some(&overflow));+ assert_eq!(bucket.tables["Settings"][0]["SyncScope"], json!("device"));+ assert_eq!(bucket.tables["Settings"][0]["Value"], json!("dark"));+}++#[test]+fn materialize_without_schema_columns_passes_rows_through_verbatim() {+ // Legacy callers (no schema map) keep today's behavior exactly: nothing filtered, empty overflow.+ let out = canonicalize_from_sqlite(basic_input(vec![CodecTableData {+ name: "Items".to_string(),+ records: vec![row(&[("Id", json!("i1")), ("AliasEnabled", json!(true))])],+ }]))+ .unwrap();+ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: None }).unwrap();+ let items = re.tables.iter().find(|t| t.name == "Items").unwrap();+ assert_eq!(items.records[0]["AliasEnabled"], json!(true));+ assert!(re.overflow.is_empty());+}+#[test]fn json_siblings_roundtrip() {let input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }]);diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rsindex 873f5bb4d..6edbad373 100644--- a/core/rust/src/vault_codec/types.rs+++ b/core/rust/src/vault_codec/types.rs@@ -62,3 +62,10 @@ pub fn blob_spec_for(table_name: &str) -> Option<&'static (&'static str, &'statipub fn is_skip_table(table_name: &str) -> bool {SKIP_TABLES.contains(&table_name)}++/// The single-column primary key that addresses a row in `table_name`, shared with the merge layer+/// (see `vault_merge::types::SYNCABLE_TABLES`) so overflow keying and LWW merge agree on row identity.+/// Defaults to "Id" for tables outside the registry.+pub fn primary_key_for(table_name: &str) -> &'static str {+ crate::vault_merge::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key).unwrap_or("Id")+}diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rsindex ec2cf61aa..599c8ddb0 100644--- a/core/rust/src/vault_merge/mod.rs+++ b/core/rust/src/vault_merge/mod.rs@@ -10,8 +10,8 @@ use serde::{Deserialize, Serialize};use std::collections::HashMap;use crate::error::VaultResult;-use types::SYNCABLE_TABLES;pub use types::SYNCABLE_TABLE_NAMES;+pub use types::{SYNCABLE_TABLES, TableConfig};/// A record is a map of column names to JSON values.pub type Record = HashMap<String, serde_json::Value>;diff --git a/core/rust/src/wasm.rs b/core/rust/src/wasm.rsindex 6b392c59c..6c4ea9c3f 100644--- a/core/rust/src/wasm.rs+++ b/core/rust/src/wasm.rs@@ -145,18 +145,21 @@ pub fn vault_codec_materialize_as_sqlite_js(input: JsValue) -> Result<JsValue, Jcodec_to_js(&output)}-/// Build a single data bucket. Input: `{ category, tables: { <name>: [rows] } }`. Output: `DataBucket`.+/// Build a single data bucket. Input: `{ category, tables: { <name>: [rows] }, overflow? }`. Output: `DataBucket`.#[wasm_bindgen(js_name = vaultCodecExtractBucket)]pub fn vault_codec_extract_bucket_js(input: JsValue) -> Result<JsValue, JsValue> {#[derive(serde::Deserialize)]+ #[serde(rename_all = "camelCase")]struct Input {category: String,#[serde(default)]tables: std::collections::HashMap<String, Vec<CodecRecord>>,+ #[serde(default)]+ overflow: Option<vault_codec::CodecOverflow>,}let input: Input = serde_wasm_bindgen::from_value(input).map_err(|e| JsValue::from_str(&format!("Failed to parse extract-bucket input: {}", e)))?;- codec_to_js(&vault_codec::extract_bucket(input.category, input.tables))+ codec_to_js(&vault_codec::extract_bucket(input.category, input.tables, input.overflow.as_ref()))}/// The bucket layout: `[{ category, tables: [<name>] }]`. Source of truth for platform bucket-only sync.