Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Add overflow mechanism to codec for forward-compat (#2159)

Leendert de Borstbe50a380e1a2
diff --git a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
index c7ece257a..d59747ff0 100644
--- a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
+++ b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts
@@ -34,7 +34,7 @@ import { VaultUploadResponse as messageVaultUploadResponse } from '@/utils/types
import { type VaultMutationScope, ALL_VAULT_MUTATION_SCOPES, DEFAULT_VAULT_MUTATION_SCOPE, dirtyScopeStorageKey, isManifestScope } from '@/utils/types/VaultMutationScope';
import { VaultCodec } from '@/utils/VaultCodec';
import { vaultMergeService } from '@/utils/VaultMergeService';
-import { vaultSyncService, SERVER_MANIFEST_REVISIONS_STORAGE_KEY } from '@/utils/VaultSyncService';
+import { vaultSyncService, SERVER_MANIFEST_REVISIONS_STORAGE_KEY, CODEC_OVERFLOW_STORAGE_KEY } from '@/utils/VaultSyncService';
import { WebApiService } from '@/utils/WebApiService';
import { t } from '@/i18n/StandaloneI18n';
@@ -440,6 +440,7 @@ export async function handleClearVaultData(): Promise<messageBoolResponse> {
'local:hiddenPrivateEmailDomains',
'local:serverRevision',
SERVER_MANIFEST_REVISIONS_STORAGE_KEY,
+ CODEC_OVERFLOW_STORAGE_KEY,
'local:isDirty',
...ALL_VAULT_MUTATION_SCOPES.map(scope => dirtyScopeStorageKey(scope)),
'local:mutationSequence',
@@ -821,7 +822,8 @@ async function uploadDirtyBucketsOnly(sqliteClient: SqliteClient, scopes: VaultM
}
const tables = VaultCodec.readNamedTables(sqliteClient, spec.tables);
- const bucket = await vaultCodecExtractBucket(category, tables);
+ // Overflow re-merges a newer client's bucket tables/columns so this partial push doesn't drop them.
+ const bucket = await vaultCodecExtractBucket(category, tables, await vaultSyncService.loadOverflow(encryptionKey));
const result = await vaultSyncService.pushDataBucketOnly(bucket, encryptionKey);
if (result.status !== 'ok') {
// Conflict persisted even after the rebase-retry, let the caller run a full re-sync (status 2).
diff --git a/apps/browser-extension/src/utils/RustCore.ts b/apps/browser-extension/src/utils/RustCore.ts
index 5a335dd0f..0de7369b9 100644
--- a/apps/browser-extension/src/utils/RustCore.ts
+++ b/apps/browser-extension/src/utils/RustCore.ts
@@ -212,6 +212,18 @@ export type CodecCanonicalized = {
blobs: Record<string, CodecBlobEntry>;
};
+/**
+ * Data a newer writer put in the manifest that this client's local SQLite schema cannot hold:
+ * whole unknown manifest tables, whole unknown bucket tables (per category), and unknown columns
+ * keyed by table > row primary-key value. Produced by materialize, persisted opaquely by the
+ * platform, and fed back into canonicalize / extractBucket so a push never drops the data.
+ */
+export type CodecOverflow = {
+ tables: Record<string, Array<Record<string, unknown>>>;
+ bucketTables: Record<string, Record<string, Array<Record<string, unknown>>>>;
+ columns: Record<string, Record<string, Record<string, unknown>>>;
+};
+
/** Input for canonicalize. */
export type CodecCanonicalizeInput = {
tables: CodecTableData[];
@@ -219,10 +231,11 @@ export type CodecCanonicalizeInput = {
migrationId: string;
version: string;
canonicalizedAt: string;
+ overflow?: CodecOverflow;
};
/** Materialized tables the platform inserts into a fresh SQLite DB. */
-export type CodecMaterialized = { tables: CodecTableData[]; migrationId: string };
+export type CodecMaterialized = { tables: CodecTableData[]; migrationId: string; overflow: CodecOverflow };
/** One entry in the bucket layout: a category and the tables it owns. */
export type CodecBucketLayoutEntry = { category: string; tables: string[] };
@@ -240,18 +253,22 @@ export async function vaultCodecCanonicalizeFromSqlite(input: CodecCanonicalizeI
/**
* Materialize the manifest + its data buckets into the table set the platform inserts.
+ * `schemaColumns` (table > column names of the local schema) makes Rust split anything the schema
+ * can't hold into `overflow` instead of emitting it; the caller must persist that overflow and pass
+ * it back into the next canonicalize so the data survives the round trip.
*/
-export async function vaultCodecMaterializeAsSqlite(manifest: CodecManifest, dataBuckets: CodecDataBucket[]): Promise<CodecMaterialized> {
+export async function vaultCodecMaterializeAsSqlite(manifest: CodecManifest, dataBuckets: CodecDataBucket[], schemaColumns?: Record<string, string[]>): Promise<CodecMaterialized> {
await initRustCore();
- return core.vaultCodecMaterializeAsSqlite({ manifest, dataBuckets }) as CodecMaterialized;
+ return core.vaultCodecMaterializeAsSqlite({ manifest, dataBuckets, schemaColumns }) as CodecMaterialized;
}
/**
- * Build a single data bucket for `category` from its tables (bucket-only push path).
+ * Build a single data bucket for `category` from its tables (bucket-only push path). `overflow`
+ * (persisted from the last materialize) re-merges a newer writer's columns/tables so they survive.
*/
-export async function vaultCodecExtractBucket(category: string, tables: Record<string, Array<Record<string, unknown>>>): Promise<CodecDataBucket> {
+export async function vaultCodecExtractBucket(category: string, tables: Record<string, Array<Record<string, unknown>>>, overflow?: CodecOverflow): Promise<CodecDataBucket> {
await initRustCore();
- return core.vaultCodecExtractBucket({ category, tables }) as CodecDataBucket;
+ return core.vaultCodecExtractBucket({ category, tables, overflow }) as CodecDataBucket;
}
/**
diff --git a/apps/browser-extension/src/utils/VaultCodec.ts b/apps/browser-extension/src/utils/VaultCodec.ts
index 32b3dce5a..39c865b5d 100644
--- a/apps/browser-extension/src/utils/VaultCodec.ts
+++ b/apps/browser-extension/src/utils/VaultCodec.ts
@@ -86,6 +86,26 @@ export class VaultCodec {
}
}
+ /**
+ * The column set of the local client schema, per table: the input Rust's materialize uses to
+ * split off anything a newer writer stored that this schema cannot hold (see `CodecOverflow`).
+ * @param schemaSql - the COMPLETE_SCHEMA_SQL string for the target client version
+ */
+ public static async getSchemaColumns(schemaSql: string): Promise<Record<string, string[]>> {
+ const db = await this.createDatabase();
+ try {
+ db.run(schemaSql);
+ const tables = db.exec("SELECT name FROM sqlite_master WHERE type='table'")[0]?.values.map(v => String(v[0])) ?? [];
+ const out: Record<string, string[]> = {};
+ for (const table of tables) {
+ out[table] = db.exec(`PRAGMA table_info("${table}")`)[0]?.values.map(v => String(v[1])) ?? [];
+ }
+ return out;
+ } finally {
+ db.close();
+ }
+ }
+
/**
* Insert the materialized tables into a fresh SQLite database and export it (base64).
* @param materialized - tables + migration id produced by the Rust `materialize_as_sqlite`
@@ -94,9 +114,7 @@ export class VaultCodec {
* @returns A base64-encoded SQLite database identical (in row content) to the original.
*/
public static async insertTables(materialized: CodecMaterialized, blobs: Map<string, Uint8Array>, schemaSql: string): Promise<string> {
- const initSqlJs = (await import('sql.js')).default;
- const SQL = await initSqlJs({ locateFile: (file: string): string => `src/${file}` });
- const db = new SQL.Database();
+ const db = await this.createDatabase();
try {
// 1) Apply the schema.
@@ -184,6 +202,22 @@ export class VaultCodec {
}
}
+ /**
+ * Open a fresh in-memory sql.js database.
+ */
+ private static async createDatabase(): Promise<import('sql.js').Database> {
+ const initSqlJs = (await import('sql.js')).default;
+ const SQL = await initSqlJs({
+ /**
+ * Locates SQL.js files from the local file system.
+ * @param file - The name of the file to locate
+ * @returns The complete URL path to the file
+ */
+ locateFile: (file: string): string => `src/${file}`
+ });
+ return new SQL.Database();
+ }
+
/**
* Enumerate the names of all user tables in the database (excluding SQLite internals).
* @param sqliteClient - opened client
diff --git a/apps/browser-extension/src/utils/VaultSyncService.ts b/apps/browser-extension/src/utils/VaultSyncService.ts
index 51844c579..f4847b49c 100644
--- a/apps/browser-extension/src/utils/VaultSyncService.ts
+++ b/apps/browser-extension/src/utils/VaultSyncService.ts
@@ -11,7 +11,7 @@ import { VaultDataBucketCategory } from '@/utils/dist/core/models/vault';
import type { VaultResponse } from '@/utils/dist/core/models/webapi';
import { VaultSqlGenerator } from '@/utils/dist/core/vault';
import { EncryptionUtility } from '@/utils/EncryptionUtility';
-import {vaultCodecComputeCiphertextHash, vaultCodecCanonicalizeFromSqlite, vaultCodecGenerateUserSalt, vaultCodecUnpackPayload, vaultCodecMaterializeAsSqlite, vaultCodecPackPayload, vaultCodecValidateManifest, vaultCodecValidateDataBucket} from '@/utils/RustCore';
+import {type CodecOverflow, vaultCodecComputeCiphertextHash, vaultCodecCanonicalizeFromSqlite, vaultCodecGenerateUserSalt, vaultCodecUnpackPayload, vaultCodecMaterializeAsSqlite, vaultCodecPackPayload, vaultCodecValidateManifest, vaultCodecValidateDataBucket} from '@/utils/RustCore';
import { SqliteClient } from '@/utils/SqliteClient';
import { ServerUpdateRequiredError } from '@/utils/types/errors/ServerUpdateRequiredError';
import { VaultProcessingError } from '@/utils/types/errors/VaultProcessingError';
@@ -96,6 +96,11 @@ const bucketRevKey = (category: string): `local:${string}` => `local:vaultV2Buck
const BLOB_CACHE_STORAGE_KEY = 'local:vaultV2BlobCipherCache';
/** Hashes the server has stored (refreshed on every pull/push). */
const SERVER_HASHES_STORAGE_KEY = 'local:vaultV2ServerBlobHashes';
+/**
+ * Codec overflow persisted between pull and push: a newer client's tables/columns that this client's
+ * local SQLite schema can't hold.
+ */
+export const CODEC_OVERFLOW_STORAGE_KEY = 'local:vaultV2CodecOverflow';
/**
* The client's last-known revision per non-Main manifest (manifestId > revision), refreshed on every materialize.
*/
@@ -237,7 +242,7 @@ export class VaultSyncService {
*/
public async pull(encryptionKey: string): Promise<VaultResponse> {
/*
- * Step 1 — network fetch. Failures here (server unreachable, HTTP error, or ServerUpdateRequiredError for an
+ * Step 1: network fetch. Failures here (server unreachable, HTTP error, or ServerUpdateRequiredError for an
* outdated server) are "can't reach / must update the server" conditions, NOT vault-processing problems, so
* they propagate unchanged and the caller maps them to the appropriate "server" message.
*/
@@ -247,14 +252,14 @@ export class VaultSyncService {
devLog(`[V2Pull] Step 1/4 done: storageFormat=${snapshot.storageFormat}, manifests=${snapshot.manifests?.length ?? 0}, rootRevision=${rootManifest?.revision}, manifestBlob=${rootManifest?.blob?.length ?? 0} chars, buckets=${snapshot.buckets?.length ?? 0}, blobRefs=${rootManifest?.blobReferences?.length ?? 0}`);
/*
- * Steps 2–4 — decrypt, materialize, and re-encrypt. Any failure here is a client-side vault-processing error
+ * Steps 2–4: decrypt, materialize, and re-encrypt. Any failure here is a client-side vault-processing error
* (codec/format mismatch, integrity failure, corrupt blob, …). We wrap it in a VaultProcessingError so the UI
* can surface the real technical detail in a copyable report instead of a misleading "server unreachable".
*/
try {
if (snapshot.storageFormat === STORAGE_FORMAT_MANIFEST) {
// Manifest-v1 user: materialize the manifest + metadata + blobs into a SQLite blob, then encrypt it.
- devLog('[V2Pull] Step 2/4: manifest format — decrypting and reassembling local SQLite...');
+ devLog('[V2Pull] Step 2/4: manifest format: decrypting and reassembling local SQLite...');
const pull = await this.materializeFromSnapshot(snapshot, encryptionKey);
devLog(`[V2Pull] Step 3/4: materialized SQLite (${pull.sqliteBase64.length} base64 chars); re-encrypting for local storage...`);
const encryptedVault = await EncryptionUtility.symmetricEncrypt(pull.sqliteBase64, encryptionKey);
@@ -264,7 +269,7 @@ export class VaultSyncService {
/*
* Not-yet-migrated (sqlite-blob fallback) user: the server returned the legacy encrypted SQLite blob. It's already in the stored
- * format (encrypted SQLite), so we pass it through unchanged — the on-open schema upgrade handles the rest.
+ * format (encrypted SQLite), so we pass it through unchanged: the on-open schema upgrade handles the rest.
*/
devLog('[V2Pull] Step 2/4: legacy blob pass-through (user not yet migrated), returning as-is.');
return this.buildResponse(
@@ -274,7 +279,7 @@ export class VaultSyncService {
snapshot
);
} catch (error) {
- devError('[V2Pull] FAILED — the last logged step above is where it broke:', error);
+ devError('[V2Pull] FAILED: the last logged step above is where it broke:', error);
throw new VaultProcessingError('vault-pull', error);
}
}
@@ -406,8 +411,8 @@ export class VaultSyncService {
/*
* Decrypt referenced blobs for reassembly and prune the persisted cache to exactly the referenced set, so
- * the cache stays bounded by the current vault size. A blob that can't be reconstituted — the server couldn't
- * serve it (missing) or its ciphertext won't decrypt with the current key (stale key / corruption) — is fatal
+ * the cache stays bounded by the current vault size. A blob that can't be reconstituted: the server couldn't
+ * serve it (missing) or its ciphertext won't decrypt with the current key (stale key / corruption): is fatal
* for attachments (silently dropping bytes would propagate permanent data loss on the next push); a favicon
* only degrades cosmetically, so it's logged and skipped. The codec inserts NULL for any skipped blob ref.
*/
@@ -441,7 +446,21 @@ export class VaultSyncService {
devLog(`[V2Pull] ${blobMap.size} blobs decrypted; running codec reassembly into a fresh SQLite...`);
const sqlGen = new VaultSqlGenerator();
const schemaSql = sqlGen.getCompleteSchemaSql();
- const materialized = await vaultCodecMaterializeAsSqlite(manifest, dataBuckets);
+ const schemaColumns = await VaultCodec.getSchemaColumns(schemaSql);
+ const materialized = await vaultCodecMaterializeAsSqlite(manifest, dataBuckets, schemaColumns);
+
+ /*
+ * Anything a newer client wrote that our schema can't hold was split into overflow instead of the
+ * insert set. Persist it so push/extract re-merge it into the manifest: without this, our next
+ * push would silently delete those tables/columns for every other client.
+ */
+ await this.saveOverflow(materialized.overflow, vek);
+ const overflowTableCount = Object.keys(materialized.overflow.tables).length + Object.values(materialized.overflow.bucketTables).reduce((n, t) => n + Object.keys(t).length, 0);
+ const overflowColumnTables = Object.keys(materialized.overflow.columns);
+ if (overflowTableCount > 0 || overflowColumnTables.length > 0) {
+ devWarn(`[V2Pull] Newer-schema data preserved as overflow: ${overflowTableCount} unknown table(s), unknown columns on [${overflowColumnTables.join(', ')}]. It will round-trip on push but is not usable locally until the app is updated.`);
+ }
+
const sqliteBase64 = await VaultCodec.insertTables(materialized, blobMap, schemaSql);
devLog('[V2Pull] Codec reassembly complete.');
@@ -513,12 +532,13 @@ export class VaultSyncService {
// Read tables from the SQLite database and apply the manifest-v1 format rules.
const tables = VaultCodec.readTables(sqliteClient);
const migrationId = VaultCodec.getLatestMigrationId(sqliteClient);
- const canonicalized = await timedStage('canonicalize (incl. Rust→JS conversion)', () => vaultCodecCanonicalizeFromSqlite({
+ const canonicalized = await timedStage('canonicalize (incl. Rust→JS conversion)', async () => vaultCodecCanonicalizeFromSqlite({
tables,
userSalt,
migrationId,
version: '2.0.0',
canonicalizedAt: new Date().toISOString(),
+ overflow: await this.loadOverflow(vek),
}));
// Plaintext blob bytes held platform-side for encryption/upload.
@@ -800,6 +820,38 @@ export class VaultSyncService {
private async saveBlobCache(cache: Record<string, string>): Promise<void> {
await storage.setItem(BLOB_CACHE_STORAGE_KEY, cache);
}
+
+ /**
+ * Load the codec overflow persisted by the last pull (undefined when none): a newer client's
+ * tables/columns our local schema can't hold. Callers pass it into canonicalize/extractBucket so a
+ * push from this client re-emits that data instead of dropping it. Stored AES-GCM encrypted; an
+ * entry that no longer decrypts (key rotated, account switched) is discarded: the next pull
+ * rebuilds it from the server manifest.
+ * @param vek - the user's symmetric key
+ */
+ public async loadOverflow(vek: string): Promise<CodecOverflow | undefined> {
+ const ciphertext = (await storage.getItem(CODEC_OVERFLOW_STORAGE_KEY)) as string | null;
+ if (!ciphertext) {
+ return undefined;
+ }
+ try {
+ return JSON.parse(await EncryptionUtility.symmetricDecrypt(ciphertext, vek)) as CodecOverflow;
+ } catch {
+ devWarn('[V2Sync] Persisted codec overflow could not be decrypted with the current key, discarding it. It will be rebuilt on the next pull.');
+ await storage.removeItem(CODEC_OVERFLOW_STORAGE_KEY);
+ return undefined;
+ }
+ }
+
+ /**
+ * Persist the codec overflow produced by materialize (rebuilt wholesale on every pull),
+ * AES-GCM encrypted so no vault row data sits in extension storage as plaintext.
+ * @param overflow - the overflow split off by the last materialize
+ * @param vek - the user's symmetric key
+ */
+ private async saveOverflow(overflow: CodecOverflow, vek: string): Promise<void> {
+ await storage.setItem(CODEC_OVERFLOW_STORAGE_KEY, await EncryptionUtility.symmetricEncrypt(JSON.stringify(overflow), vek));
+ }
}
export const vaultSyncService = new VaultSyncService();
diff --git a/apps/browser-extension/tests/e2e/13-forward-compat-overflow.spec.ts b/apps/browser-extension/tests/e2e/13-forward-compat-overflow.spec.ts
new file mode 100644
index 000000000..dbc9b9c01
--- /dev/null
+++ b/apps/browser-extension/tests/e2e/13-forward-compat-overflow.spec.ts
@@ -0,0 +1,125 @@
+/**
+ * Category 13: Forward-compat overflow — no data loss (Requires API)
+ *
+ * These tests enforce the manifest-v1 forward-compatibility guarantee: when a NEWER client writes
+ * vault data this client's schema doesn't know (a new row column, a whole new table), the current
+ * extension must (a) still load the vault, and (b) carry that unknown data through its own pushes
+ * untouched instead of silently deleting it (see `CodecOverflow` in the Rust vault codec).
+ *
+ * Scenario:
+ * 1. Extension client creates a credential and syncs (migrating the account to manifest-v1)
+ * 2. A simulated "newer client" (node-side, real Rust WASM codec) pulls the manifest, injects an
+ * unknown column (`Items.AliasEnabled`) and an unknown table (`FutureFeatures`), and pushes it
+ * back as a new revision
+ * 3. The extension pulls that manifest — the vault must load and stay fully usable
+ * 4. The extension edits the credential (rename) and pushes
+ * 5. The newer client pulls again — the rename is applied AND the unknown column/table are intact
+ */
+import { test, expect, TestClient, FieldSelectors } from '../fixtures';
+import { getVaultSnapshot, openManifest, pushManifest, pollUntil, requireRootManifest, type DecryptedManifest } from '../helpers/manifest-v2-api';
+import type { TestUser } from '../helpers/test-api';
+
+test.describe.serial('13. Forward-compat overflow', () => {
+ let client: TestClient;
+ /** The account under test — the testUser fixture is per-test, so 13.1 pins it for later tests. */
+ let user: TestUser;
+ let baseApiUrl: string;
+
+ const credentialName = `Overflow Credential ${Date.now()}`;
+ const renamedCredentialName = `${credentialName} renamed`;
+ const futureTableRow = { Id: 'f1e2e000-0000-4000-8000-000000000001', Payload: 'from-the-future' };
+
+ /** Manifest revision created by the simulated newer client in 13.2. */
+ let injectedRevision: number;
+
+ test.afterAll(async () => {
+ await client?.cleanup();
+ });
+
+ test('13.1 extension client should create a credential and sync to manifest-v1', async ({ testUser, apiUrl }) => {
+ user = testUser;
+ baseApiUrl = apiUrl;
+ client = await TestClient.create();
+ await client.login(apiUrl, testUser.username, testUser.password);
+
+ await client
+ .goToVault()
+ .then((c) => c.createCredential(credentialName, '[email protected]', 'OverflowPass123!'))
+ .then((c) => c.verifyCredentialExists(credentialName))
+ .then((c) => c.triggerSync())
+ .then((c) => c.screenshot('13.1-credential-created.png'));
+
+ // The save's push runs in the background; wait until the server actually holds a manifest-v1
+ // snapshot whose Items table contains the credential.
+ const { manifest } = await openLatestManifest(apiUrl, user.token!.token, user.encryptionKey!);
+ const item = (manifest.tables.Items ?? []).find((row) => row.Name === credentialName);
+ expect(item, 'credential row should be present in the server manifest').toBeTruthy();
+ });
+
+ test('13.2 a newer client should inject an unknown column and table into the server manifest', async () => {
+ const token = user.token!.token;
+ const { manifest, root } = await openLatestManifest(baseApiUrl, token, user.encryptionKey!);
+
+ // Simulate a newer client's schema additions: a column this extension build doesn't know on an
+ // existing row, and a whole table it doesn't know at all.
+ const item = (manifest.tables.Items ?? []).find((row) => row.Name === credentialName);
+ expect(item).toBeTruthy();
+ item!.AliasEnabled = true;
+ manifest.tables.FutureFeatures = [futureTableRow];
+
+ injectedRevision = await pushManifest(baseApiUrl, token, user.username, manifest, root.revision, root.blobReferences, user.encryptionKey!);
+ expect(injectedRevision).toBeGreaterThan(root.revision);
+ });
+
+ test('13.3 extension should load the newer manifest without crashing', async () => {
+ await client
+ .triggerSync()
+ .then((c) => c.goToVault())
+ .then((c) => c.verifyCredentialExists(credentialName))
+ .then((c) => c.screenshot('13.3-newer-manifest-loaded.png'));
+ });
+
+ test('13.4 extension should edit the credential and push', async () => {
+ await client
+ .clickCredential(credentialName)
+ .then((c) => c.openEditForm())
+ .then((c) => c.fillField(FieldSelectors.ITEM_NAME, renamedCredentialName))
+ .then((c) => c.saveCredential())
+ .then((c) => c.verifyCredentialExists(renamedCredentialName))
+ .then((c) => c.triggerSync())
+ .then((c) => c.screenshot('13.4-credential-renamed.png'));
+ });
+
+ test('13.5 the newer client data should survive the extension push (no data loss)', async () => {
+ // Wait for the extension's push to land: a revision beyond the injected one, carrying the rename.
+ const manifest = await pollUntil(async (): Promise<DecryptedManifest | undefined> => {
+ const { manifest: m, root } = await openLatestManifest(baseApiUrl, user.token!.token, user.encryptionKey!);
+ const renamed = (m.tables.Items ?? []).some((row) => row.Name === renamedCredentialName);
+ return root.revision > injectedRevision && renamed ? m : undefined;
+ });
+
+ // The unknown column re-attached to the row the extension itself rewrote (rename bumps the row).
+ const item = (manifest.tables.Items ?? []).find((row) => row.Name === renamedCredentialName);
+ expect(item, 'renamed credential row should be present').toBeTruthy();
+ expect(item!.AliasEnabled, 'unknown column must survive the old-client push').toBe(true);
+
+ // The unknown table re-emitted verbatim.
+ expect(manifest.tables.FutureFeatures, 'unknown table must survive the old-client push').toEqual([futureTableRow]);
+ });
+});
+
+/**
+ * Polls until the server snapshot is manifest-v1, then decrypts and returns its root manifest.
+ *
+ * @param apiUrl - The base URL of the API
+ * @param token - Bearer token
+ * @param encryptionKey - The user's derived vault encryption key
+ * @returns The decrypted manifest and its snapshot entry
+ */
+async function openLatestManifest(apiUrl: string, token: string, encryptionKey: Uint8Array): Promise<{ manifest: DecryptedManifest; root: ReturnType<typeof requireRootManifest> }> {
+ return pollUntil(async () => {
+ const snapshot = await getVaultSnapshot(apiUrl, token);
+ const root = requireRootManifest(snapshot);
+ return { manifest: await openManifest(root.blob, encryptionKey), root };
+ });
+}
diff --git a/apps/browser-extension/tests/helpers/manifest-v2-api.ts b/apps/browser-extension/tests/helpers/manifest-v2-api.ts
new file mode 100644
index 000000000..7d3cb2215
--- /dev/null
+++ b/apps/browser-extension/tests/helpers/manifest-v2-api.ts
@@ -0,0 +1,186 @@
+/**
+ * Node-side manifest-v1 vault client for E2E tests.
+ *
+ * Lets a test act as a second, "newer" client against the v2 Vault API: pull the snapshot,
+ * decrypt + unpack the root manifest, modify it (e.g. inject columns/tables an older client's
+ * schema doesn't know), and push it back as a new revision. Pack/unpack goes through the real
+ * Rust WASM codec so the integrity envelope's canonical content hash matches exactly what the
+ * extension verifies on pull.
+ */
+
+import { createHash } from 'crypto';
+import { readFileSync } from 'fs';
+import { join } from 'path';
+
+import { initSync, vaultCodecPackPayload, vaultCodecUnpackPayload } from '../../src/utils/dist/core/rust/aliasvault_core.js';
+
+import { normalizeUsername, symmetricDecryptBytes, symmetricEncryptBytes } from './test-api';
+
+/** One manifest entry in the v2 GET snapshot. */
+export type SnapshotManifest = {
+ manifestId: string;
+ isRoot: boolean;
+ blob: string;
+ ciphertextHash: string;
+ revision: number;
+ blobReferences: Array<{ hash: string; category: string }>;
+};
+
+/** The v2 GET /Vault snapshot (fields relevant to these tests). */
+export type VaultSnapshot = {
+ status: number;
+ /** Server StorageFormat enum: 0 = legacy sqlite-blob, 1 = manifest-v1. */
+ storageFormat: number;
+ manifests?: SnapshotManifest[];
+ buckets?: Array<{ category: string; blob: string; ciphertextHash: string; revision: number }>;
+};
+
+/** A decrypted manifest-v1 manifest: known top-level fields plus free-form tables. */
+export type DecryptedManifest = {
+ schemaVersion: number;
+ migrationId: string;
+ version: string;
+ userSalt: string;
+ canonicalizedAt: string;
+ tables: Record<string, Array<Record<string, unknown>>>;
+ [key: string]: unknown;
+};
+
+let wasmInitialized = false;
+
+/**
+ * Initializes the Rust core WASM module from the extension's dist (idempotent).
+ */
+function ensureWasm(): void {
+ if (!wasmInitialized) {
+ const wasmPath = join(process.cwd(), 'src/utils/dist/core/rust/aliasvault_core_bg.wasm');
+ initSync({ module: readFileSync(wasmPath) });
+ wasmInitialized = true;
+ }
+}
+
+/**
+ * Fetches the v2 vault snapshot for the authenticated user.
+ *
+ * @param apiBaseUrl - The base URL of the API
+ * @param token - Bearer token
+ * @returns The snapshot response
+ */
+export async function getVaultSnapshot(apiBaseUrl: string, token: string): Promise<VaultSnapshot> {
+ const response = await fetch(`${apiBaseUrl.replace(/\/$/, '')}/v2/Vault`, {
+ headers: { Authorization: `Bearer ${token}` },
+ });
+ if (!response.ok) {
+ throw new Error(`GET /v2/Vault failed with status ${response.status}: ${await response.text()}`);
+ }
+ return (await response.json()) as VaultSnapshot;
+}
+
+/**
+ * Returns the root manifest entry of a snapshot, or throws if the user isn't on manifest-v1 yet.
+ *
+ * @param snapshot - The v2 snapshot
+ * @returns The root manifest entry
+ */
+export function requireRootManifest(snapshot: VaultSnapshot): SnapshotManifest {
+ const root = (snapshot.manifests ?? []).find((m) => m.isRoot);
+ if (snapshot.storageFormat !== 1 || !root?.blob) {
+ throw new Error(`Snapshot is not manifest-v1 yet (storageFormat=${snapshot.storageFormat}, manifests=${snapshot.manifests?.length ?? 0}).`);
+ }
+ return root;
+}
+
+/**
+ * Decrypts and unpacks a manifest blob into its JSON object (content hash verified by the codec).
+ *
+ * @param blobBase64 - The encrypted manifest blob from the snapshot
+ * @param encryptionKey - The user's derived vault encryption key
+ * @returns The decrypted manifest object
+ */
+export async function openManifest(blobBase64: string, encryptionKey: Uint8Array): Promise<DecryptedManifest> {
+ ensureWasm();
+ const packedBytes = await symmetricDecryptBytes(blobBase64, encryptionKey);
+ return JSON.parse(vaultCodecUnpackPayload(packedBytes)) as DecryptedManifest;
+}
+
+/**
+ * Packs (envelope + canonical content hash + gzip via the Rust codec), encrypts, and uploads a
+ * manifest as a new revision, exactly like the extension's full push does.
+ *
+ * @param apiBaseUrl - The base URL of the API
+ * @param token - Bearer token
+ * @param username - The vault owner's username
+ * @param manifest - The (modified) manifest object to upload
+ * @param currentRevision - The revision this upload is based on (server assigns currentRevision + 1)
+ * @param blobReferences - Blob references to carry over to the new revision
+ * @param encryptionKey - The user's derived vault encryption key
+ * @returns The new manifest revision number assigned by the server
+ */
+export async function pushManifest(
+ apiBaseUrl: string,
+ token: string,
+ username: string,
+ manifest: DecryptedManifest,
+ currentRevision: number,
+ blobReferences: Array<{ hash: string; category: string }>,
+ encryptionKey: Uint8Array
+): Promise<number> {
+ ensureWasm();
+ const packedBytes = vaultCodecPackPayload(JSON.stringify(manifest));
+ const manifestBlob = await symmetricEncryptBytes(packedBytes, encryptionKey);
+ const manifestCiphertextHash = createHash('sha256').update(Buffer.from(manifestBlob, 'base64')).digest('hex');
+
+ const payload = {
+ username: normalizeUsername(username),
+ version: manifest.version,
+ manifestBlob,
+ manifestCiphertextHash,
+ currentManifestRevision: currentRevision,
+ credentialsCount: (manifest.tables.Items ?? []).length,
+ buckets: [],
+ newBlobs: [],
+ blobReferences,
+ emailRouting: { emailAddressList: [] },
+ encryptionPublicKey: '',
+ };
+
+ const response = await fetch(`${apiBaseUrl.replace(/\/$/, '')}/v2/Vault`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` },
+ body: JSON.stringify(payload),
+ });
+ if (!response.ok) {
+ throw new Error(`POST /v2/Vault failed with status ${response.status}: ${await response.text()}`);
+ }
+
+ const result = (await response.json()) as { status: number; newManifestRevision: number; missingBlobHashes?: string[] };
+ if (result.status !== 0 || (result.missingBlobHashes?.length ?? 0) > 0) {
+ throw new Error(`Manifest push rejected: status=${result.status}, missingBlobs=${result.missingBlobHashes?.join(',') ?? 'none'}`);
+ }
+ return result.newManifestRevision;
+}
+
+/**
+ * Polls until `predicate` returns a truthy value or the timeout elapses.
+ *
+ * @param predicate - Async condition; return a truthy value to stop polling
+ * @param timeoutMs - Total time budget in milliseconds
+ * @param intervalMs - Delay between attempts in milliseconds
+ * @returns The first truthy value the predicate returned
+ */
+export async function pollUntil<T>(predicate: () => Promise<T | undefined | false>, timeoutMs = 20000, intervalMs = 1000): Promise<T> {
+ const deadline = Date.now() + timeoutMs;
+ let lastError: unknown;
+ while (Date.now() < deadline) {
+ try {
+ const result = await predicate();
+ if (result) {
+ return result;
+ }
+ } catch (e) {
+ lastError = e;
+ }
+ await new Promise((resolve) => setTimeout(resolve, intervalMs));
+ }
+ throw new Error(`pollUntil timed out after ${timeoutMs}ms${lastError ? `; last error: ${lastError instanceof Error ? lastError.message : String(lastError)}` : ''}`);
+}
diff --git a/apps/browser-extension/tests/helpers/test-api.ts b/apps/browser-extension/tests/helpers/test-api.ts
index 8ae54d3ee..6174d3163 100644
--- a/apps/browser-extension/tests/helpers/test-api.ts
+++ b/apps/browser-extension/tests/helpers/test-api.ts
@@ -40,6 +40,8 @@ export type TestUser = {
token?: TokenModel;
/** TOTP secret if 2FA is enabled */
totpSecret?: string;
+ /** Argon2Id-derived vault encryption key, for tests that decrypt/re-encrypt vault blobs API-side. */
+ encryptionKey?: Uint8Array;
};
/**
@@ -190,6 +192,38 @@ async function symmetricEncrypt(plaintext: string, keyBytes: Uint8Array): Promis
return Buffer.from(combined).toString('base64');
}
+/**
+ * Encrypts raw bytes using AES-GCM (matching the browser extension's `symmetricEncryptBytes`).
+ *
+ * @param plaintextBytes - The plaintext bytes to encrypt
+ * @param keyBytes - The 256-bit encryption key as Uint8Array
+ * @returns Base64-encoded ciphertext (IV prepended to ciphertext)
+ */
+export async function symmetricEncryptBytes(plaintextBytes: Uint8Array, keyBytes: Uint8Array): Promise<string> {
+ const key = await webcrypto.subtle.importKey('raw', keyBytes, { name: 'AES-GCM', length: 256 }, false, ['encrypt']);
+ const iv = webcrypto.getRandomValues(new Uint8Array(12));
+ const ciphertext = await webcrypto.subtle.encrypt({ name: 'AES-GCM', iv }, key, plaintextBytes);
+
+ const combined = new Uint8Array(iv.length + ciphertext.byteLength);
+ combined.set(iv);
+ combined.set(new Uint8Array(ciphertext), iv.length);
+ return Buffer.from(combined).toString('base64');
+}
+
+/**
+ * Decrypts an AES-GCM ciphertext produced by the extension's `symmetricEncrypt(Bytes)` (IV-prefixed).
+ *
+ * @param base64Ciphertext - Base64-encoded ciphertext (12-byte IV prepended)
+ * @param keyBytes - The 256-bit encryption key as Uint8Array
+ * @returns The decrypted plaintext bytes
+ */
+export async function symmetricDecryptBytes(base64Ciphertext: string, keyBytes: Uint8Array): Promise<Uint8Array> {
+ const combined = new Uint8Array(Buffer.from(base64Ciphertext, 'base64'));
+ const key = await webcrypto.subtle.importKey('raw', keyBytes, { name: 'AES-GCM', length: 256 }, false, ['decrypt']);
+ const plaintext = await webcrypto.subtle.decrypt({ name: 'AES-GCM', iv: combined.slice(0, 12) }, key, combined.slice(12));
+ return new Uint8Array(plaintext);
+}
+
/**
* Creates an empty vault database with the latest schema.
*
@@ -302,7 +336,9 @@ async function uploadInitialVault(
username: string,
encryptionKey: Uint8Array
): Promise<void> {
- const baseUrl = apiBaseUrl.replace(/\/$/, '') + '/v2/';
+ // Intentionally the v1 endpoint: the initial vault is a legacy sqlite-blob (the v2 endpoint is
+ // manifest-only). This mirrors a pre-manifest account; the extension migrates it on first save.
+ const baseUrl = apiBaseUrl.replace(/\/$/, '') + '/v1/';
// Create an empty vault database
const vaultBase64 = createEmptyVaultDatabase();
@@ -359,7 +395,7 @@ export async function registerTestUser(
apiBaseUrl: string,
username: string,
password: string
-): Promise<TokenModel> {
+): Promise<{ tokenModel: TokenModel; encryptionKey: Uint8Array }> {
// Normalize the API URL
const baseUrl = apiBaseUrl.replace(/\/$/, '') + '/v2/';
@@ -392,7 +428,7 @@ export async function registerTestUser(
// Upload initial empty vault
await uploadInitialVault(apiBaseUrl, tokenModel.token, username, encryptionKey);
- return tokenModel;
+ return { tokenModel, encryptionKey };
}
/**
@@ -405,12 +441,13 @@ export async function createTestUser(apiBaseUrl: string): Promise<TestUser> {
const username = generateTestUsername();
const password = generateTestPassword();
- const token = await registerTestUser(apiBaseUrl, username, password);
+ const { tokenModel, encryptionKey } = await registerTestUser(apiBaseUrl, username, password);
return {
username,
password,
- token,
+ token: tokenModel,
+ encryptionKey,
};
}
@@ -519,15 +556,16 @@ export async function createTestUserWith2FA(apiBaseUrl: string): Promise<TestUse
const username = generateTestUsername();
const password = generateTestPassword();
- const token = await registerTestUser(apiBaseUrl, username, password);
+ const { tokenModel, encryptionKey } = await registerTestUser(apiBaseUrl, username, password);
// Enable 2FA for the user
- const totpSecret = await enableTwoFactor(apiBaseUrl, token.token);
+ const totpSecret = await enableTwoFactor(apiBaseUrl, tokenModel.token);
return {
username,
password,
- token,
+ token: tokenModel,
totpSecret,
+ encryptionKey,
};
}
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rs
index 93930a316..a05e62457 100644
--- a/core/rust/src/vault_codec/canonicalize.rs
+++ b/core/rust/src/vault_codec/canonicalize.rs
@@ -18,8 +18,9 @@ use serde_json::json;
use super::hash::salted_blob_hash;
use super::logos::dedupe_logos_by_source;
-use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedVault, DataBucket, Manifest, CodecRecord};
-use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_skip_table, SCHEMA_VERSION};
+use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedVault, CodecOverflow, DataBucket, Manifest, CodecRecord};
+use super::materialize::row_key;
+use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_skip_table, primary_key_for, SCHEMA_VERSION};
use crate::error::VaultResult;
/// Canonicalize normalized tables into the split resources: the manifest, one data bucket per declared
@@ -36,6 +37,15 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
all_tables.entry(table.name.clone()).or_default().extend(table.records.iter().cloned());
}
+ // Re-merge overflow from the last materialize (a newer writer's tables/columns this client's
+ // local schema couldn't hold) so this push doesn't drop them. See `CodecOverflow`.
+ let overflow = input.overflow.unwrap_or_default();
+ remerge_overflow_columns(&mut all_tables, &overflow);
+ for (name, rows) in &overflow.tables {
+ // Local rows win if the table somehow exists locally now (e.g. client upgraded since the pull).
+ all_tables.entry(name.clone()).or_insert_with(|| rows.clone());
+ }
+
// Collapse duplicate Logos rows sharing a Source (see `logos` module) and remap Items.LogoId so
// the manifest we write is free of any UNIQUE(Source) collision that would break materialize.
dedupe_logos_by_source(&mut all_tables);
@@ -73,6 +83,18 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
manifest_tables.insert(name, out_rows);
}
+ /*
+ * Re-emit overflow bucket tables into their original categories. A category this client's
+ * BUCKET_TABLES doesn't declare yet is added as its own bucket, preserving the newer writer's
+ * bucket structure verbatim.
+ */
+ for (category, ov_tables) in &overflow.bucket_tables {
+ let bucket = bucketed.entry(category.clone()).or_default();
+ for (name, rows) in ov_tables {
+ bucket.entry(name.clone()).or_insert_with(|| rows.clone());
+ }
+ }
+
let manifest = Manifest {
schema_version: SCHEMA_VERSION,
migration_id: input.migration_id,
@@ -124,9 +146,47 @@ fn extract_blob_cell(
json!({ "__blobRef": hash, "__blobKind": kind })
}
-/// Build a single data bucket for `category` from its already-normalized tables (name > rows).
-/// The bucket-only push path (a bucket changed but the manifest didn't). Input rows are
+/// Re-attach overflow columns (split off at materialize time because the local schema didn't know
+/// them) to their rows, matched by the table's primary-key value. A row deleted locally simply has
+/// no match and its overflow is dropped with it, which is the correct outcome. Locally-known
+/// columns always win on a (theoretical) name collision.
+fn remerge_overflow_columns(tables: &mut HashMap<String, Vec<CodecRecord>>, overflow: &CodecOverflow) {
+ for (table_name, by_pk) in &overflow.columns {
+ let rows = match tables.get_mut(table_name) {
+ Some(rows) => rows,
+ None => continue,
+ };
+ let pk_column = primary_key_for(table_name);
+ for row in rows {
+ let pk_value = match row.get(pk_column).map(row_key) {
+ Some(v) => v,
+ None => continue,
+ };
+ if let Some(extra_columns) = by_pk.get(&pk_value) {
+ for (column, value) in extra_columns {
+ row.entry(column.clone()).or_insert_with(|| value.clone());
+ }
+ }
+ }
+ }
+}
+
+/// Build a single data bucket for `category` from its already-normalized tables (name > rows).
+/// The bucket-only push path (a bucket changed but the manifest didn't). Input rows are
/// already normalized by the platform read logic.
-pub fn extract_bucket(category: String, tables: HashMap<String, Vec<CodecRecord>>) -> DataBucket {
+///
+/// `overflow` (persisted from the last materialize) is re-merged the same way the full
+/// canonicalize does it: unknown columns re-attach to this bucket's rows, and whole unknown tables
+/// recorded under this bucket's category are re-emitted, so a bucket-only push from an older
+/// client never drops a newer writer's data either.
+pub fn extract_bucket(category: String, mut tables: HashMap<String, Vec<CodecRecord>>, overflow: Option<&CodecOverflow>) -> DataBucket {
+ if let Some(overflow) = overflow {
+ remerge_overflow_columns(&mut tables, overflow);
+ if let Some(ov_tables) = overflow.bucket_tables.get(&category) {
+ for (name, rows) in ov_tables {
+ tables.entry(name.clone()).or_insert_with(|| rows.clone());
+ }
+ }
+ }
DataBucket::new(category, tables)
}
diff --git a/core/rust/src/vault_codec/manifest.rs b/core/rust/src/vault_codec/manifest.rs
index ed3ebc7f6..206e4fd4d 100644
--- a/core/rust/src/vault_codec/manifest.rs
+++ b/core/rust/src/vault_codec/manifest.rs
@@ -89,13 +89,45 @@ pub struct CodecTableData {
pub records: Vec<CodecRecord>,
}
+/// Data a newer writer put in the manifest that this client's local SQLite schema cannot hold.
+///
+/// Materialize splits it off (so inserts don't crash on unknown tables/columns) and canonicalize
+/// re-merges it (so this client's next push never drops it). The platform persists this value
+/// opaquely between pull and push; it is rebuilt from scratch on every pull, so it tracks the same
+/// staleness/LWW semantics as the rest of the row data.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CodecOverflow {
+ /// Whole manifest tables the local schema doesn't know: name > rows, re-emitted verbatim.
+ #[serde(default)]
+ pub tables: HashMap<String, Vec<CodecRecord>>,
+ /// Whole bucket tables the local schema doesn't know: category > (name > rows). Kept per
+ /// category so both full pushes and bucket-only pushes re-emit them into the right bucket.
+ #[serde(default)]
+ pub bucket_tables: HashMap<String, HashMap<String, Vec<CodecRecord>>>,
+ /// Unknown columns split off rows of known tables: table > row primary-key value > {column: value}.
+ #[serde(default)]
+ pub columns: HashMap<String, HashMap<String, CodecRecord>>,
+}
+
+impl CodecOverflow {
+ /// True when nothing was split off (the common case: reader and writer share a schema).
+ pub fn is_empty(&self) -> bool {
+ self.tables.is_empty() && self.bucket_tables.is_empty() && self.columns.is_empty()
+ }
+}
+
/// Materialized tables the platform inserts into a fresh schema DB. Blob columns carry
-/// `{ "__blobRef": hash }`; inline byte columns carry `{ "__b64": ... }`.
+/// `{ "__blobRef": hash }`; inline byte columns carry `{ "__b64": ... }`. `overflow` holds whatever
+/// the local schema couldn't accept (see [`CodecOverflow`]); the platform must persist it and feed
+/// it back into the next canonicalize.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MaterializedTables {
pub tables: Vec<CodecTableData>,
pub migration_id: String,
+ #[serde(default)]
+ pub overflow: CodecOverflow,
}
/// Input for [`crate::vault_codec::canonicalize_from_sqlite`].
@@ -108,6 +140,10 @@ pub struct CanonicalizeInput {
#[serde(default = "default_version")]
pub version: String,
pub canonicalized_at: String,
+ /// Overflow persisted from the last materialize (see [`CodecOverflow`]); re-merged into the
+ /// output so a client with an older schema never drops a newer writer's tables/columns.
+ #[serde(default)]
+ pub overflow: Option<CodecOverflow>,
}
fn default_version() -> String {
@@ -121,4 +157,9 @@ pub struct MaterializeInput {
pub manifest: Manifest,
#[serde(default)]
pub data_buckets: Vec<DataBucket>,
+ /// The caller's local SQLite schema: table > column names. When present, rows are filtered down
+ /// to what the schema can hold and the remainder lands in [`MaterializedTables::overflow`];
+ /// when absent, rows pass through verbatim (legacy behavior — unknown columns crash the insert).
+ #[serde(default)]
+ pub schema_columns: Option<HashMap<String, Vec<String>>>,
}
diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rs
index fc5866108..5497b97fa 100644
--- a/core/rust/src/vault_codec/materialize.rs
+++ b/core/rust/src/vault_codec/materialize.rs
@@ -6,36 +6,108 @@
//!
//! Rust does not handle actual SQLite operations. It only emits the destination table rows as-is
//! to the caller and let the caller handle the actual SQLite database creation and data insertion.
+//!
+//! Forward compatibility: when the caller supplies its local schema (`schema_columns`), anything a
+//! newer writer put in the manifest that this schema cannot hold — whole unknown tables or unknown
+//! columns on known tables — is split into [`CodecOverflow`] instead of being emitted (which would
+//! crash the platform insert). The platform persists the overflow and passes it back into
+//! `canonicalize_from_sqlite`, which re-merges it so this client's next push never drops the data.
+
+use std::collections::{HashMap, HashSet};
-use super::manifest::{MaterializeInput, MaterializedTables, CodecTableData};
-use super::types::is_skip_table;
+use super::manifest::{CodecOverflow, CodecRecord, CodecTableData, MaterializeInput, MaterializedTables};
+use super::types::{is_skip_table, primary_key_for};
use crate::error::VaultResult;
/// Materialize the manifest + its data buckets into the table set the platform inserts.
pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<MaterializedTables> {
- let MaterializeInput { manifest, data_buckets } = input;
+ let MaterializeInput { manifest, data_buckets, schema_columns } = input;
+ let mut overflow = CodecOverflow::default();
let mut tables: Vec<CodecTableData> = Vec::with_capacity(manifest.tables.len() + data_buckets.len());
for (name, records) in manifest.tables {
if is_skip_table(&name) {
continue;
}
- tables.push(CodecTableData { name, records });
+ match split_for_schema(&name, records, schema_columns.as_ref(), &mut overflow.columns) {
+ SplitResult::Fits(records) => tables.push(CodecTableData { name, records }),
+ SplitResult::UnknownTable(records) => {
+ overflow.tables.insert(name, records);
+ }
+ }
}
- // Reconstitute every data bucket's tables back into the flat set.
+ // Reconstitute every data bucket's tables back into the flat set. Unknown bucket tables keep
+ // their category so canonicalize / extract_bucket can re-emit them into the right bucket.
for bucket in data_buckets {
for (name, records) in bucket.tables {
if is_skip_table(&name) {
continue;
}
- tables.push(CodecTableData { name, records });
+ match split_for_schema(&name, records, schema_columns.as_ref(), &mut overflow.columns) {
+ SplitResult::Fits(records) => tables.push(CodecTableData { name, records }),
+ SplitResult::UnknownTable(records) => {
+ overflow.bucket_tables.entry(bucket.category.clone()).or_default().insert(name, records);
+ }
+ }
}
}
Ok(MaterializedTables {
tables,
migration_id: manifest.migration_id,
+ overflow,
})
}
+
+/// Outcome of fitting one table's rows to the caller's schema.
+enum SplitResult {
+ /// Rows the schema can insert (unknown columns already split off into overflow).
+ Fits(Vec<CodecRecord>),
+ /// The schema has no such table at all; the whole table belongs in overflow.
+ UnknownTable(Vec<CodecRecord>),
+}
+
+/// Fit `records` to the caller's schema. Without a schema, rows pass through verbatim. With one,
+/// unknown columns are stashed in `column_overflow` keyed by the row's primary-key value; a row
+/// whose primary key the schema doesn't know cannot be re-merged later, so its unknown columns are
+/// unavoidably dropped (real tables always keep their PK across schema versions).
+fn split_for_schema(
+ table_name: &str,
+ records: Vec<CodecRecord>,
+ schema_columns: Option<&HashMap<String, Vec<String>>>,
+ column_overflow: &mut HashMap<String, HashMap<String, CodecRecord>>,
+) -> SplitResult {
+ let known_columns: HashSet<&str> = match schema_columns {
+ None => return SplitResult::Fits(records),
+ Some(schema) => match schema.get(table_name) {
+ None => return SplitResult::UnknownTable(records),
+ Some(columns) => columns.iter().map(String::as_str).collect(),
+ },
+ };
+
+ let pk_column = primary_key_for(table_name);
+ let mut fitted: Vec<CodecRecord> = Vec::with_capacity(records.len());
+ for row in records {
+ let (known, unknown): (CodecRecord, CodecRecord) = row.into_iter().partition(|(column, _)| known_columns.contains(column.as_str()));
+ if !unknown.is_empty() {
+ if let Some(pk_value) = known.get(pk_column).map(row_key) {
+ column_overflow.entry(table_name.to_string()).or_default().insert(pk_value, unknown);
+ }
+ }
+ // A row with no insertable columns would produce invalid SQL (`INSERT INTO t () VALUES ()`); skip it.
+ if !known.is_empty() {
+ fitted.push(known);
+ }
+ }
+ SplitResult::Fits(fitted)
+}
+
+/// Stable string key for a primary-key JSON value (strings unquoted, everything else canonical JSON).
+pub(super) fn row_key(value: &serde_json::Value) -> String {
+ match value.as_str() {
+ Some(s) => s.to_string(),
+ None => value.to_string(),
+ }
+}
diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rs
index 40215c134..c3596cc8e 100644
--- a/core/rust/src/vault_codec/mod.rs
+++ b/core/rust/src/vault_codec/mod.rs
@@ -24,8 +24,8 @@ use crate::error::{VaultError, VaultResult};
pub use hash::{canonical_json, content_hash};
pub use manifest::{
- BlobEntry, BucketLayoutEntry, CanonicalizeInput, CanonicalizedVault, DataBucket, Manifest,
- MaterializeInput, MaterializedTables, CodecRecord, CodecTableData,
+ BlobEntry, BucketLayoutEntry, CanonicalizeInput, CanonicalizedVault, CodecOverflow, DataBucket,
+ Manifest, MaterializeInput, MaterializedTables, CodecRecord, CodecTableData,
};
pub use types::{
bucket_categories, bucket_category_for, tables_for_category, BLOB_COLUMNS, BUCKET_TABLES,
@@ -47,9 +47,10 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
materialize::materialize_as_sqlite(input)
}
-/// Build a single data bucket for `category` from its tables (bucket-only push path).
-pub fn extract_bucket(category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>) -> DataBucket {
- canonicalize::extract_bucket(category, tables)
+/// Build a single data bucket for `category` from its tables (bucket-only push path). `overflow`
+/// (persisted from the last materialize) re-merges a newer writer's columns/tables so they survive.
+pub fn extract_bucket(category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>, overflow: Option<&CodecOverflow>) -> DataBucket {
+ canonicalize::extract_bucket(category, tables, overflow)
}
/// The bucket layout: every category and the tables it owns, in declaration order.
@@ -159,16 +160,19 @@ pub fn materialize_as_sqlite_json(input_json: &str) -> VaultResult<String> {
Ok(serde_json::to_string(&materialize_as_sqlite(input)?)?)
}
-/// JSON-string sibling of [`extract_bucket`]. Input: `{ "category": <str>, "tables": { <name>: [rows] } }`.
+/// JSON-string sibling of [`extract_bucket`]. Input: `{ "category": <str>, "tables": { <name>: [rows] }, "overflow"?: CodecOverflow }`.
pub fn extract_bucket_json(input_json: &str) -> VaultResult<String> {
#[derive(serde::Deserialize)]
+ #[serde(rename_all = "camelCase")]
struct Input {
category: String,
#[serde(default)]
tables: std::collections::HashMap<String, Vec<CodecRecord>>,
+ #[serde(default)]
+ overflow: Option<CodecOverflow>,
}
let input: Input = serde_json::from_str(input_json)?;
- Ok(serde_json::to_string(&extract_bucket(input.category, input.tables))?)
+ Ok(serde_json::to_string(&extract_bucket(input.category, input.tables, input.overflow.as_ref()))?)
}
/// JSON-string sibling of [`bucket_layout`]. Output: `[{ "category": <str>, "tables": [<str>] }]`.
diff --git a/core/rust/src/vault_codec/tests.rs b/core/rust/src/vault_codec/tests.rs
index 22003df08..560d9ba67 100644
--- a/core/rust/src/vault_codec/tests.rs
+++ b/core/rust/src/vault_codec/tests.rs
@@ -29,6 +29,7 @@ fn basic_input(tables: Vec<CodecTableData>) -> CanonicalizeInput {
migration_id: "20250101000000_Init".to_string(),
version: "2.0.0".to_string(),
canonicalized_at: "2026-01-01T00:00:00.000Z".to_string(),
+ overflow: None,
}
}
@@ -122,7 +123,7 @@ fn inline_b64_columns_survive_roundtrip() {
let cell = &out.manifest.tables["Items"][0]["Secret"];
assert_eq!(cell["__b64"], json!(b64(&secret)));
- let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets }).unwrap();
+ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: None }).unwrap();
let items = re.tables.iter().find(|t| t.name == "Items").unwrap();
assert_eq!(items.records[0]["Secret"]["__b64"], json!(b64(&secret)));
}
@@ -135,7 +136,7 @@ fn materialize_as_sqlite_emits_settings_table_and_migration_id() {
CodecTableData { name: "Settings".to_string(), records: vec![row(&[("Key", json!("k"))])] },
]);
let out = canonicalize_from_sqlite(input).unwrap();
- let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets }).unwrap();
+ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: None }).unwrap();
assert_eq!(re.migration_id, "20250101000000_Init");
assert!(re.tables.iter().any(|t| t.name == "Settings" && t.records.len() == 1));
assert!(re.tables.iter().any(|t| t.name == "Items"));
@@ -150,7 +151,7 @@ fn materialize_as_sqlite_drops_skip_tables() {
manifest
.tables
.insert("android_metadata".to_string(), vec![row(&[("locale", json!("en_US"))])]);
- let re = materialize_as_sqlite(MaterializeInput { manifest, data_buckets: vec![] }).unwrap();
+ let re = materialize_as_sqlite(MaterializeInput { manifest, data_buckets: vec![], schema_columns: None }).unwrap();
assert!(!re.tables.iter().any(|t| t.name == "android_metadata"));
}
@@ -167,7 +168,7 @@ fn full_roundtrip_with_blobs_is_semantically_equal() {
let out = canonicalize_from_sqlite(basic_input(tables)).unwrap();
assert_eq!(out.blobs.len(), 2);
- let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest.clone(), data_buckets: out.data_buckets.clone() }).unwrap();
+ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest.clone(), data_buckets: out.data_buckets.clone(), schema_columns: None }).unwrap();
// Items/Logos/Attachments/Settings all present (skip tables aside).
for name in ["Items", "Logos", "Attachments", "Settings"] {
assert!(re.tables.iter().any(|t| t.name == name), "missing table {name}");
@@ -381,6 +382,109 @@ fn validate_manifest_rejects_duplicate_logo_sources() {
assert!(result.failed_rules.iter().any(|r| r == "logo-sources-not-unique"));
}
+/// A minimal local schema map for overflow tests: Items(Id, Name), Settings(Key, Value).
+fn old_client_schema() -> std::collections::HashMap<String, Vec<String>> {
+ [
+ ("Items".to_string(), vec!["Id".to_string(), "Name".to_string()]),
+ ("Settings".to_string(), vec!["Key".to_string(), "Value".to_string()]),
+ ]
+ .into_iter()
+ .collect()
+}
+
+#[test]
+fn materialize_splits_unknown_columns_into_overflow_and_canonicalize_remerges() {
+ // A newer client wrote Items.AliasEnabled; this client's schema doesn't know it. The column must
+ // not reach the insert (it would crash), must land in overflow keyed by row Id, and must
+ // reappear on the next canonicalize so the push doesn't drop it.
+ let out = canonicalize_from_sqlite(basic_input(vec![CodecTableData {
+ name: "Items".to_string(),
+ records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub")), ("AliasEnabled", json!(true))])],
+ }]))
+ .unwrap();
+
+ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: Some(old_client_schema()) }).unwrap();
+ let items = re.tables.iter().find(|t| t.name == "Items").unwrap();
+ assert!(!items.records[0].contains_key("AliasEnabled"), "unknown column filtered out of the insert set");
+ assert_eq!(items.records[0]["Name"], json!("GitHub"));
+ assert_eq!(re.overflow.columns["Items"]["i1"]["AliasEnabled"], json!(true));
+
+ // The old client edits Name locally, then pushes: the overflow column re-attaches to the row.
+ let mut push_input = basic_input(vec![CodecTableData {
+ name: "Items".to_string(),
+ records: vec![row(&[("Id", json!("i1")), ("Name", json!("GitHub (renamed)"))])],
+ }]);
+ push_input.overflow = Some(re.overflow);
+ let pushed = canonicalize_from_sqlite(push_input).unwrap();
+ let item = &pushed.manifest.tables["Items"][0];
+ assert_eq!(item["Name"], json!("GitHub (renamed)"));
+ assert_eq!(item["AliasEnabled"], json!(true), "newer writer's column survives the old client's push");
+}
+
+#[test]
+fn overflow_of_locally_deleted_row_is_dropped_on_canonicalize() {
+ // The row carrying the unknown column was deleted locally: its overflow must vanish with it.
+ let overflow = CodecOverflow {
+ columns: [("Items".to_string(), [("gone".to_string(), row(&[("AliasEnabled", json!(true))]))].into_iter().collect())].into_iter().collect(),
+ ..Default::default()
+ };
+ let mut input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("kept"))])] }]);
+ input.overflow = Some(overflow);
+ let out = canonicalize_from_sqlite(input).unwrap();
+ assert!(!out.manifest.tables["Items"][0].contains_key("AliasEnabled"));
+}
+
+#[test]
+fn materialize_splits_unknown_tables_into_overflow_and_canonicalize_reemits() {
+ // A newer client added a whole table (manifest-level) and a whole bucket table. Neither exists
+ // in this client's schema; both must round-trip through overflow back to their original place.
+ let mut out = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }])).unwrap();
+ out.manifest.tables.insert("NewTable".to_string(), vec![row(&[("Id", json!("n1")), ("Data", json!("x"))])]);
+ out.data_buckets[0].tables.insert("Preferences".to_string(), vec![row(&[("Key", json!("p1"))])]);
+
+ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: Some(old_client_schema()) }).unwrap();
+ assert!(!re.tables.iter().any(|t| t.name == "NewTable" || t.name == "Preferences"), "unknown tables never reach the insert set");
+ assert_eq!(re.overflow.tables["NewTable"].len(), 1);
+ assert_eq!(re.overflow.bucket_tables["Settings"]["Preferences"].len(), 1);
+
+ let mut push_input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }]);
+ push_input.overflow = Some(re.overflow.clone());
+ let pushed = canonicalize_from_sqlite(push_input).unwrap();
+ assert_eq!(pushed.manifest.tables["NewTable"].len(), 1, "unknown manifest table re-emitted");
+ assert_eq!(bucket_rows(&pushed, "Settings", "Preferences").len(), 1, "unknown bucket table re-emitted into its category");
+
+ // Bucket-only push path: extract_bucket must re-emit the bucket's overflow table too.
+ let bucket = extract_bucket("Settings".to_string(), [("Settings".to_string(), vec![row(&[("Key", json!("k")), ("Value", json!("v"))])])].into_iter().collect(), Some(&re.overflow));
+ assert_eq!(bucket.tables["Preferences"].len(), 1);
+ assert_eq!(bucket.tables["Settings"].len(), 1);
+}
+
+#[test]
+fn extract_bucket_remerges_overflow_columns() {
+ // A newer client added a column to Settings; a bucket-only push from this client must keep it.
+ let overflow = CodecOverflow {
+ columns: [("Settings".to_string(), [("theme".to_string(), row(&[("SyncScope", json!("device"))]))].into_iter().collect())].into_iter().collect(),
+ ..Default::default()
+ };
+ let bucket = extract_bucket("Settings".to_string(), [("Settings".to_string(), vec![row(&[("Key", json!("theme")), ("Value", json!("dark"))])])].into_iter().collect(), Some(&overflow));
+ assert_eq!(bucket.tables["Settings"][0]["SyncScope"], json!("device"));
+ assert_eq!(bucket.tables["Settings"][0]["Value"], json!("dark"));
+}
+
+#[test]
+fn materialize_without_schema_columns_passes_rows_through_verbatim() {
+ // Legacy callers (no schema map) keep today's behavior exactly: nothing filtered, empty overflow.
+ let out = canonicalize_from_sqlite(basic_input(vec![CodecTableData {
+ name: "Items".to_string(),
+ records: vec![row(&[("Id", json!("i1")), ("AliasEnabled", json!(true))])],
+ }]))
+ .unwrap();
+ let re = materialize_as_sqlite(MaterializeInput { manifest: out.manifest, data_buckets: out.data_buckets, schema_columns: None }).unwrap();
+ let items = re.tables.iter().find(|t| t.name == "Items").unwrap();
+ assert_eq!(items.records[0]["AliasEnabled"], json!(true));
+ assert!(re.overflow.is_empty());
+}
+
#[test]
fn json_siblings_roundtrip() {
let input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }]);
diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rs
index 873f5bb4d..6edbad373 100644
--- a/core/rust/src/vault_codec/types.rs
+++ b/core/rust/src/vault_codec/types.rs
@@ -62,3 +62,10 @@ pub fn blob_spec_for(table_name: &str) -> Option<&'static (&'static str, &'stati
pub fn is_skip_table(table_name: &str) -> bool {
SKIP_TABLES.contains(&table_name)
}
+
+/// The single-column primary key that addresses a row in `table_name`, shared with the merge layer
+/// (see `vault_merge::types::SYNCABLE_TABLES`) so overflow keying and LWW merge agree on row identity.
+/// Defaults to "Id" for tables outside the registry.
+pub fn primary_key_for(table_name: &str) -> &'static str {
+ crate::vault_merge::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key).unwrap_or("Id")
+}
diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rs
index ec2cf61aa..599c8ddb0 100644
--- a/core/rust/src/vault_merge/mod.rs
+++ b/core/rust/src/vault_merge/mod.rs
@@ -10,8 +10,8 @@ use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use crate::error::VaultResult;
-use types::SYNCABLE_TABLES;
pub use types::SYNCABLE_TABLE_NAMES;
+pub use types::{SYNCABLE_TABLES, TableConfig};
/// A record is a map of column names to JSON values.
pub type Record = HashMap<String, serde_json::Value>;
diff --git a/core/rust/src/wasm.rs b/core/rust/src/wasm.rs
index 6b392c59c..6c4ea9c3f 100644
--- a/core/rust/src/wasm.rs
+++ b/core/rust/src/wasm.rs
@@ -145,18 +145,21 @@ pub fn vault_codec_materialize_as_sqlite_js(input: JsValue) -> Result<JsValue, J
codec_to_js(&output)
}
-/// Build a single data bucket. Input: `{ category, tables: { <name>: [rows] } }`. Output: `DataBucket`.
+/// Build a single data bucket. Input: `{ category, tables: { <name>: [rows] }, overflow? }`. Output: `DataBucket`.
#[wasm_bindgen(js_name = vaultCodecExtractBucket)]
pub fn vault_codec_extract_bucket_js(input: JsValue) -> Result<JsValue, JsValue> {
#[derive(serde::Deserialize)]
+ #[serde(rename_all = "camelCase")]
struct Input {
category: String,
#[serde(default)]
tables: std::collections::HashMap<String, Vec<CodecRecord>>,
+ #[serde(default)]
+ overflow: Option<vault_codec::CodecOverflow>,
}
let input: Input = serde_wasm_bindgen::from_value(input)
.map_err(|e| JsValue::from_str(&format!("Failed to parse extract-bucket input: {}", e)))?;
- codec_to_js(&vault_codec::extract_bucket(input.category, input.tables))
+ codec_to_js(&vault_codec::extract_bucket(input.category, input.tables, input.overflow.as_ref()))
}
/// The bucket layout: `[{ category, tables: [<name>] }]`. Source of truth for platform bucket-only sync.