Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Add VaultSyncService.ts (#2159)
Leendert de Borst08158362f097
diff --git a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.tsindex a5316eaea..754337d66 100644--- a/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts+++ b/apps/browser-extension/src/entrypoints/background/VaultMessageHandler.ts@@ -34,8 +34,7 @@ import { VaultUploadResponse as messageVaultUploadResponse } from '@/utils/typesimport { type VaultMutationScope, ALL_VAULT_MUTATION_SCOPES, DEFAULT_VAULT_MUTATION_SCOPE, dirtyScopeStorageKey, isManifestScope } from '@/utils/types/VaultMutationScope';import { VaultCodec } from '@/utils/VaultCodec';import { vaultMergeService } from '@/utils/VaultMergeService';-import { vaultRetrievalService } from '@/utils/VaultRetrievalService';-import { vaultV2SyncService } from '@/utils/VaultV2SyncService';+import { vaultSyncService } from '@/utils/VaultSyncService';import { WebApiService } from '@/utils/WebApiService';import { t } from '@/i18n/StandaloneI18n';@@ -188,7 +187,7 @@ export async function handleStoreEncryptionKeyDerivationParams(}/**- * Fetch the latest vault from the server as a VaultResponse, via the v2-only {@link VaultRetrievalService}.+ * Fetch the latest vault from the server as a VaultResponse, via the v2-only {@link VaultSyncService}.** GET /v2/Vault returns either the manifest model (materialized locally into a SQLite blob) or, for a* not-yet-migrated user, the legacy SQLite blob as-is, so a migrated user never hits the legacy API's 426@@ -202,7 +201,7 @@ async function fetchLatestVaultFromServer(): Promise<VaultResponse> {}try {- return await vaultRetrievalService.retrieveVault(encryptionKey);+ return await vaultSyncService.pull(encryptionKey);} catch (error) {if (error instanceof ServerUpdateRequiredError) {throw new Error(formatErrorWithCode(await t('common.errors.serverVersionNotSupported'), AppErrorCode.SERVER_UPDATE_REQUIRED));@@ -217,7 +216,7 @@ async function fetchLatestVaultFromServer(): Promise<VaultResponse> {*/async function uploadVaultV2(sqliteClient: SqliteClient): Promise<VaultPostResponse> {// Surfaces ServerUpdateRequiredError for servers that do not support the v2 API before we attempt the push.- await vaultV2SyncService.checkStatus();+ await vaultSyncService.checkStatus();const encryptionKey = await handleGetEncryptionKey();if (!encryptionKey) {@@ -227,7 +226,7 @@ async function uploadVaultV2(sqliteClient: SqliteClient): Promise<VaultPostRespoconst username = (await storage.getItem('local:username')) as string;const emailAddresses = await getEmailAddressesForVault(sqliteClient);- const result = await vaultV2SyncService.pushVault(sqliteClient, encryptionKey, username, emailAddresses);+ const result = await vaultSyncService.push(sqliteClient, encryptionKey, username, emailAddresses);if (result.status === 'ok') {return { status: 0, newRevisionNumber: result.newManifestRevision ?? 0 };@@ -245,7 +244,7 @@ async function uploadVaultV2(sqliteClient: SqliteClient): Promise<VaultPostRespo}/*- * missing-blobs: pushVault already re-uploaded the blob bytes the server asked for and retried once; landing+ * missing-blobs: push already re-uploaded the blob bytes the server asked for and retried once; landing* here means the server still reports gaps, a genuine server-side problem, not a transient race.*/throw new Error(formatErrorWithCode('Server reported missing blobs; please retry', AppErrorCode.UPLOAD_FAILED));@@ -774,7 +773,7 @@ async function uploadDirtyBucketsOnly(sqliteClient: SqliteClient, scopes: VaultMconst tables = VaultCodec.readNamedTables(sqliteClient, spec.tables);const bucket = await vaultCodecExtractBucket(category, tables);- const result = await vaultV2SyncService.pushDataBucketOnly(bucket, encryptionKey);+ const result = await vaultSyncService.pushDataBucketOnly(bucket, encryptionKey);if (result.status !== 'ok') {// Conflict persisted even after the rebase-retry, let the caller run a full re-sync (status 2).return { status: 2, newRevisionNumber: (await storage.getItem('local:serverRevision')) as number | null ?? 0 };diff --git a/apps/browser-extension/src/entrypoints/popup/pages/auth/Login.tsx b/apps/browser-extension/src/entrypoints/popup/pages/auth/Login.tsxindex 6df0b4784..bd0717907 100644--- a/apps/browser-extension/src/entrypoints/popup/pages/auth/Login.tsx+++ b/apps/browser-extension/src/entrypoints/popup/pages/auth/Login.tsx@@ -26,7 +26,7 @@ import { hasErrorCode, getErrorMessage } from '@/utils/types/errors/AppErrorCodeimport { ServerUpdateRequiredError } from '@/utils/types/errors/ServerUpdateRequiredError';import { VaultProcessingError } from '@/utils/types/errors/VaultProcessingError';import type { MobileLoginResult } from '@/utils/types/messaging/MobileLoginResult';-import { vaultRetrievalService } from '@/utils/VaultRetrievalService';+import { vaultSyncService } from '@/utils/VaultSyncService';import { vaultStateEvents } from '@/events/VaultStateEvents';@@ -164,7 +164,7 @@ const Login: React.FC = () => {});// Fetch the latest vault.- const vaultResponseJson = await vaultRetrievalService.retrieveVault(passwordHashBase64);+ const vaultResponseJson = await vaultSyncService.pull(passwordHashBase64);/** Persist and load the vault.@@ -463,7 +463,7 @@ const Login: React.FC = () => {});// Fetch the latest vault.- const vaultResponse = await vaultRetrievalService.retrieveVault(result.decryptionKey);+ const vaultResponse = await vaultSyncService.pull(result.decryptionKey);// Persist and load the vaultawait persistAndLoadVault(vaultResponse, result.decryptionKey, result.username);diff --git a/apps/browser-extension/src/utils/VaultSyncService.ts b/apps/browser-extension/src/utils/VaultSyncService.tsnew file mode 100644index 000000000..eada50b4b--- /dev/null+++ b/apps/browser-extension/src/utils/VaultSyncService.ts@@ -0,0 +1,747 @@+/**+ * VaultSyncService.+ *+ * Handles syncing the vault with the server and interfaces with the Rust codec.+ */++import { storage } from 'wxt/utils/storage';++import { devError, devLog, devWarn } from '@/utils/devLogger/DevLogger';+import { VaultDataBucketCategory } from '@/utils/dist/core/models/vault';+import type { VaultResponse } from '@/utils/dist/core/models/webapi';+import { VaultSqlGenerator } from '@/utils/dist/core/vault';+import { EncryptionUtility } from '@/utils/EncryptionUtility';+import {vaultCodecComputeCiphertextHash, vaultCodecCanonicalizeFromSqlite, vaultCodecGenerateUserSalt, vaultCodecUnpackPayload, vaultCodecMaterializeAsSqlite, vaultCodecPackPayload, vaultCodecValidateManifest, vaultCodecValidateDataBucket} from '@/utils/RustCore';+import { SqliteClient } from '@/utils/SqliteClient';+import { ServerUpdateRequiredError } from '@/utils/types/errors/ServerUpdateRequiredError';+import { VaultProcessingError } from '@/utils/types/errors/VaultProcessingError';+import { type BlobEntry, type VaultManifest, type VaultDataBucket, VaultCodec } from '@/utils/VaultCodec';+import { WebApiService } from '@/utils/WebApiService';++// Endpoints are relative; WebApiService resolves them under the API's /v2/ prefix.+const STATUS_ENDPOINT = 'Vault/status';+const SNAPSHOT_ENDPOINT = 'Vault';+const UPLOAD_ENDPOINT = 'Vault';+const BUCKETS_ENDPOINT = 'Vault/buckets';+const BLOBS_UPLOAD_ENDPOINT = 'Vault/blobs';+const BLOBS_MISSING_ENDPOINT = 'Vault/blobs/missing';+const BLOBS_DOWNLOAD_ENDPOINT = 'Vault/blobs/download';++/**+ * True when an error from WebApiService is an HTTP 404. WebApiService surfaces non-2xx responses as a generic+ * Error whose message carries the status code (`HTTP error! status: 404`). A 404 on a v2 endpoint means the+ * server does not support the v2 API (outdated self-hosted install), surfaced as {@link ServerUpdateRequiredError}.+ * @param e - the caught error+ */+function isNotFoundError(e: unknown): boolean {+ return e instanceof Error && e.message.includes('status: 404');+}++/**+ * Human-readable size for the push/pull size logs. Lengths are base64/JSON characters, which map ~1:1 to+ * bytes on the wire.+ * @param chars - length in characters (or bytes)+ */+function formatKb(chars: number): string {+ return chars < 1024 ? `${chars} B` : `${(chars / 1024).toFixed(1)} KB`;+}++/**+ * Pack-then-encrypt a JSON payload (manifest or data bucket). The Rust codec wraps the payload in the+ * integrity envelope (`{ contentHash, payload }`) and gzips it. The platform then AES-GCM encrypts+ * the packed bytes.+ * @param payloadJson - the plaintext payload JSON string (manifest or bucket)+ * @param vek - symmetric encryption key+ * @returns The base64 ciphertext plus the intermediate packed (gzip-compressed) size (for logging purposes)+ */+async function packEncrypt(payloadJson: string, vek: string): Promise<{ ciphertext: string; compressedBytes: number }> {+ const packed = await vaultCodecPackPayload(payloadJson);+ const ciphertext = await EncryptionUtility.symmetricEncryptBytes(packed, vek);+ return { ciphertext, compressedBytes: packed.length };+}++/**+ * Dev-only stage timer: used to surface the manifest-shuffling passes that are+ * otherwise invisible (canonicalize's Rust→JS conversion, validate's JS→Rust conversion, stringify).+ * @param stage - short label for the measured step+ * @param fn - the work to run and time+ */+async function timedStage<T>(stage: string, fn: () => Promise<T> | T): Promise<T> {+ if (!import.meta.env.DEV) {+ return fn();+ }+ const start = performance.now();+ const result = await fn();+ devLog(`[V2Push] stage "${stage}": ${(performance.now() - start).toFixed(2)}ms`);+ return result;+}++/**+ * Decrypt a manifest/bucket ciphertext and open it via the Rust codec: gunzip, verify the embedded+ * content hash, and return the inner payload JSON string. Throws if the content hash mismatches+ * (possible application-layer corruption).+ * @param base64Ciphertext - base64(IV ‖ ciphertext ‖ tag) from the server+ * @param vek - symmetric encryption key+ */+async function decryptUnpack(base64Ciphertext: string, vek: string): Promise<string> {+ const encryptedBytes = Uint8Array.from(atob(base64Ciphertext), c => c.charCodeAt(0));+ const plainBytes = await EncryptionUtility.symmetricDecryptBytes(encryptedBytes, vek);+ return vaultCodecUnpackPayload(plainBytes);+}++const SALT_STORAGE_KEY = 'local:vaultV2UserSalt';+/** Per-category data-bucket revision number. */+const bucketRevKey = (category: string): `local:${string}` => `local:vaultV2BucketRev:${category}`;+/** Local cache of encrypted blobs (hash → base64 AES-GCM ciphertext). Never stores plaintext at rest. */+const BLOB_CACHE_STORAGE_KEY = 'local:vaultV2BlobCipherCache';+/** Hashes the server has stored (refreshed on every pull/push). */+const SERVER_HASHES_STORAGE_KEY = 'local:vaultV2ServerBlobHashes';++/** Max accumulated base64 ciphertext characters per POST /v2/Vault/blobs call (~4 MB request body). */+const BLOB_UPLOAD_BATCH_MAX_CHARS = 4 * 1024 * 1024;+/** Hashes per POST /v2/Vault/blobs/download call. */+const BLOB_DOWNLOAD_BATCH_SIZE = 100;++/** Bucket category for the settings data bucket (generated from the Rust BUCKET_TABLES). */+const SETTINGS_BUCKET_CATEGORY = VaultDataBucketCategory.Settings;++/**+ * Status response from the server.+ */+export type VaultSyncStatus = {+ /** Whether the user has been migrated to manifest-v1 storage (false = still legacy, migrate on next save). */+ isMigrated: boolean;+ manifestRevision: number | null;+ settingsRevision: number | null;+};++/**+ * Email routing block as returned by the snapshot (server-readable plaintext, used for email delivery).+ */+export type EmailRouting = {+ emailAddressList: string[];+ privateEmailDomainList: string[];+ hiddenPrivateEmailDomainList: string[];+ publicEmailDomainList: string[];+};++/**+ * Result of a pull: the materialized SQLite plus the side-channel data the caller needs to build a VaultResponse.+ */+export type PullResult = {+ sqliteBase64: string;+ manifestRevision: number;+ emailRouting: EmailRouting;+};++/**+ * Result of a push.+ */+export type PushResult = {+ status: 'ok' | 'outdated' | 'missing-blobs' | 'rejected';+ newManifestRevision: number | null;+ reasons?: string[];+};++type BlobRefDto = { hash: string; category: string };+type BlobDto = { hash: string; category: string; encryptedDataBase64: string };++/** A data bucket as carried in the GET snapshot / bundled upload. `category` matches the server enum name (e.g. "Settings"). */+type BucketDto = { category: string; blob?: string | null; ciphertextHash?: string | null; revision?: number };++/** Per-kind revision as carried in status / upload responses. */+type BucketRevisionDto = { category: string; revision: number };++/**+ * Numeric value of the server StorageFormat enum for the manifest-v1 format (SqliteBlob = 0, Manifest = 1).+ */+export const STORAGE_FORMAT_MANIFEST = 1;++/**+ * Raw snapshot returned by GET /v2/Vault.+ */+export type GetResponseDto = {+ status: number;+ /** The server's storage format (0 = sqlite-blob, 1 = manifest-v1). */+ storageFormat?: number;+ /** The legacy encrypted SQLite blob (for not-yet-migrated users). */+ legacyVaultBlob?: string | null;+ version?: string | null;+ manifestBlob?: string | null;+ manifestCiphertextHash?: string | null;+ manifestRevision?: number | null;+ buckets?: BucketDto[];+ blobReferences?: BlobRefDto[];+ emailRouting?: {+ emailAddressList?: string[];+ privateEmailDomainList?: string[];+ hiddenPrivateEmailDomainList?: string[];+ publicEmailDomainList?: string[];+ };+};++type StatusResponseDto = {+ storageFormat: number;+ manifestRevision: number | null;+ bucketRevisions: BucketRevisionDto[];+};++type UploadResponseDto = {+ status: number;+ newManifestRevision: number;+ newBucketRevisions: BucketRevisionDto[];+ missingBlobHashes: string[];+};++type MissingBlobsResponseDto = { missing: string[] };++/**+ * Service entry point.+ */+export class VaultSyncService {+ /**+ * Retrieve the latest vault from the server as a normalized {@link VaultResponse} (encrypted SQLite blob ++ * email routing + revision).+ *+ * Throws {@link ServerUpdateRequiredError} when the server predates the v2 API (outdated self-hosted install).+ * @param encryptionKey - the user's symmetric key (decrypts/materializes manifest-v1 and re-encrypts the+ * materialized SQLite; unused for the already-encrypted legacy-blob pass-through).+ */+ public async pull(encryptionKey: string): Promise<VaultResponse> {+ /*+ * Step 1 — network fetch. Failures here (server unreachable, HTTP error, or ServerUpdateRequiredError for an+ * outdated server) are "can't reach / must update the server" conditions, NOT vault-processing problems, so+ * they propagate unchanged and the caller maps them to the appropriate "server" message.+ */+ devLog('[V2Pull] Step 1/4: fetching vault snapshot (GET /v2/Vault)...');+ const snapshot = await this.fetchSnapshot();+ devLog(`[V2Pull] Step 1/4 done: storageFormat=${snapshot.storageFormat}, revision=${snapshot.manifestRevision}, manifestBlob=${snapshot.manifestBlob?.length ?? 0} chars, buckets=${snapshot.buckets?.length ?? 0}, blobRefs=${snapshot.blobReferences?.length ?? 0}`);++ /*+ * Steps 2–4 — decrypt, materialize, and re-encrypt. Any failure here is a client-side vault-processing error+ * (codec/format mismatch, integrity failure, corrupt blob, …). We wrap it in a VaultProcessingError so the UI+ * can surface the real technical detail in a copyable report instead of a misleading "server unreachable".+ */+ try {+ if (snapshot.storageFormat === STORAGE_FORMAT_MANIFEST) {+ // Manifest-v1 user: materialize the manifest + metadata + blobs into a SQLite blob, then encrypt it.+ devLog('[V2Pull] Step 2/4: manifest format — decrypting and reassembling local SQLite...');+ const pull = await this.materializeFromSnapshot(snapshot, encryptionKey);+ devLog(`[V2Pull] Step 3/4: materialized SQLite (${pull.sqliteBase64.length} base64 chars); re-encrypting for local storage...`);+ const encryptedVault = await EncryptionUtility.symmetricEncrypt(pull.sqliteBase64, encryptionKey);+ devLog('[V2Pull] Step 4/4: re-encryption done, returning VaultResponse.');+ return this.buildResponse(encryptedVault, '2.0.0', pull.manifestRevision, snapshot);+ }++ /*+ * Not-yet-migrated (sqlite-blob fallback) user: the server returned the legacy encrypted SQLite blob. It's already in the stored+ * format (encrypted SQLite), so we pass it through unchanged — the on-open schema upgrade handles the rest.+ */+ devLog('[V2Pull] Step 2/4: legacy blob pass-through (user not yet migrated), returning as-is.');+ return this.buildResponse(+ snapshot.legacyVaultBlob ?? '',+ snapshot.version ?? '',+ typeof snapshot.manifestRevision === 'number' ? snapshot.manifestRevision : 0,+ snapshot+ );+ } catch (error) {+ devError('[V2Pull] FAILED — the last logged step above is where it broke:', error);+ throw new VaultProcessingError('vault-pull', error);+ }+ }++ /**+ * Ask the server which storage format the user is on. Throws error if the server predates the v2 API.+ */+ public async checkStatus(): Promise<VaultSyncStatus> {+ const webApi = new WebApiService();+ try {+ const dto = await webApi.get<StatusResponseDto>(STATUS_ENDPOINT);+ const settingsRevision = (dto.bucketRevisions ?? []).find(b => b.category === SETTINGS_BUCKET_CATEGORY)?.revision ?? null;+ return {+ isMigrated: dto.storageFormat === STORAGE_FORMAT_MANIFEST,+ manifestRevision: dto.manifestRevision,+ settingsRevision,+ };+ } catch (e) {+ if (isNotFoundError(e)) {+ throw new ServerUpdateRequiredError();+ }+ throw e;+ }+ }++ /**+ * Fetch the raw snapshot (GET /v2/Vault) without decrypting/reassembling. Throws error if the server predates the v2 API.+ */+ private async fetchSnapshot(): Promise<GetResponseDto> {+ const webApi = new WebApiService();+ try {+ return await webApi.get<GetResponseDto>(SNAPSHOT_ENDPOINT);+ } catch (e) {+ if (isNotFoundError(e)) {+ throw new ServerUpdateRequiredError();+ }+ throw e;+ }+ }++ /**+ * Materialize a local SQLite database from an already-fetched manifest-v1 snapshot: verify ciphertext integrity,+ * decrypt + unwrap the manifest/metadata, fetch any missing referenced blobs, then run the codec.+ * @param snapshot - the raw GET /v2/Vault response+ * @param vek - the symmetric key used to decrypt the manifest + metadata + blobs+ */+ private async materializeFromSnapshot(snapshot: GetResponseDto, vek: string): Promise<PullResult> {+ const webApi = new WebApiService();+ if (!snapshot.manifestBlob) {+ throw new Error('VaultSyncService: server returned no manifest blob, nothing to assemble.');+ }++ // Storage-layer integrity: verify the ciphertext we received matches the hash the server stored.+ if (snapshot.manifestCiphertextHash) {+ const actual = await vaultCodecComputeCiphertextHash(snapshot.manifestBlob);+ if (actual !== snapshot.manifestCiphertextHash) {+ throw new Error('VaultSyncService: manifest ciphertext hash mismatch, refusing to load. Possible storage corruption.');+ }+ }++ devLog('[V2Pull] Manifest ciphertext hash verified; decrypting + opening manifest...');+ const manifestJson = await decryptUnpack(snapshot.manifestBlob, vek);+ const manifest = JSON.parse(manifestJson) as VaultManifest;+ devLog(`[V2Pull] Manifest opened (content hash verified): schemaVersion=${manifest.schemaVersion}, migrationId=${manifest.migrationId}, tables: ${Object.entries(manifest.tables).map(([t, rows]) => `${t}=${rows.length}`).join(', ')}`);++ // Persist the user salt locally so subsequent canonicalizes hash blobs the same way.+ await storage.setItem(SALT_STORAGE_KEY, manifest.userSalt);+ const manifestRevision = typeof snapshot.manifestRevision === 'number' ? snapshot.manifestRevision : 0;+ await storage.setItem('local:serverRevision', manifestRevision);++ // Decrypt every data bucket in the snapshot (Settings today; more categories later).+ const dataBuckets: VaultDataBucket[] = [];+ for (const bucketDto of (snapshot.buckets ?? [])) {+ if (!bucketDto.blob) {+ continue;+ }+ if (bucketDto.ciphertextHash) {+ const actual = await vaultCodecComputeCiphertextHash(bucketDto.blob);+ if (actual !== bucketDto.ciphertextHash) {+ throw new Error(`VaultSyncService: "${bucketDto.category}" bucket ciphertext hash mismatch, refusing to load.`);+ }+ }+ const bucketJson = await decryptUnpack(bucketDto.blob, vek);+ const bucket = JSON.parse(bucketJson) as VaultDataBucket;+ dataBuckets.push(bucket);+ const rowCount = Object.values(bucket.tables ?? {}).reduce((n, rows) => n + rows.length, 0);+ devLog(`[V2Pull] Data bucket "${bucketDto.category}" opened: ${rowCount} rows (revision ${bucketDto.revision}).`);+ if (typeof bucketDto.revision === 'number') {+ await storage.setItem(bucketRevKey(bucketDto.category), bucketDto.revision);+ }+ }+ if (dataBuckets.length === 0) {+ devLog('[V2Pull] No data buckets in snapshot.');+ }++ // Fetch any blobs referenced by the manifest that aren't already in the local (encrypted) cache.+ const refs = snapshot.blobReferences ?? [];+ const cache = await this.loadBlobCache();+ const missingHashes = refs.map(r => r.hash).filter(h => !(h in cache));+ devLog(`[V2Pull] Blob refs: ${refs.length} referenced, ${refs.length - missingHashes.length} cached locally, ${missingHashes.length} to download.`);+ for (let i = 0; i < missingHashes.length; i += BLOB_DOWNLOAD_BATCH_SIZE) {+ const chunk = missingHashes.slice(i, i + BLOB_DOWNLOAD_BATCH_SIZE);+ const blobs = await webApi.post<{ hashes: string[] }, BlobDto[]>(BLOBS_DOWNLOAD_ENDPOINT, { hashes: chunk });+ devLog(`[V2Pull] Downloaded blob batch ${Math.floor(i / BLOB_DOWNLOAD_BATCH_SIZE) + 1}: requested ${chunk.length}, received ${blobs.length}.`);+ for (const dto of blobs) {+ cache[dto.hash] = dto.encryptedDataBase64;+ }+ }++ /*+ * Decrypt referenced blobs for reassembly and prune the persisted cache to exactly the referenced set, so+ * the cache stays bounded by the current vault size. A referenced blob the server couldn't serve is fatal+ * for attachments (silently dropping bytes would propagate permanent data loss on the next push); a missing+ * favicon only degrades cosmetically, so it's logged and skipped.+ */+ const prunedCache: Record<string, string> = {};+ const blobMap = new Map<string, Uint8Array>();+ for (const r of refs) {+ const ciphertext = cache[r.hash];+ if (!ciphertext) {+ if (r.category === 'attachment') {+ throw new Error(`VaultSyncService: attachment blob ${r.hash} is referenced by the manifest but missing on the server, refusing to assemble an incomplete vault.`);+ }+ devWarn(`[V2Sync] Referenced ${r.category} blob ${r.hash} missing on server, continuing without it.`);+ continue;+ }+ prunedCache[r.hash] = ciphertext;+ blobMap.set(r.hash, await this.decryptBlobToBytes(ciphertext, vek));+ }+ await this.saveBlobCache(prunedCache);++ // The server demonstrably has every blob it just served or referenced, seed the upload diff with them.+ await storage.setItem(SERVER_HASHES_STORAGE_KEY, refs.map(r => r.hash));++ devLog(`[V2Pull] ${blobMap.size} blobs decrypted; running codec reassembly into a fresh SQLite...`);+ const sqlGen = new VaultSqlGenerator();+ const schemaSql = sqlGen.getCompleteSchemaSql();+ const materialized = await vaultCodecMaterializeAsSqlite(manifest, dataBuckets);+ const sqliteBase64 = await VaultCodec.insertTables(materialized, blobMap, schemaSql);+ devLog('[V2Pull] Codec reassembly complete.');++ return {+ sqliteBase64,+ manifestRevision,+ emailRouting: {+ emailAddressList: snapshot.emailRouting?.emailAddressList ?? [],+ privateEmailDomainList: snapshot.emailRouting?.privateEmailDomainList ?? [],+ hiddenPrivateEmailDomainList: snapshot.emailRouting?.hiddenPrivateEmailDomainList ?? [],+ publicEmailDomainList: snapshot.emailRouting?.publicEmailDomainList ?? [],+ },+ };+ }++ /**+ * Build a VaultResponse from a (already-encrypted) SQLite blob + the snapshot's email-routing block.+ * @param encryptedBlob - encrypted SQLite blob to store under local:encryptedVault+ * @param version - data-model version string+ * @param revision - unified content revision number+ * @param snapshot - the source snapshot (for the email-routing lists)+ */+ private buildResponse(encryptedBlob: string, version: string, revision: number, snapshot: GetResponseDto): VaultResponse {+ const er = snapshot.emailRouting ?? {};+ return {+ status: 0,+ vault: {+ blob: encryptedBlob,+ version,+ currentRevisionNumber: revision,+ encryptionPublicKey: '',+ credentialsCount: 0,+ publicEmailDomainList: er.publicEmailDomainList ?? [],+ privateEmailDomainList: er.privateEmailDomainList ?? [],+ hiddenPrivateEmailDomainList: er.hiddenPrivateEmailDomainList ?? [],+ emailAddressList: er.emailAddressList ?? [],+ createdAt: '',+ updatedAt: '',+ username: '',+ },+ } as unknown as VaultResponse;+ }++ /**+ * Canonicalize the current SQLite vault, validate, encrypt, and POST /v2/Vault. Only blobs the server doesn't+ * already have are encrypted and pre-uploaded (in size-capped batches) before the manifest POST, so a routine+ * save of a vault with hundreds of attachments uploads kilobytes, not the whole blob set. If the manifest POST+ * reports missing blobs still missing (stale local knowledge, e.g. server-side GC), the missing bytes are uploaded+ * and the POST retried once.+ * @param sqliteClient - the in-memory SQLite the user has been editing+ * @param vek - the symmetric encryption key+ * @param username - the user's username (sent in the upload payload for cross-check)+ * @param emailAddressList - claimed email aliases (server needs these in plaintext for routing)+ * @returns Push outcome.+ */+ public async push(+ sqliteClient: SqliteClient,+ vek: string,+ username: string,+ emailAddressList: string[]+ ): Promise<PushResult> {+ // 1) Canonicalize using the persisted user salt (or generate one on first save).+ let userSalt = (await storage.getItem(SALT_STORAGE_KEY)) as string | null;+ if (!userSalt) {+ userSalt = await vaultCodecGenerateUserSalt();+ await storage.setItem(SALT_STORAGE_KEY, userSalt);+ }++ // Read tables from the SQLite database and apply the manifest-v1 format rules.+ const tables = VaultCodec.readTables(sqliteClient);+ const migrationId = VaultCodec.getLatestMigrationId(sqliteClient);+ const canonicalized = await timedStage('canonicalize (incl. Rust→JS conversion)', () => vaultCodecCanonicalizeFromSqlite({+ tables,+ userSalt,+ migrationId,+ version: '2.0.0',+ canonicalizedAt: new Date().toISOString(),+ }));++ // Plaintext blob bytes held platform-side for encryption/upload.+ const blobEntries = new Map<string, BlobEntry>(+ Object.entries(canonicalized.blobs).map(([hash, entry]) => [+ hash,+ { kind: entry.kind as 'favicon' | 'attachment', bytes: VaultCodec.base64ToBytes(entry.bytesBase64) },+ ])+ );++ // Debug: full unencrypted manifest + data buckets, inspectable in the console. TODO: remove when going to production.+ devLog('[V2Push] Unencrypted manifest:', canonicalized.manifest);+ devLog(`[V2Push] Unencrypted data buckets (${canonicalized.dataBuckets.length}):`, canonicalized.dataBuckets);++ // 2) Pre-upload structural validation, refuses to upload malformed manifests / data buckets.+ const manifestValidation = await timedStage('validate-manifest (incl. JS→Rust conversion)', () => vaultCodecValidateManifest(canonicalized.manifest));+ if (!manifestValidation.ok) {+ return {+ status: 'rejected',+ newManifestRevision: null,+ reasons: [`Manifest validation failed: ${manifestValidation.failedRules.join(', ')}. ${manifestValidation.message}`.trim()],+ };+ }++ for (const bucket of canonicalized.dataBuckets) {+ const bucketValidation = await vaultCodecValidateDataBucket(bucket);+ if (!bucketValidation.ok) {+ return {+ status: 'rejected',+ newManifestRevision: null,+ reasons: [`Data bucket "${bucket.category}" validation failed: ${bucketValidation.failedRules.join(', ')}. ${bucketValidation.message}`.trim()],+ };+ }+ }++ // 3) Pack then AES-GCM encrypt.+ const manifestPlaintext = await timedStage('stringify-manifest', () => JSON.stringify(canonicalized.manifest));+ const { ciphertext: manifestCiphertext, compressedBytes: manifestCompressedBytes } = await packEncrypt(manifestPlaintext, vek);+ const manifestCiphertextHash = await vaultCodecComputeCiphertextHash(manifestCiphertext);+ devLog(`[V2Push] Manifest blob: raw ${formatKb(manifestPlaintext.length)} → compressed ${formatKb(manifestCompressedBytes)} → encrypted ${formatKb(manifestCiphertext.length)}.`);++ // Pack + encrypt each data bucket into its own upload entry.+ const bucketDtos: Array<{ category: string; blob: string; ciphertextHash: string }> = [];+ for (const bucket of canonicalized.dataBuckets) {+ const bucketPlaintext = JSON.stringify(bucket);+ const { ciphertext, compressedBytes } = await packEncrypt(bucketPlaintext, vek);+ const ciphertextHash = await vaultCodecComputeCiphertextHash(ciphertext);+ bucketDtos.push({ category: bucket.category, blob: ciphertext, ciphertextHash });+ devLog(`[V2Push] Data bucket "${bucket.category}": raw ${formatKb(bucketPlaintext.length)} → compressed ${formatKb(compressedBytes)} → encrypted ${formatKb(ciphertext.length)}.`);+ }++ /*+ * 4) Blob diff: only encrypt + upload blobs the server doesn't already have. First filter by local knowledge+ * of the server's blob set (refreshed on every pull/push), then confirm the remainder with the server via+ * POST blobs/missing.+ */+ const webApi = new WebApiService();+ const allBlobHashes = Array.from(blobEntries.keys());+ const knownServerHashes = new Set(((await storage.getItem(SERVER_HASHES_STORAGE_KEY)) as string[] | null) ?? []);+ const candidates = allBlobHashes.filter(h => !knownServerHashes.has(h));++ let hashesToUpload: string[] = [];+ if (candidates.length > 0) {+ const missingResp = await webApi.post<{ hashes: string[] }, MissingBlobsResponseDto>(BLOBS_MISSING_ENDPOINT, { hashes: candidates });+ hashesToUpload = missingResp.missing ?? [];+ }++ devLog(`[V2Push] Blob diff: ${allBlobHashes.length} blobs in vault, ${candidates.length} not known to be on server, ${hashesToUpload.length} confirmed missing → uploading ${hashesToUpload.length}.`);++ // Pre-upload the missing blobs in size-capped batches so the manifest POST below carries references only.+ const uploadedCiphertexts = await this.uploadBlobs(webApi, blobEntries, hashesToUpload, vek);++ const blobReferences: BlobRefDto[] = Array.from(blobEntries.entries()).map(([hash, entry]) => ({+ hash,+ category: entry.kind,+ }));++ const currentManifestRevision = ((await storage.getItem('local:serverRevision')) as number | null) ?? 0;+ const itemCount = (canonicalized.manifest.tables.Items ?? []).length;++ const payload = {+ username,+ version: canonicalized.manifest.version,+ manifestBlob: manifestCiphertext,+ manifestCiphertextHash,+ currentManifestRevision,+ credentialsCount: itemCount,+ buckets: bucketDtos,+ newBlobs: [],+ blobReferences,+ emailRouting: { emailAddressList },+ encryptionPublicKey: '',+ };++ let resp = await webApi.post<typeof payload, UploadResponseDto>(UPLOAD_ENDPOINT, payload);++ if (resp.missingBlobHashes && resp.missingBlobHashes.length > 0) {+ /*+ * Our local knowledge of the server's blob set was stale (e.g. the server GC'd a blob between syncs).+ * Upload the bytes it asked for and retry the manifest POST once.+ */+ const unsatisfiable = resp.missingBlobHashes.filter(h => !blobEntries.has(h));+ if (unsatisfiable.length > 0) {+ return { status: 'missing-blobs', newManifestRevision: resp.newManifestRevision, reasons: unsatisfiable };+ }++ devWarn(`[V2Sync] Server reported ${resp.missingBlobHashes.length} missing blob(s); uploading and retrying once.`);+ const retried = await this.uploadBlobs(webApi, blobEntries, resp.missingBlobHashes, vek);+ for (const [hash, ciphertext] of retried.entries()) {+ uploadedCiphertexts.set(hash, ciphertext);+ }++ resp = await webApi.post<typeof payload, UploadResponseDto>(UPLOAD_ENDPOINT, payload);+ if (resp.missingBlobHashes && resp.missingBlobHashes.length > 0) {+ return { status: 'missing-blobs', newManifestRevision: resp.newManifestRevision, reasons: resp.missingBlobHashes };+ }+ }++ if (resp.status !== 0) {+ return { status: 'outdated', newManifestRevision: resp.newManifestRevision };+ }++ // 5) Update local persisted state on success.+ await storage.setItem('local:serverRevision', resp.newManifestRevision);+ for (const br of (resp.newBucketRevisions ?? [])) {+ await storage.setItem(bucketRevKey(br.category), br.revision);+ }++ // Every referenced hash is now known to be on the server, refresh the diff baseline.+ await storage.setItem(SERVER_HASHES_STORAGE_KEY, allBlobHashes);++ /*+ * Refresh the encrypted blob cache: keep entries still referenced by the new manifest, add the ciphertexts we just uploaded.+ */+ const cache = await this.loadBlobCache();+ const newCache: Record<string, string> = {};+ for (const hash of allBlobHashes) {+ const ciphertext = uploadedCiphertexts.get(hash) ?? cache[hash];+ if (ciphertext) {+ newCache[hash] = ciphertext;+ }+ }+ await this.saveBlobCache(newCache);++ const uploadedBlobChars = Array.from(uploadedCiphertexts.values()).reduce((sum, c) => sum + c.length, 0);+ const bucketChars = bucketDtos.reduce((sum, b) => sum + b.blob.length, 0);+ const totalChars = manifestCiphertext.length + bucketChars + uploadedBlobChars;+ devLog(`[V2Push] Total pushed (encrypted): manifest ${formatKb(manifestCiphertext.length)} + ${bucketDtos.length} buckets ${formatKb(bucketChars)} + ${uploadedCiphertexts.size} blobs ${formatKb(uploadedBlobChars)} = ${formatKb(totalChars)} (+ ${blobReferences.length} blob references).`);++ return { status: 'ok', newManifestRevision: resp.newManifestRevision };+ }++ /**+ * Single-data-bucket upload, for changes scoped to a separate data bucket and not touching the (main) manifest.+ * @param bucket - the new data bucket contents (its `category` selects the server bucket)+ * @param vek - encryption key+ */+ public async pushDataBucketOnly(bucket: VaultDataBucket, vek: string): Promise<{ status: 'ok' | 'outdated'; revision: number }> {+ const { category } = bucket;+ const plaintext = JSON.stringify(bucket);+ const { ciphertext, compressedBytes } = await packEncrypt(plaintext, vek);+ const ciphertextHash = await vaultCodecComputeCiphertextHash(ciphertext);+ devLog(`[V2Push] Bucket "${category}" (bucket-only): raw ${formatKb(plaintext.length)} → compressed ${formatKb(compressedBytes)} → encrypted ${formatKb(ciphertext.length)}.`);++ const webApi = new WebApiService();+ let currentRevision = (((await storage.getItem(bucketRevKey(category))) as number | null) ?? 0);+ let resp = await webApi.post<unknown, { status: number; category: string; newRevision: number }>(BUCKETS_ENDPOINT, {+ category,+ bucketBlob: ciphertext,+ bucketCiphertextHash: ciphertextHash,+ currentRevision,+ });++ if (resp.status !== 0) {+ devWarn(`[V2Push] Bucket "${category}" outdated (server at revision ${resp.newRevision}, we assumed ${currentRevision}); rebasing and retrying once.`);+ currentRevision = resp.newRevision;+ resp = await webApi.post<unknown, { status: number; category: string; newRevision: number }>(BUCKETS_ENDPOINT, {+ category,+ bucketBlob: ciphertext,+ bucketCiphertextHash: ciphertextHash,+ currentRevision,+ });+ }++ if (resp.status !== 0) {+ return { status: 'outdated', revision: resp.newRevision };+ }++ await storage.setItem(bucketRevKey(category), resp.newRevision);+ return { status: 'ok', revision: resp.newRevision };+ }++ /**+ * Encrypt the given blobs and upload them via POST /v2/Vault/blobs in size-capped batches.+ * @param blobs - plaintext blob map keyed by content hash (source of the bytes)+ * @param hashes - the subset of hashes to upload (hashes without an entry in `blobs` are skipped)+ * @param vek - symmetric encryption key+ * @param webApi - API client to reuse+ * @returns Map of hash → uploaded ciphertext (base64), for the local encrypted blob cache.+ */+ private async uploadBlobs(webApi: WebApiService, blobs: Map<string, BlobEntry>, hashes: string[], vek: string): Promise<Map<string, string>> {+ const ciphertexts = new Map<string, string>();+ if (hashes.length === 0) {+ return ciphertexts;+ }++ let batch: BlobDto[] = [];+ let batchChars = 0;+ for (const hash of hashes) {+ const entry = blobs.get(hash);+ if (!entry) {+ continue;+ }++ const ciphertext = await this.encryptBlobBytes(entry.bytes, vek);+ ciphertexts.set(hash, ciphertext);+ devLog(`[V2Push] Blob ${hash.substring(0, 12)}… (${entry.kind}): raw ${formatKb(entry.bytes.length)} → encrypted ${formatKb(ciphertext.length)}.`);++ if (batch.length > 0 && batchChars + ciphertext.length > BLOB_UPLOAD_BATCH_MAX_CHARS) {+ devLog(`[V2Push] Uploading blob batch: ${batch.length} blobs, ${formatKb(batchChars)}.`);+ await webApi.post(BLOBS_UPLOAD_ENDPOINT, { blobs: batch });+ batch = [];+ batchChars = 0;+ }++ batch.push({ hash, category: entry.kind, encryptedDataBase64: ciphertext });+ batchChars += ciphertext.length;+ }++ if (batch.length > 0) {+ devLog(`[V2Push] Uploading blob batch: ${batch.length} blobs, ${formatKb(batchChars)}.`);+ await webApi.post(BLOBS_UPLOAD_ENDPOINT, { blobs: batch });+ }++ return ciphertexts;+ }++ /**+ * Encrypt raw blob bytes with the VEK. Result is base64-of-(IV ‖ ciphertext ‖ tag), which is what+ * symmetricEncrypt produces. Bytes round-trip through symmetricEncrypt's string interface via latin-1.+ * @param bytes - plaintext bytes+ * @param vek - symmetric encryption key+ */+ private async encryptBlobBytes(bytes: Uint8Array, vek: string): Promise<string> {+ let s = '';+ for (let i = 0; i < bytes.length; i++) {+ s += String.fromCharCode(bytes[i]);+ }++ return EncryptionUtility.symmetricEncrypt(s, vek);+ }++ /**+ * Decrypt a blob ciphertext (base64) and return raw plaintext bytes.+ * @param encryptedDataBase64 - base64 IV‖ciphertext‖tag from the server+ * @param vek - symmetric encryption key+ */+ private async decryptBlobToBytes(encryptedDataBase64: string, vek: string): Promise<Uint8Array> {+ const plaintextLatin1 = await EncryptionUtility.symmetricDecrypt(encryptedDataBase64, vek);+ const out = new Uint8Array(plaintextLatin1.length);+ for (let i = 0; i < plaintextLatin1.length; i++) {+ out[i] = plaintextLatin1.charCodeAt(i) & 0xff;+ }+ return out;+ }++ /**+ * Load the local encrypted blob cache (hash → base64 ciphertext) used to skip re-downloading known blobs.+ * Entries are stored as served to/from the server, so nothing in the cache is plaintext at rest.+ */+ private async loadBlobCache(): Promise<Record<string, string>> {+ return ((await storage.getItem(BLOB_CACHE_STORAGE_KEY)) as Record<string, string> | null) ?? {};+ }++ /**+ * Persist the local encrypted blob cache (hash → base64 ciphertext).+ * @param cache - cache to persist+ */+ private async saveBlobCache(cache: Record<string, string>): Promise<void> {+ await storage.setItem(BLOB_CACHE_STORAGE_KEY, cache);+ }+}++export const vaultSyncService = new VaultSyncService();