Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update core/client to interface with Rust vault_sync engine (#2159)
Leendert de Borst7bc1c673b97e
diff --git a/core/client/src/api/CapabilityService.ts b/core/client/src/api/CapabilityService.tsindex ffc680c09..52c2449aa 100644--- a/core/client/src/api/CapabilityService.ts+++ b/core/client/src/api/CapabilityService.ts@@ -15,9 +15,15 @@ export class CapabilityService {* @param status - the status response to take the capabilities from.*/public static async store(status: StatusResponseV2): Promise<void> {- const resolved = status.capabilities ?? {};+ await this.storeCapabilities(status.capabilities ?? {});+ }- // Only write when the status response actually changed.+ /**+ * Persist the resolved capabilities.+ * @param resolved - the capabilities, keyed by capability key.+ */+ public static async storeCapabilities(resolved: Capabilities): Promise<void> {+ // Only write when the set actually changed.if (canonical(await this.getAll()) === canonical(resolved)) {return;}diff --git a/core/client/src/api/WebApiService.ts b/core/client/src/api/WebApiService.tsindex 851dc7daa..b5259d7d5 100644--- a/core/client/src/api/WebApiService.ts+++ b/core/client/src/api/WebApiService.ts@@ -27,10 +27,7 @@ const DEFAULT_REQUEST_TIMEOUT_MS = 5000;const VAULT_TRANSFER_TIMEOUT_MS = 180000;/**- * Path prefix whose requests carry vault ciphertext and therefore run on {@link VAULT_TRANSFER_TIMEOUT_MS}:- * `Vault` itself and every subpath of it, which includes the single-manifest fetch and the batched blob- * upload/download calls that run to several MB per request. Matched lowercased and unslashed, so sibling- * endpoints such as `VaultKey` keep the short timeout.+ * Path prefix whose requests carry vault ciphertext and therefore use the larger timeout setting {@link VAULT_TRANSFER_TIMEOUT_MS}.*/const LARGE_TRANSFER_PATH = 'vault';@@ -74,6 +71,14 @@ type TokenResponse = {refreshToken: string;}+/**+ * A response as the Rust sync engine reads it: the HTTP status and the body text, never thrown on a non-2xx status.+ */+export type EngineHttpResponse = {+ status: number;+ body: string;+};+/*** Service class for interacting with the web API.*/@@ -147,7 +152,7 @@ export class WebApiService {logoutEventEmitter.emit('common.errors.sessionExpired');throw new ApiAuthError('Session expired');} else {- // Token refresh failed due to network/server error - throw NetworkError for offline handling+ // Token refresh failed due to network/server error.throw new NetworkError('Token refresh failed due to network error');}}@@ -187,16 +192,59 @@ export class WebApiService {/*** Fetch data from the API without authentication headers and without access token refresh retry.- * Throws RequestTimeoutError when the request exceeds its timeout, and NetworkError for other- * network-related failures (offline, DNS, etc.)*/public async rawFetch(endpoint: string,options: RequestInit = {}): Promise<Response> {- const baseUrl = await this.getBaseUrl();- const url = baseUrl + endpoint;+ const url = await this.getBaseUrl() + endpoint;const headers = new Headers(options.headers ?? {});+ return this.performFetch(url, endpoint, { ...options, headers }, this.isLargeTransfer(endpoint, headers));+ }++ /**+ * Run a request on behalf of the Rust sync engine.+ */+ public async engineRequest(method: string, path: string, body: string | undefined, requiresAuth: boolean, largeTransfer: boolean): Promise<EngineHttpResponse> {+ const url = (await this.getApiUrl()).replace(/\/$/, '') + '/' + path.replace(/^\/+/, '');+ const headers = new Headers({ Accept: 'application/json' });+ if (body !== undefined) {+ headers.set('Content-Type', 'application/json');+ }+ const accessToken = requiresAuth ? await this.getAccessToken() : null;+ if (accessToken) {+ headers.set('Authorization', `Bearer ${accessToken}`);+ }++ try {+ let response = await this.performFetch(url, path, { method, headers, body }, largeTransfer);+ if (response.status === 401 && requiresAuth) {+ const refreshResult = await this.refreshAccessToken();+ if (refreshResult.token) {+ headers.set('Authorization', `Bearer ${refreshResult.token}`);+ response = await this.performFetch(url, path, { method, headers, body }, largeTransfer);+ } else if (refreshResult.isAuthError) {+ // The session is truly expired; the engine turns the 401 into its logout outcome.+ logoutEventEmitter.emit('common.errors.sessionExpired');+ return { status: 401, body: '' };+ } else {+ throw new NetworkError('Token refresh failed due to network error');+ }+ }+ return { status: response.status, body: await response.text() };+ } catch (error) {+ if (error instanceof ClientUpgradeRequiredError) {+ return { status: 426, body: '' };+ }+ throw error;+ }+ }++ /**+ * Send one request with the client headers and the timeout for its size.+ */+ private async performFetch(url: string, endpoint: string, options: RequestInit & { headers: Headers }, largeTransfer: boolean): Promise<Response> {+ const headers = options.headers;// Add client version header (using API_VERSION for server compatibility)headers.set('X-AliasVault-Client', `${AppInfo.CLIENT_NAME}-${AppInfo.API_VERSION}`);@@ -206,10 +254,11 @@ export class WebApiService {headers.set(name, value);}+ const timeoutSignal = timeoutAbortSignal(largeTransfer ? VAULT_TRANSFER_TIMEOUT_MS : DEFAULT_REQUEST_TIMEOUT_MS);const requestOptions: RequestInit = {...options,headers,- signal: this.buildTimeoutSignal(endpoint, headers, options.signal),+ signal: options.signal ? anyAbortSignal([options.signal, timeoutSignal]) : timeoutSignal,};let response: Response;@@ -218,8 +267,7 @@ export class WebApiService {} catch (error) {console.error('API request failed:', error);/*- * The timeout signal aborts with a DOMException; no caller passes its own abort signal,- * so any abort here means the request exceeded its timeout.+ * The timeout signal aborts with a DOMException.*/if (error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError')) {throw new RequestTimeoutError(`Request timed out: ${endpoint}`, error);@@ -240,15 +288,12 @@ export class WebApiService {}/**- * Build an AbortSignal that bounds how long a request can run, combined with any caller-supplied- * signal.+ * Whether a request carries vault ciphertext and so runs on the long timeout.*/- private buildTimeoutSignal(endpoint: string, headers: Headers, callerSignal?: AbortSignal | null): AbortSignal {+ private isLargeTransfer(endpoint: string, headers: Headers): boolean {const path = endpoint.split('?')[0].replace(/^\/+|\/+$/g, '').toLowerCase();- const isLargeTransfer = path === LARGE_TRANSFER_PATH || path.startsWith(`${LARGE_TRANSFER_PATH}/`) ||+ return path === LARGE_TRANSFER_PATH || path.startsWith(`${LARGE_TRANSFER_PATH}/`) ||(headers.get('Accept') ?? '').toLowerCase().includes('application/octet-stream');- const timeoutSignal = timeoutAbortSignal(isLargeTransfer ? VAULT_TRANSFER_TIMEOUT_MS : DEFAULT_REQUEST_TIMEOUT_MS);- return callerSignal ? anyAbortSignal([callerSignal, timeoutSignal]) : timeoutSignal;}/**@@ -338,8 +383,6 @@ export class WebApiService {/*** Revoke only the current specific token via WebApi.* Unlike revokeTokens(), this does NOT revoke other sessions for the same device.- * Used for mobile unlock flow where we want to replace the current session without- * affecting other browser sessions.*/public async revokeCurrentTokens(): Promise<void> {try {diff --git a/core/client/src/api/errors/AppErrorCodes.ts b/core/client/src/api/errors/AppErrorCodes.tsindex 13c8eaa50..b9ca7ecf7 100644--- a/core/client/src/api/errors/AppErrorCodes.ts+++ b/core/client/src/api/errors/AppErrorCodes.ts@@ -50,6 +50,8 @@ export enum AppErrorCode {SYNC_VAULT_FETCH_FAILED = 'E-502', // Failed to fetch vault from serverSYNC_VAULT_DECRYPT_FAILED = 'E-503', // Failed to decrypt server vaultSYNC_STORE_FAILED = 'E-504', // Failed to store synced vault locally+ SYNC_SERVER_UNREACHABLE = 'E-505', // Server unreachable and no local vault to fall back on+ SYNC_SERVER_ERROR = 'E-506', // The server answered a sync request with an unexpected HTTP failure// Storage read/write errors (E-6xx)STORAGE_READ_FAILED = 'E-601',@@ -146,6 +148,8 @@ export function getErrorTranslationKey(code: AppErrorCode): string {[AppErrorCode.SYNC_VAULT_FETCH_FAILED]: 'common.errors.unknownErrorTryAgain',[AppErrorCode.SYNC_VAULT_DECRYPT_FAILED]: 'common.errors.unknownErrorTryAgain',[AppErrorCode.SYNC_STORE_FAILED]: 'common.errors.unknownErrorTryAgain',+ [AppErrorCode.SYNC_SERVER_UNREACHABLE]: 'common.errors.serverNotAvailable',+ [AppErrorCode.SYNC_SERVER_ERROR]: 'common.errors.unknownErrorTryAgain',// Storage read/write errors (E-6xx)[AppErrorCode.STORAGE_READ_FAILED]: 'common.errors.unknownErrorTryAgain',@@ -155,7 +159,7 @@ export function getErrorTranslationKey(code: AppErrorCode): string {// Merge operations (E-7xx)[AppErrorCode.MERGE_FAILED]: 'common.errors.mergeFailed',- [AppErrorCode.MERGE_CONFLICT]: 'common.errors.mergeFailed',+ [AppErrorCode.MERGE_CONFLICT]: 'common.errors.syncConflictMaxRetries',[AppErrorCode.MERGE_UPLOAD_FAILED]: 'common.errors.unknownErrorTryAgain',// Upload operations (E-8xx)diff --git a/core/client/src/auth/SrpAuthService.ts b/core/client/src/auth/SrpAuthService.tsindex 1d9879024..9be787c33 100644--- a/core/client/src/auth/SrpAuthService.ts+++ b/core/client/src/auth/SrpAuthService.ts@@ -1,14 +1,8 @@-import {- srpGenerateSalt,- srpDerivePrivateKey,- srpDeriveVerifier,- srpGenerateEphemeral,- srpDeriveSession,-} from '../../wasm/aliasvault_core.js';import { createAccountKeyHierarchy, type AccountKeyBlobs, type AccountKeyHierarchy } from '../crypto/AccountKeys';import { EncryptionUtility } from '../crypto/EncryptionUtility';-import { initRustCore } from '../rust/RustCore';+import { rustCore } from '../rust/RustCore';+import type { SrpEphemeral, SrpSession } from '../rust/RustCoreTypes';import type { TokenModel, LoginResponse, BadRequestResponse } from '@aliasvault/models/webapi';/**@@ -85,22 +79,6 @@ export const DEFAULT_ENCRYPTION = {}),} as const;-/**- * SRP ephemeral key pair type.- */-type SrpEphemeral = {- public: string;- secret: string;-};--/**- * SRP session type.- */-type SrpSession = {- proof: string;- key: string;-};-/*** SrpAuthService provides SRP-based authentication utilities using Rust WASM.*@@ -129,8 +107,7 @@ export class SrpAuthService {* @returns A random salt string (uppercase hex)*/public static async generateSalt(): Promise<string> {- await initRustCore();- return srpGenerateSalt();+ return rustCore().srpGenerateSalt();}/**@@ -169,8 +146,7 @@ export class SrpAuthService {username: string,passwordHashString: string): Promise<string> {- await initRustCore();- return srpDerivePrivateKey(salt, SrpAuthService.normalizeUsername(username), passwordHashString);+ return rustCore().srpDerivePrivateKey(salt, SrpAuthService.normalizeUsername(username), passwordHashString);}/**@@ -180,8 +156,7 @@ export class SrpAuthService {* @returns The SRP verifier (uppercase hex)*/public static async deriveVerifier(privateKey: string): Promise<string> {- await initRustCore();- return srpDeriveVerifier(privateKey);+ return rustCore().srpDeriveVerifier(privateKey);}/**@@ -190,8 +165,7 @@ export class SrpAuthService {* @returns Object containing public and secret ephemeral values (uppercase hex)*/public static async generateEphemeral(): Promise<SrpEphemeral> {- await initRustCore();- return srpGenerateEphemeral() as SrpEphemeral;+ return rustCore().srpGenerateEphemeral();}/**@@ -211,14 +185,13 @@ export class SrpAuthService {username: string,privateKey: string): Promise<SrpSession> {- await initRustCore();- return srpDeriveSession(+ return rustCore().srpDeriveSession(clientSecretEphemeral,serverPublicEphemeral,salt,SrpAuthService.normalizeUsername(username),privateKey- ) as SrpSession;+ );}/**diff --git a/core/client/src/constants/StorageKeys.ts b/core/client/src/constants/StorageKeys.tsindex 3c47e9afe..0d8cd9eda 100644--- a/core/client/src/constants/StorageKeys.ts+++ b/core/client/src/constants/StorageKeys.ts@@ -62,7 +62,7 @@ export const StorageKeys = {MUTATION_SEQUENCE: 'local:mutationSequence',/** True when the client operates against the local vault only. */IS_OFFLINE_MODE: 'local:isOfflineMode',- /** Message of the last failed sync attempt, shown in the UI. */+ /** The last failed sync attempt, as the translation key and error code the UI renders from. */LAST_SYNC_ERROR: 'local:lastSyncError',/** The client's last known server revision per manifest. */SERVER_MANIFEST_REVISIONS: 'local:serverManifestRevisions',diff --git a/core/client/src/index.ts b/core/client/src/index.tsindex 243185799..d88981be0 100644--- a/core/client/src/index.ts+++ b/core/client/src/index.ts@@ -1,7 +1,7 @@/*** @aliasvault/client: Typescript AliasVault client core.*- * Services are imported from their subpath (`@aliasvault/client/sync/VaultSyncService`). This root only exposes the+ * Services are imported from their subpath (`@aliasvault/client/sync/VaultSyncEngine`). This root only exposes the* platform contract a host app registers at startup.*/export * from './platform/index';diff --git a/core/client/src/rust/RustCore.ts b/core/client/src/rust/RustCore.tsindex aea30519b..ea8dc7193 100644--- a/core/client/src/rust/RustCore.ts+++ b/core/client/src/rust/RustCore.ts@@ -1,45 +1,36 @@/**- * Typed wrapper around the AliasVault Rust core (shared with iOS, Android,- * and the Blazor client). The browser ships the core as WebAssembly; that- * detail is encapsulated here so callers can think in terms of plain- * TypeScript functions.+ * Typed wrapper around the AliasVault Rust core (shared with iOS, Android, and the Blazor client).*- * Algorithms (URL matching, credential filtering, domain extraction) live in- * `core/rust/src/credential_matcher`. This file only handles init and- * adapts inputs/outputs to TypeScript types.+ * Algorithms (URL matching, credential filtering, domain extraction, the vault codec) live in `core/rust`.*/import { resolveDefaultLanguage } from '@aliasvault/models/defaults';import { FieldKey } from '@aliasvault/models/vault';-import initWasm, * as core from '../../wasm/aliasvault_core.js';import { getPlatform } from '../platform/ClientPlatform';import { deviceLanguage } from '../platform/DeviceLanguage';+import { AutofillMatchingMode } from './RustCoreTypes';++import type { IRustCore } from './RustCoreBinding';+import type { CodecBucketLayoutEntry, CodecCanonicalized, CodecCanonicalizeInput, CodecCanonicalMergeInput, CodecCanonicalMergeOutput, CodecDataBucket, CodecManifest, CodecMaterialized, CodecValidation, FaviconTarget, IdentityNameInput, IdentityRequest, ParsedEmail, SharingAccessPartition, SharingManifestRecord, SharingWriteSet } from './RustCoreTypes';import type { Identity } from '@aliasvault/models/identity';import type { Item, PasswordSettings } from '@aliasvault/models/vault';-export enum AutofillMatchingMode {- DEFAULT = 'default',- URL_EXACT = 'url_exact',- URL_SUBDOMAIN = 'url_subdomain'-}+export { AutofillMatchingMode } from './RustCoreTypes';+export type { CodecBlobEntry, CodecBucketLayoutEntry, CodecCanonicalized, CodecCanonicalizeInput, CodecCanonicalMergeInput, CodecCanonicalMergeOutput, CodecDataBucket, CodecManifest, CodecMaterialized, CodecValidation, FaviconTarget, IdentityNameInput, IdentityRequest, ParsedEmail, SharingAccessPartition, SharingManifestRecord, SharingWriteSet } from './RustCoreTypes';-let initPromise: Promise<void> | null = null;+/**+ * The host's Rust core binding.+ */+export function rustCore(): IRustCore {+ return getPlatform().rustCore;+}/*** Initialize the Rust core.*/export function initRustCore(): Promise<void> {- if (!initPromise) {- initPromise = (async (): Promise<void> => {- await initWasm({ module_or_path: await getPlatform().loadRustCoreWasm() });- })().catch((error: unknown) => {- // Let the next caller retry instead of pinning a failed load for the lifetime of the caller.- initPromise = null;- throw error;- });- }- return initPromise;+ return rustCore().init();}/**@@ -49,8 +40,7 @@ export function initRustCore(): Promise<void> {* reversed-TLD app bundle identifiers like `com.example.app`.*/export async function extractDomain(url: string): Promise<string> {- await initRustCore();- return core.extractDomain(url);+ return rustCore().extractDomain(url);}/**@@ -58,18 +48,9 @@ export async function extractDomain(url: string): Promise<string> {* Example: `sub.example.co.uk` > `example.co.uk`.*/export async function extractRootDomain(domain: string): Promise<string> {- await initRustCore();- return core.extractRootDomain(domain);+ return rustCore().extractRootDomain(domain);}-/**- * The URL a favicon is fetched from, paired with the `Logos.Source` key it is stored under.- */-export type FaviconTarget = {- url: string;- source: string;-};-/*** Read a URL field value as an ordered list, accepting the single-string and multi-value* shapes a field can hold.@@ -88,30 +69,20 @@ export function toUrlList(urlValue: string | string[] | undefined | null): strin* Returns null when no URL qualifies.*/export async function selectFaviconTarget(urls: string[]): Promise<FaviconTarget | null> {- await initRustCore();- return core.selectFaviconTarget(urls) as FaviconTarget | null;+ return rustCore().selectFaviconTarget(urls);}/*** Generate a password or passphrase from the given settings.- *- * The `Type` field selects the generator: `'basic'` (character-set password)- * or `'diceware'` (wordlist passphrase). Generation runs in the Rust core.- *- * Seed is an optional 64-character hex string (32 bytes) that seeds the RNG for deterministic generation- * primarily for UI comparison purposes. All normal password generation is non-deterministic.*/export async function generatePassword(settings: PasswordSettings, seed?: string): Promise<string> {- await initRustCore();const effective = await applyEffectiveDicewareLanguage(settings);const payload = seed ? { ...effective, Seed: seed } : effective;- return core.generatePassword(JSON.stringify(payload));+ return rustCore().generatePassword(JSON.stringify(payload));}/*** Resolve the effective Diceware passphrase language when none is explicitly chosen.- *- * The passphrase language is left empty by default ("auto").*/async function applyEffectiveDicewareLanguage(settings: PasswordSettings): Promise<PasswordSettings> {if (settings.Type !== 'diceware' || (settings.Language && settings.Language.trim().length > 0)) {@@ -123,59 +94,32 @@ async function applyEffectiveDicewareLanguage(settings: PasswordSettings): Promi/*** Get the list of bundled Diceware wordlist language ISO codes (first is the default, 'en').- * The set is owned by the Rust core; unknown codes fall back to English during generation.*/export async function getDicewareLanguages(): Promise<string[]> {- await initRustCore();- const languages = core.getDicewareLanguages() as string[];+ const languages = await rustCore().getDicewareLanguages();return languages.length > 0 ? languages : ['en'];}-/**- * Request for {@link generateIdentity}. All fields except `language` are optional.- */-export type IdentityRequest = {- /** Dictionary language code (e.g. 'en'); unknown codes fall back to English. */- language: string;- /** Gender preference: 'male', 'female' or 'random' (default). */- gender?: string;- /** Age range preference as stored in settings (e.g. '21-25' or 'random'). */- ageRange?: string;-};--/**- * Name and birth date input for identity-based username/email prefix generation.- */-export type IdentityNameInput = {- firstName: string;- lastName: string;- /** Birth date; only the leading yyyy year part is used. */- birthDate: string;-};-/*** Generate a random identity (alias persona) in the Rust core.* Returns the identity with a yyyy-MM-dd birth date string.*/export async function generateIdentity(request: IdentityRequest): Promise<Identity> {- await initRustCore();- return JSON.parse(core.generateIdentity(JSON.stringify(request))) as Identity;+ return JSON.parse(await rustCore().generateIdentity(JSON.stringify(request))) as Identity;}/*** Generate a username from persona name fields (alphanumeric, 6-20 characters).*/export async function generateIdentityUsername(input: IdentityNameInput): Promise<string> {- await initRustCore();- return core.generateIdentityUsername(JSON.stringify(input));+ return rustCore().generateIdentityUsername(JSON.stringify(input));}/*** Generate an email prefix from persona name fields (6-20 characters).*/export async function generateIdentityEmailPrefix(input: IdentityNameInput): Promise<string> {- await initRustCore();- return core.generateIdentityEmailPrefix(JSON.stringify(input));+ return rustCore().generateIdentityEmailPrefix(JSON.stringify(input));}/**@@ -183,8 +127,7 @@ export async function generateIdentityEmailPrefix(input: IdentityNameInput): Pro* Used for login-type credentials where no persona fields are available.*/export async function generateRandomEmailPrefix(length: number = 14): Promise<string> {- await initRustCore();- return core.generateRandomEmailPrefix(length);+ return rustCore().generateRandomEmailPrefix(length);}/**@@ -192,8 +135,7 @@ export async function generateRandomEmailPrefix(length: number = 14): Promise<st* The set is owned by the Rust core; unknown codes fall back to English during generation.*/export async function getIdentityLanguages(): Promise<string[]> {- await initRustCore();- const languages = core.getIdentityLanguages() as string[];+ const languages = await rustCore().getIdentityLanguages();return languages.length > 0 ? languages : ['en'];}@@ -201,59 +143,35 @@ export async function getIdentityLanguages(): Promise<string[]> {* Get the list of identity age range option values ('random' plus 5-year ranges).*/export async function getIdentityAgeRanges(): Promise<string[]> {- await initRustCore();- return core.getIdentityAgeRanges() as string[];+ return rustCore().getIdentityAgeRanges();}-/**- * A single attachment of a parsed email message. Metadata only: fetch the bytes with- * {@link extractEmailAttachment} using this attachment's index in {@link ParsedEmail.attachments}.- */-export type ParsedEmailAttachment = {- filename: string;- mimeType: string;- size: number;- detached: boolean;- partIndex: number | null;-};--/** Result of parsing a raw RFC 822 email source. Body fields are null when the message has no such part. */-export type ParsedEmail = {- htmlBody: string | null;- textBody: string | null;- attachments: ParsedEmailAttachment[];-};-/*** Parse a raw RFC 822 email source into its html/plain bodies and attachment metadata.*/export async function parseEmailSource(source: Uint8Array): Promise<ParsedEmail> {- await initRustCore();- return core.parseEmailSource(source) as ParsedEmail;+ return rustCore().parseEmailSource(source);}/*** Turn a stored email source into the raw RFC 822 message bytes for showing the message source without parsing it.*/export async function decodeEmailSource(source: Uint8Array): Promise<Uint8Array> {- await initRustCore();- return core.decodeEmailSource(source) as Uint8Array;+ return rustCore().decodeEmailSource(source);}/*** Extract the decoded bytes of one attachment, identified by its index in the parsed attachment list.*/export async function extractEmailAttachment(source: Uint8Array, index: number, detachedBody?: Uint8Array): Promise<Uint8Array> {- await initRustCore();- return core.extractEmailAttachment(source, index, detachedBody) as Uint8Array;+ return rustCore().extractEmailAttachment(source, index, detachedBody);}/*** Derive a 32-byte key from a password using Argon2id.*/export async function argon2DeriveKey(password: string, salt: string, encryptionSettings: string): Promise<Uint8Array> {- await initRustCore();- return core.argon2DeriveKey(password, salt, encryptionSettings);+ return rustCore().argon2DeriveKey(password, salt, encryptionSettings);}/**@@ -269,26 +187,19 @@ export function generateSeed(): string {/*** Filter items by URL/title for autofill. Returns at most 3 matches.*/-export async function filterItems(- items: Item[],- currentUrl: string,- pageTitle: string,- matchingMode: AutofillMatchingMode = AutofillMatchingMode.DEFAULT-): Promise<Item[]> {- await initRustCore();-+export async function filterItems(items: Item[], currentUrl: string, pageTitle: string, matchingMode: AutofillMatchingMode = AutofillMatchingMode.DEFAULT): Promise<Item[]> {const credentials = items.map(item => ({Id: item.Id,ItemName: item.Name ?? '',ItemUrls: getFieldValues(item, FieldKey.LoginUrl)}));- const result = core.filterCredentials({+ const result = await rustCore().filterCredentials({credentials,current_url: currentUrl,page_title: pageTitle,matching_mode: matchingMode- }) as { matched_ids: string[] };+ });return result.matched_ids.map(id => items.find(item => item.Id === id))@@ -302,183 +213,57 @@ export async function filterItems(* extractor returns no domain (app bundle identifiers).*/export async function isUrlAlreadyLinked(existingUrls: string[], newUrl: string): Promise<boolean> {- await initRustCore();- const newKey = urlComparisonKey(newUrl);+ const newKey = await urlComparisonKey(newUrl);if (!newKey) {return false;}- return existingUrls.some(existing => urlComparisonKey(existing) === newKey);+ const existingKeys = await Promise.all(existingUrls.map(existing => urlComparisonKey(existing)));+ return existingKeys.some(key => key === newKey);}/**- * Synchronous host-only comparison key. Caller must ensure `initRustCore()`- * has resolved before calling. Exposed as a sync helper for tight loops- * (see `isUrlAlreadyLinked`); async callers should use `extractDomain`.+ * Host-only comparison key of a URL: its domain, or the lowercased URL when the extractor yields none.*/-function urlComparisonKey(url: string): string {+async function urlComparisonKey(url: string): Promise<string> {const trimmed = url.trim().toLowerCase();if (!trimmed) {return trimmed;}- const domain = core.extractDomain(trimmed);+ const domain = await rustCore().extractDomain(trimmed);return domain.length > 0 ? domain : trimmed;}/** Vault codec (manifest-v1 storage format).- *- * The format logic (canonicalize/materialize, canonical hash + integrity envelope, gzip pack/unpack,- * structural validation, blob diff) lives in `core/rust/src/vault_codec`. These wrappers adapt the- * WASM exports to TypeScript types.*/-/** A single table's rows (byte columns rendered as `{ __b64 }`). */-export type CodecTableData = { name: string; records: Array<Record<string, unknown>> };--/**- * Manifest-v1 manifest. Forward-compat: unknown keys are preserved on round-trip.- *- * Carries no data-model (EF migration) version: each manifest is written by whichever client last- * pushed it, and a reader materializes into the schema it ships with anyway (anything that schema- * cannot hold is preserved structurally by `CodecOverflow`). `schemaVersion` is the wire-structure- * version and is a separate axis.- */-export type CodecManifest = {- schemaVersion: number;- manifestSalt: string;- canonicalizedAt: string;- manifestId: string;- name?: string | null;- tables: Record<string, Array<Record<string, unknown>>>;- [key: string]: unknown;-};--/**- * One manifest canonicalize should emit.- */-export type CodecManifestSpec = {- manifestId: string;- manifestSalt: string;- name?: string | null;-};--/**- * A manifest-v1 data bucket, addressed by the manifest that owns it and its category.- */-export type CodecDataBucket = {- schemaVersion: number;- manifestId: string;- category: string;- tables: Record<string, Array<Record<string, unknown>>>;- [key: string]: unknown;-};--/** A decoded blob entry: kind + plaintext bytes (base64). */-export type CodecBlobEntry = { kind: string; bytesBase64: string };--/** One manifest produced by canonicalize: the manifest and the blobs hashed with its own salt. */-export type CodecCanonicalizedManifest = { manifest: CodecManifest; blobs: Record<string, CodecBlobEntry> };--/**- * Result of canonicalize: one entry per spec, in spec order, plus the vault's data buckets.- */-export type CodecCanonicalized = {- manifests: CodecCanonicalizedManifest[];- dataBuckets: CodecDataBucket[];-};--/**- * Data a newer writer put in the manifest that this client's local SQLite schema cannot hold:- * whole unknown manifest tables, whole unknown bucket tables (per category), and unknown columns- * keyed by table > row primary-key value.- */-export type CodecOverflow = {- tables: Record<string, Array<Record<string, unknown>>>;- bucketTables: Record<string, Record<string, Array<Record<string, unknown>>>>;- columns: Record<string, Record<string, Record<string, unknown>>>;-};--/** Input for canonicalize. */-export type CodecCanonicalizeInput = {- tables: CodecTableData[];- canonicalizedAt: string;- manifests: CodecManifestSpec[];- /** For legacy sqlite-blob migration: the manifest that unstamped rows are adopted into. TODO: delete this field once the migration is complete. */- adoptUnstampedInto?: string | null;-};--/**- * Materialized tables the platform inserts into a fresh SQLite DB (`overflow` is a diagnostics copy).- * No migration id: the DB is created from this client's own schema, which is the only thing that- * decides which migration the result is at.- */-export type CodecMaterialized = { tables: CodecTableData[]; overflow: CodecOverflow };--/** One entry in the bucket layout: a category and the tables it owns. */-export type CodecBucketLayoutEntry = { category: string; tables: string[] };--/** Structural validation outcome. */-export type CodecValidation = { ok: boolean; failedRules: string[]; message: string };-/*** Canonicalize normalized tables into manifest + data buckets + blob map.*/export async function vaultCodecCanonicalizeFromSqlite(input: CodecCanonicalizeInput): Promise<CodecCanonicalized> {- await initRustCore();- return core.vaultCodecCanonicalizeFromSqlite(input) as CodecCanonicalized;+ return rustCore().vaultCodecCanonicalizeFromSqlite(input);}/*** Materialize the vault's manifests + data buckets into the table set the platform inserts.*/-export async function vaultCodecMaterializeAsSqlite(- manifests: CodecManifest[],- dataBuckets: CodecDataBucket[],- schemaColumns: Record<string, string[]>-): Promise<CodecMaterialized> {- await initRustCore();- return core.vaultCodecMaterializeAsSqlite({ manifests, dataBuckets, schemaColumns }) as CodecMaterialized;-}--/** Input of the canonical merge: server side is the base, local side the incoming changes. */-export type CodecCanonicalMergeInput = {- serverManifests: CodecManifest[];- serverBuckets: CodecDataBucket[];- contentlessServerManifestIds: string[];- localManifests: CodecManifest[];- localBuckets: CodecDataBucket[];- schemaColumns: Record<string, string[]>;-};--/** One manifest's merged result: the server manifest with merged tables, plus its merged buckets. */-export type CodecCanonicalManifestMerge = {- manifestId: string;- manifest: CodecManifest;- buckets: CodecDataBucket[];- stats: { tablesProcessed: number; recordsFromLocal: number; recordsFromServer: number; recordsCreatedLocally: number; conflicts: number; recordsInserted: number };-};--/** Output of the canonical merge. */-export type CodecCanonicalMergeOutput = {- manifests: CodecCanonicalManifestMerge[];- droppedLocalManifestIds: string[];-};+export async function vaultCodecMaterializeAsSqlite(manifests: CodecManifest[], dataBuckets: CodecDataBucket[], schemaColumns: Record<string, string[]>): Promise<CodecMaterialized> {+ return rustCore().vaultCodecMaterializeAsSqlite({ manifests, dataBuckets, schemaColumns });+}/*** Merge the local canonical vault onto the server canonical vault (the base), one manifest at a* time, rows out.*/export async function vaultCodecMergeCanonical(input: CodecCanonicalMergeInput): Promise<CodecCanonicalMergeOutput> {- await initRustCore();- return core.mergeCanonical(input) as CodecCanonicalMergeOutput;+ return rustCore().mergeCanonical(input);}/*** Extract the encryption-key row whose `PublicKey` matches `publicKey` from a decrypted manifest.*/export async function vaultCodecExtractEncryptionKeyForPublicKey(manifest: CodecManifest, publicKey: string): Promise<Record<string, unknown> | null> {- await initRustCore();- return (core.vaultCodecExtractEncryptionKeyForPublicKey(manifest, publicKey) ?? null) as Record<string, unknown> | null;+ return rustCore().vaultCodecExtractEncryptionKeyForPublicKey(manifest, publicKey);}/**@@ -486,32 +271,28 @@ export async function vaultCodecExtractEncryptionKeyForPublicKey(manifest: Codec* path).*/export async function vaultCodecExtractBuckets(category: string, manifestIds: string[], tables: Record<string, Array<Record<string, unknown>>>): Promise<CodecDataBucket[]> {- await initRustCore();- return core.vaultCodecExtractBuckets({ category, manifestIds, tables }) as CodecDataBucket[];+ return rustCore().vaultCodecExtractBuckets({ category, manifestIds, tables });}/*** The name of the client-local SQLite table that carries the codec overflow inside the vault DB.*/export async function vaultCodecOverflowTable(): Promise<string> {- await initRustCore();- return core.vaultCodecOverflowTable();+ return rustCore().vaultCodecOverflowTable();}/*** The vault tables that take part in sync, as declared by the shared vault table registry.*/export async function getSyncableTableNames(): Promise<string[]> {- await initRustCore();- return core.getSyncableTableNames();+ return rustCore().getSyncableTableNames();}/*** The bucket layout: every category and the tables it owns.*/export async function vaultCodecBucketLayout(): Promise<CodecBucketLayoutEntry[]> {- await initRustCore();- return core.vaultCodecBucketLayout() as CodecBucketLayoutEntry[];+ return rustCore().vaultCodecBucketLayout();}/**@@ -524,16 +305,14 @@ export async function vaultCodecBucketLayout(): Promise<CodecBucketLayoutEntry[]* overwrite each other.*/export async function vaultCodecLogoIdForSource(manifestId: string, source: string): Promise<string> {- await initRustCore();- return core.vaultCodecLogoIdForSource(manifestId, source);+ return rustCore().vaultCodecLogoIdForSource(manifestId, source);}/*** The `Logos.Id` to use for the logo `(kind, source)` inside the manifest with id `manifestId`.*/export async function vaultCodecLogoIdFor(manifestId: string, kind: string, source: string): Promise<string> {- await initRustCore();- return core.vaultCodecLogoIdFor(manifestId, kind, source);+ return rustCore().vaultCodecLogoIdFor(manifestId, kind, source);}/**@@ -541,56 +320,49 @@ export async function vaultCodecLogoIdFor(manifestId: string, kind: string, sour* under, which is what makes picking the same image again reuse the row that already holds it.*/export async function vaultCodecLogoContentHash(bytes: Uint8Array): Promise<string> {- await initRustCore();- return core.vaultCodecLogoContentHash(bytes);+ return rustCore().vaultCodecLogoContentHash(bytes);}/*** Generate a fresh 32-byte per-manifest blob-hashing salt (lowercase hex).*/export async function vaultCodecGenerateManifestSalt(): Promise<string> {- await initRustCore();- return core.vaultCodecGenerateManifestSalt();+ return rustCore().vaultCodecGenerateManifestSalt();}/*** Pack a payload JSON string into gzip(envelope{contentHash, payload}). The caller encrypts the result.*/export async function vaultCodecPackPayload(payloadJson: string): Promise<Uint8Array> {- await initRustCore();- return core.vaultCodecPackPayload(payloadJson);+ return rustCore().vaultCodecPackPayload(payloadJson);}/*** Unpack a (decrypted) payload: gunzip > verify content hash > return the payload JSON string.*/export async function vaultCodecUnpackPayload(plainBytes: Uint8Array): Promise<string> {- await initRustCore();- return core.vaultCodecUnpackPayload(plainBytes);+ return rustCore().vaultCodecUnpackPayload(plainBytes);}/*** Structurally validate a manifest before upload.*/export async function vaultCodecValidateManifest(manifest: CodecManifest): Promise<CodecValidation> {- await initRustCore();- return core.vaultCodecValidateManifest(manifest) as CodecValidation;+ return rustCore().vaultCodecValidateManifest(manifest);}/*** Validate a data bucket before upload.*/export async function vaultCodecValidateDataBucket(bucket: CodecDataBucket): Promise<CodecValidation> {- await initRustCore();- return core.vaultCodecValidateDataBucket(bucket) as CodecValidation;+ return rustCore().vaultCodecValidateDataBucket(bucket);}/*** SHA-256 (lowercase hex) of a base64 ciphertext string.*/export async function vaultCodecComputeCiphertextHash(base64Ciphertext: string): Promise<string> {- await initRustCore();- return core.vaultCodecComputeCiphertextHash(base64Ciphertext);+ return rustCore().vaultCodecComputeCiphertextHash(base64Ciphertext);}/**@@ -599,63 +371,23 @@ export async function vaultCodecComputeCiphertextHash(base64Ciphertext: string):* every platform uses the same fingerprinting algorithm.*/export async function vaultCodecComputeContentFingerprint(payloadJson: string): Promise<string> {- await initRustCore();- return core.vaultCodecComputeContentFingerprint(payloadJson);+ return rustCore().vaultCodecComputeContentFingerprint(payloadJson);}-/** A shared manifest's key record. */-export type SharingManifestRecord = {- manifestId: string;- salt: string;- name?: string | null;- canAdminister?: boolean;-};--/** One manifest the next push writes. */-export type SharingWriteRecord = {- manifestId: string;- isPersonal: boolean;- salt: string;- name: string | null;- canAdminister: boolean;-};--/** The manifests a push writes, personal first, plus what was left out and why. */-export type SharingWriteSet = {- records: SharingWriteRecord[];- skipped: Array<{ manifestId: string; reason: 'NO_ROWS_IN_VAULT' | 'KEY_DID_NOT_OPEN' }>;-};--/** What the vault holds but cannot write, and what it holds but has lost access to. */-export type SharingAccessPartition = { unwritable: string[]; lost: string[] };-/*** Work out which manifests the next push writes, personal manifest first.* @param input - the personal manifest, what the vault holds rows for, what opened, and the held records.*/-export async function vaultSharingResolveManifestWriteSet(input: {- personalManifestId: string;- personalManifestSalt: string;- stampedManifestIds: string[];- openedManifestIds: string[];- heldRecords: SharingManifestRecord[];- displayNames: Record<string, string>;-}): Promise<SharingWriteSet> {- await initRustCore();- return core.vaultSharingResolveManifestWriteSet(input) as SharingWriteSet;+export async function vaultSharingResolveManifestWriteSet(input: { personalManifestId: string; personalManifestSalt: string; stampedManifestIds: string[]; openedManifestIds: string[]; heldRecords: SharingManifestRecord[]; displayNames: Record<string, string> }): Promise<SharingWriteSet> {+ return rustCore().vaultSharingResolveManifestWriteSet(input);}/*** Split what the local vault holds by what this account can still open.* @param input - what the vault holds, what this session can write, and what the last snapshot served.*/-export async function vaultSharingPartitionManifestAccess(input: {- manifestIdsInVault: string[];- writableManifestIds: string[];- grantedManifestIds: string[];-}): Promise<SharingAccessPartition> {- await initRustCore();- return core.vaultSharingPartitionManifestAccess(input) as SharingAccessPartition;+export async function vaultSharingPartitionManifestAccess(input: { manifestIdsInVault: string[]; writableManifestIds: string[]; grantedManifestIds: string[] }): Promise<SharingAccessPartition> {+ return rustCore().vaultSharingPartitionManifestAccess(input);}/**diff --git a/core/client/src/rust/RustCoreBinding.ts b/core/client/src/rust/RustCoreBinding.tsnew file mode 100644index 000000000..bdd0f74be--- /dev/null+++ b/core/client/src/rust/RustCoreBinding.ts@@ -0,0 +1,75 @@+import type { CodecBucketLayoutEntry, CodecCanonicalized, CodecCanonicalizeInput, CodecCanonicalMergeInput, CodecCanonicalMergeOutput, CodecDataBucket, CodecExtractBucketsInput, CodecManifest, CodecMaterialized, CodecMaterializeInput, CodecValidation, FaviconTarget, FilterCredentialsInput, FilterCredentialsOutput, ParsedEmail, PruneTableQuery, PruneVaultInput, PruneVaultOutput, SharingAccessPartition, SharingPartitionAccessInput, SharingResolveWriteSetInput, SharingWriteSet, SrpEphemeral, SrpSession } from './RustCoreTypes';++/**+ * One running operation of the Rust vault sync engine.+ */+export interface IVaultSyncSession {+ /** The next command for the host, as JSON. */+ nextCommand(): Promise<string>;++ /** Hand the host's response to the last command back, as JSON. */+ resume(responseJson: string): Promise<void>;++ /** Release the session. */+ free(): void;+}++/**+ * The Rust core as the client core sees it: one function per exported core operation, every call asynchronous so a host may route it over a native bridge.+ */+export interface IRustCore {+ init(): Promise<void>;++ extractDomain(url: string): Promise<string>;+ extractRootDomain(domain: string): Promise<string>;+ selectFaviconTarget(urls: string[]): Promise<FaviconTarget | null>;+ filterCredentials(input: FilterCredentialsInput): Promise<FilterCredentialsOutput>;++ generatePassword(settingsJson: string): Promise<string>;+ getDicewareLanguages(): Promise<string[]>;+ generateIdentity(requestJson: string): Promise<string>;+ generateIdentityUsername(inputJson: string): Promise<string>;+ generateIdentityEmailPrefix(inputJson: string): Promise<string>;+ generateRandomEmailPrefix(length: number): Promise<string>;+ getIdentityLanguages(): Promise<string[]>;+ getIdentityAgeRanges(): Promise<string[]>;++ parseEmailSource(source: Uint8Array): Promise<ParsedEmail>;+ decodeEmailSource(source: Uint8Array): Promise<Uint8Array>;+ extractEmailAttachment(source: Uint8Array, index: number, detachedBody?: Uint8Array): Promise<Uint8Array>;++ argon2DeriveKey(password: string, salt: string, encryptionSettings: string): Promise<Uint8Array>;++ srpGenerateSalt(): Promise<string>;+ srpDerivePrivateKey(salt: string, identity: string, passwordHash: string): Promise<string>;+ srpDeriveVerifier(privateKey: string): Promise<string>;+ srpGenerateEphemeral(): Promise<SrpEphemeral>;+ srpDeriveSession(clientSecret: string, serverPublic: string, salt: string, identity: string, privateKey: string): Promise<SrpSession>;++ getSyncableTableNames(): Promise<string[]>;+ mergeCanonical(input: CodecCanonicalMergeInput): Promise<CodecCanonicalMergeOutput>;+ pruneVault(input: PruneVaultInput): Promise<PruneVaultOutput>;+ getPruneTableQueries(): Promise<PruneTableQuery[]>;++ vaultCodecCanonicalizeFromSqlite(input: CodecCanonicalizeInput): Promise<CodecCanonicalized>;+ vaultCodecMaterializeAsSqlite(input: CodecMaterializeInput): Promise<CodecMaterialized>;+ vaultCodecExtractBuckets(input: CodecExtractBucketsInput): Promise<CodecDataBucket[]>;+ vaultCodecBucketLayout(): Promise<CodecBucketLayoutEntry[]>;+ vaultCodecOverflowTable(): Promise<string>;+ vaultCodecGenerateManifestSalt(): Promise<string>;+ vaultCodecLogoIdForSource(manifestId: string, source: string): Promise<string>;+ vaultCodecLogoIdFor(manifestId: string, kind: string, source: string): Promise<string>;+ vaultCodecLogoContentHash(bytes: Uint8Array): Promise<string>;+ vaultCodecPackPayload(payloadJson: string): Promise<Uint8Array>;+ vaultCodecUnpackPayload(plainBytes: Uint8Array): Promise<string>;+ vaultCodecValidateManifest(manifest: CodecManifest): Promise<CodecValidation>;+ vaultCodecValidateDataBucket(bucket: CodecDataBucket): Promise<CodecValidation>;+ vaultCodecComputeCiphertextHash(base64Ciphertext: string): Promise<string>;+ vaultCodecComputeContentFingerprint(payloadJson: string): Promise<string>;+ vaultCodecExtractEncryptionKeyForPublicKey(manifest: CodecManifest, publicKey: string): Promise<Record<string, unknown> | null>;++ vaultSharingResolveManifestWriteSet(input: SharingResolveWriteSetInput): Promise<SharingWriteSet>;+ vaultSharingPartitionManifestAccess(input: SharingPartitionAccessInput): Promise<SharingAccessPartition>;++ createVaultSyncSession(requestJson: string): Promise<IVaultSyncSession>;+}diff --git a/core/client/src/rust/RustCoreTypes.ts b/core/client/src/rust/RustCoreTypes.tsnew file mode 100644index 000000000..bcf4014f6--- /dev/null+++ b/core/client/src/rust/RustCoreTypes.ts@@ -0,0 +1,254 @@+/**+ * Types shared by the Rust core binding and the typed wrappers in RustCore.ts.+ */++export enum AutofillMatchingMode {+ DEFAULT = 'default',+ URL_EXACT = 'url_exact',+ URL_SUBDOMAIN = 'url_subdomain'+}++/**+ * The URL a favicon is fetched from, paired with the `Logos.Source` key it is stored under.+ */+export type FaviconTarget = {+ url: string;+ source: string;+};++/**+ * Request for identity generation. All fields except `language` are optional.+ */+export type IdentityRequest = {+ /** Dictionary language code (e.g. 'en'); unknown codes fall back to English. */+ language: string;+ /** Gender preference: 'male', 'female' or 'random' (default). */+ gender?: string;+ /** Age range preference as stored in settings (e.g. '21-25' or 'random'). */+ ageRange?: string;+};++/**+ * Name and birth date input for identity-based username/email prefix generation.+ */+export type IdentityNameInput = {+ firstName: string;+ lastName: string;+ /** Birth date; only the leading yyyy year part is used. */+ birthDate: string;+};++/**+ * A single attachment of a parsed email message. Metadata only: fetch the bytes with+ * `extractEmailAttachment` using this attachment's index in {@link ParsedEmail.attachments}.+ */+export type ParsedEmailAttachment = {+ filename: string;+ mimeType: string;+ size: number;+ detached: boolean;+ partIndex: number | null;+};++/** Result of parsing a raw RFC 822 email source. Body fields are null when the message has no such part. */+export type ParsedEmail = {+ htmlBody: string | null;+ textBody: string | null;+ attachments: ParsedEmailAttachment[];+};++/** Credential filter input for autofill matching, as the Rust matcher expects it. */+export type FilterCredentialsInput = {+ credentials: Array<{ Id: string; ItemName: string; ItemUrls: string[] }>;+ current_url: string;+ page_title: string;+ matching_mode: AutofillMatchingMode;+};++/** Credential filter output: the matched item ids. */+export type FilterCredentialsOutput = { matched_ids: string[] };++/** SRP ephemeral key pair (uppercase hex). */+export type SrpEphemeral = {+ public: string;+ secret: string;+};++/** SRP client session: proof (M1) and shared key (K), uppercase hex. */+export type SrpSession = {+ proof: string;+ key: string;+};++/** One table's rows as the trash pruner reads them. */+export type RustTableData = { name: string; records: Array<Record<string, unknown>> };++/** One SQL statement the pruner emits, with `{ __b64 }` byte params. */+export type RustSqlStatement = { sql: string; params: Array<string | number | null | { __b64: string }> };++/** Input of the trash pruner. */+export type PruneVaultInput = { tables: RustTableData[]; retention_days: number; current_time: string };++/** Output of the trash pruner. */+export type PruneVaultOutput = { success: boolean; statements: RustSqlStatement[] };++/** One per-table SELECT the pruner reads its input with. */+export type PruneTableQuery = { name: string; query: string };++/*+ * Vault codec (manifest-v1 storage format).+ */++/** A single table's rows (byte columns rendered as `{ __b64 }`). */+export type CodecTableData = { name: string; records: Array<Record<string, unknown>> };++/**+ * Manifest-v1 manifest.+ */+export type CodecManifest = {+ schemaVersion: number;+ manifestSalt: string;+ canonicalizedAt: string;+ manifestId: string;+ name?: string | null;+ tables: Record<string, Array<Record<string, unknown>>>;+ [key: string]: unknown;+};++/**+ * One manifest canonicalize should emit.+ */+export type CodecManifestSpec = {+ manifestId: string;+ manifestSalt: string;+ name?: string | null;+};++/**+ * A manifest-v1 data bucket, addressed by the manifest that owns it and its category.+ */+export type CodecDataBucket = {+ schemaVersion: number;+ manifestId: string;+ category: string;+ tables: Record<string, Array<Record<string, unknown>>>;+ [key: string]: unknown;+};++/** A decoded blob entry: kind + plaintext bytes (base64). */+export type CodecBlobEntry = { kind: string; bytesBase64: string };++/** One manifest produced by canonicalize: the manifest and the blobs hashed with its own salt. */+export type CodecCanonicalizedManifest = { manifest: CodecManifest; blobs: Record<string, CodecBlobEntry> };++/**+ * Result of canonicalize: one entry per spec, in spec order, plus the vault's data buckets.+ */+export type CodecCanonicalized = {+ manifests: CodecCanonicalizedManifest[];+ dataBuckets: CodecDataBucket[];+};++/**+ * Data a newer writer put in the manifest that this client's local SQLite schema cannot hold.+ */+export type CodecOverflow = {+ tables: Record<string, Array<Record<string, unknown>>>;+ bucketTables: Record<string, Record<string, Array<Record<string, unknown>>>>;+ columns: Record<string, Record<string, Record<string, unknown>>>;+};++/** Input for canonicalize. */+export type CodecCanonicalizeInput = {+ tables: CodecTableData[];+ canonicalizedAt: string;+ manifests: CodecManifestSpec[];+ /** For legacy sqlite-blob migration: the manifest that unstamped rows are adopted into. TODO: delete this field once the migration is complete. */+ adoptUnstampedInto?: string | null;+};++/** Input for materialize. */+export type CodecMaterializeInput = {+ manifests: CodecManifest[];+ dataBuckets: CodecDataBucket[];+ schemaColumns: Record<string, string[]>;+};++/** Materialized tables the platform inserts into a fresh SQLite DB (`overflow` is a diagnostics copy). */+export type CodecMaterialized = { tables: CodecTableData[]; overflow: CodecOverflow };++/** Input for the bucket-only extraction. */+export type CodecExtractBucketsInput = { category: string; manifestIds: string[]; tables: Record<string, Array<Record<string, unknown>>> };++/** One entry in the bucket layout: a category and the tables it owns. */+export type CodecBucketLayoutEntry = { category: string; tables: string[] };++/** Structural validation outcome. */+export type CodecValidation = { ok: boolean; failedRules: string[]; message: string };++/** Input of the canonical merge: server side is the base, local side the incoming changes. */+export type CodecCanonicalMergeInput = {+ serverManifests: CodecManifest[];+ serverBuckets: CodecDataBucket[];+ contentlessServerManifestIds: string[];+ localManifests: CodecManifest[];+ localBuckets: CodecDataBucket[];+ schemaColumns: Record<string, string[]>;+};++/** One manifest's merged result: the server manifest with merged tables, plus its merged buckets. */+export type CodecCanonicalManifestMerge = {+ manifestId: string;+ manifest: CodecManifest;+ buckets: CodecDataBucket[];+ stats: { tablesProcessed: number; recordsFromLocal: number; recordsFromServer: number; recordsCreatedLocally: number; conflicts: number; recordsInserted: number };+};++/** Output of the canonical merge. */+export type CodecCanonicalMergeOutput = {+ manifests: CodecCanonicalManifestMerge[];+ droppedLocalManifestIds: string[];+};++/** A shared manifest's key record. */+export type SharingManifestRecord = {+ manifestId: string;+ salt: string;+ name?: string | null;+ canAdminister?: boolean;+};++/** One manifest the next push writes. */+export type SharingWriteRecord = {+ manifestId: string;+ isPersonal: boolean;+ salt: string;+ name: string | null;+ canAdminister: boolean;+};++/** The manifests a push writes, personal first, plus what was left out and why. */+export type SharingWriteSet = {+ records: SharingWriteRecord[];+ skipped: Array<{ manifestId: string; reason: 'NO_ROWS_IN_VAULT' | 'KEY_DID_NOT_OPEN' }>;+};++/** Input of the manifest write-set resolution. */+export type SharingResolveWriteSetInput = {+ personalManifestId: string;+ personalManifestSalt: string;+ stampedManifestIds: string[];+ openedManifestIds: string[];+ heldRecords: SharingManifestRecord[];+ displayNames: Record<string, string>;+};++/** Input of the manifest access partition. */+export type SharingPartitionAccessInput = {+ manifestIdsInVault: string[];+ writableManifestIds: string[];+ grantedManifestIds: string[];+};++/** What the vault holds but cannot write, and what it holds but has lost access to. */+export type SharingAccessPartition = { unwritable: string[]; lost: string[] };diff --git a/core/client/src/rust/WasmRustCore.ts b/core/client/src/rust/WasmRustCore.tsnew file mode 100644index 000000000..b42fe5ecb--- /dev/null+++ b/core/client/src/rust/WasmRustCore.ts@@ -0,0 +1,110 @@+/*+ * The Rust core bound to its WebAssembly build, for hosts with a WebAssembly runtime.+ */+/* eslint-disable jsdoc/require-jsdoc */+import initWasm, * as core from '../../wasm/aliasvault_core.js';++import type { IRustCore, IVaultSyncSession } from './RustCoreBinding';+import type { CodecBucketLayoutEntry, CodecCanonicalized, CodecCanonicalizeInput, CodecCanonicalMergeInput, CodecCanonicalMergeOutput, CodecDataBucket, CodecExtractBucketsInput, CodecManifest, CodecMaterialized, CodecMaterializeInput, CodecValidation, FaviconTarget, FilterCredentialsInput, FilterCredentialsOutput, ParsedEmail, PruneTableQuery, PruneVaultInput, PruneVaultOutput, SharingAccessPartition, SharingPartitionAccessInput, SharingResolveWriteSetInput, SharingWriteSet, SrpEphemeral, SrpSession } from './RustCoreTypes';++/**+ * Where the host gets the `.wasm` binary from: bytes, or a fetch response for streaming instantiation.+ */+export type WasmLoader = () => Promise<BufferSource | Response>;++/**+ * Bind the Rust core to its WebAssembly build.+ * @param loadWasm - provides the WebAssembly binary+ */+export function createWasmRustCore(loadWasm: WasmLoader): IRustCore {+ let initPromise: Promise<void> | null = null;++ /**+ * Instantiate the module.+ */+ const init = (): Promise<void> => {+ if (!initPromise) {+ initPromise = (async (): Promise<void> => {+ await initWasm({ module_or_path: await loadWasm() });+ })().catch((error: unknown) => {+ initPromise = null;+ throw error;+ });+ }+ return initPromise;+ };++ /**+ * Run a Rust core function after initialization.+ * @param call - the synchronous core call+ */+ const ready = async <T>(call: () => T): Promise<T> => {+ await init();+ return call();+ };++ return {+ init,++ extractDomain: (url): Promise<string> => ready(() => core.extractDomain(url)),+ extractRootDomain: (domain): Promise<string> => ready(() => core.extractRootDomain(domain)),+ selectFaviconTarget: (urls): Promise<FaviconTarget | null> => ready(() => (core.selectFaviconTarget(urls) ?? null) as FaviconTarget | null),+ filterCredentials: (input: FilterCredentialsInput): Promise<FilterCredentialsOutput> => ready(() => core.filterCredentials(input) as FilterCredentialsOutput),++ generatePassword: (settingsJson): Promise<string> => ready(() => core.generatePassword(settingsJson)),+ getDicewareLanguages: (): Promise<string[]> => ready(() => core.getDicewareLanguages()),+ generateIdentity: (requestJson): Promise<string> => ready(() => core.generateIdentity(requestJson)),+ generateIdentityUsername: (inputJson): Promise<string> => ready(() => core.generateIdentityUsername(inputJson)),+ generateIdentityEmailPrefix: (inputJson): Promise<string> => ready(() => core.generateIdentityEmailPrefix(inputJson)),+ generateRandomEmailPrefix: (length): Promise<string> => ready(() => core.generateRandomEmailPrefix(length)),+ getIdentityLanguages: (): Promise<string[]> => ready(() => core.getIdentityLanguages()),+ getIdentityAgeRanges: (): Promise<string[]> => ready(() => core.getIdentityAgeRanges()),++ parseEmailSource: (source): Promise<ParsedEmail> => ready(() => core.parseEmailSource(source) as ParsedEmail),+ decodeEmailSource: (source): Promise<Uint8Array> => ready(() => core.decodeEmailSource(source)),+ extractEmailAttachment: (source, index, detachedBody): Promise<Uint8Array> => ready(() => core.extractEmailAttachment(source, index, detachedBody)),++ argon2DeriveKey: (password, salt, encryptionSettings): Promise<Uint8Array> => ready(() => core.argon2DeriveKey(password, salt, encryptionSettings)),++ srpGenerateSalt: (): Promise<string> => ready(() => core.srpGenerateSalt()),+ srpDerivePrivateKey: (salt, identity, passwordHash): Promise<string> => ready(() => core.srpDerivePrivateKey(salt, identity, passwordHash)),+ srpDeriveVerifier: (privateKey): Promise<string> => ready(() => core.srpDeriveVerifier(privateKey)),+ srpGenerateEphemeral: (): Promise<SrpEphemeral> => ready(() => core.srpGenerateEphemeral() as SrpEphemeral),+ srpDeriveSession: (clientSecret, serverPublic, salt, identity, privateKey): Promise<SrpSession> =>+ ready(() => core.srpDeriveSession(clientSecret, serverPublic, salt, identity, privateKey) as SrpSession),++ getSyncableTableNames: (): Promise<string[]> => ready(() => core.getSyncableTableNames()),+ mergeCanonical: (input: CodecCanonicalMergeInput): Promise<CodecCanonicalMergeOutput> => ready(() => core.mergeCanonical(input) as CodecCanonicalMergeOutput),+ pruneVault: (input: PruneVaultInput): Promise<PruneVaultOutput> => ready(() => core.pruneVault(input) as PruneVaultOutput),+ getPruneTableQueries: (): Promise<PruneTableQuery[]> => ready(() => core.getPruneTableQueries() as PruneTableQuery[]),++ vaultCodecCanonicalizeFromSqlite: (input: CodecCanonicalizeInput): Promise<CodecCanonicalized> => ready(() => core.vaultCodecCanonicalizeFromSqlite(input) as CodecCanonicalized),+ vaultCodecMaterializeAsSqlite: (input: CodecMaterializeInput): Promise<CodecMaterialized> => ready(() => core.vaultCodecMaterializeAsSqlite(input) as CodecMaterialized),+ vaultCodecExtractBuckets: (input: CodecExtractBucketsInput): Promise<CodecDataBucket[]> => ready(() => core.vaultCodecExtractBuckets(input) as CodecDataBucket[]),+ vaultCodecBucketLayout: (): Promise<CodecBucketLayoutEntry[]> => ready(() => core.vaultCodecBucketLayout() as CodecBucketLayoutEntry[]),+ vaultCodecOverflowTable: (): Promise<string> => ready(() => core.vaultCodecOverflowTable()),+ vaultCodecGenerateManifestSalt: (): Promise<string> => ready(() => core.vaultCodecGenerateManifestSalt()),+ vaultCodecLogoIdForSource: (manifestId, source): Promise<string> => ready(() => core.vaultCodecLogoIdForSource(manifestId, source)),+ vaultCodecLogoIdFor: (manifestId, kind, source): Promise<string> => ready(() => core.vaultCodecLogoIdFor(manifestId, kind, source)),+ vaultCodecLogoContentHash: (bytes): Promise<string> => ready(() => core.vaultCodecLogoContentHash(bytes)),+ vaultCodecPackPayload: (payloadJson): Promise<Uint8Array> => ready(() => core.vaultCodecPackPayload(payloadJson)),+ vaultCodecUnpackPayload: (plainBytes): Promise<string> => ready(() => core.vaultCodecUnpackPayload(plainBytes)),+ vaultCodecValidateManifest: (manifest: CodecManifest): Promise<CodecValidation> => ready(() => core.vaultCodecValidateManifest(manifest) as CodecValidation),+ vaultCodecValidateDataBucket: (bucket: CodecDataBucket): Promise<CodecValidation> => ready(() => core.vaultCodecValidateDataBucket(bucket) as CodecValidation),+ vaultCodecComputeCiphertextHash: (base64Ciphertext): Promise<string> => ready(() => core.vaultCodecComputeCiphertextHash(base64Ciphertext)),+ vaultCodecComputeContentFingerprint: (payloadJson): Promise<string> => ready(() => core.vaultCodecComputeContentFingerprint(payloadJson)),+ vaultCodecExtractEncryptionKeyForPublicKey: (manifest: CodecManifest, publicKey): Promise<Record<string, unknown> | null> => ready(() => (core.vaultCodecExtractEncryptionKeyForPublicKey(manifest, publicKey) ?? null) as Record<string, unknown> | null),++ createVaultSyncSession: (requestJson): Promise<IVaultSyncSession> => ready((): IVaultSyncSession => {+ const session = new core.VaultSyncSession(requestJson);+ return {+ nextCommand: async (): Promise<string> => session.nextCommand(),+ resume: async (responseJson: string): Promise<void> => session.resume(responseJson),+ free: (): void => session.free(),+ };+ }),++ vaultSharingResolveManifestWriteSet: (input: SharingResolveWriteSetInput): Promise<SharingWriteSet> => ready(() => core.vaultSharingResolveManifestWriteSet(input) as SharingWriteSet),+ vaultSharingPartitionManifestAccess: (input: SharingPartitionAccessInput): Promise<SharingAccessPartition> => ready(() => core.vaultSharingPartitionManifestAccess(input) as SharingAccessPartition),+ };+}diff --git a/core/client/src/sync/EmailRouting.ts b/core/client/src/sync/EmailRouting.tsdeleted file mode 100644index 7fe649d38..000000000--- a/core/client/src/sync/EmailRouting.ts+++ /dev/null@@ -1,61 +0,0 @@-import { FieldKey } from '@aliasvault/models/vault';--import type { EmailRoutingPush } from './EmailRoutingPush';-import type { CodecManifest } from '../rust/RustCore';--/**- * Build the email routing set a push sends to the server from the canonicalized manifests.- *- * The push is one entry per (address, manifest) pair: an address carried by several manifests is emitted once per manifest,- * and the server links the claim to each of them so incoming mail gets one key wrap per linked manifest.- *- * @param manifests - every manifest one canonicalize run produced, the user's own included- * @param privateEmailDomains - domains the server hosts mail for; addresses outside them are not claimed- * @returns The (address, manifest) pairs to claim, each with its routing state, plus the manifests they were read from- */-export function buildEmailRouting(manifests: CodecManifest[], privateEmailDomains: string[]): EmailRoutingPush {- const byPair = new Map<string, { address: string, manifestId: string, paused: boolean }>();-- for (const manifest of manifests) {- const liveItemIds = new Set(- rowsOf(manifest, 'Items').filter(row => !row.IsDeleted && row.DeletedAt == null).map(row => String(row.Id))- );-- for (const fieldValue of rowsOf(manifest, 'FieldValues')) {- if (fieldValue.FieldKey !== FieldKey.LoginEmail || fieldValue.IsDeleted) {- continue;- }- if (!liveItemIds.has(String(fieldValue.ItemId))) {- continue;- }-- const address = typeof fieldValue.Value === 'string' ? fieldValue.Value.trim().toLowerCase() : '';- const domain = address.split('@')[1];- if (!domain || !privateEmailDomains.includes(domain)) {- continue;- }-- const pairKey = `${address}\0${manifest.manifestId}`;- const existing = byPair.get(pairKey);- const paused = Boolean(fieldValue.IsDisabled);- if (existing) {- // Several items in one manifest may carry the same address; one of them still wanting mail keeps it routed.- existing.paused = existing.paused && paused;- } else {- byPair.set(pairKey, { address, manifestId: manifest.manifestId, paused });- }- }- }-- return { emailAddressList: [...byPair.values()], coveredManifestIds: manifests.map(manifest => manifest.manifestId) };-}--/**- * The rows of one manifest table, or an empty list when the manifest does not carry that table.- * @param manifest - the manifest to read from- * @param table - the table name- * @returns The table's rows- */-function rowsOf(manifest: CodecManifest, table: string): Array<Record<string, unknown>> {- return manifest.tables[table] ?? [];-}diff --git a/core/client/src/sync/EmailRoutingPush.ts b/core/client/src/sync/EmailRoutingPush.tsdeleted file mode 100644index a8675bead..000000000--- a/core/client/src/sync/EmailRoutingPush.ts+++ /dev/null@@ -1,13 +0,0 @@-/**- * The email-routing set sent on a vault push: one entry per (address, manifest) pair. An address claimed by- * several manifests (personal + shared during family sharing) appears once per manifest; the server links the- * claim to each so incoming mail is key-wrapped per linked manifest.- */-export type EmailRoutingPush = {- emailAddressList: Array<{ address: string; manifestId: string; paused: boolean }>;- /**- * The manifests this push speaks for: every manifest that went into the address list above, including the ones- * that hold no alias.- */- coveredManifestIds: string[];-};diff --git a/core/client/src/sync/PendingActionProcessor.ts b/core/client/src/sync/PendingActionProcessor.tsdeleted file mode 100644index 7f0e1782b..000000000--- a/core/client/src/sync/PendingActionProcessor.ts+++ /dev/null@@ -1,117 +0,0 @@-import { devLog, devWarn } from '../platform/Logger';-import { SharingService } from '../sharing/SharingService';--import type { WebApiService } from '../api/WebApiService';-import type { SqliteClient } from '../database/SqliteClient';-import type { PendingClientAction, StatusResponseV2 } from '@aliasvault/models/webapi';--/**- * Carries out work the server hands this client with the status response.- */--/**- * The action types this build knows how to carry out. The tokens are the server's `ClientActionType` names.- */-const ACTION_ROTATE_MANIFEST_DELIVERY_KEY = 'RotateManifestDeliveryKey';--/**- * What one action left behind: whether the local vault changed, and whether the action is finished and may be retired server-side.- */-type ActionOutcome = { vaultChanged: boolean; completed: boolean };--/**- * Runs the server's outstanding work for this client.- */-export class PendingActionProcessor {- /**- * The actions the server has addressed to this client.- * @param status - the status response of the sync in progress.- */- public static pendingActions(status: StatusResponseV2): PendingClientAction[] {- return status.pendingActions ?? [];- }-- /**- * Run every action this client understands and can carry out right now, reporting each one done as it lands.- * @param webApi - API client to reuse.- * @param actions - the actions from {@link pendingActions}.- * @param sqliteClient - the open local vault, mutated in place by the actions that write to it.- * @returns Whether the vault was mutated and therefore needs to be persisted and pushed.- */- public static async process(webApi: WebApiService, actions: PendingClientAction[], sqliteClient: SqliteClient): Promise<boolean> {- let mutated = false;- let completed = 0;-- for (const action of actions) {- try {- const outcome = await this.runAction(action, sqliteClient);- mutated = outcome.vaultChanged || mutated;-- if (outcome.completed) {- await this.complete(webApi, action.id);- completed++;- }- } catch (error) {- devWarn(`[PendingActions] Could not carry out action ${action.type} (${action.id}); retrying on the next sync.`, error);- }- }-- devLog(`[PendingActions] ${completed} of ${actions.length} action(s) carried out${mutated ? '; the vault changed and rides out on this sync' : ''}.`);-- return mutated;- }-- /**- * Carry out one action, if this build knows the type and this session holds what it needs.- * @param action - the action to run.- * @param sqliteClient - the open local vault.- */- private static async runAction(action: PendingClientAction, sqliteClient: SqliteClient): Promise<ActionOutcome> {- switch (action.type) {- case ACTION_ROTATE_MANIFEST_DELIVERY_KEY:- return this.rotateDeliveryKey(action, sqliteClient);- default:- devLog(`[PendingActions] Action type ${action.type} is not known to this client; leaving it for one that knows it.`);- return { vaultChanged: false, completed: false };- }- }-- /**- * Replace a shared manifest's mail delivery keypair, which the server asks for after somebody who held the old private half was removed from the group.- * @param action - the action naming the manifest to rotate.- * @param sqliteClient - the open local vault.- */- private static async rotateDeliveryKey(action: PendingClientAction, sqliteClient: SqliteClient): Promise<ActionOutcome> {- const record = Object.values(await SharingService.getSharedManifestRecords()).find(candidate => idsEqual(candidate.manifestId, action.manifestId));-- if (!record?.canAdminister) {- devLog(`[PendingActions] Manifest ${action.manifestId} is not open to this session as an administrator; leaving its delivery key rotation for another sync.`);- return { vaultChanged: false, completed: false };- }-- await SharingService.rotateManifestEncryptionKey(sqliteClient, record.manifestId);- devLog(`[PendingActions] Rotated the mail delivery key of shared manifest ${record.manifestId}.`);-- return { vaultChanged: true, completed: true };- }-- /**- * Report an action done.- * @param webApi - API client to reuse.- * @param actionId - the action that was carried out.- */- private static async complete(webApi: WebApiService, actionId: string): Promise<void> {- await webApi.delete<void>(`ClientActions/${actionId}`);- }-}--/**- * Whether two ids name the same row, which travel in whatever casing their producer used.- * @param a - first id.- * @param b - second id.- */-function idsEqual(a: string | null | undefined, b: string | null | undefined): boolean {- return typeof a === 'string' && typeof b === 'string' && a.toLowerCase() === b.toLowerCase();-}--export default PendingActionProcessor;diff --git a/core/client/src/sync/VaultManifestMigration.ts b/core/client/src/sync/VaultManifestMigration.tsindex 5d7612b3f..b81c5502a 100644--- a/core/client/src/sync/VaultManifestMigration.ts+++ b/core/client/src/sync/VaultManifestMigration.ts@@ -7,7 +7,7 @@ import type { SqliteClient } from '../database/SqliteClient';* answered entirely from local state.** Two conditions, carried independently of each other:- * 1. A stale local schema, rebuilt via `VaultSyncService.migrateVaultToCurrentSchema`. This is the permanent path:+ * 1. A stale local schema, rebuilt by the sync engine's `migrateManifest` operation. This is the permanent path:* it delivered the initial sqlite-blob to manifest-v1 transition and fires again after every schema change we ship.* 2. A missing account key hierarchy ({@link requiresLegacyAccountKeyMigration}), one-time and deletable.*diff --git a/core/client/src/sync/VaultMergeService.ts b/core/client/src/sync/VaultMergeService.tsdeleted file mode 100644index 1c0bd86ad..000000000--- a/core/client/src/sync/VaultMergeService.ts+++ /dev/null@@ -1,219 +0,0 @@-import { TRASH_RETENTION_DAYS } from '../constants/Vault';-import { getPlatform } from '../platform/ClientPlatform';-import { devLog } from '../platform/Logger';-import { rustCore } from '../rust/RustCore';-import { base64ToBytes, bytesToBase64 } from '../utilities/Base64';--import type { SqliteClient } from '../database/SqliteClient';-import type { ISqliteDatabase, SqliteValue } from '../platform/SqliteEngine';-import type { RustSqlStatement, RustTableData } from '../rust/RustCoreTypes';--/**- * Record type for JSON data passed to/from Rust.- */-type JsonRecord = { [key: string]: unknown };--/**- * Result of a merge operation.- */-export type MergeResult = {- success: boolean;- mergedVaultBase64: string;- stats: MergeStats;-}--/**- * Statistics about what was merged.- */-export type MergeStats = {- tablesProcessed: number;- recordsFromLocal: number;- recordsFromServer: number;- recordsCreatedLocally: number;- conflicts: number;-}--/**- * Service for merging two vault SQLite databases using Last-Write-Wins (LWW) strategy.- *- * This implementation uses the Rust core for the merge logic, ensuring consistency- * across all platforms (browser, iOS, Android, server).- *- * The merge uses UpdatedAt timestamps on all SyncableEntity records to determine- * which version of a record wins in case of conflict.- */-export class VaultMergeService {- /**- * LEGACY: SQLite statement-level merge, kept only for the frozen sqlite-blob storage format (a- * not-yet-migrated local vault). Manifest-v1 vaults merge at canonical level instead, see- * `VaultSyncService.pullAndMerge`. TODO: delete this function once all users have migrated to manifest-v1.- *- * Uses the Rust core for the merge logic. The merge base is the SERVER vault (it is freshly- * materialized with the newest schema and the newest codec overflow carrier), and the local- * vault's winning changes are applied on top of it:- * 1. Load both SQLite databases- * 2. Read the syncable tables from both as JSON- * 3. Call Rust merge (returns SQL statements that bring local changes onto the server base)- * 4. Execute SQL statements on the SERVER database- * 5. Export the merged (server-based) database - the codec overflow carrier rides along untouched- *- * @param localVaultBase64 - The local vault (with offline changes) as base64 SQLite- * @param serverVaultBase64 - The server vault (latest version) as base64 SQLite- * @returns MergeResult with the merged vault as base64- */- public async merge(localVaultBase64: string, serverVaultBase64: string): Promise<MergeResult> {- try {- const core = rustCore();-- // Load both databases- const localDb = await this.loadDatabase(localVaultBase64);- const serverDb = await this.loadDatabase(serverVaultBase64);-- try {- const tableNames = await core.getSyncableTableNames();-- // Read all tables from both databases as JSON- const localTables: RustTableData[] = tableNames.map(name => ({- name,- records: this.readTableAsJson(localDb, name),- }));-- const serverTables: RustTableData[] = tableNames.map(name => ({- name,- records: this.readTableAsJson(serverDb, name),- }));-- devLog('[VaultMerge] Merge input:', {- localTableCount: localTables.length,- serverTableCount: serverTables.length,- localTables: localTables.map(t => ({ name: t.name, recordCount: t.records.length })),- serverTables: serverTables.map(t => ({ name: t.name, recordCount: t.records.length })),- });-- // JSON roundtrip so no undefined value reaches Rust/serde.- const mergeOutput = await core.mergeVaults(JSON.parse(JSON.stringify({ local_tables: localTables, server_tables: serverTables })));-- /*- * Execute SQL statements from Rust on the SERVER database (the merge base). The exported- * server DB carries the newest codec overflow carrier untouched.- */- for (const stmt of mergeOutput.statements) {- serverDb.run(stmt.sql, stmt.params.map(toBindableParam));- }-- // Export the merged (server-based) database- const mergedVaultBase64 = this.exportDatabase(serverDb);-- return {- success: mergeOutput.success,- mergedVaultBase64,- stats: {- tablesProcessed: mergeOutput.stats.tablesProcessed,- recordsFromLocal: mergeOutput.stats.recordsFromLocal,- recordsFromServer: mergeOutput.stats.recordsFromServer,- recordsCreatedLocally: mergeOutput.stats.recordsCreatedLocally,- conflicts: mergeOutput.stats.conflicts,- },- };- } finally {- // Clean up databases- localDb.close();- serverDb.close();- }- } catch (error) {- console.error('Vault merge failed:', error);- throw error;- }- }-- /**- * Prune expired trash items directly on a live SQLite client: items that have been in the trash (DeletedAt- * set) for longer than the retention period are permanently deleted (IsDeleted = true).- *- * @param client - The live SQLite client to prune- * @param retentionDays - Number of days to keep items in trash (defaults to TRASH_RETENTION_DAYS)- * @returns Number of SQL statements executed (0 when nothing was expired)- */- public async pruneInPlace(client: SqliteClient, retentionDays: number = TRASH_RETENTION_DAYS): Promise<number> {- const core = rustCore();-- const tableQueries = await core.getPruneTableQueries();- const tables: RustTableData[] = tableQueries.map(({ name, query }) => ({ name, records: client.executeQuery<JsonRecord>(query) }));-- // JSON roundtrip converts undefined to null and ensures clean JSON types for Rust/serde.- const pruneOutput = await core.pruneVault(JSON.parse(JSON.stringify({ tables, retention_days: retentionDays, current_time: new Date().toISOString() })));-- for (const stmt of pruneOutput.statements) {- client.executeUpdate(stmt.sql, stmt.params.map(toBindableParam));- }-- if (pruneOutput.statements.length > 0) {- devLog(`[VaultMerge] Pruned expired items from trash (${pruneOutput.statements.length} SQL statements executed)`);- }-- return pruneOutput.statements.length;- }-- /**- * Load a SQLite database from base64 string.- * @param base64String - The base64 encoded database- * @returns The loaded database- */- private loadDatabase(base64String: string): Promise<ISqliteDatabase> {- return getPlatform().sqlite.open(base64ToBytes(base64String));- }-- /**- * Export a SQLite database to base64 string.- * @param db - The database to export- * @returns The base64 encoded database- */- private exportDatabase(db: ISqliteDatabase): string {- db.exec('VACUUM');- return bytesToBase64(db.export());- }-- /**- * Read all records from a table as JSON objects.- * @param db - The database to query- * @param tableName - The name of the table- * @returns Array of records as JSON objects- */- private readTableAsJson(db: ISqliteDatabase, tableName: string): JsonRecord[] {- const exists = db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='${tableName}'`).length > 0;- if (!exists) {- return [];- }- return this.readQueryAsJson(db, `SELECT * FROM ${tableName}`);- }-- /**- * Read all rows of a query as JSON objects. BLOB columns come back from the engine as Uint8Array,- * which neither JSON nor serde can round-trip; they are encoded as `{ __b64 }` payloads that- * {@link toBindableParam} decodes when a merge statement writes them back.- * @param db - The database to query- * @param query - The SELECT query to run- * @returns Array of records as JSON objects- */- private readQueryAsJson(db: ISqliteDatabase, query: string): JsonRecord[] {- return db.query(query).map(row => {- const record: JsonRecord = {};- for (const [column, value] of Object.entries(row)) {- record[column] = value instanceof Uint8Array ? { __b64: bytesToBase64(value) } : value ?? null;- }- return record;- });- }-}--/**- * Decode one Rust-returned SQL parameter into a bindable value: `{ __b64 }` byte payloads- * become Uint8Array, undefined becomes null.- * @param param - the parameter as returned by the Rust merge- */-function toBindableParam(param: RustSqlStatement['params'][number] | undefined): SqliteValue {- if (param !== null && param !== undefined && typeof param === 'object' && '__b64' in param) {- return base64ToBytes(param.__b64);- }- return param ?? null;-}diff --git a/core/client/src/sync/VaultSyncEngine.ts b/core/client/src/sync/VaultSyncEngine.tsnew file mode 100644index 000000000..58497f9ac--- /dev/null+++ b/core/client/src/sync/VaultSyncEngine.ts@@ -0,0 +1,632 @@+/**+ * The TypeScript interface with the Rust vault sync engine.+ */++import { VaultSqlGenerator } from '@aliasvault/vault';++import { CapabilityService } from '../api/CapabilityService';+import { NetworkError } from '../api/errors/NetworkError';+import { RequestTimeoutError } from '../api/errors/RequestTimeoutError';+import { WebApiService } from '../api/WebApiService';+import { StorageKeys } from '../constants/StorageKeys';+import { AppInfo } from '../platform/AppInfo';+import { getPlatform } from '../platform/ClientPlatform';+import { devLog, devWarn } from '../platform/Logger';+import { TranslatableMessage } from '../platform/TranslatableMessage';+import { rustCore } from '../rust/RustCore';+import { base64ToBytes, bytesToBase64 } from '../utilities/Base64';++import { getDirtyScopes } from './VaultDirtyState';++import type { StorageKey } from '../platform/KeyValueStore';+import type { ISqliteDatabase, SqliteValue } from '../platform/SqliteEngine';++/** The operations the engine runs. */+export type VaultSyncOperation = 'fullSync' | 'migrationStatus' | 'migrateManifest' | 'statusCheck';++/** What the host hands the engine at session start (the Rust `SyncRequest`). */+export type VaultSyncEngineRequest = {+ operation: VaultSyncOperation;+ username: string;+ encryptionKey?: string;+ accountPublicKey?: string;+ accountPrivateKey?: string;+ isDirty: boolean;+ mutationSequence: number;+ dirtyScopes: string[];+ privateEmailDomains: string[];+ forcePull: boolean;+ minServerVersion: string;+ isOfflineMode: boolean;+ unnamedSharedVaultName: string;+};++/** Session values the engine changed and the host has to adopt. */+export type VaultSyncSessionUpdates = {+ encryptionKey?: string;+ accountPrivateKey?: string;+ clearAccountPrivateKey?: boolean;+};++/** The email routing a pulled vault came with. */+export type VaultSyncEmailRouting = {+ emailAddressList: string[];+ privateEmailDomainList: string[];+ hiddenPrivateEmailDomainList: string[];+ publicEmailDomainList: string[];+};++/** What every engine result carries. */+type EngineResultBase = {+ sessionUpdates: VaultSyncSessionUpdates;+ vaultChanged: boolean;+};++/** Outcome of a full sync (the Rust `SyncResult`). */+export type VaultSyncEngineResult = EngineResultBase & {+ success: boolean;+ hasNewVault: boolean;+ wasOffline: boolean;+ sqliteBlobUpgradeRequired: boolean;+ manifestMigrationRequired: boolean;+ error?: string;+ errorCode?: string;+ errorKey?: string;+ requiresLogout: boolean;+ serverVersion?: string;+ capabilities?: Record<string, string>;+ isOfflineMode: boolean;+ pulledRevision?: number;+ emailRouting?: VaultSyncEmailRouting;+};++/** Outcome of the migration classification. */+export type VaultSyncMigrationStatusResult = EngineResultBase & {+ kind: 'none' | 'schema-rebuild' | 'storage-format-upgrade';+ serverConfirmed: boolean;+};++/** Outcome of the manifest migration. */+export type VaultSyncMigrateManifestResult = EngineResultBase & {+ success: boolean;+ pushed: boolean;+ error?: string;+ errorCode?: string;+ errorKey?: string;+ requiresLogout: boolean;+};++/** Outcome of the lightweight status check. */+export type VaultSyncStatusCheckResult = EngineResultBase & {+ success: boolean;+ hasNewerVault: boolean;+ hasDirtyChanges: boolean;+ isOffline: boolean;+ requiresLogout: boolean;+ errorKey?: string;+ error?: string;+ errorCode?: string;+ serverVersion?: string;+ capabilities?: Record<string, string>;+};++/** What the engine asks to persist as the at-rest vault blob. */+export type VaultSyncStoreRequest = {+ encryptedBlob: string;+ markDirty: boolean;+ expectedMutationSeq?: number;+ revision?: number;+};++/** Outcome of a store: refused when the expected mutation sequence no longer matched. */+export type VaultSyncStoreOutcome = {+ success: boolean;+ mutationSequence: number;+};++/** What a running sync is doing: downloading a newer server vault, or uploading local changes. */+export type VaultSyncPhase = 'pull' | 'push';++/**+ * The engine's host interface.+ */+export interface IVaultSyncEngineHost {+ /**+ * The open local vault.+ */+ localDatabase(): Promise<ISqliteDatabase>;++ /**+ * The at-rest vault blob, or null when none is stored.+ */+ loadVault(): Promise<string | null>;++ /**+ * Persist the at-rest vault blob.+ */+ storeVault(request: VaultSyncStoreRequest): Promise<VaultSyncStoreOutcome>;++ /**+ * Clear the dirty flag unless a local mutation happened since the given sequence.+ */+ markClean(mutationSeqAtStart: number): Promise<boolean>;++ /**+ * Called when the run ends with changes written to the local database that no store persisted, so the host can+ * drop its in-memory copy and reload the stored vault.+ */+ discardLocalDatabase?(): void;++ /**+ * Called when the engine has decided what the sync is about to do.+ */+ onPhase?(phase: VaultSyncPhase): void;+}++/** A JSON value as the engine exchanges it. */+type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };++/** One parameterized statement of a `dbExec` command. */+type EngineSqlStatement = { sql: string; params?: JsonValue[] };++/** The engine's commands; see the Rust `Command` enum. */+type EngineCommand =+ | { kind: 'http'; method: string; path: string; body?: string; auth: boolean; largeTransfer: boolean }+ | { kind: 'stateGet'; key: string }+ | { kind: 'stateSet'; key: string; value: JsonValue }+ | { kind: 'stateRemove'; key: string }+ | { kind: 'dbOpen'; db: string; bytes?: string | null }+ | { kind: 'dbQuery'; db: string; sql: string; params: JsonValue[] }+ | { kind: 'dbExec'; db: string; statements: EngineSqlStatement[] }+ | { kind: 'dbExport'; db: string }+ | { kind: 'vaultStore'; encryptedBlob: string; markDirty: boolean; encryptionKey?: string; expectedMutationSeq?: number; revision?: number }+ | { kind: 'vaultLoad' }+ | { kind: 'markClean'; mutationSeqAtStart: number }+ | { kind: 'log'; level: 'log' | 'warn' | 'phase'; message: string }+ | { kind: 'done'; result: JsonValue };++/** The database names a command may address. */+const DB_LOCAL = 'local';+const DB_STAGING = 'staging';++/**+ * A cell as the engine binds it: `{ __b64 }` binds a BLOB, booleans bind as integers, objects as their JSON.+ * @param value - the JSON parameter+ */+function toBindValue(value: JsonValue | undefined): SqliteValue {+ if (value === null || value === undefined) {+ return null;+ }+ if (typeof value === 'string' || typeof value === 'number') {+ return value;+ }+ if (typeof value === 'boolean') {+ return value ? 1 : 0;+ }+ if (!Array.isArray(value) && typeof value.__b64 === 'string') {+ return base64ToBytes(value.__b64);+ }+ return JSON.stringify(value);+}++/**+ * A cell as the engine reads it: BLOB columns travel as `{ __b64 }`.+ * @param value - the SQLite value+ */+function toJsonCell(value: SqliteValue): JsonValue {+ return value instanceof Uint8Array ? { __b64: bytesToBase64(value) } : value;+}++/**+ * Run statements inside one transaction, rolling back when any fails.+ * @param db - the database+ * @param statements - the statements+ */+function execInTransaction(db: ISqliteDatabase, statements: EngineSqlStatement[]): void {+ db.exec('BEGIN');+ try {+ for (const statement of statements) {+ db.run(statement.sql, (statement.params ?? []).map(toBindValue));+ }+ db.exec('COMMIT');+ } catch (error) {+ try {+ db.exec('ROLLBACK');+ } catch {+ // The transaction is already gone; the original error is what matters.+ }+ throw error;+ }+}++/**+ * The storage key of an engine state key.+ * @param key - the engine's key+ */+function stateKey(key: string): StorageKey {+ if (key.includes(':')) {+ throw new Error(`The sync engine addressed a prefixed state key: ${key}`);+ }+ return `local:${key}` as StorageKey;+}++/** The SQLite bytes of a fresh database on the current client schema. */+let freshSchemaBytes: Promise<Uint8Array> | null = null;++/**+ * A fresh database on the current client schema.+ */+async function openFreshSchemaDatabase(): Promise<ISqliteDatabase> {+ freshSchemaBytes ??= (async (): Promise<Uint8Array> => {+ const db = await getPlatform().sqlite.open();+ try {+ db.exec(new VaultSqlGenerator().getCompleteSchemaSql());+ return db.export();+ } finally {+ db.close();+ }+ })().catch((error: unknown) => {+ freshSchemaBytes = null;+ throw error;+ });+ return getPlatform().sqlite.open(await freshSchemaBytes);+}++/**+ * Current time in milliseconds.+ */+function now(): number {+ return typeof performance !== 'undefined' ? performance.now() : Date.now();+}++/**+ * Carries out one engine run's commands.+ */+class EngineRun {+ private staging: ISqliteDatabase | null = null;++ /** The open local vault. */+ private local: ISqliteDatabase | null = null;++ /** Whether the local database was written to since the last store. */+ private localMutated = false;++ /** Time spent in the host per command kind (for debugging). */+ private readonly timings = new Map<string, { count: number; ms: number }>();++ /**+ * Create a run.+ * @param host - the host+ * @param webApi - the API the HTTP commands run on+ */+ public constructor(private readonly host: IVaultSyncEngineHost, private readonly webApi: WebApiService) {}++ /**+ * Carry out one command.+ * @param command - the command+ */+ public async handle(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<JsonValue> {+ const started = now();+ try {+ return await this.dispatch(command);+ } catch (error) {+ return { error: error instanceof Error ? error.message : String(error) };+ } finally {+ const timing = this.timings.get(command.kind) ?? { count: 0, ms: 0 };+ timing.count++;+ timing.ms += now() - started;+ this.timings.set(command.kind, timing);+ }+ }++ /**+ * One line of where the time went.+ * @param operation - the operation that ran+ * @param totalMs - the run's wall time+ */+ public summarize(operation: string, totalMs: number): string {+ let hostMs = 0;+ const parts: string[] = [];+ for (const [kind, timing] of [...this.timings.entries()].sort((a, b) => b[1].ms - a[1].ms)) {+ hostMs += timing.ms;+ parts.push(`${kind} ${timing.count}x ${timing.ms.toFixed(0)}ms`);+ }+ return `[VaultSyncEngine] ${operation} took ${totalMs.toFixed(0)}ms: engine ${(totalMs - hostMs).toFixed(0)}ms, host ${hostMs.toFixed(0)}ms (${parts.join(', ')})`;+ }++ /**+ * Release the staging database.+ */+ public close(): void {+ this.staging?.close();+ this.staging = null;+ if (this.localMutated) {+ this.host.discardLocalDatabase?.();+ }+ }++ /**+ * The command handlers.+ * @param command - the command+ */+ private async dispatch(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<JsonValue> {+ switch (command.kind) {+ case 'http':+ return this.http(command);+ case 'stateGet':+ return { value: await getPlatform().storage.get<JsonValue>(stateKey(command.key)) };+ case 'stateSet':+ await getPlatform().storage.set(stateKey(command.key), command.value);+ return {};+ case 'stateRemove':+ await getPlatform().storage.remove(stateKey(command.key));+ return {};+ case 'dbOpen':+ await this.openStaging(command.bytes ?? null);+ return {};+ case 'dbQuery': {+ const rows = (await this.database(command.db)).query(command.sql, command.params.map(toBindValue));+ return { rows: rows.map(row => Object.fromEntries(Object.entries(row).map(([column, value]) => [column, toJsonCell(value)]))) };+ }+ case 'dbExec':+ execInTransaction(await this.database(command.db), command.statements);+ if (command.db === DB_LOCAL) {+ this.localMutated = true;+ }+ return {};+ case 'dbExport':+ return { bytes: bytesToBase64((await this.database(command.db)).export()) };+ case 'vaultStore':+ return this.storeVault(command);+ case 'vaultLoad':+ return { encryptedBlob: await this.host.loadVault() };+ case 'markClean':+ return { cleared: await this.host.markClean(command.mutationSeqAtStart) };+ case 'log':+ this.log(command.level, command.message);+ return {};+ default:+ throw new Error(`Unknown engine command ${(command as { kind: string }).kind}`);+ }+ }++ /**+ * An API request.+ * @param command - the command+ */+ private async http(command: Extract<EngineCommand, { kind: 'http' }>): Promise<JsonValue> {+ try {+ const response = await this.webApi.engineRequest(command.method, command.path, command.body, command.auth, command.largeTransfer);+ return { status: response.status, body: response.body };+ } catch (error) {+ if (error instanceof NetworkError) {+ return { status: 0, transportError: error.message, timedOut: error instanceof RequestTimeoutError };+ }+ throw error;+ }+ }++ /**+ * Adopt the key a re-encrypted blob is under, then persist the blob.+ * @param command - the command+ */+ private async storeVault(command: Extract<EngineCommand, { kind: 'vaultStore' }>): Promise<JsonValue> {+ if (command.encryptionKey) {+ await getPlatform().storage.set(StorageKeys.ENCRYPTION_KEY, command.encryptionKey);+ }+ const outcome = await this.host.storeVault({+ encryptedBlob: command.encryptedBlob,+ markDirty: command.markDirty,+ expectedMutationSeq: command.expectedMutationSeq,+ revision: command.revision,+ });+ if (outcome.success) {+ // The local database is now the vault just stored; the host serves it on the next request.+ this.localMutated = false;+ this.local = null;+ }+ return { success: outcome.success, mutationSequence: outcome.mutationSequence };+ }++ /**+ * The database a command addresses.+ * @param name - `local` or `staging`+ */+ private async database(name: string): Promise<ISqliteDatabase> {+ if (name === DB_LOCAL) {+ this.local ??= await this.host.localDatabase();+ return this.local;+ }+ if (name === DB_STAGING) {+ if (!this.staging) {+ throw new Error('The staging database is not open');+ }+ return this.staging;+ }+ throw new Error(`Unknown database ${name}`);+ }++ /**+ * Open the staging database.+ * @param bytesBase64 - the SQLite file bytes, or null for a fresh database+ */+ private async openStaging(bytesBase64: string | null): Promise<void> {+ this.staging?.close();+ this.staging = null;+ const db = bytesBase64 ? await getPlatform().sqlite.open(base64ToBytes(bytesBase64)) : await openFreshSchemaDatabase();+ try {+ db.exec('PRAGMA foreign_keys = OFF');+ } catch (error) {+ db.close();+ throw error;+ }+ this.staging = db;+ }++ /**+ * Route an engine log line.+ * @param level - the level+ * @param message - the message+ */+ private log(level: string, message: string): void {+ switch (level) {+ case 'phase':+ this.host.onPhase?.(message as VaultSyncPhase);+ break;+ case 'warn':+ devWarn(message);+ break;+ default:+ devLog(message);+ }+ }+}++/**+ * Run one engine operation.+ * @param host - the host+ * @param request - the request+ * @param webApi - the API the HTTP commands run on+ */+export async function runVaultSyncEngine<T extends EngineResultBase>(host: IVaultSyncEngineHost, request: VaultSyncEngineRequest, webApi: WebApiService = new WebApiService()): Promise<T> {+ const started = now();+ const session = await rustCore().createVaultSyncSession(JSON.stringify(request));+ const run = new EngineRun(host, webApi);+ try {+ for (;;) {+ const command = JSON.parse(await session.nextCommand()) as EngineCommand;+ if (command.kind === 'done') {+ const result = command.result as T;+ await applySessionUpdates(result.sessionUpdates);+ devLog(run.summarize(request.operation, now() - started));+ return result;+ }+ await session.resume(JSON.stringify(await run.handle(command)));+ }+ } finally {+ run.close();+ session.free();+ }+}++/**+ * Adopt the session values the engine changed.+ * @param updates - the updates+ */+async function applySessionUpdates(updates: VaultSyncSessionUpdates | undefined): Promise<void> {+ if (!updates) {+ return;+ }+ const storage = getPlatform().storage;+ if (updates.encryptionKey) {+ await storage.set(StorageKeys.ENCRYPTION_KEY, updates.encryptionKey);+ }+ if (updates.accountPrivateKey) {+ await storage.set(StorageKeys.ACCOUNT_PRIVATE_KEY, updates.accountPrivateKey);+ } else if (updates.clearAccountPrivateKey) {+ await storage.remove(StorageKeys.ACCOUNT_PRIVATE_KEY);+ }+}++/** What a caller may ask of a full sync beyond what the revisions decide. */+export type VaultSyncOptions = {+ forcePull?: boolean;+};++/**+ * Build the engine request from the session state in the platform storage.+ * @param operation - the operation+ * @param options - what the caller asks beyond what the engine decides+ */+export async function buildVaultSyncRequest(operation: VaultSyncOperation, options: VaultSyncOptions = {}): Promise<VaultSyncEngineRequest> {+ const storage = getPlatform().storage;+ const [username, encryptionKey, accountPublicKey, accountPrivateKey, isDirty, mutationSequence, privateEmailDomains, isOfflineMode] = await Promise.all([+ storage.get<string>(StorageKeys.USERNAME),+ storage.get<string>(StorageKeys.ENCRYPTION_KEY),+ storage.get<string>(StorageKeys.ACCOUNT_PUBLIC_KEY),+ storage.get<string>(StorageKeys.ACCOUNT_PRIVATE_KEY),+ storage.get<boolean>(StorageKeys.IS_DIRTY),+ storage.get<number>(StorageKeys.MUTATION_SEQUENCE),+ storage.get<string[]>(StorageKeys.PRIVATE_EMAIL_DOMAINS),+ storage.get<boolean>(StorageKeys.IS_OFFLINE_MODE),+ ]);++ return {+ operation,+ username: username ?? '',+ encryptionKey: encryptionKey ?? undefined,+ accountPublicKey: accountPublicKey ?? undefined,+ accountPrivateKey: accountPrivateKey ?? undefined,+ isDirty: isDirty ?? false,+ mutationSequence: mutationSequence ?? 0,+ dirtyScopes: isDirty ? await getDirtyScopes() : [],+ privateEmailDomains: privateEmailDomains ?? [],+ forcePull: options.forcePull === true,+ minServerVersion: AppInfo.MIN_SERVER_VERSION,+ isOfflineMode: isOfflineMode ?? false,+ unnamedSharedVaultName: await getPlatform().translate(TranslatableMessage.UnnamedSharedVault),+ };+}++/**+ * Persist what a status call reported.+ * @param serverVersion - the server version, when the status call reached the server+ * @param capabilities - the resolved capabilities+ */+async function recordServerStatus(serverVersion: string | undefined, capabilities: Record<string, string> | undefined): Promise<void> {+ if (serverVersion && serverVersion !== '0.0.0') {+ await getPlatform().storage.set(StorageKeys.SERVER_VERSION, serverVersion);+ }+ if (capabilities) {+ await CapabilityService.storeCapabilities(capabilities);+ }+}++/**+ * Full vault sync.+ * @param host - the host+ * @param options - what the caller asks beyond what the engine decides+ * @param webApi - the API the HTTP commands run on+ */+export async function runFullVaultSync(host: IVaultSyncEngineHost, options: VaultSyncOptions = {}, webApi?: WebApiService): Promise<VaultSyncEngineResult> {+ const result = await runVaultSyncEngine<VaultSyncEngineResult>(host, await buildVaultSyncRequest('fullSync', options), webApi);+ const storage = getPlatform().storage;++ await recordServerStatus(result.serverVersion, result.capabilities);+ await storage.set(StorageKeys.IS_OFFLINE_MODE, result.isOfflineMode);+ if (result.emailRouting) {+ await storage.setMany([+ { key: StorageKeys.PUBLIC_EMAIL_DOMAINS, value: result.emailRouting.publicEmailDomainList },+ { key: StorageKeys.PRIVATE_EMAIL_DOMAINS, value: result.emailRouting.privateEmailDomainList },+ { key: StorageKeys.HIDDEN_PRIVATE_EMAIL_DOMAINS, value: result.emailRouting.hiddenPrivateEmailDomainList },+ ]);+ }+ return result;+}++/**+ * Run a status check.+ * @param host - the host+ * @param webApi - the API the HTTP commands run on+ */+export async function runVaultStatusCheck(host: IVaultSyncEngineHost, webApi?: WebApiService): Promise<VaultSyncStatusCheckResult> {+ const result = await runVaultSyncEngine<VaultSyncStatusCheckResult>(host, await buildVaultSyncRequest('statusCheck'), webApi);+ await recordServerStatus(result.serverVersion, result.capabilities);+ return result;+}++/**+ * Classify the pending local migration.+ * @param host - the host+ */+export function runVaultMigrationStatus(host: IVaultSyncEngineHost): Promise<VaultSyncMigrationStatusResult> {+ return buildVaultSyncRequest('migrationStatus').then(request => runVaultSyncEngine<VaultSyncMigrationStatusResult>(host, request));+}++/**+ * Run the local storage-model migration (schema rebuild and/or account key hierarchy) and push it.+ * @param host - the host+ */+export function runVaultManifestMigration(host: IVaultSyncEngineHost): Promise<VaultSyncMigrateManifestResult> {+ return buildVaultSyncRequest('migrateManifest').then(request => runVaultSyncEngine<VaultSyncMigrateManifestResult>(host, request));+}diff --git a/core/client/src/sync/VaultSyncService.ts b/core/client/src/sync/VaultSyncService.tsdeleted file mode 100644index 085281362..000000000--- a/core/client/src/sync/VaultSyncService.ts+++ /dev/null@@ -1,1915 +0,0 @@-/**- * VaultSyncService.- *- * Handles syncing the vault with the server and interfaces with the Rust codec.- */--import { ManifestKeyType, VaultKeyAlgorithm, type BucketRevision, type VaultResponse } from '@aliasvault/models/webapi';-import { VaultSqlGenerator } from '@aliasvault/vault';--import { VaultProcessingError } from '../api/errors/VaultProcessingError';-import { WebApiService } from '../api/WebApiService';-import { VaultKeyService } from '../auth/VaultKeyService';-import { bucketRevisionKey, StorageKeys } from '../constants/StorageKeys';-import { EncryptionUtility } from '../crypto/EncryptionUtility';-import { encryptVaultBlob } from '../crypto/VaultBlob';-import { SqliteClient } from '../database/SqliteClient';-import { getPlatform } from '../platform/ClientPlatform';-import { devError, devLog, devWarn } from '../platform/Logger';-import {vaultCodecComputeCiphertextHash, vaultCodecComputeContentFingerprint, vaultCodecCanonicalizeFromSqlite, vaultCodecExtractEncryptionKeyForPublicKey, vaultCodecGenerateManifestSalt, vaultCodecMergeCanonical, vaultCodecUnpackPayload, vaultCodecMaterializeAsSqlite, vaultCodecPackPayload, vaultCodecValidateManifest, vaultCodecValidateDataBucket, type CodecBlobEntry, type CodecCanonicalized, type CodecCanonicalManifestMerge, type CodecManifest, type CodecManifestSpec, vaultSharingPartitionManifestAccess, vaultSharingResolveManifestWriteSet, type SharingAccessPartition} from '../rust/RustCore';-import { multiManifestRendering } from '../sharing/MultiManifestRendering';-import { SharingService, type ManifestVekGrant, type SharedManifestRecord } from '../sharing/SharingService';-import { base64ToBytes, bytesToBase64 } from '../utilities/Base64';--import { buildEmailRouting } from './EmailRouting';-import { completeLegacyAccountKeyMigration, isLegacySqliteBlobSnapshot, legacyUnstampedRowAdoption, openLegacySqliteBlobSnapshot, prepareLegacyAccountKeyMigration, withOutdatedServerGuard, type LegacyAccountKeyMigration, type LegacySqliteBlobSnapshot } from './LegacyStorageModelMigration';-import { getBucketRevisions, getManifestRevisions, getPersonalManifestId, recordBucketRevisions, recordManifestRevisions, replaceBucketRevisions, replaceManifestRevisions, toManifestRevisionMap } from './ManifestRevisions';-import { type VaultManifest, type VaultDataBucket, VaultCodec, manifestIdKey } from './VaultCodec';--const VAULT_ENDPOINT = 'Vault';-const BLOBS_VAULT_ENDPOINT = 'Vault/blobs';-const BLOBS_MISSING_ENDPOINT = 'Vault/blobs/missing';-const BLOBS_DOWNLOAD_ENDPOINT = 'Vault/blobs/download';--/**- * Human-readable size for the push/pull size logs. Lengths are base64/JSON characters, which map ~1:1 to- * bytes on the wire.- * @param chars - length in characters (or bytes)- */-function formatKb(chars: number): string {- return chars < 1024 ? `${chars} B` : `${(chars / 1024).toFixed(1)} KB`;-}--/**- * Pack-then-encrypt a JSON payload (manifest or data bucket). The Rust codec wraps the payload in the- * integrity envelope (`{ contentHash, payload }`) and gzips it. The platform then AES-GCM encrypts- * the packed bytes.- * @param payloadJson - the plaintext payload JSON string (manifest or bucket)- * @param vek - symmetric encryption key- * @returns The base64 ciphertext plus the intermediate packed (gzip-compressed) size (for logging purposes)- */-async function packEncrypt(payloadJson: string, vek: string): Promise<{ ciphertext: string; compressedBytes: number }> {- const packed = await vaultCodecPackPayload(payloadJson);- const ciphertext = await EncryptionUtility.symmetricEncryptBytes(packed, vek);- return { ciphertext, compressedBytes: packed.length };-}--/**- * Dev-only stage timer: used to surface the manifest-shuffling passes that are- * otherwise invisible (canonicalize's Rust→JS conversion, validate's JS→Rust conversion, stringify).- * @param stage - short label for the measured step- * @param fn - the work to run and time- */-async function timedStage<T>(stage: string, fn: () => Promise<T> | T): Promise<T> {- if (!getPlatform().app.isDevelopment) {- return fn();- }- const start = performance.now();- const result = await fn();- devLog(`[V2Push] stage "${stage}": ${(performance.now() - start).toFixed(2)}ms`);- return result;-}--/**- * Verify a ciphertext against the hash the server stored (storage-layer integrity, skipped when the server sent- * no hash), then decrypt it and open it via the Rust codec: gunzip, verify the embedded content hash, and return- * the inner payload JSON string. Throws on either hash mismatch (storage or application-layer corruption).- * @param base64Ciphertext - base64(IV | ciphertext | tag) from the server- * @param vek - symmetric encryption key- * @param expectedCiphertextHash - the ciphertext hash the server stored, when available- * @param label - what is being opened, for the mismatch error message (e.g. `manifest`, `"Settings" bucket`)- */-async function verifyDecryptUnpack(base64Ciphertext: string, vek: string, expectedCiphertextHash: string | null | undefined, label: string): Promise<string> {- if (expectedCiphertextHash && await vaultCodecComputeCiphertextHash(base64Ciphertext) !== expectedCiphertextHash) {- throw new Error(`VaultSyncService: ${label} ciphertext hash mismatch, refusing to load. Possible storage corruption.`);- }- const encryptedBytes = base64ToBytes(base64Ciphertext);- const plainBytes = await EncryptionUtility.symmetricDecryptBytes(encryptedBytes, vek);- return vaultCodecUnpackPayload(plainBytes);-}--/**- * The grant a shared manifest is remembered by. Null when the snapshot carries no grant.- * @param dto - the snapshot manifest- */-function grantOf(dto: ManifestDto): ManifestVekGrant | null {- if (!dto.encryptedVek || !dto.encryptionPublicKey) {- return null;- }-- return { encryptedVek: dto.encryptedVek, encryptionPublicKey: dto.encryptionPublicKey, algorithm: dto.algorithm ?? VaultKeyAlgorithm.RsaOaepSha256 };-}--/**- * Whether two manifest ids match.- * @param a - first manifest id- * @param b - second manifest id- */-function manifestIdsEqual(a: string | null | undefined, b: string | null | undefined): boolean {- return typeof a === 'string' && typeof b === 'string' && a.toLowerCase() === b.toLowerCase();-}--/** Fingerprint record key for a manifest. */-const fingerprintManifestKey = (manifestId: string): string => `manifest:${manifestId}`;-/** Fingerprint record key for a data bucket, addressed by the manifest that owns it. */-const fingerprintBucketKey = (manifestId: string, category: string): string => `bucket:${manifestId}:${category}`;--/** Max amount of bytes that can be transferred in a single blob transfer request or response body. */-const BLOB_TRANSFER_BATCH_MAX_CHARS = 4 * 1024 * 1024;-/** Upper bound on the number of blobs in one transfer batch. */-const BLOB_TRANSFER_BATCH_MAX_COUNT = 100;--/** The number of base64 characters a payload of `sizeBytes` raw bytes occupies on the wire. */-function base64Chars(sizeBytes: number): number {- return Math.ceil(sizeBytes / 3) * 4;-}--/**- * Split items into request batches bounded by both {@link BLOB_TRANSFER_BATCH_MAX_CHARS} and- * {@link BLOB_TRANSFER_BATCH_MAX_COUNT}. An item larger than the byte budget itself is transferred on its own as- * we can't split it across requests.- * @param items - the items to batch- * @param costOf - the base64 character cost one item adds to the request or response body- */-function batchByTransferCost<T>(items: T[], costOf: (item: T) => number): T[][] {- const batches: T[][] = [];- let batch: T[] = [];- let chars = 0;- for (const item of items) {- const cost = costOf(item);- if (batch.length > 0 && (chars + cost > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.length >= BLOB_TRANSFER_BATCH_MAX_COUNT)) {- batches.push(batch);- batch = [];- chars = 0;- }-- batch.push(item);- chars += cost;- }-- if (batch.length > 0) {- batches.push(batch);- }-- return batches;-}--/**- * Result of a pull: the materialized SQLite plus its manifest revision.- */-type PullResult = {- sqliteBytes: Uint8Array;- manifestRevision: number;-};--/**- * What {@link VaultSyncService.openManifestsAndRecordSyncState} produces: every manifest of one snapshot,- * decrypted and ready to materialize or merge, with the local sync state already recorded from it.- */-type OpenedManifestSet = {- resolved: ResolvedManifest[];- dataBuckets: VaultDataBucket[];- blobMap: Map<string, Uint8Array>;- contentlessManifestIds: string[];- personalRevision: number;- commitRevisions: () => Promise<void>;-};--/** Aggregate canonical-merge statistics across all manifests. */-export type MergeSummary = {- tablesProcessed: number;- recordsFromLocal: number;- recordsFromServer: number;- recordsCreatedLocally: number;- conflicts: number;- recordsInserted: number;-};--/**- * Result of {@link VaultSyncService.pullAndMerge}: what the fetched snapshot turned out to be and, for the- * manifest-v1 outcomes, the merged (or server-only) vault plus the deferred revision commit.- */-export type PullAndMergeResult =- | { kind: 'legacy-server'; response: VaultResponse }- | {- kind: 'merged';- response: VaultResponse;- stats: MergeSummary;- fallbackManifestIds: string[];- droppedLocalManifestIds: string[];- commitRevisions: () => Promise<void>;- mergedSqliteBytes: Uint8Array;- pushCanonical: CanonicalizedVaultSet | null;- }- | { kind: 'server-only'; response: VaultResponse; commitRevisions: () => Promise<void> };--/**- * Result of a push. The revisions of every manifest the write carried are recorded in the per-manifest revision- * map ({@link recordManifestRevisions}) as part of the push itself; callers only act on the status.- */-export type PushResult = {- status: 'ok' | 'outdated' | 'missing-blobs' | 'rejected';- reasons?: string[];- /** The new encryption key (set by legacy migration). */- newEncryptionKey?: string;-};--type BlobRefDto = { hash: string; category: string };-type StoredBlobRefDto = BlobRefDto & { sizeBytes: number };-type BlobDto = { hash: string; category: string; encryptedDataBase64: string };--/** A plaintext blob staged for upload: its bytes plus the key that encrypts it. */-type UploadBlobEntry = { bytes: Uint8Array; kind: 'favicon' | 'attachment'; vek: string; fromPersonal: boolean };--/**- * One manifest of a pull.- */-type ResolvedManifest = {- manifestId: string;- isPersonal: boolean;- manifest: VaultManifest;- /** The key that decrypts this manifest and every blob it references. */- vek: string;- revision: number;- blobReferences: StoredBlobRefDto[];- /** Fingerprint of the plaintext exactly as the server served it: the push-side change-detection baseline. */- contentFingerprint: string;-};--/**- * A single manifest as carried in the GET snapshot / single-manifest fetch.- */-type ManifestDto = {- manifestId: string;- blob?: string | null;- ciphertextHash?: string | null;- revision: number;- blobReferences?: StoredBlobRefDto[];- /** Whether the caller may grant/revoke access to this manifest and publish its email delivery key. */- canAdminister?: boolean;- /** How the caller's access to this manifest's VEK is encrypted. */- keyType?: string | null;- /** The manifest VEK encrypted with the caller's public key; set only on manifests we open through a grant. */- encryptedVek?: string | null;- /** Algorithm of `encryptedVek`, one of the {@link VaultKeyAlgorithm} tokens. */- algorithm?: string | null;- /** The public key `encryptedVek` was encrypted with. Selects which of the caller's keypairs decrypts the grant. */- encryptionPublicKey?: string | null;-};--/** A data bucket as carried in the GET snapshot / bundled upload. `category` matches the server enum name (e.g. "Settings"). */-type BucketDto = { manifestId: string; category: string; blob?: string | null; ciphertextHash?: string | null; revision?: number };--/** Per-kind revision as carried in upload responses; the same shape the status response reports. */-type BucketRevisionDto = BucketRevision;--/**- * Pick the user's personal manifest out of a snapshot. The server names it by id rather than flagging each entry, so- * that one id is the only thing that decides which manifest is the caller's own. Strict on purpose: with no id, or an- * id matching nothing, there is no safe fallback (grabbing an arbitrary manifest could assemble the wrong vault), so- * callers that require a personal manifest must fail loudly on `undefined`.- */-function selectPersonalManifest(snapshot: GetResponseDto | undefined | null): ManifestDto | undefined {- const personalId = snapshot?.personalManifestId;- return personalId ? (snapshot?.manifests ?? []).find(m => m.manifestId === personalId) : undefined;-}--/**- * Raw snapshot returned by GET /v2/Vault. The `storageFormat` / `legacyVaultBlob` / `legacyRevision` / `version`- * fields are only populated for a not-yet-migrated user and are declared by {@link LegacySqliteBlobSnapshot}.- */-export type GetResponseDto = LegacySqliteBlobSnapshot & {- status: number;- /**- * The caller's personal manifest id: the one manifest owned by their personal group, every other entry being a- * shared one. Also set on the sqlite-blob path, where the manifests list is empty.- */- personalManifestId?: string | null;- /** The manifests making up the logical vault. Each carries its own blob references. */- manifests?: ManifestDto[];- buckets?: BucketDto[];- emailRouting?: {- emailAddressList?: string[];- privateEmailDomainList?: string[];- hiddenPrivateEmailDomainList?: string[];- publicEmailDomainList?: string[];- };-};--/**- * One manifest element in a POST /v2/Vault write. Every manifest is addressed by `manifestId`, the personal manifest- * included, so the server never has to infer the target; it authorizes each id against what the caller may write.- */-type ManifestWriteDto = {- manifestId: string;- manifestBlob: string;- manifestCiphertextHash: string;- currentRevision: number;- credentialsCount: number;- blobReferences: BlobRefDto[];- encryptionPublicKey?: string;-};--/** Per-manifest result of a write: the new revision (Ok) or the current server revision (Outdated). */-type ManifestWriteResultDto = { manifestId: string; revision: number };--type VaultWriteResponseDto = {- status: number;- manifestRevisions: ManifestWriteResultDto[];- bucketRevisions: BucketRevisionDto[];- missingBlobHashes: string[];-};--type MissingBlobsResponseDto = { missing: string[] };--/**- * One manifest canonicalize produced, staged as a write candidate.- */-type PushManifest = {- manifestId: string;- isPersonal: boolean;- manifest: CodecManifest;- /** The key this manifest and its blobs encrypt with: the vault's content key for the personal manifest, the grant's VEK otherwise. */- vek: string;- blobs: Record<string, CodecBlobEntry>;- /** The revision this write rebases on; the server rejects the whole batch when any is stale. */- currentRevision: number;-};--/**- * One manifest this vault can write, as resolved from local state before canonicalizing. The list is uniform on- * purpose: the codec treats every manifest alike and the push drives one loop over all of them. `isPersonal` marks the- * few places where the user's own manifest genuinely differs (see {@link resolveManifestRecords}).- */-export type ManifestRecord = {- manifestId: string;- isPersonal: boolean;- /** Salt this manifest's blob hashes are derived with. */- salt: string;- /** The key this manifest encrypts with; null for the personal manifest, whose content key the push supplies (it can be freshly minted). */- vek: string | null;- /** Display name, as read from (and written back into) the encrypted manifest; null for the personal manifest. */- name: string | null;- /** Whether the caller may publish this manifest's email delivery key. */- canAdminister: boolean;-};--/**- * The canonicalized vault plus the manifest records it was split against, personal manifest first (the order canonicalize- * requires: the first spec is the manifest being written from).- */-export type CanonicalizedVaultSet = {- canonicalized: CodecCanonicalized;- manifestRecords: ManifestRecord[];-};--let canonicalizeCache: ({ client: SqliteClient; mutationSequence: number } & CanonicalizedVaultSet) | null = null;--/**- * Drop the cached canonicalize result so the next push re-canonicalizes from the live SQLite state.- */-export function invalidateCanonicalizeCache(): void {- canonicalizeCache = null;-}--/**- * Hand the push a canonicalize result to prevent double-canonicalization where possible.- * @param client - the client the result describes; reused for this instance only- * @param mutationSequence - the sequence the vault is at; a mutation past it invalidates the result- * @param set - the canonicalized vault- */-export function primeCanonicalizeCache(client: SqliteClient, mutationSequence: number, set: CanonicalizedVaultSet): void {- canonicalizeCache = { client, mutationSequence, ...set };-}--/**- * Service entry point.- */-export class VaultSyncService {- /** The manifest ids the last snapshot carried, recorded before any of them is opened. */- private lastServedManifestIds: string[] = [];-- /** Whether the last snapshot was still on the legacy sqlite-blob storage format. */- private lastSnapshotWasLegacySqliteBlob: boolean = false;-- /**- * Retrieve the latest vault from the server as a normalized {@link VaultResponse} (encrypted SQLite blob +- * email routing + revision).- *- * Throws {@link ServerUpdateRequiredError} when the server predates the v2 API (outdated self-hosted install).- * @param encryptionKey - the user's symmetric key (decrypts/materializes manifest-v1 and re-encrypts the- * materialized SQLite; unused for the already-encrypted legacy-blob pass-through).- */- public async pull(encryptionKey: string): Promise<VaultResponse> {- /*- * Step 1: network fetch. Failures here (server unreachable, HTTP error, or ServerUpdateRequiredError for an- * outdated server) are "can't reach / must update the server" conditions, NOT vault-processing problems, so- * they propagate unchanged and the caller maps them to the appropriate "server" message.- */- devLog('[V2Pull] Step 1/4: fetching vault snapshot (GET /v2/Vault)...');- const snapshot = await this.fetchSnapshot();- this.lastServedManifestIds = (snapshot.manifests ?? []).map(m => m.manifestId);- this.lastSnapshotWasLegacySqliteBlob = isLegacySqliteBlobSnapshot(snapshot);- const personalManifest = selectPersonalManifest(snapshot);- devLog(`[V2Pull] Step 1/4 done: storageFormat=${snapshot.storageFormat}, manifests=${snapshot.manifests?.length ?? 0}, personalRevision=${personalManifest?.revision}, manifestBlob=${personalManifest?.blob?.length ?? 0} chars, buckets=${snapshot.buckets?.length ?? 0}, blobRefs=${personalManifest?.blobReferences?.length ?? 0}`);-- /*- * Steps 2–4: decrypt, materialize, and re-encrypt. Any failure here is a client-side vault-processing error- * (codec/format mismatch, integrity failure, corrupt blob, …). We wrap it in a VaultProcessingError so the UI- * can surface the real technical detail in a copyable report instead of a misleading "server unreachable".- */- try {- // LEGACY: a not-yet-migrated user's blob is passed through unchanged.- if (isLegacySqliteBlobSnapshot(snapshot)) {- const legacy = await openLegacySqliteBlobSnapshot(snapshot);- return this.buildResponse(legacy.encryptedBlob, legacy.version, legacy.revision, snapshot);- }-- // Manifest-v1 user: materialize the manifest + metadata + blobs into a SQLite blob, then encrypt it.- devLog('[V2Pull] Step 2/4: manifest format: decrypting and reassembling local SQLite...');- const pull = await this.materializeFromSnapshot(snapshot, encryptionKey);- devLog(`[V2Pull] Step 3/4: materialized SQLite (${pull.sqliteBytes.length} bytes); re-encrypting for local storage...`);- const encryptedVault = await encryptVaultBlob(pull.sqliteBytes, encryptionKey);- devLog('[V2Pull] Step 4/4: re-encryption done, returning VaultResponse.');- return this.buildResponse(encryptedVault, '2.0.0', pull.manifestRevision, snapshot);- } catch (error) {- devError('[V2Pull] FAILED: the last logged step above is where it broke:', error);- throw new VaultProcessingError('vault-pull', error);- }- }-- /**- * Pull the latest snapshot and merge the local vault onto it at canonical level, one manifest at a time.- * The server side is the base; the local side is canonicalized from `localClient`.- * @param encryptionKey - the personal manifest's symmetric key- * @param localClient - the local vault, freshly decrypted from storage- */- public async pullAndMerge(encryptionKey: string, localClient: SqliteClient): Promise<PullAndMergeResult> {- devLog('[V2Merge] Fetching vault snapshot for canonical merge (GET /v2/Vault)...');-- const snapshotPromise = this.fetchSnapshot();- const localCanonicalized = this.canonicalizeVault(localClient);- void localCanonicalized.catch(() => null);-- const snapshot = await snapshotPromise;- this.lastServedManifestIds = (snapshot.manifests ?? []).map(m => m.manifestId);- this.lastSnapshotWasLegacySqliteBlob = isLegacySqliteBlobSnapshot(snapshot);-- // LEGACY: a server still on the sqlite-blob format cannot merge with a manifest-v1 vault; the caller pushes over it.- if (isLegacySqliteBlobSnapshot(snapshot)) {- const legacy = await openLegacySqliteBlobSnapshot(snapshot);- return { kind: 'legacy-server', response: this.buildResponse(legacy.encryptedBlob, legacy.version, legacy.revision, snapshot) };- }-- try {- const opened = await this.openManifestsAndRecordSyncState(snapshot, encryptionKey, { deferRevisionCommit: true });-- try {- const localSide = await this.localSideForMerge(localCanonicalized, localClient, opened);- return await this.mergeOntoOpenedManifests(opened, snapshot, encryptionKey, localSide);- } catch (mergeError) {- // Today's merge-failure fallback, from the same snapshot: the server vault stands, local changes are dropped.- devError('[V2Merge] Canonical merge failed, falling back to the server vault:', mergeError);- const sqliteBytes = await this.materializeToSqlite(opened.resolved.map(m => m.manifest), opened.dataBuckets, opened.blobMap);- const encryptedVault = await encryptVaultBlob(sqliteBytes, encryptionKey);- return { kind: 'server-only', response: this.buildResponse(encryptedVault, '2.0.0', opened.personalRevision, snapshot), commitRevisions: opened.commitRevisions };- }- } catch (error) {- devError('[V2Merge] FAILED: the last logged step above is where it broke:', error);- throw new VaultProcessingError('vault-pull', error);- }- }-- /**- * The local side of the merge.- * @param parallelRun - the canonicalize started before the fetch- * @param localClient - the local vault, for the re-run- * @param opened - the opened server manifests, which decide whether the parallel result still holds- */- private async localSideForMerge(parallelRun: Promise<CanonicalizedVaultSet>, localClient: SqliteClient, opened: OpenedManifestSet): Promise<CanonicalizedVaultSet> {- const set = await parallelRun;- const moved = this.groundMovedUnderCanonicalize(set.manifestRecords, opened);- if (moved === null) {- return set;- }-- devWarn(`[V2Merge] This snapshot re-minted ${moved}; canonicalizing the local vault again so its blob hashes match the server's.`);- return this.canonicalizeVault(localClient);- }-- /**- * What this snapshot changed under a canonicalize that ran before it was opened.- * @param records - the manifest records the parallel canonicalize was split against- * @param opened - the opened server manifests- */- private groundMovedUnderCanonicalize(records: ManifestRecord[], opened: OpenedManifestSet): string | null {- const personalRecord = records.find(record => record.isPersonal);- const serverPersonal = opened.resolved.find(entry => entry.isPersonal);- if (serverPersonal && !manifestIdsEqual(personalRecord?.manifestId, serverPersonal.manifestId)) {- return `the personal manifest id (${personalRecord?.manifestId ?? 'none'} became ${serverPersonal.manifestId})`;- }-- const saltByManifestId = new Map(opened.resolved.map(entry => [manifestIdKey(entry.manifestId), entry.manifest.manifestSalt]));- for (const record of records) {- const serverSalt = saltByManifestId.get(manifestIdKey(record.manifestId));- if (serverSalt !== undefined && serverSalt !== record.salt) {- return `the blob salt of manifest ${record.manifestId}`;- }- }-- return null;- }-- /**- * The merge core of {@link pullAndMerge}: run the Rust canonical merge of the local side against the manifests- * just opened, validate the result per manifest, and materialize it.- * @param opened - the opened server manifests (the merge base)- * @param snapshot - the raw snapshot (for the email-routing block of the response)- * @param encryptionKey - the personal manifest's symmetric key- * @param localSide - the canonicalized local vault, the incoming side of the merge- */- private async mergeOntoOpenedManifests(opened: OpenedManifestSet, snapshot: GetResponseDto, encryptionKey: string, localSide: CanonicalizedVaultSet): Promise<PullAndMergeResult> {- const { canonicalized, manifestRecords } = localSide;-- const schemaColumns = await VaultCodec.getSchemaColumns(new VaultSqlGenerator().getCompleteSchemaSql());- devLog(`[V2Merge] Merging ${canonicalized.manifests.length} local manifest(s) onto ${opened.resolved.length} server manifest(s)...`);- const mergeOutput = await vaultCodecMergeCanonical({- serverManifests: opened.resolved.map(m => m.manifest),- serverBuckets: opened.dataBuckets,- contentlessServerManifestIds: opened.contentlessManifestIds,- localManifests: canonicalized.manifests.map(m => m.manifest),- localBuckets: canonicalized.dataBuckets,- schemaColumns,- });-- // Validation gate, per manifest: the server's version of a failing manifest stands.- const serverManifestById = new Map(opened.resolved.map(m => [m.manifestId.toLowerCase(), m.manifest]));- const serverBucketsById = new Map<string, VaultDataBucket[]>();- for (const bucket of opened.dataBuckets) {- const key = bucket.manifestId.toLowerCase();- serverBucketsById.set(key, [...(serverBucketsById.get(key) ?? []), bucket]);- }- const contentlessIds = new Set(opened.contentlessManifestIds.map(id => id.toLowerCase()));-- const manifests: VaultManifest[] = [];- const dataBuckets: VaultDataBucket[] = [];- const fallbackManifestIds: string[] = [];- const stats: MergeSummary = { tablesProcessed: 0, recordsFromLocal: 0, recordsFromServer: 0, recordsCreatedLocally: 0, conflicts: 0, recordsInserted: 0 };- for (const entry of mergeOutput.manifests) {- const failure = await this.validateMergedManifest(entry);- if (failure !== null && !contentlessIds.has(entry.manifestId.toLowerCase())) {- devWarn(`[V2Merge] Merged manifest ${entry.manifestId} failed validation (${failure}); the server's version stands and local changes to it are dropped.`);- fallbackManifestIds.push(entry.manifestId);- const serverManifest = serverManifestById.get(entry.manifestId.toLowerCase());- if (serverManifest) {- manifests.push(serverManifest);- dataBuckets.push(...(serverBucketsById.get(entry.manifestId.toLowerCase()) ?? []));- }- continue;- }- if (failure !== null) {- // A contentless pass-through has no server base to fall back to; keep the local rows, the push gate decides.- devWarn(`[V2Merge] Pass-through manifest ${entry.manifestId} failed validation (${failure}); keeping its local rows.`);- }- manifests.push(entry.manifest);- dataBuckets.push(...entry.buckets);- stats.tablesProcessed += entry.stats.tablesProcessed;- stats.recordsFromLocal += entry.stats.recordsFromLocal;- stats.recordsFromServer += entry.stats.recordsFromServer;- stats.recordsCreatedLocally += entry.stats.recordsCreatedLocally;- stats.conflicts += entry.stats.conflicts;- stats.recordsInserted += entry.stats.recordsInserted;- }- for (const dropped of mergeOutput.droppedLocalManifestIds) {- devWarn(`[V2Merge] Local manifest ${dropped} is no longer served; its rows are dropped from the merged vault.`);- }-- // Blob bytes for materialize: the server download plus everything the local canonicalize extracted.- const blobMap = opened.blobMap;- const localBlobs = new Map<string, CodecBlobEntry>();- for (const { blobs } of canonicalized.manifests) {- for (const [hash, blob] of Object.entries(blobs)) {- localBlobs.set(hash, blob);- if (!blobMap.has(hash)) {- blobMap.set(hash, VaultCodec.base64ToBytes(blob.bytesBase64));- }- }- }- const mergedBlobs = this.resolveMergedBlobRefs(manifests, blobMap, opened.resolved[0]?.manifestId ?? '', localBlobs);-- const sqliteBytes = await this.materializeToSqlite(manifests, dataBuckets, blobMap);- const encryptedVault = await encryptVaultBlob(sqliteBytes, encryptionKey);- devLog(`[V2Merge] Canonical merge complete: ${stats.conflicts} conflict(s), ${stats.recordsInserted} offline row(s) kept, ${fallbackManifestIds.length} validation fallback(s), ${mergeOutput.droppedLocalManifestIds.length} dropped local manifest(s).`);-- return {- kind: 'merged',- response: this.buildResponse(encryptedVault, '2.0.0', opened.personalRevision, snapshot),- stats,- fallbackManifestIds,- droppedLocalManifestIds: mergeOutput.droppedLocalManifestIds,- commitRevisions: opened.commitRevisions,- mergedSqliteBytes: sqliteBytes,- pushCanonical: this.mergeOutputForPush(manifests, dataBuckets, mergedBlobs, manifestRecords, fallbackManifestIds.length > 0),- };- }-- /**- * The merged vault in the shape the push writes from.- * @param manifests - the merged manifests, as materialized- * @param dataBuckets - the merged data buckets, as materialized- * @param blobsByManifest - the blob entries per manifest- * @param manifestRecords - the records the local side was split against- * @param hadFallbacks - whether any manifest fell back to the server's copy- */- private mergeOutputForPush(- manifests: VaultManifest[],- dataBuckets: VaultDataBucket[],- blobsByManifest: Map<string, Record<string, CodecBlobEntry>> | null,- manifestRecords: ManifestRecord[],- hadFallbacks: boolean- ): CanonicalizedVaultSet | null {- if (!blobsByManifest || hadFallbacks) {- return null;- }-- const mergedIds = new Set(manifests.map(manifest => manifestIdKey(manifest.manifestId)));- const recordIds = manifestRecords.map(record => manifestIdKey(record.manifestId));- if (mergedIds.size !== recordIds.length || recordIds.some(id => !mergedIds.has(id))) {- return null;- }-- return {- canonicalized: {- manifests: manifests.map(manifest => ({ manifest, blobs: blobsByManifest.get(manifestIdKey(manifest.manifestId)) ?? {} })),- dataBuckets,- },- manifestRecords,- };- }-- /**- * Validate one merged manifest and its buckets. Returns the failed rules, or null when valid.- * @param entry - one manifest's merge result- */- private async validateMergedManifest(entry: CodecCanonicalManifestMerge): Promise<string | null> {- const validation = await vaultCodecValidateManifest(entry.manifest);- if (!validation.ok) {- return validation.failedRules.join(', ');- }- for (const bucket of entry.buckets) {- const bucketValidation = await vaultCodecValidateDataBucket(bucket);- if (!bucketValidation.ok) {- return `bucket "${bucket.category}": ${bucketValidation.failedRules.join(', ')}`;- }- }- return null;- }-- /**- * Every blob marker in the merged manifests must resolve to bytes.- * @param manifests - the manifests about to materialize- * @param blobMap - the bytes available by hash- * @param personalManifestId - the personal manifest's id- * @param localBlobs - the blob entries the local canonicalize produced, by hash- * @returns The blob entries per manifest id, or null when a marker resolved to no bytes or no kind- */- private resolveMergedBlobRefs(- manifests: VaultManifest[],- blobMap: Map<string, Uint8Array>,- personalManifestId: string,- localBlobs: Map<string, CodecBlobEntry>- ): Map<string, Record<string, CodecBlobEntry>> | null {- const byManifestId = new Map<string, Record<string, CodecBlobEntry>>();- let complete = true;-- for (const manifest of manifests) {- const isPersonal = manifestIdsEqual(manifest.manifestId, personalManifestId);- const blobs: Record<string, CodecBlobEntry> = {};- byManifestId.set(manifestIdKey(manifest.manifestId), blobs);-- for (const rows of Object.values(manifest.tables)) {- for (const row of rows) {- for (const value of Object.values(row)) {- if (typeof value !== 'object' || value === null || !('__blobRef' in value)) {- continue;- }- const ref = value as { __blobRef: string; __blobKind?: string };- const bytes = blobMap.get(ref.__blobRef);- if (!bytes) {- if (ref.__blobKind === 'attachment' && isPersonal) {- throw new Error(`VaultSyncService: merged vault references attachment blob ${ref.__blobRef} with no bytes available, refusing to materialize an incomplete vault.`);- }- devWarn(`[V2Merge] Merged ${ref.__blobKind ?? 'blob'} ${ref.__blobRef} has no bytes available; it will materialize as empty.`);- complete = false;- continue;- }-- const local = localBlobs.get(ref.__blobRef);- if (local) {- blobs[ref.__blobRef] = local;- } else if (ref.__blobKind) {- blobs[ref.__blobRef] = { kind: ref.__blobKind, bytesBase64: bytesToBase64(bytes) };- } else {- // A marker without a kind cannot be staged for upload; the push derives its own blob set instead.- complete = false;- }- }- }- }- }-- return complete ? byManifestId : null;- }-- /**- * Migrate the local vault onto the current full schema, entirely locally: canonicalize the database into- * manifest-v1 form and materialize it straight back out again so it works offline too.- *- * This is the permanent delivery path for client schema changes (`SqliteClient.requiresSchemaMigration`).- * It runs for a legacy `sqlite-blob` vault whose schema stops at the frozen sqlite-blob upgrade chain, and equally for a- * manifest-v1 vault materialized by an older client whenever a newer COMPLETE_SCHEMA_SQL ships.- * @param sqliteClient - the local vault to migrate- * @returns The migrated SQLite database (plaintext bytes; the caller encrypts and stores it)- */- public async migrateVaultToCurrentSchema(sqliteClient: SqliteClient): Promise<Uint8Array> {- devLog('[ManifestMigration] Migrating local vault onto the current schema (local round-trip, no server involved)...');-- try {- // LEGACY: a sqlite-blob vault's rows carry no ManifestId yet, so this one canonicalize adopts them.- const { canonicalized } = await this.canonicalizeVault(sqliteClient, legacyUnstampedRowAdoption(await this.resolvePersonalManifestId()));-- /*- * Canonicalize already handed us every extracted favicon/attachment as plaintext bytes, so materialize can- * resolve its blob references without a single fetch.- */- const blobMap = new Map<string, Uint8Array>();- for (const { blobs } of canonicalized.manifests) {- for (const [hash, entry] of Object.entries(blobs)) {- blobMap.set(hash, VaultCodec.base64ToBytes(entry.bytesBase64));- }- }-- const schemaSql = new VaultSqlGenerator().getCompleteSchemaSql();- const schemaColumns = await VaultCodec.getSchemaColumns(schemaSql);- const materialized = await vaultCodecMaterializeAsSqlite(canonicalized.manifests.map(m => m.manifest), canonicalized.dataBuckets, schemaColumns);-- const sqliteBytes = await VaultCodec.insertTables(materialized, blobMap, schemaSql);-- devLog(`[ManifestMigration] Migration complete: ${blobMap.size} blobs re-embedded, ${sqliteBytes.length} bytes.`);- return sqliteBytes;- } catch (error) {- devError('[ManifestMigration] FAILED to migrate the local vault:', error);- throw new VaultProcessingError('vault-storage-migration', error);- }- }-- /**- * Fetch the raw snapshot (GET /v2/Vault) without decrypting/reassembling. Throws error if the server predates the v2 API.- */- private async fetchSnapshot(): Promise<GetResponseDto> {- return withOutdatedServerGuard(() => new WebApiService().get<GetResponseDto>(VAULT_ENDPOINT));- }-- /**- * Materialize a local SQLite database from an already-fetched manifest-v1 snapshot: verify ciphertext integrity,- * decrypt every manifest and the data buckets, fetch any missing referenced blobs, then run the codec.- * @param snapshot - the raw GET /v2/Vault response- * @param vek - the personal manifest's symmetric key (from the unlock chain); decrypts the personal manifest and the data buckets- */- private async materializeFromSnapshot(snapshot: GetResponseDto, vek: string): Promise<PullResult> {- const opened = await this.openManifestsAndRecordSyncState(snapshot, vek);- const sqliteBytes = await this.materializeToSqlite(opened.resolved.map(m => m.manifest), opened.dataBuckets, opened.blobMap);- return { sqliteBytes, manifestRevision: opened.personalRevision };- }-- /**- * Open every manifest a snapshot carries (the personal one plus each shared one) and record the snapshot as- * this device's sync state.- * @param snapshot - the raw GET /v2/Vault response- * @param vek - the personal manifest's symmetric key (from the unlock chain); every other manifest key resolves from it- * @param options - set deferRevisionCommit to hand the revision write back to the caller as `commitRevisions`,- * for when the pulled revisions may only become local truth once a later step has succeeded- */- private async openManifestsAndRecordSyncState(snapshot: GetResponseDto, vek: string, options?: { deferRevisionCommit?: boolean }): Promise<OpenedManifestSet> {- const personalDto = selectPersonalManifest(snapshot);- if (!personalDto) {- throw new Error('VaultSyncService: server returned no personal manifest, refusing to assemble.');- }-- if (!personalDto.blob) {- throw new Error('VaultSyncService: server returned no manifest blob, nothing to assemble.');- }-- // Content baselines for the push-side change detection: fingerprint every target as served by the server.- const pulledFingerprints: Record<string, string> = {};-- /*- * 1) Open every manifest in the snapshot through one path, personal manifest first. Every failure to resolve- * a key or open a blob throws on purpose.- */- const accountKeyVeks = new Map<string, string>([[personalDto.manifestId, vek]]);- const resolved: ResolvedManifest[] = [];- const sharedManifestRecords: Record<string, SharedManifestRecord> = {};- const contentlessRevisions: Record<string, number> = {};- for (const dto of [personalDto, ...(snapshot.manifests ?? []).filter(m => m.manifestId !== personalDto.manifestId)]) {- const isPersonal = dto.manifestId === personalDto.manifestId;- const manifestKey = await this.resolveManifestVek(dto, accountKeyVeks, isPersonal, resolved[0]?.manifest ?? null);- if (!dto.blob) {- // A shared manifest served without content yet (created but never written); its grant and revision are still tracked.- const contentlessGrant = grantOf(dto);- if (!contentlessGrant) {- throw new Error(`VaultSyncService: shared manifest ${dto.manifestId} was served without content and without a grant, refusing to assemble.`);- }- sharedManifestRecords[dto.manifestId] = {- manifestId: dto.manifestId,- ...contentlessGrant,- salt: await vaultCodecGenerateManifestSalt(),- name: null,- canAdminister: dto.canAdminister ?? false,- };- contentlessRevisions[dto.manifestId] = dto.revision;- continue;- }-- devLog(`[V2Pull] Verifying ciphertext hash; decrypting + opening ${isPersonal ? 'personal manifest' : `shared manifest ${dto.manifestId}`}...`);- const entry = await this.openManifest(dto, manifestKey, isPersonal);- resolved.push(entry);-- devLog(`[V2Pull] Manifest ${entry.manifestId} opened (content hash verified, ${isPersonal ? 'personal' : `shared "${entry.manifest.name ?? 'unnamed'}"`}): tables: ${Object.entries(entry.manifest.tables).map(([t, rows]) => `${t}=${rows.length}`).join(', ')}`);-- if (isPersonal) {- await getPlatform().storage.set(StorageKeys.VAULT_MANIFEST_SALT, entry.manifest.manifestSalt);- await getPlatform().storage.set(StorageKeys.VAULT_PERSONAL_MANIFEST_ID, entry.manifestId);- continue;- }-- const grant = grantOf(dto);- if (!grant) {- throw new Error(`VaultSyncService: shared manifest ${entry.manifestId} carries no grant this account can re-open, refusing to assemble.`);- }-- sharedManifestRecords[entry.manifestId] = {- manifestId: entry.manifestId,- ...grant,- salt: entry.manifest.manifestSalt,- name: entry.manifest.name ?? null,- canAdminister: dto.canAdminister ?? false,- };- }- await SharingService.setSharedManifestRecords(sharedManifestRecords, vek);-- const personal = resolved[0];-- // 2) Open the data buckets belonging to those manifests.- const { dataBuckets, bucketFingerprints, bucketRevisions } = await this.openDataBuckets(snapshot, resolved);- Object.assign(pulledFingerprints, bucketFingerprints);-- // 3) Revision baselines: what the next status check compares this device against.- const manifestRevisions = { ...Object.fromEntries(resolved.map(m => [m.manifestId, m.revision])), ...contentlessRevisions };- /**- * Commit the snapshot's revision maps, manifests and data buckets alike, as the local believed-current- * revisions. Both are replaced rather than merged, so a manifest that is gone takes its buckets with it.- * The two move independently and the status check compares both, so they have to be committed together:- * committing the buckets earlier would clear a pending bucket pull that the manifests cannot re-trigger.- */- const commitRevisions = async (): Promise<void> => {- await replaceManifestRevisions(manifestRevisions);- await replaceBucketRevisions(bucketRevisions);- devLog(`[V2Pull] Stored local manifest revisions from snapshot: ${Object.entries(manifestRevisions).map(([id, rev]) => `${id}=${rev}`).join(', ')}. Next status check compares against these.`);- devLog(`[V2Pull] Stored local data bucket revisions from snapshot: ${Object.entries(bucketRevisions).map(([key, rev]) => `${key}=${rev}`).join(', ') || 'none'}.`);- };- if (options?.deferRevisionCommit !== true) {- await commitRevisions();- }-- // 4) Fetch and decrypt every blob the opened manifests reference.- const blobMap = await this.downloadReferencedBlobs(resolved, vek);-- /*- * 5) Fingerprint baselines: what the next push diffs against. Replace rather than merge, the record must- * mirror exactly the manifests and buckets the server holds right now, so entries of revoked/removed- * manifests drop out. One entry per opened manifest, the personal one included.- */- for (const entry of resolved) {- pulledFingerprints[fingerprintManifestKey(entry.manifestId)] = entry.contentFingerprint;- }- await getPlatform().storage.set(StorageKeys.VAULT_CONTENT_FINGERPRINTS, pulledFingerprints);- devLog(`[V2Pull] Stored ${Object.keys(pulledFingerprints).length} content fingerprint baseline(s) for push-side change detection.`);-- return {- resolved,- dataBuckets,- blobMap,- contentlessManifestIds: Object.keys(contentlessRevisions),- personalRevision: personal.revision,- commitRevisions,- };- }-- /**- * Open the data buckets a snapshot carries and record their revisions as the local baseline.- * @param snapshot - the raw GET /v2/Vault response- * @param resolved - the manifests already opened; a bucket addressed to any other manifest is refused- * @returns The decrypted buckets, one content fingerprint per bucket for the push-side change detection, and the- * snapshot's bucket revisions, which the caller commits together with the manifest revisions- */- private async openDataBuckets(snapshot: GetResponseDto, resolved: ResolvedManifest[]): Promise<{ dataBuckets: VaultDataBucket[]; bucketFingerprints: Record<string, string>; bucketRevisions: Record<string, number> }> {- const bucketFingerprints: Record<string, string> = {};- const keyByManifestId = new Map(resolved.map(entry => [entry.manifestId, entry.vek]));- const dataBuckets: VaultDataBucket[] = [];- const pulledBucketRevisions: Record<string, number> = {};- for (const bucketDto of (snapshot.buckets ?? [])) {- if (!bucketDto.blob) {- continue;- }- const bucketKey = keyByManifestId.get(bucketDto.manifestId);- if (!bucketKey) {- throw new Error(`VaultSyncService: data bucket "${bucketDto.category}" belongs to manifest ${bucketDto.manifestId}, which this vault did not open, refusing to assemble.`);- }- const label = `"${bucketDto.category}" bucket of manifest ${bucketDto.manifestId}`;- const bucketJson = await verifyDecryptUnpack(bucketDto.blob, bucketKey, bucketDto.ciphertextHash, label);- const bucket = JSON.parse(bucketJson) as VaultDataBucket;-- // Bind the payload to the address the server delivered it under- if (!manifestIdsEqual(bucket.manifestId, bucketDto.manifestId) || bucket.category !== bucketDto.category) {- throw new Error(`VaultSyncService: ${label} declares a different address (manifest ${bucket.manifestId}, category "${bucket.category}") inside its encrypted payload, refusing to assemble.`);- }-- dataBuckets.push(bucket);- bucketFingerprints[fingerprintBucketKey(bucketDto.manifestId, bucketDto.category)] = await vaultCodecComputeContentFingerprint(bucketJson);- const rowCount = Object.values(bucket.tables ?? {}).reduce((n, rows) => n + rows.length, 0);- devLog(`[V2Pull] Data bucket ${label} opened: ${rowCount} rows (revision ${bucketDto.revision}).`);- if (typeof bucketDto.revision === 'number') {- pulledBucketRevisions[bucketRevisionKey(bucketDto.manifestId, bucketDto.category)] = bucketDto.revision;- }- }- if (dataBuckets.length === 0) {- devLog('[V2Pull] No data buckets in snapshot.');- }-- return { dataBuckets, bucketFingerprints, bucketRevisions: pulledBucketRevisions };- }-- /**- * Fetch every blob the opened manifests reference that is not already cached locally, decrypt it, and prune- * the persisted cache to exactly that referenced set, so the cache stays bounded by the current vault size.- * @param resolved - the opened manifests, whose references decide what is downloaded and which key opens it- * @param vek - fallback key for a reference whose owning manifest cannot be determined- * @returns Plaintext bytes per blob hash, for the codec to re-embed- */- private async downloadReferencedBlobs(resolved: ResolvedManifest[], vek: string): Promise<Map<string, Uint8Array>> {- const webApi = new WebApiService();- const refOwners = new Map<string, ResolvedManifest>();- const refs: StoredBlobRefDto[] = [];- for (const entry of resolved) {- for (const r of entry.blobReferences) {- if (refOwners.has(r.hash)) {- continue;- }- refs.push(r);- refOwners.set(r.hash, entry);- }- }- const cache = await this.loadBlobCache();- const missingRefs = refs.filter(r => !(r.hash in cache));- devLog(`[V2Pull] Blob refs: ${refs.length} referenced, ${refs.length - missingRefs.length} cached locally, ${missingRefs.length} to download.`);-- // Batch downloads by the bytes each blob adds to the response, not by hash count.- const downloadBatches = batchByTransferCost(missingRefs, r => base64Chars(r.sizeBytes));- for (const [index, chunk] of downloadBatches.entries()) {- const chunkChars = chunk.reduce((sum, r) => sum + base64Chars(r.sizeBytes), 0);- const blobs = await webApi.post<{ hashes: string[] }, BlobDto[]>(BLOBS_DOWNLOAD_ENDPOINT, { hashes: chunk.map(r => r.hash) });- devLog(`[V2Pull] Downloaded blob batch ${index + 1}/${downloadBatches.length}: requested ${chunk.length} (${formatKb(chunkChars)}), received ${blobs.length}.`);- for (const dto of blobs) {- cache[dto.hash] = dto.encryptedDataBase64;- }- }-- // Decrypt the referenced blobs and prune the cache to exactly the referenced set, so the cache stays bounded by the current vault size.- const prunedCache: Record<string, string> = {};- const blobMap = new Map<string, Uint8Array>();- for (const r of refs) {- const ciphertext = cache[r.hash];- const owner = refOwners.get(r.hash);- const blobKey = owner?.vek ?? vek;- if (!ciphertext) {- devWarn(`[V2Sync] Referenced ${r.category} blob ${r.hash} missing on server, continuing without it.`);- continue;- }- try {- blobMap.set(r.hash, await this.decryptBlobToBytes(ciphertext, blobKey));- prunedCache[r.hash] = ciphertext;- } catch {- devWarn(`[V2Sync] Referenced ${r.category} blob ${r.hash} failed to decrypt with the current key, continuing without it.`);- }- }- await this.saveBlobCache(prunedCache);-- // The server demonstrably has every blob it just served or referenced, seed the upload diff with them.- await getPlatform().storage.set(StorageKeys.VAULT_SERVER_BLOB_HASHES, refs.map(r => r.hash));-- return blobMap;- }-- /**- * Materialize manifests + data buckets into a fresh SQLite database via the codec.- * @param manifests - the manifests to combine, the caller's own first- * @param dataBuckets - the data buckets belonging to those manifests- * @param blobMap - plaintext bytes for every blob hash the manifests reference- * @returns The materialized SQLite database (plaintext bytes)- */- private async materializeToSqlite(manifests: VaultManifest[], dataBuckets: VaultDataBucket[], blobMap: Map<string, Uint8Array>): Promise<Uint8Array> {- devLog(`[V2Pull] ${blobMap.size} blobs decrypted; running codec reassembly into a fresh SQLite (${manifests.length} manifest(s) combined)...`);- const sqlGen = new VaultSqlGenerator();- const schemaSql = sqlGen.getCompleteSchemaSql();- const schemaColumns = await VaultCodec.getSchemaColumns(schemaSql);- const materialized = await vaultCodecMaterializeAsSqlite(manifests, dataBuckets, schemaColumns);-- const overflowTableCount = Object.keys(materialized.overflow.tables).length + Object.values(materialized.overflow.bucketTables).reduce((n, t) => n + Object.keys(t).length, 0);- const overflowColumnTables = Object.keys(materialized.overflow.columns);- if (overflowTableCount > 0 || overflowColumnTables.length > 0) {- devWarn(`[V2Pull] Newer-schema data preserved as overflow: ${overflowTableCount} unknown table(s), unknown columns on [${overflowColumnTables.join(', ')}]. It will round-trip on push but is not usable locally until the app is updated.`);- }-- const sqliteBytes = await VaultCodec.insertTables(materialized, blobMap, schemaSql);- devLog('[V2Pull] Codec reassembly complete.');-- return sqliteBytes;- }-- /**- * Verify one snapshot manifest's ciphertext, decrypt and open it into a {@link ResolvedManifest}.- * @param dto - the snapshot manifest (must carry a blob)- * @param vek - the key that decrypts it- * @param isPersonal - whether this is the manifest the snapshot named as the caller's own- */- private async openManifest(dto: ManifestDto, vek: string, isPersonal: boolean): Promise<ResolvedManifest> {- const manifestJson = await verifyDecryptUnpack(dto.blob!, vek, dto.ciphertextHash, isPersonal ? 'manifest' : `shared manifest ${dto.manifestId}`);- const manifest = JSON.parse(manifestJson) as VaultManifest;-- if (!manifestIdsEqual(manifest.manifestId, dto.manifestId)) {- throw new Error(`VaultSyncService: manifest ${dto.manifestId} declares a different id (${manifest.manifestId}) inside its encrypted payload, refusing to open it.`);- }-- return {- manifestId: dto.manifestId,- isPersonal,- manifest,- vek,- revision: typeof dto.revision === 'number' ? dto.revision : 0,- blobReferences: dto.blobReferences ?? [],- contentFingerprint: await vaultCodecComputeContentFingerprint(manifestJson),- };- }-- /**- * The key that opens one snapshot manifest.- * @param dto - the snapshot manifest- * @param accountKeyVeks - the VEKs already held per manifest id, keyed by the account key hierarchy- * @param isPersonal - whether this is the manifest the snapshot named as the caller's own- * @param personalManifest - the already-decrypted personal manifest (the durable home of rotated private keys), or null when none is open yet- * @returns The manifest's VEK. Throws when this client holds no key for it.- */- private async resolveManifestVek(dto: ManifestDto, accountKeyVeks: Map<string, string>, isPersonal: boolean, personalManifest: CodecManifest | null): Promise<string> {- // A server that predates the field says nothing, and there the home manifest is the account-key one by definition.- const keyType = dto.keyType ?? (isPersonal ? ManifestKeyType.AccountKey : ManifestKeyType.GrantKey);-- if (keyType === ManifestKeyType.AccountKey) {- const accountKeyVek = accountKeyVeks.get(dto.manifestId);- if (!accountKeyVek) {- // Our unlock chain produced a VEK for a different manifest than the one this key is filed under.- throw new Error(`VaultSyncService: manifest ${dto.manifestId} is unlocked by the account key hierarchy, but this session holds no key for it, refusing to assemble.`);- }- return accountKeyVek;- }-- if (keyType !== ManifestKeyType.GrantKey) {- throw new Error(`VaultSyncService: manifest ${dto.manifestId} states an unknown key type "${keyType}" (newer server?), refusing to assemble.`);- }-- if (!personalManifest) {- throw new Error(`VaultSyncService: manifest ${dto.manifestId} is opened through a grant, but no personal manifest is open to resolve the private key from, refusing to assemble.`);- }-- return this.resolveGrantedVek(personalManifest, dto);- }-- /**- * Decrypt the VEK the server granted the caller on a manifest.- * @param personalManifest - the already-decrypted personal manifest, the durable home of rotated private keys- * @param dto - the snapshot manifest carrying the grant- */- private async resolveGrantedVek(personalManifest: CodecManifest, dto: ManifestDto): Promise<string> {- if (!dto.encryptedVek || !dto.encryptionPublicKey) {- throw new Error(`VaultSyncService: shared manifest ${dto.manifestId} carries no grant to open it with, refusing to assemble.`);- }-- /*- * The algorithm, not the key type, decides how the ciphertext opens: a grant encrypted under an algorithm we- * do not implement would otherwise be handed to an RSA-OAEP decrypt that cannot possibly be right. A server- * predating the field says nothing, and back then every grant was RSA-OAEP.- */- const algorithm = dto.algorithm ?? VaultKeyAlgorithm.RsaOaepSha256;- if (algorithm !== VaultKeyAlgorithm.RsaOaepSha256) {- throw new Error(`VaultSyncService: shared manifest ${dto.manifestId} grants its VEK under an unsupported algorithm "${algorithm}" (newer server?), refusing to assemble.`);- }-- const privateKeyJwk = await this.resolvePrivateKeyJwk(personalManifest, dto.encryptionPublicKey);- if (!privateKeyJwk) {- throw new Error(`VaultSyncService: no private key in this vault opens the grant on shared manifest ${dto.manifestId}, refusing to assemble.`);- }-- try {- return await SharingService.decryptManifestVek(dto.encryptedVek, privateKeyJwk);- } catch (e) {- throw new Error(`VaultSyncService: failed to decrypt the VEK of shared manifest ${dto.manifestId}, refusing to assemble. Underlying: ${e instanceof Error ? e.message : String(e)}`);- }- }-- /**- * Resolve the user's asymmetric private key (JWK string) for decrypting a shared manifest's VEK.- * @param personalManifest - the decrypted personal manifest- * @param encryptionPublicKey - the public key the grant's VEK was encrypted with- */- private async resolvePrivateKeyJwk(personalManifest: CodecManifest, encryptionPublicKey: string): Promise<string | null> {- const accountPublicKey = await VaultKeyService.getAccountPublicKey();- if (accountPublicKey === encryptionPublicKey) {- const sessionPrivateKey = await VaultKeyService.getSessionAccountPrivateKey();- if (sessionPrivateKey) {- return sessionPrivateKey;- }- }-- const keyRow = await vaultCodecExtractEncryptionKeyForPublicKey(personalManifest, encryptionPublicKey);- return typeof keyRow?.PrivateKey === 'string' ? keyRow.PrivateKey : null;- }-- /**- * Build a VaultResponse from a (already-encrypted) SQLite blob + the snapshot's email-routing block.- * @param encryptedBlob - encrypted SQLite blob to store under local:encryptedVault- * @param version - data-model version string- * @param revision - unified content revision number- * @param snapshot - the source snapshot (for the email-routing lists)- */- private buildResponse(encryptedBlob: string, version: string, revision: number, snapshot: GetResponseDto): VaultResponse {- const er = snapshot.emailRouting ?? {};- return {- status: 0,- vault: {- blob: encryptedBlob,- version,- currentRevisionNumber: revision,- encryptionPublicKey: '',- credentialsCount: 0,- publicEmailDomainList: er.publicEmailDomainList ?? [],- privateEmailDomainList: er.privateEmailDomainList ?? [],- hiddenPrivateEmailDomainList: er.hiddenPrivateEmailDomainList ?? [],- emailAddressList: er.emailAddressList ?? [],- createdAt: '',- updatedAt: '',- username: '',- },- } as unknown as VaultResponse;- }-- /**- * Canonicalize the current SQLite vault, validate, encrypt, and POST /v2/Vault. The write is as narrow as- * possible: content-fingerprint gating keeps every manifest and bucket whose canonical content still matches- * its last-known server state OUT of the write, so a credential edit uploads the personal manifest alone and a- * shared-manifest edit uploads that manifest alone. Only blobs the server doesn't- * already have are encrypted and pre-uploaded (in size-capped batches) before the manifest POST, so a routine- * save of a vault with hundreds of attachments uploads kilobytes, not the whole blob set. If the manifest POST- * reports missing blobs still missing (stale local knowledge, e.g. server-side GC), the missing bytes are uploaded- * and the POST retried once.- * @param sqliteClient - the in-memory SQLite the user has been editing- * @param vek - the symmetric encryption key (on a KEK/VEK migration push this is the password-derived key,- * which becomes the KEK; on a normal push it is the VEK itself)- * @param username - the user's username (sent in the upload payload for cross-check)- * @param options - set createVaultKey to perform the KEK/VEK migration as part of this push (decided once, in- * handleUploadVault); set forceFullWrite to bypass the content-fingerprint gating and rewrite every manifest- * and bucket, for when the fingerprints cannot be trusted to describe the server's state- * @returns Push outcome.- */- public async push(- sqliteClient: SqliteClient,- vek: string,- username: string,- options?: { createVaultKey?: boolean; forceFullWrite?: boolean }- ): Promise<PushResult> {- return withOutdatedServerGuard(() => this.pushInternal(sqliteClient, vek, username, options));- }-- /**- * The push implementation; {@link push} wraps it with the outdated-server guard.- * @param sqliteClient - the in-memory SQLite the user has been editing- * @param vek - the symmetric encryption key- * @param username - the user's username- * @param options - see {@link push}- */- private async pushInternal(- sqliteClient: SqliteClient,- vek: string,- username: string,- options?: { createVaultKey?: boolean; forceFullWrite?: boolean }- ): Promise<PushResult> {- /*- * Nothing is written while the vault holds manifests this session cannot write: those rows would be dropped- * from the write rather than fail it. Reported as outdated, which is the status that sends the caller back- * through a sync — and the sync pulls, which is what puts the grants and the vault back in step.- */- const unwritable = await this.findUnwritableManifests(sqliteClient);- if (unwritable.length > 0) {- devWarn(`[V2Push] Vault holds rows for manifest(s) this session cannot write (${unwritable.join(', ')}); refusing the write until a pull restores them.`);- return { status: 'outdated', reasons: [`Manifest(s) ${unwritable.join(', ')} are not open to this session`] };- }-- const migration: LegacyAccountKeyMigration | null = options?.createVaultKey === true ? await prepareLegacyAccountKeyMigration(vek) : null;- const contentKey = migration?.contentKey ?? vek;- if (migration) {- devLog('[V2Push] Account-key migration: generated new VEK, AK and account keypair; vault content and all blobs will be re-encrypted and re-uploaded.');- }-- /**- * Canonicalize.- */- const currentMutationSequence = ((await getPlatform().storage.get(StorageKeys.MUTATION_SEQUENCE)) as number | null) ?? 0;- const cachedSet = (canonicalizeCache && canonicalizeCache.client === sqliteClient && canonicalizeCache.mutationSequence === currentMutationSequence) ? canonicalizeCache : null;- if (cachedSet) {- devLog('[V2Push] Reusing the canonicalize result already produced for this vault.');- }- const { canonicalized, manifestRecords } = cachedSet ?? await this.canonicalizeVault(sqliteClient);- // Personal manifest first by construction (see `resolveManifestRecords`), which is also the order canonicalize requires.- const [personalRecord] = manifestRecords;- const privateEmailDomains = (await getPlatform().storage.get<string[]>(StorageKeys.PRIVATE_EMAIL_DOMAINS)) ?? [];- const emailRouting = buildEmailRouting(canonicalized.manifests.map(m => m.manifest), privateEmailDomains);-- /*- * Debug: manifest-set summary, shapes only.- */- const canonicalizedBuckets = canonicalized.dataBuckets;- devLog(`[V2Push] Canonicalize produced ${canonicalized.manifests.length} manifest(s) + ${canonicalizedBuckets.length} data bucket(s).`);- for (const { manifest } of canonicalized.manifests) {- const isPersonal = manifest.manifestId === personalRecord.manifestId;- devLog(`[V2Push] ${isPersonal ? 'Personal manifest' : `Shared manifest ${manifest.manifestId}`}: tables: ${Object.entries(manifest.tables).map(([t, rows]) => `${t}=${rows.length}`).join(', ')}`);- }-- /*- * 2) Content-fingerprint gating: compare every canonicalized target (each manifest below, each data bucket)- * against the fingerprint of its last-known server state and only write the targets that actually changed. A- * missing baseline means "server state unknown" and always writes. Two cases force a blanket write: a KEK/VEK- * migration re-keys the personal manifest and all buckets (their ciphertext must be re-encrypted with the new VEK- * even when the content is unchanged), and forceFullWrite rewrites everything for a caller that knows the- * fingerprints no longer describe what the server holds.- */- const forceFullWrite = options?.forceFullWrite === true;- const fingerprints = await this.loadContentFingerprints();-- const storedRevisions = await getManifestRevisions();- const recordByManifestId = new Map(manifestRecords.map(r => [r.manifestId, r]));- const candidates: PushManifest[] = canonicalized.manifests.flatMap(({ manifest, blobs }): PushManifest[] => {- const record = recordByManifestId.get(manifest.manifestId);- if (!record) {- return [];- }- return [{- manifestId: record.manifestId,- isPersonal: record.isPersonal,- manifest,- vek: record.vek ?? contentKey,- blobs,- currentRevision: storedRevisions[record.manifestId] ?? 0,- }];- });-- // Set up the blob entries for the write.- const blobEntries = new Map<string, UploadBlobEntry>();- for (const candidate of candidates) {- for (const [hash, blob] of Object.entries(candidate.blobs)) {- if (blobEntries.has(hash)) {- continue;- }- blobEntries.set(hash, { kind: blob.kind as 'favicon' | 'attachment', bytes: VaultCodec.base64ToBytes(blob.bytesBase64), vek: candidate.vek, fromPersonal: candidate.isPersonal });- }- }-- /*- * Pack + encrypt each changed data bucket, carrying the client's believed-current revision so each bucket- * participates in the same all-or-nothing revision gate as the manifests. Unchanged buckets are skipped- * (unless the write is forced, see the gating comment above).- */- const bucketDtos: Array<{ manifestId: string; category: string; blob: string; ciphertextHash: string; currentRevision: number }> = [];- const writtenBucketFingerprints: Record<string, string> = {};- const storedBucketRevisions = await getBucketRevisions();- const keyByManifestId = new Map(candidates.map(candidate => [candidate.manifestId, candidate.vek]));- for (const bucket of canonicalizedBuckets) {- const label = `Data bucket "${bucket.category}" of manifest ${bucket.manifestId}`;- const fingerprintKey = fingerprintBucketKey(bucket.manifestId, bucket.category);- const bucketPlaintext = JSON.stringify(bucket);- const bucketFingerprint = await vaultCodecComputeContentFingerprint(bucketPlaintext);- if (!forceFullWrite && !migration && fingerprints[fingerprintKey] === bucketFingerprint) {- devLog(`[V2Push] ${label} unchanged versus server baseline, leaving it out of this write.`);- continue;- }-- // A bucket is encrypted with the key of the manifest that owns it.- const bucketKey = keyByManifestId.get(bucket.manifestId);- if (!bucketKey) {- devWarn(`[V2Push] ${label} names a manifest this vault cannot write; leaving it out of this write.`);- continue;- }-- const bucketValidation = await vaultCodecValidateDataBucket(bucket);- if (!bucketValidation.ok) {- return {- status: 'rejected',- reasons: [`${label} validation failed: ${bucketValidation.failedRules.join(', ')}. ${bucketValidation.message}`.trim()],- };- }-- const { ciphertext, compressedBytes } = await packEncrypt(bucketPlaintext, bucketKey);- const ciphertextHash = await vaultCodecComputeCiphertextHash(ciphertext);- const currentRevision = storedBucketRevisions[bucketRevisionKey(bucket.manifestId, bucket.category)] ?? 0;- bucketDtos.push({ manifestId: bucket.manifestId, category: bucket.category, blob: ciphertext, ciphertextHash, currentRevision });- writtenBucketFingerprints[fingerprintKey] = bucketFingerprint;- devLog(`[V2Push] ${label}: raw ${formatKb(bucketPlaintext.length)} → compressed ${formatKb(compressedBytes)} → encrypted ${formatKb(ciphertext.length)}.`);- }-- /*- * Gate, validate, pack and encrypt every candidate manifest into the write batch, each with its own VEK. An- * unchanged manifest is skipped entirely: that avoids re-uploading it and keeps its (possibly stale) revision- * out of the all-or-nothing gate, so another member's update to a shared folder can't fail an unrelated write.- */- const manifestWrites: ManifestWriteDto[] = [];- const writtenManifestFingerprints: Record<string, string> = {};- for (const candidate of candidates) {- const label = candidate.isPersonal ? 'Personal manifest' : `Shared manifest "${candidate.manifest.name ?? candidate.manifestId}"`;- const plaintext = await timedStage(`stringify-manifest ${candidate.manifestId}`, () => JSON.stringify(candidate.manifest));- const fingerprint = await vaultCodecComputeContentFingerprint(plaintext);-- /*- * A KEK/VEK migration re-keys the personal manifest and all buckets (their ciphertext must be re-encrypted with- * the new VEK even when the content is unchanged) but not the shared manifests, which keep their own VEK.- */- if (!forceFullWrite && !(migration && candidate.isPersonal) && fingerprints[fingerprintManifestKey(candidate.manifestId)] === fingerprint) {- devLog(`[V2Push] ${label} unchanged versus server baseline, leaving it out of this write.`);- continue;- }-- const validation = await timedStage(`validate-manifest ${candidate.manifestId} (incl. JS→Rust conversion)`, () => vaultCodecValidateManifest(candidate.manifest));- if (!validation.ok) {- if (candidate.isPersonal) {- return {- status: 'rejected',- reasons: [`Manifest validation failed: ${validation.failedRules.join(', ')}. ${validation.message}`.trim()],- };- }- devWarn(`[V2Push] ${label} failed validation (${validation.failedRules.join(', ')}), dropping it from this write.`);- continue;- }-- const { ciphertext, compressedBytes } = await packEncrypt(plaintext, candidate.vek);- const ciphertextHash = await vaultCodecComputeCiphertextHash(ciphertext);- devLog(`[V2Push] ${label}: raw ${formatKb(plaintext.length)} → compressed ${formatKb(compressedBytes)} → encrypted ${formatKb(ciphertext.length)}.`);-- /*- * Publish the public half of this manifest's RSA key pair, which is used to e.g. encrypt incoming emails.- * Only admins are allowed to publish a new shared manifest's key.- */- const mayPublish = candidate.isPersonal || recordByManifestId.get(candidate.manifestId)?.canAdminister === true;- const manifestKey = mayPublish ? sqliteClient.encryptionKeys.getActiveForManifest(candidate.manifestId) : null;- if (mayPublish && !manifestKey && !candidate.isPersonal) {- /*- * Sharing a folder mints its keypair in the same vault mutation, so an owned shared manifest without one- * means that mutation was interrupted after the manifest was created server-side. Aliases in its subtree- * stay personal (readable by the owner only) until sharing is toggled again.- */- devWarn(`[V2Push] ${label} is missing its email keypair; its aliases stay personal until sharing is re-enabled.`);- }-- manifestWrites.push({- manifestId: candidate.manifestId,- manifestBlob: ciphertext,- manifestCiphertextHash: ciphertextHash,- currentRevision: candidate.currentRevision,- credentialsCount: (candidate.manifest.tables.Items ?? []).length,- blobReferences: Object.entries(candidate.blobs).map(([hash, blob]) => ({ hash, category: blob.kind })),- ...(manifestKey ? { encryptionPublicKey: manifestKey.PublicKey } : {}),- });- writtenManifestFingerprints[candidate.manifestId] = fingerprint;- }-- /*- * Nothing changed versus the server baselines: skip the write (and the blob diff) entirely. Reachable when a- * mutation was recorded but produced no canonical content change (e.g. an edit reverted to the original value).- */- if (manifestWrites.length === 0 && bucketDtos.length === 0) {- devLog('[V2Push] No content changes detected (every manifest and data bucket matches the server baselines); skipping upload.');- return { status: 'ok' };- }-- /*- * 4) Blob diff across the personal manifest and every shared manifest in this write: only encrypt + upload blobs the- * server doesn't already have. On a KEK/VEK migration all personal-manifest blobs are re-encrypted with the new VEK and- * overwritten in place; shared-manifest blobs keep their own (unchanged) VEK and only fill genuine gaps.- */- const webApi = new WebApiService();- const personalHashes = Array.from(blobEntries).filter(([, entry]) => entry.fromPersonal).map(([hash]) => hash);- const sharedHashes = Array.from(blobEntries).filter(([, entry]) => !entry.fromPersonal).map(([hash]) => hash);- const allBlobHashes = Array.from(blobEntries.keys());- const knownServerHashes = new Set(((await getPlatform().storage.get(StorageKeys.VAULT_SERVER_BLOB_HASHES)) as string[] | null) ?? []);-- let personalToUpload: string[] = [];- let sharedToUpload: string[] = [];- if (migration) {- personalToUpload = personalHashes;- const sharedCandidates = sharedHashes.filter(h => !knownServerHashes.has(h));- if (sharedCandidates.length > 0) {- const missingResp = await webApi.post<{ hashes: string[] }, MissingBlobsResponseDto>(BLOBS_MISSING_ENDPOINT, { hashes: sharedCandidates });- sharedToUpload = missingResp.missing ?? [];- }- } else {- const uploadCandidates = allBlobHashes.filter(h => !knownServerHashes.has(h));- let toUpload: string[] = [];- if (uploadCandidates.length > 0) {- const missingResp = await webApi.post<{ hashes: string[] }, MissingBlobsResponseDto>(BLOBS_MISSING_ENDPOINT, { hashes: uploadCandidates });- toUpload = missingResp.missing ?? [];- }- const toUploadSet = new Set(toUpload);- personalToUpload = personalHashes.filter(h => toUploadSet.has(h));- sharedToUpload = sharedHashes.filter(h => toUploadSet.has(h));- }-- devLog(`[V2Push] Blob diff: ${allBlobHashes.length} blobs across ${candidates.length} manifest(s), uploading ${personalToUpload.length} personal + ${sharedToUpload.length} shared${migration ? ' (personal manifest re-encrypted, VEK migration)' : ''}.`);-- // Pre-upload the missing bytes so the write below carries references only; each staged entry carries its own key.- const uploadedCiphertexts = await this.uploadBlobs(webApi, blobEntries, personalToUpload, migration !== null);- for (const [hash, ciphertext] of await this.uploadBlobs(webApi, blobEntries, sharedToUpload)) {- uploadedCiphertexts.set(hash, ciphertext);- }-- const payload = {- username,- manifests: manifestWrites,- buckets: bucketDtos,- newBlobs: [] as BlobDto[],- emailRouting,- // Legacy: only the one-time KEK/VEK migration push carries a key hierarchy for the server to store.- accountKeys: migration?.accountKeys ?? null,- };-- let resp = await webApi.post<typeof payload, VaultWriteResponseDto>(VAULT_ENDPOINT, payload);-- if (resp.missingBlobHashes && resp.missingBlobHashes.length > 0) {- /*- * Our local knowledge of the server's blob set was stale (e.g. the server GC'd a blob between syncs).- * Upload the bytes it asked for (each with its correct key) and retry the write once. Nothing was committed- * (all-or-nothing), so the retry sends the identical payload.- */- const unsatisfiable = resp.missingBlobHashes.filter(h => !blobEntries.has(h));- if (unsatisfiable.length > 0) {- return { status: 'missing-blobs', reasons: unsatisfiable };- }-- devWarn(`[V2Sync] Server reported ${resp.missingBlobHashes.length} missing blob(s); uploading and retrying once.`);- const retriedPersonal = await this.uploadBlobs(webApi, blobEntries, resp.missingBlobHashes.filter(h => blobEntries.get(h)?.fromPersonal === true), migration !== null);- const retriedShared = await this.uploadBlobs(webApi, blobEntries, resp.missingBlobHashes.filter(h => blobEntries.get(h)?.fromPersonal === false));- for (const [hash, ciphertext] of [...retriedPersonal, ...retriedShared]) {- uploadedCiphertexts.set(hash, ciphertext);- }-- resp = await webApi.post<typeof payload, VaultWriteResponseDto>(VAULT_ENDPOINT, payload);- if (resp.missingBlobHashes && resp.missingBlobHashes.length > 0) {- return { status: 'missing-blobs', reasons: resp.missingBlobHashes };- }- }-- if (resp.status !== 0) {- // All-or-nothing: a single stale manifest or bucket rejected the whole write; the orchestrator pulls/merges/retries.- return { status: 'outdated' };- }-- // 5) Update local persisted state on success.- if ((resp.bucketRevisions ?? []).length > 0) {- await recordBucketRevisions(Object.fromEntries(resp.bucketRevisions.map(br => [bucketRevisionKey(br.manifestId, br.category), br.revision])));- }-- // Advance the baseline of the manifests this write included.- await recordManifestRevisions(toManifestRevisionMap(resp.manifestRevisions));-- /*- * Record the new content baselines for every target this write actually carried, so the next push can skip- * them again when unchanged. Targets left out of the write keep their existing baselines.- */- for (const [key, fingerprint] of Object.entries(writtenBucketFingerprints)) {- fingerprints[key] = fingerprint;- }- for (const [manifestId, fingerprint] of Object.entries(writtenManifestFingerprints)) {- fingerprints[fingerprintManifestKey(manifestId)] = fingerprint;- }- await this.saveContentFingerprints(fingerprints);-- // Every referenced hash is now known to be on the server, refresh the diff baseline.- await getPlatform().storage.set(StorageKeys.VAULT_SERVER_BLOB_HASHES, allBlobHashes);-- /*- * Refresh the encrypted blob cache: keep entries still referenced by the new manifests, add the ciphertexts we just uploaded.- */- const cache = await this.loadBlobCache();- const newCache: Record<string, string> = {};- for (const hash of allBlobHashes) {- const ciphertext = uploadedCiphertexts.get(hash) ?? cache[hash];- if (ciphertext) {- newCache[hash] = ciphertext;- }- }- await this.saveBlobCache(newCache);-- if (migration) {- await completeLegacyAccountKeyMigration(migration);- devLog('[V2Push] Account-key migration complete: hierarchy created server-side, blob chain cached locally.');- }-- const uploadedBlobChars = Array.from(uploadedCiphertexts.values()).reduce((sum, c) => sum + c.length, 0);- const bucketChars = bucketDtos.reduce((sum, b) => sum + b.blob.length, 0);- const manifestChars = manifestWrites.reduce((sum, m) => sum + m.manifestBlob.length, 0);- const totalChars = manifestChars + bucketChars + uploadedBlobChars;- devLog(`[V2Push] Total pushed (encrypted): ${manifestWrites.length} manifest(s) ${formatKb(manifestChars)} + ${bucketDtos.length} buckets ${formatKb(bucketChars)} + ${uploadedCiphertexts.size} blobs ${formatKb(uploadedBlobChars)} = ${formatKb(totalChars)}.`);-- return { status: 'ok', newEncryptionKey: migration ? contentKey : undefined };- }-- /**- * Canonicalize the local vault into the manifest-v1 format against every manifest this vault writes, routing each- * row into its own manifest by the ManifestId it carries.- * @param sqliteClient - the in-memory SQLite database to canonicalize- * @returns The canonicalized set plus the manifest records it was split against- */- private async canonicalizeVault(sqliteClient: SqliteClient, options?: { adoptUnstampedInto?: string | null }): Promise<CanonicalizedVaultSet> {- // Read tables from the SQLite database and apply the manifest-v1 format rules.- const tables = VaultCodec.readTables(sqliteClient);-- const manifestRecords = await this.resolveManifestRecords(sqliteClient);- /*- * Membership is the ManifestId stamp on each row (written by the repositories at insert, move and share time),- * so a spec only has to name the manifest and supply its blob salt. The personal manifest goes first: the codec reads the- * first spec as the manifest being written from, which is the routing key every row is matched against and- * which keeps any table the registry does not scope per manifest.- */- const manifests: CodecManifestSpec[] = manifestRecords.map(r => ({ manifestId: r.manifestId, manifestSalt: r.salt, name: r.name }));-- const canonicalized = await timedStage('canonicalize (incl. Rust→JS conversion)', () => vaultCodecCanonicalizeFromSqlite({- tables,- canonicalizedAt: new Date().toISOString(),- manifests,- adoptUnstampedInto: options?.adoptUnstampedInto ?? null,- }));-- return { canonicalized, manifestRecords };- }-- /**- * Whether the local vault is canonically identical to the last-known server state, meaning a push would be- * skipped in its entirety by the content-fingerprint gating (a "no-op mutation", e.g. an entry opened for edit- * and saved without changes). Runs the same canonicalize + fingerprint comparison the push runs and caches the- * canonicalize result, so a push right after (the vault DID change) does not canonicalize a second time.- * Returns false whenever the state cannot be proven unchanged: any changed or missing baseline, or a pending- * KEK/VEK migration (which forces a blanket push regardless of content).- * @param sqliteClient - the local vault- * @param mutationSequence - the current mutation sequence; the cached canonicalize result is handed to the push only while this sequence is unchanged- */- public async detectNoOpMutation(sqliteClient: SqliteClient, mutationSequence: number): Promise<boolean> {- const canonicalizedSet = await this.canonicalizeVault(sqliteClient);- primeCanonicalizeCache(sqliteClient, mutationSequence, canonicalizedSet);-- const { canonicalized, manifestRecords } = canonicalizedSet;- const fingerprints = await this.loadContentFingerprints();-- /*- * Same join the push does (see the candidate list there): a manifest with no record is not pushed, so it- * cannot make this push non-empty either.- */- const writableManifestIds = new Set(manifestRecords.map(r => r.manifestId));- const manifests: CodecManifest[] = canonicalized.manifests.map(m => m.manifest).filter(m => writableManifestIds.has(m.manifestId));- for (const manifest of manifests) {- if (fingerprints[fingerprintManifestKey(manifest.manifestId)] !== await vaultCodecComputeContentFingerprint(JSON.stringify(manifest))) {- return false;- }- }- for (const bucket of canonicalized.dataBuckets) {- if (fingerprints[fingerprintBucketKey(bucket.manifestId, bucket.category)] !== await vaultCodecComputeContentFingerprint(JSON.stringify(bucket))) {- return false;- }- }-- return true;- }-- /**- * Single-data-bucket upload, for changes scoped to a separate data bucket and not touching any manifest. Goes- * through the unified POST /v2/Vault write with an empty manifest list and one bucket; the server's all-or-nothing- * revision gate reports the current bucket revision on conflict, which we rebase onto and retry once.- * @param bucket - the new data bucket contents (its `manifestId` and `category` select the server bucket)- * @param vek - the key of the manifest that owns the bucket- * @param username - the user's username (the unified write cross-checks it against the auth session)- */- public async pushDataBucketOnly(bucket: VaultDataBucket, vek: string, username: string): Promise<{ status: 'ok' | 'outdated'; revision: number }> {- return withOutdatedServerGuard(() => this.pushDataBucketOnlyInternal(bucket, vek, username));- }-- /**- * The bucket-only push implementation; {@link pushDataBucketOnly} wraps it with the outdated-server guard.- * @param bucket - the new data bucket contents- * @param vek - the key of the manifest that owns the bucket- * @param username - the user's username- */- private async pushDataBucketOnlyInternal(bucket: VaultDataBucket, vek: string, username: string): Promise<{ status: 'ok' | 'outdated'; revision: number }> {- const { manifestId, category } = bucket;- const label = `Bucket "${category}" of manifest ${manifestId}`;- const revisionKey = bucketRevisionKey(manifestId, category);-- const plaintext = JSON.stringify(bucket);-- // Same content-fingerprint gate as the full push: a mutation that ended up changing nothing skips the write.- const fingerprints = await this.loadContentFingerprints();- const bucketRevisions = await getBucketRevisions();- const bucketFingerprint = await vaultCodecComputeContentFingerprint(plaintext);- if (fingerprints[fingerprintBucketKey(manifestId, category)] === bucketFingerprint) {- devLog(`[V2Push] ${label} (bucket-only) unchanged versus server baseline, skipping upload.`);- return { status: 'ok', revision: bucketRevisions[revisionKey] ?? 0 };- }-- const { ciphertext, compressedBytes } = await packEncrypt(plaintext, vek);- const ciphertextHash = await vaultCodecComputeCiphertextHash(ciphertext);- devLog(`[V2Push] ${label} (bucket-only): raw ${formatKb(plaintext.length)} → compressed ${formatKb(compressedBytes)} → encrypted ${formatKb(ciphertext.length)}.`);-- const webApi = new WebApiService();- /** POST the single-bucket write with the given believed-current revision (called again on the rebase retry). */- const postBucket = (currentRevision: number): Promise<VaultWriteResponseDto> => webApi.post<Record<string, unknown>, VaultWriteResponseDto>(VAULT_ENDPOINT, {- username, manifests: [], buckets: [{ manifestId, category, blob: ciphertext, ciphertextHash, currentRevision }], newBlobs: [], emailRouting: null,- });-- /** This bucket's revision as the server reported it, matched on both halves of its address. */- const reportedRevision = (resp: VaultWriteResponseDto): number | undefined =>- (resp.bucketRevisions ?? []).find(b => manifestIdsEqual(b.manifestId, manifestId) && b.category === category)?.revision;-- let currentRevision = bucketRevisions[revisionKey] ?? 0;- let resp = await postBucket(currentRevision);-- if (resp.status !== 0) {- const serverRevision = reportedRevision(resp) ?? currentRevision;- devWarn(`[V2Push] ${label} outdated (server at revision ${serverRevision}, we assumed ${currentRevision}); rebasing and retrying once.`);- currentRevision = serverRevision;- resp = await postBucket(currentRevision);- }-- if (resp.status !== 0) {- return { status: 'outdated', revision: reportedRevision(resp) ?? currentRevision };- }-- const newRevision = reportedRevision(resp) ?? currentRevision + 1;- await recordBucketRevisions({ [revisionKey]: newRevision });-- // New server baseline for this bucket, so an unchanged follow-up push (bucket-only or full) can skip it.- fingerprints[fingerprintBucketKey(manifestId, category)] = bucketFingerprint;- await this.saveContentFingerprints(fingerprints);-- return { status: 'ok', revision: newRevision };- }-- /**- * The key each manifest's data buckets are encrypted with: the personal VEK for the personal manifest,- * and its own VEK for every shared manifest whose grant this session can unwrap.- * @param sqliteClient - the open local vault, which holds the private keys the grants are unwrapped with- * @param personalVek - the personal manifest's symmetric key- * @returns manifest id → the key its buckets are written under- */- public async resolveBucketWriteKeys(sqliteClient: SqliteClient, personalVek: string): Promise<Map<string, string>> {- const keys = new Map<string, string>([[await this.resolvePersonalManifestId(), personalVek]]);- for (const [manifestId, vek] of await SharingService.openSharedManifestVeks(sqliteClient)) {- keys.set(manifestId, vek);- }- return keys;- }-- /**- * The manifests the local vault holds rows for that this session cannot write: no grant recorded for them, or- * one that will not unwrap.- * @param sqliteClient - the open local vault- * @returns The manifest ids the vault holds but this session cannot write, empty when the two agree- */- public async findUnwritableManifests(sqliteClient: SqliteClient): Promise<string[]> {- return (await this.partitionManifestAccess(sqliteClient, [])).unwritable;- }-- /**- * Split what the vault holds against what this account may still open.- * @param sqliteClient - the open local vault- * @param grantedManifestIds - the manifests the last snapshot served, empty when that is not being asked- */- private async partitionManifestAccess(sqliteClient: SqliteClient, grantedManifestIds: Iterable<string>): Promise<SharingAccessPartition> {- const writable = [...(await SharingService.openSharedManifestVeks(sqliteClient)).keys()];-- const personalManifestId = await getPersonalManifestId();- if (personalManifestId) {- writable.push(personalManifestId);- }-- return vaultSharingPartitionManifestAccess({- manifestIdsInVault: [...VaultCodec.manifestIdsInVault(sqliteClient)],- writableManifestIds: writable,- grantedManifestIds: [...grantedManifestIds],- });- }-- /**- * The manifests the last snapshot served this account. This is what the account may still open.- * @returns The served manifest ids, empty when nothing has been pulled in this session- */- public manifestIdsServedByLastSnapshot(): string[] {- return [...this.lastServedManifestIds];- }-- /**- * Whether the last snapshot was served in the legacy sqlite-blob storage format rather than manifest-v1.- *- * TODO: delete once all users have migrated to the manifest-v1 storage model.- * @returns True when the server has not migrated yet, false when it served manifest-v1 or nothing was pulled in this session- */- public lastSnapshotServedLegacySqliteBlob(): boolean {- return this.lastSnapshotWasLegacySqliteBlob;- }-- /**- * Drop the rows of every manifest the server no longer serves this account.- * @param sqliteClient - the open local vault, mutated in place- * @param grantedManifestIds - the manifests the snapshot last served- * @returns The manifest ids whose rows were dropped, empty when there was nothing to drop- */- public async dropRowsOfManifestsNoLongerServed(sqliteClient: SqliteClient, grantedManifestIds: Iterable<string>): Promise<string[]> {- const { lost } = await this.partitionManifestAccess(sqliteClient, grantedManifestIds);- if (lost.length === 0) {- return [];- }-- for (const table of VaultCodec.stampedTables(sqliteClient)) {- for (const manifestId of lost) {- sqliteClient.executeUpdate(`DELETE FROM "${table}" WHERE lower(ManifestId) = ?`, [manifestIdKey(manifestId)]);- }- }-- devWarn(`[VaultSync] Dropped the rows of manifest(s) this account no longer holds (${lost.join(', ')}); they are gone from the server for this account.`);- return lost;- }-- /**- * Resolve every manifest this vault can write, personal manifest first, as one uniform list.- *- * The core decides which, so a second client cannot arrive at a different write set. It is told which- * manifests opened, never what they opened with.- * @param sqliteClient - the open local vault- */- private async resolveManifestRecords(sqliteClient: SqliteClient): Promise<ManifestRecord[]> {- let manifestSalt = (await getPlatform().storage.get(StorageKeys.VAULT_MANIFEST_SALT)) as string | null;- if (!manifestSalt) {- manifestSalt = await vaultCodecGenerateManifestSalt();- await getPlatform().storage.set(StorageKeys.VAULT_MANIFEST_SALT, manifestSalt);- }-- const sharedVeks = await SharingService.openSharedManifestVeks(sqliteClient);- const writeSet = await vaultSharingResolveManifestWriteSet({- personalManifestId: await this.resolvePersonalManifestId(),- personalManifestSalt: manifestSalt,- stampedManifestIds: [...VaultCodec.manifestIdsInVault(sqliteClient)],- openedManifestIds: [...sharedVeks.keys()],- heldRecords: Object.values(await SharingService.getSharedManifestRecords()),- displayNames: multiManifestRendering.displayNames(sqliteClient),- });-- for (const skipped of writeSet.skipped) {- const why = skipped.reason === 'NO_ROWS_IN_VAULT'- ? 'has no rows in this vault; leaving it out of the write rather than emptying it server-side'- : 'did not open; leaving it out of the write';- devLog(`[V2Push] Shared manifest ${skipped.manifestId} ${why}.`);- }-- // Re-attach the keys. Without one a shared manifest falls back to the personal key downstream, so assert it.- const vekByManifestId = new Map([...sharedVeks].map(([manifestId, vek]) => [manifestIdKey(manifestId), vek]));- return writeSet.records.map(record => {- const vek = record.isPersonal ? null : vekByManifestId.get(manifestIdKey(record.manifestId)) ?? null;- if (!record.isPersonal && !vek) {- throw new Error(`VaultSyncService: manifest ${record.manifestId} is in the write set without a key, refusing to write it.`);- }- return { ...record, vek };- });- }-- /**- * The id of this vault's own (personal) manifest, as reported by the server on the last pull. Pushing without it- * is impossible: it addresses the write and keys the revision the write rebases on.- */- private async resolvePersonalManifestId(): Promise<string> {- const personalManifestId = await getPersonalManifestId();- if (!personalManifestId) {- throw new Error('VaultSyncService: no personal manifest id available (no snapshot baseline recorded); pull once before pushing.');- }- return personalManifestId;- }-- /**- * Encrypt the given blobs (each staged with its own VEK: the personal VEK or a folder VEK) and upload them via- * POST /v2/Vault/blobs in size-capped batches. `overwrite` is set only for personal-manifest blobs on a KEK/VEK migration;- * shared-manifest blobs keep their own VEK across a migration, so their ciphertext is never re-keyed.- * @param webApi - API client to reuse- * @param entries - hash → staged blob for every candidate (hashes without an entry are skipped)- * @param hashes - the subset of hashes to upload- * @param overwrite - ask the server to replace the ciphertext of blobs it already has (KEK/VEK migration)- * @returns Map of hash → uploaded ciphertext (base64), for the local encrypted blob cache.- */- private async uploadBlobs(webApi: WebApiService, entries: Map<string, UploadBlobEntry>, hashes: string[], overwrite: boolean = false): Promise<Map<string, string>> {- const ciphertexts = new Map<string, string>();- if (hashes.length === 0) {- return ciphertexts;- }-- let batch: BlobDto[] = [];- let batchChars = 0;- for (const hash of hashes) {- const entry = entries.get(hash);- if (!entry) {- continue;- }-- const ciphertext = await this.encryptBlobBytes(entry.bytes, entry.vek);- ciphertexts.set(hash, ciphertext);- devLog(`[V2Push] Blob ${hash.substring(0, 12)}… (${entry.kind}): raw ${formatKb(entry.bytes.length)} → encrypted ${formatKb(ciphertext.length)}.`);-- // Flush before adding when this blob would take the request past either bound.- if (batch.length > 0 && (batchChars + ciphertext.length > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.length >= BLOB_TRANSFER_BATCH_MAX_COUNT)) {- devLog(`[V2Push] Uploading blob batch: ${batch.length} blobs, ${formatKb(batchChars)}.`);- await webApi.post(BLOBS_VAULT_ENDPOINT, { blobs: batch, overwrite });- batch = [];- batchChars = 0;- }-- batch.push({ hash, category: entry.kind, encryptedDataBase64: ciphertext });- batchChars += ciphertext.length;- }-- if (batch.length > 0) {- devLog(`[V2Push] Uploading blob batch: ${batch.length} blobs, ${formatKb(batchChars)}.`);- await webApi.post(BLOBS_VAULT_ENDPOINT, { blobs: batch, overwrite });- }-- return ciphertexts;- }-- /**- * Encrypt raw blob bytes with the VEK. Result is base64-of-(IV | ciphertext | tag).- * @param bytes - plaintext bytes- * @param vek - symmetric encryption key- */- private async encryptBlobBytes(bytes: Uint8Array, vek: string): Promise<string> {- return EncryptionUtility.symmetricEncryptBytes(bytes, vek);- }-- /**- * Decrypt a blob ciphertext (base64) and return raw plaintext bytes. Counterpart of {@link encryptBlobBytes}.- * @param encryptedDataBase64 - base64 IV | ciphertext | tag from the server- * @param vek - symmetric encryption key- */- private async decryptBlobToBytes(encryptedDataBase64: string, vek: string): Promise<Uint8Array> {- return EncryptionUtility.symmetricDecryptBytes(base64ToBytes(encryptedDataBase64), vek);- }-- /**- * Load the per-target content-fingerprint baselines (see {@link StorageKeys.VAULT_CONTENT_FINGERPRINTS}).- */- private async loadContentFingerprints(): Promise<Record<string, string>> {- return ((await getPlatform().storage.get(StorageKeys.VAULT_CONTENT_FINGERPRINTS)) as Record<string, string> | null) ?? {};- }-- /**- * Persist the per-target content-fingerprint baselines.- * @param fingerprints - fingerprint record to persist- */- private async saveContentFingerprints(fingerprints: Record<string, string>): Promise<void> {- await getPlatform().storage.set(StorageKeys.VAULT_CONTENT_FINGERPRINTS, fingerprints);- }-- /**- * Load the local encrypted blob cache (hash → base64 ciphertext) used to skip re-downloading known blobs.- * Entries are stored as served to/from the server, so nothing in the cache is plaintext at rest.- */- private async loadBlobCache(): Promise<Record<string, string>> {- return ((await getPlatform().storage.get(StorageKeys.VAULT_BLOB_CIPHER_CACHE)) as Record<string, string> | null) ?? {};- }-- /**- * Persist the local encrypted blob cache (hash → base64 ciphertext).- * @param cache - cache to persist- */- private async saveBlobCache(cache: Record<string, string>): Promise<void> {- await getPlatform().storage.set(StorageKeys.VAULT_BLOB_CIPHER_CACHE, cache);- }--}