Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Make sync engine pass database exports as raw bytes where possible to save on JSON overhead
Leendert de Borst052c9eb9546f
diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.ktindex 13ee81aba..db6b6de2c 100644--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.kt+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.kt@@ -1,6 +1,5 @@package net.aliasvault.app.vaultstore-import android.util.Base64import android.util.Logimport kotlinx.coroutines.Dispatchersimport kotlinx.coroutines.withContext@@ -67,10 +66,10 @@ class VaultSyncEngine(return result}val startNanos = System.nanoTime()- val response = handle(kind, command)- log.recordCommand(kind, command, response, System.nanoTime() - startNanos)+ val (response, bytes) = respond(kind, command)+ log.recordCommand(kind, command, response, bytes, System.nanoTime() - startNanos)val responseJson = log.json { response.toString() }- log.engine { session.resume(responseJson) }+ log.engine { session.resume(responseJson, bytes) }}} finally {closeStaging()@@ -107,6 +106,21 @@ class VaultSyncEngine(// region Commands+ /**+ * The JSON response to one command.+ */+ @Suppress("TooGenericExceptionCaught")+ private suspend fun respond(kind: String, command: JSONObject): Pair<JSONObject, ByteArray?> {+ if (kind != "dbExport") {+ return Pair(handle(kind, command), null)+ }+ return try {+ Pair(JSONObject(), exportDatabase(command.optString("db")))+ } catch (e: Exception) {+ Pair(errorResponse(kind, e), null)+ }+ }+@Suppress("TooGenericExceptionCaught")private suspend fun handle(kind: String, command: JSONObject): JSONObject {return try {@@ -122,7 +136,7 @@ class VaultSyncEngine(JSONObject()}"dbOpen" -> {- openStaging(command.optString("bytes", null).takeIf { command.has("bytes") && !command.isNull("bytes") })+ openStaging()JSONObject()}"dbQuery" -> {@@ -137,14 +151,6 @@ class VaultSyncEngine(}JSONObject()}- "dbExport" -> {- val bytes = when (val db = command.optString("db")) {- "local" -> vaultStore.database.export()- "staging" -> stagingDatabase().export()- else -> error("Unknown database $db")- }- JSONObject().put("bytes", Base64.encodeToString(bytes, Base64.NO_WRAP))- }"vaultStore" -> storeVault(command)"vaultLoad" -> JSONObject().put("encryptedBlob", if (vaultStore.hasEncryptedDatabase()) vaultStore.getEncryptedDatabase() else JSONObject.NULL)"markClean" -> {@@ -158,11 +164,27 @@ class VaultSyncEngine(else -> JSONObject().put("error", "Unknown engine command $kind")}} catch (e: Exception) {- Log.w(TAG, "Engine command $kind failed", e)- JSONObject().put("error", e.message ?: e.toString())+ errorResponse(kind, e)}}+ /**+ * Serialize a database to SQLite bytes.+ */+ private fun exportDatabase(name: String): ByteArray = when (name) {+ "local" -> vaultStore.database.export()+ "staging" -> stagingDatabase().export()+ else -> error("Unknown database $name")+ }++ /**+ * The `{ "error": ... }` response for a failed command.+ */+ private fun errorResponse(kind: String, e: Exception): JSONObject {+ Log.w(TAG, "Engine command $kind failed", e)+ return JSONObject().put("error", e.message ?: e.toString())+ }+@Suppress("TooGenericExceptionCaught")private suspend fun http(command: JSONObject): JSONObject = withContext(Dispatchers.IO) {val body = if (command.has("body") && !command.isNull("body")) command.getString("body") else null@@ -274,13 +296,9 @@ class VaultSyncEngine(/*** Open the staging database in memory.*/- private fun openStaging(bytesBase64: String?) {+ private fun openStaging() {closeStaging()- staging = if (bytesBase64 != null) {- SqliteMemoryDatabase.fromBytes(Base64.decode(bytesBase64, Base64.NO_WRAP))- } else {- SqliteMemoryDatabase.withSchema(VaultSql.completeSchema)- }.also {+ staging = SqliteMemoryDatabase.withSchema(VaultSql.completeSchema).also {// The schema script ends by turning foreign keys on; the engine inserts rows in codec order, not FK order.it.executeBatch("PRAGMA foreign_keys = OFF")}diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncRunLog.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncRunLog.ktindex 9efa75926..97ab005d1 100644--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncRunLog.kt+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncRunLog.kt@@ -35,8 +35,12 @@ class VaultSyncRunLog(private val operation: String) {/*** The size of a text payload (JSON or base64) for the logs, by character count.*/- private fun formatSize(value: Any?): String {- val size = (value as? String)?.length ?: 0+ private fun formatSize(value: Any?): String = formatByteCount((value as? String)?.length ?: 0)++ /**+ * A byte count for the logs.+ */+ private fun formatByteCount(size: Int): String {return when {size < BYTES_PER_KB -> "$size B"size < BYTES_PER_KB * BYTES_PER_KB -> String.format(Locale.US, "%.1f KB", size / BYTES_PER_KB)@@ -107,10 +111,10 @@ class VaultSyncRunLog(private val operation: String) {/*** Count one handled command, and log it on its own when it is a transfer or slow.*/- fun recordCommand(kind: String, command: JSONObject, response: JSONObject, nanos: Long) {+ fun recordCommand(kind: String, command: JSONObject, response: JSONObject, responseBytes: ByteArray?, nanos: Long) {commandNanos[kind] = (commandNanos[kind] ?: 0L) + nanoscommandCounts[kind] = (commandCounts[kind] ?: 0) + 1- describe(kind, command, response, nanos / NANOS_PER_MS)?.let { note(it) }+ describe(kind, command, response, responseBytes, nanos / NANOS_PER_MS)?.let { note(it) }}/**@@ -135,7 +139,7 @@ class VaultSyncRunLog(private val operation: String) {/*** One line for a command relevant for performance analysis, or null for unrelated calls.*/- private fun describe(kind: String, command: JSONObject, response: JSONObject, ms: Long): String? {+ private fun describe(kind: String, command: JSONObject, response: JSONObject, responseBytes: ByteArray?, ms: Long): String? {val took = "$kind took ${ms}ms"if (response.has("error")) {return "$took: failed (${response.optString("error")})"@@ -154,8 +158,7 @@ class VaultSyncRunLog(private val operation: String) {}"vaultStore" -> "$took: ${formatSize(command.opt("encryptedBlob"))}""vaultLoad" -> "$took: ${formatSize(response.opt("encryptedBlob"))}"- "dbOpen" -> "$took: ${if (command.isNull("bytes")) "fresh schema" else formatSize(command.opt("bytes"))}"- "dbExport" -> "$took: $db ${formatSize(response.opt("bytes"))}"+ "dbExport" -> "$took: $db ${formatByteCount(responseBytes?.size ?: 0)}""dbExec" -> if (slow) "$took: ${command.optJSONArray("statements")?.length() ?: 0} statement(s) on $db" else null"dbQuery" -> if (slow) "$took: ${response.optJSONArray("rows")?.length() ?: 0} row(s) from $db" else nullelse -> if (slow) took else nulldiff --git a/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swift b/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swiftindex 5f28accb5..adce51901 100644--- a/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swift+++ b/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swift@@ -63,10 +63,10 @@ public final class VaultSyncEngine {return result}let commandStartedAt = Date()- let response = await handle(kind: kind, command: command)- log.recordCommand(kind, command: command, response: response, since: commandStartedAt)+ let (response, bytes) = await respond(kind: kind, command: command)+ log.recordCommand(kind, command: command, response: response, responseBytes: bytes, since: commandStartedAt)let responseJson = try log.json { try Self.serializeJson(response) }- try log.engine { try session.resume(responseJson: responseJson) }+ try log.engine { try session.resume(responseJson: responseJson, bytes: bytes) }}}@@ -103,6 +103,18 @@ public final class VaultSyncEngine {// MARK: - Commands+ /// The JSON response to one command.+ private func respond(kind: String, command: [String: Any]) async -> (json: [String: Any], bytes: Data?) {+ guard kind == "dbExport" else {+ return (await handle(kind: kind, command: command), nil)+ }+ do {+ return ([:], try exportDatabase(named: command["db"] as? String ?? ""))+ } catch {+ return (Self.errorResponse(error), nil)+ }+ }+private func handle(kind: String, command: [String: Any]) async -> [String: Any] {do {switch kind {@@ -117,7 +129,7 @@ public final class VaultSyncEngine {setState(nil, forKey: command["key"] as? String ?? "")return [:]case "dbOpen":- try openStaging(bytesBase64: command["bytes"] as? String)+ try openStaging()return [:]case "dbQuery":let db = try database(named: command["db"] as? String ?? "")@@ -130,17 +142,6 @@ public final class VaultSyncEngine {localMutated = true}return [:]- case "dbExport":- let bytes: Data- switch command["db"] as? String ?? "" {- case "local":- bytes = try vaultStore.exportDatabase()- case "staging":- bytes = try database(named: "staging").export()- case let name:- throw AppError.unknownError(message: "Unknown database \(name)")- }- return ["bytes": bytes.base64EncodedString()]case "vaultStore":return try storeVault(command)case "vaultLoad":@@ -154,13 +155,31 @@ public final class VaultSyncEngine {default:return ["error": "Unknown engine command \(kind)"]}- } catch let error as AppError {- return ["error": error.message]} catch {- return ["error": "\(error)"]+ return Self.errorResponse(error)+ }+ }++ /// Serialize a database to SQLite bytes.+ private func exportDatabase(named name: String) throws -> Data {+ switch name {+ case "local":+ return try vaultStore.exportDatabase()+ case "staging":+ return try database(named: "staging").export()+ default:+ throw AppError.unknownError(message: "Unknown database \(name)")}}+ /// The `{ "error": ... }` response for a failed command.+ private static func errorResponse(_ error: Error) -> [String: Any] {+ if let appError = error as? AppError {+ return ["error": appError.message]+ }+ return ["error": "\(error)"]+ }+private func http(_ command: [String: Any]) async -> [String: Any] {let method = command["method"] as? String ?? "GET"let path = command["path"] as? String ?? ""@@ -279,15 +298,10 @@ public final class VaultSyncEngine {}}- /// Open the staging database in memory: from SQLite bytes, or fresh with the current client schema.- private func openStaging(bytesBase64: String?) throws {+ /// Open a fresh staging database in memory with the current client schema.+ private func openStaging() throws {closeStaging()- let opened: SqliteMemoryDatabase- if let bytesBase64 = bytesBase64, let bytes = Data(base64Encoded: bytesBase64) {- opened = try SqliteMemoryDatabase.fromBytes(bytes: bytes)- } else {- opened = try SqliteMemoryDatabase.withSchema(schemaSql: VaultSql.completeSchema)- }+ let opened = try SqliteMemoryDatabase.withSchema(schemaSql: VaultSql.completeSchema)// The schema script ends by turning foreign keys on; the engine inserts rows in codec order, not FK order.try opened.executeBatch(sql: "PRAGMA foreign_keys = OFF")staging = openeddiff --git a/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncRunLog.swift b/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncRunLog.swiftindex 159a7b261..cb978a229 100644--- a/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncRunLog.swift+++ b/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncRunLog.swift@@ -21,7 +21,11 @@ internal final class VaultSyncRunLog {/// The size of a text payload (JSON or base64) for the logs, by UTF-8 byte count.private static func formatSize(_ value: Any?) -> String {- let size = (value as? String)?.utf8.count ?? 0+ return formatByteCount((value as? String)?.utf8.count ?? 0)+ }++ /// A byte count for the logs.+ private static func formatByteCount(_ size: Int) -> String {if size < 1024 {return "\(size) B"}@@ -78,11 +82,11 @@ internal final class VaultSyncRunLog {}/// Count one handled command, and log it on its own when it is a transfer or slow.- func recordCommand(_ kind: String, command: [String: Any], response: [String: Any], since commandStartedAt: Date) {+ func recordCommand(_ kind: String, command: [String: Any], response: [String: Any], responseBytes: Data?, since commandStartedAt: Date) {let durationMs = Date().timeIntervalSince(commandStartedAt) * 1000let timing = commandTimings[kind] ?? (count: 0, totalMs: 0)commandTimings[kind] = (count: timing.count + 1, totalMs: timing.totalMs + durationMs)- if let line = describe(kind: kind, command: command, response: response, durationMs: durationMs) {+ if let line = describe(kind: kind, command: command, response: response, responseBytes: responseBytes, durationMs: durationMs) {note(line)}}@@ -105,7 +109,7 @@ internal final class VaultSyncRunLog {}/// One line for a command relevant for performance analysis, or nil for unrelated calls.- private func describe(kind: String, command: [String: Any], response: [String: Any], durationMs: Double) -> String? {+ private func describe(kind: String, command: [String: Any], response: [String: Any], responseBytes: Data?, durationMs: Double) -> String? {let took = "\(kind) took \(Int(durationMs))ms"if let error = response["error"] as? String {return "\(took): failed (\(error))"@@ -123,10 +127,8 @@ internal final class VaultSyncRunLog {return "\(took): \(Self.formatSize(command["encryptedBlob"]))"case "vaultLoad":return "\(took): \(Self.formatSize(response["encryptedBlob"]))"- case "dbOpen":- return "\(took): \(command["bytes"] is String ? Self.formatSize(command["bytes"]) : "fresh schema")"case "dbExport":- return "\(took): \(database) \(Self.formatSize(response["bytes"]))"+ return "\(took): \(database) \(Self.formatByteCount(responseBytes?.count ?? 0))"case "dbExec":return slow ? "\(took): \((command["statements"] as? [Any])?.count ?? 0) statement(s) on \(database)" : nilcase "dbQuery":diff --git a/core/client/src/rust/RustCoreBinding.ts b/core/client/src/rust/RustCoreBinding.tsindex 6b10c1981..0134f0a58 100644--- a/core/client/src/rust/RustCoreBinding.ts+++ b/core/client/src/rust/RustCoreBinding.ts@@ -7,8 +7,8 @@ export interface IVaultSyncSession {/** The next command for the host, as JSON. */nextCommand(): Promise<string>;- /** Hand the host's response to the last command back, as JSON. */- resume(responseJson: string): Promise<void>;+ /** Hand the host's response to the last command back, as JSON, with raw bytes for a `dbExport`. */+ resume(responseJson: string, bytes: Uint8Array | null): Promise<void>;/** Release the session. */free(): void;diff --git a/core/client/src/rust/WasmRustCore.ts b/core/client/src/rust/WasmRustCore.tsindex 3b7960cb8..fd45fc668 100644--- a/core/client/src/rust/WasmRustCore.ts+++ b/core/client/src/rust/WasmRustCore.ts@@ -98,7 +98,7 @@ export function createWasmRustCore(loadWasm: WasmLoader): IRustCore {const session = new core.VaultSyncSession(requestJson);return {nextCommand: async (): Promise<string> => session.nextCommand(),- resume: async (responseJson: string): Promise<void> => session.resume(responseJson),+ resume: async (responseJson: string, bytes: Uint8Array | null): Promise<void> => session.resume(responseJson, bytes),free: (): void => session.free(),};}),diff --git a/core/client/src/sync/VaultSyncEngine.ts b/core/client/src/sync/VaultSyncEngine.tsindex bd87639da..5936d621e 100644--- a/core/client/src/sync/VaultSyncEngine.ts+++ b/core/client/src/sync/VaultSyncEngine.ts@@ -175,7 +175,7 @@ type EngineCommand =| { kind: 'stateGet'; key: string }| { kind: 'stateSet'; key: string; value: JsonValue }| { kind: 'stateRemove'; key: string }- | { kind: 'dbOpen'; db: string; bytes?: string | null }+ | { kind: 'dbOpen'; db: string }| { kind: 'dbQuery'; db: string; sql: string; params: JsonValue[] }| { kind: 'dbExec'; db: string; statements: EngineSqlStatement[] }| { kind: 'dbExport'; db: string }@@ -185,6 +185,9 @@ type EngineCommand =| { kind: 'log'; level: 'log' | 'warn' | 'phase'; message: string }| { kind: 'done'; result: JsonValue };+/** The host's response to one command: the JSON, plus the SQLite file for a `dbExport`. */+type HostResponse = { json: JsonValue; bytes: Uint8Array | null };+/** The database names a command may address. */const DB_LOCAL = 'local';const DB_STAGING = 'staging';@@ -302,15 +305,16 @@ class EngineRun {public constructor(private readonly host: IVaultSyncEngineHost, private readonly webApi: WebApiService) {}/**- * Carry out one command.+ * Carry out one command and return the host's response.* @param command - the command*/- public async handle(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<JsonValue> {+ public async handle(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<HostResponse> {const started = now();try {- return await this.dispatch(command);+ const response = await this.dispatch(command);+ return response instanceof Uint8Array ? { json: {}, bytes: response } : { json: response, bytes: null };} catch (error) {- return { error: error instanceof Error ? error.message : String(error) };+ return { json: { error: error instanceof Error ? error.message : String(error) }, bytes: null };} finally {const timing = this.timings.get(command.kind) ?? { count: 0, ms: 0 };timing.count++;@@ -346,10 +350,10 @@ class EngineRun {}/**- * The command handlers.+ * The command handlers. Raw bytes (a `dbExport`) go back to the engine outside the JSON.* @param command - the command*/- private async dispatch(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<JsonValue> {+ private async dispatch(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<JsonValue | Uint8Array> {switch (command.kind) {case 'http':return this.http(command);@@ -362,7 +366,7 @@ class EngineRun {await getPlatform().storage.remove(stateKey(command.key));return {};case 'dbOpen':- await this.openStaging(command.bytes ?? null);+ await this.openStaging();return {};case 'dbQuery': {const rows = (await this.database(command.db)).query(command.sql, command.params.map(toBindValue));@@ -375,7 +379,7 @@ class EngineRun {}return {};case 'dbExport':- return { bytes: bytesToBase64((await this.database(command.db)).export()) };+ return (await this.database(command.db)).export();case 'vaultStore':return this.storeVault(command);case 'vaultLoad':@@ -444,13 +448,12 @@ class EngineRun {}/**- * Open the staging database.- * @param bytesBase64 - the SQLite file bytes, or null for a fresh database+ * Open a fresh staging database.*/- private async openStaging(bytesBase64: string | null): Promise<void> {+ private async openStaging(): Promise<void> {this.staging?.close();this.staging = null;- const db = bytesBase64 ? await getPlatform().sqlite.open(base64ToBytes(bytesBase64)) : await openFreshSchemaDatabase();+ const db = await openFreshSchemaDatabase();try {db.exec('PRAGMA foreign_keys = OFF');} catch (error) {@@ -496,7 +499,8 @@ export async function runVaultSyncEngine<T extends VaultSyncEngineResultBase>(hodevLog(run.summarize(request.operation, now() - started));return command.result as T;}- await session.resume(JSON.stringify(await run.handle(command)));+ const response = await run.handle(command);+ await session.resume(JSON.stringify(response.json), response.bytes);}} finally {run.close();diff --git a/core/rust/src/bindings/uniffi_api.rs b/core/rust/src/bindings/uniffi_api.rsindex 8f1c412eb..6620926b0 100644--- a/core/rust/src/bindings/uniffi_api.rs+++ b/core/rust/src/bindings/uniffi_api.rs@@ -291,9 +291,9 @@ impl VaultSyncSession {self.inner.next_command()}- /// Hand the host's response to the last command back, as JSON.- pub fn resume(&self, response_json: String) -> Result<(), VaultError> {- self.inner.resume(&response_json)+ /// Hand the host's response to the last command back, as JSON, with raw bytes for a `dbExport`.+ pub fn resume(&self, response_json: String, bytes: Option<Vec<u8>>) -> Result<(), VaultError> {+ self.inner.resume(&response_json, bytes)}}diff --git a/core/rust/src/bindings/wasm.rs b/core/rust/src/bindings/wasm.rsindex f1f543eca..822a1f1e2 100644--- a/core/rust/src/bindings/wasm.rs+++ b/core/rust/src/bindings/wasm.rs@@ -292,9 +292,9 @@ impl VaultSyncSessionJs {self.inner.next_command().map_err(js_err)}- /// Hand the host's response to the last command back, as JSON.- pub fn resume(&self, response_json: &str) -> Result<(), JsValue> {- self.inner.resume(response_json).map_err(js_err)+ /// Hand the host's response to the last command back, as JSON, with raw bytes for a `dbExport`.+ pub fn resume(&self, response_json: &str, bytes: Option<Vec<u8>>) -> Result<(), JsValue> {+ self.inner.resume(response_json, bytes).map_err(js_err)}}diff --git a/core/rust/src/vault_sync/db.rs b/core/rust/src/vault_sync/db.rsindex 87c8b7fd1..b5427ff16 100644--- a/core/rust/src/vault_sync/db.rs+++ b/core/rust/src/vault_sync/db.rs@@ -6,9 +6,9 @@ use serde_json::{json, Map, Value};use super::errors::{SyncError, SyncResult};use super::session::Host;-use super::types::{Ack, Command, Db, DbBytes, DbRows, LogLevel};+use super::types::{Ack, Command, Db, DbRows, LogLevel};use super::legacy;-use crate::common::encoding::{base64_decode, base64_encode, uuid_from_bytes};+use crate::common::encoding::uuid_from_bytes;use crate::sqlite_host::SqlStatement;use crate::common::timestamp::{now_iso_utc, now_vault_datetime};use crate::vault_codec::row::{blob_ref_of, inline_bytes};@@ -35,16 +35,16 @@ pub(crate) async fn exec(host: &Host, db: Db, statements: Vec<SqlStatement>) ->Ok(())}-/// Open the staging database: fresh with the current schema, or from SQLite bytes.-pub(crate) async fn open_staging(host: &Host, bytes: Option<&[u8]>) -> SyncResult<()> {- host.call::<Ack>(Command::DbOpen { db: Db::Staging, bytes: bytes.map(base64_encode) }).await?;+/// Open the staging database fresh with the current schema.+pub(crate) async fn open_staging(host: &Host) -> SyncResult<()> {+ host.call::<Ack>(Command::DbOpen { db: Db::Staging }).await?;Ok(())}/// Serialize a database to SQLite bytes.pub(crate) async fn export(host: &Host, db: Db) -> SyncResult<Vec<u8>> {- let response: DbBytes = host.call(Command::DbExport { db }).await?;- base64_decode(&response.bytes).map_err(|_| SyncError::Other("host returned invalid database bytes".to_string()))+ let (_, bytes) = host.call_with_bytes::<Ack>(Command::DbExport { db }).await?;+ bytes.ok_or_else(|| SyncError::Other("host returned no database bytes".to_string()))}/// A cell as text: strings as they are, null as empty, anything else in its JSON form.diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rsindex b9a5dac7c..916d78445 100644--- a/core/rust/src/vault_sync/pull.rs+++ b/core/rust/src/vault_sync/pull.rs@@ -353,7 +353,7 @@ pub(crate) async fn materialize_to_sqlite(ctx: &mut Ctx, manifests: &[Manifest],}// Use a fresh staging database for every materialize.- db::open_staging(&ctx.host, None).await?;+ db::open_staging(&ctx.host).await?;db::insert_materialized(&ctx.host, &materialized, &schema.columns, blob_map).await?;// Names are not part of a manifest: the ones the vault already shows are kept, the ones this run opened win.diff --git a/core/rust/src/vault_sync/session.rs b/core/rust/src/vault_sync/session.rsindex 4b5dd7d26..2cc412027 100644--- a/core/rust/src/vault_sync/session.rs+++ b/core/rust/src/vault_sync/session.rs@@ -18,6 +18,8 @@ use crate::common::error::{VaultError, VaultResult};pub(crate) struct Slot {pub pending: Option<Command>,pub response: Option<Value>,+ /// Raw bytes the host handed back with the response (the SQLite file of a `dbExport`).+ pub response_bytes: Option<Vec<u8>>,}/// The engine's handle to the host.@@ -33,12 +35,18 @@ impl Host {/// Send a command and wait for the host's typed response. A `{ "error": ... }` response becomes an error.pub async fn call<R: DeserializeOwned>(&self, command: Command) -> SyncResult<R> {+ Ok(self.call_with_bytes(command).await?.0)+ }++ /// Like [`Host::call`], also returning the raw bytes the host attached to its response, if any.+ pub async fn call_with_bytes<R: DeserializeOwned>(&self, command: Command) -> SyncResult<(R, Option<Vec<u8>>)> {let kind = command.kind();- let response = CommandFuture { slot: self.slot.clone(), command: Some(command), sent: false }.await;+ let (response, bytes) = CommandFuture { slot: self.slot.clone(), command: Some(command), sent: false }.await;if let Some(error) = response.get("error").and_then(Value::as_str) {return Err(SyncError::Host { command: kind, message: error.to_string() });}- serde_json::from_value(response).map_err(|e| SyncError::Host { command: kind, message: format!("unexpected response shape: {}", e) })+ let typed = serde_json::from_value(response).map_err(|e| SyncError::Host { command: kind, message: format!("unexpected response shape: {}", e) })?;+ Ok((typed, bytes))}/// Emit a log line.@@ -55,17 +63,18 @@ struct CommandFuture {}impl Future for CommandFuture {- type Output = Value;+ type Output = (Value, Option<Vec<u8>>);- fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Value> {+ fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {let this = &mut *self;if !this.sent {this.slot.lock().expect("sync host slot poisoned").pending = this.command.take();this.sent = true;return Poll::Pending;}- match this.slot.lock().expect("sync host slot poisoned").response.take() {- Some(response) => Poll::Ready(response),+ let mut slot = this.slot.lock().expect("sync host slot poisoned");+ match slot.response.take() {+ Some(response) => Poll::Ready((response, slot.response_bytes.take())),None => Poll::Pending,}}@@ -117,15 +126,17 @@ impl SyncSession {}}- /// Hand the host's response to the last command back.- pub fn resume(&self, response_json: &str) -> VaultResult<()> {+ /// Hand the host's response to the last command back, with raw bytes for a `dbExport` (the SQLite file).+ pub fn resume(&self, response_json: &str, bytes: Option<Vec<u8>>) -> VaultResult<()> {let response: Value = if response_json.trim().is_empty() {Value::Object(Default::default())} else {serde_json::from_str(response_json).map_err(|e| VaultError::General(format!("Invalid host response: {}", e)))?};let inner = self.inner.lock().map_err(|_| VaultError::General("sync session poisoned".to_string()))?;- inner.slot.lock().map_err(|_| VaultError::General("sync host slot poisoned".to_string()))?.response = Some(response);+ let mut slot = inner.slot.lock().map_err(|_| VaultError::General("sync host slot poisoned".to_string()))?;+ slot.response = Some(response);+ slot.response_bytes = bytes;Ok(())}}diff --git a/core/rust/src/vault_sync/state.rs b/core/rust/src/vault_sync/state.rsindex f9a93707b..3eaaad293 100644--- a/core/rust/src/vault_sync/state.rs+++ b/core/rust/src/vault_sync/state.rs@@ -85,7 +85,7 @@ impl Ctx {if let Some(schema) = &self.schema {return Ok(schema.clone());}- db::open_staging(&self.host, None).await?;+ db::open_staging(&self.host).await?;let schema = SchemaInfo { columns: db::schema_columns(&self.host, Db::Staging).await?, migration_id: db::latest_migration_id(&self.host, Db::Staging).await? };self.schema = Some(schema.clone());Ok(schema)diff --git a/core/rust/src/vault_sync/tests/mod.rs b/core/rust/src/vault_sync/tests/mod.rsindex 440ce9517..fdffe90dc 100644--- a/core/rust/src/vault_sync/tests/mod.rs+++ b/core/rust/src/vault_sync/tests/mod.rs@@ -781,7 +781,7 @@ fn session_reports_when_a_response_is_missing() {let first: Value = serde_json::from_str(&session.next_command().unwrap()).unwrap();assert_eq!(first["kind"], "log", "the engine announces the sync before touching the host");assert!(session.next_command().is_err(), "the session refuses to advance without a response");- session.resume("{}").unwrap();+ session.resume("{}", None).unwrap();let second: Value = serde_json::from_str(&session.next_command().unwrap()).unwrap();assert_eq!(second["kind"], "http");assert_eq!(second["path"], "Status");diff --git a/core/rust/src/vault_sync/tests/test_host.rs b/core/rust/src/vault_sync/tests/test_host.rsindex 9d9a8d058..6a7b45c98 100644--- a/core/rust/src/vault_sync/tests/test_host.rs+++ b/core/rust/src/vault_sync/tests/test_host.rs@@ -6,7 +6,7 @@ use rusqlite::{Connection, MAIN_DB};use serde_json::{json, Map, Value};use crate::crypto;-use crate::common::encoding::{base64_decode, base64_encode};+use crate::common::encoding::base64_decode;use crate::sqlite_host::{self, SqlStatement};use crate::vault_sync::types::{Command, Db};use crate::vault_sync::session::SyncSession;@@ -123,11 +123,8 @@ impl TestHost {self.state.remove(&key);json!({})}- Command::DbOpen { bytes, .. } => {- let conn = match bytes {- None => open_schema_db(&self.schema_sql),- Some(b64) => open_from_bytes(&base64_decode(&b64).unwrap()),- };+ Command::DbOpen { .. } => {+ let conn = open_schema_db(&self.schema_sql);conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();self.staging = Some(conn);json!({})@@ -144,7 +141,12 @@ impl TestHost {Ok(()) => json!({}),Err(error) => json!({ "error": error }),},- Command::DbExport { db } => json!({ "bytes": base64_encode(&self.db(db).serialize(MAIN_DB).unwrap()) }),+ Command::DbExport { db } => {+ // The SQLite file travels as raw bytes, outside the JSON.+ let bytes = self.db(db).serialize(MAIN_DB).unwrap().to_vec();+ session.resume("{}", Some(bytes)).unwrap();+ continue;+ }Command::VaultStore { encrypted_blob, mark_dirty, encryption_key, expected_mutation_seq, revision } => {self.store_calls.push(Command::VaultStore { encrypted_blob: String::new(), mark_dirty, encryption_key: encryption_key.clone(), expected_mutation_seq, revision });if let Some(key) = encryption_key {@@ -180,7 +182,7 @@ impl TestHost {json!({})}};- session.resume(&response.to_string()).unwrap();+ session.resume(&response.to_string(), None).unwrap();}}diff --git a/core/rust/src/vault_sync/types.rs b/core/rust/src/vault_sync/types.rsindex 7d5756ece..562e60086 100644--- a/core/rust/src/vault_sync/types.rs+++ b/core/rust/src/vault_sync/types.rs@@ -322,18 +322,14 @@ pub enum Command {StateSet { key: String, value: Value },/// Delete an engine-owned persisted value; response [`Ack`].StateRemove { key: String },- /// Open the staging database, response [`Ack`]: from the given SQLite bytes (base64), or fresh with the- /// current client schema applied when `bytes` is null.- DbOpen {- db: Db,- #[serde(skip_serializing_if = "Option::is_none")]- bytes: Option<String>,- },+ /// Open the staging database fresh with the current client schema applied; response [`Ack`].+ DbOpen { db: Db },/// Run a SELECT; response [`DbRows`].DbQuery { db: Db, sql: String, params: Vec<Value> },/// Run statements inside one transaction; response [`Ack`]. A `{ "__b64": ... }` parameter binds a BLOB.DbExec { db: Db, statements: Vec<SqlStatement> },- /// Serialize a database; response [`DbBytes`].+ /// Serialize a database; response [`Ack`], with the SQLite file handed back as raw bytes through+ /// `SyncSession::resume` (never base64 inside the JSON).DbExport { db: Db },/// Persist the at-rest vault blob; response [`StoreOutcome`].VaultStore {@@ -450,12 +446,6 @@ pub struct DbRows {pub rows: Vec<Map<String, Value>>,}-/// Response of `dbExport`: the SQLite file, base64.-#[derive(Debug, Deserialize)]-pub struct DbBytes {- pub bytes: String,-}-/// Response of `vaultStore`: whether the store went through, and the host's mutation sequence after it.#[derive(Debug, Deserialize)]#[serde(rename_all = "camelCase")]