Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Make sync engine pass database exports as raw bytes where possible to save on JSON overhead

Leendert de Borst052c9eb9546f
diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.kt
index 13ee81aba..db6b6de2c 100644
--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.kt
+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.kt
@@ -1,6 +1,5 @@
package net.aliasvault.app.vaultstore
-import android.util.Base64
import android.util.Log
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
@@ -67,10 +66,10 @@ class VaultSyncEngine(
return result
}
val startNanos = System.nanoTime()
- val response = handle(kind, command)
- log.recordCommand(kind, command, response, System.nanoTime() - startNanos)
+ val (response, bytes) = respond(kind, command)
+ log.recordCommand(kind, command, response, bytes, System.nanoTime() - startNanos)
val responseJson = log.json { response.toString() }
- log.engine { session.resume(responseJson) }
+ log.engine { session.resume(responseJson, bytes) }
}
} finally {
closeStaging()
@@ -107,6 +106,21 @@ class VaultSyncEngine(
// region Commands
+ /**
+ * The JSON response to one command.
+ */
+ @Suppress("TooGenericExceptionCaught")
+ private suspend fun respond(kind: String, command: JSONObject): Pair<JSONObject, ByteArray?> {
+ if (kind != "dbExport") {
+ return Pair(handle(kind, command), null)
+ }
+ return try {
+ Pair(JSONObject(), exportDatabase(command.optString("db")))
+ } catch (e: Exception) {
+ Pair(errorResponse(kind, e), null)
+ }
+ }
+
@Suppress("TooGenericExceptionCaught")
private suspend fun handle(kind: String, command: JSONObject): JSONObject {
return try {
@@ -122,7 +136,7 @@ class VaultSyncEngine(
JSONObject()
}
"dbOpen" -> {
- openStaging(command.optString("bytes", null).takeIf { command.has("bytes") && !command.isNull("bytes") })
+ openStaging()
JSONObject()
}
"dbQuery" -> {
@@ -137,14 +151,6 @@ class VaultSyncEngine(
}
JSONObject()
}
- "dbExport" -> {
- val bytes = when (val db = command.optString("db")) {
- "local" -> vaultStore.database.export()
- "staging" -> stagingDatabase().export()
- else -> error("Unknown database $db")
- }
- JSONObject().put("bytes", Base64.encodeToString(bytes, Base64.NO_WRAP))
- }
"vaultStore" -> storeVault(command)
"vaultLoad" -> JSONObject().put("encryptedBlob", if (vaultStore.hasEncryptedDatabase()) vaultStore.getEncryptedDatabase() else JSONObject.NULL)
"markClean" -> {
@@ -158,11 +164,27 @@ class VaultSyncEngine(
else -> JSONObject().put("error", "Unknown engine command $kind")
}
} catch (e: Exception) {
- Log.w(TAG, "Engine command $kind failed", e)
- JSONObject().put("error", e.message ?: e.toString())
+ errorResponse(kind, e)
}
}
+ /**
+ * Serialize a database to SQLite bytes.
+ */
+ private fun exportDatabase(name: String): ByteArray = when (name) {
+ "local" -> vaultStore.database.export()
+ "staging" -> stagingDatabase().export()
+ else -> error("Unknown database $name")
+ }
+
+ /**
+ * The `{ "error": ... }` response for a failed command.
+ */
+ private fun errorResponse(kind: String, e: Exception): JSONObject {
+ Log.w(TAG, "Engine command $kind failed", e)
+ return JSONObject().put("error", e.message ?: e.toString())
+ }
+
@Suppress("TooGenericExceptionCaught")
private suspend fun http(command: JSONObject): JSONObject = withContext(Dispatchers.IO) {
val body = if (command.has("body") && !command.isNull("body")) command.getString("body") else null
@@ -274,13 +296,9 @@ class VaultSyncEngine(
/**
* Open the staging database in memory.
*/
- private fun openStaging(bytesBase64: String?) {
+ private fun openStaging() {
closeStaging()
- staging = if (bytesBase64 != null) {
- SqliteMemoryDatabase.fromBytes(Base64.decode(bytesBase64, Base64.NO_WRAP))
- } else {
- SqliteMemoryDatabase.withSchema(VaultSql.completeSchema)
- }.also {
+ staging = SqliteMemoryDatabase.withSchema(VaultSql.completeSchema).also {
// The schema script ends by turning foreign keys on; the engine inserts rows in codec order, not FK order.
it.executeBatch("PRAGMA foreign_keys = OFF")
}
diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncRunLog.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncRunLog.kt
index 9efa75926..97ab005d1 100644
--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncRunLog.kt
+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncRunLog.kt
@@ -35,8 +35,12 @@ class VaultSyncRunLog(private val operation: String) {
/**
* The size of a text payload (JSON or base64) for the logs, by character count.
*/
- private fun formatSize(value: Any?): String {
- val size = (value as? String)?.length ?: 0
+ private fun formatSize(value: Any?): String = formatByteCount((value as? String)?.length ?: 0)
+
+ /**
+ * A byte count for the logs.
+ */
+ private fun formatByteCount(size: Int): String {
return when {
size < BYTES_PER_KB -> "$size B"
size < BYTES_PER_KB * BYTES_PER_KB -> String.format(Locale.US, "%.1f KB", size / BYTES_PER_KB)
@@ -107,10 +111,10 @@ class VaultSyncRunLog(private val operation: String) {
/**
* Count one handled command, and log it on its own when it is a transfer or slow.
*/
- fun recordCommand(kind: String, command: JSONObject, response: JSONObject, nanos: Long) {
+ fun recordCommand(kind: String, command: JSONObject, response: JSONObject, responseBytes: ByteArray?, nanos: Long) {
commandNanos[kind] = (commandNanos[kind] ?: 0L) + nanos
commandCounts[kind] = (commandCounts[kind] ?: 0) + 1
- describe(kind, command, response, nanos / NANOS_PER_MS)?.let { note(it) }
+ describe(kind, command, response, responseBytes, nanos / NANOS_PER_MS)?.let { note(it) }
}
/**
@@ -135,7 +139,7 @@ class VaultSyncRunLog(private val operation: String) {
/**
* One line for a command relevant for performance analysis, or null for unrelated calls.
*/
- private fun describe(kind: String, command: JSONObject, response: JSONObject, ms: Long): String? {
+ private fun describe(kind: String, command: JSONObject, response: JSONObject, responseBytes: ByteArray?, ms: Long): String? {
val took = "$kind took ${ms}ms"
if (response.has("error")) {
return "$took: failed (${response.optString("error")})"
@@ -154,8 +158,7 @@ class VaultSyncRunLog(private val operation: String) {
}
"vaultStore" -> "$took: ${formatSize(command.opt("encryptedBlob"))}"
"vaultLoad" -> "$took: ${formatSize(response.opt("encryptedBlob"))}"
- "dbOpen" -> "$took: ${if (command.isNull("bytes")) "fresh schema" else formatSize(command.opt("bytes"))}"
- "dbExport" -> "$took: $db ${formatSize(response.opt("bytes"))}"
+ "dbExport" -> "$took: $db ${formatByteCount(responseBytes?.size ?: 0)}"
"dbExec" -> if (slow) "$took: ${command.optJSONArray("statements")?.length() ?: 0} statement(s) on $db" else null
"dbQuery" -> if (slow) "$took: ${response.optJSONArray("rows")?.length() ?: 0} row(s) from $db" else null
else -> if (slow) took else null
diff --git a/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swift b/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swift
index 5f28accb5..adce51901 100644
--- a/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swift
+++ b/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swift
@@ -63,10 +63,10 @@ public final class VaultSyncEngine {
return result
}
let commandStartedAt = Date()
- let response = await handle(kind: kind, command: command)
- log.recordCommand(kind, command: command, response: response, since: commandStartedAt)
+ let (response, bytes) = await respond(kind: kind, command: command)
+ log.recordCommand(kind, command: command, response: response, responseBytes: bytes, since: commandStartedAt)
let responseJson = try log.json { try Self.serializeJson(response) }
- try log.engine { try session.resume(responseJson: responseJson) }
+ try log.engine { try session.resume(responseJson: responseJson, bytes: bytes) }
}
}
@@ -103,6 +103,18 @@ public final class VaultSyncEngine {
// MARK: - Commands
+ /// The JSON response to one command.
+ private func respond(kind: String, command: [String: Any]) async -> (json: [String: Any], bytes: Data?) {
+ guard kind == "dbExport" else {
+ return (await handle(kind: kind, command: command), nil)
+ }
+ do {
+ return ([:], try exportDatabase(named: command["db"] as? String ?? ""))
+ } catch {
+ return (Self.errorResponse(error), nil)
+ }
+ }
+
private func handle(kind: String, command: [String: Any]) async -> [String: Any] {
do {
switch kind {
@@ -117,7 +129,7 @@ public final class VaultSyncEngine {
setState(nil, forKey: command["key"] as? String ?? "")
return [:]
case "dbOpen":
- try openStaging(bytesBase64: command["bytes"] as? String)
+ try openStaging()
return [:]
case "dbQuery":
let db = try database(named: command["db"] as? String ?? "")
@@ -130,17 +142,6 @@ public final class VaultSyncEngine {
localMutated = true
}
return [:]
- case "dbExport":
- let bytes: Data
- switch command["db"] as? String ?? "" {
- case "local":
- bytes = try vaultStore.exportDatabase()
- case "staging":
- bytes = try database(named: "staging").export()
- case let name:
- throw AppError.unknownError(message: "Unknown database \(name)")
- }
- return ["bytes": bytes.base64EncodedString()]
case "vaultStore":
return try storeVault(command)
case "vaultLoad":
@@ -154,13 +155,31 @@ public final class VaultSyncEngine {
default:
return ["error": "Unknown engine command \(kind)"]
}
- } catch let error as AppError {
- return ["error": error.message]
} catch {
- return ["error": "\(error)"]
+ return Self.errorResponse(error)
+ }
+ }
+
+ /// Serialize a database to SQLite bytes.
+ private func exportDatabase(named name: String) throws -> Data {
+ switch name {
+ case "local":
+ return try vaultStore.exportDatabase()
+ case "staging":
+ return try database(named: "staging").export()
+ default:
+ throw AppError.unknownError(message: "Unknown database \(name)")
}
}
+ /// The `{ "error": ... }` response for a failed command.
+ private static func errorResponse(_ error: Error) -> [String: Any] {
+ if let appError = error as? AppError {
+ return ["error": appError.message]
+ }
+ return ["error": "\(error)"]
+ }
+
private func http(_ command: [String: Any]) async -> [String: Any] {
let method = command["method"] as? String ?? "GET"
let path = command["path"] as? String ?? ""
@@ -279,15 +298,10 @@ public final class VaultSyncEngine {
}
}
- /// Open the staging database in memory: from SQLite bytes, or fresh with the current client schema.
- private func openStaging(bytesBase64: String?) throws {
+ /// Open a fresh staging database in memory with the current client schema.
+ private func openStaging() throws {
closeStaging()
- let opened: SqliteMemoryDatabase
- if let bytesBase64 = bytesBase64, let bytes = Data(base64Encoded: bytesBase64) {
- opened = try SqliteMemoryDatabase.fromBytes(bytes: bytes)
- } else {
- opened = try SqliteMemoryDatabase.withSchema(schemaSql: VaultSql.completeSchema)
- }
+ let opened = try SqliteMemoryDatabase.withSchema(schemaSql: VaultSql.completeSchema)
// The schema script ends by turning foreign keys on; the engine inserts rows in codec order, not FK order.
try opened.executeBatch(sql: "PRAGMA foreign_keys = OFF")
staging = opened
diff --git a/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncRunLog.swift b/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncRunLog.swift
index 159a7b261..cb978a229 100644
--- a/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncRunLog.swift
+++ b/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncRunLog.swift
@@ -21,7 +21,11 @@ internal final class VaultSyncRunLog {
/// The size of a text payload (JSON or base64) for the logs, by UTF-8 byte count.
private static func formatSize(_ value: Any?) -> String {
- let size = (value as? String)?.utf8.count ?? 0
+ return formatByteCount((value as? String)?.utf8.count ?? 0)
+ }
+
+ /// A byte count for the logs.
+ private static func formatByteCount(_ size: Int) -> String {
if size < 1024 {
return "\(size) B"
}
@@ -78,11 +82,11 @@ internal final class VaultSyncRunLog {
}
/// Count one handled command, and log it on its own when it is a transfer or slow.
- func recordCommand(_ kind: String, command: [String: Any], response: [String: Any], since commandStartedAt: Date) {
+ func recordCommand(_ kind: String, command: [String: Any], response: [String: Any], responseBytes: Data?, since commandStartedAt: Date) {
let durationMs = Date().timeIntervalSince(commandStartedAt) * 1000
let timing = commandTimings[kind] ?? (count: 0, totalMs: 0)
commandTimings[kind] = (count: timing.count + 1, totalMs: timing.totalMs + durationMs)
- if let line = describe(kind: kind, command: command, response: response, durationMs: durationMs) {
+ if let line = describe(kind: kind, command: command, response: response, responseBytes: responseBytes, durationMs: durationMs) {
note(line)
}
}
@@ -105,7 +109,7 @@ internal final class VaultSyncRunLog {
}
/// One line for a command relevant for performance analysis, or nil for unrelated calls.
- private func describe(kind: String, command: [String: Any], response: [String: Any], durationMs: Double) -> String? {
+ private func describe(kind: String, command: [String: Any], response: [String: Any], responseBytes: Data?, durationMs: Double) -> String? {
let took = "\(kind) took \(Int(durationMs))ms"
if let error = response["error"] as? String {
return "\(took): failed (\(error))"
@@ -123,10 +127,8 @@ internal final class VaultSyncRunLog {
return "\(took): \(Self.formatSize(command["encryptedBlob"]))"
case "vaultLoad":
return "\(took): \(Self.formatSize(response["encryptedBlob"]))"
- case "dbOpen":
- return "\(took): \(command["bytes"] is String ? Self.formatSize(command["bytes"]) : "fresh schema")"
case "dbExport":
- return "\(took): \(database) \(Self.formatSize(response["bytes"]))"
+ return "\(took): \(database) \(Self.formatByteCount(responseBytes?.count ?? 0))"
case "dbExec":
return slow ? "\(took): \((command["statements"] as? [Any])?.count ?? 0) statement(s) on \(database)" : nil
case "dbQuery":
diff --git a/core/client/src/rust/RustCoreBinding.ts b/core/client/src/rust/RustCoreBinding.ts
index 6b10c1981..0134f0a58 100644
--- a/core/client/src/rust/RustCoreBinding.ts
+++ b/core/client/src/rust/RustCoreBinding.ts
@@ -7,8 +7,8 @@ export interface IVaultSyncSession {
/** The next command for the host, as JSON. */
nextCommand(): Promise<string>;
- /** Hand the host's response to the last command back, as JSON. */
- resume(responseJson: string): Promise<void>;
+ /** Hand the host's response to the last command back, as JSON, with raw bytes for a `dbExport`. */
+ resume(responseJson: string, bytes: Uint8Array | null): Promise<void>;
/** Release the session. */
free(): void;
diff --git a/core/client/src/rust/WasmRustCore.ts b/core/client/src/rust/WasmRustCore.ts
index 3b7960cb8..fd45fc668 100644
--- a/core/client/src/rust/WasmRustCore.ts
+++ b/core/client/src/rust/WasmRustCore.ts
@@ -98,7 +98,7 @@ export function createWasmRustCore(loadWasm: WasmLoader): IRustCore {
const session = new core.VaultSyncSession(requestJson);
return {
nextCommand: async (): Promise<string> => session.nextCommand(),
- resume: async (responseJson: string): Promise<void> => session.resume(responseJson),
+ resume: async (responseJson: string, bytes: Uint8Array | null): Promise<void> => session.resume(responseJson, bytes),
free: (): void => session.free(),
};
}),
diff --git a/core/client/src/sync/VaultSyncEngine.ts b/core/client/src/sync/VaultSyncEngine.ts
index bd87639da..5936d621e 100644
--- a/core/client/src/sync/VaultSyncEngine.ts
+++ b/core/client/src/sync/VaultSyncEngine.ts
@@ -175,7 +175,7 @@ type EngineCommand =
| { kind: 'stateGet'; key: string }
| { kind: 'stateSet'; key: string; value: JsonValue }
| { kind: 'stateRemove'; key: string }
- | { kind: 'dbOpen'; db: string; bytes?: string | null }
+ | { kind: 'dbOpen'; db: string }
| { kind: 'dbQuery'; db: string; sql: string; params: JsonValue[] }
| { kind: 'dbExec'; db: string; statements: EngineSqlStatement[] }
| { kind: 'dbExport'; db: string }
@@ -185,6 +185,9 @@ type EngineCommand =
| { kind: 'log'; level: 'log' | 'warn' | 'phase'; message: string }
| { kind: 'done'; result: JsonValue };
+/** The host's response to one command: the JSON, plus the SQLite file for a `dbExport`. */
+type HostResponse = { json: JsonValue; bytes: Uint8Array | null };
+
/** The database names a command may address. */
const DB_LOCAL = 'local';
const DB_STAGING = 'staging';
@@ -302,15 +305,16 @@ class EngineRun {
public constructor(private readonly host: IVaultSyncEngineHost, private readonly webApi: WebApiService) {}
/**
- * Carry out one command.
+ * Carry out one command and return the host's response.
* @param command - the command
*/
- public async handle(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<JsonValue> {
+ public async handle(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<HostResponse> {
const started = now();
try {
- return await this.dispatch(command);
+ const response = await this.dispatch(command);
+ return response instanceof Uint8Array ? { json: {}, bytes: response } : { json: response, bytes: null };
} catch (error) {
- return { error: error instanceof Error ? error.message : String(error) };
+ return { json: { error: error instanceof Error ? error.message : String(error) }, bytes: null };
} finally {
const timing = this.timings.get(command.kind) ?? { count: 0, ms: 0 };
timing.count++;
@@ -346,10 +350,10 @@ class EngineRun {
}
/**
- * The command handlers.
+ * The command handlers. Raw bytes (a `dbExport`) go back to the engine outside the JSON.
* @param command - the command
*/
- private async dispatch(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<JsonValue> {
+ private async dispatch(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<JsonValue | Uint8Array> {
switch (command.kind) {
case 'http':
return this.http(command);
@@ -362,7 +366,7 @@ class EngineRun {
await getPlatform().storage.remove(stateKey(command.key));
return {};
case 'dbOpen':
- await this.openStaging(command.bytes ?? null);
+ await this.openStaging();
return {};
case 'dbQuery': {
const rows = (await this.database(command.db)).query(command.sql, command.params.map(toBindValue));
@@ -375,7 +379,7 @@ class EngineRun {
}
return {};
case 'dbExport':
- return { bytes: bytesToBase64((await this.database(command.db)).export()) };
+ return (await this.database(command.db)).export();
case 'vaultStore':
return this.storeVault(command);
case 'vaultLoad':
@@ -444,13 +448,12 @@ class EngineRun {
}
/**
- * Open the staging database.
- * @param bytesBase64 - the SQLite file bytes, or null for a fresh database
+ * Open a fresh staging database.
*/
- private async openStaging(bytesBase64: string | null): Promise<void> {
+ private async openStaging(): Promise<void> {
this.staging?.close();
this.staging = null;
- const db = bytesBase64 ? await getPlatform().sqlite.open(base64ToBytes(bytesBase64)) : await openFreshSchemaDatabase();
+ const db = await openFreshSchemaDatabase();
try {
db.exec('PRAGMA foreign_keys = OFF');
} catch (error) {
@@ -496,7 +499,8 @@ export async function runVaultSyncEngine<T extends VaultSyncEngineResultBase>(ho
devLog(run.summarize(request.operation, now() - started));
return command.result as T;
}
- await session.resume(JSON.stringify(await run.handle(command)));
+ const response = await run.handle(command);
+ await session.resume(JSON.stringify(response.json), response.bytes);
}
} finally {
run.close();
diff --git a/core/rust/src/bindings/uniffi_api.rs b/core/rust/src/bindings/uniffi_api.rs
index 8f1c412eb..6620926b0 100644
--- a/core/rust/src/bindings/uniffi_api.rs
+++ b/core/rust/src/bindings/uniffi_api.rs
@@ -291,9 +291,9 @@ impl VaultSyncSession {
self.inner.next_command()
}
- /// Hand the host's response to the last command back, as JSON.
- pub fn resume(&self, response_json: String) -> Result<(), VaultError> {
- self.inner.resume(&response_json)
+ /// Hand the host's response to the last command back, as JSON, with raw bytes for a `dbExport`.
+ pub fn resume(&self, response_json: String, bytes: Option<Vec<u8>>) -> Result<(), VaultError> {
+ self.inner.resume(&response_json, bytes)
}
}
diff --git a/core/rust/src/bindings/wasm.rs b/core/rust/src/bindings/wasm.rs
index f1f543eca..822a1f1e2 100644
--- a/core/rust/src/bindings/wasm.rs
+++ b/core/rust/src/bindings/wasm.rs
@@ -292,9 +292,9 @@ impl VaultSyncSessionJs {
self.inner.next_command().map_err(js_err)
}
- /// Hand the host's response to the last command back, as JSON.
- pub fn resume(&self, response_json: &str) -> Result<(), JsValue> {
- self.inner.resume(response_json).map_err(js_err)
+ /// Hand the host's response to the last command back, as JSON, with raw bytes for a `dbExport`.
+ pub fn resume(&self, response_json: &str, bytes: Option<Vec<u8>>) -> Result<(), JsValue> {
+ self.inner.resume(response_json, bytes).map_err(js_err)
}
}
diff --git a/core/rust/src/vault_sync/db.rs b/core/rust/src/vault_sync/db.rs
index 87c8b7fd1..b5427ff16 100644
--- a/core/rust/src/vault_sync/db.rs
+++ b/core/rust/src/vault_sync/db.rs
@@ -6,9 +6,9 @@ use serde_json::{json, Map, Value};
use super::errors::{SyncError, SyncResult};
use super::session::Host;
-use super::types::{Ack, Command, Db, DbBytes, DbRows, LogLevel};
+use super::types::{Ack, Command, Db, DbRows, LogLevel};
use super::legacy;
-use crate::common::encoding::{base64_decode, base64_encode, uuid_from_bytes};
+use crate::common::encoding::uuid_from_bytes;
use crate::sqlite_host::SqlStatement;
use crate::common::timestamp::{now_iso_utc, now_vault_datetime};
use crate::vault_codec::row::{blob_ref_of, inline_bytes};
@@ -35,16 +35,16 @@ pub(crate) async fn exec(host: &Host, db: Db, statements: Vec<SqlStatement>) ->
Ok(())
}
-/// Open the staging database: fresh with the current schema, or from SQLite bytes.
-pub(crate) async fn open_staging(host: &Host, bytes: Option<&[u8]>) -> SyncResult<()> {
- host.call::<Ack>(Command::DbOpen { db: Db::Staging, bytes: bytes.map(base64_encode) }).await?;
+/// Open the staging database fresh with the current schema.
+pub(crate) async fn open_staging(host: &Host) -> SyncResult<()> {
+ host.call::<Ack>(Command::DbOpen { db: Db::Staging }).await?;
Ok(())
}
/// Serialize a database to SQLite bytes.
pub(crate) async fn export(host: &Host, db: Db) -> SyncResult<Vec<u8>> {
- let response: DbBytes = host.call(Command::DbExport { db }).await?;
- base64_decode(&response.bytes).map_err(|_| SyncError::Other("host returned invalid database bytes".to_string()))
+ let (_, bytes) = host.call_with_bytes::<Ack>(Command::DbExport { db }).await?;
+ bytes.ok_or_else(|| SyncError::Other("host returned no database bytes".to_string()))
}
/// A cell as text: strings as they are, null as empty, anything else in its JSON form.
diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rs
index b9a5dac7c..916d78445 100644
--- a/core/rust/src/vault_sync/pull.rs
+++ b/core/rust/src/vault_sync/pull.rs
@@ -353,7 +353,7 @@ pub(crate) async fn materialize_to_sqlite(ctx: &mut Ctx, manifests: &[Manifest],
}
// Use a fresh staging database for every materialize.
- db::open_staging(&ctx.host, None).await?;
+ db::open_staging(&ctx.host).await?;
db::insert_materialized(&ctx.host, &materialized, &schema.columns, blob_map).await?;
// Names are not part of a manifest: the ones the vault already shows are kept, the ones this run opened win.
diff --git a/core/rust/src/vault_sync/session.rs b/core/rust/src/vault_sync/session.rs
index 4b5dd7d26..2cc412027 100644
--- a/core/rust/src/vault_sync/session.rs
+++ b/core/rust/src/vault_sync/session.rs
@@ -18,6 +18,8 @@ use crate::common::error::{VaultError, VaultResult};
pub(crate) struct Slot {
pub pending: Option<Command>,
pub response: Option<Value>,
+ /// Raw bytes the host handed back with the response (the SQLite file of a `dbExport`).
+ pub response_bytes: Option<Vec<u8>>,
}
/// The engine's handle to the host.
@@ -33,12 +35,18 @@ impl Host {
/// Send a command and wait for the host's typed response. A `{ "error": ... }` response becomes an error.
pub async fn call<R: DeserializeOwned>(&self, command: Command) -> SyncResult<R> {
+ Ok(self.call_with_bytes(command).await?.0)
+ }
+
+ /// Like [`Host::call`], also returning the raw bytes the host attached to its response, if any.
+ pub async fn call_with_bytes<R: DeserializeOwned>(&self, command: Command) -> SyncResult<(R, Option<Vec<u8>>)> {
let kind = command.kind();
- let response = CommandFuture { slot: self.slot.clone(), command: Some(command), sent: false }.await;
+ let (response, bytes) = CommandFuture { slot: self.slot.clone(), command: Some(command), sent: false }.await;
if let Some(error) = response.get("error").and_then(Value::as_str) {
return Err(SyncError::Host { command: kind, message: error.to_string() });
}
- serde_json::from_value(response).map_err(|e| SyncError::Host { command: kind, message: format!("unexpected response shape: {}", e) })
+ let typed = serde_json::from_value(response).map_err(|e| SyncError::Host { command: kind, message: format!("unexpected response shape: {}", e) })?;
+ Ok((typed, bytes))
}
/// Emit a log line.
@@ -55,17 +63,18 @@ struct CommandFuture {
}
impl Future for CommandFuture {
- type Output = Value;
+ type Output = (Value, Option<Vec<u8>>);
- fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Value> {
+ fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = &mut *self;
if !this.sent {
this.slot.lock().expect("sync host slot poisoned").pending = this.command.take();
this.sent = true;
return Poll::Pending;
}
- match this.slot.lock().expect("sync host slot poisoned").response.take() {
- Some(response) => Poll::Ready(response),
+ let mut slot = this.slot.lock().expect("sync host slot poisoned");
+ match slot.response.take() {
+ Some(response) => Poll::Ready((response, slot.response_bytes.take())),
None => Poll::Pending,
}
}
@@ -117,15 +126,17 @@ impl SyncSession {
}
}
- /// Hand the host's response to the last command back.
- pub fn resume(&self, response_json: &str) -> VaultResult<()> {
+ /// Hand the host's response to the last command back, with raw bytes for a `dbExport` (the SQLite file).
+ pub fn resume(&self, response_json: &str, bytes: Option<Vec<u8>>) -> VaultResult<()> {
let response: Value = if response_json.trim().is_empty() {
Value::Object(Default::default())
} else {
serde_json::from_str(response_json).map_err(|e| VaultError::General(format!("Invalid host response: {}", e)))?
};
let inner = self.inner.lock().map_err(|_| VaultError::General("sync session poisoned".to_string()))?;
- inner.slot.lock().map_err(|_| VaultError::General("sync host slot poisoned".to_string()))?.response = Some(response);
+ let mut slot = inner.slot.lock().map_err(|_| VaultError::General("sync host slot poisoned".to_string()))?;
+ slot.response = Some(response);
+ slot.response_bytes = bytes;
Ok(())
}
}
diff --git a/core/rust/src/vault_sync/state.rs b/core/rust/src/vault_sync/state.rs
index f9a93707b..3eaaad293 100644
--- a/core/rust/src/vault_sync/state.rs
+++ b/core/rust/src/vault_sync/state.rs
@@ -85,7 +85,7 @@ impl Ctx {
if let Some(schema) = &self.schema {
return Ok(schema.clone());
}
- db::open_staging(&self.host, None).await?;
+ db::open_staging(&self.host).await?;
let schema = SchemaInfo { columns: db::schema_columns(&self.host, Db::Staging).await?, migration_id: db::latest_migration_id(&self.host, Db::Staging).await? };
self.schema = Some(schema.clone());
Ok(schema)
diff --git a/core/rust/src/vault_sync/tests/mod.rs b/core/rust/src/vault_sync/tests/mod.rs
index 440ce9517..fdffe90dc 100644
--- a/core/rust/src/vault_sync/tests/mod.rs
+++ b/core/rust/src/vault_sync/tests/mod.rs
@@ -781,7 +781,7 @@ fn session_reports_when_a_response_is_missing() {
let first: Value = serde_json::from_str(&session.next_command().unwrap()).unwrap();
assert_eq!(first["kind"], "log", "the engine announces the sync before touching the host");
assert!(session.next_command().is_err(), "the session refuses to advance without a response");
- session.resume("{}").unwrap();
+ session.resume("{}", None).unwrap();
let second: Value = serde_json::from_str(&session.next_command().unwrap()).unwrap();
assert_eq!(second["kind"], "http");
assert_eq!(second["path"], "Status");
diff --git a/core/rust/src/vault_sync/tests/test_host.rs b/core/rust/src/vault_sync/tests/test_host.rs
index 9d9a8d058..6a7b45c98 100644
--- a/core/rust/src/vault_sync/tests/test_host.rs
+++ b/core/rust/src/vault_sync/tests/test_host.rs
@@ -6,7 +6,7 @@ use rusqlite::{Connection, MAIN_DB};
use serde_json::{json, Map, Value};
use crate::crypto;
-use crate::common::encoding::{base64_decode, base64_encode};
+use crate::common::encoding::base64_decode;
use crate::sqlite_host::{self, SqlStatement};
use crate::vault_sync::types::{Command, Db};
use crate::vault_sync::session::SyncSession;
@@ -123,11 +123,8 @@ impl TestHost {
self.state.remove(&key);
json!({})
}
- Command::DbOpen { bytes, .. } => {
- let conn = match bytes {
- None => open_schema_db(&self.schema_sql),
- Some(b64) => open_from_bytes(&base64_decode(&b64).unwrap()),
- };
+ Command::DbOpen { .. } => {
+ let conn = open_schema_db(&self.schema_sql);
conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap();
self.staging = Some(conn);
json!({})
@@ -144,7 +141,12 @@ impl TestHost {
Ok(()) => json!({}),
Err(error) => json!({ "error": error }),
},
- Command::DbExport { db } => json!({ "bytes": base64_encode(&self.db(db).serialize(MAIN_DB).unwrap()) }),
+ Command::DbExport { db } => {
+ // The SQLite file travels as raw bytes, outside the JSON.
+ let bytes = self.db(db).serialize(MAIN_DB).unwrap().to_vec();
+ session.resume("{}", Some(bytes)).unwrap();
+ continue;
+ }
Command::VaultStore { encrypted_blob, mark_dirty, encryption_key, expected_mutation_seq, revision } => {
self.store_calls.push(Command::VaultStore { encrypted_blob: String::new(), mark_dirty, encryption_key: encryption_key.clone(), expected_mutation_seq, revision });
if let Some(key) = encryption_key {
@@ -180,7 +182,7 @@ impl TestHost {
json!({})
}
};
- session.resume(&response.to_string()).unwrap();
+ session.resume(&response.to_string(), None).unwrap();
}
}
diff --git a/core/rust/src/vault_sync/types.rs b/core/rust/src/vault_sync/types.rs
index 7d5756ece..562e60086 100644
--- a/core/rust/src/vault_sync/types.rs
+++ b/core/rust/src/vault_sync/types.rs
@@ -322,18 +322,14 @@ pub enum Command {
StateSet { key: String, value: Value },
/// Delete an engine-owned persisted value; response [`Ack`].
StateRemove { key: String },
- /// Open the staging database, response [`Ack`]: from the given SQLite bytes (base64), or fresh with the
- /// current client schema applied when `bytes` is null.
- DbOpen {
- db: Db,
- #[serde(skip_serializing_if = "Option::is_none")]
- bytes: Option<String>,
- },
+ /// Open the staging database fresh with the current client schema applied; response [`Ack`].
+ DbOpen { db: Db },
/// Run a SELECT; response [`DbRows`].
DbQuery { db: Db, sql: String, params: Vec<Value> },
/// Run statements inside one transaction; response [`Ack`]. A `{ "__b64": ... }` parameter binds a BLOB.
DbExec { db: Db, statements: Vec<SqlStatement> },
- /// Serialize a database; response [`DbBytes`].
+ /// Serialize a database; response [`Ack`], with the SQLite file handed back as raw bytes through
+ /// `SyncSession::resume` (never base64 inside the JSON).
DbExport { db: Db },
/// Persist the at-rest vault blob; response [`StoreOutcome`].
VaultStore {
@@ -450,12 +446,6 @@ pub struct DbRows {
pub rows: Vec<Map<String, Value>>,
}
-/// Response of `dbExport`: the SQLite file, base64.
-#[derive(Debug, Deserialize)]
-pub struct DbBytes {
- pub bytes: String,
-}
-
/// Response of `vaultStore`: whether the store went through, and the host's mutation sequence after it.
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]