Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Make vault download call be in bytes to avoid json and base64 overhead
Leendert de Borst47f8bdcca526
diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.ktindex 3fccddfe7..ef9a8e9ae 100644--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.kt+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/VaultSyncEngine.kt@@ -112,6 +112,9 @@ class VaultSyncEngine(*/@Suppress("TooGenericExceptionCaught")private suspend fun respond(kind: String, command: JSONObject): Pair<JSONObject, ByteArray?> {+ if (kind == "http") {+ return http(command)+ }if (kind != "dbExport") {return Pair(handle(kind, command), null)}@@ -126,7 +129,6 @@ class VaultSyncEngine(private suspend fun handle(kind: String, command: JSONObject): JSONObject {return try {when (kind) {- "http" -> http(command)"stateGet" -> JSONObject().put("value", state(command.optString("key")) ?: JSONObject.NULL)"stateSet" -> {setState(command.optString("key"), command.opt("value"))@@ -186,10 +188,14 @@ class VaultSyncEngine(return JSONObject().put("error", e.message ?: e.toString())}+ /**+ * An API request. On success a binary response returns its body as raw bytes instead of as `body` text in the JSON.+ */@Suppress("TooGenericExceptionCaught")- private suspend fun http(command: JSONObject): JSONObject = withContext(Dispatchers.IO) {+ private suspend fun http(command: JSONObject): Pair<JSONObject, ByteArray?> = withContext(Dispatchers.IO) {val body = if (command.has("body") && !command.isNull("body")) command.getString("body") else null- val headers = mutableMapOf("Accept" to "application/json")+ val binaryResponse = command.optBoolean("binaryResponse", false)+ val headers = mutableMapOf("Accept" to if (binaryResponse) "application/octet-stream" else "application/json")if (body != null) {headers["Content-Type"] = "application/json"}@@ -202,11 +208,16 @@ class VaultSyncEngine(requiresAuth = command.optBoolean("auth", true),largeTransfer = command.optBoolean("largeTransfer", false),)- JSONObject().put("status", response.statusCode).put("body", response.body)+ val bytes = response.bodyBytes+ if (binaryResponse && bytes != null) {+ Pair(JSONObject().put("status", response.statusCode), bytes)+ } else {+ Pair(JSONObject().put("status", response.statusCode).put("body", response.body), null)+ }} catch (e: SocketTimeoutException) {- JSONObject().put("status", 0).put("transportError", e.toString()).put("timedOut", true)+ Pair(JSONObject().put("status", 0).put("transportError", e.toString()).put("timedOut", true), null)} catch (e: Exception) {- JSONObject().put("status", 0).put("transportError", e.toString()).put("timedOut", false)+ Pair(JSONObject().put("status", 0).put("transportError", e.toString()).put("timedOut", false), null)}}diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/webapi/WebApiService.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/webapi/WebApiService.ktindex 4938aabe9..65e9ad79a 100644--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/webapi/WebApiService.kt+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/webapi/WebApiService.kt@@ -24,6 +24,8 @@ data class WebApiResponse(val body: String,/** The response headers as a map of key-value pairs. */val headers: Map<String, String>,+ /** The body as raw bytes, set on success when the request accepted `application/octet-stream`. */+ val bodyBytes: ByteArray? = null,)/**@@ -322,6 +324,7 @@ class WebApiService(private val context: Context) {val statusCode = connection.responseCode// Read response body+ var bodyBytes: ByteArray? = nullval responseBody = try {val acceptHeader = headers.entries.firstOrNull { it.key.equals("Accept", ignoreCase = true) }?.value ?: ""val isBinary = acceptHeader.contains("application/octet-stream", ignoreCase = true)@@ -331,6 +334,7 @@ class WebApiService(private val context: Context) {// Read binary data and encode as base64connection.inputStream.use { inputStream ->val bytes = inputStream.readBytes()+ bodyBytes = bytesandroid.util.Base64.encodeToString(bytes, android.util.Base64.NO_WRAP)}} else {@@ -361,6 +365,7 @@ class WebApiService(private val context: Context) {statusCode = statusCode,body = responseBody,headers = responseHeaders,+ bodyBytes = bodyBytes,)} catch (e: Exception) {Log.e(TAG, "Error executing request", e)diff --git a/apps/mobile-app/ios/VaultStoreKit/Models/WebApiResponse.swift b/apps/mobile-app/ios/VaultStoreKit/Models/WebApiResponse.swiftindex 5b4361959..345f6bcc3 100644--- a/apps/mobile-app/ios/VaultStoreKit/Models/WebApiResponse.swift+++ b/apps/mobile-app/ios/VaultStoreKit/Models/WebApiResponse.swift@@ -8,11 +8,14 @@ public struct WebApiResponse {public let body: String/// The headers of the responsepublic let headers: [String: String]+ /// The body as raw bytes, set on success when the request accepted `application/octet-stream`+ public let bodyData: Data?/// Initialize a new WebApiResponse- public init(statusCode: Int, body: String, headers: [String: String]) {+ public init(statusCode: Int, body: String, headers: [String: String], bodyData: Data? = nil) {self.statusCode = statusCodeself.body = bodyself.headers = headers+ self.bodyData = bodyData}}diff --git a/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swift b/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swiftindex e669ab408..8d86217f1 100644--- a/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swift+++ b/apps/mobile-app/ios/VaultStoreKit/Services/VaultSyncEngine.swift@@ -108,6 +108,9 @@ public final class VaultSyncEngine {/// The JSON response to one command.private func respond(kind: String, command: [String: Any]) async -> (json: [String: Any], bytes: Data?) {+ if kind == "http" {+ return await http(command)+ }guard kind == "dbExport" else {return (await handle(kind: kind, command: command), nil)}@@ -121,8 +124,6 @@ public final class VaultSyncEngine {private func handle(kind: String, command: [String: Any]) async -> [String: Any] {do {switch kind {- case "http":- return await http(command)case "stateGet":return ["value": state(forKey: command["key"] as? String ?? "") ?? NSNull()]case "stateSet":@@ -183,22 +184,27 @@ public final class VaultSyncEngine {return ["error": "\(error)"]}- private func http(_ command: [String: Any]) async -> [String: Any] {+ /// An API request. On success a binary response returns its body as raw bytes instead of as `body` text in the JSON.+ private func http(_ command: [String: Any]) async -> (json: [String: Any], bytes: Data?) {let method = command["method"] as? String ?? "GET"let path = command["path"] as? String ?? ""let body = command["body"] as? Stringlet auth = command["auth"] as? Bool ?? truelet largeTransfer = command["largeTransfer"] as? Bool ?? false- var headers: [String: String] = ["Accept": "application/json"]+ let binaryResponse = command["binaryResponse"] as? Bool ?? false+ var headers: [String: String] = ["Accept": binaryResponse ? "application/octet-stream" : "application/json"]if body != nil {headers["Content-Type"] = "application/json"}do {let response = try await webApiService.executeRequest(method: method, endpoint: path, body: body, headers: headers, requiresAuth: auth, largeTransfer: largeTransfer)- return ["status": response.statusCode, "body": response.body]+ if binaryResponse, let data = response.bodyData {+ return (["status": response.statusCode], data)+ }+ return (["status": response.statusCode, "body": response.body], nil)} catch {let timedOut = (error as? URLError)?.code == .timedOut- return ["status": 0, "transportError": error.localizedDescription, "timedOut": timedOut]+ return (["status": 0, "transportError": error.localizedDescription, "timedOut": timedOut], nil)}}diff --git a/apps/mobile-app/ios/VaultStoreKit/Services/WebApiService.swift b/apps/mobile-app/ios/VaultStoreKit/Services/WebApiService.swiftindex f6a7f0ee0..f8550b5df 100644--- a/apps/mobile-app/ios/VaultStoreKit/Services/WebApiService.swift+++ b/apps/mobile-app/ios/VaultStoreKit/Services/WebApiService.swift@@ -300,11 +300,11 @@ public class WebApiService {}let acceptHeader = headers.first { $0.key.caseInsensitiveCompare("Accept") == .orderedSame }?.value ?? ""- let isBinary = acceptHeader.lowercased().contains("application/octet-stream")+ let isBinary = acceptHeader.lowercased().contains("application/octet-stream") && httpResponse.statusCode >= 200 && httpResponse.statusCode < 300// Parse response bodylet responseBody: String- if isBinary && httpResponse.statusCode >= 200 && httpResponse.statusCode < 300 {+ if isBinary {// Encode binary data as base64responseBody = data.base64EncodedString()} else {@@ -315,7 +315,8 @@ public class WebApiService {return WebApiResponse(statusCode: httpResponse.statusCode,body: responseBody,- headers: responseHeaders+ headers: responseHeaders,+ bodyData: isBinary ? data : nil)}diff --git a/apps/server/AliasVault.Api/Controllers/V2/VaultController.cs b/apps/server/AliasVault.Api/Controllers/V2/VaultController.csindex aa0e5e62b..530226565 100644--- a/apps/server/AliasVault.Api/Controllers/V2/VaultController.cs+++ b/apps/server/AliasVault.Api/Controllers/V2/VaultController.cs@@ -71,9 +71,10 @@ public class VaultController(};/// <summary>- /// Atomic snapshot. Returns the latest encrypted manifest + metadata + blob refs + email routing.+ /// Atomic snapshot. Returns the latest encrypted manifest + metadata + blob refs + email routing as a binary frame:+ /// a <see cref="GetResponse"/> JSON header, then the manifest and bucket ciphertexts (see <see cref="BinaryFrame"/>)./// </summary>- /// <returns>Snapshot DTO.</returns>+ /// <returns>Snapshot frame.</returns>[HttpGet("")]public async Task<IActionResult> Get(){@@ -100,7 +101,7 @@ public class VaultController(.OrderByDescending(x => x.RevisionNumber).FirstOrDefaultAsync();- return Ok(new GetResponse+ var legacyResponse = new GetResponse{StorageFormat = StorageFormat.SqliteBlob,LegacyVaultBlob = legacy?.VaultBlob ?? string.Empty,@@ -108,7 +109,8 @@ public class VaultController(LegacyRevision = legacy?.RevisionNumber ?? 0,PersonalManifestId = legacy?.ManifestId,EmailRouting = emailRouting,- });+ };+ return File(BinaryFrame.Write(legacyResponse, []), BinaryFrame.ContentType);}var manifestIds = latestManifests.Select(m => m.ManifestId).ToList();@@ -118,17 +120,17 @@ public class VaultController({ManifestId = x.ManifestId,Category = x.Category,- Blob = Convert.ToBase64String(x.EncryptedData),+ Size = x.EncryptedData.Length,+ Data = x.EncryptedData,CiphertextHash = x.CiphertextHash,Revision = x.RevisionNumber,}).ToListAsync();- var currentRevisionByManifest = latestManifests.ToDictionary(m => m.ManifestId, m => m.RevisionNumber);- var refsByManifest = (await context.VaultBlobReferences- .Where(r => manifestIds.Contains(r.ManifestId))- .Join(context.VaultBlobObjects, r => new { r.ManifestId, Hash = r.BlobHash }, b => new { b.ManifestId, b.Hash }, (r, b) => new { r.ManifestId, r.RevisionNumber, b.Hash, b.Category, b.SizeBytes })++ var refsByManifest = (await AccessibleManifests(context, accessScope)+ .Join(context.VaultBlobReferences, m => new { m.ManifestId, m.RevisionNumber }, r => new { r.ManifestId, r.RevisionNumber }, (m, r) => r)+ .Join(context.VaultBlobObjects, r => new { r.ManifestId, Hash = r.BlobHash }, b => new { b.ManifestId, b.Hash }, (r, b) => new { r.ManifestId, b.Hash, b.Category, b.SizeBytes }).ToListAsync())- .Where(x => currentRevisionByManifest.TryGetValue(x.ManifestId, out var rev) && rev == x.RevisionNumber).GroupBy(x => x.ManifestId).ToDictionary(g => g.Key, g => g.Select(x => new BlobReference { Hash = x.Hash, Category = x.Category, SizeBytes = x.SizeBytes }).ToList());@@ -145,7 +147,8 @@ public class VaultController(return new Manifest{ManifestId = m.ManifestId,- Blob = m.ManifestBlob != null ? Convert.ToBase64String(m.ManifestBlob) : null,+ Size = m.ManifestBlob?.Length ?? 0,+ Data = m.ManifestBlob,CiphertextHash = m.ManifestCiphertextHash,Revision = m.RevisionNumber,BlobReferences = refsByManifest.TryGetValue(m.ManifestId, out var refs) ? refs : [],@@ -161,14 +164,16 @@ public class VaultController(};}).ToList();- return Ok(new GetResponse+ var response = new GetResponse{StorageFormat = StorageFormat.Manifest,Manifests = manifests,PersonalManifestId = latestManifests.First(m => m.OwnerGroupId == user.PersonalGroupId).ManifestId,Buckets = buckets,EmailRouting = emailRouting,- });+ };+ var parts = manifests.Where(m => m.Data != null).Select(m => m.Data!).Concat(buckets.Select(b => b.Data)).ToList();+ return File(BinaryFrame.Write(response, parts), BinaryFrame.ContentType);}/// <summary>@@ -176,7 +181,7 @@ public class VaultController(/// plus its blob references, without the rest of the snapshot./// </summary>/// <param name="manifestId">The stable identifier of the logical manifest to fetch.</param>- /// <returns>The manifest DTO, or 404 when the user has no such manifest-v1 manifest.</returns>+ /// <returns>A binary frame of the manifest DTO and its ciphertext, or 404 when the user has no such manifest-v1 manifest.</returns>[HttpGet("manifest/{manifestId:guid}")]public async Task<IActionResult> GetManifest(Guid manifestId){@@ -206,7 +211,8 @@ public class VaultController(var manifest = new Manifest{ManifestId = latest.ManifestId,- Blob = latest.ManifestBlob != null ? Convert.ToBase64String(latest.ManifestBlob) : null,+ Size = latest.ManifestBlob?.Length ?? 0,+ Data = latest.ManifestBlob,CiphertextHash = latest.ManifestCiphertextHash,Revision = latest.RevisionNumber,BlobReferences = blobRefs,@@ -235,7 +241,7 @@ public class VaultController(manifest.CanAdminister = await GroupHelper.IsGroupAdminAsync(context, latest.OwnerGroupId, user.Id);}- return Ok(manifest);+ return File(BinaryFrame.Write(manifest, manifest.Data != null ? [manifest.Data] : []), BinaryFrame.ContentType);}/// <summary>@@ -768,8 +774,8 @@ public class VaultController(}/// <summary>- /// Download a batch of one manifest's encrypted blobs by hash. TODO: Returns base64-encoded payloads in JSON- /// because for now we kept the codec language-agnostic. Look into switching to multipart binary in the future.+ /// Download a batch of one manifest's encrypted blobs by hash as raw bytes (application/octet-stream). The body is a+ /// 4-byte big-endian header length, a <see cref="BlobDownloadResponse"/> JSON header, then each blob's ciphertext in header order./// </summary>/// <param name="model">Hash list request.</param>/// <returns>The stored blobs among the requested hashes.</returns>@@ -790,23 +796,13 @@ public class VaultController(}var wanted = model.Hashes.Distinct().ToList();- if (wanted.Count == 0)- {- return Ok(new BlobDownloadResponse());- }-var rows = await context.VaultBlobObjects.Where(b => b.ManifestId == model.ManifestId && wanted.Contains(b.Hash))- .Select(b => new Blob- {- Hash = b.Hash,- Category = b.Category,- EncryptedDataBase64 = Convert.ToBase64String(b.EncryptedData),- EncryptedBlobKey = b.EncryptedBlobKey,- })+ .Select(b => new { b.Hash, b.Category, b.EncryptedBlobKey, b.EncryptedData }).ToListAsync();- return Ok(new BlobDownloadResponse { Blobs = rows });+ var header = new BlobDownloadResponse { Blobs = rows.Select(r => new BlobDownloadEntry { Hash = r.Hash, Category = r.Category, EncryptedBlobKey = r.EncryptedBlobKey, Size = r.EncryptedData.Length }).ToList() };+ return File(BinaryFrame.Write(header, rows.Select(r => r.EncryptedData).ToList()), BinaryFrame.ContentType);}/// <summary>diff --git a/apps/server/AliasVault.Api/Helpers/BinaryFrame.cs b/apps/server/AliasVault.Api/Helpers/BinaryFrame.csnew file mode 100644index 000000000..8fc8d0981--- /dev/null+++ b/apps/server/AliasVault.Api/Helpers/BinaryFrame.cs@@ -0,0 +1,45 @@+//-----------------------------------------------------------------------+// <copyright file="BinaryFrame.cs" company="aliasvault">+// Copyright (c) aliasvault. All rights reserved.+// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.+// </copyright>+//-----------------------------------------------------------------------++namespace AliasVault.Api.Helpers;++using System.Buffers.Binary;+using System.Text.Json;++/// <summary>+/// Writes the binary body of the v2 vault downloads: a 4-byte big-endian header length, the header as UTF-8 JSON, then+/// the raw ciphertexts back to back. The header carries each ciphertext's size, in the order the bytes follow.+/// </summary>+public static class BinaryFrame+{+ /// <summary>+ /// The media type of a binary frame.+ /// </summary>+ public const string ContentType = "application/octet-stream";++ /// <summary>+ /// Builds a frame from a header and the ciphertexts that follow it.+ /// </summary>+ /// <param name="header">The JSON header.</param>+ /// <param name="parts">The ciphertexts, in the order the header lists their sizes.</param>+ /// <returns>The frame bytes.</returns>+ public static byte[] Write(object header, IReadOnlyList<byte[]> parts)+ {+ var headerBytes = JsonSerializer.SerializeToUtf8Bytes(header, JsonSerializerOptions.Web);+ var body = new byte[4 + headerBytes.Length + parts.Sum(p => p.Length)];+ BinaryPrimitives.WriteUInt32BigEndian(body, (uint)headerBytes.Length);+ headerBytes.CopyTo(body, 4);+ var offset = 4 + headerBytes.Length;+ foreach (var part in parts)+ {+ part.CopyTo(body, offset);+ offset += part.Length;+ }++ return body;+ }+}diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobDownloadEntry.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobDownloadEntry.csnew file mode 100644index 000000000..4022e0dc1--- /dev/null+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobDownloadEntry.cs@@ -0,0 +1,26 @@+//-----------------------------------------------------------------------+// <copyright file="BlobDownloadEntry.cs" company="aliasvault">+// Copyright (c) aliasvault. All rights reserved.+// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.+// </copyright>+//-----------------------------------------------------------------------++namespace AliasVault.Shared.Models.WebApi.V2.Vault;++/// <summary>+/// One blob in the header of the binary blob download response.+/// </summary>+public class BlobDownloadEntry+{+ /// <summary>Gets or sets the per-manifest salted SHA-256 hex of the plaintext.</summary>+ public required string Hash { get; set; }++ /// <summary>Gets or sets the blob category ("favicon" or "attachment").</summary>+ public required string Category { get; set; }++ /// <summary>Gets or sets the blob's own key, encrypted with the manifest's VEK.</summary>+ public required string EncryptedBlobKey { get; set; }++ /// <summary>Gets or sets the length in bytes of the blob's ciphertext in the response body.</summary>+ public int Size { get; set; }+}diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobDownloadResponse.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobDownloadResponse.csindex 5bea07030..ddccfa011 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobDownloadResponse.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobDownloadResponse.cs@@ -8,10 +8,11 @@namespace AliasVault.Shared.Models.WebApi.V2.Vault;/// <summary>-/// Response for POST /v2/Vault/blobs/download.+/// JSON header of the binary POST /v2/Vault/blobs/download response. The body is a 4-byte big-endian header length,+/// this header as UTF-8 JSON, then each blob's ciphertext bytes in header order./// </summary>public class BlobDownloadResponse{- /// <summary>Gets or sets the requested blobs the server has stored.</summary>- public List<Blob> Blobs { get; set; } = [];+ /// <summary>Gets or sets the requested blobs the server has stored, in the order their bytes follow the header.</summary>+ public List<BlobDownloadEntry> Blobs { get; set; } = [];}diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Bucket.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Bucket.csindex b419f454c..110e6e14f 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Bucket.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Bucket.cs@@ -7,6 +7,8 @@namespace AliasVault.Shared.Models.WebApi.V2.Vault;+using System.Text.Json.Serialization;+/// <summary>/// A single data bucket as carried in list-based payloads./// </summary>@@ -18,8 +20,12 @@ public class Bucket/// <summary>Gets or sets the bucket kind discriminator.</summary>public required VaultDataBucketCategory Category { get; set; }- /// <summary>Gets or sets the encrypted bucket blob (base64 of AES-GCM ciphertext).</summary>- public required string Blob { get; set; }+ /// <summary>Gets or sets the length of the bucket ciphertext in the binary response body.</summary>+ public int Size { get; set; }++ /// <summary>Gets or sets the bucket ciphertext (AES-GCM). Sent after the JSON header, not inside it.</summary>+ [JsonIgnore]+ public byte[] Data { get; set; } = [];/// <summary>Gets or sets the SHA-256 (hex) of the ciphertext for client-side storage-integrity check.</summary>public string? CiphertextHash { get; set; }diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/GetResponse.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/GetResponse.csindex 43b668315..fe85b750a 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/GetResponse.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/GetResponse.cs@@ -8,7 +8,8 @@namespace AliasVault.Shared.Models.WebApi.V2.Vault;/// <summary>-/// Atomic snapshot returned by GET /v2/Vault.+/// Atomic snapshot returned by GET /v2/Vault, as the JSON header of a binary body: the manifest ciphertexts follow it+/// in <see cref="Manifests"/> order, then the bucket ciphertexts in <see cref="Buckets"/> order./// </summary>public class GetResponse{diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Manifest.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Manifest.csindex 04e719b21..b06b565f2 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Manifest.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Manifest.cs@@ -7,6 +7,8 @@namespace AliasVault.Shared.Models.WebApi.V2.Vault;+using System.Text.Json.Serialization;+/// <summary>/// A single vault manifest as carried in list-based payloads. A user's logical vault is assembled from one or more/// manifests: the one owned by their personal group plus any number owned by shared groups they belong to.@@ -18,8 +20,12 @@ public class Manifest/// <summary>Gets or sets the stable identifier of the logical manifest (constant across its revisions).</summary>public required Guid ManifestId { get; set; }- /// <summary>Gets or sets the encrypted manifest blob (base64 of AES-GCM ciphertext), null on empty vault.</summary>- public string? Blob { get; set; }+ /// <summary>Gets or sets the length of the manifest ciphertext in the binary response body, 0 on empty vault.</summary>+ public int Size { get; set; }++ /// <summary>Gets or sets the manifest ciphertext (AES-GCM), null on empty vault. Sent after the JSON header, not inside it.</summary>+ [JsonIgnore]+ public byte[]? Data { get; set; }/// <summary>Gets or sets the SHA-256 (hex) of the manifest ciphertext for client-side storage-integrity check.</summary>public string? CiphertextHash { get; set; }diff --git a/core/client/src/api/WebApiService.ts b/core/client/src/api/WebApiService.tsindex 00703a35d..1ef5d1fa1 100644--- a/core/client/src/api/WebApiService.ts+++ b/core/client/src/api/WebApiService.ts@@ -80,6 +80,8 @@ type TokenResponse = {export type EngineHttpResponse = {status: number;body: string;+ /** The body as raw bytes, set on success when the request asked for a binary response. */+ bytes?: Uint8Array;};/**@@ -211,9 +213,9 @@ export class WebApiService {/*** Run a request on behalf of the Rust sync engine.*/- public async engineRequest(method: string, path: string, body: string | undefined, requiresAuth: boolean, largeTransfer: boolean): Promise<EngineHttpResponse> {+ public async engineRequest(method: string, path: string, body: string | undefined, requiresAuth: boolean, largeTransfer: boolean, binaryResponse = false): Promise<EngineHttpResponse> {const url = await this.resolveUrl(path);- const headers = new Headers({ Accept: 'application/json' });+ const headers = new Headers({ Accept: binaryResponse ? 'application/octet-stream' : 'application/json' });if (body !== undefined) {headers.set('Content-Type', 'application/json');}@@ -237,6 +239,9 @@ export class WebApiService {throw new NetworkError('Token refresh failed due to network error');}}+ if (binaryResponse && response.ok) {+ return { status: response.status, body: '', bytes: new Uint8Array(await response.arrayBuffer()) };+ }return { status: response.status, body: await response.text() };} catch (error) {if (error instanceof ClientUpgradeRequiredError) {diff --git a/core/client/src/sync/VaultSyncEngine.ts b/core/client/src/sync/VaultSyncEngine.tsindex c63ed947a..be6882d4e 100644--- a/core/client/src/sync/VaultSyncEngine.ts+++ b/core/client/src/sync/VaultSyncEngine.ts@@ -174,7 +174,7 @@ type EngineSqlStatement = { sql: string; params?: JsonValue[] };/** The engine's commands; see the Rust `Command` enum. */type EngineCommand =- | { kind: 'http'; method: string; path: string; body?: string; auth: boolean; largeTransfer: boolean }+ | { kind: 'http'; method: string; path: string; body?: string; auth: boolean; largeTransfer: boolean; binaryResponse?: boolean }| { kind: 'stateGet'; key: string }| { kind: 'stateSet'; key: string; value: JsonValue }| { kind: 'stateRemove'; key: string }@@ -188,7 +188,7 @@ type EngineCommand =| { kind: 'log'; level: 'log' | 'warn' | 'phase'; message: string }| { kind: 'done'; result: JsonValue };-/** The host's response to one command: the JSON, plus the SQLite file for a `dbExport`. */+/** The host's response to one command: the JSON, plus the raw bytes of a `dbExport` or a binary `http` response. */type HostResponse = { json: JsonValue; bytes: Uint8Array | null };/** The database names a command may address. */@@ -314,6 +314,9 @@ class EngineRun {public async handle(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<HostResponse> {const started = now();try {+ if (command.kind === 'http') {+ return await this.http(command);+ }const response = await this.dispatch(command);return response instanceof Uint8Array ? { json: {}, bytes: response } : { json: response, bytes: null };} catch (error) {@@ -356,10 +359,8 @@ class EngineRun {* The command handlers. Raw bytes (a `dbExport`) go back to the engine outside the JSON.* @param command - the command*/- private async dispatch(command: Exclude<EngineCommand, { kind: 'done' }>): Promise<JsonValue | Uint8Array> {+ private async dispatch(command: Exclude<EngineCommand, { kind: 'done' | 'http' }>): Promise<JsonValue | Uint8Array> {switch (command.kind) {- case 'http':- return this.http(command);case 'stateGet':return { value: await getPlatform().storage.get<JsonValue>(stateKey(command.key)) };case 'stateSet':@@ -398,16 +399,16 @@ class EngineRun {}/**- * An API request.+ * An API request. On success a binary response returns its body as raw bytes instead of as `body` text in the JSON.* @param command - the command*/- private async http(command: Extract<EngineCommand, { kind: 'http' }>): Promise<JsonValue> {+ private async http(command: Extract<EngineCommand, { kind: 'http' }>): Promise<HostResponse> {try {- const response = await this.webApi.engineRequest(command.method, command.path, command.body, command.auth, command.largeTransfer);- return { status: response.status, body: response.body };+ const response = await this.webApi.engineRequest(command.method, command.path, command.body, command.auth, command.largeTransfer, command.binaryResponse);+ return { json: { status: response.status, body: response.body }, bytes: response.bytes ?? null };} catch (error) {if (error instanceof NetworkError) {- return { status: 0, transportError: error.message, timedOut: error instanceof RequestTimeoutError };+ return { json: { status: 0, transportError: error.message, timedOut: error instanceof RequestTimeoutError }, bytes: null };}throw error;}diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rsindex 8c34a3294..e489f8c31 100644--- a/core/rust/src/vault_codec/mod.rs+++ b/core/rust/src/vault_codec/mod.rs@@ -125,10 +125,15 @@ pub fn unpack_versioned_payload(plain_bytes: &[u8]) -> VaultResult<UnpackedPayloOk(UnpackedPayload::Readable(serde_json::to_string(payload)?))}+/// SHA-256 (lowercase hex) of a raw ciphertext: storage-layer integrity.+pub fn compute_ciphertext_hash_bytes(ciphertext: &[u8]) -> String {+ hash::sha256_hex(ciphertext)+}+/// SHA-256 (lowercase hex) of a base64 ciphertext string: storage-layer integrity.pub fn compute_ciphertext_hash(base64_ciphertext: &str) -> String {match base64_decode(base64_ciphertext) {- Ok(raw) => hash::sha256_hex(&raw),+ Ok(raw) => compute_ciphertext_hash_bytes(&raw),// An undecodable input still yields a stable hash; callers compare equality, so a malformed// input simply fails the check.Err(_) => hash::sha256_hex(base64_ciphertext.as_bytes()),diff --git a/core/rust/src/vault_sync/http.rs b/core/rust/src/vault_sync/http.rsindex 4d53675ce..d91a32a28 100644--- a/core/rust/src/vault_sync/http.rs+++ b/core/rust/src/vault_sync/http.rs@@ -8,7 +8,11 @@ use super::session::Host;use super::types::{Command, HttpMethod, HttpResponse, LogLevel, StatusResponse};async fn send(host: &Host, method: HttpMethod, path: &str, body: Option<String>, auth: bool, large_transfer: bool) -> SyncResult<HttpResponse> {- let response: HttpResponse = host.call(Command::Http { method, path: path.to_string(), body, auth, large_transfer }).await?;+ Ok(send_with_bytes(host, method, path, body, auth, large_transfer, false).await?.0)+}++async fn send_with_bytes(host: &Host, method: HttpMethod, path: &str, body: Option<String>, auth: bool, large_transfer: bool, binary_response: bool) -> SyncResult<(HttpResponse, Option<Vec<u8>>)> {+ let (response, bytes): (HttpResponse, _) = host.call_with_bytes(Command::Http { method, path: path.to_string(), body, auth, large_transfer, binary_response }).await?;if response.status == 0 {let message = response.transport_error.unwrap_or_else(|| "request failed".to_string());return Err(if response.timed_out { SyncError::Timeout(message) } else { SyncError::Network(message) });@@ -16,7 +20,7 @@ async fn send(host: &Host, method: HttpMethod, path: &str, body: Option<String>,if response.status == 426 {return Err(SyncError::ClientUpgradeRequired);}- Ok(response)+ Ok((response, bytes))}fn check(response: HttpResponse) -> SyncResult<String> {@@ -58,6 +62,50 @@ pub(crate) async fn post<B: Serialize, T: DeserializeOwned>(host: &Host, path: &parse(&body)}+/// Authenticated GET of a v2 endpoint that answers with raw bytes.+pub(crate) async fn get_binary(host: &Host, path: &str, large_transfer: bool) -> SyncResult<Vec<u8>> {+ let (response, bytes) = send_with_bytes(host, HttpMethod::Get, path, None, true, large_transfer, true).await?;+ check(response)?;+ Ok(bytes.unwrap_or_default())+}++/// Authenticated POST of a v2 endpoint with a JSON body that answers with raw bytes.+pub(crate) async fn post_binary<B: Serialize>(host: &Host, path: &str, body: &B, large_transfer: bool) -> SyncResult<Vec<u8>> {+ let (response, bytes) = send_with_bytes(host, HttpMethod::Post, path, Some(serde_json::to_string(body)?), true, large_transfer, true).await?;+ check(response)?;+ Ok(bytes.unwrap_or_default())+}++/// The ciphertexts of a binary vault download, after its JSON header, taken in the order the header lists their sizes.+pub(crate) struct Frame<'a> {+ data: &'a [u8],+}++fn malformed_frame(reason: &str) -> SyncError {+ SyncError::ServerVaultUnreadable(format!("binary response is malformed: {}", reason))+}++/// Open a binary vault download body: a 4-byte big-endian header length, the JSON header, then the raw ciphertexts.+pub(crate) fn open_frame<T: DeserializeOwned>(body: &[u8]) -> SyncResult<(T, Frame<'_>)> {+ let (length, rest) = body.split_first_chunk::<4>().ok_or_else(|| malformed_frame("no header length"))?;+ let (header, data) = rest.split_at_checked(u32::from_be_bytes(*length) as usize).ok_or_else(|| malformed_frame("header exceeds the body"))?;+ Ok((serde_json::from_slice(header).map_err(|e| malformed_frame(&e.to_string()))?, Frame { data }))+}++impl<'a> Frame<'a> {+ /// The next `size` bytes.+ pub fn take(&mut self, size: usize) -> SyncResult<&'a [u8]> {+ let (part, rest) = self.data.split_at_checked(size).ok_or_else(|| malformed_frame("a ciphertext exceeds the body"))?;+ self.data = rest;+ Ok(part)+ }++ /// Refuse bytes the header did not account for.+ pub fn finish(self) -> SyncResult<()> {+ if self.data.is_empty() { Ok(()) } else { Err(malformed_frame("trailing bytes")) }+ }+}+/// Authenticated POST that returns no body.pub(crate) async fn post_no_content<B: Serialize>(host: &Host, path: &str, body: &B) -> SyncResult<()> {check(send(host, HttpMethod::Post, path, Some(serde_json::to_string(body)?), true, false).await?)?;diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rsindex 31ef85e52..b5a7962a5 100644--- a/core/rust/src/vault_sync/pull.rs+++ b/core/rust/src/vault_sync/pull.rs@@ -6,9 +6,10 @@ use std::collections::HashMap;use crate::vault_model::{id_key, ids_equal};use super::errors::{SyncError, SyncResult};use super::state::{self, Ctx};-use super::types::{self, BlobDownloadResponse, BlobHashesRequest, Db, EmailRoutingDto, GetResponse, ManifestDto, SharedManifestDto, StoredBlobRef};+use super::types::{self, BlobDownloadEntry, BlobDownloadResponse, BlobHashesRequest, Db, EmailRoutingDto, GetResponse, ManifestDto, SharedManifestDto, StoredBlobRef};use super::blob_keys::{self, EncryptedBlob};use super::{db, http, keys, legacy};+use crate::common::encoding::base64_encode;use crate::crypto;use crate::vault_codec::{self, CanonicalizeInput, CodecTableData, DataBucket, Manifest, ManifestSpec, MaterializeInput, UnpackedPayload};use crate::vault_model::SYNCABLE_TABLE_NAMES;@@ -73,7 +74,8 @@ pub(crate) enum Snapshot {/// `GET v2/Vault`. A snapshot in a storage format newer than this build knows is refused, never read as legacy.pub(crate) async fn fetch_snapshot(ctx: &Ctx) -> SyncResult<Snapshot> {- let snapshot = http::with_outdated_server_guard(http::get::<GetResponse>(&ctx.host, http::VAULT_ENDPOINT, true).await)?;+ let body = http::with_outdated_server_guard(http::get_binary(&ctx.host, http::VAULT_ENDPOINT, true).await)?;+ let snapshot = decode_snapshot(&body)?;match snapshot.storage_format.as_deref() {None | Some(legacy::STORAGE_FORMAT_SQLITE_BLOB) => Ok(Snapshot::LegacySqliteBlob(snapshot)),Some(STORAGE_FORMAT_MANIFEST) => Ok(Snapshot::Manifest(snapshot)),@@ -81,20 +83,28 @@ pub(crate) async fn fetch_snapshot(ctx: &Ctx) -> SyncResult<Snapshot> {}}-fn base64_chars(size_bytes: i64) -> usize {- ((size_bytes.max(0) as usize).div_ceil(3)) * 4+/// Split a binary snapshot body into its header, filling each manifest and bucket with its ciphertext.+pub(crate) fn decode_snapshot(body: &[u8]) -> SyncResult<GetResponse> {+ let (mut snapshot, mut frame) = http::open_frame::<GetResponse>(body)?;+ for manifest in &mut snapshot.manifests {+ manifest.blob = frame.take(manifest.size)?.to_vec();+ }+ for bucket in &mut snapshot.buckets {+ bucket.blob = frame.take(bucket.size)?.to_vec();+ }+ frame.finish()?;+ Ok(snapshot)}/// Verify a ciphertext against the server's hash, decrypt it bound to `aad` and open it via the codec.-fn verify_decrypt_unpack(base64_ciphertext: &str, vek: &str, aad: &[u8], expected_ciphertext_hash: Option<&str>, label: &str) -> SyncResult<String> {+fn verify_decrypt_unpack(ciphertext: &[u8], vek: &str, aad: &[u8], expected_ciphertext_hash: Option<&str>, label: &str) -> SyncResult<String> {if let Some(expected) = expected_ciphertext_hash.filter(|hash| !hash.is_empty()) {- if vault_codec::compute_ciphertext_hash(base64_ciphertext) != expected {+ if vault_codec::compute_ciphertext_hash_bytes(ciphertext) != expected {return Err(SyncError::ServerVaultUnreadable(format!("{} ciphertext hash mismatch, refusing to load (possible storage corruption)", label)));}}let unreadable = |e: crate::common::error::VaultError| SyncError::ServerVaultUnreadable(format!("{}: {}", label, e));- let encrypted = crate::common::encoding::base64_decode(base64_ciphertext).map_err(unreadable)?;- let plain = crypto::symmetric_decrypt_bytes_with_aad(&encrypted, vek, aad).map_err(unreadable)?;+ let plain = crypto::symmetric_decrypt_bytes_with_aad(ciphertext, vek, aad).map_err(unreadable)?;match vault_codec::unpack_versioned_payload(&plain).map_err(unreadable)? {UnpackedPayload::Readable(payload_json) => Ok(payload_json),UnpackedPayload::NewerFormat(version) => Err(newer_format(label, version)),@@ -271,10 +281,12 @@ async fn open_data_buckets(ctx: &Ctx, snapshot: &GetResponse, resolved: &[Resolvlet mut fingerprints = HashMap::new();let mut revisions = HashMap::new();for dto in &snapshot.buckets {- let Some(blob) = dto.blob.as_deref().filter(|b| !b.is_empty()) else { continue };+ if dto.blob.is_empty() {+ continue;+ }let key = key_by_manifest.get(dto.manifest_id.as_str()).ok_or_else(|| SyncError::Snapshot(format!("data bucket \"{}\" belongs to manifest {}, which this vault did not open, refusing to assemble", dto.category, dto.manifest_id)))?;let label = format!("\"{}\" bucket of manifest {}", dto.category, dto.manifest_id);- let bucket_json = verify_decrypt_unpack(blob, key, &crypto::aad::bucket(&dto.manifest_id, &dto.category), dto.ciphertext_hash.as_deref(), &label)?;+ let bucket_json = verify_decrypt_unpack(&dto.blob, key, &crypto::aad::bucket(&dto.manifest_id, &dto.category), dto.ciphertext_hash.as_deref(), &label)?;let bucket: DataBucket = serde_json::from_str(&bucket_json)?;ensure_readable(bucket.schema_version, &label)?;if !ids_equal(&bucket.manifest_id, &dto.manifest_id) || bucket.category != dto.category {@@ -313,14 +325,15 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest]) ->let mut batches: Vec<(String, Vec<StoredBlobRef>)> = Vec::new();for entry in resolved {let owned: Vec<StoredBlobRef> = missing.iter().filter(|r| owners.get(&r.hash).is_some_and(|o| o.manifest_id == entry.manifest_id)).cloned().collect();- batches.extend(http::batch_by_transfer_cost(owned, |r| base64_chars(r.size_bytes)).into_iter().map(|chunk| (entry.manifest_id.clone(), chunk)));+ batches.extend(http::batch_by_transfer_cost(owned, |r| r.size_bytes.max(0) as usize).into_iter().map(|chunk| (entry.manifest_id.clone(), chunk)));}let batch_count = batches.len();for (index, (manifest_id, chunk)) in batches.into_iter().enumerate() {- let blobs = http::post::<_, BlobDownloadResponse>(&ctx.host, BLOBS_DOWNLOAD_ENDPOINT, &BlobHashesRequest { manifest_id, hashes: chunk.iter().map(|r| r.hash.clone()).collect() }, true).await?.blobs;- ctx.log(format!("[Pull] Downloaded blob batch {}/{}: requested {}, received {}.", index + 1, batch_count, chunk.len(), blobs.len())).await;- for dto in blobs {- cache.insert(dto.hash, EncryptedBlob { encrypted_data_base64: dto.encrypted_data_base64, encrypted_blob_key: dto.encrypted_blob_key });+ let body = http::post_binary(&ctx.host, BLOBS_DOWNLOAD_ENDPOINT, &BlobHashesRequest { manifest_id, hashes: chunk.iter().map(|r| r.hash.clone()).collect() }, true).await?;+ let blobs = decode_blob_download(&body)?;+ ctx.log(format!("[Pull] Downloaded blob batch {}/{}: requested {}, received {} ({} bytes).", index + 1, batch_count, chunk.len(), blobs.len(), body.len())).await;+ for (entry, ciphertext) in blobs {+ cache.insert(entry.hash, EncryptedBlob { encrypted_data_base64: base64_encode(ciphertext), encrypted_blob_key: entry.encrypted_blob_key });}}@@ -345,6 +358,14 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest]) ->Ok(blob_map)}+/// Split a binary blob download body into its blobs.+pub(crate) fn decode_blob_download(body: &[u8]) -> SyncResult<Vec<(BlobDownloadEntry, &[u8])>> {+ let (header, mut frame) = http::open_frame::<BlobDownloadResponse>(body)?;+ let blobs = header.blobs.into_iter().map(|entry| frame.take(entry.size).map(|ciphertext| (entry, ciphertext))).collect::<SyncResult<Vec<_>>>()?;+ frame.finish()?;+ Ok(blobs)+}+/// Materialize manifests and buckets into a fresh database and return its bytes.pub(crate) async fn materialize_to_sqlite(ctx: &mut Ctx, manifests: &[Manifest], data_buckets: &[DataBucket], blob_map: &HashMap<String, Vec<u8>>, opened_names: &HashMap<String, String>) -> SyncResult<Vec<u8>> {ctx.log(format!("[Pull] {} blobs decrypted; running codec reassembly into a fresh SQLite ({} manifest(s) combined)...", blob_map.len(), manifests.len())).await;@@ -371,7 +392,7 @@ pub(crate) async fn materialize_to_sqlite(ctx: &mut Ctx, manifests: &[Manifest],fn open_manifest(dto: &ManifestDto, vek: &str, is_personal: bool) -> SyncResult<ResolvedManifest> {let label = if is_personal { "manifest".to_string() } else { format!("shared manifest {}", dto.manifest_id) };- let manifest_json = verify_decrypt_unpack(dto.blob.as_deref().unwrap_or(""), vek, &crypto::aad::manifest(&dto.manifest_id), dto.ciphertext_hash.as_deref(), &label)?;+ let manifest_json = verify_decrypt_unpack(&dto.blob, vek, &crypto::aad::manifest(&dto.manifest_id), dto.ciphertext_hash.as_deref(), &label)?;let manifest: Manifest = serde_json::from_str(&manifest_json)?;ensure_readable(manifest.schema_version, &label)?;if !ids_equal(&manifest.manifest_id, &dto.manifest_id) {diff --git a/core/rust/src/vault_sync/types.rs b/core/rust/src/vault_sync/types.rsindex ca94ac56d..c8a83c610 100644--- a/core/rust/src/vault_sync/types.rs+++ b/core/rust/src/vault_sync/types.rs@@ -257,6 +257,9 @@ pub enum Command {auth: bool,/// A vault transfer, which the host may give a longer timeout than a status call (which uses default).large_transfer: bool,+ /// On success the host returns the body as raw bytes (the `bytes` argument of `SyncSession::resume`), not as `body` text.+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]+ binary_response: bool,},/// Read an engine-owned persisted value; response [`StateValue`].StateGet { key: String },@@ -456,8 +459,12 @@ pub struct GetResponse {#[serde(rename_all = "camelCase")]pub struct ManifestDto {pub manifest_id: String,+ /// Length of the manifest ciphertext in the binary snapshot body, 0 on an empty vault.#[serde(default)]- pub blob: Option<String>,+ pub size: usize,+ /// The manifest ciphertext, read from the binary snapshot body after the JSON header.+ #[serde(skip)]+ pub blob: Vec<u8>,#[serde(default)]pub ciphertext_hash: Option<String>,#[serde(default)]@@ -487,7 +494,7 @@ pub struct ManifestDto {impl ManifestDto {/// Whether the server served this manifest with content.pub fn has_content(&self) -> bool {- self.blob.as_deref().is_some_and(|blob| !blob.is_empty())+ !self.blob.is_empty()}}@@ -496,8 +503,12 @@ impl ManifestDto {pub struct BucketDto {pub manifest_id: String,pub category: String,+ /// Length of the bucket ciphertext in the binary snapshot body.#[serde(default)]- pub blob: Option<String>,+ pub size: usize,+ /// The bucket ciphertext, read from the binary snapshot body after the JSON header.+ #[serde(skip)]+ pub blob: Vec<u8>,#[serde(default)]pub ciphertext_hash: Option<String>,#[serde(default)]@@ -659,11 +670,21 @@ pub struct MissingBlobsResponse {pub missing: Vec<String>,}+/// JSON header of the binary `POST v2/Vault/blobs/download` response.#[derive(Debug, Clone, Default, Deserialize)]#[serde(rename_all = "camelCase")]pub struct BlobDownloadResponse {#[serde(default)]- pub blobs: Vec<BlobDto>,+ pub blobs: Vec<BlobDownloadEntry>,+}++/// One blob in the JSON header of the binary `POST v2/Vault/blobs/download` response; its ciphertext follows the header.+#[derive(Debug, Clone, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct BlobDownloadEntry {+ pub hash: String,+ pub encrypted_blob_key: String,+ pub size: usize,}/// `GET v2/VaultKey/Password`.