Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Add frame offset pointers to vault GET and POST for backwards compatibility (#2484)
Leendert de Borstf916c5ae2fa6
diff --git a/apps/browser-extension/tests/helpers/manifest-v2-api.ts b/apps/browser-extension/tests/helpers/manifest-v2-api.tsindex fac0f97b1..686adade4 100644--- a/apps/browser-extension/tests/helpers/manifest-v2-api.ts+++ b/apps/browser-extension/tests/helpers/manifest-v2-api.ts@@ -17,6 +17,7 @@ import { bucketAad, manifestAad, symmetricDecryptBytes, symmetricEncryptBytes }/** One manifest entry in the v2 GET snapshot. */export type SnapshotManifest = {manifestId: string;+ offset: number;size: number;blob: string;ciphertextHash: string;@@ -30,7 +31,7 @@ export type VaultSnapshot = {manifests?: SnapshotManifest[];/** The manifest owned by the caller's personal group; every other entry is a shared one. */personalManifestId?: string | null;- buckets?: Array<{ category: string; size: number; blob: string; ciphertextHash: string; revision: number }>;+ buckets?: Array<{ category: string; offset: number; size: number; blob: string; ciphertextHash: string; revision: number }>;};/** The account-key unlock chain from GET /v2/VaultKey/{type} (fields relevant to these tests). */@@ -89,14 +90,13 @@ export async function getVaultSnapshot(apiBaseUrl: string, token: string): Promithrow new Error(`GET /v2/Vault failed with status ${response.status}: ${await response.text()}`);}- // The body is a 4-byte big-endian header length, the JSON header, then the manifest and bucket ciphertexts in header order.+ // The body is a 4-byte big-endian header length, the JSON header, then the ciphertexts, each at its entry's offset after the header.const body = Buffer.from(await response.arrayBuffer());const headerLength = body.readUInt32BE(0);const snapshot = JSON.parse(body.subarray(4, 4 + headerLength).toString('utf8')) as VaultSnapshot;- let offset = 4 + headerLength;+ const data = body.subarray(4 + headerLength);for (const entry of [...(snapshot.manifests ?? []), ...(snapshot.buckets ?? [])]) {- entry.blob = body.subarray(offset, offset + entry.size).toString('base64');- offset += entry.size;+ entry.blob = data.subarray(entry.offset, entry.offset + entry.size).toString('base64');}return snapshot;}@@ -334,12 +334,19 @@ async function postVaultWrite(manifests: ManifestWrite[],buckets: BucketWrite[]): Promise<VaultWriteResult> {- // The body is a 4-byte big-endian header length, the JSON header (sizes in place of the base64 ciphertexts), then the ciphertexts.- const ciphertexts = [...manifests.map((m) => Buffer.from(m.manifestBlob, 'base64')), ...buckets.map((b) => Buffer.from(b.blob, 'base64'))];+ // The body is a 4-byte big-endian header length, the JSON header (offset and size in place of each base64 ciphertext), then the ciphertexts.+ const ciphertexts: Buffer[] = [];+ let dataLength = 0;+ const place = (base64: string): { offset: number; size: number } => {+ const bytes = Buffer.from(base64, 'base64');+ ciphertexts.push(bytes);+ dataLength += bytes.length;+ return { offset: dataLength - bytes.length, size: bytes.length };+ };const header = Buffer.from(JSON.stringify({username: SrpAuthService.normalizeUsername(username),- manifests: manifests.map(({ manifestBlob, ...rest }) => ({ ...rest, size: Buffer.from(manifestBlob, 'base64').length })),- buckets: buckets.map(({ blob, ...rest }) => ({ ...rest, size: Buffer.from(blob, 'base64').length })),+ manifests: manifests.map(({ manifestBlob, ...rest }) => ({ ...rest, ...place(manifestBlob) })),+ buckets: buckets.map(({ blob, ...rest }) => ({ ...rest, ...place(blob) })),emailRouting: null,}));const headerLength = Buffer.alloc(4);diff --git a/apps/mobile-app/ios/VaultStoreKit/Models/WebApiResponse.swift b/apps/mobile-app/ios/VaultStoreKit/Models/WebApiResponse.swiftindex 34ecc2c61..943d3094a 100644--- a/apps/mobile-app/ios/VaultStoreKit/Models/WebApiResponse.swift+++ b/apps/mobile-app/ios/VaultStoreKit/Models/WebApiResponse.swift@@ -1,3 +1,5 @@+import Foundation+/*** Response object from a WebAPI request containing status code, body, and headers*/diff --git a/apps/server/AliasVault.Api/Controllers/V2/VaultController.cs b/apps/server/AliasVault.Api/Controllers/V2/VaultController.csindex 5a114963b..9dde6527b 100644--- a/apps/server/AliasVault.Api/Controllers/V2/VaultController.cs+++ b/apps/server/AliasVault.Api/Controllers/V2/VaultController.cs@@ -71,8 +71,7 @@ public class VaultController(};/// <summary>- /// Atomic snapshot. Returns the latest encrypted manifest + metadata + blob refs + email routing as a binary frame:- /// a <see cref="GetResponse"/> JSON header, then the manifest and bucket ciphertexts (see <see cref="BinaryFrame"/>).+ /// Atomic snapshot. Returns the latest encrypted manifest + metadata + blob refs + email routing as a <see cref="BinaryFrame"/>./// </summary>/// <returns>Snapshot frame.</returns>[HttpGet("")]@@ -110,7 +109,7 @@ public class VaultController(PersonalManifestId = legacy?.ManifestId,EmailRouting = emailRouting,};- return File(BinaryFrame.Write(legacyResponse, []), BinaryFrame.ContentType);+ return File(BinaryFrame.Write(legacyResponse), BinaryFrame.ContentType);}var manifestIds = latestManifests.Select(m => m.ManifestId).ToList();@@ -120,7 +119,6 @@ public class VaultController({ManifestId = x.ManifestId,Category = x.Category,- Size = x.EncryptedData.Length,Data = x.EncryptedData,CiphertextHash = x.CiphertextHash,Revision = x.RevisionNumber,@@ -147,8 +145,7 @@ public class VaultController(return new Manifest{ManifestId = m.ManifestId,- Size = m.ManifestBlob?.Length ?? 0,- Data = m.ManifestBlob,+ Data = m.ManifestBlob ?? [],CiphertextHash = m.ManifestCiphertextHash,Revision = m.RevisionNumber,BlobReferences = refsByManifest.TryGetValue(m.ManifestId, out var refs) ? refs : [],@@ -172,8 +169,7 @@ public class VaultController(Buckets = buckets,EmailRouting = emailRouting,};- var parts = manifests.Where(m => m.Data != null).Select(m => m.Data!).Concat(buckets.Select(b => b.Data)).ToList();- return File(BinaryFrame.Write(response, parts), BinaryFrame.ContentType);+ return File(BinaryFrame.Write(response), BinaryFrame.ContentType);}/// <summary>@@ -181,7 +177,7 @@ public class VaultController(/// plus its blob references, without the rest of the snapshot./// </summary>/// <param name="manifestId">The stable identifier of the logical manifest to fetch.</param>- /// <returns>A binary frame of the manifest DTO and its ciphertext, or 404 when the user has no such manifest-v1 manifest.</returns>+ /// <returns>The manifest as a <see cref="BinaryFrame"/>, or 404 when the user has no such manifest-v1 manifest.</returns>[HttpGet("manifest/{manifestId:guid}")]public async Task<IActionResult> GetManifest(Guid manifestId){@@ -211,8 +207,7 @@ public class VaultController(var manifest = new Manifest{ManifestId = latest.ManifestId,- Size = latest.ManifestBlob?.Length ?? 0,- Data = latest.ManifestBlob,+ Data = latest.ManifestBlob ?? [],CiphertextHash = latest.ManifestCiphertextHash,Revision = latest.RevisionNumber,BlobReferences = blobRefs,@@ -241,7 +236,7 @@ public class VaultController(manifest.CanAdminister = await GroupHelper.IsGroupAdminAsync(context, latest.OwnerGroupId, user.Id);}- return File(BinaryFrame.Write(manifest, manifest.Data != null ? [manifest.Data] : []), BinaryFrame.ContentType);+ return File(BinaryFrame.Write(manifest), BinaryFrame.ContentType);}/// <summary>@@ -774,8 +769,7 @@ public class VaultController(}/// <summary>- /// Download a batch of one manifest's encrypted blobs by hash as raw bytes (application/octet-stream). The body is a- /// 4-byte big-endian header length, a <see cref="BlobDownloadResponse"/> JSON header, then each blob's ciphertext in header order.+ /// Download a batch of one manifest's encrypted blobs by hash, as a <see cref="BinaryFrame"/>./// </summary>/// <param name="model">Hash list request.</param>/// <returns>The stored blobs among the requested hashes.</returns>@@ -796,13 +790,12 @@ public class VaultController(}var wanted = model.Hashes.Distinct().ToList();- var rows = await context.VaultBlobObjects+ var blobs = await context.VaultBlobObjects.Where(b => b.ManifestId == model.ManifestId && wanted.Contains(b.Hash))- .Select(b => new { b.Hash, b.Category, b.EncryptedBlobKey, b.EncryptedData })+ .Select(b => new BlobEntry { Hash = b.Hash, Category = b.Category, EncryptedBlobKey = b.EncryptedBlobKey, Data = b.EncryptedData }).ToListAsync();- var header = new BlobDownloadResponse { Blobs = rows.Select(r => new BlobEntry { Hash = r.Hash, Category = r.Category, EncryptedBlobKey = r.EncryptedBlobKey, Size = r.EncryptedData.Length }).ToList() };- return File(BinaryFrame.Write(header, rows.Select(r => r.EncryptedData).ToList()), BinaryFrame.ContentType);+ return File(BinaryFrame.Write(new BlobDownloadResponse { Blobs = blobs }), BinaryFrame.ContentType);}/// <summary>@@ -1091,15 +1084,9 @@ public class VaultController(foreach (var dto in blobs){- byte[]? data = null;- if (!existing.TryGetValue(dto.Hash, out var row) || overwrite)+ if ((!existing.TryGetValue(dto.Hash, out var row) || overwrite) && !CiphertextHelper.IsCiphertext(dto.Data)){- if (!CiphertextHelper.IsCiphertext(dto.Data))- {- return false;- }-- data = dto.Data;+ return false;}if (row != null)@@ -1107,9 +1094,9 @@ public class VaultController(if (overwrite){row.Category = dto.Category;- row.EncryptedData = data!;+ row.EncryptedData = dto.Data;row.EncryptedBlobKey = dto.EncryptedBlobKey;- row.SizeBytes = data!.Length;+ row.SizeBytes = dto.Data.Length;}continue;@@ -1120,9 +1107,9 @@ public class VaultController(Hash = dto.Hash,ManifestId = manifestId,Category = dto.Category,- EncryptedData = data!,+ EncryptedData = dto.Data,EncryptedBlobKey = dto.EncryptedBlobKey,- SizeBytes = data!.Length,+ SizeBytes = dto.Data.Length,CreatedAt = nowUtc,};context.VaultBlobObjects.Add(entity);diff --git a/apps/server/AliasVault.Api/Helpers/BinaryFrame.cs b/apps/server/AliasVault.Api/Helpers/BinaryFrame.csindex 8fc8d0981..83b762aef 100644--- a/apps/server/AliasVault.Api/Helpers/BinaryFrame.cs+++ b/apps/server/AliasVault.Api/Helpers/BinaryFrame.cs@@ -9,10 +9,12 @@ namespace AliasVault.Api.Helpers;using System.Buffers.Binary;using System.Text.Json;+using AliasVault.Shared.Models.WebApi.V2.Vault;/// <summary>-/// Writes the binary body of the v2 vault downloads: a 4-byte big-endian header length, the header as UTF-8 JSON, then-/// the raw ciphertexts back to back. The header carries each ciphertext's size, in the order the bytes follow.+/// The binary body of the v2 vault transfers: a 4-byte big-endian header length, a UTF-8 JSON header, then the raw+/// ciphertexts. Each <see cref="IFramePart"/> in the header gives the offset (counted from the first byte after the+/// header) and size of its ciphertext, so readers skip entries and bytes they do not know./// </summary>public static class BinaryFrame{@@ -22,22 +24,29 @@ public static class BinaryFramepublic const string ContentType = "application/octet-stream";/// <summary>- /// Builds a frame from a header and the ciphertexts that follow it.+ /// Builds a frame from a header, setting each part's offset and size./// </summary>/// <param name="header">The JSON header.</param>- /// <param name="parts">The ciphertexts, in the order the header lists their sizes.</param>/// <returns>The frame bytes.</returns>- public static byte[] Write(object header, IReadOnlyList<byte[]> parts)+ public static byte[] Write(IFrameBody header){- var headerBytes = JsonSerializer.SerializeToUtf8Bytes(header, JsonSerializerOptions.Web);- var body = new byte[4 + headerBytes.Length + parts.Sum(p => p.Length)];+ var partList = header.FrameParts.ToList();+ var dataLength = 0;+ foreach (var part in partList)+ {+ part.Offset = dataLength;+ part.Size = part.Data.Length;+ dataLength += part.Data.Length;+ }++ var headerBytes = JsonSerializer.SerializeToUtf8Bytes(header, header.GetType(), JsonSerializerOptions.Web);+ var dataStart = 4 + headerBytes.Length;+ var body = new byte[dataStart + dataLength];BinaryPrimitives.WriteUInt32BigEndian(body, (uint)headerBytes.Length);headerBytes.CopyTo(body, 4);- var offset = 4 + headerBytes.Length;- foreach (var part in parts)+ foreach (var part in partList){- part.CopyTo(body, offset);- offset += part.Length;+ part.Data.CopyTo(body, dataStart + part.Offset);}return body;diff --git a/apps/server/AliasVault.Api/Helpers/BinaryFrameInputFormatter.cs b/apps/server/AliasVault.Api/Helpers/BinaryFrameInputFormatter.csindex b61c5da2b..ae573f9c9 100644--- a/apps/server/AliasVault.Api/Helpers/BinaryFrameInputFormatter.cs+++ b/apps/server/AliasVault.Api/Helpers/BinaryFrameInputFormatter.cs@@ -32,16 +32,16 @@ public class BinaryFrameInputFormatter : InputFormatterawait context.HttpContext.Request.Body.CopyToAsync(buffer, context.HttpContext.RequestAborted);var body = buffer.GetBuffer().AsMemory(0, (int)buffer.Length);- if (body.Length < 4 || BinaryPrimitives.ReadUInt32BigEndian(body.Span) > (uint)(body.Length - 4))+ var headerLength = body.Length < 4 ? uint.MaxValue : BinaryPrimitives.ReadUInt32BigEndian(body.Span);+ if (headerLength > body.Length - 4){return Malformed(context, "The frame header length exceeds the body.");}- var headerLength = (int)BinaryPrimitives.ReadUInt32BigEndian(body.Span);IFrameBody? model;try{- model = JsonSerializer.Deserialize(body.Span.Slice(4, headerLength), context.ModelType, JsonSerializerOptions.Web) as IFrameBody;+ model = JsonSerializer.Deserialize(body.Span.Slice(4, (int)headerLength), context.ModelType, JsonSerializerOptions.Web) as IFrameBody;}catch (JsonException ex){@@ -53,19 +53,19 @@ public class BinaryFrameInputFormatter : InputFormatterreturn Malformed(context, "The frame header is empty.");}- var offset = 4 + headerLength;+ // Bytes no known part points at are ignored, as unknown JSON fields are.+ var data = body[(4 + (int)headerLength)..];foreach (var part in model.FrameParts){- if (part.Size < 0 || part.Size > body.Length - offset)+ if (part.Offset < 0 || part.Size < 0 || (long)part.Offset + part.Size > data.Length){return Malformed(context, "A ciphertext exceeds the body.");}- part.Data = body.Slice(offset, part.Size).ToArray();- offset += part.Size;+ part.Data = data.Slice(part.Offset, part.Size).ToArray();}- return offset == body.Length ? await InputFormatterResult.SuccessAsync(model) : Malformed(context, "The body carries bytes the header does not list.");+ return await InputFormatterResult.SuccessAsync(model);}/// <inheritdoc/>diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobDownloadResponse.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobDownloadResponse.csindex da5464a0c..13d4fb1db 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobDownloadResponse.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobDownloadResponse.cs@@ -7,12 +7,17 @@namespace AliasVault.Shared.Models.WebApi.V2.Vault;+using System.Text.Json.Serialization;+/// <summary>-/// JSON header of the binary POST /v2/Vault/blobs/download response. The body is a 4-byte big-endian header length,-/// this header as UTF-8 JSON, then each blob's ciphertext bytes in header order.+/// Response for POST /v2/Vault/blobs/download, as the JSON header of a binary frame (see <see cref="IFrameBody"/>)./// </summary>-public class BlobDownloadResponse+public class BlobDownloadResponse : IFrameBody{- /// <summary>Gets or sets the requested blobs the server has stored, in the order their bytes follow the header.</summary>+ /// <summary>Gets or sets the requested blobs the server has stored.</summary>public List<BlobEntry> Blobs { get; set; } = [];++ /// <inheritdoc/>+ [JsonIgnore]+ public IEnumerable<IFramePart> FrameParts => Blobs;}diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobEntry.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobEntry.csindex 540ff5ae0..e0e52a8e8 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobEntry.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobEntry.cs@@ -11,7 +11,7 @@ using System.ComponentModel.DataAnnotations;using System.Text.Json.Serialization;/// <summary>-/// One blob in the header of a binary blob upload or download; its ciphertext follows the header.+/// A single encrypted blob of a blob upload or download./// </summary>public class BlobEntry : IFramePart{@@ -27,10 +27,13 @@ public class BlobEntry : IFramePart[StringLength(255, MinimumLength = 1)]public required string EncryptedBlobKey { get; set; }- /// <summary>Gets or sets the length in bytes of the blob's ciphertext in the frame.</summary>+ /// <inheritdoc/>+ public int Offset { get; set; }++ /// <inheritdoc/>public int Size { get; set; }- /// <summary>Gets or sets the blob's ciphertext (encrypted with the blob's own key). Sent after the JSON header, not inside it.</summary>+ /// <summary>Gets or sets the bytes encrypted with the blob's own key. Sent after the JSON header, not inside it.</summary>[JsonIgnore]public byte[] Data { get; set; } = [];}diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Bucket.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Bucket.csindex 110e6e14f..2e1f619d6 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Bucket.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Bucket.cs@@ -12,7 +12,7 @@ using System.Text.Json.Serialization;/// <summary>/// A single data bucket as carried in list-based payloads./// </summary>-public class Bucket+public class Bucket : IFramePart{/// <summary>Gets or sets the id of the manifest that owns this bucket.</summary>public required Guid ManifestId { get; set; }@@ -20,10 +20,13 @@ public class Bucket/// <summary>Gets or sets the bucket kind discriminator.</summary>public required VaultDataBucketCategory Category { get; set; }- /// <summary>Gets or sets the length of the bucket ciphertext in the binary response body.</summary>+ /// <inheritdoc/>+ public int Offset { get; set; }++ /// <inheritdoc/>public int Size { get; set; }- /// <summary>Gets or sets the bucket ciphertext (AES-GCM). Sent after the JSON header, not inside it.</summary>+ /// <inheritdoc/>[JsonIgnore]public byte[] Data { get; set; } = [];diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BucketWrite.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BucketWrite.csindex 0fc68ac3e..35a873d68 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BucketWrite.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BucketWrite.cs@@ -21,10 +21,13 @@ public class BucketWrite : IFramePart/// <summary>Gets or sets the bucket kind discriminator.</summary>public required VaultDataBucketCategory Category { get; set; }- /// <summary>Gets or sets the length of the bucket ciphertext in the frame.</summary>+ /// <inheritdoc/>+ public int Offset { get; set; }++ /// <inheritdoc/>public int Size { get; set; }- /// <summary>Gets or sets the bucket ciphertext (AES-GCM). Sent after the JSON header, not inside it.</summary>+ /// <inheritdoc/>[JsonIgnore]public byte[] Data { get; set; } = [];diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/GetResponse.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/GetResponse.csindex fe85b750a..4d49e5ef3 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/GetResponse.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/GetResponse.cs@@ -7,11 +7,12 @@namespace AliasVault.Shared.Models.WebApi.V2.Vault;+using System.Text.Json.Serialization;+/// <summary>-/// Atomic snapshot returned by GET /v2/Vault, as the JSON header of a binary body: the manifest ciphertexts follow it-/// in <see cref="Manifests"/> order, then the bucket ciphertexts in <see cref="Buckets"/> order.+/// Atomic snapshot returned by GET /v2/Vault, as the JSON header of a binary frame (see <see cref="IFrameBody"/>)./// </summary>-public class GetResponse+public class GetResponse : IFrameBody{/// <summary>/// Gets or sets the storage format of the returned vault.@@ -43,4 +44,8 @@ public class GetResponse/// <summary>Gets or sets the plaintext email routing data (private/public domains + claimed addresses).</summary>public EmailRouting EmailRouting { get; set; } = new();++ /// <inheritdoc/>+ [JsonIgnore]+ public IEnumerable<IFramePart> FrameParts => Manifests.Cast<IFramePart>().Concat(Buckets);}diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/IFrameBody.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/IFrameBody.csindex 121a63484..1b33fd5a0 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/IFrameBody.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/IFrameBody.cs@@ -8,11 +8,10 @@namespace AliasVault.Shared.Models.WebApi.V2.Vault;/// <summary>-/// A request body that arrives as a binary frame: a 4-byte big-endian header length, this object as JSON, then the-/// ciphertexts of its <see cref="FrameParts"/> back to back.+/// A body sent as a binary frame: this object is the JSON header, the ciphertexts of its parts follow it./// </summary>public interface IFrameBody{- /// <summary>Gets the entries that carry a ciphertext, in the order their bytes follow the header.</summary>+ /// <summary>Gets the entries whose ciphertext follows the header, in frame order.</summary>IEnumerable<IFramePart> FrameParts { get; }}diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/IFramePart.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/IFramePart.csindex f6c6a6ac0..a9ddb161a 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/IFramePart.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/IFramePart.cs@@ -8,13 +8,16 @@namespace AliasVault.Shared.Models.WebApi.V2.Vault;/// <summary>-/// An entry of a binary frame header whose ciphertext follows the header.+/// A header entry of a binary frame whose ciphertext follows the header, addressed by offset and size./// </summary>public interface IFramePart{- /// <summary>Gets the length of the ciphertext in the frame.</summary>- int Size { get; }+ /// <summary>Gets or sets where the ciphertext starts, counted from the first byte after the header.</summary>+ int Offset { get; set; }- /// <summary>Gets or sets the ciphertext, read from the frame after the header.</summary>+ /// <summary>Gets or sets the length of the ciphertext, 0 when the entry carries none.</summary>+ int Size { get; set; }++ /// <summary>Gets or sets the ciphertext. Sent after the JSON header, not inside it.</summary>byte[] Data { get; set; }}diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Manifest.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Manifest.csindex b06b565f2..1101d2c0f 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Manifest.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/Manifest.cs@@ -15,17 +15,20 @@ using System.Text.Json.Serialization;/// Which is which follows from <see cref="GetResponse.PersonalManifestId"/>, not from a flag on each entry. Each/// manifest is independently encrypted and revisioned, and carries its own blob references./// </summary>-public class Manifest+public class Manifest : IFramePart, IFrameBody{/// <summary>Gets or sets the stable identifier of the logical manifest (constant across its revisions).</summary>public required Guid ManifestId { get; set; }- /// <summary>Gets or sets the length of the manifest ciphertext in the binary response body, 0 on empty vault.</summary>+ /// <inheritdoc/>+ public int Offset { get; set; }++ /// <inheritdoc/>public int Size { get; set; }- /// <summary>Gets or sets the manifest ciphertext (AES-GCM), null on empty vault. Sent after the JSON header, not inside it.</summary>+ /// <summary>Gets or sets the manifest ciphertext (AES-GCM), empty on an empty vault.</summary>[JsonIgnore]- public byte[]? Data { get; set; }+ public byte[] Data { get; set; } = [];/// <summary>Gets or sets the SHA-256 (hex) of the manifest ciphertext for client-side storage-integrity check.</summary>public string? CiphertextHash { get; set; }@@ -72,4 +75,8 @@ public class Manifest/// <summary>Gets or sets the version of the VEK in <see cref="EncryptedVek"/>, which the grant is signed for.</summary>public int KeyVersion { get; set; }++ /// <summary>Gets this manifest as its own frame part, for GET /v2/Vault/manifest/{id}.</summary>+ [JsonIgnore]+ public IEnumerable<IFramePart> FrameParts => [this];}diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/ManifestWrite.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/ManifestWrite.csindex 9ddfa3610..02c9909c7 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/ManifestWrite.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/ManifestWrite.cs@@ -18,10 +18,13 @@ public class ManifestWrite : IFramePart/// <summary>Gets or sets the manifest this write targets.</summary>public required Guid ManifestId { get; set; }- /// <summary>Gets or sets the length of the manifest ciphertext in the frame.</summary>+ /// <inheritdoc/>+ public int Offset { get; set; }++ /// <inheritdoc/>public int Size { get; set; }- /// <summary>Gets or sets the manifest ciphertext (AES-GCM). Sent after the JSON header, not inside it.</summary>+ /// <inheritdoc/>[JsonIgnore]public byte[] Data { get; set; } = [];diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/VaultWriteRequest.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/VaultWriteRequest.csindex 778f48782..aaa92c09a 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/VaultWriteRequest.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/VaultWriteRequest.cs@@ -11,8 +11,7 @@ using System.ComponentModel.DataAnnotations;using System.Text.Json.Serialization;/// <summary>-/// Unified atomic write for POST /v2/Vault, sent as a binary frame: the manifest ciphertexts follow this JSON header in-/// <see cref="Manifests"/> order, then the bucket ciphertexts in <see cref="Buckets"/> order.+/// Unified atomic write for POST /v2/Vault, sent as a binary frame (see <see cref="IFrameBody"/>)./// </summary>public class VaultWriteRequest : IFrameBody{diff --git a/apps/web/tests/helpers/vault-tamper.ts b/apps/web/tests/helpers/vault-tamper.tsindex d3e21e97d..99b3ac84c 100644--- a/apps/web/tests/helpers/vault-tamper.ts+++ b/apps/web/tests/helpers/vault-tamper.ts@@ -79,6 +79,7 @@ async function writePersonalManifest(apiUrl: string, user: TestUser, blob: strinusername: SrpAuthService.normalizeUsername(user.username),manifests: [{manifestId,+ offset: 0,size: ciphertext.length,manifestCiphertextHash: createHash('sha256').update(ciphertext).digest('hex'),currentRevision: revision,diff --git a/core/rust/src/bindings/uniffi_api.rs b/core/rust/src/bindings/uniffi_api.rsindex 8fbb24265..2569ffaf6 100644--- a/core/rust/src/bindings/uniffi_api.rs+++ b/core/rust/src/bindings/uniffi_api.rs@@ -335,7 +335,7 @@ impl VaultSyncSession {self.inner.command_bytes()}- /// Hand the host's response to the last command back, as JSON, with raw bytes for a `dbExport`.+ /// Hand the host's response to the last command back, as JSON, with raw bytes for a `dbExport` or a binary `http` response.pub fn resume(&self, response_json: String, bytes: Option<Vec<u8>>) -> Result<(), VaultError> {self.inner.resume(&response_json, bytes)}diff --git a/core/rust/src/bindings/wasm.rs b/core/rust/src/bindings/wasm.rsindex a5a928c2a..7efb1a67e 100644--- a/core/rust/src/bindings/wasm.rs+++ b/core/rust/src/bindings/wasm.rs@@ -331,7 +331,7 @@ impl VaultSyncSessionJs {self.inner.command_bytes().map_err(js_err)}- /// Hand the host's response to the last command back, as JSON, with raw bytes for a `dbExport`.+ /// Hand the host's response to the last command back, as JSON, with raw bytes for a `dbExport` or a binary `http` response.pub fn resume(&self, response_json: &str, bytes: Option<Vec<u8>>) -> Result<(), JsValue> {self.inner.resume(response_json, bytes).map_err(js_err)}diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rsindex e489f8c31..2d54eac90 100644--- a/core/rust/src/vault_codec/mod.rs+++ b/core/rust/src/vault_codec/mod.rs@@ -20,7 +20,7 @@ mod validate;use serde_json::json;-use crate::common::encoding::{base64_decode, hex_encode_lower};+use crate::common::encoding::hex_encode_lower;use crate::common::error::{VaultError, VaultResult};pub use types::SCHEMA_VERSION;@@ -125,21 +125,11 @@ pub fn unpack_versioned_payload(plain_bytes: &[u8]) -> VaultResult<UnpackedPayloOk(UnpackedPayload::Readable(serde_json::to_string(payload)?))}-/// SHA-256 (lowercase hex) of a raw ciphertext: storage-layer integrity.-pub fn compute_ciphertext_hash_bytes(ciphertext: &[u8]) -> String {+/// SHA-256 (lowercase hex) of a ciphertext: storage-layer integrity.+pub fn compute_ciphertext_hash(ciphertext: &[u8]) -> String {hash::sha256_hex(ciphertext)}-/// SHA-256 (lowercase hex) of a base64 ciphertext string: storage-layer integrity.-pub fn compute_ciphertext_hash(base64_ciphertext: &str) -> String {- match base64_decode(base64_ciphertext) {- Ok(raw) => compute_ciphertext_hash_bytes(&raw),- // An undecodable input still yields a stable hash; callers compare equality, so a malformed- // input simply fails the check.- Err(_) => hash::sha256_hex(base64_ciphertext.as_bytes()),- }-}-/// Content fingerprint of a manifest / data-bucket payload for client-side change detection: SHA-256/// (lowercase hex) of the canonical JSON with the volatile top-level `canonicalizedAt` timestamp removed/// (it is regenerated on every canonicalize and must not read as a content change).diff --git a/core/rust/src/vault_sync/blob_keys.rs b/core/rust/src/vault_sync/blob_keys.rsindex 7d9658dff..4e999d925 100644--- a/core/rust/src/vault_sync/blob_keys.rs+++ b/core/rust/src/vault_sync/blob_keys.rs@@ -10,7 +10,7 @@ use crate::common::error::VaultResult;#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]pub(crate) struct EncryptedBlob {- /// The bytes, encrypted with the blob key. Kept as base64 in the persisted cache (the field name predates raw transport).+ /// The bytes, encrypted with the blob key. Persisted as base64 under the old field name, so existing caches still read.#[serde(rename = "encryptedDataBase64", with = "base64_bytes")]pub encrypted_data: Vec<u8>,/// The blob key, encrypted with the manifest's VEK.diff --git a/core/rust/src/vault_sync/db.rs b/core/rust/src/vault_sync/db.rsindex 285b589f1..12d8e8054 100644--- a/core/rust/src/vault_sync/db.rs+++ b/core/rust/src/vault_sync/db.rs@@ -43,7 +43,7 @@ pub(crate) async fn open_staging(host: &Host) -> SyncResult<()> {/// Serialize a database to SQLite bytes.pub(crate) async fn export(host: &Host, db: Db) -> SyncResult<Vec<u8>> {- let (_, bytes) = host.call_with_bytes::<Ack>(Command::DbExport { db }).await?;+ let (_, bytes) = host.call_with_bytes::<Ack>(Command::DbExport { db }, None).await?;bytes.ok_or_else(|| SyncError::Other("host returned no database bytes".to_string()))}diff --git a/core/rust/src/vault_sync/frame.rs b/core/rust/src/vault_sync/frame.rsnew file mode 100644index 000000000..edd0bfeae--- /dev/null+++ b/core/rust/src/vault_sync/frame.rs@@ -0,0 +1,117 @@+//! The binary body of the v2 vault transfers: a 4-byte big-endian header length, a JSON header, then the raw+//! ciphertexts. Each header entry that carries a ciphertext gives its `offset` (counted from the first byte after the+//! header) and `size`, so a reader skips entries and bytes it does not know, as it skips unknown JSON fields.+//! Mirrors the server's `BinaryFrame`.++use serde::de::DeserializeOwned;+use serde::{Deserialize, Serialize};++use super::errors::{SyncError, SyncResult};++/// A ciphertext sent after the frame header; the header carries its `offset` and `size`.+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]+pub struct FramePart {+ #[serde(default)]+ pub offset: usize,+ #[serde(default)]+ pub size: usize,+ #[serde(skip)]+ pub bytes: Vec<u8>,+}++impl From<Vec<u8>> for FramePart {+ fn from(bytes: Vec<u8>) -> Self {+ Self { offset: 0, size: bytes.len(), bytes }+ }+}++/// A frame header: the entries whose ciphertext follows it, in frame order.+pub(crate) trait FrameBody {+ fn parts(&mut self) -> Vec<&mut FramePart>;+}++/// Lay out `body` as a frame, setting each part's `offset` and `size`.+pub(crate) fn encode<B: FrameBody + Serialize>(body: &mut B) -> SyncResult<Vec<u8>> {+ let mut data_length = 0;+ for part in body.parts() {+ part.offset = data_length;+ part.size = part.bytes.len();+ data_length += part.size;+ }+ let header = serde_json::to_vec(body)?;+ let mut frame = Vec::with_capacity(4 + header.len() + data_length);+ frame.extend_from_slice(&(header.len() as u32).to_be_bytes());+ frame.extend_from_slice(&header);+ for part in body.parts() {+ frame.extend_from_slice(&part.bytes);+ }+ Ok(frame)+}++/// Read a frame back into its header type, filling each part with its ciphertext.+pub(crate) fn decode<T: FrameBody + DeserializeOwned>(frame: &[u8]) -> SyncResult<T> {+ let malformed = |reason: &str| SyncError::ServerVaultUnreadable(format!("binary response is malformed: {}", reason));+ let (length, rest) = frame.split_first_chunk::<4>().ok_or_else(|| malformed("no header length"))?;+ let (header, data) = rest.split_at_checked(u32::from_be_bytes(*length) as usize).ok_or_else(|| malformed("header exceeds the body"))?;+ let mut body: T = serde_json::from_slice(header).map_err(|e| malformed(&e.to_string()))?;+ for part in body.parts() {+ let bytes = part.offset.checked_add(part.size).and_then(|end| data.get(part.offset..end)).ok_or_else(|| malformed("a ciphertext exceeds the body"))?;+ part.bytes = bytes.to_vec();+ }+ Ok(body)+}++#[cfg(test)]+mod tests {+ use serde_json::{json, Value};++ use super::*;+ use crate::vault_sync::types::{BlobDownloadResponse, BlobDto, BlobUploadRequest, GetResponse};++ fn frame(header: Value, data: &[u8]) -> Vec<u8> {+ let header = serde_json::to_vec(&header).unwrap();+ [&(header.len() as u32).to_be_bytes()[..], &header, data].concat()+ }++ #[test]+ fn an_upload_keeps_each_ciphertext_out_of_the_json_header() {+ let blob = |hash: &str, bytes: Vec<u8>| BlobDto { hash: hash.to_string(), category: "attachment".to_string(), ciphertext: bytes.into(), encrypted_blob_key: format!("k{}", hash) };+ let mut request = BlobUploadRequest { manifest_id: "m1".to_string(), overwrite: false, blobs: vec![blob("a", vec![1, 2, 3]), blob("b", vec![]), blob("c", vec![4])] };+ let encoded = encode(&mut request).unwrap();++ let header_length = u32::from_be_bytes(encoded[..4].try_into().unwrap()) as usize;+ let header: Value = serde_json::from_slice(&encoded[4..4 + header_length]).unwrap();+ assert_eq!(header["blobs"][0], json!({ "hash": "a", "category": "attachment", "offset": 0, "size": 3, "encryptedBlobKey": "ka" }));+ assert_eq!(&encoded[4 + header_length..], &[1, 2, 3, 4]);++ let decoded: BlobDownloadResponse = decode(&encoded).unwrap();+ assert_eq!(decoded.blobs.iter().map(|b| b.ciphertext.bytes.clone()).collect::<Vec<_>>(), vec![vec![1, 2, 3], vec![], vec![4]]);+ }++ #[test]+ fn a_download_reads_each_part_by_offset_and_skips_what_it_does_not_know() {+ // A newer server: parts in another order, an unknown list with a part of its own, and a part nothing known points at.+ let encoded = frame(+ json!({+ "storageFormat": "manifest",+ "futureList": [{ "offset": 0, "size": 2 }],+ "manifests": [{ "manifestId": "m1", "offset": 5, "size": 3, "futureField": true }, { "manifestId": "m2" }],+ "buckets": [{ "manifestId": "m1", "category": "settings", "offset": 2, "size": 3 }],+ }),+ &[9, 9, 4, 5, 6, 1, 2, 3, 7, 7, 7],+ );+ let snapshot: GetResponse = decode(&encoded).unwrap();+ assert_eq!(snapshot.manifests[0].ciphertext.bytes, vec![1, 2, 3]);+ assert!(!snapshot.manifests[1].has_content());+ assert_eq!(snapshot.buckets[0].ciphertext.bytes, vec![4, 5, 6]);+ }++ #[test]+ fn a_malformed_frame_is_refused() {+ let encoded = frame(json!({ "blobs": [{ "hash": "a", "category": "favicon", "encryptedBlobKey": "ka", "offset": 0, "size": 3 }] }), &[1, 2, 3]);+ assert!(decode::<BlobDownloadResponse>(&encoded).is_ok());+ assert!(decode::<BlobDownloadResponse>(&encoded[..encoded.len() - 1]).is_err(), "a truncated ciphertext");+ assert!(decode::<BlobDownloadResponse>(&[]).is_err(), "no header length");+ assert!(decode::<BlobDownloadResponse>(&[0, 0, 1, 0, b'{']).is_err(), "a header longer than the body");+ }+}diff --git a/core/rust/src/vault_sync/http.rs b/core/rust/src/vault_sync/http.rsindex b55aaaf38..9bfc6a6af 100644--- a/core/rust/src/vault_sync/http.rs+++ b/core/rust/src/vault_sync/http.rs@@ -4,29 +4,26 @@ use serde::de::DeserializeOwned;use serde::Serialize;use super::errors::{SyncError, SyncResult};+use super::frame::{self, FrameBody};use super::session::Host;use super::types::{Command, HttpMethod, HttpResponse, LogLevel, StatusResponse};-/// The body of a request: JSON text, or raw bytes the host sends as `application/octet-stream`.+/// A request body: none, JSON text, or a binary frame the host sends as `application/octet-stream`.enum Body {None,Json(String),- Binary(Vec<u8>),+ Frame(Vec<u8>),}-async fn send(host: &Host, method: HttpMethod, path: &str, body: Option<String>, auth: bool, large_transfer: bool) -> SyncResult<HttpResponse> {- let body = body.map_or(Body::None, Body::Json);- Ok(send_with_bytes(host, method, path, body, auth, large_transfer, false).await?.0)-}--async fn send_with_bytes(host: &Host, method: HttpMethod, path: &str, body: Body, auth: bool, large_transfer: bool, binary_response: bool) -> SyncResult<(HttpResponse, Option<Vec<u8>>)> {- let (body, binary_body, command_bytes) = match body {+/// Send a request; with `binary_response` a successful response comes back as raw bytes instead of as `body` text.+async fn send(host: &Host, method: HttpMethod, path: &str, body: Body, large_transfer: bool, binary_response: bool) -> SyncResult<(HttpResponse, Option<Vec<u8>>)> {+ let (body, binary_body, bytes) = match body {Body::None => (None, false, None),Body::Json(json) => (Some(json), false, None),- Body::Binary(bytes) => (None, true, Some(bytes)),+ Body::Frame(frame) => (None, true, Some(frame)),};- let command = Command::Http { method, path: path.to_string(), body, auth, large_transfer, binary_response, binary_body };- let (response, bytes): (HttpResponse, _) = host.exchange(command, command_bytes).await?;+ let command = Command::Http { method, path: path.to_string(), body, auth: true, large_transfer, binary_response, binary_body };+ let (response, bytes): (HttpResponse, _) = host.call_with_bytes(command, bytes).await?;if response.status == 0 {let message = response.transport_error.unwrap_or_else(|| "request failed".to_string());return Err(if response.timed_out { SyncError::Timeout(message) } else { SyncError::Network(message) });@@ -37,6 +34,19 @@ async fn send_with_bytes(host: &Host, method: HttpMethod, path: &str, body: BodyOk((response, bytes))}+/// Send a request that answers with JSON and return the checked body.+async fn send_json(host: &Host, method: HttpMethod, path: &str, body: Body, large_transfer: bool) -> SyncResult<String> {+ check(send(host, method, path, body, large_transfer, false).await?.0)+}++/// Send a request that answers with a binary frame and decode it.+async fn send_for_frame<T: FrameBody + DeserializeOwned>(host: &Host, method: HttpMethod, path: &str, body: Body) -> SyncResult<T> {+ let (response, bytes) = send(host, method, path, body, true, true).await?;+ check(response)?;+ // A host that could not read the body hands back no bytes: a transfer failure, not a damaged vault.+ frame::decode(&bytes.ok_or_else(|| SyncError::Network(format!("{} answered without a body", path)))?)+}+fn check(response: HttpResponse) -> SyncResult<String> {let code = api_error_code(&response.body);match response.status {@@ -66,103 +76,39 @@ fn parse<T: DeserializeOwned>(body: &str) -> SyncResult<T> {/// Authenticated GET of a v2 endpoint.pub(crate) async fn get<T: DeserializeOwned>(host: &Host, path: &str, large_transfer: bool) -> SyncResult<T> {- let body = check(send(host, HttpMethod::Get, path, None, true, large_transfer).await?)?;- parse(&body)+ parse(&send_json(host, HttpMethod::Get, path, Body::None, large_transfer).await?)}/// Authenticated POST of a v2 endpoint with a JSON body.pub(crate) async fn post<B: Serialize, T: DeserializeOwned>(host: &Host, path: &str, body: &B, large_transfer: bool) -> SyncResult<T> {- let body = check(send(host, HttpMethod::Post, path, Some(serde_json::to_string(body)?), true, large_transfer).await?)?;- parse(&body)+ parse(&send_json(host, HttpMethod::Post, path, Body::Json(serde_json::to_string(body)?), large_transfer).await?)}-/// Authenticated GET of a v2 endpoint that answers with raw bytes.-pub(crate) async fn get_binary(host: &Host, path: &str, large_transfer: bool) -> SyncResult<Vec<u8>> {- let (response, bytes) = send_with_bytes(host, HttpMethod::Get, path, Body::None, true, large_transfer, true).await?;- check(response)?;- Ok(bytes.unwrap_or_default())-}--/// Authenticated POST of a v2 endpoint with a JSON body that answers with raw bytes.-pub(crate) async fn post_binary<B: Serialize>(host: &Host, path: &str, body: &B, large_transfer: bool) -> SyncResult<Vec<u8>> {- let (response, bytes) = send_with_bytes(host, HttpMethod::Post, path, Body::Json(serde_json::to_string(body)?), true, large_transfer, true).await?;- check(response)?;- Ok(bytes.unwrap_or_default())-}--/// A request body sent as a binary frame: its JSON header (which lists each ciphertext's size), then the ciphertexts.-pub(crate) trait FrameBody: Serialize {- /// The ciphertexts, in the order the header lists their sizes.- fn parts(&self) -> Vec<&[u8]>;-}--/// Authenticated POST of a v2 endpoint with a binary frame body and a JSON response.-pub(crate) async fn post_frame<B: FrameBody, T: DeserializeOwned>(host: &Host, path: &str, body: &B) -> SyncResult<T> {- let (response, _) = send_with_bytes(host, HttpMethod::Post, path, Body::Binary(encode_frame(body)?), true, true, false).await?;- parse(&check(response)?)-}--/// Authenticated POST of a v2 endpoint with a binary frame body and no response body.-pub(crate) async fn post_frame_no_content<B: FrameBody>(host: &Host, path: &str, body: &B) -> SyncResult<()> {- let (response, _) = send_with_bytes(host, HttpMethod::Post, path, Body::Binary(encode_frame(body)?), true, true, false).await?;- check(response)?;+/// Authenticated POST that returns no body.+pub(crate) async fn post_no_content<B: Serialize>(host: &Host, path: &str, body: &B) -> SyncResult<()> {+ send_json(host, HttpMethod::Post, path, Body::Json(serde_json::to_string(body)?), false).await?;Ok(())}-/// A binary frame: 4-byte big-endian header length, the JSON header, then the parts back to back.-pub(crate) fn encode_frame<B: FrameBody>(body: &B) -> SyncResult<Vec<u8>> {- let header = serde_json::to_vec(body)?;- let parts = body.parts();- let header_length = u32::try_from(header.len()).map_err(|_| SyncError::Other("binary frame header exceeds 4 GiB".to_string()))?;- let mut frame = Vec::with_capacity(4 + header.len() + parts.iter().map(|p| p.len()).sum::<usize>());- frame.extend_from_slice(&header_length.to_be_bytes());- frame.extend_from_slice(&header);- for part in parts {- frame.extend_from_slice(part);- }- Ok(frame)-}--/// The ciphertexts of a binary vault download, after its JSON header, taken in the order the header lists their sizes.-pub(crate) struct Frame<'a> {- data: &'a [u8],-}--fn malformed_frame(reason: &str) -> SyncError {- SyncError::ServerVaultUnreadable(format!("binary response is malformed: {}", reason))-}--/// Open a binary vault download body: a 4-byte big-endian header length, the JSON header, then the raw ciphertexts.-pub(crate) fn open_frame<T: DeserializeOwned>(body: &[u8]) -> SyncResult<(T, Frame<'_>)> {- let (length, rest) = body.split_first_chunk::<4>().ok_or_else(|| malformed_frame("no header length"))?;- let (header, data) = rest.split_at_checked(u32::from_be_bytes(*length) as usize).ok_or_else(|| malformed_frame("header exceeds the body"))?;- Ok((serde_json::from_slice(header).map_err(|e| malformed_frame(&e.to_string()))?, Frame { data }))+/// Authenticated DELETE of a v2 endpoint.+pub(crate) async fn delete(host: &Host, path: &str) -> SyncResult<()> {+ send_json(host, HttpMethod::Delete, path, Body::None, false).await?;+ Ok(())}-impl<'a> Frame<'a> {- /// The next `size` bytes.- pub fn take(&mut self, size: usize) -> SyncResult<&'a [u8]> {- let (part, rest) = self.data.split_at_checked(size).ok_or_else(|| malformed_frame("a ciphertext exceeds the body"))?;- self.data = rest;- Ok(part)- }-- /// Refuse bytes the header did not account for.- pub fn finish(self) -> SyncResult<()> {- if self.data.is_empty() { Ok(()) } else { Err(malformed_frame("trailing bytes")) }- }+/// Authenticated GET of a v2 endpoint that answers with a binary frame.+pub(crate) async fn get_frame<T: FrameBody + DeserializeOwned>(host: &Host, path: &str) -> SyncResult<T> {+ send_for_frame(host, HttpMethod::Get, path, Body::None).await}-/// Authenticated POST that returns no body.-pub(crate) async fn post_no_content<B: Serialize>(host: &Host, path: &str, body: &B) -> SyncResult<()> {- check(send(host, HttpMethod::Post, path, Some(serde_json::to_string(body)?), true, false).await?)?;- Ok(())+/// Authenticated POST of a v2 endpoint with a JSON body that answers with a binary frame.+pub(crate) async fn post_for_frame<B: Serialize, T: FrameBody + DeserializeOwned>(host: &Host, path: &str, body: &B) -> SyncResult<T> {+ send_for_frame(host, HttpMethod::Post, path, Body::Json(serde_json::to_string(body)?)).await}-/// Authenticated DELETE of a v2 endpoint.-pub(crate) async fn delete(host: &Host, path: &str) -> SyncResult<()> {- check(send(host, HttpMethod::Delete, path, None, true, false).await?)?;- Ok(())+/// Authenticated POST of a v2 endpoint with a binary frame body and a JSON response.+pub(crate) async fn post_frame<B: FrameBody + Serialize, T: DeserializeOwned>(host: &Host, path: &str, body: &mut B) -> SyncResult<T> {+ parse(&send_json(host, HttpMethod::Post, path, Body::Frame(frame::encode(body)?), true).await?)}/// The v2 vault endpoint (`GET`/`POST v2/Vault`).@@ -171,8 +117,8 @@ pub(crate) const VAULT_ENDPOINT: &str = "Vault";/// The v2 vault key endpoint (`GET v2/VaultKey/Password`): the account's password-wrapped key chain.pub(crate) const VAULT_KEY_PASSWORD_ENDPOINT: &str = "VaultKey/Password";-/// Max base64 characters in one blob transfer request or response body.-pub(crate) const BLOB_TRANSFER_BATCH_MAX_CHARS: usize = 4 * 1024 * 1024;+/// Max ciphertext bytes in one blob transfer request or response body.+pub(crate) const BLOB_TRANSFER_BATCH_MAX_BYTES: usize = 4 * 1024 * 1024;/// Upper bound on the number of blobs in one transfer batch (the server's `VaultWriteLimits.MaxBlobsPerUpload`).pub(crate) const BLOB_TRANSFER_BATCH_MAX_COUNT: usize = 100;@@ -184,15 +130,15 @@ pub(crate) const BLOB_HASH_REQUEST_MAX_COUNT: usize = 1000;pub(crate) fn batch_by_transfer_cost<T>(items: Vec<T>, cost_of: impl Fn(&T) -> usize) -> Vec<Vec<T>> {let mut batches = Vec::new();let mut batch = Vec::new();- let mut chars = 0usize;+ let mut bytes = 0usize;for item in items {let cost = cost_of(&item);- if !batch.is_empty() && (chars + cost > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.len() >= BLOB_TRANSFER_BATCH_MAX_COUNT) {+ if !batch.is_empty() && (bytes + cost > BLOB_TRANSFER_BATCH_MAX_BYTES || batch.len() >= BLOB_TRANSFER_BATCH_MAX_COUNT) {batches.push(std::mem::take(&mut batch));- chars = 0;+ bytes = 0;}batch.push(item);- chars += cost;+ bytes += cost;}if !batch.is_empty() {batches.push(batch);diff --git a/core/rust/src/vault_sync/mod.rs b/core/rust/src/vault_sync/mod.rsindex 2dbb01ff0..af645676e 100644--- a/core/rust/src/vault_sync/mod.rs+++ b/core/rust/src/vault_sync/mod.rs@@ -5,6 +5,7 @@ pub(crate) mod db;mod email_routing;mod engine;pub(crate) mod errors;+mod frame;mod http;mod keys;mod legacy;diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rsindex 65a1f673d..65e78dc34 100644--- a/core/rust/src/vault_sync/pull.rs+++ b/core/rust/src/vault_sync/pull.rs@@ -6,7 +6,7 @@ use std::collections::HashMap;use crate::vault_model::{id_key, ids_equal};use super::errors::{SyncError, SyncResult};use super::state::{self, Ctx};-use super::types::{self, BlobDownloadEntry, BlobDownloadResponse, BlobHashesRequest, Db, EmailRoutingDto, GetResponse, ManifestDto, SharedManifestDto, StoredBlobRef};+use super::types::{self, BlobDownloadResponse, BlobHashesRequest, Db, EmailRoutingDto, GetResponse, ManifestDto, SharedManifestDto, StoredBlobRef};use super::blob_keys::{self, EncryptedBlob};use super::{db, http, keys, legacy};use crate::crypto;@@ -73,8 +73,7 @@ pub(crate) enum Snapshot {/// `GET v2/Vault`. A snapshot in a storage format newer than this build knows is refused, never read as legacy.pub(crate) async fn fetch_snapshot(ctx: &Ctx) -> SyncResult<Snapshot> {- let body = http::with_outdated_server_guard(http::get_binary(&ctx.host, http::VAULT_ENDPOINT, true).await)?;- let snapshot = decode_snapshot(&body)?;+ let snapshot: GetResponse = http::with_outdated_server_guard(http::get_frame(&ctx.host, http::VAULT_ENDPOINT).await)?;match snapshot.storage_format.as_deref() {None | Some(legacy::STORAGE_FORMAT_SQLITE_BLOB) => Ok(Snapshot::LegacySqliteBlob(snapshot)),Some(STORAGE_FORMAT_MANIFEST) => Ok(Snapshot::Manifest(snapshot)),@@ -82,23 +81,10 @@ pub(crate) async fn fetch_snapshot(ctx: &Ctx) -> SyncResult<Snapshot> {}}-/// Split a binary snapshot body into its header, filling each manifest and bucket with its ciphertext.-pub(crate) fn decode_snapshot(body: &[u8]) -> SyncResult<GetResponse> {- let (mut snapshot, mut frame) = http::open_frame::<GetResponse>(body)?;- for manifest in &mut snapshot.manifests {- manifest.blob = frame.take(manifest.size)?.to_vec();- }- for bucket in &mut snapshot.buckets {- bucket.blob = frame.take(bucket.size)?.to_vec();- }- frame.finish()?;- Ok(snapshot)-}-/// Verify a ciphertext against the server's hash, decrypt it bound to `aad` and open it via the codec.fn verify_decrypt_unpack(ciphertext: &[u8], vek: &str, aad: &[u8], expected_ciphertext_hash: Option<&str>, label: &str) -> SyncResult<String> {if let Some(expected) = expected_ciphertext_hash.filter(|hash| !hash.is_empty()) {- if vault_codec::compute_ciphertext_hash_bytes(ciphertext) != expected {+ if vault_codec::compute_ciphertext_hash(ciphertext) != expected {return Err(SyncError::ServerVaultUnreadable(format!("{} ciphertext hash mismatch, refusing to load (possible storage corruption)", label)));}}@@ -280,12 +266,12 @@ async fn open_data_buckets(ctx: &Ctx, snapshot: &GetResponse, resolved: &[Resolvlet mut fingerprints = HashMap::new();let mut revisions = HashMap::new();for dto in &snapshot.buckets {- if dto.blob.is_empty() {+ if dto.ciphertext.bytes.is_empty() {continue;}let key = key_by_manifest.get(dto.manifest_id.as_str()).ok_or_else(|| SyncError::Snapshot(format!("data bucket \"{}\" belongs to manifest {}, which this vault did not open, refusing to assemble", dto.category, dto.manifest_id)))?;let label = format!("\"{}\" bucket of manifest {}", dto.category, dto.manifest_id);- let bucket_json = verify_decrypt_unpack(&dto.blob, key, &crypto::aad::bucket(&dto.manifest_id, &dto.category), dto.ciphertext_hash.as_deref(), &label)?;+ let bucket_json = verify_decrypt_unpack(&dto.ciphertext.bytes, key, &crypto::aad::bucket(&dto.manifest_id, &dto.category), dto.ciphertext_hash.as_deref(), &label)?;let bucket: DataBucket = serde_json::from_str(&bucket_json)?;ensure_readable(bucket.schema_version, &label)?;if !ids_equal(&bucket.manifest_id, &dto.manifest_id) || bucket.category != dto.category {@@ -328,11 +314,12 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest]) ->}let batch_count = batches.len();for (index, (manifest_id, chunk)) in batches.into_iter().enumerate() {- let body = http::post_binary(&ctx.host, BLOBS_DOWNLOAD_ENDPOINT, &BlobHashesRequest { manifest_id, hashes: chunk.iter().map(|r| r.hash.clone()).collect() }, true).await?;- let blobs = decode_blob_download(&body)?;- ctx.log(format!("[Pull] Downloaded blob batch {}/{}: requested {}, received {} ({} bytes).", index + 1, batch_count, chunk.len(), blobs.len(), body.len())).await;- for (entry, ciphertext) in blobs {- cache.insert(entry.hash, EncryptedBlob { encrypted_data: ciphertext.to_vec(), encrypted_blob_key: entry.encrypted_blob_key });+ let request = BlobHashesRequest { manifest_id, hashes: chunk.iter().map(|r| r.hash.clone()).collect() };+ let blobs = http::post_for_frame::<_, BlobDownloadResponse>(&ctx.host, BLOBS_DOWNLOAD_ENDPOINT, &request).await?.blobs;+ let bytes: usize = blobs.iter().map(|b| b.ciphertext.size).sum();+ ctx.log(format!("[Pull] Downloaded blob batch {}/{}: requested {}, received {} ({} bytes).", index + 1, batch_count, chunk.len(), blobs.len(), bytes)).await;+ for dto in blobs {+ cache.insert(dto.hash, EncryptedBlob { encrypted_data: dto.ciphertext.bytes, encrypted_blob_key: dto.encrypted_blob_key });}}@@ -357,14 +344,6 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest]) ->Ok(blob_map)}-/// Split a binary blob download body into its blobs.-pub(crate) fn decode_blob_download(body: &[u8]) -> SyncResult<Vec<(BlobDownloadEntry, &[u8])>> {- let (header, mut frame) = http::open_frame::<BlobDownloadResponse>(body)?;- let blobs = header.blobs.into_iter().map(|entry| frame.take(entry.size).map(|ciphertext| (entry, ciphertext))).collect::<SyncResult<Vec<_>>>()?;- frame.finish()?;- Ok(blobs)-}-/// Materialize manifests and buckets into a fresh database and return its bytes.pub(crate) async fn materialize_to_sqlite(ctx: &mut Ctx, manifests: &[Manifest], data_buckets: &[DataBucket], blob_map: &HashMap<String, Vec<u8>>, opened_names: &HashMap<String, String>) -> SyncResult<Vec<u8>> {ctx.log(format!("[Pull] {} blobs decrypted; running codec reassembly into a fresh SQLite ({} manifest(s) combined)...", blob_map.len(), manifests.len())).await;@@ -391,7 +370,7 @@ pub(crate) async fn materialize_to_sqlite(ctx: &mut Ctx, manifests: &[Manifest],fn open_manifest(dto: &ManifestDto, vek: &str, is_personal: bool) -> SyncResult<ResolvedManifest> {let label = if is_personal { "manifest".to_string() } else { format!("shared manifest {}", dto.manifest_id) };- let manifest_json = verify_decrypt_unpack(&dto.blob, vek, &crypto::aad::manifest(&dto.manifest_id), dto.ciphertext_hash.as_deref(), &label)?;+ let manifest_json = verify_decrypt_unpack(&dto.ciphertext.bytes, vek, &crypto::aad::manifest(&dto.manifest_id), dto.ciphertext_hash.as_deref(), &label)?;let manifest: Manifest = serde_json::from_str(&manifest_json)?;ensure_readable(manifest.schema_version, &label)?;if !ids_equal(&manifest.manifest_id, &dto.manifest_id) {diff --git a/core/rust/src/vault_sync/push.rs b/core/rust/src/vault_sync/push.rsindex dcd73d559..642942634 100644--- a/core/rust/src/vault_sync/push.rs+++ b/core/rust/src/vault_sync/push.rs@@ -3,12 +3,14 @@use std::collections::{HashMap, HashSet};+use serde::de::IgnoredAny;+use crate::vault_model::names::ITEMS_TABLE;use crate::vault_model::{id_key, OVERFLOW_TABLE, TRASH_RETENTION_DEFAULT_DAYS};use super::email_routing::build_email_routing;use super::errors::{SyncError, SyncResult};use super::state::{self, Ctx};-use super::types::{BlobHashesRequest, BlobUpload, BlobRef, BlobUploadRequest, BucketRevision, BucketWrite, Db, ManifestRevision, ManifestWrite, MissingBlobsResponse, VaultWriteMigration, VaultWriteRequest, VaultWriteResponse, VaultWriteStatus, ALGORITHM_RSA_OAEP_SHA256};+use super::types::{BlobDto, BlobHashesRequest, BlobRef, BlobUploadRequest, BucketRevision, BucketWrite, Db, ManifestRevision, ManifestWrite, MissingBlobsResponse, VaultWriteMigration, VaultWriteRequest, VaultWriteResponse, VaultWriteStatus, ALGORITHM_RSA_OAEP_SHA256};use super::blob_keys::{self, EncryptedBlob};use super::write_set::{self, ManifestRecord, SkipReason};use super::{db, http, keys};@@ -221,8 +223,8 @@ async fn write_dirty_buckets(ctx: &Ctx, categories: &[String]) -> SyncResult<Pusctx.log("[Push] No bucket changed versus the server baselines; skipping upload.").await;return Ok(PushStatus::Ok);}- let payload = VaultWriteRequest { username: ctx.request.username.clone(), manifests: Vec::new(), buckets: writes, email_routing: None, migration: None };- let response: VaultWriteResponse = http::with_outdated_server_guard(http::post_frame(&ctx.host, http::VAULT_ENDPOINT, &payload).await)?;+ let mut payload = VaultWriteRequest { username: ctx.request.username.clone(), manifests: Vec::new(), buckets: writes, email_routing: None, migration: None };+ let response: VaultWriteResponse = http::with_outdated_server_guard(http::post_frame(&ctx.host, http::VAULT_ENDPOINT, &mut payload).await)?;if response.status != VaultWriteStatus::Ok {ctx.warn("[Push] Bucket-only write outdated; pulling and merging before the next attempt.").await;return Ok(PushStatus::Outdated);@@ -406,7 +408,7 @@ async fn encrypt_payload(ctx: &Ctx, label: &str, plaintext: &str, key: &str, aadlet packed = vault_codec::pack_payload(plaintext)?;let ciphertext = crypto::symmetric_encrypt_raw_with_aad(&packed, key, aad)?;ctx.log(format!("[Push] {}: raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(packed.len()), format_kb(ciphertext.len()))).await;- let hash = vault_codec::compute_ciphertext_hash_bytes(&ciphertext);+ let hash = vault_codec::compute_ciphertext_hash(&ciphertext);Ok(EncryptedPayload { ciphertext, hash })}@@ -459,8 +461,8 @@ async fn write_manifests_and_buckets(ctx: &mut Ctx, cached: Option<Canonicalizedlet mut uploaded = upload_missing_blobs(ctx, &blobs, &baselines, overwrite_personal_blobs).await?;let routed_manifests: Vec<_> = canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect();let email_routing = build_email_routing(&routed_manifests, &ctx.request.private_email_domains, &baselines.manifest_revisions);- let payload = VaultWriteRequest { username: ctx.request.username.clone(), manifests: manifest_writes, buckets: bucket_writes, email_routing: Some(email_routing), migration: migration.as_ref().map(|m| VaultWriteMigration { account_keys: Some(m.account_keys.clone()) }) };- let response = write_vault(ctx, &payload, &blobs, overwrite_personal_blobs, &mut uploaded).await?;+ let mut payload = VaultWriteRequest { username: ctx.request.username.clone(), manifests: manifest_writes, buckets: bucket_writes, email_routing: Some(email_routing), migration: migration.as_ref().map(|m| VaultWriteMigration { account_keys: Some(m.account_keys.clone()) }) };+ let response = write_vault(ctx, &mut payload, &blobs, overwrite_personal_blobs, &mut uploaded).await?;if response.status != VaultWriteStatus::Ok {// All-or-nothing: a single stale manifest or bucket rejected the whole write; the caller pulls, merges and retries.@@ -550,7 +552,7 @@ async fn encrypt_changed_buckets(ctx: &Ctx, buckets: &[DataBucket], keys_by_manireturn Err(SyncError::UploadRejected(vec![format!("{} validation failed: {}. {}", label, validation.failed_rules.join(", "), validation.message).trim().to_string()]));}let encrypted = encrypt_payload(ctx, &label, &plaintext, bucket_key, &crypto::aad::bucket(&bucket.manifest_id, &bucket.category)).await?;- writes.push(BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: encrypted.ciphertext, ciphertext_hash: encrypted.hash, current_revision: baselines.bucket_revision(bucket) });+ writes.push(BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), ciphertext: encrypted.ciphertext.into(), ciphertext_hash: encrypted.hash, current_revision: baselines.bucket_revision(bucket) });written.insert(fingerprint_key, fingerprint);}Ok(writes)@@ -597,7 +599,7 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], baselet blob_refs: Vec<BlobRef> = candidate.manifest.referenced_blobs().into_iter().map(|(hash, category)| BlobRef { hash, category }).collect();writes.push(ManifestWrite {manifest_id: candidate.record.manifest_id.clone(),- manifest_blob: encrypted.ciphertext,+ ciphertext: encrypted.ciphertext.into(),manifest_ciphertext_hash: encrypted.hash,current_revision: candidate.current_revision,credentials_count: candidate.manifest.tables.get(ITEMS_TABLE).map(Vec::len).unwrap_or(0),@@ -645,7 +647,7 @@ async fn missing_on_server(ctx: &Ctx, blobs: &UploadBlobs, hashes: &[String]) ->/// `POST v2/Vault`. When the server reports blobs it lacks (stale local knowledge of its blob set), upload/// them and retry the identical write once; blobs this client cannot supply fail the push.-async fn write_vault(ctx: &Ctx, payload: &VaultWriteRequest, blobs: &UploadBlobs, overwrite_personal_blobs: bool, uploaded: &mut HashMap<String, EncryptedBlob>) -> SyncResult<VaultWriteResponse> {+async fn write_vault(ctx: &Ctx, payload: &mut VaultWriteRequest, blobs: &UploadBlobs, overwrite_personal_blobs: bool, uploaded: &mut HashMap<String, EncryptedBlob>) -> SyncResult<VaultWriteResponse> {let response: VaultWriteResponse = http::post_frame(&ctx.host, http::VAULT_ENDPOINT, payload).await?;if response.missing_blob_hashes.is_empty() {return Ok(response);@@ -722,12 +724,12 @@ async fn upload_blobs(ctx: &Ctx, blobs: &UploadBlobs, hashes: &[String], overwrilet entry = &blobs.entries[&hash];let encrypted = blob_keys::encrypt_blob(&entry.bytes, &entry.vek, &manifest_id, &hash)?;encrypted_blobs.insert(hash.clone(), encrypted.clone());- dtos.push(BlobUpload { hash, category: entry.kind.clone(), encrypted_data: encrypted.encrypted_data, encrypted_blob_key: encrypted.encrypted_blob_key });+ dtos.push(BlobDto { hash, category: entry.kind.clone(), ciphertext: encrypted.encrypted_data.into(), encrypted_blob_key: encrypted.encrypted_blob_key });}- for batch in http::batch_by_transfer_cost(dtos, |dto| dto.encrypted_data.len()) {- let bytes: usize = batch.iter().map(|dto| dto.encrypted_data.len()).sum();+ for batch in http::batch_by_transfer_cost(dtos, |dto| dto.ciphertext.size) {+ let bytes: usize = batch.iter().map(|dto| dto.ciphertext.size).sum();ctx.log(format!("[Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(bytes))).await;- http::post_frame_no_content(&ctx.host, BLOBS_ENDPOINT, &BlobUploadRequest { manifest_id: manifest_id.clone(), blobs: batch, overwrite }).await?;+ http::post_frame::<_, IgnoredAny>(&ctx.host, BLOBS_ENDPOINT, &mut BlobUploadRequest { manifest_id: manifest_id.clone(), blobs: batch, overwrite }).await?;}}Ok(encrypted_blobs)diff --git a/core/rust/src/vault_sync/session.rs b/core/rust/src/vault_sync/session.rsindex 7c135e588..fcd960c23 100644--- a/core/rust/src/vault_sync/session.rs+++ b/core/rust/src/vault_sync/session.rs@@ -17,8 +17,10 @@ use crate::common::error::{VaultError, VaultResult};#[derive(Default)]pub(crate) struct Slot {pub pending: Option<Command>,+ /// Raw bytes sent with the pending command (the body of a binary `http` request), until the host takes them.pub pending_bytes: Option<Vec<u8>>,pub response: Option<Value>,+ /// Raw bytes the host handed back with the response (a `dbExport` file or a binary `http` response).pub response_bytes: Option<Vec<u8>>,}@@ -35,16 +37,11 @@ impl Host {/// Send a command and wait for the host's typed response. A `{ "error": ... }` response becomes an error.pub async fn call<R: DeserializeOwned>(&self, command: Command) -> SyncResult<R> {- Ok(self.call_with_bytes(command).await?.0)+ Ok(self.call_with_bytes(command, None).await?.0)}- /// Like [`Host::call`], also returning the raw bytes the host attached to its response, if any.- pub async fn call_with_bytes<R: DeserializeOwned>(&self, command: Command) -> SyncResult<(R, Option<Vec<u8>>)> {- self.exchange(command, None).await- }-- /// Send a command with raw bytes attached for the host, returning the typed response and the bytes the host attached to it.- pub async fn exchange<R: DeserializeOwned>(&self, command: Command, command_bytes: Option<Vec<u8>>) -> SyncResult<(R, Option<Vec<u8>>)> {+ /// Like [`Host::call`], with raw bytes sent alongside the command and those the host attached to its response.+ pub async fn call_with_bytes<R: DeserializeOwned>(&self, command: Command, command_bytes: Option<Vec<u8>>) -> SyncResult<(R, Option<Vec<u8>>)> {let kind = command.name();let (response, bytes) = CommandFuture { slot: self.slot.clone(), command: Some(command), command_bytes, sent: false }.await;if let Some(error) = response.get("error").and_then(Value::as_str) {@@ -77,7 +74,6 @@ impl Future for CommandFuture {let mut slot = this.slot.lock().expect("sync host slot poisoned");slot.pending = this.command.take();slot.pending_bytes = this.command_bytes.take();- drop(slot);this.sent = true;return Poll::Pending;}@@ -98,8 +94,6 @@ struct SessionInner {slot: Arc<Mutex<Slot>>,future: Option<Pin<Box<dyn Future<Output = Value> + Send>>>,finished: Option<Value>,- /// Raw bytes attached to the last command handed out, until the host takes them.- command_bytes: Option<Vec<u8>>,}impl SyncSession {@@ -109,7 +103,7 @@ impl SyncSession {let slot = Arc::new(Mutex::new(Slot::default()));let host = Host::new(slot.clone());let future: Pin<Box<dyn Future<Output = Value> + Send>> = Box::pin(engine::run(host, request));- Ok(Self { inner: Mutex::new(SessionInner { slot, future: Some(future), finished: None, command_bytes: None }) })+ Ok(Self { inner: Mutex::new(SessionInner { slot, future: Some(future), finished: None }) })}/// The next command for the host, as JSON.@@ -128,11 +122,7 @@ impl SyncSession {Ok(serde_json::to_string(&Command::Done { result })?)}Poll::Pending => {- let (pending, pending_bytes) = {- let mut slot = inner.slot.lock().map_err(|_| VaultError::General("sync host slot poisoned".to_string()))?;- (slot.pending.take(), slot.pending_bytes.take())- };- inner.command_bytes = pending_bytes;+ let pending = inner.slot.lock().map_err(|_| VaultError::General("sync host slot poisoned".to_string()))?.pending.take();match pending {Some(command) => Ok(serde_json::to_string(&command)?),None => Err(VaultError::General("sync session is waiting for a response to its last command".to_string())),@@ -143,11 +133,12 @@ impl SyncSession {/// Take the raw bytes attached to the last command (the body of a binary `http` request), if any.pub fn command_bytes(&self) -> VaultResult<Option<Vec<u8>>> {- let mut inner = self.inner.lock().map_err(|_| VaultError::General("sync session poisoned".to_string()))?;- Ok(inner.command_bytes.take())+ let inner = self.inner.lock().map_err(|_| VaultError::General("sync session poisoned".to_string()))?;+ let mut slot = inner.slot.lock().map_err(|_| VaultError::General("sync host slot poisoned".to_string()))?;+ Ok(slot.pending_bytes.take())}- /// Hand the host's response to the last command back, with raw bytes for a `dbExport` (the SQLite file).+ /// Hand the host's response to the last command back, with raw bytes for a `dbExport` or a binary `http` response.pub fn resume(&self, response_json: &str, bytes: Option<Vec<u8>>) -> VaultResult<()> {let response: Value = if response_json.trim().is_empty() {Value::Object(Default::default())diff --git a/core/rust/src/vault_sync/tests/blobs.rs b/core/rust/src/vault_sync/tests/blobs.rsindex ef38509cb..48c43dcf7 100644--- a/core/rust/src/vault_sync/tests/blobs.rs+++ b/core/rust/src/vault_sync/tests/blobs.rs@@ -5,12 +5,11 @@use serde_json::{json, Value};use super::fake_server::FakeServer;-use super::test_host::{decode_upload_frame, encode_blob_download, encode_snapshot, open_schema_db, query, TestHost};+use super::test_host::{open_schema_db, query, TestHost};use super::{insert_attachment, insert_delivery_key, insert_item, insert_logo, rename_item, synced, Synced, ITEM_A, ITEM_B, PERSONAL_MANIFEST_ID};use crate::crypto;use crate::vault_codec;use crate::vault_sync::blob_keys;-use crate::vault_sync::pull::{decode_blob_download, decode_snapshot};/// A server vault holding one item with a logo and an attachment.fn vault_with_blobs(db: &rusqlite::Connection) {@@ -177,56 +176,3 @@ fn an_attachment_row_that_references_no_blob_does_not_fail_the_pull() {assert_eq!(rows.len(), 1);assert_eq!((&rows[0]["Blob"], &rows[0]["BlobHash"]), (&Value::Null, &Value::Null));}--#[test]-fn a_binary_blob_download_splits_into_its_blobs() {- let frame = encode_blob_download(&json!({ "blobs": [- { "hash": "a", "category": "favicon", "encryptedDataBase64": "AQID", "encryptedBlobKey": "ka" },- { "hash": "b", "category": "attachment", "encryptedDataBase64": "", "encryptedBlobKey": "kb" },- { "hash": "c", "category": "attachment", "encryptedDataBase64": "BAUGBw==", "encryptedBlobKey": "kc" },- ] }));- let blobs = decode_blob_download(&frame).unwrap();- let decoded: Vec<(&str, &str, &[u8])> = blobs.iter().map(|(entry, bytes)| (entry.hash.as_str(), entry.encrypted_blob_key.as_str(), *bytes)).collect();- assert_eq!(decoded, vec![("a", "ka", &[1u8, 2, 3][..]), ("b", "kb", &[][..]), ("c", "kc", &[4u8, 5, 6, 7][..])]);- assert!(decode_blob_download(&encode_blob_download(&json!({ "blobs": [] }))).unwrap().is_empty());-}--#[test]-fn a_malformed_binary_blob_download_is_refused() {- let frame = encode_blob_download(&json!({ "blobs": [{ "hash": "a", "category": "favicon", "encryptedDataBase64": "AQID", "encryptedBlobKey": "ka" }] }));- assert!(decode_blob_download(&[]).is_err());- assert!(decode_blob_download(&frame[..frame.len() - 1]).is_err(), "a truncated blob");- assert!(decode_blob_download(&[frame.as_slice(), &[0]].concat()).is_err(), "trailing bytes");- assert!(decode_blob_download(&[0, 0, 1, 0, b'{']).is_err(), "a header longer than the body");-}--#[test]-fn a_binary_snapshot_fills_each_manifest_and_bucket_with_its_ciphertext() {- let frame = encode_snapshot(&json!({- "storageFormat": "manifest",- "manifests": [{ "manifestId": "m1", "blob": "AQID", "revision": 3 }, { "manifestId": "m2", "blob": null, "revision": 0 }],- "buckets": [{ "manifestId": "m1", "category": "settings", "blob": "BAU=", "revision": 1 }],- }));- let snapshot = decode_snapshot(&frame).unwrap();- assert_eq!(snapshot.manifests.iter().map(|m| m.blob.clone()).collect::<Vec<_>>(), vec![vec![1u8, 2, 3], vec![]]);- assert!(!snapshot.manifests[1].has_content());- assert_eq!(snapshot.buckets[0].blob, vec![4u8, 5]);- assert!(decode_snapshot(&frame[..frame.len() - 1]).is_err(), "a truncated bucket");-}--#[test]-fn a_binary_upload_keeps_each_ciphertext_out_of_the_json_header() {- use crate::vault_sync::http::encode_frame;- use crate::vault_sync::types::{BlobUpload, BlobUploadRequest};-- let request = BlobUploadRequest { manifest_id: "m1".to_string(), overwrite: false, blobs: vec![- BlobUpload { hash: "a".to_string(), category: "favicon".to_string(), encrypted_data: vec![1, 2, 3], encrypted_blob_key: "ka".to_string() },- BlobUpload { hash: "b".to_string(), category: "attachment".to_string(), encrypted_data: vec![4], encrypted_blob_key: "kb".to_string() },- ] };- let frame = encode_frame(&request).unwrap();- let header_length = u32::from_be_bytes(frame[..4].try_into().unwrap()) as usize;- let header: Value = serde_json::from_slice(&frame[4..4 + header_length]).unwrap();- assert_eq!(header["blobs"][0], json!({ "hash": "a", "category": "favicon", "size": 3, "encryptedBlobKey": "ka" }));- assert_eq!(&frame[4 + header_length..], &[1, 2, 3, 4]);- assert_eq!(decode_upload_frame("Vault/blobs", &frame)["blobs"][1]["encryptedDataBase64"], json!("BA=="));-}diff --git a/core/rust/src/vault_sync/tests/fake_server.rs b/core/rust/src/vault_sync/tests/fake_server.rsindex e6c8789e6..796c86677 100644--- a/core/rust/src/vault_sync/tests/fake_server.rs+++ b/core/rust/src/vault_sync/tests/fake_server.rs@@ -393,9 +393,9 @@ impl FakeServer {/// A manifest or bucket payload packed and encrypted under `vek`, with its ciphertext hash.fn encrypt_payload(json: &str, vek: &str, aad: &[u8]) -> (String, String) {let packed = vault_codec::pack_payload(json).unwrap();- let blob = crypto::symmetric_encrypt_bytes_with_aad(&packed, vek, aad).unwrap();+ let blob = crypto::symmetric_encrypt_raw_with_aad(&packed, vek, aad).unwrap();let hash = vault_codec::compute_ciphertext_hash(&blob);- (blob, hash)+ (crate::common::encoding::base64_encode(&blob), hash)}/// The `vaultKey` dto of `GET VaultKey/Password` around the account key fields.diff --git a/core/rust/src/vault_sync/tests/test_host.rs b/core/rust/src/vault_sync/tests/test_host.rsindex a75573086..16a64d554 100644--- a/core/rust/src/vault_sync/tests/test_host.rs+++ b/core/rust/src/vault_sync/tests/test_host.rs@@ -203,8 +203,7 @@ impl TestHost {Command::Done { result } => return result,Command::Http { method, path, body, binary_response, binary_body, .. } => {let body_json = if binary_body {- // Upload frames reach the responders as the JSON they stand for, each ciphertext back in its base64 field.- Some(decode_upload_frame(&path, &session.command_bytes().unwrap().expect("a binary body carries bytes")))+ Some(frame_to_json(&path, &session.command_bytes().unwrap().expect("a binary body carries bytes")))} else {body.as_deref().map(|b| serde_json::from_str(b).unwrap_or(Value::String(b.to_string())))};@@ -212,9 +211,7 @@ impl TestHost {self.requests.push(RecordedRequest { method: method.clone(), path: path.clone(), body: body_json.clone() });match self.responders.iter().find_map(|r| r(&method, &path, body_json.as_ref())) {Some((status, body)) if binary_response && (200..300).contains(&status) => {- // Responders script binary downloads as JSON; the host hands them over in the server's binary frame.- let frame = if path == "Vault/blobs/download" { encode_blob_download(&body) } else { encode_snapshot(&body) };- session.resume(&json!({ "status": status }).to_string(), Some(frame)).unwrap();+ session.resume(&json!({ "status": status }).to_string(), Some(json_to_frame(&path, &body))).unwrap();continue;}Some((status, body)) => json!({ "status": status, "body": body.to_string() }),@@ -360,56 +357,52 @@ pub fn exec(conn: &Connection, statements: &[SqlStatement]) -> Result<(), Stringsqlite_host::exec(conn, statements).map_err(|e| e.to_string())}-/// A binary vault download: 4-byte big-endian header length, the JSON header, then the parts.-fn encode_frame(header: &Value, parts: Vec<Vec<u8>>) -> Vec<u8> {- let header = serde_json::to_vec(header).unwrap();- let mut frame = (header.len() as u32).to_be_bytes().to_vec();- frame.extend(header);- frame.extend(parts.into_iter().flatten());- frame-}--/// Take the base64 `blob` out of a JSON entry, leaving its `size`.-fn take_blob(entry: &mut Value, field: &str) -> Vec<u8> {- let bytes = entry.as_object_mut().unwrap().remove(field).and_then(|b| b.as_str().map(str::to_string)).map(|b| crate::common::encoding::base64_decode(&b).unwrap()).unwrap_or_default();- entry["size"] = json!(bytes.len());- bytes+/*+ * Responders and the fake server speak JSON with each ciphertext as a base64 field; the engine sends and reads binary+ * frames. These convert between the two.+ */++/// The base64 fields that travel as frame parts, per list, for a request to or response from `path`.+fn frame_fields(path: &str, upload: bool) -> &'static [(&'static str, &'static str)] {+ match path {+ "Vault/blobs" | "Vault/blobs/download" => &[("blobs", "encryptedDataBase64")],+ _ if upload => &[("manifests", "manifestBlob"), ("buckets", "blob")],+ _ => &[("manifests", "blob"), ("buckets", "blob")],+ }}-/// The server's binary snapshot frame for a `GET v2/Vault` JSON body that carries base64 `blob`s.-pub fn encode_snapshot(body: &Value) -> Vec<u8> {+/// The server's binary frame for a scripted JSON response.+fn json_to_frame(path: &str, body: &Value) -> Vec<u8> {let mut header = body.clone();- let mut parts = Vec::new();- for list in ["manifests", "buckets"] {- for entry in header.get_mut(list).and_then(Value::as_array_mut).map(|a| a.iter_mut().collect::<Vec<_>>()).unwrap_or_default() {- parts.push(take_blob(entry, "blob"));+ let mut data = Vec::new();+ for (list, field) in frame_fields(path, false) {+ for entry in header.get_mut(*list).and_then(Value::as_array_mut).into_iter().flatten() {+ let base64 = entry.as_object_mut().unwrap().remove(*field);+ let bytes = base64.as_ref().and_then(Value::as_str).map(|b| crate::common::encoding::base64_decode(b).unwrap()).unwrap_or_default();+ entry["offset"] = json!(data.len());+ entry["size"] = json!(bytes.len());+ data.extend(bytes);}}- encode_frame(&header, parts)-}--/// The server's binary blob download frame for a `{ "blobs": [{ hash, category, encryptedDataBase64, encryptedBlobKey }] }` JSON body.-pub fn encode_blob_download(body: &Value) -> Vec<u8> {- let mut header = body.clone();- let parts = header.get_mut("blobs").and_then(Value::as_array_mut).map(|a| a.iter_mut().map(|blob| take_blob(blob, "encryptedDataBase64")).collect()).unwrap_or_default();- encode_frame(&header, parts)+ let header = serde_json::to_vec(&header).unwrap();+ [&(header.len() as u32).to_be_bytes()[..], &header, &data].concat()}-/// The JSON a binary upload frame stands for: each listed `size` replaced by its ciphertext as base64 in the named field.-pub fn decode_upload_frame(path: &str, frame: &[u8]) -> Value {+/// The JSON an uploaded binary frame stands for.+fn frame_to_json(path: &str, frame: &[u8]) -> Value {let header_length = u32::from_be_bytes(frame[..4].try_into().unwrap()) as usize;let mut header: Value = serde_json::from_slice(&frame[4..4 + header_length]).unwrap();- let mut data = &frame[4 + header_length..];- let lists: &[(&str, &str)] = if path == "Vault/blobs" { &[("blobs", "encryptedDataBase64")] } else { &[("manifests", "manifestBlob"), ("buckets", "blob")] };- for (list, field) in lists {- for entry in header.get_mut(*list).and_then(Value::as_array_mut).map(|a| a.iter_mut().collect::<Vec<_>>()).unwrap_or_default() {+ let data = &frame[4 + header_length..];+ let mut listed = 0;+ for (list, field) in frame_fields(path, true) {+ for entry in header.get_mut(*list).and_then(Value::as_array_mut).into_iter().flatten() {let entry = entry.as_object_mut().unwrap();- let size = entry.remove("size").and_then(|s| s.as_u64()).expect("each listed ciphertext has a size") as usize;- let (ciphertext, rest) = data.split_at(size);- entry.insert(field.to_string(), json!(crate::common::encoding::base64_encode(ciphertext)));- data = rest;+ let offset = entry.remove("offset").and_then(|o| o.as_u64()).expect("each part has an offset") as usize;+ let size = entry.remove("size").and_then(|s| s.as_u64()).expect("each part has a size") as usize;+ entry.insert(field.to_string(), json!(crate::common::encoding::base64_encode(&data[offset..offset + size])));+ listed += size;}}- assert!(data.is_empty(), "the frame carries only the ciphertexts its header lists");+ assert_eq!(listed, data.len(), "the client sends only the ciphertexts its header lists");header}diff --git a/core/rust/src/vault_sync/types.rs b/core/rust/src/vault_sync/types.rsindex 019ba7975..56c2bdbb3 100644--- a/core/rust/src/vault_sync/types.rs+++ b/core/rust/src/vault_sync/types.rs@@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize};use serde_json::{Map, Value};use super::errors::{ErrorCode, Failure, LogoutReason, SyncError};+use super::frame::{FrameBody, FramePart};use crate::sqlite_host::SqlStatement;/// The operation a session runs.@@ -458,16 +459,19 @@ pub struct GetResponse {pub email_routing: Option<EmailRoutingDto>,}+impl FrameBody for GetResponse {+ fn parts(&mut self) -> Vec<&mut FramePart> {+ self.manifests.iter_mut().map(|m| &mut m.ciphertext).chain(self.buckets.iter_mut().map(|b| &mut b.ciphertext)).collect()+ }+}+#[derive(Debug, Clone, Default, Deserialize)]#[serde(rename_all = "camelCase")]pub struct ManifestDto {pub manifest_id: String,- /// Length of the manifest ciphertext in the binary snapshot body, 0 on an empty vault.- #[serde(default)]- pub size: usize,- /// The manifest ciphertext, read from the binary snapshot body after the JSON header.- #[serde(skip)]- pub blob: Vec<u8>,+ /// The manifest ciphertext, empty on an empty vault.+ #[serde(flatten)]+ pub ciphertext: FramePart,#[serde(default)]pub ciphertext_hash: Option<String>,#[serde(default)]@@ -497,7 +501,7 @@ pub struct ManifestDto {impl ManifestDto {/// Whether the server served this manifest with content.pub fn has_content(&self) -> bool {- !self.blob.is_empty()+ !self.ciphertext.bytes.is_empty()}}@@ -506,12 +510,8 @@ impl ManifestDto {pub struct BucketDto {pub manifest_id: String,pub category: String,- /// Length of the bucket ciphertext in the binary snapshot body.- #[serde(default)]- pub size: usize,- /// The bucket ciphertext, read from the binary snapshot body after the JSON header.- #[serde(skip)]- pub blob: Vec<u8>,+ #[serde(flatten)]+ pub ciphertext: FramePart,#[serde(default)]pub ciphertext_hash: Option<String>,#[serde(default)]@@ -550,9 +550,8 @@ pub const ALGORITHM_AES256_GCM: &str = crate::crypto::key_chain::ACCOUNT_KEY_WRA#[serde(rename_all = "camelCase")]pub struct ManifestWrite {pub manifest_id: String,- /// The manifest ciphertext; the JSON header carries its length as `size`, the bytes follow the header.- #[serde(rename = "size", serialize_with = "serialize_len")]- pub manifest_blob: Vec<u8>,+ #[serde(flatten)]+ pub ciphertext: FramePart,pub manifest_ciphertext_hash: String,pub current_revision: i64,pub credentials_count: usize,@@ -577,30 +576,23 @@ pub struct BlobRef {pub struct BucketWrite {pub manifest_id: String,pub category: String,- /// The bucket ciphertext; the JSON header carries its length as `size`, the bytes follow the header.- #[serde(rename = "size", serialize_with = "serialize_len")]- pub blob: Vec<u8>,+ #[serde(flatten)]+ pub ciphertext: FramePart,pub ciphertext_hash: String,pub current_revision: i64,}-/// One blob of a binary `POST v2/Vault/blobs` upload.-#[derive(Debug, Clone, Serialize)]+/// One blob of a `POST v2/Vault/blobs` upload or `POST v2/Vault/blobs/download` response.+#[derive(Debug, Clone, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]-pub struct BlobUpload {+pub struct BlobDto {pub hash: String,pub category: String,- /// The blob ciphertext; the JSON header carries its length as `size`, the bytes follow the header.- #[serde(rename = "size", serialize_with = "serialize_len")]- pub encrypted_data: Vec<u8>,+ #[serde(flatten)]+ pub ciphertext: FramePart,pub encrypted_blob_key: String,}-/// Serialize a ciphertext field as its length: the bytes travel after the JSON header of a binary frame.-fn serialize_len<S: serde::Serializer>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error> {- serializer.serialize_u64(bytes.len() as u64)-}-#[derive(Debug, Clone, Serialize)]#[serde(rename_all = "camelCase")]pub struct ClaimedEmailAddress {@@ -629,9 +621,9 @@ pub struct VaultWriteRequest {pub migration: Option<VaultWriteMigration>,}-impl super::http::FrameBody for VaultWriteRequest {- fn parts(&self) -> Vec<&[u8]> {- self.manifests.iter().map(|m| m.manifest_blob.as_slice()).chain(self.buckets.iter().map(|b| b.blob.as_slice())).collect()+impl FrameBody for VaultWriteRequest {+ fn parts(&mut self) -> Vec<&mut FramePart> {+ self.manifests.iter_mut().map(|m| &mut m.ciphertext).chain(self.buckets.iter_mut().map(|b| &mut b.ciphertext)).collect()}}@@ -673,13 +665,13 @@ pub enum VaultWriteStatus {#[serde(rename_all = "camelCase")]pub struct BlobUploadRequest {pub manifest_id: String,- pub blobs: Vec<BlobUpload>,+ pub blobs: Vec<BlobDto>,pub overwrite: bool,}-impl super::http::FrameBody for BlobUploadRequest {- fn parts(&self) -> Vec<&[u8]> {- self.blobs.iter().map(|b| b.encrypted_data.as_slice()).collect()+impl FrameBody for BlobUploadRequest {+ fn parts(&mut self) -> Vec<&mut FramePart> {+ self.blobs.iter_mut().map(|b| &mut b.ciphertext).collect()}}@@ -697,21 +689,17 @@ pub struct MissingBlobsResponse {pub missing: Vec<String>,}-/// JSON header of the binary `POST v2/Vault/blobs/download` response.#[derive(Debug, Clone, Default, Deserialize)]#[serde(rename_all = "camelCase")]pub struct BlobDownloadResponse {#[serde(default)]- pub blobs: Vec<BlobDownloadEntry>,+ pub blobs: Vec<BlobDto>,}-/// One blob in the JSON header of the binary `POST v2/Vault/blobs/download` response; its ciphertext follows the header.-#[derive(Debug, Clone, Deserialize)]-#[serde(rename_all = "camelCase")]-pub struct BlobDownloadEntry {- pub hash: String,- pub encrypted_blob_key: String,- pub size: usize,+impl FrameBody for BlobDownloadResponse {+ fn parts(&mut self) -> Vec<&mut FramePart> {+ self.blobs.iter_mut().map(|b| &mut b.ciphertext).collect()+ }}/// `GET v2/VaultKey/Password`.