Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update VaultBlobObject sync logic (#2159)
Leendert de Borst7cc09a3f5e14
diff --git a/apps/browser-extension/src/utils/VaultSyncService.ts b/apps/browser-extension/src/utils/VaultSyncService.tsindex 316874209..1667027c0 100644--- a/apps/browser-extension/src/utils/VaultSyncService.ts+++ b/apps/browser-extension/src/utils/VaultSyncService.ts@@ -113,10 +113,45 @@ const fingerprintManifestKey = (manifestId: string): string => `manifest:${manif/** Fingerprint record key for a data bucket, addressed by the manifest that owns it. */const fingerprintBucketKey = (manifestId: string, category: string): string => `bucket:${manifestId}:${category}`;-/** Max accumulated base64 ciphertext characters per POST /v2/Vault/blobs call (~4 MB request body). */-const BLOB_UPLOAD_BATCH_MAX_CHARS = 4 * 1024 * 1024;-/** Hashes per POST /v2/Vault/blobs/download call. */-const BLOB_DOWNLOAD_BATCH_SIZE = 100;+/** Max amount of bytes that can be transferred in a single blob transfer request or response body. */+const BLOB_TRANSFER_BATCH_MAX_CHARS = 4 * 1024 * 1024;+/** Upper bound on the number of blobs in one transfer batch. */+const BLOB_TRANSFER_BATCH_MAX_COUNT = 100;++/** The number of base64 characters a payload of `sizeBytes` raw bytes occupies on the wire. */+function base64Chars(sizeBytes: number): number {+ return Math.ceil(sizeBytes / 3) * 4;+}++/**+ * Split items into request batches bounded by both {@link BLOB_TRANSFER_BATCH_MAX_CHARS} and+ * {@link BLOB_TRANSFER_BATCH_MAX_COUNT}. An item larger than the byte budget itself is transferred on its own as+ * we can't split it across requests.+ * @param items - the items to batch+ * @param costOf - the base64 character cost one item adds to the request or response body+ */+function batchByTransferCost<T>(items: T[], costOf: (item: T) => number): T[][] {+ const batches: T[][] = [];+ let batch: T[] = [];+ let chars = 0;+ for (const item of items) {+ const cost = costOf(item);+ if (batch.length > 0 && (chars + cost > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.length >= BLOB_TRANSFER_BATCH_MAX_COUNT)) {+ batches.push(batch);+ batch = [];+ chars = 0;+ }++ batch.push(item);+ chars += cost;+ }++ if (batch.length > 0) {+ batches.push(batch);+ }++ return batches;+}/*** Result of a pull: the materialized SQLite plus its manifest revision.@@ -138,6 +173,7 @@ export type PushResult = {};type BlobRefDto = { hash: string; category: string };+type StoredBlobRefDto = BlobRefDto & { sizeBytes: number };type BlobDto = { hash: string; category: string; encryptedDataBase64: string };/** A plaintext blob staged for upload: its bytes plus the key that encrypts it. */@@ -153,7 +189,7 @@ type ResolvedManifest = {/** The key that decrypts this manifest and every blob it references. */vek: string;revision: number;- blobReferences: BlobRefDto[];+ blobReferences: StoredBlobRefDto[];/** Fingerprint of the plaintext exactly as the server served it: the push-side change-detection baseline. */contentFingerprint: string;};@@ -166,7 +202,7 @@ type ManifestDto = {blob?: string | null;ciphertextHash?: string | null;revision: number;- blobReferences?: BlobRefDto[];+ blobReferences?: StoredBlobRefDto[];/** Whether the caller may grant/revoke access to this manifest and publish its email delivery key. */canAdminister?: boolean;/** How the caller's access to this manifest's VEK is encrypted. */@@ -548,7 +584,7 @@ export class VaultSyncService {// Fetch any referenced blobs that aren't already in the local (encrypted) cache.const refOwners = new Map<string, ResolvedManifest>();- const refs: BlobRefDto[] = [];+ const refs: StoredBlobRefDto[] = [];for (const entry of resolved) {for (const r of entry.blobReferences) {if (refOwners.has(r.hash)) {@@ -559,12 +595,15 @@ export class VaultSyncService {}}const cache = await this.loadBlobCache();- const missingHashes = refs.map(r => r.hash).filter(h => !(h in cache));- devLog(`[V2Pull] Blob refs: ${refs.length} referenced, ${refs.length - missingHashes.length} cached locally, ${missingHashes.length} to download.`);- for (let i = 0; i < missingHashes.length; i += BLOB_DOWNLOAD_BATCH_SIZE) {- const chunk = missingHashes.slice(i, i + BLOB_DOWNLOAD_BATCH_SIZE);- const blobs = await webApi.post<{ hashes: string[] }, BlobDto[]>(BLOBS_DOWNLOAD_ENDPOINT, { hashes: chunk });- devLog(`[V2Pull] Downloaded blob batch ${Math.floor(i / BLOB_DOWNLOAD_BATCH_SIZE) + 1}: requested ${chunk.length}, received ${blobs.length}.`);+ const missingRefs = refs.filter(r => !(r.hash in cache));+ devLog(`[V2Pull] Blob refs: ${refs.length} referenced, ${refs.length - missingRefs.length} cached locally, ${missingRefs.length} to download.`);++ // Batch downloads by the bytes each blob adds to the response, not by hash count.+ const downloadBatches = batchByTransferCost(missingRefs, r => base64Chars(r.sizeBytes));+ for (const [index, chunk] of downloadBatches.entries()) {+ const chunkChars = chunk.reduce((sum, r) => sum + base64Chars(r.sizeBytes), 0);+ const blobs = await webApi.post<{ hashes: string[] }, BlobDto[]>(BLOBS_DOWNLOAD_ENDPOINT, { hashes: chunk.map(r => r.hash) });+ devLog(`[V2Pull] Downloaded blob batch ${index + 1}/${downloadBatches.length}: requested ${chunk.length} (${formatKb(chunkChars)}), received ${blobs.length}.`);for (const dto of blobs) {cache[dto.hash] = dto.encryptedDataBase64;}@@ -1477,7 +1516,8 @@ export class VaultSyncService {ciphertexts.set(hash, ciphertext);devLog(`[V2Push] Blob ${hash.substring(0, 12)}… (${entry.kind}): raw ${formatKb(entry.bytes.length)} → encrypted ${formatKb(ciphertext.length)}.`);- if (batch.length > 0 && batchChars + ciphertext.length > BLOB_UPLOAD_BATCH_MAX_CHARS) {+ // Flush before adding when this blob would take the request past either bound.+ if (batch.length > 0 && (batchChars + ciphertext.length > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.length >= BLOB_TRANSFER_BATCH_MAX_COUNT)) {devLog(`[V2Push] Uploading blob batch: ${batch.length} blobs, ${formatKb(batchChars)}.`);await webApi.post(BLOBS_VAULT_ENDPOINT, { blobs: batch, overwrite });batch = [];@@ -1497,32 +1537,21 @@ export class VaultSyncService {}/**- * Encrypt raw blob bytes with the VEK. Result is base64-of-(IV ‖ ciphertext ‖ tag), which is what- * symmetricEncrypt produces. Bytes round-trip through symmetricEncrypt's string interface via latin-1.+ * Encrypt raw blob bytes with the VEK. Result is base64-of-(IV | ciphertext | tag).* @param bytes - plaintext bytes* @param vek - symmetric encryption key*/private async encryptBlobBytes(bytes: Uint8Array, vek: string): Promise<string> {- let s = '';- for (let i = 0; i < bytes.length; i++) {- s += String.fromCharCode(bytes[i]);- }-- return EncryptionUtility.symmetricEncrypt(s, vek);+ return EncryptionUtility.symmetricEncryptBytes(bytes, vek);}/**- * Decrypt a blob ciphertext (base64) and return raw plaintext bytes.- * @param encryptedDataBase64 - base64 IV‖ciphertext‖tag from the server+ * Decrypt a blob ciphertext (base64) and return raw plaintext bytes. Counterpart of {@link encryptBlobBytes}.+ * @param encryptedDataBase64 - base64 IV | ciphertext | tag from the server* @param vek - symmetric encryption key*/private async decryptBlobToBytes(encryptedDataBase64: string, vek: string): Promise<Uint8Array> {- const plaintextLatin1 = await EncryptionUtility.symmetricDecrypt(encryptedDataBase64, vek);- const out = new Uint8Array(plaintextLatin1.length);- for (let i = 0; i < plaintextLatin1.length; i++) {- out[i] = plaintextLatin1.charCodeAt(i) & 0xff;- }- return out;+ return EncryptionUtility.symmetricDecryptBytes(base64ToBytes(encryptedDataBase64), vek);}/**diff --git a/apps/server/AliasVault.Api/Controllers/V2/VaultController.cs b/apps/server/AliasVault.Api/Controllers/V2/VaultController.csindex ad8e11e46..336e15427 100644--- a/apps/server/AliasVault.Api/Controllers/V2/VaultController.cs+++ b/apps/server/AliasVault.Api/Controllers/V2/VaultController.cs@@ -129,11 +129,11 @@ public class VaultController(var currentRevisionByManifest = latestManifests.ToDictionary(m => m.ManifestId, m => m.RevisionNumber);var refsByManifest = (await context.VaultBlobReferences.Where(r => manifestIds.Contains(r.ManifestId))- .Join(context.VaultBlobObjects, r => r.BlobHash, b => b.Hash, (r, b) => new { r.ManifestId, r.RevisionNumber, b.Hash, b.Category })+ .Join(context.VaultBlobObjects, r => r.BlobHash, b => b.Hash, (r, b) => new { r.ManifestId, r.RevisionNumber, b.Hash, b.Category, b.SizeBytes }).ToListAsync()).Where(x => currentRevisionByManifest.TryGetValue(x.ManifestId, out var rev) && rev == x.RevisionNumber).GroupBy(x => x.ManifestId)- .ToDictionary(g => g.Key, g => g.GroupBy(x => x.Hash, StringComparer.Ordinal).Select(h => new BlobReference { Hash = h.Key, Category = h.First().Category }).ToList());+ .ToDictionary(g => g.Key, g => g.GroupBy(x => x.Hash, StringComparer.Ordinal).Select(h => new BlobReference { Hash = h.Key, Category = h.First().Category, SizeBytes = h.First().SizeBytes }).ToList());var accessKeysByManifest = await GetAccessKeysAsync(context, user.Id, manifestIds);var encryptionPublicKeys = await GetEncryptionPublicKeysAsync(context, accessKeysByManifest.Values.Where(g => g.UserGrantKeyId != null).Select(g => g.UserGrantKeyId!.Value));@@ -197,10 +197,10 @@ public class VaultController(var blobRefs = (await context.VaultBlobReferences.Where(r => r.ManifestId == latest.ManifestId && r.RevisionNumber == latest.RevisionNumber)- .Join(context.VaultBlobObjects, r => r.BlobHash, b => b.Hash, (r, b) => new { b.Hash, b.Category })+ .Join(context.VaultBlobObjects, r => r.BlobHash, b => b.Hash, (r, b) => new { b.Hash, b.Category, b.SizeBytes }).Distinct().ToListAsync())- .Select(x => new BlobReference { Hash = x.Hash, Category = x.Category })+ .Select(x => new BlobReference { Hash = x.Hash, Category = x.Category, SizeBytes = x.SizeBytes }).ToList();var manifest = new Manifestdiff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobReference.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobReference.csindex 26f3b659d..8c55ddea8 100644--- a/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobReference.cs+++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/V2/Vault/BlobReference.cs@@ -17,4 +17,10 @@ public class BlobReference/// <summary>Gets or sets the blob category (e.g. "favicon" or "attachment").</summary>public required string Category { get; set; }++ /// <summary>+ /// Gets or sets the size of the encrypted payload in bytes. Server-populated on read so a client can size its+ /// download batches by bytes instead of by count; ignored on write (the server measures the bytes it stores).+ /// </summary>+ public int SizeBytes { get; set; }}