Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Add VaultSyncService.cs (#2159)

Leendert de Borst954936875933
diff --git a/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncService.cs b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncService.cs
new file mode 100644
index 000000000..faffe3550
--- /dev/null
+++ b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncService.cs
@@ -0,0 +1,679 @@
+//-----------------------------------------------------------------------
+// <copyright file="VaultSyncService.cs" company="aliasvault">
+// Copyright (c) aliasvault. All rights reserved.
+// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
+// </copyright>
+//-----------------------------------------------------------------------
+
+namespace AliasVault.Client.Services.VaultSync;
+
+using System.Net.Http.Json;
+using System.Text;
+using System.Text.Json;
+using AliasVault.Client.Services.Auth;
+using AliasVault.Client.Services.Database;
+using AliasVault.Client.Services.JsInterop;
+using AliasVault.Client.Services.JsInterop.RustCore;
+using AliasVault.Client.Services.VaultSync.Exceptions;
+using AliasVault.Client.Services.VaultSync.Models;
+using AliasVault.Shared.Models.Enums;
+using AliasVault.Shared.Models.WebApi.V2.Vault;
+using Microsoft.Data.Sqlite;
+using Microsoft.JSInterop;
+
+/// <summary>
+/// Syncs the vault with the server.
+/// </summary>
+/// <param name="httpClient">The HTTP client.</param>
+/// <param name="authService">AuthService instance, the session's key holder.</param>
+/// <param name="vaultKeyService">VaultKeyService instance.</param>
+/// <param name="jsInteropService">JsInteropService instance.</param>
+/// <param name="rustCoreService">RustCoreService instance.</param>
+/// <param name="state">The sync state this pull adopts the snapshot into.</param>
+/// <param name="logger">ILogger instance.</param>
+public sealed class VaultSyncService(HttpClient httpClient, AuthService authService, VaultKeyService vaultKeyService, JsInteropService jsInteropService, RustCoreService rustCoreService, VaultSyncState state, ILogger<VaultSyncService> logger)
+{
+ private const string VaultEndpoint = "v2/Vault";
+ private const string BlobsDownloadEndpoint = "v2/Vault/blobs/download";
+ private const string AttachmentBlobCategory = "attachment";
+
+ /// <summary>
+ /// Max amount of base64 characters transferred in a single blob transfer request or response body.
+ /// </summary>
+ private const int BlobTransferBatchMaxChars = 4 * 1024 * 1024;
+
+ /// <summary>
+ /// Upper bound on the number of blobs in one transfer batch.
+ /// </summary>
+ private const int BlobTransferBatchMaxCount = 100;
+
+ /// <summary>
+ /// Gets the sync state adopted from the last pull.
+ /// </summary>
+ public VaultSyncState State => state;
+
+ /// <summary>
+ /// Retrieve the latest vault from the server
+ /// </summary>
+ /// <returns>The pull result.</returns>
+ /// <exception cref="VaultProcessingException">Thrown when the snapshot was fetched but could not be opened or materialized.</exception>
+ public async Task<PullResult> PullAsync()
+ {
+ /*
+ * Step 1: network fetch. Failures here (server unreachable, HTTP error) are "can't reach the server" conditions,
+ * not vault-processing problems, so they propagate unchanged.
+ */
+ logger.LogInformation("[V2Pull] Step 1/3: fetching vault snapshot (GET /v2/Vault)...");
+ var snapshot = await FetchSnapshotAsync();
+ state.LastServedManifestIds.Clear();
+ state.LastServedManifestIds.AddRange(snapshot.Manifests.Select(m => m.ManifestId));
+ state.LastSnapshotWasLegacySqliteBlob = snapshot.StorageFormat != StorageFormat.Manifest;
+
+ // LEGACY: a not-yet-migrated account's blob is passed through unchanged.
+ if (state.LastSnapshotWasLegacySqliteBlob)
+ {
+ return OpenLegacySqliteBlobSnapshot(snapshot);
+ }
+
+ var personalDto = SelectPersonalManifest(snapshot) ?? throw new VaultProcessingException("vault-pull", new InvalidOperationException("The server returned no personal manifest, refusing to assemble."));
+ if (string.IsNullOrEmpty(personalDto.Blob))
+ {
+ // The personal manifest exists but was never written (fresh account): the client creates the vault.
+ state.PersonalManifestId = personalDto.ManifestId;
+ logger.LogInformation("[V2Pull] Personal manifest {ManifestId} has no content yet; a new vault will be created.", personalDto.ManifestId);
+ return new PullResult { Kind = PullKind.Empty, Revision = personalDto.Revision };
+ }
+
+ logger.LogInformation("[V2Pull] Step 1/3 done: manifests={ManifestCount}, personalRevision={Revision}, buckets={BucketCount}, blobRefs={BlobRefCount}.", snapshot.Manifests.Count, personalDto.Revision, snapshot.Buckets.Count, personalDto.BlobReferences.Count);
+
+ /*
+ * Steps 2 and 3: decrypt, download blobs, materialize. Any failure here is a client-side vault-processing
+ * error (codec/format mismatch, integrity failure, corrupt blob) and is wrapped so the UI can surface the real
+ * technical detail.
+ */
+ try
+ {
+ logger.LogInformation("[V2Pull] Step 2/3: decrypting manifests, buckets and blobs...");
+ var opened = await OpenManifestsAndAdoptSyncStateAsync(snapshot, personalDto);
+ var blobs = await DownloadReferencedBlobsAsync(opened);
+
+ logger.LogInformation("[V2Pull] Step 3/3: materializing {ManifestCount} manifest(s) into a fresh SQLite database...", opened.Resolved.Count);
+ var database = await MaterializeAsync(opened, blobs);
+ return new PullResult { Kind = PullKind.Materialized, Database = database, Revision = opened.PersonalRevision };
+ }
+ catch (Exception ex)
+ {
+ logger.LogError(ex, "[V2Pull] FAILED: the last logged step above is where it broke.");
+ throw new VaultProcessingException("vault-pull", ex);
+ }
+ }
+
+ /// <summary>
+ /// Pick the user's personal manifest out of a snapshot.
+ /// </summary>
+ /// <param name="snapshot">The raw snapshot.</param>
+ /// <returns>The personal manifest, or null.</returns>
+ private static Manifest? SelectPersonalManifest(GetResponse snapshot)
+ {
+ return snapshot.PersonalManifestId is { } personalId ? snapshot.Manifests.FirstOrDefault(m => m.ManifestId == personalId) : null;
+ }
+
+ /// <summary>
+ /// The grant a shared manifest is remembered by, or null when the snapshot carries none.
+ /// </summary>
+ /// <param name="dto">The snapshot manifest.</param>
+ /// <returns>Tuple with the sealed VEK, the public key it was sealed with and the algorithm token.</returns>
+ private static (string EncryptedVek, string EncryptionPublicKey, string Algorithm)? GrantOf(Manifest dto)
+ {
+ if (string.IsNullOrEmpty(dto.EncryptedVek) || string.IsNullOrEmpty(dto.EncryptionPublicKey))
+ {
+ return null;
+ }
+
+ return (dto.EncryptedVek, dto.EncryptionPublicKey, dto.Algorithm ?? VaultKeyAlgorithms.ToToken(VaultKeyAlgorithm.RsaOaepSha256));
+ }
+
+ /// <summary>
+ /// The number of base64 characters a payload of the given raw size occupies on the wire.
+ /// </summary>
+ /// <param name="sizeBytes">Raw byte count.</param>
+ /// <returns>Base64 character count.</returns>
+ private static int Base64Chars(int sizeBytes)
+ {
+ return (int)Math.Ceiling(sizeBytes / 3.0) * 4;
+ }
+
+ /// <summary>
+ /// Split items into request batches bounded by both the byte budget and the item count. An item larger than the
+ /// byte budget itself is transferred on its own.
+ /// </summary>
+ /// <typeparam name="T">Item type.</typeparam>
+ /// <param name="items">The items to batch.</param>
+ /// <param name="costOf">The base64 character cost one item adds to the body.</param>
+ /// <returns>The batches.</returns>
+ private static List<List<T>> BatchByTransferCost<T>(IEnumerable<T> items, Func<T, int> costOf)
+ {
+ var batches = new List<List<T>>();
+ var batch = new List<T>();
+ var chars = 0;
+ foreach (var item in items)
+ {
+ var cost = costOf(item);
+ if (batch.Count > 0 && (chars + cost > BlobTransferBatchMaxChars || batch.Count >= BlobTransferBatchMaxCount))
+ {
+ batches.Add(batch);
+ batch = [];
+ chars = 0;
+ }
+
+ batch.Add(item);
+ chars += cost;
+ }
+
+ if (batch.Count > 0)
+ {
+ batches.Add(batch);
+ }
+
+ return batches;
+ }
+
+ /// <summary>
+ /// Read the addressing fields off a decrypted manifest or bucket payload without materializing its rows.
+ /// </summary>
+ /// <param name="payloadJson">The payload JSON.</param>
+ /// <returns>The manifest id the payload declares, its salt, its name, its category and a row count summary per table.</returns>
+ private static (Guid? ManifestId, string? Salt, string? Name, string? Category, string TableSummary) ReadPayloadHeader(string payloadJson)
+ {
+ using var document = JsonDocument.Parse(payloadJson);
+ var root = document.RootElement;
+ Guid? manifestId = root.TryGetProperty("manifestId", out var idElement) && idElement.ValueKind == JsonValueKind.String && Guid.TryParse(idElement.GetString(), out var parsed) ? parsed : null;
+ var salt = root.TryGetProperty("manifestSalt", out var saltElement) && saltElement.ValueKind == JsonValueKind.String ? saltElement.GetString() : null;
+ var name = root.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null;
+ var category = root.TryGetProperty("category", out var categoryElement) && categoryElement.ValueKind == JsonValueKind.String ? categoryElement.GetString() : null;
+
+ var summary = new StringBuilder();
+ if (root.TryGetProperty("tables", out var tables) && tables.ValueKind == JsonValueKind.Object)
+ {
+ foreach (var table in tables.EnumerateObject())
+ {
+ summary.Append(summary.Length > 0 ? ", " : string.Empty).Append(table.Name).Append('=').Append(table.Value.ValueKind == JsonValueKind.Array ? table.Value.GetArrayLength() : 0);
+ }
+ }
+
+ return (manifestId, salt, name, category, summary.ToString());
+ }
+
+ /// <summary>
+ /// Build the JSON input of the Rust materialize call from the raw payload strings.
+ /// </summary>
+ /// <param name="manifestJsons">The decrypted manifests, the caller's own first.</param>
+ /// <param name="bucketJsons">The decrypted data buckets.</param>
+ /// <param name="schemaColumns">The column set of the local schema, per table.</param>
+ /// <returns>The MaterializeInput JSON.</returns>
+ private static string BuildMaterializeInput(IEnumerable<string> manifestJsons, IEnumerable<string> bucketJsons, IReadOnlyDictionary<string, List<string>> schemaColumns)
+ {
+ using var stream = new MemoryStream();
+ using (var writer = new Utf8JsonWriter(stream))
+ {
+ writer.WriteStartObject();
+ writer.WriteStartArray("manifests");
+ foreach (var manifestJson in manifestJsons)
+ {
+ writer.WriteRawValue(manifestJson, skipInputValidation: true);
+ }
+
+ writer.WriteEndArray();
+ writer.WriteStartArray("dataBuckets");
+ foreach (var bucketJson in bucketJsons)
+ {
+ writer.WriteRawValue(bucketJson, skipInputValidation: true);
+ }
+
+ writer.WriteEndArray();
+ writer.WriteStartObject("schemaColumns");
+ foreach (var (table, columns) in schemaColumns)
+ {
+ writer.WriteStartArray(table);
+ foreach (var column in columns)
+ {
+ writer.WriteStringValue(column);
+ }
+
+ writer.WriteEndArray();
+ }
+
+ writer.WriteEndObject();
+ writer.WriteEndObject();
+ }
+
+ return Encoding.UTF8.GetString(stream.ToArray());
+ }
+
+ /// <summary>
+ /// Fetch the raw snapshot (GET /v2/Vault).
+ /// </summary>
+ /// <returns>The snapshot.</returns>
+ private async Task<GetResponse> FetchSnapshotAsync()
+ {
+ return await httpClient.GetFromJsonAsync<GetResponse>(VaultEndpoint) ?? throw new InvalidOperationException("Empty vault snapshot response.");
+ }
+
+ /// <summary>
+ /// Take a legacy snapshot apart: the blob passes through untouched. TODO: remove when all accounts have been migrated to the new manifest-v1 format.
+ /// </summary>
+ /// <param name="snapshot">The raw snapshot.</param>
+ /// <returns>The pass-through result.</returns>
+ private PullResult OpenLegacySqliteBlobSnapshot(GetResponse snapshot)
+ {
+ // Clear any manifest-v1 state from the sync state in case there is any.
+ state.ContentFingerprints.Clear();
+ state.SharedManifests.Clear();
+ state.BucketRevisions.Clear();
+
+ var revision = snapshot.LegacyRevision ?? 0;
+ state.ManifestRevisions.Clear();
+ if (snapshot.PersonalManifestId is { } personalId)
+ {
+ // A legacy vault is one logical manifest; seed the per-manifest baseline with the server's reserved id.
+ state.PersonalManifestId = personalId;
+ state.ManifestRevisions[personalId] = revision;
+ }
+
+ logger.LogInformation("[V2Pull] Legacy sqlite-blob pass-through (account not yet migrated), returning the blob as-is.");
+ return new PullResult { Kind = PullKind.LegacySqliteBlob, LegacyVaultBlob = snapshot.LegacyVaultBlob ?? string.Empty, LegacyVersion = snapshot.Version ?? string.Empty, Revision = revision };
+ }
+
+ /// <summary>
+ /// Open every manifest a snapshot carries (the personal one plus each shared one) and adopt the snapshot as this device's sync state.
+ /// </summary>
+ /// <param name="snapshot">The raw snapshot.</param>
+ /// <param name="personalDto">The caller's own manifest.</param>
+ /// <returns>The opened set.</returns>
+ private async Task<OpenedManifestSet> OpenManifestsAndAdoptSyncStateAsync(GetResponse snapshot, Manifest personalDto)
+ {
+ var personalVek = authService.GetEncryptionKeyAsBase64Async();
+ var resolved = new List<ResolvedManifest>();
+ var sharedRecords = new Dictionary<Guid, SharedManifestRecord>();
+ var contentlessRevisions = new Dictionary<Guid, long>();
+ var fingerprints = new Dictionary<string, string>(StringComparer.Ordinal);
+
+ // 1) Open every manifest through one path, personal manifest first.
+ var ordered = new List<Manifest> { personalDto };
+ ordered.AddRange(snapshot.Manifests.Where(m => m.ManifestId != personalDto.ManifestId));
+ foreach (var dto in ordered)
+ {
+ var isPersonal = dto.ManifestId == personalDto.ManifestId;
+ var manifestKey = await ResolveManifestVekAsync(dto, personalVek, isPersonal, resolved.Count > 0 ? resolved[0] : null);
+ if (string.IsNullOrEmpty(dto.Blob))
+ {
+ // A shared manifest served without content yet (created but never written); its grant and revision are still tracked.
+ var contentlessGrant = GrantOf(dto) ?? throw new InvalidOperationException($"Shared manifest {dto.ManifestId} was served without content and without a grant, refusing to assemble.");
+ sharedRecords[dto.ManifestId] = new SharedManifestRecord(dto.ManifestId, contentlessGrant.EncryptedVek, contentlessGrant.EncryptionPublicKey, contentlessGrant.Algorithm, await rustCoreService.VaultCodecGenerateManifestSaltAsync(), null, dto.CanAdminister, manifestKey);
+ contentlessRevisions[dto.ManifestId] = dto.Revision;
+ continue;
+ }
+
+ var entry = await OpenManifestAsync(dto, manifestKey, isPersonal);
+ resolved.Add(entry);
+ fingerprints[VaultSyncState.ManifestFingerprintKey(entry.ManifestId)] = entry.ContentFingerprint;
+
+ if (isPersonal)
+ {
+ state.PersonalManifestSalt = entry.ManifestSalt;
+ state.PersonalManifestId = entry.ManifestId;
+ continue;
+ }
+
+ var grant = GrantOf(dto) ?? throw new InvalidOperationException($"Shared manifest {entry.ManifestId} carries no grant this account can re-open, refusing to assemble.");
+ sharedRecords[entry.ManifestId] = new SharedManifestRecord(entry.ManifestId, grant.EncryptedVek, grant.EncryptionPublicKey, grant.Algorithm, entry.ManifestSalt, entry.Name, dto.CanAdminister, manifestKey);
+ }
+
+ // 2) Open the data buckets belonging to those manifests.
+ var (dataBuckets, bucketRevisions) = await OpenDataBucketsAsync(snapshot, resolved, fingerprints);
+
+ // 3) Adopt the snapshot as local truth.
+ state.SharedManifests.Clear();
+ foreach (var (id, record) in sharedRecords)
+ {
+ state.SharedManifests[id] = record;
+ }
+
+ state.ManifestRevisions.Clear();
+ foreach (var entry in resolved)
+ {
+ state.ManifestRevisions[entry.ManifestId] = entry.Revision;
+ }
+
+ foreach (var (id, revision) in contentlessRevisions)
+ {
+ state.ManifestRevisions[id] = revision;
+ }
+
+ state.BucketRevisions.Clear();
+ foreach (var (key, revision) in bucketRevisions)
+ {
+ state.BucketRevisions[key] = revision;
+ }
+
+ state.ContentFingerprints.Clear();
+ foreach (var (key, fingerprint) in fingerprints)
+ {
+ state.ContentFingerprints[key] = fingerprint;
+ }
+
+ logger.LogInformation("[V2Pull] Adopted revisions of {ManifestCount} manifest(s) and {BucketCount} bucket(s); {FingerprintCount} fingerprint baseline(s) stored.", state.ManifestRevisions.Count, state.BucketRevisions.Count, state.ContentFingerprints.Count);
+ return new OpenedManifestSet(resolved, dataBuckets, contentlessRevisions.Keys.ToList(), resolved[0].Revision);
+ }
+
+ /// <summary>
+ /// Verify one snapshot manifest's ciphertext, decrypt and open it.
+ /// </summary>
+ /// <param name="dto">The snapshot manifest (must carry a blob).</param>
+ /// <param name="vek">The key that decrypts it.</param>
+ /// <param name="isPersonal">Whether this is the caller's own manifest.</param>
+ /// <returns>The opened manifest.</returns>
+ private async Task<ResolvedManifest> OpenManifestAsync(Manifest dto, string vek, bool isPersonal)
+ {
+ var label = isPersonal ? "manifest" : $"shared manifest {dto.ManifestId}";
+ var manifestJson = await VerifyDecryptUnpackAsync(dto.Blob!, vek, dto.CiphertextHash, label);
+ var header = ReadPayloadHeader(manifestJson);
+
+ // Bind the payload to the address the server delivered it under (anti-rehoming).
+ if (header.ManifestId != dto.ManifestId)
+ {
+ throw new InvalidOperationException($"Manifest {dto.ManifestId} declares a different id ({header.ManifestId?.ToString() ?? "none"}) inside its encrypted payload, refusing to open it.");
+ }
+
+ if (string.IsNullOrEmpty(header.Salt))
+ {
+ throw new InvalidOperationException($"Manifest {dto.ManifestId} carries no manifest salt, refusing to open it.");
+ }
+
+ logger.LogInformation("[V2Pull] Opened {Label} (content hash verified): tables: {Tables}", label, header.TableSummary);
+ return new ResolvedManifest(dto.ManifestId, isPersonal, manifestJson, header.Salt, header.Name, vek, dto.Revision, dto.BlobReferences, await rustCoreService.VaultCodecComputeContentFingerprintAsync(manifestJson));
+ }
+
+ /// <summary>
+ /// Verify a ciphertext against the hash the server stored, then decrypt and unpack it via the Rust codec.
+ /// </summary>
+ /// <param name="base64Ciphertext">The ciphertext as served.</param>
+ /// <param name="vek">The symmetric key.</param>
+ /// <param name="expectedCiphertextHash">The ciphertext hash the server stored, when available.</param>
+ /// <param name="label">What is being opened, for the error message.</param>
+ /// <returns>The payload JSON.</returns>
+ private async Task<string> VerifyDecryptUnpackAsync(string base64Ciphertext, string vek, string? expectedCiphertextHash, string label)
+ {
+ if (!string.IsNullOrEmpty(expectedCiphertextHash) && await rustCoreService.VaultCodecComputeCiphertextHashAsync(base64Ciphertext) != expectedCiphertextHash)
+ {
+ throw new InvalidOperationException($"The {label} ciphertext hash does not match what the server stored, refusing to load. Possible storage corruption.");
+ }
+
+ try
+ {
+ return await rustCoreService.VaultCodecDecryptAndUnpackPayloadAsync(base64Ciphertext, vek);
+ }
+ catch (JSException ex)
+ {
+ throw new InvalidOperationException($"The {label} could not be decrypted or unpacked: {ex.Message}", ex);
+ }
+ }
+
+ /// <summary>
+ /// The key that opens one snapshot manifest.
+ /// </summary>
+ /// <param name="dto">The snapshot manifest.</param>
+ /// <param name="personalVek">The session's VEK, which opens the manifest the account key hierarchy is filed under.</param>
+ /// <param name="isPersonal">Whether this is the caller's own manifest.</param>
+ /// <param name="personalManifest">The already-opened personal manifest (the durable home of rotated private keys), or null when none is open yet.</param>
+ /// <returns>The manifest's VEK. Throws when this client holds no key for it.</returns>
+ private async Task<string> ResolveManifestVekAsync(Manifest dto, string personalVek, bool isPersonal, ResolvedManifest? personalManifest)
+ {
+ // Without an explicit key type the home manifest is the account-key one by definition, any other manifest a grant.
+ var keyTypeToken = dto.KeyType ?? ManifestKeyTypes.ToToken(isPersonal ? ManifestKeyType.AccountKey : ManifestKeyType.GrantKey);
+ if (!ManifestKeyTypes.TryParse(keyTypeToken, out var keyType))
+ {
+ throw new InvalidOperationException($"Manifest {dto.ManifestId} states an unknown key type \"{keyTypeToken}\" (newer server?), refusing to assemble.");
+ }
+
+ if (keyType == ManifestKeyType.AccountKey)
+ {
+ if (!isPersonal)
+ {
+ // Our unlock chain produced a VEK for a different manifest than the one this key is filed under.
+ throw new InvalidOperationException($"Manifest {dto.ManifestId} is unlocked by the account key hierarchy, but this session holds no key for it, refusing to assemble.");
+ }
+
+ return personalVek;
+ }
+
+ if (personalManifest is null)
+ {
+ throw new InvalidOperationException($"Manifest {dto.ManifestId} is opened through a grant, but no personal manifest is open to resolve the private key from, refusing to assemble.");
+ }
+
+ return await ResolveGrantedVekAsync(personalManifest, dto);
+ }
+
+ /// <summary>
+ /// Decrypt the VEK the server granted the caller on a manifest.
+ /// </summary>
+ /// <param name="personalManifest">The opened personal manifest, the durable home of rotated private keys.</param>
+ /// <param name="dto">The snapshot manifest carrying the grant.</param>
+ /// <returns>The manifest's VEK.</returns>
+ private async Task<string> ResolveGrantedVekAsync(ResolvedManifest personalManifest, Manifest dto)
+ {
+ if (string.IsNullOrEmpty(dto.EncryptedVek) || string.IsNullOrEmpty(dto.EncryptionPublicKey))
+ {
+ throw new InvalidOperationException($"Shared manifest {dto.ManifestId} carries no grant to open it with, refusing to assemble.");
+ }
+
+ // The algorithm, not the key type, decides how the ciphertext opens; RSA-OAEP is the default when none is stated.
+ var algorithmToken = dto.Algorithm ?? VaultKeyAlgorithms.ToToken(VaultKeyAlgorithm.RsaOaepSha256);
+ if (!VaultKeyAlgorithms.TryParse(algorithmToken, out var algorithm) || algorithm != VaultKeyAlgorithm.RsaOaepSha256)
+ {
+ throw new InvalidOperationException($"Shared manifest {dto.ManifestId} grants its VEK under an unsupported algorithm \"{algorithmToken}\" (newer server?), refusing to assemble.");
+ }
+
+ var privateKeyJwk = await ResolvePrivateKeyJwkAsync(personalManifest, dto.EncryptionPublicKey) ?? throw new InvalidOperationException($"No private key in this vault opens the grant on shared manifest {dto.ManifestId}, refusing to assemble.");
+ try
+ {
+ var plaintextBase64 = await jsInteropService.DecryptWithPrivateKey(dto.EncryptedVek, privateKeyJwk);
+ return Encoding.UTF8.GetString(Convert.FromBase64String(plaintextBase64));
+ }
+ catch (Exception ex)
+ {
+ throw new InvalidOperationException($"Failed to decrypt the VEK of shared manifest {dto.ManifestId}, refusing to assemble. Underlying: {ex.Message}", ex);
+ }
+ }
+
+ /// <summary>
+ /// Resolve the private key (JWK) that opens a grant.
+ /// </summary>
+ /// <param name="personalManifest">The opened personal manifest.</param>
+ /// <param name="encryptionPublicKey">The public key the grant was sealed with.</param>
+ /// <returns>The private key as a JWK JSON string, or null.</returns>
+ private async Task<string?> ResolvePrivateKeyJwkAsync(ResolvedManifest personalManifest, string encryptionPublicKey)
+ {
+ var accountPublicKey = await vaultKeyService.GetAccountPublicKeyAsync();
+ if (accountPublicKey == encryptionPublicKey && authService.GetAccountPrivateKey() is { } sessionPrivateKey)
+ {
+ return sessionPrivateKey;
+ }
+
+ var keyRowJson = await rustCoreService.VaultCodecExtractEncryptionKeyForPublicKeyAsync(personalManifest.ManifestJson, encryptionPublicKey);
+ if (keyRowJson is null)
+ {
+ return null;
+ }
+
+ using var document = JsonDocument.Parse(keyRowJson);
+ return document.RootElement.TryGetProperty("PrivateKey", out var privateKey) && privateKey.ValueKind == JsonValueKind.String ? privateKey.GetString() : null;
+ }
+
+ /// <summary>
+ /// Open the data buckets a snapshot carries.
+ /// </summary>
+ /// <param name="snapshot">The raw snapshot.</param>
+ /// <param name="resolved">The manifests already opened; a bucket addressed to any other manifest is refused.</param>
+ /// <param name="fingerprints">The fingerprint map to add one entry per bucket to.</param>
+ /// <returns>The decrypted bucket payloads and the snapshot's bucket revisions.</returns>
+ private async Task<(List<string> DataBuckets, Dictionary<string, long> BucketRevisions)> OpenDataBucketsAsync(GetResponse snapshot, List<ResolvedManifest> resolved, Dictionary<string, string> fingerprints)
+ {
+ var keyByManifestId = resolved.ToDictionary(entry => entry.ManifestId, entry => entry.VaultEncryptionKey);
+ var dataBuckets = new List<string>();
+ var bucketRevisions = new Dictionary<string, long>(StringComparer.Ordinal);
+ foreach (var bucketDto in snapshot.Buckets)
+ {
+ if (string.IsNullOrEmpty(bucketDto.Blob))
+ {
+ continue;
+ }
+
+ var category = bucketDto.Category.ToString();
+ if (!keyByManifestId.TryGetValue(bucketDto.ManifestId, out var bucketKey))
+ {
+ throw new InvalidOperationException($"Data bucket \"{category}\" belongs to manifest {bucketDto.ManifestId}, which this vault did not open, refusing to assemble.");
+ }
+
+ var label = $"\"{category}\" bucket of manifest {bucketDto.ManifestId}";
+ var bucketJson = await VerifyDecryptUnpackAsync(bucketDto.Blob, bucketKey, bucketDto.CiphertextHash, label);
+ var header = ReadPayloadHeader(bucketJson);
+
+ // Bind the payload to the address the server delivered it under.
+ if (header.ManifestId != bucketDto.ManifestId || !string.Equals(header.Category, category, StringComparison.Ordinal))
+ {
+ throw new InvalidOperationException($"The {label} declares a different address (manifest {header.ManifestId?.ToString() ?? "none"}, category \"{header.Category}\") inside its encrypted payload, refusing to assemble.");
+ }
+
+ dataBuckets.Add(bucketJson);
+ fingerprints[VaultSyncState.BucketFingerprintKey(bucketDto.ManifestId, category)] = await rustCoreService.VaultCodecComputeContentFingerprintAsync(bucketJson);
+ bucketRevisions[VaultSyncState.BucketRevisionKey(bucketDto.ManifestId, category)] = bucketDto.Revision;
+ logger.LogInformation("[V2Pull] Opened {Label} (revision {Revision}): tables: {Tables}", label, bucketDto.Revision, header.TableSummary);
+ }
+
+ return (dataBuckets, bucketRevisions);
+ }
+
+ /// <summary>
+ /// Fetch every blob the opened manifests reference that is not cached yet, decrypt it with the owning manifest's
+ /// key, and prune the cache to exactly the referenced set.
+ /// </summary>
+ /// <param name="opened">The opened manifests.</param>
+ /// <returns>Plaintext bytes per blob hash, for the codec to re-embed.</returns>
+ private async Task<Dictionary<string, byte[]>> DownloadReferencedBlobsAsync(OpenedManifestSet opened)
+ {
+ var refOwners = new Dictionary<string, ResolvedManifest>(StringComparer.Ordinal);
+ var refs = new List<BlobReference>();
+ foreach (var entry in opened.Resolved)
+ {
+ foreach (var reference in entry.BlobReferences)
+ {
+ if (refOwners.TryAdd(reference.Hash, entry))
+ {
+ refs.Add(reference);
+ }
+ }
+ }
+
+ var cache = state.BlobCipherCache;
+ var missingRefs = refs.Where(reference => !cache.ContainsKey(reference.Hash)).ToList();
+ logger.LogInformation("[V2Pull] Blob refs: {Referenced} referenced, {Cached} cached, {Missing} to download.", refs.Count, refs.Count - missingRefs.Count, missingRefs.Count);
+
+ // Batch downloads by the bytes each blob adds to the response, not by hash count.
+ var batches = BatchByTransferCost(missingRefs, reference => Base64Chars(reference.SizeBytes));
+ for (var index = 0; index < batches.Count; index++)
+ {
+ var batch = batches[index];
+ using var response = await httpClient.PostAsJsonAsync(BlobsDownloadEndpoint, new BlobHashesRequest { Hashes = batch.Select(reference => reference.Hash).ToList() });
+ response.EnsureSuccessStatusCode();
+ var blobs = await response.Content.ReadFromJsonAsync<List<Blob>>() ?? [];
+ foreach (var blob in blobs)
+ {
+ cache[blob.Hash] = blob.EncryptedDataBase64;
+ }
+
+ logger.LogInformation("[V2Pull] Downloaded blob batch {Index}/{Total}: requested {Requested}, received {Received}.", index + 1, batches.Count, batch.Count, blobs.Count);
+ }
+
+ // Decrypt the referenced blobs and prune the cache to the referenced set, so it stays bounded by the current vault size.
+ var prunedCache = new Dictionary<string, string>(StringComparer.Ordinal);
+ var blobMap = new Dictionary<string, byte[]>(StringComparer.Ordinal);
+ foreach (var reference in refs)
+ {
+ var owner = refOwners[reference.Hash];
+ if (!cache.TryGetValue(reference.Hash, out var ciphertext))
+ {
+ HandleUnavailableBlob(reference, owner, "is missing on the server");
+ continue;
+ }
+
+ try
+ {
+ blobMap[reference.Hash] = await jsInteropService.SymmetricDecryptBase64ToBytes(ciphertext, owner.VaultEncryptionKey);
+ prunedCache[reference.Hash] = ciphertext;
+ }
+ catch (JSException)
+ {
+ HandleUnavailableBlob(reference, owner, "failed to decrypt with the current key");
+ }
+ }
+
+ cache.Clear();
+ foreach (var (hash, ciphertext) in prunedCache)
+ {
+ cache[hash] = ciphertext;
+ }
+
+ // The server demonstrably has every blob it just served or referenced, seed the upload diff with them.
+ state.ServerBlobHashes.Clear();
+ state.ServerBlobHashes.UnionWith(refs.Select(reference => reference.Hash));
+ return blobMap;
+ }
+
+ /// <summary>
+ /// A referenced blob that cannot be materialized. A personal attachment is fatal: materializing it as empty would
+ /// propagate silent data loss on the next push. A favicon or a shared manifest's blob is skipped with a warning.
+ /// </summary>
+ /// <param name="reference">The blob reference.</param>
+ /// <param name="owner">The manifest that references it.</param>
+ /// <param name="reason">Why it is unavailable.</param>
+ private void HandleUnavailableBlob(BlobReference reference, ResolvedManifest owner, string reason)
+ {
+ if (owner.IsPersonal && string.Equals(reference.Category, AttachmentBlobCategory, StringComparison.OrdinalIgnoreCase))
+ {
+ throw new InvalidOperationException($"Referenced attachment blob {reference.Hash} {reason}, refusing to materialize an incomplete vault.");
+ }
+
+ logger.LogWarning("[V2Pull] Referenced {Category} blob {Hash} {Reason}, continuing without it.", reference.Category, reference.Hash, reason);
+ }
+
+ /// <summary>
+ /// Materialize manifests and data buckets into a fresh in-memory SQLite database via the codec.
+ /// </summary>
+ /// <param name="opened">The opened manifests and buckets.</param>
+ /// <param name="blobs">Plaintext bytes for every blob hash the manifests reference.</param>
+ /// <returns>The open database connection; the caller owns it.</returns>
+ private async Task<SqliteConnection> MaterializeAsync(OpenedManifestSet opened, Dictionary<string, byte[]> blobs)
+ {
+ var connection = new SqliteConnection("Data Source=:memory:");
+ await connection.OpenAsync();
+ try
+ {
+ var schemaSql = await jsInteropService.GetCompleteSchemaSqlAsync();
+ await VaultMaterializer.ApplySchemaAsync(connection, schemaSql);
+
+ // The local schema's column set routes anything a newer writer stored into the overflow carrier instead of crashing the insert.
+ var schemaColumns = await VaultMaterializer.ReadSchemaColumnsAsync(connection);
+ var inputJson = BuildMaterializeInput(opened.Resolved.Select(entry => entry.ManifestJson), opened.DataBuckets, schemaColumns);
+ var materializedJson = await rustCoreService.VaultCodecMaterializeAsSqliteAsync(inputJson);
+ await VaultMaterializer.InsertMaterializedTablesAsync(connection, materializedJson, blobs, logger);
+ return connection;
+ }
+ catch
+ {
+ await connection.DisposeAsync();
+ throw;
+ }
+ }
+}
diff --git a/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncState.cs b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncState.cs
new file mode 100644
index 000000000..bb5105059
--- /dev/null
+++ b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncState.cs
@@ -0,0 +1,107 @@
+//-----------------------------------------------------------------------
+// <copyright file="VaultSyncState.cs" company="aliasvault">
+// Copyright (c) aliasvault. All rights reserved.
+// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
+// </copyright>
+//-----------------------------------------------------------------------
+
+namespace AliasVault.Client.Services.VaultSync;
+
+using AliasVault.Client.Services.VaultSync.Models;
+
+/// <summary>
+/// The client's sync state as adopted from the last pull.
+/// </summary>
+public sealed class VaultSyncState
+{
+ /// <summary>
+ /// Gets or sets the id of the user's personal manifest, as reported by the server.
+ /// </summary>
+ public Guid? PersonalManifestId { get; set; }
+
+ /// <summary>
+ /// Gets or sets the personal manifest's blob-hashing salt.
+ /// </summary>
+ public string? PersonalManifestSalt { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether the last snapshot was still on the legacy sqlite-blob storage format.
+ /// </summary>
+ public bool LastSnapshotWasLegacySqliteBlob { get; set; }
+
+ /// <summary>
+ /// Gets the last-known server revision per manifest.
+ /// </summary>
+ public Dictionary<Guid, long> ManifestRevisions { get; } = [];
+
+ /// <summary>
+ /// Gets the last-known server revision per data bucket, keyed by <see cref="BucketRevisionKey"/>.
+ /// </summary>
+ public Dictionary<string, long> BucketRevisions { get; } = new(StringComparer.Ordinal);
+
+ /// <summary>
+ /// Gets the content fingerprints of every manifest and bucket as last served or pushed, keyed by
+ /// <see cref="ManifestFingerprintKey"/> and <see cref="BucketFingerprintKey"/>.
+ /// </summary>
+ public Dictionary<string, string> ContentFingerprints { get; } = new(StringComparer.Ordinal);
+
+ /// <summary>
+ /// Gets the blob hashes the server is known to hold.
+ /// </summary>
+ public HashSet<string> ServerBlobHashes { get; } = new(StringComparer.Ordinal);
+
+ /// <summary>
+ /// Gets the encrypted blob cache (hash to base64 ciphertext), pruned to the referenced set on every pull.
+ /// </summary>
+ public Dictionary<string, string> BlobCipherCache { get; } = new(StringComparer.Ordinal);
+
+ /// <summary>
+ /// Gets the shared manifests this session holds a grant on, keyed by manifest id.
+ /// </summary>
+ public Dictionary<Guid, SharedManifestRecord> SharedManifests { get; } = [];
+
+ /// <summary>
+ /// Gets the manifest ids the last snapshot carried, recorded before any of them is opened.
+ /// </summary>
+ public List<Guid> LastServedManifestIds { get; } = [];
+
+ /// <summary>
+ /// The key of one data bucket's revision.
+ /// </summary>
+ /// <param name="manifestId">The manifest that owns the bucket.</param>
+ /// <param name="category">The bucket category.</param>
+ /// <returns>The record key.</returns>
+ public static string BucketRevisionKey(Guid manifestId, string category) => $"{manifestId}:{category}";
+
+ /// <summary>
+ /// The fingerprint record key of a manifest.
+ /// </summary>
+ /// <param name="manifestId">The manifest id.</param>
+ /// <returns>The record key.</returns>
+ public static string ManifestFingerprintKey(Guid manifestId) => $"manifest:{manifestId}";
+
+ /// <summary>
+ /// The fingerprint record key of a data bucket.
+ /// </summary>
+ /// <param name="manifestId">The manifest that owns the bucket.</param>
+ /// <param name="category">The bucket category.</param>
+ /// <returns>The record key.</returns>
+ public static string BucketFingerprintKey(Guid manifestId, string category) => $"bucket:{manifestId}:{category}";
+
+ /// <summary>
+ /// Forgets everything; the next pull rebuilds it.
+ /// </summary>
+ public void Clear()
+ {
+ PersonalManifestId = null;
+ PersonalManifestSalt = null;
+ LastSnapshotWasLegacySqliteBlob = false;
+ ManifestRevisions.Clear();
+ BucketRevisions.Clear();
+ ContentFingerprints.Clear();
+ ServerBlobHashes.Clear();
+ BlobCipherCache.Clear();
+ SharedManifests.Clear();
+ LastServedManifestIds.Clear();
+ }
+}