Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update services for manifest-v1 canonicalize push (#2159)
Leendert de Borst048918ae18ea
diff --git a/apps/server/AliasVault.Client/Services/Database/DbService.cs b/apps/server/AliasVault.Client/Services/Database/DbService.csindex f82e2ad70..705b2d445 100644--- a/apps/server/AliasVault.Client/Services/Database/DbService.cs+++ b/apps/server/AliasVault.Client/Services/Database/DbService.cs@@ -8,9 +8,6 @@namespace AliasVault.Client.Services.Database;using System.Data;-using System.Net;-using System.Net.Http.Json;-using System.Text.Json;using AliasClientDb;using AliasClientDb.Models;using AliasVault.Client.Services;@@ -21,9 +18,6 @@ using AliasVault.Client.Services.JsInterop.RustCore.Models;using AliasVault.Client.Services.VaultSync;using AliasVault.Client.Services.VaultSync.Exceptions;using AliasVault.Client.Services.VaultSync.Models;-using AliasVault.Client.Utilities;-using AliasVault.Shared.Models.Enums;-using AliasVault.Shared.Models.WebApi.V1.Vault;using Microsoft.Data.Sqlite;using Microsoft.EntityFrameworkCore;using Microsoft.Extensions.Localization;@@ -36,23 +30,30 @@ using Microsoft.Extensions.Localization;public sealed class DbService : IDisposable{private const string _UNKNOWN_VERSION = "Unknown";++ /// <summary>+ /// How many times a save pulls, merges and pushes again after the server reported newer state before giving up.+ /// </summary>+ private const int MaxPushAttempts = 3;+private readonly AuthService _authService;private readonly JsInteropService _jsInteropService;private readonly RustCoreService _rustCore;private readonly VaultSyncService _vaultSync;+ private readonly VaultKeyService _vaultKeyService;private readonly VaultSyncState _syncState;- private readonly HttpClient _httpClient;private readonly DbServiceState _state = new();private readonly Config _config;private readonly ILogger<DbService> _logger;private readonly GlobalNotificationService _globalNotificationService;private readonly IStringLocalizer _sharedLocalizer;private readonly CancellationTokenSource _backgroundSyncCts = new();+ private readonly SemaphoreSlim _saveLock = new(1, 1);private SettingsService _settingsService = new();private SqliteConnection? _sqlConnection;private AliasClientDbContext _dbContext;- private long _vaultRevisionNumber;private bool _isSuccessfullyInitialized;+ private bool _forceFullWriteOnNextPush;private int _retryCount;private bool _disposed;@@ -62,21 +63,21 @@ public sealed class DbService : IDisposable/// <param name="authService">AuthService.</param>/// <param name="jsInteropService">JsInteropService.</param>/// <param name="rustCore">RustCoreService for WASM interop.</param>- /// <param name="vaultSync">VaultSyncService that pulls the vault from the server.</param>- /// <param name="syncState">The sync state recorded by the last pull.</param>- /// <param name="httpClient">HttpClient.</param>+ /// <param name="vaultSync">VaultSyncService that pulls, pushes and merges the vault.</param>+ /// <param name="vaultKeyService">VaultKeyService that holds the account key chain.</param>+ /// <param name="syncState">The sync state recorded by the last pull or push.</param>/// <param name="config">Config instance.</param>/// <param name="globalNotificationService">Global notification service.</param>/// <param name="localizerFactory">IStringLocalizerFactory instance.</param>/// <param name="logger">ILogger instance.</param>- public DbService(AuthService authService, JsInteropService jsInteropService, RustCoreService rustCore, VaultSyncService vaultSync, VaultSyncState syncState, HttpClient httpClient, Config config, GlobalNotificationService globalNotificationService, IStringLocalizerFactory localizerFactory, ILogger<DbService> logger)+ public DbService(AuthService authService, JsInteropService jsInteropService, RustCoreService rustCore, VaultSyncService vaultSync, VaultKeyService vaultKeyService, VaultSyncState syncState, Config config, GlobalNotificationService globalNotificationService, IStringLocalizerFactory localizerFactory, ILogger<DbService> logger){_authService = authService;_jsInteropService = jsInteropService;_rustCore = rustCore;_vaultSync = vaultSync;+ _vaultKeyService = vaultKeyService;_syncState = syncState;- _httpClient = httpClient;_config = config;_globalNotificationService = globalNotificationService;_sharedLocalizer = localizerFactory.Create("SharedResources", "AliasVault.Client");@@ -95,6 +96,12 @@ public sealed class DbService : IDisposable/// <returns>SettingsService.</returns>public SettingsService Settings => _settingsService;+ /// <summary>+ /// Gets the id of the user's personal manifest, the scope every row this client creates belongs to.+ /// </summary>+ /// <exception cref="InvalidOperationException">Thrown when no vault has been pulled yet.</exception>+ public Guid PersonalManifestId => _syncState.PersonalManifestId ?? throw new InvalidOperationException("No personal manifest id is recorded; the vault has not been loaded from the server yet.");+/// <summary>/// Gets database service state object which can be subscribed to./// </summary>@@ -124,15 +131,6 @@ public sealed class DbService : IDisposable}}- /// <summary>- /// Stores / updates the vault revision number. Should be called after a successful vault update to the server.- /// </summary>- /// <param name="newRevisionNumber">New revision number.</param>- public void StoreVaultRevisionNumber(long newRevisionNumber)- {- _vaultRevisionNumber = newRevisionNumber;- }-/// <summary>/// Returns the AliasClientDbContext instance./// </summary>@@ -156,22 +154,6 @@ public sealed class DbService : IDisposablereturn _dbContext;}- /// <summary>- /// Generate encrypted base64 string representation of current state of database in order to save it- /// to the server.- /// </summary>- /// <returns>Base64 encoded vault blob.</returns>- public async Task<string> GetEncryptedDatabaseBase64String()- {- // Save the actual dbContext.- await _dbContext.SaveChangesAsync();-- string base64String = await ExportSqliteToBase64Async();-- // SymmetricEncrypt base64 string using IJSInterop.- return await _jsInteropService.SymmetricEncrypt(base64String, _authService.GetEncryptionKeyAsBase64Async());- }-/// <summary>/// Saves the database to the remote server./// </summary>@@ -184,24 +166,16 @@ public sealed class DbService : IDisposable_state.UpdateState(DbServiceState.DatabaseStatus.SavingToServer);}- // Prune expired items from trash before saving.- await PruneExpiredTrashItemsAsync();-- // Make sure a public/private RSA encryption key exists before saving the database.- await GetOrCreateEncryptionKeyAsync();-- var encryptedBase64String = await GetEncryptedDatabaseBase64String();-- // Save to webapi.- var success = await SaveToServerAsync(encryptedBase64String);+ var success = await SaveAndPushAsync();if (success){_logger.LogInformation("Database successfully saved to server.");- if (_state.CurrentState.Status != DbServiceState.DatabaseStatus.Creating)- {- // If database is not in the process of being created, update status to ready which is reflected in the UI.- _state.UpdateState(DbServiceState.DatabaseStatus.Ready);- }+ }++ if (_state.CurrentState.Status != DbServiceState.DatabaseStatus.Creating)+ {+ // If database is not in the process of being created, update status to ready which is reflected in the UI.+ _state.UpdateState(DbServiceState.DatabaseStatus.Ready);}return success;@@ -219,8 +193,12 @@ public sealed class DbService : IDisposable/// </remarks>public void SaveDatabaseInBackground(){- // Set state to indicate background sync is pending- _state.UpdateState(DbServiceState.DatabaseStatus.BackgroundSyncPending);+ var creating = _state.CurrentState.Status == DbServiceState.DatabaseStatus.Creating;+ if (!creating)+ {+ // Set state to indicate background sync is pending+ _state.UpdateState(DbServiceState.DatabaseStatus.BackgroundSyncPending);+ }// Capture cancellation token for this background operationvar cancellationToken = _backgroundSyncCts.Token;@@ -236,43 +214,19 @@ public sealed class DbService : IDisposablereturn;}- // Prune expired items from trash before saving.- await PruneExpiredTrashItemsAsync();-- if (cancellationToken.IsCancellationRequested || _disposed)- {- return;- }-- // Make sure a public/private RSA encryption key exists before saving the database.- await GetOrCreateEncryptionKeyAsync();-- if (cancellationToken.IsCancellationRequested || _disposed)- {- return;- }-- var encryptedBase64String = await GetEncryptedDatabaseBase64String();-- if (cancellationToken.IsCancellationRequested || _disposed)- {- return;- }-- // Update state to show we're actively syncing- _state.UpdateState(DbServiceState.DatabaseStatus.SavingToServer);-- // Save to webapi.- var success = await SaveToServerAsync(encryptedBase64String);+ var success = await SaveAndPushAsync();if (success){_logger.LogInformation("Database successfully saved to server (background sync).");- _state.UpdateState(DbServiceState.DatabaseStatus.Ready);}else{- // SaveToServerAsync already raised the user-facing error notification (targeted or generic).+ // SaveAndPushAsync already raised the user-facing error notification (targeted or generic)._logger.LogWarning("Background sync to server failed.");+ }++ if (!creating && !_disposed)+ {_state.UpdateState(DbServiceState.DatabaseStatus.Ready);}}@@ -290,7 +244,10 @@ public sealed class DbService : IDisposable{_logger.LogError(ex, "Error during background database sync.");_globalNotificationService.AddErrorMessage(_sharedLocalizer["ErrorUnknown"], true);- _state.UpdateState(DbServiceState.DatabaseStatus.Ready);+ if (!creating)+ {+ _state.UpdateState(DbServiceState.DatabaseStatus.Ready);+ }}},cancellationToken);@@ -331,6 +288,9 @@ public sealed class DbService : IDisposable{try{+ // Every row this vault will hold is stamped with the personal manifest, so its id must be known up front.+ _ = PersonalManifestId;+// Call JS interop to get SQL commands to create a new vault with the latest schema.var sqlCommands = await _jsInteropService.GetCreateVaultSqlAsync();@@ -354,30 +314,35 @@ public sealed class DbService : IDisposable}/// <summary>- /// Migrate the database structure to the latest version.+ /// Migrate the loaded legacy vault onto the current storage model: walk the frozen sqlite-blob upgrade chain when+ /// the vault predates its end, then rebuild the database from its manifest-v1 form on the complete schema. The+ /// result is held locally until <see cref="SaveDatabaseAsync"/> pushes it, which also creates the account key chain+ /// for an account that has none yet./// </summary>/// <returns>Bool which indicates if migration was successful.</returns>public async Task<bool> MigrateDatabaseAsync(){try{- // Get current version of database.- var currentVersion = await GetCurrentDatabaseVersionAsync();-- // Get latest version from JsInteropService.- var latestVersion = await _jsInteropService.GetLatestVaultVersionAsync();-- // Call JS interop to get SQL commands to create a new vault with the latest schema.- var sqlCommands = await _jsInteropService.GetUpgradeVaultSqlAsync(currentVersion.Revision, latestVersion.Revision);-- // Execute the SQL commands to create a new vault with the latest schema.- foreach (var sqlCommand in sqlCommands.SqlCommands)+ if (await HasPendingMigrationsAsync()){- await _dbContext.Database.ExecuteSqlRawAsync(sqlCommand);+ // LEGACY: the frozen sqlite-blob upgrade chain has to bring the vault to its end first; the codec cannot canonicalize what came before.+ var currentVersion = await GetCurrentDatabaseVersionAsync();+ var latestVersion = await _jsInteropService.GetLatestVaultVersionAsync();+ var sqlCommands = await _jsInteropService.GetUpgradeVaultSqlAsync(currentVersion.Revision, latestVersion.Revision);+ foreach (var sqlCommand in sqlCommands.SqlCommands)+ {+ await _dbContext.Database.ExecuteSqlRawAsync(sqlCommand);+ }}+ // Rebuild on the complete schema via the codec; this stamps every row with the personal manifest.+ AdoptDatabase(await _vaultSync.MigrateVaultToCurrentSchemaAsync(_sqlConnection!));++ // The server holds no manifest-v1 state for this vault yet, so the next push must write everything.+ _forceFullWriteOnNextPush = true;_isSuccessfullyInitialized = true;- await _settingsService.InitializeAsync(this);+ await _settingsService.ReloadAsync(this);}catch (Exception ex){@@ -476,36 +441,6 @@ public sealed class DbService : IDisposable};}- /// <summary>- /// Prepare a vault object for upload to the server.- /// </summary>- /// <param name="encryptedDatabase">Encrypted database as string.</param>- /// <returns>Vault object.</returns>- public async Task<Vault> PrepareVaultForUploadAsync(string encryptedDatabase)- {- var username = _authService.GetUsername();- var databaseVersion = await GetCurrentDatabaseVersionAsync();- var encryptionKey = await GetOrCreateEncryptionKeyAsync();- var credentialsCount = await _dbContext.Items.Where(x => !x.IsDeleted && x.DeletedAt == null).CountAsync();- var emailAddresses = await GetEmailClaimListAsync();- var currentDateTime = DateTime.UtcNow;- return new Vault- {- Username = username,- Blob = encryptedDatabase,- Version = databaseVersion.Version,- CurrentRevisionNumber = _vaultRevisionNumber,- EncryptionPublicKey = encryptionKey.PublicKey,- CredentialsCount = credentialsCount,- EmailAddressList = emailAddresses,- PrivateEmailDomainList = [],- HiddenPrivateEmailDomainList = [],- PublicEmailDomainList = [],- CreatedAt = currentDateTime,- UpdatedAt = currentDateTime,- };- }-/// <summary>/// Clears the database connection and creates a new one so that the database is empty./// </summary>@@ -516,6 +451,7 @@ public sealed class DbService : IDisposableconnection.Open();SetLiveConnection(connection);_syncState.Clear();+ _forceFullWriteOnNextPush = false;// Reset the database state._state.UpdateState(DbServiceState.DatabaseStatus.Uninitialized);@@ -605,6 +541,7 @@ public sealed class DbService : IDisposable/// <summary>/// Imports a base64-encoded SQLite database into the given connection, replacing its contents.+ /// TODO: remove when all accounts have been migrated to the new manifest-v1 format./// </summary>/// <param name="base64String">The base64-encoded SQLite database.</param>/// <param name="connection">The connection to import into.</param>@@ -699,25 +636,6 @@ public sealed class DbService : IDisposableFile.Delete(tempFileName);}- /// <summary>- /// Converts a JsonElement to its appropriate .NET value for SQLite parameters.- /// </summary>- /// <param name="element">The JsonElement to convert.</param>- /// <returns>The converted value.</returns>- private static object? ConvertJsonElementToValue(JsonElement element)- {- return element.ValueKind switch- {- JsonValueKind.String => element.GetString(),- JsonValueKind.Number => element.TryGetInt64(out var longVal) ? longVal : element.GetDouble(),- JsonValueKind.True => 1L, // SQLite stores booleans as integers- JsonValueKind.False => 0L,- JsonValueKind.Null => null,- JsonValueKind.Undefined => null,- _ => element.ToString(),- };- }-/// <summary>/// Replace first occurrence of a string./// </summary>@@ -737,111 +655,140 @@ public sealed class DbService : IDisposable}/// <summary>- /// Fetches the latest vault from server, merges with local changes using Rust WASM, and saves the merged result.- /// Called when server responds with "Outdated" status, indicating another client has uploaded a newer vault.+ /// Commit the pending EF changes and push the vault to the server, one save at a time. Saves that arrive while+ /// another is running wait for it, so two pushes never canonicalize the same database concurrently./// </summary>- /// <returns>Bool which indicates if merge and save was successful.</returns>- private async Task<bool> MergeWithServerAndSaveAsync()+ /// <returns>True when the server holds the local vault afterwards.</returns>+ private async Task<bool> SaveAndPushAsync(){+ await _saveLock.WaitAsync();try{- _logger.LogInformation("Local vault is outdated. Fetching latest vault from server for merge...");+ // Prune expired items from trash before saving.+ await PruneExpiredTrashItemsAsync();- // Fetch the latest vault from server.- var response = await _httpClient.GetFromJsonAsync<VaultGetResponse>(ApiRoute("Vault"));- if (response?.Vault == null || string.IsNullOrEmpty(response.Vault.Blob))- {- _logger.LogError("Failed to fetch vault from server for merge.");- _globalNotificationService.AddErrorMessage(_sharedLocalizer["ErrorUnknown"], true);- return false;- }-- var serverVault = response.Vault;- _logger.LogInformation("Fetched server vault at revision {Revision}.", serverVault.CurrentRevisionNumber);+ // Make sure a public/private RSA encryption key exists before saving the database.+ await EnsurePersonalEncryptionKeyAsync();+ await _dbContext.SaveChangesAsync();- // Store username of the loaded vault in memory to send to server as sanity check when updating the vault later.- _authService.StoreUsername(serverVault.Username);-- // Decrypt server vault.- var decryptedBase64String = await _jsInteropService.SymmetricDecrypt(serverVault.Blob, _authService.GetEncryptionKeyAsBase64Async());+ return await PushWithConflictResolutionAsync();+ }+ catch (Exception ex)+ {+ _logger.LogError(ex, "Error saving database to server.");+ _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultSaveError"], true);+ return false;+ }+ finally+ {+ _saveLock.Release();+ }+ }- // Get the list of syncable table names from Rust core.- var tableNames = await _rustCore.GetSyncableTableNamesAsync();+ /// <summary>+ /// Push the vault. When the server reports newer state, pull it, merge the local changes onto it and push again,+ /// a bounded number of times. Errors are reported to the user here so callers only need the bool.+ /// </summary>+ /// <returns>True when the push succeeded.</returns>+ private async Task<bool> PushWithConflictResolutionAsync()+ {+ for (var attempt = 1; attempt <= MaxPushAttempts; attempt++)+ {+ // LEGACY: an account without a key chain creates one on its first manifest-v1 push, unless another device already did.+ var createVaultKey = !await _vaultKeyService.HasLocalVaultKeyAsync() && !await TryAdoptRemoteVaultKeyAsync();- // Read local tables as JSON.- var localTables = await ReadTablesAsJsonAsync(_sqlConnection!, tableNames);- _logger.LogDebug("Read {Count} local tables.", localTables.Count);+ PushResult result;+ try+ {+ result = await _vaultSync.PushAsync(_sqlConnection!, new PushOptions(createVaultKey, _forceFullWriteOnNextPush));+ }+ catch (VaultTooLargeException)+ {+ // 413: server / reverse-proxy rejected the upload because the vault exceeded MAX_UPLOAD_SIZE_MB.+ _logger.LogError("Vault upload rejected by server with 413 Request Entity Too Large. The vault exceeds the server's configured MAX_UPLOAD_SIZE_MB.");+ _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultTooLargeError"], true);+ return false;+ }+ catch (VaultKeyDecryptException ex)+ {+ _logger.LogError(ex, "The server's key chain does not open with this session's key; the password was changed elsewhere. Log in again.");+ _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultSaveError"], true);+ return false;+ }- // Create a temporary in-memory SQLite database for the server vault.- await using var serverConnection = new SqliteConnection("Data Source=:memory:");- await serverConnection.OpenAsync();- await ImportDbContextFromBase64Async(decryptedBase64String, serverConnection);+ switch (result.Status)+ {+ case PushStatus.Ok:+ _forceFullWriteOnNextPush = false;+ return true;- // Read server tables as JSON.- var serverTables = await ReadTablesAsJsonAsync(serverConnection, tableNames);- _logger.LogDebug("Read {Count} server tables.", serverTables.Count);+ case PushStatus.Outdated:+ _logger.LogInformation("Push attempt {Attempt}/{Max}: the server holds newer state ({Reasons}); pulling and merging.", attempt, MaxPushAttempts, string.Join("; ", result.Reasons ?? []));+ if (attempt == MaxPushAttempts)+ {+ break;+ }- // Create the merge input (local has our pending changes, server is the merge base).- var mergeInput = new MergeInput- {- LocalTables = localTables,- ServerTables = serverTables,- };+ await MergeWithServerAsync();+ continue;- // Call Rust WASM merge (LWW - Last Write Wins based on UpdatedAt). The merge base is the- // server vault; the returned statements bring the local changes onto that base.- var mergeOutput = await _rustCore.MergeVaultsAsync(mergeInput);-- _logger.LogInformation(- "Merge completed: {TablesProcessed} tables, {FromLocal} kept local, {FromServer} from server, {Inserted} inserted, {Conflicts} conflicts.",- mergeOutput.Stats.TablesProcessed,- mergeOutput.Stats.RecordsFromLocal,- mergeOutput.Stats.RecordsFromServer,- mergeOutput.Stats.RecordsInserted,- mergeOutput.Stats.Conflicts);-- // Apply the local changes onto the server base (statements target the server database),- // then make the merged (server-based) database the live local database.- await ExecuteMergeSqlStatementsAsync(mergeOutput.Statements, serverConnection);- var mergedBase64 = await ExportConnectionToBase64Async(serverConnection);- await ImportDbContextFromBase64Async(mergedBase64, _sqlConnection!);-- // Verify foreign key integrity after merge.- await using (var command = _sqlConnection!.CreateCommand())- {- command.CommandText = "PRAGMA foreign_key_check;";- await using var reader = await command.ExecuteReaderAsync();- if (await reader.ReadAsync())- {- _logger.LogError("Foreign key violation detected after merge.");- _globalNotificationService.AddErrorMessage(_sharedLocalizer["ErrorUnknown"], true);+ default:+ _logger.LogError("Vault push {Status}: {Reasons}", result.Status, string.Join("; ", result.Reasons ?? []));+ _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultSaveError"], true);return false;- }}+ }- // Update the db context with the merged database.- _dbContext = new AliasClientDbContext(_sqlConnection, log => _logger.LogDebug("{Message}", log));-- // Update the local revision number to the server's revision.- // When we upload, server will calculate new revision = this + 1.- StoreVaultRevisionNumber(serverVault.CurrentRevisionNumber);+ _logger.LogError("Vault push still outdated after {Max} attempts, giving up.", MaxPushAttempts);+ _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultSaveError"], true);+ return false;+ }- _logger.LogInformation("Local merge completed. Uploading merged vault to server...");+ /// <summary>+ /// Fetch the server vault and merge the local changes onto it; the merged database becomes the live one. When the+ /// server holds nothing to merge with (legacy format or a never-written manifest), the next push writes everything.+ /// </summary>+ /// <returns>Task.</returns>+ private async Task MergeWithServerAsync()+ {+ var result = await _vaultSync.PullAndMergeAsync(_sqlConnection!);+ if (result.Kind == PullAndMergeKind.NothingToMergeWith)+ {+ _logger.LogInformation("The server holds no manifest-v1 vault to merge with; pushing the local vault over it whole.");+ _forceFullWriteOnNextPush = true;+ return;+ }- // Now save the merged vault to server. This recursive call handles the case where- // another client uploaded during our merge (returns Outdated again).- return await SaveDatabaseAsync();+ foreach (var manifestId in result.FallbackManifestIds)+ {+ _logger.LogWarning("Canonical merge fell back to the server's rows for manifest {ManifestId}; local changes to it were dropped.", manifestId);}- catch (Exception ex)++ AdoptDatabase(result.Database!);+ await _settingsService.ReloadAsync(this);+ }++ /// <summary>+ /// Adopt a key chain the server holds but this device does not: the account was migrated on another device while+ /// this session still holds the old password-derived key, which opens the chain as its KEK.+ /// </summary>+ /// <returns>True when a chain was adopted, false when the server holds none.</returns>+ /// <exception cref="VaultKeyDecryptException">Thrown when the server's chain does not open with the session key.</exception>+ private async Task<bool> TryAdoptRemoteVaultKeyAsync()+ {+ var resolved = await _vaultKeyService.AdoptRemoteVaultKeyAsync(_authService.GetEncryptionKeyAsBase64Async());+ if (resolved is null){- _globalNotificationService.AddErrorMessage(_sharedLocalizer["ErrorUnknown"], true);- _logger.LogError(ex, "Error merging with server vault.");return false;}++ _logger.LogInformation("Adopted the account key chain another device created; the session key is now the VEK.");+ await _authService.StoreSessionKeysAsync(resolved);+ return true;}/// <summary>- /// Checks if there are any pending migrations.+ /// Checks if there are any pending migrations of the frozen sqlite-blob upgrade chain./// </summary>/// <returns>Bool which indicates if there are any pending migrations.</returns>private async Task<bool> HasPendingMigrationsAsync()@@ -895,7 +842,6 @@ public sealed class DbService : IDisposable{case PullKind.Empty:// The vault was never written: create the database structure from scratch to get an empty ready-to-use database.- StoreVaultRevisionNumber(pull.Revision);_state.UpdateState(DbServiceState.DatabaseStatus.Creating);return false;@@ -904,7 +850,6 @@ public sealed class DbService : IDisposabledefault:SetLiveConnection(pull.Database!);- StoreVaultRevisionNumber(pull.Revision);_isSuccessfullyInitialized = true;await _settingsService.InitializeAsync(this);_state.UpdateState(DbServiceState.DatabaseStatus.Ready);@@ -920,15 +865,14 @@ public sealed class DbService : IDisposable}/// <summary>- /// Import an account's not-yet-migrated sqlite-blob vault. The derived key is the encryption key for these accounts.- /// TODO: remove when all accounts have been migrated to the new manifest-v1 format.+ /// Import an account's not-yet-migrated sqlite-blob vault. The derived key is the encryption key for these+ /// accounts. A legacy vault cannot be saved before it is migrated onto the manifest-v1 storage model, so it+ /// always lands on the upgrade page. TODO: remove when all accounts have been migrated to the new manifest-v1 format./// </summary>/// <param name="pull">The pass-through pull result.</param>/// <returns>True when the database is ready for use.</returns>private async Task<bool> LoadLegacySqliteBlobAsync(PullResult pull){- StoreVaultRevisionNumber(pull.Revision);-// An account that registered but never uploaded a vault: create the database structure from scratch.if (string.IsNullOrEmpty(pull.LegacyVaultBlob)){@@ -941,16 +885,12 @@ public sealed class DbService : IDisposableawait ImportDbContextFromBase64Async(decryptedBase64String, _sqlConnection!);// Refresh the db context with the new database to invalidate any cached data if the _dbContext was already used.- _dbContext = new AliasClientDbContext(_sqlConnection!, log => _logger.LogDebug("{Message}", log));+ _dbContext = CreateDbContext(_sqlConnection!);- // Check if database is up-to-date with migrations.+ // A vault this client cannot place on the upgrade chain (newer major version) cannot be migrated either.try{- if (await HasPendingMigrationsAsync())- {- _state.UpdateState(DbServiceState.DatabaseStatus.PendingMigrations);- return false;- }+ await HasPendingMigrationsAsync();}catch (DataException){@@ -958,10 +898,8 @@ public sealed class DbService : IDisposablereturn false;}- _isSuccessfullyInitialized = true;- await _settingsService.InitializeAsync(this);- _state.UpdateState(DbServiceState.DatabaseStatus.Ready);- return true;+ _state.UpdateState(DbServiceState.DatabaseStatus.PendingMigrations);+ return false;}/// <summary>@@ -972,7 +910,7 @@ public sealed class DbService : IDisposable{var previous = _sqlConnection;_sqlConnection = connection;- _dbContext = new AliasClientDbContext(_sqlConnection, log => _logger.LogDebug("{Message}", log));+ _dbContext = CreateDbContext(connection);if (previous is not null && !ReferenceEquals(previous, connection)){@@ -981,57 +919,27 @@ public sealed class DbService : IDisposable}/// <summary>- /// Saves encrypted database blob to server and updates the local revision number.+ /// Copy another database into the live connection and refresh the context over it. The connection object stays+ /// the same, so a context handed out earlier keeps working and simply sees the new content./// </summary>- /// <param name="encryptedDatabase">Encrypted database as string.</param>- /// <returns>True if save action succeeded and revision number was updated, false otherwise.</returns>- private async Task<bool> SaveToServerAsync(string encryptedDatabase)+ /// <param name="source">The open in-memory database to adopt; disposed afterwards.</param>+ private void AdoptDatabase(SqliteConnection source){- var vaultObject = await PrepareVaultForUploadAsync(encryptedDatabase);-- try- {- var response = await _httpClient.PostAsJsonAsync(ApiRoute("Vault"), vaultObject);-- // 413: server / reverse-proxy rejected the upload because the vault exceeded MAX_UPLOAD_SIZE_MB.- // Show the targeted message and skip the generic notification fired in the catch / fallthrough.- if (response.StatusCode == HttpStatusCode.RequestEntityTooLarge)- {- _logger.LogError("Vault upload rejected by server with 413 Request Entity Too Large. The vault exceeds the server's configured MAX_UPLOAD_SIZE_MB.");- _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultTooLargeError"], true);- return false;- }-- // Ensure the request was successful- response.EnsureSuccessStatusCode();-- // Deserialize the response content- var vaultUpdateResponse = await response.Content.ReadFromJsonAsync<VaultUpdateResponse>();-- if (vaultUpdateResponse != null)- {- if (vaultUpdateResponse.Status == VaultStatus.Outdated)- {- // Server has a newer vault. Fetch it, merge with our local changes, and re-upload.- // The merge uses LWW (Last Write Wins) based on UpdatedAt timestamps.- return await MergeWithServerAndSaveAsync();- }-- _vaultRevisionNumber = vaultUpdateResponse.NewRevisionNumber;- return true;- }-- _logger.LogError("Error during save: server response was empty or could not be deserialized.");- }- catch (Exception ex)- {- _logger.LogError(ex, "Error saving database to server.");- }+ source.BackupDatabase(_sqlConnection!);+ source.Dispose();+ _dbContext = CreateDbContext(_sqlConnection!);+ }- // Generic save failure (server error, network issue, malformed response, etc.). DbService owns- // the user-facing error so callers only need to react to the bool return value.- _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultSaveError"], true);- return false;+ /// <summary>+ /// Create the EF context over a connection, with every save stamping new rows with their manifest.+ /// </summary>+ /// <param name="connection">The open connection.</param>+ /// <returns>The context.</returns>+ private AliasClientDbContext CreateDbContext(SqliteConnection connection)+ {+ var context = new AliasClientDbContext(connection, log => _logger.LogDebug("{Message}", log));+ context.SavingChanges += (sender, _) => ManifestStamper.Stamp((DbContext)sender!, PersonalManifestId);+ return context;}/// <summary>@@ -1096,31 +1004,32 @@ public sealed class DbService : IDisposable}/// <summary>- /// Get the default public/private encryption key, if it does not yet exist, create it.+ /// Make sure the personal manifest holds its email delivery keypair; create one when it does not./// </summary>/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>- private async Task<EncryptionKey> GetOrCreateEncryptionKeyAsync()+ private async Task EnsurePersonalEncryptionKeyAsync(){- var encryptionKey = await _dbContext.EncryptionKeys.FirstOrDefaultAsync(x => x.IsPrimary);+ var personalManifestId = PersonalManifestId;+ var encryptionKey = await _dbContext.EncryptionKeys.FirstOrDefaultAsync(x => x.ManifestId == personalManifestId && x.IsPrimary && !x.IsDeleted);if (encryptionKey is not null){- return encryptionKey;+ return;}// Create a new encryption key via JSInterop, .NET WASM does not support crypto operations natively (yet).var keyPair = await _jsInteropService.GenerateRsaKeyPair();var currentDateTime = DateTime.UtcNow;- encryptionKey = new EncryptionKey+ _dbContext.EncryptionKeys.Add(new EncryptionKey{+ Id = Guid.NewGuid(),+ ManifestId = personalManifestId,PublicKey = keyPair.PublicKey,PrivateKey = keyPair.PrivateKey,IsPrimary = true,CreatedAt = currentDateTime,UpdatedAt = currentDateTime,- };- _dbContext.EncryptionKeys.Add(encryptionKey);- return encryptionKey;+ });}/// <summary>@@ -1177,137 +1086,6 @@ public sealed class DbService : IDisposablereturn tables;}- /// <summary>- /// Reads all specified tables from a SQLite connection as JSON data for the Rust merge.- /// </summary>- /// <param name="connection">The SQLite connection to read from.</param>- /// <param name="tableNames">The names of tables to read.</param>- /// <returns>List of TableData objects containing the table records.</returns>- private async Task<List<TableData>> ReadTablesAsJsonAsync(SqliteConnection connection, string[] tableNames)- {- var tables = new List<TableData>();-- foreach (var tableName in tableNames)- {- var tableData = new TableData { Name = tableName };-- // Check if table exists in the database.- await using var checkCommand = connection.CreateCommand();- checkCommand.CommandText = "SELECT name FROM sqlite_master WHERE type='table' AND name=@tableName";- checkCommand.Parameters.AddWithValue("@tableName", tableName);- var exists = await checkCommand.ExecuteScalarAsync();-- if (exists == null)- {- // Table doesn't exist, add empty table data.- tables.Add(tableData);- continue;- }-- // Get column names for the table.- await using var columnsCommand = connection.CreateCommand();- columnsCommand.CommandText = $"PRAGMA table_info({tableName})";- var columns = new List<string>();- await using (var columnsReader = await columnsCommand.ExecuteReaderAsync())- {- while (await columnsReader.ReadAsync())- {- columns.Add(columnsReader.GetString(1));- }- }-- // Read all records from the table.- await using var selectCommand = connection.CreateCommand();- selectCommand.CommandText = $"SELECT * FROM {tableName}";- await using var reader = await selectCommand.ExecuteReaderAsync();-- while (await reader.ReadAsync())- {- var record = new Dictionary<string, object?>();- for (var i = 0; i < columns.Count; i++)- {- var value = reader.GetValue(i);-- // Convert DBNull to null for proper JSON serialization.- record[columns[i]] = value == DBNull.Value ? null : value;- }-- tableData.Records.Add(record);- }-- tables.Add(tableData);- }-- return tables;- }-- /// <summary>- /// Executes the SQL statements returned by the Rust merge operation against the given connection.- /// </summary>- /// <param name="statements">The SQL statements to execute.</param>- /// <param name="connection">The connection to execute the statements against (the merge base).</param>- /// <returns>Task.</returns>- private async Task ExecuteMergeSqlStatementsAsync(List<SqlStatement> statements, SqliteConnection connection)- {- if (statements.Count == 0)- {- _logger.LogDebug("No SQL statements to execute from merge.");- return;- }-- _logger.LogDebug("Executing {Count} SQL statements from merge.", statements.Count);-- // Disable foreign key checks during merge execution.- await using (var pragmaCommand = connection.CreateCommand())- {- pragmaCommand.CommandText = "PRAGMA foreign_keys = OFF;";- await pragmaCommand.ExecuteNonQueryAsync();- }-- try- {- foreach (var statement in statements)- {- await using var command = connection.CreateCommand();- command.CommandText = statement.Sql;-- // Add parameters in order (SQLite uses positional parameters with ?).- for (var i = 0; i < statement.Params.Count; i++)- {- var value = statement.Params[i];-- // Handle JsonElement values from deserialization.- if (value is JsonElement jsonElement)- {- value = ConvertJsonElementToValue(jsonElement);- }-- command.Parameters.AddWithValue($"@p{i}", value ?? DBNull.Value);- }-- // Replace ? placeholders with named parameters.- var paramIndex = 0;- var sql = statement.Sql;- while (sql.Contains('?'))- {- var pos = sql.IndexOf('?');- sql = sql[..pos] + $"@p{paramIndex}" + sql[(pos + 1)..];- paramIndex++;- }-- command.CommandText = sql;- await command.ExecuteNonQueryAsync();- }- }- finally- {- // Re-enable foreign key checks.- await using var pragmaCommand = connection.CreateCommand();- pragmaCommand.CommandText = "PRAGMA foreign_keys = ON;";- await pragmaCommand.ExecuteNonQueryAsync();- }- }-/// <summary>/// Disposes the service./// </summary>@@ -1324,6 +1102,7 @@ public sealed class DbService : IDisposable// Cancel any pending background sync operations first_backgroundSyncCts.Cancel();_backgroundSyncCts.Dispose();+ _saveLock.Dispose();_sqlConnection?.Dispose();}diff --git a/apps/server/AliasVault.Client/Services/SettingsService.cs b/apps/server/AliasVault.Client/Services/SettingsService.csindex 029491108..1954c6691 100644--- a/apps/server/AliasVault.Client/Services/SettingsService.cs+++ b/apps/server/AliasVault.Client/Services/SettingsService.cs@@ -231,15 +231,25 @@ public sealed class SettingsService// Store the DbService instance for later use._dbService = dbService;+ await ReloadAsync(dbService);+ _initialized = true;+ }+ /// <summary>+ /// Re-read every setting from the database, for after the live database was replaced (a merge or migration).+ /// </summary>+ /// <param name="dbService">The database service the settings are read from.</param>+ /// <returns>Task.</returns>+ public async Task ReloadAsync(DbService dbService)+ {+ _dbService = dbService;var db = await _dbService.GetDbContextAsync();var settings = await db.Settings.ToListAsync();+ _settings.Clear();foreach (var setting in settings){_settings[setting.Key] = setting.Value;}-- _initialized = true;}/// <summary>@@ -373,11 +383,12 @@ public sealed class SettingsService}var db = await _dbService!.GetDbContextAsync();- var setting = await db.Settings.FindAsync(key);+ var setting = await db.Settings.FindAsync(_dbService.PersonalManifestId, key);if (setting == null){setting = new Setting{+ ManifestId = _dbService.PersonalManifestId,Key = key,Value = value,CreatedAt = DateTime.UtcNow,diff --git a/apps/server/AliasVault.Client/Services/VaultSync/VaultKeyService.cs b/apps/server/AliasVault.Client/Services/VaultSync/VaultKeyService.csindex 9f8a6d582..44adc28da 100644--- a/apps/server/AliasVault.Client/Services/VaultSync/VaultKeyService.cs+++ b/apps/server/AliasVault.Client/Services/VaultSync/VaultKeyService.cs@@ -8,6 +8,7 @@namespace AliasVault.Client.Services.VaultSync;using System.Net.Http.Json;+using System.Security.Cryptography;using AliasVault.Client.Services.JsInterop;using AliasVault.Client.Services.JsInterop.RustCore;using AliasVault.Client.Services.VaultSync.Exceptions;@@ -15,6 +16,7 @@ using AliasVault.Client.Services.VaultSync.Models;using AliasVault.Shared.Models.Enums;using AliasVault.Shared.Models.WebApi.V1.Auth;using AliasVault.Shared.Models.WebApi.V2.Auth;+using AliasVault.Shared.Models.WebApi.V2.Vault;using Blazored.LocalStorage;using Microsoft.JSInterop;@@ -69,7 +71,7 @@ public sealed class VaultKeyService(HttpClient httpClient, ILocalStorageService/// </summary>/// <param name="username">The username, needed to look up the salt of a legacy account.</param>/// <param name="password">The master password.</param>- /// <returns>The resolved keys. A legacy result still has to be validated by the caller.</returns>+ /// <returns>The resolved keys.</returns>/// <exception cref="VaultKeyDecryptException">Thrown when the password is wrong.</exception>/// <exception cref="VaultKeyUnavailableException">Thrown when offline and the chain this account unlocks with is not cached.</exception>public async Task<ResolvedVaultKey> UnlockWithPasswordAsync(string username, string password)@@ -153,6 +155,121 @@ public sealed class VaultKeyService(HttpClient httpClient, ILocalStorageServicereturn await localStorage.GetItemAsStringAsync(StorageKeys.AccountPublicKey);}+ /// <summary>+ /// Whether this device holds the account's encrypted key chain, meaning the account is on the account-key model.+ /// </summary>+ /// <returns>True when the chain is cached.</returns>+ public async Task<bool> HasLocalVaultKeyAsync()+ {+ return !string.IsNullOrEmpty(await localStorage.GetItemAsStringAsync(StorageKeys.EncryptedAccountKey));+ }++ /// <summary>+ /// Create a complete account key hierarchy: a random Account Key wrapped by the KEK, a random VEK and the account+ /// keypair both wrapped by the Account Key. Used on registration and on the one-time legacy migration push.+ /// </summary>+ /// <param name="kekBase64">The password-derived key that becomes the KEK.</param>+ /// <returns>The plaintext keys to adopt and the wrapped chain to upload.</returns>+ public async Task<AccountKeyHierarchy> CreateAccountKeyHierarchyAsync(string kekBase64)+ {+ var vek = RandomNumberGenerator.GetBytes(32);+ var accountKey = RandomNumberGenerator.GetBytes(32);+ var (publicKey, privateKey) = await jsInteropService.GenerateRsaKeyPair();++ var keys = new AccountKeysUpload+ {+ EncryptedAccountKey = await WrapKeyAsync(accountKey, Convert.FromBase64String(kekBase64)),+ EncryptedVek = await WrapKeyAsync(vek, accountKey),+ AccountPublicKey = publicKey,+ EncryptedAccountPrivateKey = await jsInteropService.SymmetricEncrypt(privateKey, Convert.ToBase64String(accountKey)),+ };++ return new AccountKeyHierarchy(Convert.ToBase64String(vek), privateKey, keys);+ }++ /// <summary>+ /// Adopt a hierarchy the server just committed: cache the wrapped chain for offline unlock and mark the cached+ /// derivation parameters as belonging to an account-key account.+ /// </summary>+ /// <param name="hierarchy">The hierarchy that was uploaded.</param>+ /// <returns>Task.</returns>+ public async Task AdoptLocalAccountKeysAsync(AccountKeyHierarchy hierarchy)+ {+ await localStorage.SetItemAsStringAsync(StorageKeys.EncryptedAccountKey, hierarchy.Keys.EncryptedAccountKey!);+ await localStorage.SetItemAsStringAsync(StorageKeys.EncryptedVek, hierarchy.Keys.EncryptedVek!);+ await localStorage.SetItemAsStringAsync(StorageKeys.AccountPublicKey, hierarchy.Keys.AccountPublicKey!);+ await localStorage.SetItemAsStringAsync(StorageKeys.EncryptedAccountPrivateKey, hierarchy.Keys.EncryptedAccountPrivateKey!);++ var parameters = await GetDerivationParamsAsync();+ if (parameters is not null)+ {+ await StoreDerivationParamsAsync(parameters with { HasKeyChain = true });+ }+ }++ /// <summary>+ /// Adopt a chain the server holds but this device does not (the account was migrated on another device while+ /// this session still holds the old password-derived key): open it with the session key, which then is the KEK.+ /// </summary>+ /// <param name="sessionKeyBase64">The session's current key.</param>+ /// <returns>The resolved keys when a chain was adopted, null when the server holds none. Throws when the chain does not open.</returns>+ public async Task<ResolvedVaultKey?> AdoptRemoteVaultKeyAsync(string sessionKeyBase64)+ {+ var vaultKey = await FetchVaultKeyAsync();+ if (vaultKey is null)+ {+ return null;+ }++ var resolved = await DecryptKeyChainAsync(vaultKey.EncryptedAccountKey, vaultKey.EncryptedVek, vaultKey.EncryptedAccountPrivateKey, sessionKeyBase64);+ await CacheVaultKeyBlobsAsync(vaultKey);+ return resolved;+ }++ /// <summary>+ /// Decrypt the locally cached chain with the given KEK.+ /// </summary>+ /// <param name="derivedKeyBase64">The password-derived key.</param>+ /// <returns>The resolved keys, or null when no chain is cached.</returns>+ /// <exception cref="VaultKeyDecryptException">Thrown when the chain does not open with the derived key.</exception>+ public async Task<ResolvedVaultKey?> UnlockCachedChainAsync(string derivedKeyBase64)+ {+ return await ResolveFromLocalCacheAsync(derivedKeyBase64);+ }++ /// <summary>+ /// Re-wrap the cached Account Key for a new password: open it with the current KEK, wrap it with the new one.+ /// The VEK, the account keypair and every grant stay as they are.+ /// </summary>+ /// <param name="currentKekBase64">The KEK derived from the current password.</param>+ /// <param name="newKekBase64">The KEK derived from the new password.</param>+ /// <returns>The Account Key wrapped with the new KEK.</returns>+ /// <exception cref="VaultKeyDecryptException">Thrown when the current password is wrong.</exception>+ /// <exception cref="VaultKeyUnavailableException">Thrown when no chain is cached (the account has not migrated yet).</exception>+ public async Task<string> RewrapAccountKeyAsync(string currentKekBase64, string newKekBase64)+ {+ var encryptedAccountKey = await localStorage.GetItemAsStringAsync(StorageKeys.EncryptedAccountKey);+ if (string.IsNullOrEmpty(encryptedAccountKey))+ {+ throw new VaultKeyUnavailableException(null);+ }++ var accountKey = await DecryptKeyOrThrowAsync(encryptedAccountKey, currentKekBase64);+ return await WrapKeyAsync(Convert.FromBase64String(accountKey), Convert.FromBase64String(newKekBase64));+ }++ /// <summary>+ /// Persist the re-wrapped Account Key and the new derivation parameters after a password change.+ /// </summary>+ /// <param name="newEncryptedAccountKey">The Account Key wrapped with the new KEK.</param>+ /// <param name="parameters">The new derivation parameters.</param>+ /// <returns>Task.</returns>+ public async Task PersistNewAccountKeyAsync(string newEncryptedAccountKey, EncryptionKeyDerivationParams parameters)+ {+ await localStorage.SetItemAsStringAsync(StorageKeys.EncryptedAccountKey, newEncryptedAccountKey);+ await StoreDerivationParamsAsync(parameters with { HasKeyChain = true });+ }+/// <summary>/// Persist the KEK derivation parameters./// </summary>@@ -295,6 +412,17 @@ public sealed class VaultKeyService(HttpClient httpClient, ILocalStorageService}}+ /// <summary>+ /// Wrap a 32-byte key with another key: AES-256-GCM, output base64(IV | ciphertext | tag).+ /// </summary>+ /// <param name="key">The key to wrap.</param>+ /// <param name="wrappingKey">The key that wraps it.</param>+ /// <returns>The wrapped key.</returns>+ private async Task<string> WrapKeyAsync(byte[] key, byte[] wrappingKey)+ {+ return Convert.ToBase64String(await jsInteropService.SymmetricEncryptBytes(key, wrappingKey));+ }+/// <summary>/// Decrypt a wrapped key, mapping an AES-GCM authentication failure onto <see cref="VaultKeyDecryptException"/>./// </summary>diff --git a/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncService.cs b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncService.csindex 12b740a03..f1890eea4 100644--- a/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncService.cs+++ b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncService.cs@@ -7,6 +7,7 @@namespace AliasVault.Client.Services.VaultSync;+using System.Net;using System.Net.Http.Json;using System.Text;using System.Text.Json;@@ -22,18 +23,22 @@ using Microsoft.Data.Sqlite;using Microsoft.JSInterop;/// <summary>-/// Syncs the vault with the server.+/// Syncs the vault with the server: pulls and materializes the manifest snapshot, canonicalizes and pushes the local+/// vault, and merges the two when they diverged./// </summary>/// <param name="httpClient">The HTTP client.</param>/// <param name="authService">AuthService instance, the session's key holder.</param>/// <param name="vaultKeyService">VaultKeyService instance.</param>/// <param name="jsInteropService">JsInteropService instance.</param>/// <param name="rustCoreService">RustCoreService instance.</param>-/// <param name="state">The sync state this pull records the snapshot into.</param>+/// <param name="state">The sync state this service records the server's state into.</param>+/// <param name="config">Config instance, for the private email domains the routing push claims.</param>/// <param name="logger">ILogger instance.</param>-public sealed class VaultSyncService(HttpClient httpClient, AuthService authService, VaultKeyService vaultKeyService, JsInteropService jsInteropService, RustCoreService rustCoreService, VaultSyncState state, ILogger<VaultSyncService> logger)+public sealed class VaultSyncService(HttpClient httpClient, AuthService authService, VaultKeyService vaultKeyService, JsInteropService jsInteropService, RustCoreService rustCoreService, VaultSyncState state, Config config, ILogger<VaultSyncService> logger){private const string AttachmentBlobCategory = "attachment";+ private const string BlobRefMarker = "__blobRef";+ private const string BlobKindMarker = "__blobKind";/// <summary>/// Max amount of base64 characters transferred in a single blob transfer request or response body.@@ -46,10 +51,15 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServprivate const int BlobTransferBatchMaxCount = 100;private static readonly string VaultEndpoint = ApiRoute("Vault");+ private static readonly string BlobsEndpoint = ApiRoute("Vault/blobs");+ private static readonly string BlobsMissingEndpoint = ApiRoute("Vault/blobs/missing");private static readonly string BlobsDownloadEndpoint = ApiRoute("Vault/blobs/download");+ private string? _schemaSql;+ private Dictionary<string, List<string>>? _schemaColumns;+/// <summary>- /// Gets the sync state recorded by the last pull.+ /// Gets the sync state recorded by the last pull or push./// </summary>public VaultSyncState State => state;@@ -66,9 +76,7 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ*/logger.LogInformation("[V2Pull] Step 1/3: fetching vault snapshot (GET /v2/Vault)...");var snapshot = await FetchSnapshotAsync();- state.LastServedManifestIds.Clear();- state.LastServedManifestIds.AddRange(snapshot.Manifests.Select(m => m.ManifestId));- state.LastSnapshotWasLegacySqliteBlob = snapshot.StorageFormat != StorageFormat.Manifest;+ RecordServedManifests(snapshot);// LEGACY: a not-yet-migrated account's blob is passed through unchanged.if (state.LastSnapshotWasLegacySqliteBlob)@@ -80,9 +88,9 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServif (string.IsNullOrEmpty(personalDto.Blob)){// The personal manifest exists but was never written (fresh account): the client creates the vault.- state.PersonalManifestId = personalDto.ManifestId;+ RecordEmptyPersonalManifest(personalDto);logger.LogInformation("[V2Pull] Personal manifest {ManifestId} has no content yet; a new vault will be created.", personalDto.ManifestId);- return new PullResult { Kind = PullKind.Empty, Revision = personalDto.Revision };+ return new PullResult { Kind = PullKind.Empty };}logger.LogInformation("[V2Pull] Step 1/3 done: manifests={ManifestCount}, personalRevision={Revision}, buckets={BucketCount}, blobRefs={BlobRefCount}.", snapshot.Manifests.Count, personalDto.Revision, snapshot.Buckets.Count, personalDto.BlobReferences.Count);@@ -99,8 +107,8 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServvar blobs = await DownloadReferencedBlobsAsync(opened);logger.LogInformation("[V2Pull] Step 3/3: materializing {ManifestCount} manifest(s) into a fresh SQLite database...", opened.Resolved.Count);- var database = await MaterializeAsync(opened, blobs);- return new PullResult { Kind = PullKind.Materialized, Database = database, Revision = opened.PersonalRevision };+ var database = await MaterializeAsync(opened.Resolved.Select(entry => entry.ManifestJson), opened.DataBuckets, blobs);+ return new PullResult { Kind = PullKind.Materialized, Database = database };}catch (Exception ex){@@ -109,6 +117,339 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ}}+ /// <summary>+ /// Pull the latest snapshot and merge the local vault onto it at canonical level, one manifest at a time. The+ /// server side is the base; the local side is canonicalized from the given connection.+ /// </summary>+ /// <param name="localConnection">The local vault holding the changes to keep.</param>+ /// <returns>The merged database, or the signal that the server holds nothing to merge with.</returns>+ /// <exception cref="VaultProcessingException">Thrown when the snapshot could not be opened or the merge failed.</exception>+ public async Task<PullAndMergeResult> PullAndMergeAsync(SqliteConnection localConnection)+ {+ logger.LogInformation("[V2Merge] Fetching vault snapshot for canonical merge (GET /v2/Vault)...");+ var snapshot = await FetchSnapshotAsync();+ RecordServedManifests(snapshot);++ // LEGACY: a server still on the sqlite-blob format cannot merge with a manifest-v1 vault; the caller pushes over it.+ if (state.LastSnapshotWasLegacySqliteBlob)+ {+ OpenLegacySqliteBlobSnapshot(snapshot);+ return new PullAndMergeResult { Kind = PullAndMergeKind.NothingToMergeWith };+ }++ var personalDto = SelectPersonalManifest(snapshot) ?? throw new VaultProcessingException("vault-merge", new InvalidOperationException("The server returned no personal manifest, refusing to merge."));+ if (string.IsNullOrEmpty(personalDto.Blob))+ {+ RecordEmptyPersonalManifest(personalDto);+ return new PullAndMergeResult { Kind = PullAndMergeKind.NothingToMergeWith };+ }++ var stateBeforeMerge = state.Clone();+ try+ {+ var opened = await OpenManifestsAndRecordSyncStateAsync(snapshot, personalDto);+ var serverBlobs = await DownloadReferencedBlobsAsync(opened);+ var local = await CanonicalizeAsync(localConnection, authService.GetEncryptionKeyAsBase64Async(), null);+ return await MergeOntoOpenedManifestsAsync(opened, serverBlobs, local);+ }+ catch (Exception ex)+ {+ // The local vault is still the live one, so the state must keep describing what it was pulled from.+ state.CopyFrom(stateBeforeMerge);+ logger.LogError(ex, "[V2Merge] FAILED: the last logged step above is where it broke.");+ throw new VaultProcessingException("vault-merge", ex);+ }+ }++ /// <summary>+ /// Migrate the local vault onto the current full schema canonicalize the database into+ /// manifest-v1 form (adopting rows that predate the manifest stamp into the personal manifest) and materialize+ /// it straight back out again. This is the permanent delivery path for client schema changes.+ /// </summary>+ /// <param name="connection">The local vault to migrate.</param>+ /// <returns>The migrated in-memory database; the caller owns it.</returns>+ /// <exception cref="VaultProcessingException">Thrown when the vault could not be canonicalized or materialized.</exception>+ public async Task<SqliteConnection> MigrateVaultToCurrentSchemaAsync(SqliteConnection connection)+ {+ logger.LogInformation("[ManifestMigration] Migrating the local vault onto the current schema (local round-trip, no server involved)...");+ try+ {+ var personalManifestId = state.PersonalManifestId ?? throw new InvalidOperationException("No personal manifest id is recorded; pull once before migrating.");+ var vault = await CanonicalizeAsync(connection, authService.GetEncryptionKeyAsBase64Async(), personalManifestId);++ // Canonicalize already extracted every favicon and attachment as plaintext bytes, so materialize resolves its references without a fetch.+ var blobs = new Dictionary<string, byte[]>(StringComparer.Ordinal);+ foreach (var manifest in vault.Manifests)+ {+ foreach (var (hash, blob) in manifest.Blobs)+ {+ blobs[hash] = blob.Bytes;+ }+ }++ var database = await MaterializeAsync(vault.Manifests.Select(manifest => manifest.ManifestJson), vault.Buckets.Select(bucket => bucket.BucketJson), blobs);+ logger.LogInformation("[ManifestMigration] Migration complete: {BlobCount} blob(s) re-embedded.", blobs.Count);+ return database;+ }+ catch (Exception ex)+ {+ logger.LogError(ex, "[ManifestMigration] FAILED to migrate the local vault.");+ throw new VaultProcessingException("vault-storage-migration", ex);+ }+ }++ /// <summary>+ /// Canonicalize the local vault, validate, encrypt and POST /v2/Vault.+ /// </summary>+ /// <param name="connection">The local vault to push.</param>+ /// <param name="options">How to write.</param>+ /// <returns>The outcome. Revisions, fingerprints and the blob caches are updated on success.</returns>+ /// <exception cref="VaultTooLargeException">Thrown when the server refuses an upload as too large.</exception>+ public async Task<PushResult> PushAsync(SqliteConnection connection, PushOptions options)+ {+ var personalManifestId = state.PersonalManifestId ?? throw new InvalidOperationException("No personal manifest id is recorded; pull once before pushing.");++ var unwritable = await FindUnwritableManifestsAsync(connection, personalManifestId);+ if (unwritable.Count > 0)+ {+ logger.LogWarning("[V2Push] Vault holds rows for manifest(s) this session cannot write ({Manifests}); refusing the write until a pull restores them.", string.Join(", ", unwritable));+ return new PushResult(PushStatus.Outdated, [$"Manifest(s) {string.Join(", ", unwritable)} are not open to this session"]);+ }++ // LEGACY: the one-time migration push creates the account key hierarchy and re-keys the personal manifest under the new VEK.+ var migration = options.CreateVaultKey ? await vaultKeyService.CreateAccountKeyHierarchyAsync(authService.GetEncryptionKeyAsBase64Async()) : null;+ var contentKey = migration?.VaultEncryptionKey ?? authService.GetEncryptionKeyAsBase64Async();+ if (migration is not null)+ {+ logger.LogInformation("[V2Push] Account-key migration: generated a new VEK, AK and account keypair; vault content and all blobs will be re-encrypted and re-uploaded.");+ }++ var vault = await CanonicalizeAsync(connection, contentKey, null);+ var emailRouting = EmailRoutingBuilder.Build(vault.Manifests.Select(manifest => manifest.ManifestJson), PrivateEmailDomains());+ logger.LogInformation("[V2Push] Canonicalize produced {ManifestCount} manifest(s) and {BucketCount} data bucket(s).", vault.Manifests.Count, vault.Buckets.Count);++ // Content-fingerprint gating: compare every canonicalized target against the fingerprint of its last-known server state and only write the targets that changed.+ var keyByManifestId = vault.Manifests.ToDictionary(manifest => manifest.Record.ManifestId, manifest => manifest.Record.VaultEncryptionKey);+ var bucketWrites = new List<BucketWrite>();+ var writtenBucketFingerprints = new Dictionary<string, string>(StringComparer.Ordinal);+ foreach (var bucket in vault.Buckets)+ {+ var label = $"Data bucket \"{bucket.Category}\" of manifest {bucket.ManifestId}";+ var fingerprintKey = VaultSyncState.BucketFingerprintKey(bucket.ManifestId, bucket.Category);+ var fingerprint = await rustCoreService.VaultCodecComputeContentFingerprintAsync(bucket.BucketJson);+ if (!options.ForceFullWrite && migration is null && state.ContentFingerprints.TryGetValue(fingerprintKey, out var baseline) && baseline == fingerprint)+ {+ continue;+ }++ if (!keyByManifestId.TryGetValue(bucket.ManifestId, out var bucketKey))+ {+ logger.LogWarning("[V2Push] {Label} names a manifest this vault cannot write; leaving it out of this write.", label);+ continue;+ }++ if (!Enum.TryParse<VaultDataBucketCategory>(bucket.Category, true, out var category))+ {+ logger.LogWarning("[V2Push] {Label} has a category this client does not know; leaving it out of this write.", label);+ continue;+ }++ var validation = await rustCoreService.VaultCodecValidateDataBucketAsync(bucket.BucketJson);+ if (!validation.Ok)+ {+ return new PushResult(PushStatus.Rejected, [$"{label} validation failed: {string.Join(", ", validation.FailedRules)}. {validation.Message}".Trim()]);+ }++ var ciphertext = await rustCoreService.VaultCodecPackAndEncryptPayloadAsync(bucket.BucketJson, bucketKey);+ bucketWrites.Add(new BucketWrite+ {+ ManifestId = bucket.ManifestId,+ Category = category,+ Blob = ciphertext,+ CiphertextHash = await rustCoreService.VaultCodecComputeCiphertextHashAsync(ciphertext),+ CurrentRevision = state.BucketRevisions.GetValueOrDefault(VaultSyncState.BucketRevisionKey(bucket.ManifestId, bucket.Category), 0),+ });+ writtenBucketFingerprints[fingerprintKey] = fingerprint;+ }++ // Gate, validate, pack and encrypt every candidate manifest into the write, each with its own VEK.+ var blobEntries = new Dictionary<string, UploadBlobEntry>(StringComparer.Ordinal);+ var manifestWrites = new List<ManifestWrite>();+ var writtenManifestFingerprints = new Dictionary<Guid, string>();+ foreach (var manifest in vault.Manifests)+ {+ foreach (var (hash, blob) in manifest.Blobs)+ {+ blobEntries.TryAdd(hash, new UploadBlobEntry(blob.Kind, blob.Bytes, manifest.Record.VaultEncryptionKey, manifest.Record.IsPersonal));+ }++ var label = manifest.Record.IsPersonal ? "Personal manifest" : $"Shared manifest \"{manifest.Record.Name ?? manifest.Record.ManifestId.ToString()}\"";+ var fingerprint = await rustCoreService.VaultCodecComputeContentFingerprintAsync(manifest.ManifestJson);+ var rekeyed = migration is not null && manifest.Record.IsPersonal;+ if (!options.ForceFullWrite && !rekeyed && state.ContentFingerprints.TryGetValue(VaultSyncState.ManifestFingerprintKey(manifest.Record.ManifestId), out var baseline) && baseline == fingerprint)+ {+ continue;+ }++ var validation = await rustCoreService.VaultCodecValidateManifestAsync(manifest.ManifestJson);+ if (!validation.Ok)+ {+ if (manifest.Record.IsPersonal)+ {+ return new PushResult(PushStatus.Rejected, [$"Manifest validation failed: {string.Join(", ", validation.FailedRules)}. {validation.Message}".Trim()]);+ }++ logger.LogWarning("[V2Push] {Label} failed validation ({Rules}), dropping it from this write.", label, string.Join(", ", validation.FailedRules));+ continue;+ }++ var ciphertext = await rustCoreService.VaultCodecPackAndEncryptPayloadAsync(manifest.ManifestJson, manifest.Record.VaultEncryptionKey);++ // Publish the public half of the manifest's email delivery keypair; only admins may publish a shared manifest's key.+ var mayPublish = manifest.Record.IsPersonal || manifest.Record.CanAdminister;+ var publicKey = mayPublish ? await VaultTableReader.ReadActivePublicKeyAsync(connection, manifest.Record.ManifestId) : null;+ if (mayPublish && publicKey is null && !manifest.Record.IsPersonal)+ {+ logger.LogWarning("[V2Push] {Label} is missing its email keypair; its aliases stay personal until sharing is re-enabled.", label);+ }++ manifestWrites.Add(new ManifestWrite+ {+ ManifestId = manifest.Record.ManifestId,+ ManifestBlob = ciphertext,+ ManifestCiphertextHash = await rustCoreService.VaultCodecComputeCiphertextHashAsync(ciphertext),+ CurrentRevision = state.ManifestRevisions.GetValueOrDefault(manifest.Record.ManifestId, 0),+ CredentialsCount = manifest.ItemCount,+ BlobReferences = manifest.Blobs.Select(pair => new BlobReference { Hash = pair.Key, Category = pair.Value.Kind }).ToList(),+ EncryptionPublicKey = publicKey,+ });+ writtenManifestFingerprints[manifest.Record.ManifestId] = fingerprint;+ logger.LogInformation("[V2Push] {Label}: {Items} item(s), {Blobs} blob reference(s), {Chars} encrypted characters.", label, manifest.ItemCount, manifest.Blobs.Count, ciphertext.Length);+ }++ // Nothing changed versus the server baselines: skip the write (and the blob diff) entirely.+ if (manifestWrites.Count == 0 && bucketWrites.Count == 0)+ {+ logger.LogInformation("[V2Push] No content changes detected (every manifest and data bucket matches the server baselines); skipping upload.");+ return new PushResult(PushStatus.Ok);+ }++ // Blob diff across every manifest in this write: only encrypt and upload blobs the server does not already have.+ var allBlobHashes = blobEntries.Keys.ToList();+ var personalHashes = allBlobHashes.Where(hash => blobEntries[hash].FromPersonal).ToList();+ var sharedHashes = allBlobHashes.Where(hash => !blobEntries[hash].FromPersonal).ToList();+ List<string> personalToUpload;+ List<string> sharedToUpload;+ if (migration is not null)+ {+ personalToUpload = personalHashes;+ sharedToUpload = await MissingOnServerAsync(sharedHashes.Where(hash => !state.ServerBlobHashes.Contains(hash)));+ }+ else+ {+ var toUpload = new HashSet<string>(await MissingOnServerAsync(allBlobHashes.Where(hash => !state.ServerBlobHashes.Contains(hash))), StringComparer.Ordinal);+ personalToUpload = personalHashes.Where(toUpload.Contains).ToList();+ sharedToUpload = sharedHashes.Where(toUpload.Contains).ToList();+ }++ logger.LogInformation("[V2Push] Blob diff: {Total} blob(s) across {Manifests} manifest(s), uploading {Personal} personal and {Shared} shared.", allBlobHashes.Count, vault.Manifests.Count, personalToUpload.Count, sharedToUpload.Count);+ var uploadedCiphertexts = new Dictionary<string, string>(StringComparer.Ordinal);+ await UploadBlobsAsync(blobEntries, personalToUpload, migration is not null, uploadedCiphertexts);+ await UploadBlobsAsync(blobEntries, sharedToUpload, false, uploadedCiphertexts);++ var request = new VaultWriteRequest+ {+ Username = await authService.GetUsernameAsync(),+ Manifests = manifestWrites,+ Buckets = bucketWrites,+ NewBlobs = [],+ EmailRouting = emailRouting,+ AccountKeys = migration?.Keys,+ };++ var response = await PostWriteAsync(request);+ if (response.MissingBlobHashes.Count > 0)+ {+ // Upload any blobs the server reports as missing and retry the write.+ var unsatisfiable = response.MissingBlobHashes.Where(hash => !blobEntries.ContainsKey(hash)).ToList();+ if (unsatisfiable.Count > 0)+ {+ return new PushResult(PushStatus.MissingBlobs, unsatisfiable);+ }++ logger.LogWarning("[V2Push] Server reported {Count} missing blob(s); uploading and retrying once.", response.MissingBlobHashes.Count);+ await UploadBlobsAsync(blobEntries, response.MissingBlobHashes.Where(hash => blobEntries[hash].FromPersonal), migration is not null, uploadedCiphertexts);+ await UploadBlobsAsync(blobEntries, response.MissingBlobHashes.Where(hash => !blobEntries[hash].FromPersonal), false, uploadedCiphertexts);+ response = await PostWriteAsync(request);+ if (response.MissingBlobHashes.Count > 0)+ {+ return new PushResult(PushStatus.MissingBlobs, response.MissingBlobHashes);+ }+ }++ if (response.Status != VaultStatus.Ok)+ {+ // A single stale manifest or bucket rejected the whole write; the caller pulls, merges and retries.+ return new PushResult(PushStatus.Outdated);+ }++ // Advance the baselines of exactly the targets this write carried.+ foreach (var bucketRevision in response.BucketRevisions)+ {+ state.BucketRevisions[VaultSyncState.BucketRevisionKey(bucketRevision.ManifestId, bucketRevision.Category.ToString())] = bucketRevision.Revision;+ }++ foreach (var manifestRevision in response.ManifestRevisions)+ {+ state.ManifestRevisions[manifestRevision.ManifestId] = manifestRevision.Revision;+ }++ foreach (var (key, fingerprint) in writtenBucketFingerprints)+ {+ state.ContentFingerprints[key] = fingerprint;+ }++ foreach (var (manifestId, fingerprint) in writtenManifestFingerprints)+ {+ state.ContentFingerprints[VaultSyncState.ManifestFingerprintKey(manifestId)] = fingerprint;+ }++ // Every referenced hash is now known to be on the server; refresh the diff baseline and the cache.+ state.ServerBlobHashes.Clear();+ state.ServerBlobHashes.UnionWith(allBlobHashes);+ var refreshedCache = new Dictionary<string, string>(StringComparer.Ordinal);+ foreach (var hash in allBlobHashes)+ {+ if (uploadedCiphertexts.TryGetValue(hash, out var uploaded))+ {+ refreshedCache[hash] = uploaded;+ }+ else if (state.BlobCipherCache.TryGetValue(hash, out var cached))+ {+ refreshedCache[hash] = cached;+ }+ }++ state.BlobCipherCache.Clear();+ foreach (var (hash, ciphertext) in refreshedCache)+ {+ state.BlobCipherCache[hash] = ciphertext;+ }++ state.LastSnapshotWasLegacySqliteBlob = false;++ // Adopt the newly created account keys if this push included the one-time sqlite-blob to manifest-v1 migration.+ if (migration is not null)+ {+ await vaultKeyService.AdoptLocalAccountKeysAsync(migration);+ await authService.StoreSessionKeysAsync(Convert.FromBase64String(migration.VaultEncryptionKey), migration.AccountPrivateKey);+ logger.LogInformation("[V2Push] Account-key migration complete: hierarchy created server-side, chain cached locally.");+ }++ logger.LogInformation("[V2Push] Pushed {Manifests} manifest(s), {Buckets} bucket(s) and {Blobs} blob(s).", manifestWrites.Count, bucketWrites.Count, uploadedCiphertexts.Count);+ return new PushResult(PushStatus.Ok);+ }+/// <summary>/// Pick the user's personal manifest out of a snapshot./// </summary>@@ -218,37 +559,150 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServusing (var writer = new Utf8JsonWriter(stream)){writer.WriteStartObject();- writer.WriteStartArray("manifests");- foreach (var manifestJson in manifestJsons)+ WriteRawArray(writer, "manifests", manifestJsons);+ WriteRawArray(writer, "dataBuckets", bucketJsons);+ WriteSchemaColumns(writer, schemaColumns);+ writer.WriteEndObject();+ }++ return Encoding.UTF8.GetString(stream.ToArray());+ }++ /// <summary>+ /// Build the JSON input of the Rust canonical merge call.+ /// </summary>+ /// <param name="opened">The opened server manifests, the merge base.</param>+ /// <param name="local">The canonicalized local vault, the incoming side.</param>+ /// <param name="schemaColumns">The column set of the local schema, per table.</param>+ /// <returns>The canonical merge input JSON.</returns>+ private static string BuildMergeInput(OpenedManifestSet opened, CanonicalizedVault local, IReadOnlyDictionary<string, List<string>> schemaColumns)+ {+ using var stream = new MemoryStream();+ using (var writer = new Utf8JsonWriter(stream))+ {+ writer.WriteStartObject();+ WriteRawArray(writer, "serverManifests", opened.Resolved.Select(entry => entry.ManifestJson));+ WriteRawArray(writer, "serverBuckets", opened.DataBuckets);+ writer.WriteStartArray("contentlessServerManifestIds");+ foreach (var id in opened.ContentlessManifestIds){- writer.WriteRawValue(manifestJson, skipInputValidation: true);+ writer.WriteStringValue(id.ToString());}writer.WriteEndArray();- writer.WriteStartArray("dataBuckets");- foreach (var bucketJson in bucketJsons)+ WriteRawArray(writer, "localManifests", local.Manifests.Select(manifest => manifest.ManifestJson));+ WriteRawArray(writer, "localBuckets", local.Buckets.Select(bucket => bucket.BucketJson));+ WriteSchemaColumns(writer, schemaColumns);+ writer.WriteEndObject();+ }++ return Encoding.UTF8.GetString(stream.ToArray());+ }++ /// <summary>+ /// Write a named array of pre-serialized JSON values.+ /// </summary>+ /// <param name="writer">The writer.</param>+ /// <param name="name">The property name.</param>+ /// <param name="rawValues">The JSON values.</param>+ private static void WriteRawArray(Utf8JsonWriter writer, string name, IEnumerable<string> rawValues)+ {+ writer.WriteStartArray(name);+ foreach (var rawValue in rawValues)+ {+ writer.WriteRawValue(rawValue, skipInputValidation: true);+ }++ writer.WriteEndArray();+ }++ /// <summary>+ /// Write the schemaColumns object.+ /// </summary>+ /// <param name="writer">The writer.</param>+ /// <param name="schemaColumns">The column set of the local schema, per table.</param>+ private static void WriteSchemaColumns(Utf8JsonWriter writer, IReadOnlyDictionary<string, List<string>> schemaColumns)+ {+ writer.WriteStartObject("schemaColumns");+ foreach (var (table, columns) in schemaColumns)+ {+ writer.WriteStartArray(table);+ foreach (var column in columns){- writer.WriteRawValue(bucketJson, skipInputValidation: true);+ writer.WriteStringValue(column);}writer.WriteEndArray();- writer.WriteStartObject("schemaColumns");- foreach (var (table, columns) in schemaColumns)+ }++ writer.WriteEndObject();+ }++ /// <summary>+ /// Every blob reference marker in a manifest payload: hash and kind.+ /// </summary>+ /// <param name="manifestJson">The manifest payload JSON.</param>+ /// <returns>The references.</returns>+ private static List<(string Hash, string? Kind)> BlobReferencesOf(string manifestJson)+ {+ var references = new List<(string Hash, string? Kind)>();+ using var document = JsonDocument.Parse(manifestJson);+ if (!document.RootElement.TryGetProperty("tables", out var tables) || tables.ValueKind != JsonValueKind.Object)+ {+ return references;+ }++ foreach (var table in tables.EnumerateObject())+ {+ if (table.Value.ValueKind != JsonValueKind.Array){- writer.WriteStartArray(table);- foreach (var column in columns)+ continue;+ }++ foreach (var row in table.Value.EnumerateArray())+ {+ foreach (var cell in row.EnumerateObject()){- writer.WriteStringValue(column);+ if (cell.Value.ValueKind == JsonValueKind.Object && cell.Value.TryGetProperty(BlobRefMarker, out var hash) && hash.ValueKind == JsonValueKind.String)+ {+ var kind = cell.Value.TryGetProperty(BlobKindMarker, out var kindElement) && kindElement.ValueKind == JsonValueKind.String ? kindElement.GetString() : null;+ references.Add((hash.GetString()!, kind));+ }}-- writer.WriteEndArray();}+ }- writer.WriteEndObject();- writer.WriteEndObject();+ return references;+ }++ /// <summary>+ /// Turn a failed upload response into an exception that names the failure.+ /// </summary>+ /// <param name="response">The response.</param>+ /// <param name="what">What was uploaded, for the message.</param>+ /// <returns>Task.</returns>+ private static async Task EnsureUploadSucceededAsync(HttpResponseMessage response, string what)+ {+ if (response.StatusCode == HttpStatusCode.RequestEntityTooLarge)+ {+ throw new VaultTooLargeException();}- return Encoding.UTF8.GetString(stream.ToArray());+ if (!response.IsSuccessStatusCode)+ {+ var body = await response.Content.ReadAsStringAsync();+ throw new HttpRequestException($"{what} failed with HTTP {(int)response.StatusCode}: {body}", null, response.StatusCode);+ }+ }++ /// <summary>+ /// The domains the server hosts mail for; addresses outside them are never claimed.+ /// </summary>+ /// <returns>The domains.</returns>+ private List<string> PrivateEmailDomains()+ {+ // "DISABLED.TLD" was a placeholder used before 0.22.0 that has been replaced by an empty string. TODO: remove in a future release.+ return config.PrivateEmailDomains.Where(domain => !string.IsNullOrWhiteSpace(domain) && !string.Equals(domain, "DISABLED.TLD", StringComparison.OrdinalIgnoreCase)).ToList();}/// <summary>@@ -260,6 +714,31 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServreturn await httpClient.GetFromJsonAsync<GetResponse>(VaultEndpoint) ?? throw new InvalidOperationException("Empty vault snapshot response.");}+ /// <summary>+ /// Record which manifests a snapshot served, before any of them is opened: it is the authority for held access.+ /// </summary>+ /// <param name="snapshot">The raw snapshot.</param>+ private void RecordServedManifests(GetResponse snapshot)+ {+ state.LastServedManifestIds.Clear();+ state.LastServedManifestIds.AddRange(snapshot.Manifests.Select(m => m.ManifestId));+ state.LastSnapshotWasLegacySqliteBlob = snapshot.StorageFormat != StorageFormat.Manifest;+ }++ /// <summary>+ /// Record a personal manifest that exists but was never written: its id and revision are the baseline the first push rebases on.+ /// </summary>+ /// <param name="personalDto">The personal manifest.</param>+ private void RecordEmptyPersonalManifest(Manifest personalDto)+ {+ state.PersonalManifestId = personalDto.ManifestId;+ state.ContentFingerprints.Clear();+ state.SharedManifests.Clear();+ state.BucketRevisions.Clear();+ state.ManifestRevisions.Clear();+ state.ManifestRevisions[personalDto.ManifestId] = personalDto.Revision;+ }+/// <summary>/// Take a legacy snapshot apart: the blob passes through untouched. TODO: remove when all accounts have been migrated to the new manifest-v1 format./// </summary>@@ -282,7 +761,7 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ}logger.LogInformation("[V2Pull] Legacy sqlite-blob pass-through (account not yet migrated), returning the blob as-is.");- return new PullResult { Kind = PullKind.LegacySqliteBlob, LegacyVaultBlob = snapshot.LegacyVaultBlob ?? string.Empty, LegacyVersion = snapshot.Version ?? string.Empty, Revision = revision };+ return new PullResult { Kind = PullKind.LegacySqliteBlob, LegacyVaultBlob = snapshot.LegacyVaultBlob ?? string.Empty, LegacyVersion = snapshot.Version ?? string.Empty };}/// <summary>@@ -649,24 +1128,416 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServlogger.LogWarning("[V2Pull] Referenced {Category} blob {Hash} {Reason}, continuing without it.", reference.Category, reference.Hash, reason);}+ /// <summary>+ /// The merge core: run the Rust canonical merge of the local side against the manifests just opened, validate the+ /// result per manifest (the server's copy stands for one that fails), and materialize it.+ /// </summary>+ /// <param name="opened">The opened server manifests, the merge base.</param>+ /// <param name="serverBlobs">The plaintext blobs the server manifests reference.</param>+ /// <param name="local">The canonicalized local vault, the incoming side.</param>+ /// <returns>The merged result.</returns>+ private async Task<PullAndMergeResult> MergeOntoOpenedManifestsAsync(OpenedManifestSet opened, Dictionary<string, byte[]> serverBlobs, CanonicalizedVault local)+ {+ logger.LogInformation("[V2Merge] Merging {Local} local manifest(s) onto {Server} server manifest(s)...", local.Manifests.Count, opened.Resolved.Count);+ var mergeOutputJson = await rustCoreService.MergeCanonicalAsync(BuildMergeInput(opened, local, await GetSchemaColumnsAsync()));++ var serverManifestById = opened.Resolved.ToDictionary(entry => entry.ManifestId, entry => entry.ManifestJson);+ var serverBucketsById = new Dictionary<Guid, List<string>>();+ foreach (var bucketJson in opened.DataBuckets)+ {+ var owner = ReadPayloadHeader(bucketJson).ManifestId ?? Guid.Empty;+ serverBucketsById.TryAdd(owner, []);+ serverBucketsById[owner].Add(bucketJson);+ }++ var contentless = new HashSet<Guid>(opened.ContentlessManifestIds);+ var manifestJsons = new List<(Guid ManifestId, string Json)>();+ var bucketJsons = new List<string>();+ var fallbackManifestIds = new List<Guid>();+ var droppedLocalManifestIds = new List<Guid>();+ using (var document = JsonDocument.Parse(mergeOutputJson))+ {+ var root = document.RootElement;+ foreach (var entry in root.GetProperty("manifests").EnumerateArray())+ {+ var manifestId = Guid.Parse(entry.GetProperty("manifestId").GetString()!);+ var manifestJson = entry.GetProperty("manifest").GetRawText();+ var mergedBuckets = entry.TryGetProperty("buckets", out var buckets) && buckets.ValueKind == JsonValueKind.Array ? buckets.EnumerateArray().Select(bucket => bucket.GetRawText()).ToList() : [];+ var failure = await ValidateMergedManifestAsync(manifestJson, mergedBuckets);+ if (failure is not null && !contentless.Contains(manifestId))+ {+ logger.LogWarning("[V2Merge] Merged manifest {ManifestId} failed validation ({Failure}); the server's version stands and local changes to it are dropped.", manifestId, failure);+ fallbackManifestIds.Add(manifestId);+ if (serverManifestById.TryGetValue(manifestId, out var serverJson))+ {+ manifestJsons.Add((manifestId, serverJson));+ bucketJsons.AddRange(serverBucketsById.GetValueOrDefault(manifestId, []));+ }++ continue;+ }++ if (failure is not null)+ {+ // A contentless pass-through has no server base to fall back to; keep the local rows, the push gate decides.+ logger.LogWarning("[V2Merge] Pass-through manifest {ManifestId} failed validation ({Failure}); keeping its local rows.", manifestId, failure);+ }++ manifestJsons.Add((manifestId, manifestJson));+ bucketJsons.AddRange(mergedBuckets);+ if (entry.TryGetProperty("stats", out var stats))+ {+ logger.LogInformation("[V2Merge] Manifest {ManifestId}: {Stats}", manifestId, stats.GetRawText());+ }+ }++ if (root.TryGetProperty("droppedLocalManifestIds", out var dropped) && dropped.ValueKind == JsonValueKind.Array)+ {+ foreach (var id in dropped.EnumerateArray())+ {+ if (Guid.TryParse(id.GetString(), out var droppedId))+ {+ droppedLocalManifestIds.Add(droppedId);+ logger.LogWarning("[V2Merge] Local manifest {ManifestId} is no longer served; its rows are dropped from the merged vault.", droppedId);+ }+ }+ }+ }++ // Blob bytes for materialize: the server download plus everything the local canonicalize extracted.+ var blobMap = new Dictionary<string, byte[]>(serverBlobs, StringComparer.Ordinal);+ foreach (var manifest in local.Manifests)+ {+ foreach (var (hash, blob) in manifest.Blobs)+ {+ blobMap.TryAdd(hash, blob.Bytes);+ }+ }++ var personalManifestId = opened.Resolved[0].ManifestId;+ foreach (var (manifestId, manifestJson) in manifestJsons)+ {+ foreach (var (hash, kind) in BlobReferencesOf(manifestJson))+ {+ if (blobMap.ContainsKey(hash))+ {+ continue;+ }++ if (manifestId == personalManifestId && string.Equals(kind, AttachmentBlobCategory, StringComparison.OrdinalIgnoreCase))+ {+ throw new InvalidOperationException($"The merged vault references attachment blob {hash} with no bytes available, refusing to materialize an incomplete vault.");+ }++ logger.LogWarning("[V2Merge] Merged {Kind} blob {Hash} has no bytes available; it will materialize as empty.", kind ?? "blob", hash);+ }+ }++ var database = await MaterializeAsync(manifestJsons.Select(entry => entry.Json), bucketJsons, blobMap);+ logger.LogInformation("[V2Merge] Canonical merge complete: {Fallbacks} validation fallback(s), {Dropped} dropped local manifest(s).", fallbackManifestIds.Count, droppedLocalManifestIds.Count);+ return new PullAndMergeResult { Kind = PullAndMergeKind.Merged, Database = database, FallbackManifestIds = fallbackManifestIds, DroppedLocalManifestIds = droppedLocalManifestIds };+ }++ /// <summary>+ /// Validate one merged manifest and its buckets.+ /// </summary>+ /// <param name="manifestJson">The merged manifest.</param>+ /// <param name="bucketJsons">Its merged buckets.</param>+ /// <returns>The failed rules, or null when valid.</returns>+ private async Task<string?> ValidateMergedManifestAsync(string manifestJson, List<string> bucketJsons)+ {+ var validation = await rustCoreService.VaultCodecValidateManifestAsync(manifestJson);+ if (!validation.Ok)+ {+ return string.Join(", ", validation.FailedRules);+ }++ foreach (var bucketJson in bucketJsons)+ {+ var bucketValidation = await rustCoreService.VaultCodecValidateDataBucketAsync(bucketJson);+ if (!bucketValidation.Ok)+ {+ return $"bucket \"{ReadPayloadHeader(bucketJson).Category}\": {string.Join(", ", bucketValidation.FailedRules)}";+ }+ }++ return null;+ }++ /// <summary>+ /// Canonicalize the local vault into the manifest-v1 format against every manifest this vault writes, routing each+ /// row into its own manifest by the ManifestId it carries.+ /// </summary>+ /// <param name="connection">The local vault.</param>+ /// <param name="personalVek">The key the personal manifest encrypts with.</param>+ /// <param name="adoptUnstampedInto">One-time migration only: the manifest rows without a stamp are adopted into.</param>+ /// <returns>The canonicalized vault, personal manifest first.</returns>+ private async Task<CanonicalizedVault> CanonicalizeAsync(SqliteConnection connection, string personalVek, Guid? adoptUnstampedInto)+ {+ var records = await ResolveManifestRecordsAsync(connection, personalVek);+ var tablesJson = await VaultTableReader.ReadTablesAsCodecJsonAsync(connection);++ string inputJson;+ using (var stream = new MemoryStream())+ {+ using (var writer = new Utf8JsonWriter(stream))+ {+ writer.WriteStartObject();+ writer.WritePropertyName("tables");+ writer.WriteRawValue(tablesJson, skipInputValidation: true);+ writer.WriteString("canonicalizedAt", DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ", System.Globalization.CultureInfo.InvariantCulture));+ writer.WriteStartArray("manifests");+ foreach (var record in records)+ {+ writer.WriteStartObject();+ writer.WriteString("manifestId", record.ManifestId.ToString());+ writer.WriteString("manifestSalt", record.Salt);+ if (record.Name is not null)+ {+ writer.WriteString("name", record.Name);+ }++ writer.WriteEndObject();+ }++ writer.WriteEndArray();+ if (adoptUnstampedInto is { } adoptInto)+ {+ writer.WriteString("adoptUnstampedInto", adoptInto.ToString());+ }++ writer.WriteEndObject();+ }++ inputJson = Encoding.UTF8.GetString(stream.ToArray());+ }++ var outputJson = await rustCoreService.VaultCodecCanonicalizeFromSqliteAsync(inputJson);+ return ParseCanonicalized(outputJson, records);+ }++ /// <summary>+ /// Read the codec's canonicalize output into the shapes the push works with.+ /// </summary>+ /// <param name="outputJson">The CanonicalizedVault JSON.</param>+ /// <param name="records">The records the vault was split against.</param>+ /// <returns>The canonicalized vault.</returns>+ private CanonicalizedVault ParseCanonicalized(string outputJson, List<ManifestRecord> records)+ {+ using var document = JsonDocument.Parse(outputJson);+ var root = document.RootElement;+ var manifests = new List<CanonicalizedManifest>();+ foreach (var entry in root.GetProperty("manifests").EnumerateArray())+ {+ var manifest = entry.GetProperty("manifest");+ var manifestId = Guid.Parse(manifest.GetProperty("manifestId").GetString()!);+ var record = records.FirstOrDefault(candidate => candidate.ManifestId == manifestId);+ if (record is null)+ {+ logger.LogWarning("[V2Push] Canonicalize produced manifest {ManifestId}, which is not in the write set; leaving it out.", manifestId);+ continue;+ }++ var itemCount = manifest.TryGetProperty("tables", out var tables) && tables.TryGetProperty("Items", out var items) && items.ValueKind == JsonValueKind.Array ? items.GetArrayLength() : 0;+ var blobs = new Dictionary<string, CanonicalizedBlob>(StringComparer.Ordinal);+ if (entry.TryGetProperty("blobs", out var blobsElement) && blobsElement.ValueKind == JsonValueKind.Object)+ {+ foreach (var blob in blobsElement.EnumerateObject())+ {+ blobs[blob.Name] = new CanonicalizedBlob(blob.Value.GetProperty("kind").GetString() ?? string.Empty, blob.Value.GetProperty("bytesBase64").GetBytesFromBase64());+ }+ }++ manifests.Add(new CanonicalizedManifest(record, manifest.GetRawText(), itemCount, blobs));+ }++ var buckets = new List<CanonicalizedBucket>();+ if (root.TryGetProperty("dataBuckets", out var dataBuckets) && dataBuckets.ValueKind == JsonValueKind.Array)+ {+ foreach (var bucket in dataBuckets.EnumerateArray())+ {+ buckets.Add(new CanonicalizedBucket(Guid.Parse(bucket.GetProperty("manifestId").GetString()!), bucket.GetProperty("category").GetString() ?? string.Empty, bucket.GetRawText()));+ }+ }++ return new CanonicalizedVault(manifests, buckets);+ }++ /// <summary>+ /// Resolve every manifest this vault can write, personal manifest first.+ /// </summary>+ /// <param name="connection">The local vault.</param>+ /// <param name="personalVek">The key the personal manifest encrypts with.</param>+ /// <returns>The records.</returns>+ private async Task<List<ManifestRecord>> ResolveManifestRecordsAsync(SqliteConnection connection, string personalVek)+ {+ var personalManifestId = state.PersonalManifestId ?? throw new InvalidOperationException("No personal manifest id is recorded; pull once before pushing.");+ state.PersonalManifestSalt ??= await rustCoreService.VaultCodecGenerateManifestSaltAsync();++ var opened = state.SharedManifests.Values.Where(record => !string.IsNullOrEmpty(record.VaultEncryptionKey)).ToList();+ var request = new+ {+ personalManifestId = personalManifestId.ToString(),+ personalManifestSalt = state.PersonalManifestSalt,+ stampedManifestIds = (await VaultTableReader.ManifestIdsInVaultAsync(connection)).ToList(),+ openedManifestIds = opened.Select(record => record.ManifestId.ToString()).ToList(),+ heldRecords = state.SharedManifests.Values.Select(record => new { manifestId = record.ManifestId.ToString(), salt = record.Salt, name = record.Name, canAdminister = record.CanAdminister }).ToList(),+ displayNames = await VaultTableReader.ReadDisplayNamesAsync(connection),+ };++ var writeSetJson = await rustCoreService.VaultSharingResolveManifestWriteSetAsync(JsonSerializer.Serialize(request));+ using var document = JsonDocument.Parse(writeSetJson);+ var root = document.RootElement;+ if (root.TryGetProperty("skipped", out var skipped) && skipped.ValueKind == JsonValueKind.Array)+ {+ foreach (var entry in skipped.EnumerateArray())+ {+ logger.LogInformation("[V2Push] Shared manifest {ManifestId} is left out of the write: {Reason}.", entry.GetProperty("manifestId").GetString(), entry.GetProperty("reason").GetString());+ }+ }++ var records = new List<ManifestRecord>();+ foreach (var entry in root.GetProperty("records").EnumerateArray())+ {+ var manifestId = Guid.Parse(entry.GetProperty("manifestId").GetString()!);+ var isPersonal = entry.GetProperty("isPersonal").GetBoolean();+ var name = entry.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null;+ var canAdminister = entry.TryGetProperty("canAdminister", out var adminElement) && adminElement.ValueKind == JsonValueKind.True;+ var vek = isPersonal ? personalVek : state.SharedManifests.GetValueOrDefault(manifestId)?.VaultEncryptionKey;+ if (string.IsNullOrEmpty(vek))+ {+ throw new InvalidOperationException($"Manifest {manifestId} is in the write set without a key, refusing to write it.");+ }++ records.Add(new ManifestRecord(manifestId, isPersonal, entry.GetProperty("salt").GetString()!, vek, name, canAdminister));+ }++ return records;+ }++ /// <summary>+ /// The manifests the local vault holds rows for that this session cannot write.+ /// </summary>+ /// <param name="connection">The local vault.</param>+ /// <param name="personalManifestId">The caller's own manifest.</param>+ /// <returns>The manifest ids, empty when the vault and the session agree.</returns>+ private async Task<List<string>> FindUnwritableManifestsAsync(SqliteConnection connection, Guid personalManifestId)+ {+ var writable = state.SharedManifests.Values.Where(record => !string.IsNullOrEmpty(record.VaultEncryptionKey)).Select(record => record.ManifestId.ToString()).Append(personalManifestId.ToString()).ToList();+ var request = new+ {+ manifestIdsInVault = (await VaultTableReader.ManifestIdsInVaultAsync(connection)).ToList(),+ writableManifestIds = writable,+ grantedManifestIds = Array.Empty<string>(),+ };++ var partitionJson = await rustCoreService.VaultSharingPartitionManifestAccessAsync(JsonSerializer.Serialize(request));+ using var document = JsonDocument.Parse(partitionJson);+ return document.RootElement.TryGetProperty("unwritable", out var unwritable) && unwritable.ValueKind == JsonValueKind.Array ? unwritable.EnumerateArray().Select(id => id.GetString() ?? string.Empty).ToList() : [];+ }++ /// <summary>+ /// Ask the server which of the given blobs it does not hold.+ /// </summary>+ /// <param name="hashes">The candidate hashes.</param>+ /// <returns>The hashes unknown to the server.</returns>+ private async Task<List<string>> MissingOnServerAsync(IEnumerable<string> hashes)+ {+ var candidates = hashes.ToList();+ if (candidates.Count == 0)+ {+ return [];+ }++ using var response = await httpClient.PostAsJsonAsync(BlobsMissingEndpoint, new BlobHashesRequest { Hashes = candidates });+ response.EnsureSuccessStatusCode();+ var body = await response.Content.ReadFromJsonAsync<MissingBlobsResponse>();+ return body?.Missing ?? [];+ }++ /// <summary>+ /// Encrypt the given blobs, each with the key of the manifest that owns it, and upload them in size-capped batches+ /// ahead of the manifest write.+ /// </summary>+ /// <param name="entries">Every staged blob, by hash.</param>+ /// <param name="hashes">The subset to upload.</param>+ /// <param name="overwrite">Ask the server to replace the ciphertext of blobs it already has (the migration re-keys them).</param>+ /// <param name="uploaded">Receives the uploaded ciphertext per hash, for the local cipher cache.</param>+ /// <returns>Task.</returns>+ private async Task UploadBlobsAsync(Dictionary<string, UploadBlobEntry> entries, IEnumerable<string> hashes, bool overwrite, Dictionary<string, string> uploaded)+ {+ var batch = new List<Blob>();+ var batchChars = 0;+ foreach (var hash in hashes)+ {+ if (!entries.TryGetValue(hash, out var entry))+ {+ continue;+ }++ var ciphertext = Convert.ToBase64String(await jsInteropService.SymmetricEncryptBytes(entry.Bytes, Convert.FromBase64String(entry.VaultEncryptionKey)));+ uploaded[hash] = ciphertext;++ // Flush before adding when this blob would take the request past either bound.+ if (batch.Count > 0 && (batchChars + ciphertext.Length > BlobTransferBatchMaxChars || batch.Count >= BlobTransferBatchMaxCount))+ {+ await PostBlobBatchAsync(batch, overwrite);+ batch = [];+ batchChars = 0;+ }++ batch.Add(new Blob { Hash = hash, Category = entry.Kind, EncryptedDataBase64 = ciphertext });+ batchChars += ciphertext.Length;+ }++ if (batch.Count > 0)+ {+ await PostBlobBatchAsync(batch, overwrite);+ }+ }++ /// <summary>+ /// Upload one batch of encrypted blobs.+ /// </summary>+ /// <param name="batch">The blobs.</param>+ /// <param name="overwrite">Whether the server replaces ciphertext it already holds.</param>+ /// <returns>Task.</returns>+ private async Task PostBlobBatchAsync(List<Blob> batch, bool overwrite)+ {+ logger.LogInformation("[V2Push] Uploading blob batch: {Count} blob(s).", batch.Count);+ using var response = await httpClient.PostAsJsonAsync(BlobsEndpoint, new BlobUploadRequest { Blobs = batch, Overwrite = overwrite });+ await EnsureUploadSucceededAsync(response, "Blob upload");+ }++ /// <summary>+ /// POST the unified vault write.+ /// </summary>+ /// <param name="request">The write.</param>+ /// <returns>The server's response.</returns>+ private async Task<VaultWriteResponse> PostWriteAsync(VaultWriteRequest request)+ {+ using var response = await httpClient.PostAsJsonAsync(VaultEndpoint, request);+ await EnsureUploadSucceededAsync(response, "Vault write");+ return await response.Content.ReadFromJsonAsync<VaultWriteResponse>() ?? throw new InvalidOperationException("Empty vault write response.");+ }+/// <summary>/// Materialize manifests and data buckets into a fresh in-memory SQLite database via the codec./// </summary>- /// <param name="opened">The opened manifests and buckets.</param>+ /// <param name="manifestJsons">The manifests, the caller's own first.</param>+ /// <param name="bucketJsons">The data buckets belonging to those manifests.</param>/// <param name="blobs">Plaintext bytes for every blob hash the manifests reference.</param>/// <returns>The open database connection; the caller owns it.</returns>- private async Task<SqliteConnection> MaterializeAsync(OpenedManifestSet opened, Dictionary<string, byte[]> blobs)+ private async Task<SqliteConnection> MaterializeAsync(IEnumerable<string> manifestJsons, IEnumerable<string> bucketJsons, Dictionary<string, byte[]> blobs){var connection = new SqliteConnection("Data Source=:memory:");await connection.OpenAsync();try{- var schemaSql = await jsInteropService.GetCompleteSchemaSqlAsync();- await VaultMaterializer.ApplySchemaAsync(connection, schemaSql);+ await VaultMaterializer.ApplySchemaAsync(connection, await GetSchemaSqlAsync());// The local schema's column set routes anything a newer writer stored into the overflow carrier instead of crashing the insert.- var schemaColumns = await VaultMaterializer.ReadSchemaColumnsAsync(connection);- var inputJson = BuildMaterializeInput(opened.Resolved.Select(entry => entry.ManifestJson), opened.DataBuckets, schemaColumns);+ var inputJson = BuildMaterializeInput(manifestJsons, bucketJsons, await GetSchemaColumnsAsync());var materializedJson = await rustCoreService.VaultCodecMaterializeAsSqliteAsync(inputJson);await VaultMaterializer.InsertMaterializedTablesAsync(connection, materializedJson, blobs, logger);return connection;@@ -677,4 +1548,40 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServthrow;}}++ /// <summary>+ /// The complete client schema, read once.+ /// </summary>+ /// <returns>The COMPLETE_SCHEMA_SQL script.</returns>+ private async Task<string> GetSchemaSqlAsync()+ {+ return _schemaSql ??= await jsInteropService.GetCompleteSchemaSqlAsync();+ }++ /// <summary>+ /// The column set of the complete client schema, per table, read once from a scratch database.+ /// </summary>+ /// <returns>Column names per table.</returns>+ private async Task<Dictionary<string, List<string>>> GetSchemaColumnsAsync()+ {+ if (_schemaColumns is not null)+ {+ return _schemaColumns;+ }++ await using var scratch = new SqliteConnection("Data Source=:memory:");+ await scratch.OpenAsync();+ await VaultMaterializer.ApplySchemaAsync(scratch, await GetSchemaSqlAsync());+ _schemaColumns = await VaultMaterializer.ReadSchemaColumnsAsync(scratch);+ return _schemaColumns;+ }++ /// <summary>+ /// A plaintext blob staged for upload: its bytes plus the key that encrypts it.+ /// </summary>+ /// <param name="Kind">The blob kind.</param>+ /// <param name="Bytes">The plaintext bytes.</param>+ /// <param name="VaultEncryptionKey">The key of the manifest that owns it.</param>+ /// <param name="FromPersonal">Whether the personal manifest owns it.</param>+ private sealed record UploadBlobEntry(string Kind, byte[] Bytes, string VaultEncryptionKey, bool FromPersonal);}diff --git a/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncState.cs b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncState.csindex 3329a17bc..d20f969bc 100644--- a/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncState.cs+++ b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncState.cs@@ -88,6 +88,56 @@ public sealed class VaultSyncState/// <returns>The record key.</returns>public static string BucketFingerprintKey(Guid manifestId, string category) => $"bucket:{manifestId}:{category}";+ /// <summary>+ /// A detached copy, so a failed operation can restore the state it started from.+ /// </summary>+ /// <returns>The copy.</returns>+ public VaultSyncState Clone()+ {+ var copy = new VaultSyncState();+ copy.CopyFrom(this);+ return copy;+ }++ /// <summary>+ /// Replace everything with another state's content.+ /// </summary>+ /// <param name="other">The state to copy.</param>+ public void CopyFrom(VaultSyncState other)+ {+ Clear();+ PersonalManifestId = other.PersonalManifestId;+ PersonalManifestSalt = other.PersonalManifestSalt;+ LastSnapshotWasLegacySqliteBlob = other.LastSnapshotWasLegacySqliteBlob;+ foreach (var (key, value) in other.ManifestRevisions)+ {+ ManifestRevisions[key] = value;+ }++ foreach (var (key, value) in other.BucketRevisions)+ {+ BucketRevisions[key] = value;+ }++ foreach (var (key, value) in other.ContentFingerprints)+ {+ ContentFingerprints[key] = value;+ }++ ServerBlobHashes.UnionWith(other.ServerBlobHashes);+ foreach (var (key, value) in other.BlobCipherCache)+ {+ BlobCipherCache[key] = value;+ }++ foreach (var (key, value) in other.SharedManifests)+ {+ SharedManifests[key] = value;+ }++ LastServedManifestIds.AddRange(other.LastServedManifestIds);+ }+/// <summary>/// Forgets everything; the next pull rebuilds it./// </summary>