Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update services for manifest-v1 canonicalize push (#2159)

Leendert de Borst048918ae18ea
diff --git a/apps/server/AliasVault.Client/Services/Database/DbService.cs b/apps/server/AliasVault.Client/Services/Database/DbService.cs
index f82e2ad70..705b2d445 100644
--- a/apps/server/AliasVault.Client/Services/Database/DbService.cs
+++ b/apps/server/AliasVault.Client/Services/Database/DbService.cs
@@ -8,9 +8,6 @@
namespace AliasVault.Client.Services.Database;
using System.Data;
-using System.Net;
-using System.Net.Http.Json;
-using System.Text.Json;
using AliasClientDb;
using AliasClientDb.Models;
using AliasVault.Client.Services;
@@ -21,9 +18,6 @@ using AliasVault.Client.Services.JsInterop.RustCore.Models;
using AliasVault.Client.Services.VaultSync;
using AliasVault.Client.Services.VaultSync.Exceptions;
using AliasVault.Client.Services.VaultSync.Models;
-using AliasVault.Client.Utilities;
-using AliasVault.Shared.Models.Enums;
-using AliasVault.Shared.Models.WebApi.V1.Vault;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Localization;
@@ -36,23 +30,30 @@ using Microsoft.Extensions.Localization;
public sealed class DbService : IDisposable
{
private const string _UNKNOWN_VERSION = "Unknown";
+
+ /// <summary>
+ /// How many times a save pulls, merges and pushes again after the server reported newer state before giving up.
+ /// </summary>
+ private const int MaxPushAttempts = 3;
+
private readonly AuthService _authService;
private readonly JsInteropService _jsInteropService;
private readonly RustCoreService _rustCore;
private readonly VaultSyncService _vaultSync;
+ private readonly VaultKeyService _vaultKeyService;
private readonly VaultSyncState _syncState;
- private readonly HttpClient _httpClient;
private readonly DbServiceState _state = new();
private readonly Config _config;
private readonly ILogger<DbService> _logger;
private readonly GlobalNotificationService _globalNotificationService;
private readonly IStringLocalizer _sharedLocalizer;
private readonly CancellationTokenSource _backgroundSyncCts = new();
+ private readonly SemaphoreSlim _saveLock = new(1, 1);
private SettingsService _settingsService = new();
private SqliteConnection? _sqlConnection;
private AliasClientDbContext _dbContext;
- private long _vaultRevisionNumber;
private bool _isSuccessfullyInitialized;
+ private bool _forceFullWriteOnNextPush;
private int _retryCount;
private bool _disposed;
@@ -62,21 +63,21 @@ public sealed class DbService : IDisposable
/// <param name="authService">AuthService.</param>
/// <param name="jsInteropService">JsInteropService.</param>
/// <param name="rustCore">RustCoreService for WASM interop.</param>
- /// <param name="vaultSync">VaultSyncService that pulls the vault from the server.</param>
- /// <param name="syncState">The sync state recorded by the last pull.</param>
- /// <param name="httpClient">HttpClient.</param>
+ /// <param name="vaultSync">VaultSyncService that pulls, pushes and merges the vault.</param>
+ /// <param name="vaultKeyService">VaultKeyService that holds the account key chain.</param>
+ /// <param name="syncState">The sync state recorded by the last pull or push.</param>
/// <param name="config">Config instance.</param>
/// <param name="globalNotificationService">Global notification service.</param>
/// <param name="localizerFactory">IStringLocalizerFactory instance.</param>
/// <param name="logger">ILogger instance.</param>
- public DbService(AuthService authService, JsInteropService jsInteropService, RustCoreService rustCore, VaultSyncService vaultSync, VaultSyncState syncState, HttpClient httpClient, Config config, GlobalNotificationService globalNotificationService, IStringLocalizerFactory localizerFactory, ILogger<DbService> logger)
+ public DbService(AuthService authService, JsInteropService jsInteropService, RustCoreService rustCore, VaultSyncService vaultSync, VaultKeyService vaultKeyService, VaultSyncState syncState, Config config, GlobalNotificationService globalNotificationService, IStringLocalizerFactory localizerFactory, ILogger<DbService> logger)
{
_authService = authService;
_jsInteropService = jsInteropService;
_rustCore = rustCore;
_vaultSync = vaultSync;
+ _vaultKeyService = vaultKeyService;
_syncState = syncState;
- _httpClient = httpClient;
_config = config;
_globalNotificationService = globalNotificationService;
_sharedLocalizer = localizerFactory.Create("SharedResources", "AliasVault.Client");
@@ -95,6 +96,12 @@ public sealed class DbService : IDisposable
/// <returns>SettingsService.</returns>
public SettingsService Settings => _settingsService;
+ /// <summary>
+ /// Gets the id of the user's personal manifest, the scope every row this client creates belongs to.
+ /// </summary>
+ /// <exception cref="InvalidOperationException">Thrown when no vault has been pulled yet.</exception>
+ public Guid PersonalManifestId => _syncState.PersonalManifestId ?? throw new InvalidOperationException("No personal manifest id is recorded; the vault has not been loaded from the server yet.");
+
/// <summary>
/// Gets database service state object which can be subscribed to.
/// </summary>
@@ -124,15 +131,6 @@ public sealed class DbService : IDisposable
}
}
- /// <summary>
- /// Stores / updates the vault revision number. Should be called after a successful vault update to the server.
- /// </summary>
- /// <param name="newRevisionNumber">New revision number.</param>
- public void StoreVaultRevisionNumber(long newRevisionNumber)
- {
- _vaultRevisionNumber = newRevisionNumber;
- }
-
/// <summary>
/// Returns the AliasClientDbContext instance.
/// </summary>
@@ -156,22 +154,6 @@ public sealed class DbService : IDisposable
return _dbContext;
}
- /// <summary>
- /// Generate encrypted base64 string representation of current state of database in order to save it
- /// to the server.
- /// </summary>
- /// <returns>Base64 encoded vault blob.</returns>
- public async Task<string> GetEncryptedDatabaseBase64String()
- {
- // Save the actual dbContext.
- await _dbContext.SaveChangesAsync();
-
- string base64String = await ExportSqliteToBase64Async();
-
- // SymmetricEncrypt base64 string using IJSInterop.
- return await _jsInteropService.SymmetricEncrypt(base64String, _authService.GetEncryptionKeyAsBase64Async());
- }
-
/// <summary>
/// Saves the database to the remote server.
/// </summary>
@@ -184,24 +166,16 @@ public sealed class DbService : IDisposable
_state.UpdateState(DbServiceState.DatabaseStatus.SavingToServer);
}
- // Prune expired items from trash before saving.
- await PruneExpiredTrashItemsAsync();
-
- // Make sure a public/private RSA encryption key exists before saving the database.
- await GetOrCreateEncryptionKeyAsync();
-
- var encryptedBase64String = await GetEncryptedDatabaseBase64String();
-
- // Save to webapi.
- var success = await SaveToServerAsync(encryptedBase64String);
+ var success = await SaveAndPushAsync();
if (success)
{
_logger.LogInformation("Database successfully saved to server.");
- if (_state.CurrentState.Status != DbServiceState.DatabaseStatus.Creating)
- {
- // If database is not in the process of being created, update status to ready which is reflected in the UI.
- _state.UpdateState(DbServiceState.DatabaseStatus.Ready);
- }
+ }
+
+ if (_state.CurrentState.Status != DbServiceState.DatabaseStatus.Creating)
+ {
+ // If database is not in the process of being created, update status to ready which is reflected in the UI.
+ _state.UpdateState(DbServiceState.DatabaseStatus.Ready);
}
return success;
@@ -219,8 +193,12 @@ public sealed class DbService : IDisposable
/// </remarks>
public void SaveDatabaseInBackground()
{
- // Set state to indicate background sync is pending
- _state.UpdateState(DbServiceState.DatabaseStatus.BackgroundSyncPending);
+ var creating = _state.CurrentState.Status == DbServiceState.DatabaseStatus.Creating;
+ if (!creating)
+ {
+ // Set state to indicate background sync is pending
+ _state.UpdateState(DbServiceState.DatabaseStatus.BackgroundSyncPending);
+ }
// Capture cancellation token for this background operation
var cancellationToken = _backgroundSyncCts.Token;
@@ -236,43 +214,19 @@ public sealed class DbService : IDisposable
return;
}
- // Prune expired items from trash before saving.
- await PruneExpiredTrashItemsAsync();
-
- if (cancellationToken.IsCancellationRequested || _disposed)
- {
- return;
- }
-
- // Make sure a public/private RSA encryption key exists before saving the database.
- await GetOrCreateEncryptionKeyAsync();
-
- if (cancellationToken.IsCancellationRequested || _disposed)
- {
- return;
- }
-
- var encryptedBase64String = await GetEncryptedDatabaseBase64String();
-
- if (cancellationToken.IsCancellationRequested || _disposed)
- {
- return;
- }
-
- // Update state to show we're actively syncing
- _state.UpdateState(DbServiceState.DatabaseStatus.SavingToServer);
-
- // Save to webapi.
- var success = await SaveToServerAsync(encryptedBase64String);
+ var success = await SaveAndPushAsync();
if (success)
{
_logger.LogInformation("Database successfully saved to server (background sync).");
- _state.UpdateState(DbServiceState.DatabaseStatus.Ready);
}
else
{
- // SaveToServerAsync already raised the user-facing error notification (targeted or generic).
+ // SaveAndPushAsync already raised the user-facing error notification (targeted or generic).
_logger.LogWarning("Background sync to server failed.");
+ }
+
+ if (!creating && !_disposed)
+ {
_state.UpdateState(DbServiceState.DatabaseStatus.Ready);
}
}
@@ -290,7 +244,10 @@ public sealed class DbService : IDisposable
{
_logger.LogError(ex, "Error during background database sync.");
_globalNotificationService.AddErrorMessage(_sharedLocalizer["ErrorUnknown"], true);
- _state.UpdateState(DbServiceState.DatabaseStatus.Ready);
+ if (!creating)
+ {
+ _state.UpdateState(DbServiceState.DatabaseStatus.Ready);
+ }
}
},
cancellationToken);
@@ -331,6 +288,9 @@ public sealed class DbService : IDisposable
{
try
{
+ // Every row this vault will hold is stamped with the personal manifest, so its id must be known up front.
+ _ = PersonalManifestId;
+
// Call JS interop to get SQL commands to create a new vault with the latest schema.
var sqlCommands = await _jsInteropService.GetCreateVaultSqlAsync();
@@ -354,30 +314,35 @@ public sealed class DbService : IDisposable
}
/// <summary>
- /// Migrate the database structure to the latest version.
+ /// Migrate the loaded legacy vault onto the current storage model: walk the frozen sqlite-blob upgrade chain when
+ /// the vault predates its end, then rebuild the database from its manifest-v1 form on the complete schema. The
+ /// result is held locally until <see cref="SaveDatabaseAsync"/> pushes it, which also creates the account key chain
+ /// for an account that has none yet.
/// </summary>
/// <returns>Bool which indicates if migration was successful.</returns>
public async Task<bool> MigrateDatabaseAsync()
{
try
{
- // Get current version of database.
- var currentVersion = await GetCurrentDatabaseVersionAsync();
-
- // Get latest version from JsInteropService.
- var latestVersion = await _jsInteropService.GetLatestVaultVersionAsync();
-
- // Call JS interop to get SQL commands to create a new vault with the latest schema.
- var sqlCommands = await _jsInteropService.GetUpgradeVaultSqlAsync(currentVersion.Revision, latestVersion.Revision);
-
- // Execute the SQL commands to create a new vault with the latest schema.
- foreach (var sqlCommand in sqlCommands.SqlCommands)
+ if (await HasPendingMigrationsAsync())
{
- await _dbContext.Database.ExecuteSqlRawAsync(sqlCommand);
+ // LEGACY: the frozen sqlite-blob upgrade chain has to bring the vault to its end first; the codec cannot canonicalize what came before.
+ var currentVersion = await GetCurrentDatabaseVersionAsync();
+ var latestVersion = await _jsInteropService.GetLatestVaultVersionAsync();
+ var sqlCommands = await _jsInteropService.GetUpgradeVaultSqlAsync(currentVersion.Revision, latestVersion.Revision);
+ foreach (var sqlCommand in sqlCommands.SqlCommands)
+ {
+ await _dbContext.Database.ExecuteSqlRawAsync(sqlCommand);
+ }
}
+ // Rebuild on the complete schema via the codec; this stamps every row with the personal manifest.
+ AdoptDatabase(await _vaultSync.MigrateVaultToCurrentSchemaAsync(_sqlConnection!));
+
+ // The server holds no manifest-v1 state for this vault yet, so the next push must write everything.
+ _forceFullWriteOnNextPush = true;
_isSuccessfullyInitialized = true;
- await _settingsService.InitializeAsync(this);
+ await _settingsService.ReloadAsync(this);
}
catch (Exception ex)
{
@@ -476,36 +441,6 @@ public sealed class DbService : IDisposable
};
}
- /// <summary>
- /// Prepare a vault object for upload to the server.
- /// </summary>
- /// <param name="encryptedDatabase">Encrypted database as string.</param>
- /// <returns>Vault object.</returns>
- public async Task<Vault> PrepareVaultForUploadAsync(string encryptedDatabase)
- {
- var username = _authService.GetUsername();
- var databaseVersion = await GetCurrentDatabaseVersionAsync();
- var encryptionKey = await GetOrCreateEncryptionKeyAsync();
- var credentialsCount = await _dbContext.Items.Where(x => !x.IsDeleted && x.DeletedAt == null).CountAsync();
- var emailAddresses = await GetEmailClaimListAsync();
- var currentDateTime = DateTime.UtcNow;
- return new Vault
- {
- Username = username,
- Blob = encryptedDatabase,
- Version = databaseVersion.Version,
- CurrentRevisionNumber = _vaultRevisionNumber,
- EncryptionPublicKey = encryptionKey.PublicKey,
- CredentialsCount = credentialsCount,
- EmailAddressList = emailAddresses,
- PrivateEmailDomainList = [],
- HiddenPrivateEmailDomainList = [],
- PublicEmailDomainList = [],
- CreatedAt = currentDateTime,
- UpdatedAt = currentDateTime,
- };
- }
-
/// <summary>
/// Clears the database connection and creates a new one so that the database is empty.
/// </summary>
@@ -516,6 +451,7 @@ public sealed class DbService : IDisposable
connection.Open();
SetLiveConnection(connection);
_syncState.Clear();
+ _forceFullWriteOnNextPush = false;
// Reset the database state.
_state.UpdateState(DbServiceState.DatabaseStatus.Uninitialized);
@@ -605,6 +541,7 @@ public sealed class DbService : IDisposable
/// <summary>
/// Imports a base64-encoded SQLite database into the given connection, replacing its contents.
+ /// TODO: remove when all accounts have been migrated to the new manifest-v1 format.
/// </summary>
/// <param name="base64String">The base64-encoded SQLite database.</param>
/// <param name="connection">The connection to import into.</param>
@@ -699,25 +636,6 @@ public sealed class DbService : IDisposable
File.Delete(tempFileName);
}
- /// <summary>
- /// Converts a JsonElement to its appropriate .NET value for SQLite parameters.
- /// </summary>
- /// <param name="element">The JsonElement to convert.</param>
- /// <returns>The converted value.</returns>
- private static object? ConvertJsonElementToValue(JsonElement element)
- {
- return element.ValueKind switch
- {
- JsonValueKind.String => element.GetString(),
- JsonValueKind.Number => element.TryGetInt64(out var longVal) ? longVal : element.GetDouble(),
- JsonValueKind.True => 1L, // SQLite stores booleans as integers
- JsonValueKind.False => 0L,
- JsonValueKind.Null => null,
- JsonValueKind.Undefined => null,
- _ => element.ToString(),
- };
- }
-
/// <summary>
/// Replace first occurrence of a string.
/// </summary>
@@ -737,111 +655,140 @@ public sealed class DbService : IDisposable
}
/// <summary>
- /// Fetches the latest vault from server, merges with local changes using Rust WASM, and saves the merged result.
- /// Called when server responds with "Outdated" status, indicating another client has uploaded a newer vault.
+ /// Commit the pending EF changes and push the vault to the server, one save at a time. Saves that arrive while
+ /// another is running wait for it, so two pushes never canonicalize the same database concurrently.
/// </summary>
- /// <returns>Bool which indicates if merge and save was successful.</returns>
- private async Task<bool> MergeWithServerAndSaveAsync()
+ /// <returns>True when the server holds the local vault afterwards.</returns>
+ private async Task<bool> SaveAndPushAsync()
{
+ await _saveLock.WaitAsync();
try
{
- _logger.LogInformation("Local vault is outdated. Fetching latest vault from server for merge...");
+ // Prune expired items from trash before saving.
+ await PruneExpiredTrashItemsAsync();
- // Fetch the latest vault from server.
- var response = await _httpClient.GetFromJsonAsync<VaultGetResponse>(ApiRoute("Vault"));
- if (response?.Vault == null || string.IsNullOrEmpty(response.Vault.Blob))
- {
- _logger.LogError("Failed to fetch vault from server for merge.");
- _globalNotificationService.AddErrorMessage(_sharedLocalizer["ErrorUnknown"], true);
- return false;
- }
-
- var serverVault = response.Vault;
- _logger.LogInformation("Fetched server vault at revision {Revision}.", serverVault.CurrentRevisionNumber);
+ // Make sure a public/private RSA encryption key exists before saving the database.
+ await EnsurePersonalEncryptionKeyAsync();
+ await _dbContext.SaveChangesAsync();
- // Store username of the loaded vault in memory to send to server as sanity check when updating the vault later.
- _authService.StoreUsername(serverVault.Username);
-
- // Decrypt server vault.
- var decryptedBase64String = await _jsInteropService.SymmetricDecrypt(serverVault.Blob, _authService.GetEncryptionKeyAsBase64Async());
+ return await PushWithConflictResolutionAsync();
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Error saving database to server.");
+ _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultSaveError"], true);
+ return false;
+ }
+ finally
+ {
+ _saveLock.Release();
+ }
+ }
- // Get the list of syncable table names from Rust core.
- var tableNames = await _rustCore.GetSyncableTableNamesAsync();
+ /// <summary>
+ /// Push the vault. When the server reports newer state, pull it, merge the local changes onto it and push again,
+ /// a bounded number of times. Errors are reported to the user here so callers only need the bool.
+ /// </summary>
+ /// <returns>True when the push succeeded.</returns>
+ private async Task<bool> PushWithConflictResolutionAsync()
+ {
+ for (var attempt = 1; attempt <= MaxPushAttempts; attempt++)
+ {
+ // LEGACY: an account without a key chain creates one on its first manifest-v1 push, unless another device already did.
+ var createVaultKey = !await _vaultKeyService.HasLocalVaultKeyAsync() && !await TryAdoptRemoteVaultKeyAsync();
- // Read local tables as JSON.
- var localTables = await ReadTablesAsJsonAsync(_sqlConnection!, tableNames);
- _logger.LogDebug("Read {Count} local tables.", localTables.Count);
+ PushResult result;
+ try
+ {
+ result = await _vaultSync.PushAsync(_sqlConnection!, new PushOptions(createVaultKey, _forceFullWriteOnNextPush));
+ }
+ catch (VaultTooLargeException)
+ {
+ // 413: server / reverse-proxy rejected the upload because the vault exceeded MAX_UPLOAD_SIZE_MB.
+ _logger.LogError("Vault upload rejected by server with 413 Request Entity Too Large. The vault exceeds the server's configured MAX_UPLOAD_SIZE_MB.");
+ _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultTooLargeError"], true);
+ return false;
+ }
+ catch (VaultKeyDecryptException ex)
+ {
+ _logger.LogError(ex, "The server's key chain does not open with this session's key; the password was changed elsewhere. Log in again.");
+ _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultSaveError"], true);
+ return false;
+ }
- // Create a temporary in-memory SQLite database for the server vault.
- await using var serverConnection = new SqliteConnection("Data Source=:memory:");
- await serverConnection.OpenAsync();
- await ImportDbContextFromBase64Async(decryptedBase64String, serverConnection);
+ switch (result.Status)
+ {
+ case PushStatus.Ok:
+ _forceFullWriteOnNextPush = false;
+ return true;
- // Read server tables as JSON.
- var serverTables = await ReadTablesAsJsonAsync(serverConnection, tableNames);
- _logger.LogDebug("Read {Count} server tables.", serverTables.Count);
+ case PushStatus.Outdated:
+ _logger.LogInformation("Push attempt {Attempt}/{Max}: the server holds newer state ({Reasons}); pulling and merging.", attempt, MaxPushAttempts, string.Join("; ", result.Reasons ?? []));
+ if (attempt == MaxPushAttempts)
+ {
+ break;
+ }
- // Create the merge input (local has our pending changes, server is the merge base).
- var mergeInput = new MergeInput
- {
- LocalTables = localTables,
- ServerTables = serverTables,
- };
+ await MergeWithServerAsync();
+ continue;
- // Call Rust WASM merge (LWW - Last Write Wins based on UpdatedAt). The merge base is the
- // server vault; the returned statements bring the local changes onto that base.
- var mergeOutput = await _rustCore.MergeVaultsAsync(mergeInput);
-
- _logger.LogInformation(
- "Merge completed: {TablesProcessed} tables, {FromLocal} kept local, {FromServer} from server, {Inserted} inserted, {Conflicts} conflicts.",
- mergeOutput.Stats.TablesProcessed,
- mergeOutput.Stats.RecordsFromLocal,
- mergeOutput.Stats.RecordsFromServer,
- mergeOutput.Stats.RecordsInserted,
- mergeOutput.Stats.Conflicts);
-
- // Apply the local changes onto the server base (statements target the server database),
- // then make the merged (server-based) database the live local database.
- await ExecuteMergeSqlStatementsAsync(mergeOutput.Statements, serverConnection);
- var mergedBase64 = await ExportConnectionToBase64Async(serverConnection);
- await ImportDbContextFromBase64Async(mergedBase64, _sqlConnection!);
-
- // Verify foreign key integrity after merge.
- await using (var command = _sqlConnection!.CreateCommand())
- {
- command.CommandText = "PRAGMA foreign_key_check;";
- await using var reader = await command.ExecuteReaderAsync();
- if (await reader.ReadAsync())
- {
- _logger.LogError("Foreign key violation detected after merge.");
- _globalNotificationService.AddErrorMessage(_sharedLocalizer["ErrorUnknown"], true);
+ default:
+ _logger.LogError("Vault push {Status}: {Reasons}", result.Status, string.Join("; ", result.Reasons ?? []));
+ _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultSaveError"], true);
return false;
- }
}
+ }
- // Update the db context with the merged database.
- _dbContext = new AliasClientDbContext(_sqlConnection, log => _logger.LogDebug("{Message}", log));
-
- // Update the local revision number to the server's revision.
- // When we upload, server will calculate new revision = this + 1.
- StoreVaultRevisionNumber(serverVault.CurrentRevisionNumber);
+ _logger.LogError("Vault push still outdated after {Max} attempts, giving up.", MaxPushAttempts);
+ _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultSaveError"], true);
+ return false;
+ }
- _logger.LogInformation("Local merge completed. Uploading merged vault to server...");
+ /// <summary>
+ /// Fetch the server vault and merge the local changes onto it; the merged database becomes the live one. When the
+ /// server holds nothing to merge with (legacy format or a never-written manifest), the next push writes everything.
+ /// </summary>
+ /// <returns>Task.</returns>
+ private async Task MergeWithServerAsync()
+ {
+ var result = await _vaultSync.PullAndMergeAsync(_sqlConnection!);
+ if (result.Kind == PullAndMergeKind.NothingToMergeWith)
+ {
+ _logger.LogInformation("The server holds no manifest-v1 vault to merge with; pushing the local vault over it whole.");
+ _forceFullWriteOnNextPush = true;
+ return;
+ }
- // Now save the merged vault to server. This recursive call handles the case where
- // another client uploaded during our merge (returns Outdated again).
- return await SaveDatabaseAsync();
+ foreach (var manifestId in result.FallbackManifestIds)
+ {
+ _logger.LogWarning("Canonical merge fell back to the server's rows for manifest {ManifestId}; local changes to it were dropped.", manifestId);
}
- catch (Exception ex)
+
+ AdoptDatabase(result.Database!);
+ await _settingsService.ReloadAsync(this);
+ }
+
+ /// <summary>
+ /// Adopt a key chain the server holds but this device does not: the account was migrated on another device while
+ /// this session still holds the old password-derived key, which opens the chain as its KEK.
+ /// </summary>
+ /// <returns>True when a chain was adopted, false when the server holds none.</returns>
+ /// <exception cref="VaultKeyDecryptException">Thrown when the server's chain does not open with the session key.</exception>
+ private async Task<bool> TryAdoptRemoteVaultKeyAsync()
+ {
+ var resolved = await _vaultKeyService.AdoptRemoteVaultKeyAsync(_authService.GetEncryptionKeyAsBase64Async());
+ if (resolved is null)
{
- _globalNotificationService.AddErrorMessage(_sharedLocalizer["ErrorUnknown"], true);
- _logger.LogError(ex, "Error merging with server vault.");
return false;
}
+
+ _logger.LogInformation("Adopted the account key chain another device created; the session key is now the VEK.");
+ await _authService.StoreSessionKeysAsync(resolved);
+ return true;
}
/// <summary>
- /// Checks if there are any pending migrations.
+ /// Checks if there are any pending migrations of the frozen sqlite-blob upgrade chain.
/// </summary>
/// <returns>Bool which indicates if there are any pending migrations.</returns>
private async Task<bool> HasPendingMigrationsAsync()
@@ -895,7 +842,6 @@ public sealed class DbService : IDisposable
{
case PullKind.Empty:
// The vault was never written: create the database structure from scratch to get an empty ready-to-use database.
- StoreVaultRevisionNumber(pull.Revision);
_state.UpdateState(DbServiceState.DatabaseStatus.Creating);
return false;
@@ -904,7 +850,6 @@ public sealed class DbService : IDisposable
default:
SetLiveConnection(pull.Database!);
- StoreVaultRevisionNumber(pull.Revision);
_isSuccessfullyInitialized = true;
await _settingsService.InitializeAsync(this);
_state.UpdateState(DbServiceState.DatabaseStatus.Ready);
@@ -920,15 +865,14 @@ public sealed class DbService : IDisposable
}
/// <summary>
- /// Import an account's not-yet-migrated sqlite-blob vault. The derived key is the encryption key for these accounts.
- /// TODO: remove when all accounts have been migrated to the new manifest-v1 format.
+ /// Import an account's not-yet-migrated sqlite-blob vault. The derived key is the encryption key for these
+ /// accounts. A legacy vault cannot be saved before it is migrated onto the manifest-v1 storage model, so it
+ /// always lands on the upgrade page. TODO: remove when all accounts have been migrated to the new manifest-v1 format.
/// </summary>
/// <param name="pull">The pass-through pull result.</param>
/// <returns>True when the database is ready for use.</returns>
private async Task<bool> LoadLegacySqliteBlobAsync(PullResult pull)
{
- StoreVaultRevisionNumber(pull.Revision);
-
// An account that registered but never uploaded a vault: create the database structure from scratch.
if (string.IsNullOrEmpty(pull.LegacyVaultBlob))
{
@@ -941,16 +885,12 @@ public sealed class DbService : IDisposable
await ImportDbContextFromBase64Async(decryptedBase64String, _sqlConnection!);
// Refresh the db context with the new database to invalidate any cached data if the _dbContext was already used.
- _dbContext = new AliasClientDbContext(_sqlConnection!, log => _logger.LogDebug("{Message}", log));
+ _dbContext = CreateDbContext(_sqlConnection!);
- // Check if database is up-to-date with migrations.
+ // A vault this client cannot place on the upgrade chain (newer major version) cannot be migrated either.
try
{
- if (await HasPendingMigrationsAsync())
- {
- _state.UpdateState(DbServiceState.DatabaseStatus.PendingMigrations);
- return false;
- }
+ await HasPendingMigrationsAsync();
}
catch (DataException)
{
@@ -958,10 +898,8 @@ public sealed class DbService : IDisposable
return false;
}
- _isSuccessfullyInitialized = true;
- await _settingsService.InitializeAsync(this);
- _state.UpdateState(DbServiceState.DatabaseStatus.Ready);
- return true;
+ _state.UpdateState(DbServiceState.DatabaseStatus.PendingMigrations);
+ return false;
}
/// <summary>
@@ -972,7 +910,7 @@ public sealed class DbService : IDisposable
{
var previous = _sqlConnection;
_sqlConnection = connection;
- _dbContext = new AliasClientDbContext(_sqlConnection, log => _logger.LogDebug("{Message}", log));
+ _dbContext = CreateDbContext(connection);
if (previous is not null && !ReferenceEquals(previous, connection))
{
@@ -981,57 +919,27 @@ public sealed class DbService : IDisposable
}
/// <summary>
- /// Saves encrypted database blob to server and updates the local revision number.
+ /// Copy another database into the live connection and refresh the context over it. The connection object stays
+ /// the same, so a context handed out earlier keeps working and simply sees the new content.
/// </summary>
- /// <param name="encryptedDatabase">Encrypted database as string.</param>
- /// <returns>True if save action succeeded and revision number was updated, false otherwise.</returns>
- private async Task<bool> SaveToServerAsync(string encryptedDatabase)
+ /// <param name="source">The open in-memory database to adopt; disposed afterwards.</param>
+ private void AdoptDatabase(SqliteConnection source)
{
- var vaultObject = await PrepareVaultForUploadAsync(encryptedDatabase);
-
- try
- {
- var response = await _httpClient.PostAsJsonAsync(ApiRoute("Vault"), vaultObject);
-
- // 413: server / reverse-proxy rejected the upload because the vault exceeded MAX_UPLOAD_SIZE_MB.
- // Show the targeted message and skip the generic notification fired in the catch / fallthrough.
- if (response.StatusCode == HttpStatusCode.RequestEntityTooLarge)
- {
- _logger.LogError("Vault upload rejected by server with 413 Request Entity Too Large. The vault exceeds the server's configured MAX_UPLOAD_SIZE_MB.");
- _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultTooLargeError"], true);
- return false;
- }
-
- // Ensure the request was successful
- response.EnsureSuccessStatusCode();
-
- // Deserialize the response content
- var vaultUpdateResponse = await response.Content.ReadFromJsonAsync<VaultUpdateResponse>();
-
- if (vaultUpdateResponse != null)
- {
- if (vaultUpdateResponse.Status == VaultStatus.Outdated)
- {
- // Server has a newer vault. Fetch it, merge with our local changes, and re-upload.
- // The merge uses LWW (Last Write Wins) based on UpdatedAt timestamps.
- return await MergeWithServerAndSaveAsync();
- }
-
- _vaultRevisionNumber = vaultUpdateResponse.NewRevisionNumber;
- return true;
- }
-
- _logger.LogError("Error during save: server response was empty or could not be deserialized.");
- }
- catch (Exception ex)
- {
- _logger.LogError(ex, "Error saving database to server.");
- }
+ source.BackupDatabase(_sqlConnection!);
+ source.Dispose();
+ _dbContext = CreateDbContext(_sqlConnection!);
+ }
- // Generic save failure (server error, network issue, malformed response, etc.). DbService owns
- // the user-facing error so callers only need to react to the bool return value.
- _globalNotificationService.AddErrorMessage(_sharedLocalizer["VaultSaveError"], true);
- return false;
+ /// <summary>
+ /// Create the EF context over a connection, with every save stamping new rows with their manifest.
+ /// </summary>
+ /// <param name="connection">The open connection.</param>
+ /// <returns>The context.</returns>
+ private AliasClientDbContext CreateDbContext(SqliteConnection connection)
+ {
+ var context = new AliasClientDbContext(connection, log => _logger.LogDebug("{Message}", log));
+ context.SavingChanges += (sender, _) => ManifestStamper.Stamp((DbContext)sender!, PersonalManifestId);
+ return context;
}
/// <summary>
@@ -1096,31 +1004,32 @@ public sealed class DbService : IDisposable
}
/// <summary>
- /// Get the default public/private encryption key, if it does not yet exist, create it.
+ /// Make sure the personal manifest holds its email delivery keypair; create one when it does not.
/// </summary>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
- private async Task<EncryptionKey> GetOrCreateEncryptionKeyAsync()
+ private async Task EnsurePersonalEncryptionKeyAsync()
{
- var encryptionKey = await _dbContext.EncryptionKeys.FirstOrDefaultAsync(x => x.IsPrimary);
+ var personalManifestId = PersonalManifestId;
+ var encryptionKey = await _dbContext.EncryptionKeys.FirstOrDefaultAsync(x => x.ManifestId == personalManifestId && x.IsPrimary && !x.IsDeleted);
if (encryptionKey is not null)
{
- return encryptionKey;
+ return;
}
// Create a new encryption key via JSInterop, .NET WASM does not support crypto operations natively (yet).
var keyPair = await _jsInteropService.GenerateRsaKeyPair();
var currentDateTime = DateTime.UtcNow;
- encryptionKey = new EncryptionKey
+ _dbContext.EncryptionKeys.Add(new EncryptionKey
{
+ Id = Guid.NewGuid(),
+ ManifestId = personalManifestId,
PublicKey = keyPair.PublicKey,
PrivateKey = keyPair.PrivateKey,
IsPrimary = true,
CreatedAt = currentDateTime,
UpdatedAt = currentDateTime,
- };
- _dbContext.EncryptionKeys.Add(encryptionKey);
- return encryptionKey;
+ });
}
/// <summary>
@@ -1177,137 +1086,6 @@ public sealed class DbService : IDisposable
return tables;
}
- /// <summary>
- /// Reads all specified tables from a SQLite connection as JSON data for the Rust merge.
- /// </summary>
- /// <param name="connection">The SQLite connection to read from.</param>
- /// <param name="tableNames">The names of tables to read.</param>
- /// <returns>List of TableData objects containing the table records.</returns>
- private async Task<List<TableData>> ReadTablesAsJsonAsync(SqliteConnection connection, string[] tableNames)
- {
- var tables = new List<TableData>();
-
- foreach (var tableName in tableNames)
- {
- var tableData = new TableData { Name = tableName };
-
- // Check if table exists in the database.
- await using var checkCommand = connection.CreateCommand();
- checkCommand.CommandText = "SELECT name FROM sqlite_master WHERE type='table' AND name=@tableName";
- checkCommand.Parameters.AddWithValue("@tableName", tableName);
- var exists = await checkCommand.ExecuteScalarAsync();
-
- if (exists == null)
- {
- // Table doesn't exist, add empty table data.
- tables.Add(tableData);
- continue;
- }
-
- // Get column names for the table.
- await using var columnsCommand = connection.CreateCommand();
- columnsCommand.CommandText = $"PRAGMA table_info({tableName})";
- var columns = new List<string>();
- await using (var columnsReader = await columnsCommand.ExecuteReaderAsync())
- {
- while (await columnsReader.ReadAsync())
- {
- columns.Add(columnsReader.GetString(1));
- }
- }
-
- // Read all records from the table.
- await using var selectCommand = connection.CreateCommand();
- selectCommand.CommandText = $"SELECT * FROM {tableName}";
- await using var reader = await selectCommand.ExecuteReaderAsync();
-
- while (await reader.ReadAsync())
- {
- var record = new Dictionary<string, object?>();
- for (var i = 0; i < columns.Count; i++)
- {
- var value = reader.GetValue(i);
-
- // Convert DBNull to null for proper JSON serialization.
- record[columns[i]] = value == DBNull.Value ? null : value;
- }
-
- tableData.Records.Add(record);
- }
-
- tables.Add(tableData);
- }
-
- return tables;
- }
-
- /// <summary>
- /// Executes the SQL statements returned by the Rust merge operation against the given connection.
- /// </summary>
- /// <param name="statements">The SQL statements to execute.</param>
- /// <param name="connection">The connection to execute the statements against (the merge base).</param>
- /// <returns>Task.</returns>
- private async Task ExecuteMergeSqlStatementsAsync(List<SqlStatement> statements, SqliteConnection connection)
- {
- if (statements.Count == 0)
- {
- _logger.LogDebug("No SQL statements to execute from merge.");
- return;
- }
-
- _logger.LogDebug("Executing {Count} SQL statements from merge.", statements.Count);
-
- // Disable foreign key checks during merge execution.
- await using (var pragmaCommand = connection.CreateCommand())
- {
- pragmaCommand.CommandText = "PRAGMA foreign_keys = OFF;";
- await pragmaCommand.ExecuteNonQueryAsync();
- }
-
- try
- {
- foreach (var statement in statements)
- {
- await using var command = connection.CreateCommand();
- command.CommandText = statement.Sql;
-
- // Add parameters in order (SQLite uses positional parameters with ?).
- for (var i = 0; i < statement.Params.Count; i++)
- {
- var value = statement.Params[i];
-
- // Handle JsonElement values from deserialization.
- if (value is JsonElement jsonElement)
- {
- value = ConvertJsonElementToValue(jsonElement);
- }
-
- command.Parameters.AddWithValue($"@p{i}", value ?? DBNull.Value);
- }
-
- // Replace ? placeholders with named parameters.
- var paramIndex = 0;
- var sql = statement.Sql;
- while (sql.Contains('?'))
- {
- var pos = sql.IndexOf('?');
- sql = sql[..pos] + $"@p{paramIndex}" + sql[(pos + 1)..];
- paramIndex++;
- }
-
- command.CommandText = sql;
- await command.ExecuteNonQueryAsync();
- }
- }
- finally
- {
- // Re-enable foreign key checks.
- await using var pragmaCommand = connection.CreateCommand();
- pragmaCommand.CommandText = "PRAGMA foreign_keys = ON;";
- await pragmaCommand.ExecuteNonQueryAsync();
- }
- }
-
/// <summary>
/// Disposes the service.
/// </summary>
@@ -1324,6 +1102,7 @@ public sealed class DbService : IDisposable
// Cancel any pending background sync operations first
_backgroundSyncCts.Cancel();
_backgroundSyncCts.Dispose();
+ _saveLock.Dispose();
_sqlConnection?.Dispose();
}
diff --git a/apps/server/AliasVault.Client/Services/SettingsService.cs b/apps/server/AliasVault.Client/Services/SettingsService.cs
index 029491108..1954c6691 100644
--- a/apps/server/AliasVault.Client/Services/SettingsService.cs
+++ b/apps/server/AliasVault.Client/Services/SettingsService.cs
@@ -231,15 +231,25 @@ public sealed class SettingsService
// Store the DbService instance for later use.
_dbService = dbService;
+ await ReloadAsync(dbService);
+ _initialized = true;
+ }
+ /// <summary>
+ /// Re-read every setting from the database, for after the live database was replaced (a merge or migration).
+ /// </summary>
+ /// <param name="dbService">The database service the settings are read from.</param>
+ /// <returns>Task.</returns>
+ public async Task ReloadAsync(DbService dbService)
+ {
+ _dbService = dbService;
var db = await _dbService.GetDbContextAsync();
var settings = await db.Settings.ToListAsync();
+ _settings.Clear();
foreach (var setting in settings)
{
_settings[setting.Key] = setting.Value;
}
-
- _initialized = true;
}
/// <summary>
@@ -373,11 +383,12 @@ public sealed class SettingsService
}
var db = await _dbService!.GetDbContextAsync();
- var setting = await db.Settings.FindAsync(key);
+ var setting = await db.Settings.FindAsync(_dbService.PersonalManifestId, key);
if (setting == null)
{
setting = new Setting
{
+ ManifestId = _dbService.PersonalManifestId,
Key = key,
Value = value,
CreatedAt = DateTime.UtcNow,
diff --git a/apps/server/AliasVault.Client/Services/VaultSync/VaultKeyService.cs b/apps/server/AliasVault.Client/Services/VaultSync/VaultKeyService.cs
index 9f8a6d582..44adc28da 100644
--- a/apps/server/AliasVault.Client/Services/VaultSync/VaultKeyService.cs
+++ b/apps/server/AliasVault.Client/Services/VaultSync/VaultKeyService.cs
@@ -8,6 +8,7 @@
namespace AliasVault.Client.Services.VaultSync;
using System.Net.Http.Json;
+using System.Security.Cryptography;
using AliasVault.Client.Services.JsInterop;
using AliasVault.Client.Services.JsInterop.RustCore;
using AliasVault.Client.Services.VaultSync.Exceptions;
@@ -15,6 +16,7 @@ using AliasVault.Client.Services.VaultSync.Models;
using AliasVault.Shared.Models.Enums;
using AliasVault.Shared.Models.WebApi.V1.Auth;
using AliasVault.Shared.Models.WebApi.V2.Auth;
+using AliasVault.Shared.Models.WebApi.V2.Vault;
using Blazored.LocalStorage;
using Microsoft.JSInterop;
@@ -69,7 +71,7 @@ public sealed class VaultKeyService(HttpClient httpClient, ILocalStorageService
/// </summary>
/// <param name="username">The username, needed to look up the salt of a legacy account.</param>
/// <param name="password">The master password.</param>
- /// <returns>The resolved keys. A legacy result still has to be validated by the caller.</returns>
+ /// <returns>The resolved keys.</returns>
/// <exception cref="VaultKeyDecryptException">Thrown when the password is wrong.</exception>
/// <exception cref="VaultKeyUnavailableException">Thrown when offline and the chain this account unlocks with is not cached.</exception>
public async Task<ResolvedVaultKey> UnlockWithPasswordAsync(string username, string password)
@@ -153,6 +155,121 @@ public sealed class VaultKeyService(HttpClient httpClient, ILocalStorageService
return await localStorage.GetItemAsStringAsync(StorageKeys.AccountPublicKey);
}
+ /// <summary>
+ /// Whether this device holds the account's encrypted key chain, meaning the account is on the account-key model.
+ /// </summary>
+ /// <returns>True when the chain is cached.</returns>
+ public async Task<bool> HasLocalVaultKeyAsync()
+ {
+ return !string.IsNullOrEmpty(await localStorage.GetItemAsStringAsync(StorageKeys.EncryptedAccountKey));
+ }
+
+ /// <summary>
+ /// Create a complete account key hierarchy: a random Account Key wrapped by the KEK, a random VEK and the account
+ /// keypair both wrapped by the Account Key. Used on registration and on the one-time legacy migration push.
+ /// </summary>
+ /// <param name="kekBase64">The password-derived key that becomes the KEK.</param>
+ /// <returns>The plaintext keys to adopt and the wrapped chain to upload.</returns>
+ public async Task<AccountKeyHierarchy> CreateAccountKeyHierarchyAsync(string kekBase64)
+ {
+ var vek = RandomNumberGenerator.GetBytes(32);
+ var accountKey = RandomNumberGenerator.GetBytes(32);
+ var (publicKey, privateKey) = await jsInteropService.GenerateRsaKeyPair();
+
+ var keys = new AccountKeysUpload
+ {
+ EncryptedAccountKey = await WrapKeyAsync(accountKey, Convert.FromBase64String(kekBase64)),
+ EncryptedVek = await WrapKeyAsync(vek, accountKey),
+ AccountPublicKey = publicKey,
+ EncryptedAccountPrivateKey = await jsInteropService.SymmetricEncrypt(privateKey, Convert.ToBase64String(accountKey)),
+ };
+
+ return new AccountKeyHierarchy(Convert.ToBase64String(vek), privateKey, keys);
+ }
+
+ /// <summary>
+ /// Adopt a hierarchy the server just committed: cache the wrapped chain for offline unlock and mark the cached
+ /// derivation parameters as belonging to an account-key account.
+ /// </summary>
+ /// <param name="hierarchy">The hierarchy that was uploaded.</param>
+ /// <returns>Task.</returns>
+ public async Task AdoptLocalAccountKeysAsync(AccountKeyHierarchy hierarchy)
+ {
+ await localStorage.SetItemAsStringAsync(StorageKeys.EncryptedAccountKey, hierarchy.Keys.EncryptedAccountKey!);
+ await localStorage.SetItemAsStringAsync(StorageKeys.EncryptedVek, hierarchy.Keys.EncryptedVek!);
+ await localStorage.SetItemAsStringAsync(StorageKeys.AccountPublicKey, hierarchy.Keys.AccountPublicKey!);
+ await localStorage.SetItemAsStringAsync(StorageKeys.EncryptedAccountPrivateKey, hierarchy.Keys.EncryptedAccountPrivateKey!);
+
+ var parameters = await GetDerivationParamsAsync();
+ if (parameters is not null)
+ {
+ await StoreDerivationParamsAsync(parameters with { HasKeyChain = true });
+ }
+ }
+
+ /// <summary>
+ /// Adopt a chain the server holds but this device does not (the account was migrated on another device while
+ /// this session still holds the old password-derived key): open it with the session key, which then is the KEK.
+ /// </summary>
+ /// <param name="sessionKeyBase64">The session's current key.</param>
+ /// <returns>The resolved keys when a chain was adopted, null when the server holds none. Throws when the chain does not open.</returns>
+ public async Task<ResolvedVaultKey?> AdoptRemoteVaultKeyAsync(string sessionKeyBase64)
+ {
+ var vaultKey = await FetchVaultKeyAsync();
+ if (vaultKey is null)
+ {
+ return null;
+ }
+
+ var resolved = await DecryptKeyChainAsync(vaultKey.EncryptedAccountKey, vaultKey.EncryptedVek, vaultKey.EncryptedAccountPrivateKey, sessionKeyBase64);
+ await CacheVaultKeyBlobsAsync(vaultKey);
+ return resolved;
+ }
+
+ /// <summary>
+ /// Decrypt the locally cached chain with the given KEK.
+ /// </summary>
+ /// <param name="derivedKeyBase64">The password-derived key.</param>
+ /// <returns>The resolved keys, or null when no chain is cached.</returns>
+ /// <exception cref="VaultKeyDecryptException">Thrown when the chain does not open with the derived key.</exception>
+ public async Task<ResolvedVaultKey?> UnlockCachedChainAsync(string derivedKeyBase64)
+ {
+ return await ResolveFromLocalCacheAsync(derivedKeyBase64);
+ }
+
+ /// <summary>
+ /// Re-wrap the cached Account Key for a new password: open it with the current KEK, wrap it with the new one.
+ /// The VEK, the account keypair and every grant stay as they are.
+ /// </summary>
+ /// <param name="currentKekBase64">The KEK derived from the current password.</param>
+ /// <param name="newKekBase64">The KEK derived from the new password.</param>
+ /// <returns>The Account Key wrapped with the new KEK.</returns>
+ /// <exception cref="VaultKeyDecryptException">Thrown when the current password is wrong.</exception>
+ /// <exception cref="VaultKeyUnavailableException">Thrown when no chain is cached (the account has not migrated yet).</exception>
+ public async Task<string> RewrapAccountKeyAsync(string currentKekBase64, string newKekBase64)
+ {
+ var encryptedAccountKey = await localStorage.GetItemAsStringAsync(StorageKeys.EncryptedAccountKey);
+ if (string.IsNullOrEmpty(encryptedAccountKey))
+ {
+ throw new VaultKeyUnavailableException(null);
+ }
+
+ var accountKey = await DecryptKeyOrThrowAsync(encryptedAccountKey, currentKekBase64);
+ return await WrapKeyAsync(Convert.FromBase64String(accountKey), Convert.FromBase64String(newKekBase64));
+ }
+
+ /// <summary>
+ /// Persist the re-wrapped Account Key and the new derivation parameters after a password change.
+ /// </summary>
+ /// <param name="newEncryptedAccountKey">The Account Key wrapped with the new KEK.</param>
+ /// <param name="parameters">The new derivation parameters.</param>
+ /// <returns>Task.</returns>
+ public async Task PersistNewAccountKeyAsync(string newEncryptedAccountKey, EncryptionKeyDerivationParams parameters)
+ {
+ await localStorage.SetItemAsStringAsync(StorageKeys.EncryptedAccountKey, newEncryptedAccountKey);
+ await StoreDerivationParamsAsync(parameters with { HasKeyChain = true });
+ }
+
/// <summary>
/// Persist the KEK derivation parameters.
/// </summary>
@@ -295,6 +412,17 @@ public sealed class VaultKeyService(HttpClient httpClient, ILocalStorageService
}
}
+ /// <summary>
+ /// Wrap a 32-byte key with another key: AES-256-GCM, output base64(IV | ciphertext | tag).
+ /// </summary>
+ /// <param name="key">The key to wrap.</param>
+ /// <param name="wrappingKey">The key that wraps it.</param>
+ /// <returns>The wrapped key.</returns>
+ private async Task<string> WrapKeyAsync(byte[] key, byte[] wrappingKey)
+ {
+ return Convert.ToBase64String(await jsInteropService.SymmetricEncryptBytes(key, wrappingKey));
+ }
+
/// <summary>
/// Decrypt a wrapped key, mapping an AES-GCM authentication failure onto <see cref="VaultKeyDecryptException"/>.
/// </summary>
diff --git a/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncService.cs b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncService.cs
index 12b740a03..f1890eea4 100644
--- a/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncService.cs
+++ b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncService.cs
@@ -7,6 +7,7 @@
namespace AliasVault.Client.Services.VaultSync;
+using System.Net;
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
@@ -22,18 +23,22 @@ using Microsoft.Data.Sqlite;
using Microsoft.JSInterop;
/// <summary>
-/// Syncs the vault with the server.
+/// Syncs the vault with the server: pulls and materializes the manifest snapshot, canonicalizes and pushes the local
+/// vault, and merges the two when they diverged.
/// </summary>
/// <param name="httpClient">The HTTP client.</param>
/// <param name="authService">AuthService instance, the session's key holder.</param>
/// <param name="vaultKeyService">VaultKeyService instance.</param>
/// <param name="jsInteropService">JsInteropService instance.</param>
/// <param name="rustCoreService">RustCoreService instance.</param>
-/// <param name="state">The sync state this pull records the snapshot into.</param>
+/// <param name="state">The sync state this service records the server's state into.</param>
+/// <param name="config">Config instance, for the private email domains the routing push claims.</param>
/// <param name="logger">ILogger instance.</param>
-public sealed class VaultSyncService(HttpClient httpClient, AuthService authService, VaultKeyService vaultKeyService, JsInteropService jsInteropService, RustCoreService rustCoreService, VaultSyncState state, ILogger<VaultSyncService> logger)
+public sealed class VaultSyncService(HttpClient httpClient, AuthService authService, VaultKeyService vaultKeyService, JsInteropService jsInteropService, RustCoreService rustCoreService, VaultSyncState state, Config config, ILogger<VaultSyncService> logger)
{
private const string AttachmentBlobCategory = "attachment";
+ private const string BlobRefMarker = "__blobRef";
+ private const string BlobKindMarker = "__blobKind";
/// <summary>
/// Max amount of base64 characters transferred in a single blob transfer request or response body.
@@ -46,10 +51,15 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ
private const int BlobTransferBatchMaxCount = 100;
private static readonly string VaultEndpoint = ApiRoute("Vault");
+ private static readonly string BlobsEndpoint = ApiRoute("Vault/blobs");
+ private static readonly string BlobsMissingEndpoint = ApiRoute("Vault/blobs/missing");
private static readonly string BlobsDownloadEndpoint = ApiRoute("Vault/blobs/download");
+ private string? _schemaSql;
+ private Dictionary<string, List<string>>? _schemaColumns;
+
/// <summary>
- /// Gets the sync state recorded by the last pull.
+ /// Gets the sync state recorded by the last pull or push.
/// </summary>
public VaultSyncState State => state;
@@ -66,9 +76,7 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ
*/
logger.LogInformation("[V2Pull] Step 1/3: fetching vault snapshot (GET /v2/Vault)...");
var snapshot = await FetchSnapshotAsync();
- state.LastServedManifestIds.Clear();
- state.LastServedManifestIds.AddRange(snapshot.Manifests.Select(m => m.ManifestId));
- state.LastSnapshotWasLegacySqliteBlob = snapshot.StorageFormat != StorageFormat.Manifest;
+ RecordServedManifests(snapshot);
// LEGACY: a not-yet-migrated account's blob is passed through unchanged.
if (state.LastSnapshotWasLegacySqliteBlob)
@@ -80,9 +88,9 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ
if (string.IsNullOrEmpty(personalDto.Blob))
{
// The personal manifest exists but was never written (fresh account): the client creates the vault.
- state.PersonalManifestId = personalDto.ManifestId;
+ RecordEmptyPersonalManifest(personalDto);
logger.LogInformation("[V2Pull] Personal manifest {ManifestId} has no content yet; a new vault will be created.", personalDto.ManifestId);
- return new PullResult { Kind = PullKind.Empty, Revision = personalDto.Revision };
+ return new PullResult { Kind = PullKind.Empty };
}
logger.LogInformation("[V2Pull] Step 1/3 done: manifests={ManifestCount}, personalRevision={Revision}, buckets={BucketCount}, blobRefs={BlobRefCount}.", snapshot.Manifests.Count, personalDto.Revision, snapshot.Buckets.Count, personalDto.BlobReferences.Count);
@@ -99,8 +107,8 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ
var blobs = await DownloadReferencedBlobsAsync(opened);
logger.LogInformation("[V2Pull] Step 3/3: materializing {ManifestCount} manifest(s) into a fresh SQLite database...", opened.Resolved.Count);
- var database = await MaterializeAsync(opened, blobs);
- return new PullResult { Kind = PullKind.Materialized, Database = database, Revision = opened.PersonalRevision };
+ var database = await MaterializeAsync(opened.Resolved.Select(entry => entry.ManifestJson), opened.DataBuckets, blobs);
+ return new PullResult { Kind = PullKind.Materialized, Database = database };
}
catch (Exception ex)
{
@@ -109,6 +117,339 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ
}
}
+ /// <summary>
+ /// Pull the latest snapshot and merge the local vault onto it at canonical level, one manifest at a time. The
+ /// server side is the base; the local side is canonicalized from the given connection.
+ /// </summary>
+ /// <param name="localConnection">The local vault holding the changes to keep.</param>
+ /// <returns>The merged database, or the signal that the server holds nothing to merge with.</returns>
+ /// <exception cref="VaultProcessingException">Thrown when the snapshot could not be opened or the merge failed.</exception>
+ public async Task<PullAndMergeResult> PullAndMergeAsync(SqliteConnection localConnection)
+ {
+ logger.LogInformation("[V2Merge] Fetching vault snapshot for canonical merge (GET /v2/Vault)...");
+ var snapshot = await FetchSnapshotAsync();
+ RecordServedManifests(snapshot);
+
+ // LEGACY: a server still on the sqlite-blob format cannot merge with a manifest-v1 vault; the caller pushes over it.
+ if (state.LastSnapshotWasLegacySqliteBlob)
+ {
+ OpenLegacySqliteBlobSnapshot(snapshot);
+ return new PullAndMergeResult { Kind = PullAndMergeKind.NothingToMergeWith };
+ }
+
+ var personalDto = SelectPersonalManifest(snapshot) ?? throw new VaultProcessingException("vault-merge", new InvalidOperationException("The server returned no personal manifest, refusing to merge."));
+ if (string.IsNullOrEmpty(personalDto.Blob))
+ {
+ RecordEmptyPersonalManifest(personalDto);
+ return new PullAndMergeResult { Kind = PullAndMergeKind.NothingToMergeWith };
+ }
+
+ var stateBeforeMerge = state.Clone();
+ try
+ {
+ var opened = await OpenManifestsAndRecordSyncStateAsync(snapshot, personalDto);
+ var serverBlobs = await DownloadReferencedBlobsAsync(opened);
+ var local = await CanonicalizeAsync(localConnection, authService.GetEncryptionKeyAsBase64Async(), null);
+ return await MergeOntoOpenedManifestsAsync(opened, serverBlobs, local);
+ }
+ catch (Exception ex)
+ {
+ // The local vault is still the live one, so the state must keep describing what it was pulled from.
+ state.CopyFrom(stateBeforeMerge);
+ logger.LogError(ex, "[V2Merge] FAILED: the last logged step above is where it broke.");
+ throw new VaultProcessingException("vault-merge", ex);
+ }
+ }
+
+ /// <summary>
+ /// Migrate the local vault onto the current full schema canonicalize the database into
+ /// manifest-v1 form (adopting rows that predate the manifest stamp into the personal manifest) and materialize
+ /// it straight back out again. This is the permanent delivery path for client schema changes.
+ /// </summary>
+ /// <param name="connection">The local vault to migrate.</param>
+ /// <returns>The migrated in-memory database; the caller owns it.</returns>
+ /// <exception cref="VaultProcessingException">Thrown when the vault could not be canonicalized or materialized.</exception>
+ public async Task<SqliteConnection> MigrateVaultToCurrentSchemaAsync(SqliteConnection connection)
+ {
+ logger.LogInformation("[ManifestMigration] Migrating the local vault onto the current schema (local round-trip, no server involved)...");
+ try
+ {
+ var personalManifestId = state.PersonalManifestId ?? throw new InvalidOperationException("No personal manifest id is recorded; pull once before migrating.");
+ var vault = await CanonicalizeAsync(connection, authService.GetEncryptionKeyAsBase64Async(), personalManifestId);
+
+ // Canonicalize already extracted every favicon and attachment as plaintext bytes, so materialize resolves its references without a fetch.
+ var blobs = new Dictionary<string, byte[]>(StringComparer.Ordinal);
+ foreach (var manifest in vault.Manifests)
+ {
+ foreach (var (hash, blob) in manifest.Blobs)
+ {
+ blobs[hash] = blob.Bytes;
+ }
+ }
+
+ var database = await MaterializeAsync(vault.Manifests.Select(manifest => manifest.ManifestJson), vault.Buckets.Select(bucket => bucket.BucketJson), blobs);
+ logger.LogInformation("[ManifestMigration] Migration complete: {BlobCount} blob(s) re-embedded.", blobs.Count);
+ return database;
+ }
+ catch (Exception ex)
+ {
+ logger.LogError(ex, "[ManifestMigration] FAILED to migrate the local vault.");
+ throw new VaultProcessingException("vault-storage-migration", ex);
+ }
+ }
+
+ /// <summary>
+ /// Canonicalize the local vault, validate, encrypt and POST /v2/Vault.
+ /// </summary>
+ /// <param name="connection">The local vault to push.</param>
+ /// <param name="options">How to write.</param>
+ /// <returns>The outcome. Revisions, fingerprints and the blob caches are updated on success.</returns>
+ /// <exception cref="VaultTooLargeException">Thrown when the server refuses an upload as too large.</exception>
+ public async Task<PushResult> PushAsync(SqliteConnection connection, PushOptions options)
+ {
+ var personalManifestId = state.PersonalManifestId ?? throw new InvalidOperationException("No personal manifest id is recorded; pull once before pushing.");
+
+ var unwritable = await FindUnwritableManifestsAsync(connection, personalManifestId);
+ if (unwritable.Count > 0)
+ {
+ logger.LogWarning("[V2Push] Vault holds rows for manifest(s) this session cannot write ({Manifests}); refusing the write until a pull restores them.", string.Join(", ", unwritable));
+ return new PushResult(PushStatus.Outdated, [$"Manifest(s) {string.Join(", ", unwritable)} are not open to this session"]);
+ }
+
+ // LEGACY: the one-time migration push creates the account key hierarchy and re-keys the personal manifest under the new VEK.
+ var migration = options.CreateVaultKey ? await vaultKeyService.CreateAccountKeyHierarchyAsync(authService.GetEncryptionKeyAsBase64Async()) : null;
+ var contentKey = migration?.VaultEncryptionKey ?? authService.GetEncryptionKeyAsBase64Async();
+ if (migration is not null)
+ {
+ logger.LogInformation("[V2Push] Account-key migration: generated a new VEK, AK and account keypair; vault content and all blobs will be re-encrypted and re-uploaded.");
+ }
+
+ var vault = await CanonicalizeAsync(connection, contentKey, null);
+ var emailRouting = EmailRoutingBuilder.Build(vault.Manifests.Select(manifest => manifest.ManifestJson), PrivateEmailDomains());
+ logger.LogInformation("[V2Push] Canonicalize produced {ManifestCount} manifest(s) and {BucketCount} data bucket(s).", vault.Manifests.Count, vault.Buckets.Count);
+
+ // Content-fingerprint gating: compare every canonicalized target against the fingerprint of its last-known server state and only write the targets that changed.
+ var keyByManifestId = vault.Manifests.ToDictionary(manifest => manifest.Record.ManifestId, manifest => manifest.Record.VaultEncryptionKey);
+ var bucketWrites = new List<BucketWrite>();
+ var writtenBucketFingerprints = new Dictionary<string, string>(StringComparer.Ordinal);
+ foreach (var bucket in vault.Buckets)
+ {
+ var label = $"Data bucket \"{bucket.Category}\" of manifest {bucket.ManifestId}";
+ var fingerprintKey = VaultSyncState.BucketFingerprintKey(bucket.ManifestId, bucket.Category);
+ var fingerprint = await rustCoreService.VaultCodecComputeContentFingerprintAsync(bucket.BucketJson);
+ if (!options.ForceFullWrite && migration is null && state.ContentFingerprints.TryGetValue(fingerprintKey, out var baseline) && baseline == fingerprint)
+ {
+ continue;
+ }
+
+ if (!keyByManifestId.TryGetValue(bucket.ManifestId, out var bucketKey))
+ {
+ logger.LogWarning("[V2Push] {Label} names a manifest this vault cannot write; leaving it out of this write.", label);
+ continue;
+ }
+
+ if (!Enum.TryParse<VaultDataBucketCategory>(bucket.Category, true, out var category))
+ {
+ logger.LogWarning("[V2Push] {Label} has a category this client does not know; leaving it out of this write.", label);
+ continue;
+ }
+
+ var validation = await rustCoreService.VaultCodecValidateDataBucketAsync(bucket.BucketJson);
+ if (!validation.Ok)
+ {
+ return new PushResult(PushStatus.Rejected, [$"{label} validation failed: {string.Join(", ", validation.FailedRules)}. {validation.Message}".Trim()]);
+ }
+
+ var ciphertext = await rustCoreService.VaultCodecPackAndEncryptPayloadAsync(bucket.BucketJson, bucketKey);
+ bucketWrites.Add(new BucketWrite
+ {
+ ManifestId = bucket.ManifestId,
+ Category = category,
+ Blob = ciphertext,
+ CiphertextHash = await rustCoreService.VaultCodecComputeCiphertextHashAsync(ciphertext),
+ CurrentRevision = state.BucketRevisions.GetValueOrDefault(VaultSyncState.BucketRevisionKey(bucket.ManifestId, bucket.Category), 0),
+ });
+ writtenBucketFingerprints[fingerprintKey] = fingerprint;
+ }
+
+ // Gate, validate, pack and encrypt every candidate manifest into the write, each with its own VEK.
+ var blobEntries = new Dictionary<string, UploadBlobEntry>(StringComparer.Ordinal);
+ var manifestWrites = new List<ManifestWrite>();
+ var writtenManifestFingerprints = new Dictionary<Guid, string>();
+ foreach (var manifest in vault.Manifests)
+ {
+ foreach (var (hash, blob) in manifest.Blobs)
+ {
+ blobEntries.TryAdd(hash, new UploadBlobEntry(blob.Kind, blob.Bytes, manifest.Record.VaultEncryptionKey, manifest.Record.IsPersonal));
+ }
+
+ var label = manifest.Record.IsPersonal ? "Personal manifest" : $"Shared manifest \"{manifest.Record.Name ?? manifest.Record.ManifestId.ToString()}\"";
+ var fingerprint = await rustCoreService.VaultCodecComputeContentFingerprintAsync(manifest.ManifestJson);
+ var rekeyed = migration is not null && manifest.Record.IsPersonal;
+ if (!options.ForceFullWrite && !rekeyed && state.ContentFingerprints.TryGetValue(VaultSyncState.ManifestFingerprintKey(manifest.Record.ManifestId), out var baseline) && baseline == fingerprint)
+ {
+ continue;
+ }
+
+ var validation = await rustCoreService.VaultCodecValidateManifestAsync(manifest.ManifestJson);
+ if (!validation.Ok)
+ {
+ if (manifest.Record.IsPersonal)
+ {
+ return new PushResult(PushStatus.Rejected, [$"Manifest validation failed: {string.Join(", ", validation.FailedRules)}. {validation.Message}".Trim()]);
+ }
+
+ logger.LogWarning("[V2Push] {Label} failed validation ({Rules}), dropping it from this write.", label, string.Join(", ", validation.FailedRules));
+ continue;
+ }
+
+ var ciphertext = await rustCoreService.VaultCodecPackAndEncryptPayloadAsync(manifest.ManifestJson, manifest.Record.VaultEncryptionKey);
+
+ // Publish the public half of the manifest's email delivery keypair; only admins may publish a shared manifest's key.
+ var mayPublish = manifest.Record.IsPersonal || manifest.Record.CanAdminister;
+ var publicKey = mayPublish ? await VaultTableReader.ReadActivePublicKeyAsync(connection, manifest.Record.ManifestId) : null;
+ if (mayPublish && publicKey is null && !manifest.Record.IsPersonal)
+ {
+ logger.LogWarning("[V2Push] {Label} is missing its email keypair; its aliases stay personal until sharing is re-enabled.", label);
+ }
+
+ manifestWrites.Add(new ManifestWrite
+ {
+ ManifestId = manifest.Record.ManifestId,
+ ManifestBlob = ciphertext,
+ ManifestCiphertextHash = await rustCoreService.VaultCodecComputeCiphertextHashAsync(ciphertext),
+ CurrentRevision = state.ManifestRevisions.GetValueOrDefault(manifest.Record.ManifestId, 0),
+ CredentialsCount = manifest.ItemCount,
+ BlobReferences = manifest.Blobs.Select(pair => new BlobReference { Hash = pair.Key, Category = pair.Value.Kind }).ToList(),
+ EncryptionPublicKey = publicKey,
+ });
+ writtenManifestFingerprints[manifest.Record.ManifestId] = fingerprint;
+ logger.LogInformation("[V2Push] {Label}: {Items} item(s), {Blobs} blob reference(s), {Chars} encrypted characters.", label, manifest.ItemCount, manifest.Blobs.Count, ciphertext.Length);
+ }
+
+ // Nothing changed versus the server baselines: skip the write (and the blob diff) entirely.
+ if (manifestWrites.Count == 0 && bucketWrites.Count == 0)
+ {
+ logger.LogInformation("[V2Push] No content changes detected (every manifest and data bucket matches the server baselines); skipping upload.");
+ return new PushResult(PushStatus.Ok);
+ }
+
+ // Blob diff across every manifest in this write: only encrypt and upload blobs the server does not already have.
+ var allBlobHashes = blobEntries.Keys.ToList();
+ var personalHashes = allBlobHashes.Where(hash => blobEntries[hash].FromPersonal).ToList();
+ var sharedHashes = allBlobHashes.Where(hash => !blobEntries[hash].FromPersonal).ToList();
+ List<string> personalToUpload;
+ List<string> sharedToUpload;
+ if (migration is not null)
+ {
+ personalToUpload = personalHashes;
+ sharedToUpload = await MissingOnServerAsync(sharedHashes.Where(hash => !state.ServerBlobHashes.Contains(hash)));
+ }
+ else
+ {
+ var toUpload = new HashSet<string>(await MissingOnServerAsync(allBlobHashes.Where(hash => !state.ServerBlobHashes.Contains(hash))), StringComparer.Ordinal);
+ personalToUpload = personalHashes.Where(toUpload.Contains).ToList();
+ sharedToUpload = sharedHashes.Where(toUpload.Contains).ToList();
+ }
+
+ logger.LogInformation("[V2Push] Blob diff: {Total} blob(s) across {Manifests} manifest(s), uploading {Personal} personal and {Shared} shared.", allBlobHashes.Count, vault.Manifests.Count, personalToUpload.Count, sharedToUpload.Count);
+ var uploadedCiphertexts = new Dictionary<string, string>(StringComparer.Ordinal);
+ await UploadBlobsAsync(blobEntries, personalToUpload, migration is not null, uploadedCiphertexts);
+ await UploadBlobsAsync(blobEntries, sharedToUpload, false, uploadedCiphertexts);
+
+ var request = new VaultWriteRequest
+ {
+ Username = await authService.GetUsernameAsync(),
+ Manifests = manifestWrites,
+ Buckets = bucketWrites,
+ NewBlobs = [],
+ EmailRouting = emailRouting,
+ AccountKeys = migration?.Keys,
+ };
+
+ var response = await PostWriteAsync(request);
+ if (response.MissingBlobHashes.Count > 0)
+ {
+ // Upload any blobs the server reports as missing and retry the write.
+ var unsatisfiable = response.MissingBlobHashes.Where(hash => !blobEntries.ContainsKey(hash)).ToList();
+ if (unsatisfiable.Count > 0)
+ {
+ return new PushResult(PushStatus.MissingBlobs, unsatisfiable);
+ }
+
+ logger.LogWarning("[V2Push] Server reported {Count} missing blob(s); uploading and retrying once.", response.MissingBlobHashes.Count);
+ await UploadBlobsAsync(blobEntries, response.MissingBlobHashes.Where(hash => blobEntries[hash].FromPersonal), migration is not null, uploadedCiphertexts);
+ await UploadBlobsAsync(blobEntries, response.MissingBlobHashes.Where(hash => !blobEntries[hash].FromPersonal), false, uploadedCiphertexts);
+ response = await PostWriteAsync(request);
+ if (response.MissingBlobHashes.Count > 0)
+ {
+ return new PushResult(PushStatus.MissingBlobs, response.MissingBlobHashes);
+ }
+ }
+
+ if (response.Status != VaultStatus.Ok)
+ {
+ // A single stale manifest or bucket rejected the whole write; the caller pulls, merges and retries.
+ return new PushResult(PushStatus.Outdated);
+ }
+
+ // Advance the baselines of exactly the targets this write carried.
+ foreach (var bucketRevision in response.BucketRevisions)
+ {
+ state.BucketRevisions[VaultSyncState.BucketRevisionKey(bucketRevision.ManifestId, bucketRevision.Category.ToString())] = bucketRevision.Revision;
+ }
+
+ foreach (var manifestRevision in response.ManifestRevisions)
+ {
+ state.ManifestRevisions[manifestRevision.ManifestId] = manifestRevision.Revision;
+ }
+
+ foreach (var (key, fingerprint) in writtenBucketFingerprints)
+ {
+ state.ContentFingerprints[key] = fingerprint;
+ }
+
+ foreach (var (manifestId, fingerprint) in writtenManifestFingerprints)
+ {
+ state.ContentFingerprints[VaultSyncState.ManifestFingerprintKey(manifestId)] = fingerprint;
+ }
+
+ // Every referenced hash is now known to be on the server; refresh the diff baseline and the cache.
+ state.ServerBlobHashes.Clear();
+ state.ServerBlobHashes.UnionWith(allBlobHashes);
+ var refreshedCache = new Dictionary<string, string>(StringComparer.Ordinal);
+ foreach (var hash in allBlobHashes)
+ {
+ if (uploadedCiphertexts.TryGetValue(hash, out var uploaded))
+ {
+ refreshedCache[hash] = uploaded;
+ }
+ else if (state.BlobCipherCache.TryGetValue(hash, out var cached))
+ {
+ refreshedCache[hash] = cached;
+ }
+ }
+
+ state.BlobCipherCache.Clear();
+ foreach (var (hash, ciphertext) in refreshedCache)
+ {
+ state.BlobCipherCache[hash] = ciphertext;
+ }
+
+ state.LastSnapshotWasLegacySqliteBlob = false;
+
+ // Adopt the newly created account keys if this push included the one-time sqlite-blob to manifest-v1 migration.
+ if (migration is not null)
+ {
+ await vaultKeyService.AdoptLocalAccountKeysAsync(migration);
+ await authService.StoreSessionKeysAsync(Convert.FromBase64String(migration.VaultEncryptionKey), migration.AccountPrivateKey);
+ logger.LogInformation("[V2Push] Account-key migration complete: hierarchy created server-side, chain cached locally.");
+ }
+
+ logger.LogInformation("[V2Push] Pushed {Manifests} manifest(s), {Buckets} bucket(s) and {Blobs} blob(s).", manifestWrites.Count, bucketWrites.Count, uploadedCiphertexts.Count);
+ return new PushResult(PushStatus.Ok);
+ }
+
/// <summary>
/// Pick the user's personal manifest out of a snapshot.
/// </summary>
@@ -218,37 +559,150 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ
using (var writer = new Utf8JsonWriter(stream))
{
writer.WriteStartObject();
- writer.WriteStartArray("manifests");
- foreach (var manifestJson in manifestJsons)
+ WriteRawArray(writer, "manifests", manifestJsons);
+ WriteRawArray(writer, "dataBuckets", bucketJsons);
+ WriteSchemaColumns(writer, schemaColumns);
+ writer.WriteEndObject();
+ }
+
+ return Encoding.UTF8.GetString(stream.ToArray());
+ }
+
+ /// <summary>
+ /// Build the JSON input of the Rust canonical merge call.
+ /// </summary>
+ /// <param name="opened">The opened server manifests, the merge base.</param>
+ /// <param name="local">The canonicalized local vault, the incoming side.</param>
+ /// <param name="schemaColumns">The column set of the local schema, per table.</param>
+ /// <returns>The canonical merge input JSON.</returns>
+ private static string BuildMergeInput(OpenedManifestSet opened, CanonicalizedVault local, IReadOnlyDictionary<string, List<string>> schemaColumns)
+ {
+ using var stream = new MemoryStream();
+ using (var writer = new Utf8JsonWriter(stream))
+ {
+ writer.WriteStartObject();
+ WriteRawArray(writer, "serverManifests", opened.Resolved.Select(entry => entry.ManifestJson));
+ WriteRawArray(writer, "serverBuckets", opened.DataBuckets);
+ writer.WriteStartArray("contentlessServerManifestIds");
+ foreach (var id in opened.ContentlessManifestIds)
{
- writer.WriteRawValue(manifestJson, skipInputValidation: true);
+ writer.WriteStringValue(id.ToString());
}
writer.WriteEndArray();
- writer.WriteStartArray("dataBuckets");
- foreach (var bucketJson in bucketJsons)
+ WriteRawArray(writer, "localManifests", local.Manifests.Select(manifest => manifest.ManifestJson));
+ WriteRawArray(writer, "localBuckets", local.Buckets.Select(bucket => bucket.BucketJson));
+ WriteSchemaColumns(writer, schemaColumns);
+ writer.WriteEndObject();
+ }
+
+ return Encoding.UTF8.GetString(stream.ToArray());
+ }
+
+ /// <summary>
+ /// Write a named array of pre-serialized JSON values.
+ /// </summary>
+ /// <param name="writer">The writer.</param>
+ /// <param name="name">The property name.</param>
+ /// <param name="rawValues">The JSON values.</param>
+ private static void WriteRawArray(Utf8JsonWriter writer, string name, IEnumerable<string> rawValues)
+ {
+ writer.WriteStartArray(name);
+ foreach (var rawValue in rawValues)
+ {
+ writer.WriteRawValue(rawValue, skipInputValidation: true);
+ }
+
+ writer.WriteEndArray();
+ }
+
+ /// <summary>
+ /// Write the schemaColumns object.
+ /// </summary>
+ /// <param name="writer">The writer.</param>
+ /// <param name="schemaColumns">The column set of the local schema, per table.</param>
+ private static void WriteSchemaColumns(Utf8JsonWriter writer, IReadOnlyDictionary<string, List<string>> schemaColumns)
+ {
+ writer.WriteStartObject("schemaColumns");
+ foreach (var (table, columns) in schemaColumns)
+ {
+ writer.WriteStartArray(table);
+ foreach (var column in columns)
{
- writer.WriteRawValue(bucketJson, skipInputValidation: true);
+ writer.WriteStringValue(column);
}
writer.WriteEndArray();
- writer.WriteStartObject("schemaColumns");
- foreach (var (table, columns) in schemaColumns)
+ }
+
+ writer.WriteEndObject();
+ }
+
+ /// <summary>
+ /// Every blob reference marker in a manifest payload: hash and kind.
+ /// </summary>
+ /// <param name="manifestJson">The manifest payload JSON.</param>
+ /// <returns>The references.</returns>
+ private static List<(string Hash, string? Kind)> BlobReferencesOf(string manifestJson)
+ {
+ var references = new List<(string Hash, string? Kind)>();
+ using var document = JsonDocument.Parse(manifestJson);
+ if (!document.RootElement.TryGetProperty("tables", out var tables) || tables.ValueKind != JsonValueKind.Object)
+ {
+ return references;
+ }
+
+ foreach (var table in tables.EnumerateObject())
+ {
+ if (table.Value.ValueKind != JsonValueKind.Array)
{
- writer.WriteStartArray(table);
- foreach (var column in columns)
+ continue;
+ }
+
+ foreach (var row in table.Value.EnumerateArray())
+ {
+ foreach (var cell in row.EnumerateObject())
{
- writer.WriteStringValue(column);
+ if (cell.Value.ValueKind == JsonValueKind.Object && cell.Value.TryGetProperty(BlobRefMarker, out var hash) && hash.ValueKind == JsonValueKind.String)
+ {
+ var kind = cell.Value.TryGetProperty(BlobKindMarker, out var kindElement) && kindElement.ValueKind == JsonValueKind.String ? kindElement.GetString() : null;
+ references.Add((hash.GetString()!, kind));
+ }
}
-
- writer.WriteEndArray();
}
+ }
- writer.WriteEndObject();
- writer.WriteEndObject();
+ return references;
+ }
+
+ /// <summary>
+ /// Turn a failed upload response into an exception that names the failure.
+ /// </summary>
+ /// <param name="response">The response.</param>
+ /// <param name="what">What was uploaded, for the message.</param>
+ /// <returns>Task.</returns>
+ private static async Task EnsureUploadSucceededAsync(HttpResponseMessage response, string what)
+ {
+ if (response.StatusCode == HttpStatusCode.RequestEntityTooLarge)
+ {
+ throw new VaultTooLargeException();
}
- return Encoding.UTF8.GetString(stream.ToArray());
+ if (!response.IsSuccessStatusCode)
+ {
+ var body = await response.Content.ReadAsStringAsync();
+ throw new HttpRequestException($"{what} failed with HTTP {(int)response.StatusCode}: {body}", null, response.StatusCode);
+ }
+ }
+
+ /// <summary>
+ /// The domains the server hosts mail for; addresses outside them are never claimed.
+ /// </summary>
+ /// <returns>The domains.</returns>
+ private List<string> PrivateEmailDomains()
+ {
+ // "DISABLED.TLD" was a placeholder used before 0.22.0 that has been replaced by an empty string. TODO: remove in a future release.
+ return config.PrivateEmailDomains.Where(domain => !string.IsNullOrWhiteSpace(domain) && !string.Equals(domain, "DISABLED.TLD", StringComparison.OrdinalIgnoreCase)).ToList();
}
/// <summary>
@@ -260,6 +714,31 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ
return await httpClient.GetFromJsonAsync<GetResponse>(VaultEndpoint) ?? throw new InvalidOperationException("Empty vault snapshot response.");
}
+ /// <summary>
+ /// Record which manifests a snapshot served, before any of them is opened: it is the authority for held access.
+ /// </summary>
+ /// <param name="snapshot">The raw snapshot.</param>
+ private void RecordServedManifests(GetResponse snapshot)
+ {
+ state.LastServedManifestIds.Clear();
+ state.LastServedManifestIds.AddRange(snapshot.Manifests.Select(m => m.ManifestId));
+ state.LastSnapshotWasLegacySqliteBlob = snapshot.StorageFormat != StorageFormat.Manifest;
+ }
+
+ /// <summary>
+ /// Record a personal manifest that exists but was never written: its id and revision are the baseline the first push rebases on.
+ /// </summary>
+ /// <param name="personalDto">The personal manifest.</param>
+ private void RecordEmptyPersonalManifest(Manifest personalDto)
+ {
+ state.PersonalManifestId = personalDto.ManifestId;
+ state.ContentFingerprints.Clear();
+ state.SharedManifests.Clear();
+ state.BucketRevisions.Clear();
+ state.ManifestRevisions.Clear();
+ state.ManifestRevisions[personalDto.ManifestId] = personalDto.Revision;
+ }
+
/// <summary>
/// Take a legacy snapshot apart: the blob passes through untouched. TODO: remove when all accounts have been migrated to the new manifest-v1 format.
/// </summary>
@@ -282,7 +761,7 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ
}
logger.LogInformation("[V2Pull] Legacy sqlite-blob pass-through (account not yet migrated), returning the blob as-is.");
- return new PullResult { Kind = PullKind.LegacySqliteBlob, LegacyVaultBlob = snapshot.LegacyVaultBlob ?? string.Empty, LegacyVersion = snapshot.Version ?? string.Empty, Revision = revision };
+ return new PullResult { Kind = PullKind.LegacySqliteBlob, LegacyVaultBlob = snapshot.LegacyVaultBlob ?? string.Empty, LegacyVersion = snapshot.Version ?? string.Empty };
}
/// <summary>
@@ -649,24 +1128,416 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ
logger.LogWarning("[V2Pull] Referenced {Category} blob {Hash} {Reason}, continuing without it.", reference.Category, reference.Hash, reason);
}
+ /// <summary>
+ /// The merge core: run the Rust canonical merge of the local side against the manifests just opened, validate the
+ /// result per manifest (the server's copy stands for one that fails), and materialize it.
+ /// </summary>
+ /// <param name="opened">The opened server manifests, the merge base.</param>
+ /// <param name="serverBlobs">The plaintext blobs the server manifests reference.</param>
+ /// <param name="local">The canonicalized local vault, the incoming side.</param>
+ /// <returns>The merged result.</returns>
+ private async Task<PullAndMergeResult> MergeOntoOpenedManifestsAsync(OpenedManifestSet opened, Dictionary<string, byte[]> serverBlobs, CanonicalizedVault local)
+ {
+ logger.LogInformation("[V2Merge] Merging {Local} local manifest(s) onto {Server} server manifest(s)...", local.Manifests.Count, opened.Resolved.Count);
+ var mergeOutputJson = await rustCoreService.MergeCanonicalAsync(BuildMergeInput(opened, local, await GetSchemaColumnsAsync()));
+
+ var serverManifestById = opened.Resolved.ToDictionary(entry => entry.ManifestId, entry => entry.ManifestJson);
+ var serverBucketsById = new Dictionary<Guid, List<string>>();
+ foreach (var bucketJson in opened.DataBuckets)
+ {
+ var owner = ReadPayloadHeader(bucketJson).ManifestId ?? Guid.Empty;
+ serverBucketsById.TryAdd(owner, []);
+ serverBucketsById[owner].Add(bucketJson);
+ }
+
+ var contentless = new HashSet<Guid>(opened.ContentlessManifestIds);
+ var manifestJsons = new List<(Guid ManifestId, string Json)>();
+ var bucketJsons = new List<string>();
+ var fallbackManifestIds = new List<Guid>();
+ var droppedLocalManifestIds = new List<Guid>();
+ using (var document = JsonDocument.Parse(mergeOutputJson))
+ {
+ var root = document.RootElement;
+ foreach (var entry in root.GetProperty("manifests").EnumerateArray())
+ {
+ var manifestId = Guid.Parse(entry.GetProperty("manifestId").GetString()!);
+ var manifestJson = entry.GetProperty("manifest").GetRawText();
+ var mergedBuckets = entry.TryGetProperty("buckets", out var buckets) && buckets.ValueKind == JsonValueKind.Array ? buckets.EnumerateArray().Select(bucket => bucket.GetRawText()).ToList() : [];
+ var failure = await ValidateMergedManifestAsync(manifestJson, mergedBuckets);
+ if (failure is not null && !contentless.Contains(manifestId))
+ {
+ logger.LogWarning("[V2Merge] Merged manifest {ManifestId} failed validation ({Failure}); the server's version stands and local changes to it are dropped.", manifestId, failure);
+ fallbackManifestIds.Add(manifestId);
+ if (serverManifestById.TryGetValue(manifestId, out var serverJson))
+ {
+ manifestJsons.Add((manifestId, serverJson));
+ bucketJsons.AddRange(serverBucketsById.GetValueOrDefault(manifestId, []));
+ }
+
+ continue;
+ }
+
+ if (failure is not null)
+ {
+ // A contentless pass-through has no server base to fall back to; keep the local rows, the push gate decides.
+ logger.LogWarning("[V2Merge] Pass-through manifest {ManifestId} failed validation ({Failure}); keeping its local rows.", manifestId, failure);
+ }
+
+ manifestJsons.Add((manifestId, manifestJson));
+ bucketJsons.AddRange(mergedBuckets);
+ if (entry.TryGetProperty("stats", out var stats))
+ {
+ logger.LogInformation("[V2Merge] Manifest {ManifestId}: {Stats}", manifestId, stats.GetRawText());
+ }
+ }
+
+ if (root.TryGetProperty("droppedLocalManifestIds", out var dropped) && dropped.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var id in dropped.EnumerateArray())
+ {
+ if (Guid.TryParse(id.GetString(), out var droppedId))
+ {
+ droppedLocalManifestIds.Add(droppedId);
+ logger.LogWarning("[V2Merge] Local manifest {ManifestId} is no longer served; its rows are dropped from the merged vault.", droppedId);
+ }
+ }
+ }
+ }
+
+ // Blob bytes for materialize: the server download plus everything the local canonicalize extracted.
+ var blobMap = new Dictionary<string, byte[]>(serverBlobs, StringComparer.Ordinal);
+ foreach (var manifest in local.Manifests)
+ {
+ foreach (var (hash, blob) in manifest.Blobs)
+ {
+ blobMap.TryAdd(hash, blob.Bytes);
+ }
+ }
+
+ var personalManifestId = opened.Resolved[0].ManifestId;
+ foreach (var (manifestId, manifestJson) in manifestJsons)
+ {
+ foreach (var (hash, kind) in BlobReferencesOf(manifestJson))
+ {
+ if (blobMap.ContainsKey(hash))
+ {
+ continue;
+ }
+
+ if (manifestId == personalManifestId && string.Equals(kind, AttachmentBlobCategory, StringComparison.OrdinalIgnoreCase))
+ {
+ throw new InvalidOperationException($"The merged vault references attachment blob {hash} with no bytes available, refusing to materialize an incomplete vault.");
+ }
+
+ logger.LogWarning("[V2Merge] Merged {Kind} blob {Hash} has no bytes available; it will materialize as empty.", kind ?? "blob", hash);
+ }
+ }
+
+ var database = await MaterializeAsync(manifestJsons.Select(entry => entry.Json), bucketJsons, blobMap);
+ logger.LogInformation("[V2Merge] Canonical merge complete: {Fallbacks} validation fallback(s), {Dropped} dropped local manifest(s).", fallbackManifestIds.Count, droppedLocalManifestIds.Count);
+ return new PullAndMergeResult { Kind = PullAndMergeKind.Merged, Database = database, FallbackManifestIds = fallbackManifestIds, DroppedLocalManifestIds = droppedLocalManifestIds };
+ }
+
+ /// <summary>
+ /// Validate one merged manifest and its buckets.
+ /// </summary>
+ /// <param name="manifestJson">The merged manifest.</param>
+ /// <param name="bucketJsons">Its merged buckets.</param>
+ /// <returns>The failed rules, or null when valid.</returns>
+ private async Task<string?> ValidateMergedManifestAsync(string manifestJson, List<string> bucketJsons)
+ {
+ var validation = await rustCoreService.VaultCodecValidateManifestAsync(manifestJson);
+ if (!validation.Ok)
+ {
+ return string.Join(", ", validation.FailedRules);
+ }
+
+ foreach (var bucketJson in bucketJsons)
+ {
+ var bucketValidation = await rustCoreService.VaultCodecValidateDataBucketAsync(bucketJson);
+ if (!bucketValidation.Ok)
+ {
+ return $"bucket \"{ReadPayloadHeader(bucketJson).Category}\": {string.Join(", ", bucketValidation.FailedRules)}";
+ }
+ }
+
+ return null;
+ }
+
+ /// <summary>
+ /// Canonicalize the local vault into the manifest-v1 format against every manifest this vault writes, routing each
+ /// row into its own manifest by the ManifestId it carries.
+ /// </summary>
+ /// <param name="connection">The local vault.</param>
+ /// <param name="personalVek">The key the personal manifest encrypts with.</param>
+ /// <param name="adoptUnstampedInto">One-time migration only: the manifest rows without a stamp are adopted into.</param>
+ /// <returns>The canonicalized vault, personal manifest first.</returns>
+ private async Task<CanonicalizedVault> CanonicalizeAsync(SqliteConnection connection, string personalVek, Guid? adoptUnstampedInto)
+ {
+ var records = await ResolveManifestRecordsAsync(connection, personalVek);
+ var tablesJson = await VaultTableReader.ReadTablesAsCodecJsonAsync(connection);
+
+ string inputJson;
+ using (var stream = new MemoryStream())
+ {
+ using (var writer = new Utf8JsonWriter(stream))
+ {
+ writer.WriteStartObject();
+ writer.WritePropertyName("tables");
+ writer.WriteRawValue(tablesJson, skipInputValidation: true);
+ writer.WriteString("canonicalizedAt", DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ", System.Globalization.CultureInfo.InvariantCulture));
+ writer.WriteStartArray("manifests");
+ foreach (var record in records)
+ {
+ writer.WriteStartObject();
+ writer.WriteString("manifestId", record.ManifestId.ToString());
+ writer.WriteString("manifestSalt", record.Salt);
+ if (record.Name is not null)
+ {
+ writer.WriteString("name", record.Name);
+ }
+
+ writer.WriteEndObject();
+ }
+
+ writer.WriteEndArray();
+ if (adoptUnstampedInto is { } adoptInto)
+ {
+ writer.WriteString("adoptUnstampedInto", adoptInto.ToString());
+ }
+
+ writer.WriteEndObject();
+ }
+
+ inputJson = Encoding.UTF8.GetString(stream.ToArray());
+ }
+
+ var outputJson = await rustCoreService.VaultCodecCanonicalizeFromSqliteAsync(inputJson);
+ return ParseCanonicalized(outputJson, records);
+ }
+
+ /// <summary>
+ /// Read the codec's canonicalize output into the shapes the push works with.
+ /// </summary>
+ /// <param name="outputJson">The CanonicalizedVault JSON.</param>
+ /// <param name="records">The records the vault was split against.</param>
+ /// <returns>The canonicalized vault.</returns>
+ private CanonicalizedVault ParseCanonicalized(string outputJson, List<ManifestRecord> records)
+ {
+ using var document = JsonDocument.Parse(outputJson);
+ var root = document.RootElement;
+ var manifests = new List<CanonicalizedManifest>();
+ foreach (var entry in root.GetProperty("manifests").EnumerateArray())
+ {
+ var manifest = entry.GetProperty("manifest");
+ var manifestId = Guid.Parse(manifest.GetProperty("manifestId").GetString()!);
+ var record = records.FirstOrDefault(candidate => candidate.ManifestId == manifestId);
+ if (record is null)
+ {
+ logger.LogWarning("[V2Push] Canonicalize produced manifest {ManifestId}, which is not in the write set; leaving it out.", manifestId);
+ continue;
+ }
+
+ var itemCount = manifest.TryGetProperty("tables", out var tables) && tables.TryGetProperty("Items", out var items) && items.ValueKind == JsonValueKind.Array ? items.GetArrayLength() : 0;
+ var blobs = new Dictionary<string, CanonicalizedBlob>(StringComparer.Ordinal);
+ if (entry.TryGetProperty("blobs", out var blobsElement) && blobsElement.ValueKind == JsonValueKind.Object)
+ {
+ foreach (var blob in blobsElement.EnumerateObject())
+ {
+ blobs[blob.Name] = new CanonicalizedBlob(blob.Value.GetProperty("kind").GetString() ?? string.Empty, blob.Value.GetProperty("bytesBase64").GetBytesFromBase64());
+ }
+ }
+
+ manifests.Add(new CanonicalizedManifest(record, manifest.GetRawText(), itemCount, blobs));
+ }
+
+ var buckets = new List<CanonicalizedBucket>();
+ if (root.TryGetProperty("dataBuckets", out var dataBuckets) && dataBuckets.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var bucket in dataBuckets.EnumerateArray())
+ {
+ buckets.Add(new CanonicalizedBucket(Guid.Parse(bucket.GetProperty("manifestId").GetString()!), bucket.GetProperty("category").GetString() ?? string.Empty, bucket.GetRawText()));
+ }
+ }
+
+ return new CanonicalizedVault(manifests, buckets);
+ }
+
+ /// <summary>
+ /// Resolve every manifest this vault can write, personal manifest first.
+ /// </summary>
+ /// <param name="connection">The local vault.</param>
+ /// <param name="personalVek">The key the personal manifest encrypts with.</param>
+ /// <returns>The records.</returns>
+ private async Task<List<ManifestRecord>> ResolveManifestRecordsAsync(SqliteConnection connection, string personalVek)
+ {
+ var personalManifestId = state.PersonalManifestId ?? throw new InvalidOperationException("No personal manifest id is recorded; pull once before pushing.");
+ state.PersonalManifestSalt ??= await rustCoreService.VaultCodecGenerateManifestSaltAsync();
+
+ var opened = state.SharedManifests.Values.Where(record => !string.IsNullOrEmpty(record.VaultEncryptionKey)).ToList();
+ var request = new
+ {
+ personalManifestId = personalManifestId.ToString(),
+ personalManifestSalt = state.PersonalManifestSalt,
+ stampedManifestIds = (await VaultTableReader.ManifestIdsInVaultAsync(connection)).ToList(),
+ openedManifestIds = opened.Select(record => record.ManifestId.ToString()).ToList(),
+ heldRecords = state.SharedManifests.Values.Select(record => new { manifestId = record.ManifestId.ToString(), salt = record.Salt, name = record.Name, canAdminister = record.CanAdminister }).ToList(),
+ displayNames = await VaultTableReader.ReadDisplayNamesAsync(connection),
+ };
+
+ var writeSetJson = await rustCoreService.VaultSharingResolveManifestWriteSetAsync(JsonSerializer.Serialize(request));
+ using var document = JsonDocument.Parse(writeSetJson);
+ var root = document.RootElement;
+ if (root.TryGetProperty("skipped", out var skipped) && skipped.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var entry in skipped.EnumerateArray())
+ {
+ logger.LogInformation("[V2Push] Shared manifest {ManifestId} is left out of the write: {Reason}.", entry.GetProperty("manifestId").GetString(), entry.GetProperty("reason").GetString());
+ }
+ }
+
+ var records = new List<ManifestRecord>();
+ foreach (var entry in root.GetProperty("records").EnumerateArray())
+ {
+ var manifestId = Guid.Parse(entry.GetProperty("manifestId").GetString()!);
+ var isPersonal = entry.GetProperty("isPersonal").GetBoolean();
+ var name = entry.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null;
+ var canAdminister = entry.TryGetProperty("canAdminister", out var adminElement) && adminElement.ValueKind == JsonValueKind.True;
+ var vek = isPersonal ? personalVek : state.SharedManifests.GetValueOrDefault(manifestId)?.VaultEncryptionKey;
+ if (string.IsNullOrEmpty(vek))
+ {
+ throw new InvalidOperationException($"Manifest {manifestId} is in the write set without a key, refusing to write it.");
+ }
+
+ records.Add(new ManifestRecord(manifestId, isPersonal, entry.GetProperty("salt").GetString()!, vek, name, canAdminister));
+ }
+
+ return records;
+ }
+
+ /// <summary>
+ /// The manifests the local vault holds rows for that this session cannot write.
+ /// </summary>
+ /// <param name="connection">The local vault.</param>
+ /// <param name="personalManifestId">The caller's own manifest.</param>
+ /// <returns>The manifest ids, empty when the vault and the session agree.</returns>
+ private async Task<List<string>> FindUnwritableManifestsAsync(SqliteConnection connection, Guid personalManifestId)
+ {
+ var writable = state.SharedManifests.Values.Where(record => !string.IsNullOrEmpty(record.VaultEncryptionKey)).Select(record => record.ManifestId.ToString()).Append(personalManifestId.ToString()).ToList();
+ var request = new
+ {
+ manifestIdsInVault = (await VaultTableReader.ManifestIdsInVaultAsync(connection)).ToList(),
+ writableManifestIds = writable,
+ grantedManifestIds = Array.Empty<string>(),
+ };
+
+ var partitionJson = await rustCoreService.VaultSharingPartitionManifestAccessAsync(JsonSerializer.Serialize(request));
+ using var document = JsonDocument.Parse(partitionJson);
+ return document.RootElement.TryGetProperty("unwritable", out var unwritable) && unwritable.ValueKind == JsonValueKind.Array ? unwritable.EnumerateArray().Select(id => id.GetString() ?? string.Empty).ToList() : [];
+ }
+
+ /// <summary>
+ /// Ask the server which of the given blobs it does not hold.
+ /// </summary>
+ /// <param name="hashes">The candidate hashes.</param>
+ /// <returns>The hashes unknown to the server.</returns>
+ private async Task<List<string>> MissingOnServerAsync(IEnumerable<string> hashes)
+ {
+ var candidates = hashes.ToList();
+ if (candidates.Count == 0)
+ {
+ return [];
+ }
+
+ using var response = await httpClient.PostAsJsonAsync(BlobsMissingEndpoint, new BlobHashesRequest { Hashes = candidates });
+ response.EnsureSuccessStatusCode();
+ var body = await response.Content.ReadFromJsonAsync<MissingBlobsResponse>();
+ return body?.Missing ?? [];
+ }
+
+ /// <summary>
+ /// Encrypt the given blobs, each with the key of the manifest that owns it, and upload them in size-capped batches
+ /// ahead of the manifest write.
+ /// </summary>
+ /// <param name="entries">Every staged blob, by hash.</param>
+ /// <param name="hashes">The subset to upload.</param>
+ /// <param name="overwrite">Ask the server to replace the ciphertext of blobs it already has (the migration re-keys them).</param>
+ /// <param name="uploaded">Receives the uploaded ciphertext per hash, for the local cipher cache.</param>
+ /// <returns>Task.</returns>
+ private async Task UploadBlobsAsync(Dictionary<string, UploadBlobEntry> entries, IEnumerable<string> hashes, bool overwrite, Dictionary<string, string> uploaded)
+ {
+ var batch = new List<Blob>();
+ var batchChars = 0;
+ foreach (var hash in hashes)
+ {
+ if (!entries.TryGetValue(hash, out var entry))
+ {
+ continue;
+ }
+
+ var ciphertext = Convert.ToBase64String(await jsInteropService.SymmetricEncryptBytes(entry.Bytes, Convert.FromBase64String(entry.VaultEncryptionKey)));
+ uploaded[hash] = ciphertext;
+
+ // Flush before adding when this blob would take the request past either bound.
+ if (batch.Count > 0 && (batchChars + ciphertext.Length > BlobTransferBatchMaxChars || batch.Count >= BlobTransferBatchMaxCount))
+ {
+ await PostBlobBatchAsync(batch, overwrite);
+ batch = [];
+ batchChars = 0;
+ }
+
+ batch.Add(new Blob { Hash = hash, Category = entry.Kind, EncryptedDataBase64 = ciphertext });
+ batchChars += ciphertext.Length;
+ }
+
+ if (batch.Count > 0)
+ {
+ await PostBlobBatchAsync(batch, overwrite);
+ }
+ }
+
+ /// <summary>
+ /// Upload one batch of encrypted blobs.
+ /// </summary>
+ /// <param name="batch">The blobs.</param>
+ /// <param name="overwrite">Whether the server replaces ciphertext it already holds.</param>
+ /// <returns>Task.</returns>
+ private async Task PostBlobBatchAsync(List<Blob> batch, bool overwrite)
+ {
+ logger.LogInformation("[V2Push] Uploading blob batch: {Count} blob(s).", batch.Count);
+ using var response = await httpClient.PostAsJsonAsync(BlobsEndpoint, new BlobUploadRequest { Blobs = batch, Overwrite = overwrite });
+ await EnsureUploadSucceededAsync(response, "Blob upload");
+ }
+
+ /// <summary>
+ /// POST the unified vault write.
+ /// </summary>
+ /// <param name="request">The write.</param>
+ /// <returns>The server's response.</returns>
+ private async Task<VaultWriteResponse> PostWriteAsync(VaultWriteRequest request)
+ {
+ using var response = await httpClient.PostAsJsonAsync(VaultEndpoint, request);
+ await EnsureUploadSucceededAsync(response, "Vault write");
+ return await response.Content.ReadFromJsonAsync<VaultWriteResponse>() ?? throw new InvalidOperationException("Empty vault write response.");
+ }
+
/// <summary>
/// Materialize manifests and data buckets into a fresh in-memory SQLite database via the codec.
/// </summary>
- /// <param name="opened">The opened manifests and buckets.</param>
+ /// <param name="manifestJsons">The manifests, the caller's own first.</param>
+ /// <param name="bucketJsons">The data buckets belonging to those manifests.</param>
/// <param name="blobs">Plaintext bytes for every blob hash the manifests reference.</param>
/// <returns>The open database connection; the caller owns it.</returns>
- private async Task<SqliteConnection> MaterializeAsync(OpenedManifestSet opened, Dictionary<string, byte[]> blobs)
+ private async Task<SqliteConnection> MaterializeAsync(IEnumerable<string> manifestJsons, IEnumerable<string> bucketJsons, Dictionary<string, byte[]> blobs)
{
var connection = new SqliteConnection("Data Source=:memory:");
await connection.OpenAsync();
try
{
- var schemaSql = await jsInteropService.GetCompleteSchemaSqlAsync();
- await VaultMaterializer.ApplySchemaAsync(connection, schemaSql);
+ await VaultMaterializer.ApplySchemaAsync(connection, await GetSchemaSqlAsync());
// The local schema's column set routes anything a newer writer stored into the overflow carrier instead of crashing the insert.
- var schemaColumns = await VaultMaterializer.ReadSchemaColumnsAsync(connection);
- var inputJson = BuildMaterializeInput(opened.Resolved.Select(entry => entry.ManifestJson), opened.DataBuckets, schemaColumns);
+ var inputJson = BuildMaterializeInput(manifestJsons, bucketJsons, await GetSchemaColumnsAsync());
var materializedJson = await rustCoreService.VaultCodecMaterializeAsSqliteAsync(inputJson);
await VaultMaterializer.InsertMaterializedTablesAsync(connection, materializedJson, blobs, logger);
return connection;
@@ -677,4 +1548,40 @@ public sealed class VaultSyncService(HttpClient httpClient, AuthService authServ
throw;
}
}
+
+ /// <summary>
+ /// The complete client schema, read once.
+ /// </summary>
+ /// <returns>The COMPLETE_SCHEMA_SQL script.</returns>
+ private async Task<string> GetSchemaSqlAsync()
+ {
+ return _schemaSql ??= await jsInteropService.GetCompleteSchemaSqlAsync();
+ }
+
+ /// <summary>
+ /// The column set of the complete client schema, per table, read once from a scratch database.
+ /// </summary>
+ /// <returns>Column names per table.</returns>
+ private async Task<Dictionary<string, List<string>>> GetSchemaColumnsAsync()
+ {
+ if (_schemaColumns is not null)
+ {
+ return _schemaColumns;
+ }
+
+ await using var scratch = new SqliteConnection("Data Source=:memory:");
+ await scratch.OpenAsync();
+ await VaultMaterializer.ApplySchemaAsync(scratch, await GetSchemaSqlAsync());
+ _schemaColumns = await VaultMaterializer.ReadSchemaColumnsAsync(scratch);
+ return _schemaColumns;
+ }
+
+ /// <summary>
+ /// A plaintext blob staged for upload: its bytes plus the key that encrypts it.
+ /// </summary>
+ /// <param name="Kind">The blob kind.</param>
+ /// <param name="Bytes">The plaintext bytes.</param>
+ /// <param name="VaultEncryptionKey">The key of the manifest that owns it.</param>
+ /// <param name="FromPersonal">Whether the personal manifest owns it.</param>
+ private sealed record UploadBlobEntry(string Kind, byte[] Bytes, string VaultEncryptionKey, bool FromPersonal);
}
diff --git a/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncState.cs b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncState.cs
index 3329a17bc..d20f969bc 100644
--- a/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncState.cs
+++ b/apps/server/AliasVault.Client/Services/VaultSync/VaultSyncState.cs
@@ -88,6 +88,56 @@ public sealed class VaultSyncState
/// <returns>The record key.</returns>
public static string BucketFingerprintKey(Guid manifestId, string category) => $"bucket:{manifestId}:{category}";
+ /// <summary>
+ /// A detached copy, so a failed operation can restore the state it started from.
+ /// </summary>
+ /// <returns>The copy.</returns>
+ public VaultSyncState Clone()
+ {
+ var copy = new VaultSyncState();
+ copy.CopyFrom(this);
+ return copy;
+ }
+
+ /// <summary>
+ /// Replace everything with another state's content.
+ /// </summary>
+ /// <param name="other">The state to copy.</param>
+ public void CopyFrom(VaultSyncState other)
+ {
+ Clear();
+ PersonalManifestId = other.PersonalManifestId;
+ PersonalManifestSalt = other.PersonalManifestSalt;
+ LastSnapshotWasLegacySqliteBlob = other.LastSnapshotWasLegacySqliteBlob;
+ foreach (var (key, value) in other.ManifestRevisions)
+ {
+ ManifestRevisions[key] = value;
+ }
+
+ foreach (var (key, value) in other.BucketRevisions)
+ {
+ BucketRevisions[key] = value;
+ }
+
+ foreach (var (key, value) in other.ContentFingerprints)
+ {
+ ContentFingerprints[key] = value;
+ }
+
+ ServerBlobHashes.UnionWith(other.ServerBlobHashes);
+ foreach (var (key, value) in other.BlobCipherCache)
+ {
+ BlobCipherCache[key] = value;
+ }
+
+ foreach (var (key, value) in other.SharedManifests)
+ {
+ SharedManifests[key] = value;
+ }
+
+ LastServedManifestIds.AddRange(other.LastServedManifestIds);
+ }
+
/// <summary>
/// Forgets everything; the next pull rebuilds it.
/// </summary>