Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Add ManifestStamper and VaultTableReader scaffolding (#2159)

Leendert de Borst5b7c998901bc
diff --git a/apps/server/AliasVault.Client/Services/Database/ManifestStamper.cs b/apps/server/AliasVault.Client/Services/Database/ManifestStamper.cs
new file mode 100644
index 000000000..30e30994a
--- /dev/null
+++ b/apps/server/AliasVault.Client/Services/Database/ManifestStamper.cs
@@ -0,0 +1,236 @@
+//-----------------------------------------------------------------------
+// <copyright file="ManifestStamper.cs" company="aliasvault">
+// Copyright (c) aliasvault. All rights reserved.
+// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
+// </copyright>
+//-----------------------------------------------------------------------
+
+namespace AliasVault.Client.Services.Database;
+
+using AliasClientDb;
+using AliasClientDb.Abstracts;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.ChangeTracking;
+
+/// <summary>
+/// Stamps every new row with the manifest it belongs to before it is saved. Row identity is (ManifestId, Id) and the
+/// codec refuses to write a row that names no manifest, so nothing may reach the database unstamped.
+/// </summary>
+internal static class ManifestStamper
+{
+ /// <summary>
+ /// Stamp every added row that carries no manifest yet.
+ /// </summary>
+ /// <param name="context">The context about to save.</param>
+ /// <param name="personalManifestId">The caller's own manifest, the default scope.</param>
+ public static void Stamp(DbContext context, Guid personalManifestId)
+ {
+ context.ChangeTracker.DetectChanges();
+ var unstamped = context.ChangeTracker.Entries<ManifestScopedEntity>().Where(entry => entry.State == EntityState.Added && entry.Entity.ManifestId == Guid.Empty).ToList();
+ if (unstamped.Count == 0)
+ {
+ return;
+ }
+
+ var resolver = new ScopeResolver(context, personalManifestId);
+
+ // Containers first, so the rows inside them can look their scope up.
+ foreach (var entry in unstamped.Where(entry => entry.Entity is Folder))
+ {
+ Apply(entry, resolver.ForFolder((Folder)entry.Entity));
+ }
+
+ foreach (var entry in unstamped.Where(entry => entry.Entity is Item))
+ {
+ Apply(entry, resolver.ForItem((Item)entry.Entity));
+ }
+
+ foreach (var entry in unstamped)
+ {
+ var itemId = ItemIdOf(entry.Entity);
+ if (itemId is { } id)
+ {
+ Apply(entry, resolver.OfItem(id));
+ }
+ }
+
+ // Rows that other rows point at follow the row pointing at them.
+ var referenced = ReferencedScopes(context);
+ foreach (var entry in unstamped.Where(entry => entry.Entity.ManifestId == Guid.Empty))
+ {
+ Apply(entry, referenced.TryGetValue(ReferenceKey(entry.Entity), out var scope) ? scope : personalManifestId);
+ }
+ }
+
+ /// <summary>
+ /// Set the stamp on an added entry.
+ /// </summary>
+ /// <param name="entry">The entry.</param>
+ /// <param name="manifestId">The manifest.</param>
+ private static void Apply(EntityEntry<ManifestScopedEntity> entry, Guid manifestId)
+ {
+ if (entry.Entity.ManifestId == Guid.Empty)
+ {
+ entry.Property(nameof(ManifestScopedEntity.ManifestId)).CurrentValue = manifestId;
+ }
+ }
+
+ /// <summary>
+ /// The item a child row belongs to, or null when the row is not an item child.
+ /// </summary>
+ /// <param name="entity">The entity.</param>
+ /// <returns>The item id.</returns>
+ private static Guid? ItemIdOf(ManifestScopedEntity entity)
+ {
+ return entity switch
+ {
+ FieldValue row => row.ItemId,
+ FieldHistory row => row.ItemId,
+ Attachment row => row.ItemId,
+ Passkey row => row.ItemId,
+ TotpCode row => row.ItemId,
+ ItemTag row => row.ItemId,
+ ItemStat row => row.Id,
+ _ => null,
+ };
+ }
+
+ /// <summary>
+ /// The lookup key a referenced row is found under.
+ /// </summary>
+ /// <param name="entity">The entity.</param>
+ /// <returns>The key.</returns>
+ private static (Type Type, Guid Id) ReferenceKey(ManifestScopedEntity entity)
+ {
+ return entity switch
+ {
+ Tag row => (typeof(Tag), row.Id),
+ FieldDefinition row => (typeof(FieldDefinition), row.Id),
+ Logo row => (typeof(Logo), row.Id),
+ _ => (entity.GetType(), Guid.Empty),
+ };
+ }
+
+ /// <summary>
+ /// The manifest every tag, field definition and logo is referenced from by a tracked, stamped row.
+ /// </summary>
+ /// <param name="context">The context.</param>
+ /// <returns>Reference key to manifest.</returns>
+ private static Dictionary<(Type Type, Guid Id), Guid> ReferencedScopes(DbContext context)
+ {
+ var scopes = new Dictionary<(Type Type, Guid Id), Guid>();
+ foreach (var entry in context.ChangeTracker.Entries<ManifestScopedEntity>().Where(entry => entry.State is EntityState.Added or EntityState.Modified && entry.Entity.ManifestId != Guid.Empty))
+ {
+ switch (entry.Entity)
+ {
+ case ItemTag row:
+ scopes.TryAdd((typeof(Tag), row.TagId), row.ManifestId);
+ break;
+ case FieldValue { FieldDefinitionId: { } definitionId } row:
+ scopes.TryAdd((typeof(FieldDefinition), definitionId), row.ManifestId);
+ break;
+ case FieldHistory { FieldDefinitionId: { } definitionId } row:
+ scopes.TryAdd((typeof(FieldDefinition), definitionId), row.ManifestId);
+ break;
+ case Item { LogoId: { } logoId } row:
+ scopes.TryAdd((typeof(Logo), logoId), row.ManifestId);
+ break;
+ }
+ }
+
+ return scopes;
+ }
+
+ /// <summary>
+ /// Resolves the manifest of folders and items, from the change tracker first and the database second.
+ /// </summary>
+ private sealed class ScopeResolver(DbContext context, Guid personalManifestId)
+ {
+ private readonly Dictionary<Guid, Guid> _folderScopes = [];
+ private readonly Dictionary<Guid, Guid> _itemScopes = [];
+
+ /// <summary>
+ /// The manifest a folder belongs to: its parent's, or personal at the root.
+ /// </summary>
+ /// <param name="folder">The folder.</param>
+ /// <returns>The manifest.</returns>
+ public Guid ForFolder(Folder folder)
+ {
+ var scope = folder.ParentFolderId is { } parentId ? OfFolder(parentId) : personalManifestId;
+ _folderScopes[folder.Id] = scope;
+ return scope;
+ }
+
+ /// <summary>
+ /// The manifest an item belongs to: its folder's, or personal without one.
+ /// </summary>
+ /// <param name="item">The item.</param>
+ /// <returns>The manifest.</returns>
+ public Guid ForItem(Item item)
+ {
+ var scope = item.FolderId is { } folderId ? OfFolder(folderId) : personalManifestId;
+ _itemScopes[item.Id] = scope;
+ return scope;
+ }
+
+ /// <summary>
+ /// The manifest of an existing item.
+ /// </summary>
+ /// <param name="itemId">The item id.</param>
+ /// <returns>The manifest, personal when the item is unknown.</returns>
+ public Guid OfItem(Guid itemId)
+ {
+ if (_itemScopes.TryGetValue(itemId, out var known))
+ {
+ return known;
+ }
+
+ var tracked = context.ChangeTracker.Entries<Item>().FirstOrDefault(entry => entry.Entity.Id == itemId && entry.Entity.ManifestId != Guid.Empty)?.Entity.ManifestId;
+ var scope = tracked ?? context.Set<Item>().AsNoTracking().Where(item => item.Id == itemId).Select(item => item.ManifestId).FirstOrDefault();
+ if (scope == Guid.Empty)
+ {
+ scope = personalManifestId;
+ }
+
+ _itemScopes[itemId] = scope;
+ return scope;
+ }
+
+ /// <summary>
+ /// The manifest of an existing folder.
+ /// </summary>
+ /// <param name="folderId">The folder id.</param>
+ /// <returns>The manifest, personal when the folder is unknown.</returns>
+ private Guid OfFolder(Guid folderId)
+ {
+ if (_folderScopes.TryGetValue(folderId, out var known))
+ {
+ return known;
+ }
+
+ var trackedEntry = context.ChangeTracker.Entries<Folder>().FirstOrDefault(entry => entry.Entity.Id == folderId);
+ Guid scope;
+ if (trackedEntry is not null && trackedEntry.Entity.ManifestId != Guid.Empty)
+ {
+ scope = trackedEntry.Entity.ManifestId;
+ }
+ else if (trackedEntry is not null)
+ {
+ // An added parent that is not stamped yet: resolve it through its own parent chain.
+ _folderScopes[folderId] = personalManifestId;
+ scope = ForFolder(trackedEntry.Entity);
+ }
+ else
+ {
+ scope = context.Set<Folder>().AsNoTracking().Where(folder => folder.Id == folderId).Select(folder => folder.ManifestId).FirstOrDefault();
+ if (scope == Guid.Empty)
+ {
+ scope = personalManifestId;
+ }
+ }
+
+ _folderScopes[folderId] = scope;
+ return scope;
+ }
+ }
+}
diff --git a/apps/server/AliasVault.Client/Services/Database/VaultTableReader.cs b/apps/server/AliasVault.Client/Services/Database/VaultTableReader.cs
new file mode 100644
index 000000000..4e4e4babb
--- /dev/null
+++ b/apps/server/AliasVault.Client/Services/Database/VaultTableReader.cs
@@ -0,0 +1,247 @@
+//-----------------------------------------------------------------------
+// <copyright file="VaultTableReader.cs" company="aliasvault">
+// Copyright (c) aliasvault. All rights reserved.
+// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
+// </copyright>
+//-----------------------------------------------------------------------
+
+namespace AliasVault.Client.Services.Database;
+
+using System.Text;
+using System.Text.Json;
+using Microsoft.Data.Sqlite;
+
+/// <summary>
+/// Reads the local vault database in the row-JSON shape the Rust codec consumes: every user table, byte columns as
+/// <c>{ "__b64": base64 }</c>, and the manifest bookkeeping the push needs alongside it.
+/// </summary>
+public static class VaultTableReader
+{
+ private const string ManifestIdColumn = "ManifestId";
+ private const string InlineBytesMarker = "__b64";
+
+ /// <summary>
+ /// Read every user table into the codec's <c>CodecTableData[]</c> JSON.
+ /// </summary>
+ /// <param name="connection">The open connection.</param>
+ /// <returns>The JSON array.</returns>
+ public static async Task<string> ReadTablesAsCodecJsonAsync(SqliteConnection connection)
+ {
+ using var stream = new MemoryStream();
+ using (var writer = new Utf8JsonWriter(stream))
+ {
+ writer.WriteStartArray();
+ foreach (var table in await ListUserTablesAsync(connection))
+ {
+ writer.WriteStartObject();
+ writer.WriteString("name", table);
+ writer.WriteStartArray("records");
+ await WriteRowsAsync(connection, table, writer);
+ writer.WriteEndArray();
+ writer.WriteEndObject();
+ }
+
+ writer.WriteEndArray();
+ }
+
+ return Encoding.UTF8.GetString(stream.ToArray());
+ }
+
+ /// <summary>
+ /// The names of all user tables, in name order.
+ /// </summary>
+ /// <param name="connection">The open connection.</param>
+ /// <returns>The table names.</returns>
+ public static async Task<List<string>> ListUserTablesAsync(SqliteConnection connection)
+ {
+ await using var command = connection.CreateCommand();
+ command.CommandText = "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name";
+ await using var reader = await command.ExecuteReaderAsync();
+ var tables = new List<string>();
+ while (await reader.ReadAsync())
+ {
+ tables.Add(reader.GetString(0));
+ }
+
+ return tables;
+ }
+
+ /// <summary>
+ /// The tables carrying a manifest stamp.
+ /// </summary>
+ /// <param name="connection">The open connection.</param>
+ /// <returns>The table names.</returns>
+ public static async Task<List<string>> ListStampedTablesAsync(SqliteConnection connection)
+ {
+ var stamped = new List<string>();
+ foreach (var table in await ListUserTablesAsync(connection))
+ {
+ if (await HasColumnAsync(connection, table, ManifestIdColumn))
+ {
+ stamped.Add(table);
+ }
+ }
+
+ return stamped;
+ }
+
+ /// <summary>
+ /// Every manifest id this vault holds a row for, spelled as stored.
+ /// </summary>
+ /// <param name="connection">The open connection.</param>
+ /// <returns>The manifest ids.</returns>
+ public static async Task<HashSet<string>> ManifestIdsInVaultAsync(SqliteConnection connection)
+ {
+ var ids = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+ foreach (var table in await ListStampedTablesAsync(connection))
+ {
+ await using var command = connection.CreateCommand();
+ command.CommandText = $"SELECT DISTINCT \"{ManifestIdColumn}\" FROM \"{table}\" WHERE \"{ManifestIdColumn}\" IS NOT NULL AND \"{ManifestIdColumn}\" != ''";
+ await using var reader = await command.ExecuteReaderAsync();
+ while (await reader.ReadAsync())
+ {
+ var id = reader.GetString(0);
+ if (Guid.TryParse(id, out var parsed) && parsed != Guid.Empty)
+ {
+ ids.Add(id);
+ }
+ }
+ }
+
+ return ids;
+ }
+
+ /// <summary>
+ /// The display name of every shared manifest rendered as a folder (a folder whose id is the manifest id), keyed by
+ /// lowercase manifest id. This is the authority for the name the push writes into a shared manifest.
+ /// </summary>
+ /// <param name="connection">The open connection.</param>
+ /// <returns>Manifest id to name.</returns>
+ public static async Task<Dictionary<string, string>> ReadDisplayNamesAsync(SqliteConnection connection)
+ {
+ var names = new Dictionary<string, string>(StringComparer.Ordinal);
+ if (!await HasColumnAsync(connection, "Folders", ManifestIdColumn))
+ {
+ return names;
+ }
+
+ await using var command = connection.CreateCommand();
+ command.CommandText = "SELECT ManifestId, Name FROM Folders WHERE IsDeleted = 0 AND ManifestId IS NOT NULL AND UPPER(Id) = UPPER(ManifestId)";
+ await using var reader = await command.ExecuteReaderAsync();
+ while (await reader.ReadAsync())
+ {
+ names[reader.GetString(0).ToLowerInvariant()] = reader.IsDBNull(1) ? string.Empty : reader.GetString(1);
+ }
+
+ return names;
+ }
+
+ /// <summary>
+ /// The public half of a manifest's active email delivery keypair, or null when the manifest holds none.
+ /// </summary>
+ /// <param name="connection">The open connection.</param>
+ /// <param name="manifestId">The manifest.</param>
+ /// <returns>The public key.</returns>
+ public static async Task<string?> ReadActivePublicKeyAsync(SqliteConnection connection, Guid manifestId)
+ {
+ await using var command = connection.CreateCommand();
+ command.CommandText = "SELECT PublicKey FROM EncryptionKeys WHERE UPPER(ManifestId) = UPPER(@manifestId) AND IsPrimary = 1 AND IsDeleted = 0 LIMIT 1";
+ command.Parameters.AddWithValue("@manifestId", manifestId.ToString());
+ return await command.ExecuteScalarAsync() as string;
+ }
+
+ /// <summary>
+ /// The highest EF migration id the database is stamped with, or null when it carries none.
+ /// </summary>
+ /// <param name="connection">The open connection.</param>
+ /// <returns>The migration id.</returns>
+ public static async Task<string?> ReadLatestMigrationIdAsync(SqliteConnection connection)
+ {
+ if (!await HasColumnAsync(connection, "__EFMigrationsHistory", "MigrationId"))
+ {
+ return null;
+ }
+
+ await using var command = connection.CreateCommand();
+ command.CommandText = "SELECT MigrationId FROM __EFMigrationsHistory ORDER BY MigrationId DESC LIMIT 1";
+ return await command.ExecuteScalarAsync() as string;
+ }
+
+ /// <summary>
+ /// Whether a table has the given column.
+ /// </summary>
+ /// <param name="connection">The open connection.</param>
+ /// <param name="table">The table.</param>
+ /// <param name="column">The column.</param>
+ /// <returns>True when present.</returns>
+ public static async Task<bool> HasColumnAsync(SqliteConnection connection, string table, string column)
+ {
+ await using var command = connection.CreateCommand();
+ command.CommandText = $"SELECT COUNT(*) FROM pragma_table_info(\"{table}\") WHERE name = @column";
+ command.Parameters.AddWithValue("@column", column);
+ return Convert.ToInt64(await command.ExecuteScalarAsync()) > 0;
+ }
+
+ /// <summary>
+ /// Write every row of a table as a JSON object.
+ /// </summary>
+ /// <param name="connection">The open connection.</param>
+ /// <param name="table">The table.</param>
+ /// <param name="writer">The writer positioned inside an array.</param>
+ /// <returns>Task.</returns>
+ private static async Task WriteRowsAsync(SqliteConnection connection, string table, Utf8JsonWriter writer)
+ {
+ await using var command = connection.CreateCommand();
+ command.CommandText = $"SELECT * FROM \"{table}\"";
+ await using var reader = await command.ExecuteReaderAsync();
+ var columns = new string[reader.FieldCount];
+ for (var index = 0; index < columns.Length; index++)
+ {
+ columns[index] = reader.GetName(index);
+ }
+
+ while (await reader.ReadAsync())
+ {
+ writer.WriteStartObject();
+ for (var index = 0; index < columns.Length; index++)
+ {
+ writer.WritePropertyName(columns[index]);
+ WriteCell(writer, reader.IsDBNull(index) ? null : reader.GetValue(index));
+ }
+
+ writer.WriteEndObject();
+ }
+ }
+
+ /// <summary>
+ /// Write one cell: bytes as the inline marker, JSON scalars by their SQLite affinity.
+ /// </summary>
+ /// <param name="writer">The writer.</param>
+ /// <param name="value">The cell value.</param>
+ private static void WriteCell(Utf8JsonWriter writer, object? value)
+ {
+ switch (value)
+ {
+ case null:
+ writer.WriteNullValue();
+ break;
+ case byte[] bytes:
+ writer.WriteStartObject();
+ writer.WriteBase64String(InlineBytesMarker, bytes);
+ writer.WriteEndObject();
+ break;
+ case long integer:
+ writer.WriteNumberValue(integer);
+ break;
+ case double real:
+ writer.WriteNumberValue(real);
+ break;
+ case string text:
+ writer.WriteStringValue(text);
+ break;
+ default:
+ writer.WriteStringValue(Convert.ToString(value, System.Globalization.CultureInfo.InvariantCulture));
+ break;
+ }
+ }
+}