Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update retention logic to work for both manifests and buckets (#2159)
Leendert de Borstd85ab9107405
diff --git a/apps/server/AliasVault.Api/Vault/RetentionRules/DailyRetentionRule.cs b/apps/server/AliasVault.Api/Vault/RetentionRules/DailyRetentionRule.csindex fd3ae57a0..38908dc00 100644--- a/apps/server/AliasVault.Api/Vault/RetentionRules/DailyRetentionRule.cs+++ b/apps/server/AliasVault.Api/Vault/RetentionRules/DailyRetentionRule.cs@@ -10,20 +10,20 @@ namespace AliasVault.Api.Vault.RetentionRules;using AliasServerDb;/// <summary>-/// Daily retention rule that keeps the latest vault for each day.+/// Daily retention rule that keeps the latest revision for each day./// </summary>public class DailyRetentionRule : IRetentionRule{/// <summary>- /// Gets the amount of days to keep vault.+ /// Gets the amount of days to keep a revision for./// </summary>public int DaysToKeep { get; init; }/// <inheritdoc cref="IRetentionRule.ApplyRule"/>- public IEnumerable<VaultManifestBase> ApplyRule(List<VaultManifestBase> vaults, DateTime now)+ public IEnumerable<IVaultRevision> ApplyRule(List<IVaultRevision> revisions, DateTime now){- // For the specified amount of days, take last vault per day.- return vaults+ // For the specified amount of days, take the last revision per day.+ return revisions.GroupBy(x => x.UpdatedAt.Date).Select(g => g.OrderByDescending(x => x.UpdatedAt).First()).OrderByDescending(x => x.UpdatedAt)diff --git a/apps/server/AliasVault.Api/Vault/RetentionRules/DbVersionRetentionRule.cs b/apps/server/AliasVault.Api/Vault/RetentionRules/DbVersionRetentionRule.csindex 498d57386..354a6d3b8 100644--- a/apps/server/AliasVault.Api/Vault/RetentionRules/DbVersionRetentionRule.cs+++ b/apps/server/AliasVault.Api/Vault/RetentionRules/DbVersionRetentionRule.cs@@ -10,7 +10,8 @@ namespace AliasVault.Api.Vault.RetentionRules;using AliasServerDb;/// <summary>-/// Version retention rule that keeps the latest X unique db versions of the vault.+/// Version retention rule that keeps the latest X unique db versions of the vault. Only applies to manifest+/// revisions (other revision types carry no db version); it keeps nothing extra for other types./// </summary>public class DbVersionRetentionRule : IRetentionRule{@@ -20,10 +21,11 @@ public class DbVersionRetentionRule : IRetentionRulepublic int VersionsToKeep { get; init; }/// <inheritdoc cref="IRetentionRule.ApplyRule"/>- public IEnumerable<VaultManifestBase> ApplyRule(List<VaultManifestBase> vaults, DateTime now)+ public IEnumerable<IVaultRevision> ApplyRule(List<IVaultRevision> revisions, DateTime now){// For the specified amount of versions, take last vault per version.- return vaults+ return revisions+ .OfType<VaultManifestBase>().GroupBy(x => x.Version).Select(g => g.OrderByDescending(x => x.UpdatedAt).First()).OrderByDescending(x => x.UpdatedAt)diff --git a/apps/server/AliasVault.Api/Vault/RetentionRules/IRetentionRule.cs b/apps/server/AliasVault.Api/Vault/RetentionRules/IRetentionRule.csindex 2acfcc386..20888dbf7 100644--- a/apps/server/AliasVault.Api/Vault/RetentionRules/IRetentionRule.cs+++ b/apps/server/AliasVault.Api/Vault/RetentionRules/IRetentionRule.cs@@ -17,8 +17,8 @@ public interface IRetentionRule/// <summary>/// Apply retention rule./// </summary>- /// <param name="vaults">List of existing vaults to apply the retention rule to.</param>+ /// <param name="revisions">List of existing revisions to apply the retention rule to.</param>/// <param name="now">Current DateTime.</param>- /// <returns>Vaults that should be kept according to the retention rule.</returns>- IEnumerable<VaultManifestBase> ApplyRule(List<VaultManifestBase> vaults, DateTime now);+ /// <returns>Revisions that should be kept according to the retention rule.</returns>+ IEnumerable<IVaultRevision> ApplyRule(List<IVaultRevision> revisions, DateTime now);}diff --git a/apps/server/AliasVault.Api/Vault/RetentionRules/LoginCredentialRetentionRule.cs b/apps/server/AliasVault.Api/Vault/RetentionRules/LoginCredentialRetentionRule.csindex bb560b9d6..7b6e4b7b8 100644--- a/apps/server/AliasVault.Api/Vault/RetentionRules/LoginCredentialRetentionRule.cs+++ b/apps/server/AliasVault.Api/Vault/RetentionRules/LoginCredentialRetentionRule.cs@@ -10,7 +10,8 @@ namespace AliasVault.Api.Vault.RetentionRules;using AliasServerDb;/// <summary>-/// Credential retention rule that keeps the latest X unique login credentials and vaults associated with it.+/// Credential retention rule that keeps the latest X unique login credentials and vaults associated with it. Only+/// applies to manifest revisions (other revision types carry no credentials); it keeps nothing extra for other types./// </summary>public class LoginCredentialRetentionRule : IRetentionRule{@@ -20,10 +21,11 @@ public class LoginCredentialRetentionRule : IRetentionRulepublic int CredentialsToKeep { get; set; }/// <inheritdoc cref="IRetentionRule.ApplyRule"/>- public IEnumerable<VaultManifestBase> ApplyRule(List<VaultManifestBase> vaults, DateTime now)+ public IEnumerable<IVaultRevision> ApplyRule(List<IVaultRevision> revisions, DateTime now){// For the specified amount of versions, take last vault per version.- return vaults+ return revisions+ .OfType<VaultManifestBase>().GroupBy(x => new { x.Salt, x.Verifier }).Select(g => g.OrderByDescending(x => x.UpdatedAt).First()).OrderByDescending(x => x.UpdatedAt)diff --git a/apps/server/AliasVault.Api/Vault/RetentionRules/MonthlyRetentionRule.cs b/apps/server/AliasVault.Api/Vault/RetentionRules/MonthlyRetentionRule.csindex 6dd49dc2c..edc5ae6af 100644--- a/apps/server/AliasVault.Api/Vault/RetentionRules/MonthlyRetentionRule.cs+++ b/apps/server/AliasVault.Api/Vault/RetentionRules/MonthlyRetentionRule.cs@@ -10,19 +10,19 @@ namespace AliasVault.Api.Vault.RetentionRules;using AliasServerDb;/// <summary>-/// Monthly retention rule that keeps the latest vault for each month.+/// Monthly retention rule that keeps the latest revision for each month./// </summary>public class MonthlyRetentionRule : IRetentionRule{/// <summary>- /// Gets the amount of months to keep vault.+ /// Gets the amount of months to keep a revision for./// </summary>public int MonthsToKeep { get; init; }/// <inheritdoc cref="IRetentionRule.ApplyRule"/>- public IEnumerable<VaultManifestBase> ApplyRule(List<VaultManifestBase> vaults, DateTime now)+ public IEnumerable<IVaultRevision> ApplyRule(List<IVaultRevision> revisions, DateTime now){- return vaults+ return revisions.GroupBy(x => x.UpdatedAt.Month).Select(g => g.OrderByDescending(x => x.UpdatedAt).First()).OrderByDescending(x => x.UpdatedAt)diff --git a/apps/server/AliasVault.Api/Vault/RetentionRules/RevisionRetentionRule.cs b/apps/server/AliasVault.Api/Vault/RetentionRules/RevisionRetentionRule.csindex 16cc56231..13fff47c5 100644--- a/apps/server/AliasVault.Api/Vault/RetentionRules/RevisionRetentionRule.cs+++ b/apps/server/AliasVault.Api/Vault/RetentionRules/RevisionRetentionRule.cs@@ -10,20 +10,20 @@ namespace AliasVault.Api.Vault.RetentionRules;using AliasServerDb;/// <summary>-/// Revision retention rule that keeps the latest X unique revisions of the vault.+/// Revision retention rule that keeps the latest X unique revisions./// </summary>public class RevisionRetentionRule : IRetentionRule{/// <summary>- /// Gets the amount of revisions to keep the vault.+ /// Gets the amount of revisions to keep./// </summary>public int RevisionsToKeep { get; init; }/// <inheritdoc cref="IRetentionRule.ApplyRule"/>- public IEnumerable<VaultManifestBase> ApplyRule(List<VaultManifestBase> vaults, DateTime now)+ public IEnumerable<IVaultRevision> ApplyRule(List<IVaultRevision> revisions, DateTime now){- // For the specified amount of versions, take last vault per version.- return vaults+ // For the specified amount of revisions, take the last one per revision number.+ return revisions.GroupBy(x => x.RevisionNumber).Select(g => g.OrderByDescending(x => x.UpdatedAt).First()).OrderByDescending(x => x.UpdatedAt)diff --git a/apps/server/AliasVault.Api/Vault/RetentionRules/WeeklyRetentionRule.cs b/apps/server/AliasVault.Api/Vault/RetentionRules/WeeklyRetentionRule.csindex 25c953e1e..c8266f088 100644--- a/apps/server/AliasVault.Api/Vault/RetentionRules/WeeklyRetentionRule.cs+++ b/apps/server/AliasVault.Api/Vault/RetentionRules/WeeklyRetentionRule.cs@@ -10,17 +10,17 @@ namespace AliasVault.Api.Vault.RetentionRules;using AliasServerDb;/// <summary>-/// Weekly retention rule that keeps the latest vault for each week.+/// Weekly retention rule that keeps the latest revision for each week./// </summary>public class WeeklyRetentionRule : IRetentionRule{/// <summary>- /// Gets the amount of weeks to keep vault.+ /// Gets the amount of weeks to keep a revision for./// </summary>public int WeeksToKeep { get; init; }/// <inheritdoc cref="IRetentionRule.ApplyRule"/>- public IEnumerable<VaultManifestBase> ApplyRule(List<VaultManifestBase> vaults, DateTime now)+ public IEnumerable<IVaultRevision> ApplyRule(List<IVaultRevision> revisions, DateTime now){// Helper function to get the start of the week with Monday as the first day of the week.DateTime GetStartOfWeek(DateTime date)@@ -29,7 +29,7 @@ public class WeeklyRetentionRule : IRetentionRulereturn date.Date.AddDays(-1 * diff).Date;}- return vaults+ return revisions.GroupBy(x => GetStartOfWeek(x.UpdatedAt)).Select(g => g.OrderByDescending(x => x.UpdatedAt).First()).OrderByDescending(x => x.UpdatedAt)diff --git a/apps/server/AliasVault.Api/Vault/VaultRetentionManager.cs b/apps/server/AliasVault.Api/Vault/VaultRetentionManager.csindex 06a5d5f78..848c9d1ab 100644--- a/apps/server/AliasVault.Api/Vault/VaultRetentionManager.cs+++ b/apps/server/AliasVault.Api/Vault/VaultRetentionManager.cs@@ -13,24 +13,26 @@ using System.Linq;using AliasServerDb;/// <summary>-/// History manager for vault manifests that applies retention rules to determine how many superseded revisions to-/// keep as backups and returns the history revisions that should be deleted.+/// History manager for revisioned vault storage entities (manifests, data buckets) that applies retention rules to+/// determine how many superseded revisions to keep as backups and returns the history revisions that should be deleted./// </summary>public static class VaultRetentionManager{/// <summary>- /// Applies retention policies to the superseded (history) revisions of a manifest. The current revision is- /// passed in so the rules see the full revision timeline, but it is never eligible for deletion — only history- /// revisions are returned.+ /// Applies retention policies to the superseded (history) revisions of a revisioned entity. The current revision+ /// can be passed in so the rules see the full revision timeline, but it is never eligible for deletion — only+ /// history revisions are returned./// </summary>+ /// <typeparam name="THistory">The history row type of the entity (e.g. <see cref="VaultManifestsHistory"/> or <see cref="VaultDataBucketsHistory"/>).</typeparam>/// <param name="retentionPolicy">List of retention policies to apply.</param>- /// <param name="historyRevisions">Superseded revisions of the manifest (including the one just archived).</param>+ /// <param name="historyRevisions">Superseded revisions of the entity (including the one just archived).</param>/// <param name="now">DateTime which represents current time.</param>- /// <param name="currentRevision">The current revision of the manifest, taken into account by the rules but always kept.</param>+ /// <param name="currentRevision">The current revision of the entity, taken into account by the rules but always kept.</param>/// <returns>List of history revisions to delete according to the retention policies.</returns>- public static List<VaultManifestsHistory> ApplyRetention(RetentionPolicy retentionPolicy, List<VaultManifestsHistory> historyRevisions, DateTime now, VaultManifest? currentRevision = null)+ public static List<THistory> ApplyRetention<THistory>(RetentionPolicy retentionPolicy, List<THistory> historyRevisions, DateTime now, IVaultRevision? currentRevision = null)+ where THistory : class, IVaultRevision{- var allRevisions = new List<VaultManifestBase>(historyRevisions);+ var allRevisions = new List<IVaultRevision>(historyRevisions);if (currentRevision is not null){allRevisions.Add(currentRevision);@@ -39,7 +41,7 @@ public static class VaultRetentionManager// Sort revisions by UpdatedAt in descending order.allRevisions = allRevisions.OrderByDescending(v => v.UpdatedAt).ToList();- var revisionsToKeep = new HashSet<VaultManifestBase>();+ var revisionsToKeep = new HashSet<IVaultRevision>();// Process retention rules.foreach (var rule in retentionPolicy.Rules)@@ -57,6 +59,6 @@ public static class VaultRetentionManager}// Only history revisions are deletable; the current revision is always kept implicitly.- return allRevisions.Except(revisionsToKeep).OfType<VaultManifestsHistory>().ToList();+ return allRevisions.Except(revisionsToKeep).OfType<THistory>().ToList();}}diff --git a/apps/server/Databases/AliasServerDb/IVaultRevision.cs b/apps/server/Databases/AliasServerDb/IVaultRevision.csnew file mode 100644index 000000000..e747da8b4--- /dev/null+++ b/apps/server/Databases/AliasServerDb/IVaultRevision.cs@@ -0,0 +1,24 @@+//-----------------------------------------------------------------------+// <copyright file="IVaultRevision.cs" company="aliasvault">+// Copyright (c) aliasvault. All rights reserved.+// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.+// </copyright>+//-----------------------------------------------------------------------+namespace AliasServerDb;++/// <summary>+/// A single revision of a versioned vault storage entity (manifest or data bucket). Exposes the revision metadata+/// that retention rules operate on, so one retention implementation can prune the history of any revisioned table.+/// </summary>+public interface IVaultRevision+{+ /// <summary>+ /// Gets the revision number, incremented on every write.+ /// </summary>+ long RevisionNumber { get; }++ /// <summary>+ /// Gets the timestamp at which this revision was last updated.+ /// </summary>+ DateTime UpdatedAt { get; }+}diff --git a/apps/server/Databases/AliasServerDb/VaultDataBucketBase.cs b/apps/server/Databases/AliasServerDb/VaultDataBucketBase.csindex cee3cccf9..bcfd0f23e 100644--- a/apps/server/Databases/AliasServerDb/VaultDataBucketBase.cs+++ b/apps/server/Databases/AliasServerDb/VaultDataBucketBase.cs@@ -13,7 +13,7 @@ using System.ComponentModel.DataAnnotations;/// <see cref="VaultDataBucket"/> holds the current revision of each (owner, category) bucket./// <see cref="VaultDataBucketsHistory"/> holds superseded revisions./// </summary>-public abstract class VaultDataBucketBase+public abstract class VaultDataBucketBase : IVaultRevision{/// <summary>/// Gets or sets the encrypted bucket payload (AES-GCM ciphertext, base64-encoded).diff --git a/apps/server/Databases/AliasServerDb/VaultManifestBase.cs b/apps/server/Databases/AliasServerDb/VaultManifestBase.csindex c8cfd5b35..e7e92ae89 100644--- a/apps/server/Databases/AliasServerDb/VaultManifestBase.cs+++ b/apps/server/Databases/AliasServerDb/VaultManifestBase.cs@@ -12,7 +12,7 @@ using System.ComponentModel.DataAnnotations;/// Shared revision payload columns for a vault manifest. <see cref="VaultManifest"/> holds the current revision of/// each logical manifest (one row per manifest); <see cref="VaultManifestsHistory"/> holds superseded revisions./// </summary>-public abstract class VaultManifestBase+public abstract class VaultManifestBase : IVaultRevision{/// <summary>/// Gets or sets the encrypted vault blob (only used by legacy sqlite-blob format). Not used anymore in new format.diff --git a/apps/server/Tests/AliasVault.UnitTests/Vault/RetentionManager/BucketRetentionTests.cs b/apps/server/Tests/AliasVault.UnitTests/Vault/RetentionManager/BucketRetentionTests.csnew file mode 100644index 000000000..294a8d1a7--- /dev/null+++ b/apps/server/Tests/AliasVault.UnitTests/Vault/RetentionManager/BucketRetentionTests.cs@@ -0,0 +1,163 @@+//-----------------------------------------------------------------------+// <copyright file="BucketRetentionTests.cs" company="aliasvault">+// Copyright (c) aliasvault. All rights reserved.+// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.+// </copyright>+//-----------------------------------------------------------------------++namespace AliasVault.UnitTests.Vault.RetentionManager;++using AliasServerDb;+using AliasVault.Api.Vault;+using AliasVault.Api.Vault.RetentionRules;+using AliasVault.Shared.Models.WebApi.V2.Vault;++/// <summary>+/// Tests for applying the retention rules to vault data bucket history revisions via the same VaultRetentionManager+/// that the manifest write path uses.+/// </summary>+public class BucketRetentionTests+{+ private List<VaultDataBucketsHistory> testRevisions;+ private DateTime now;++ /// <summary>+ /// Common setup for all tests.+ /// </summary>+ [SetUp]+ public void Setup()+ {+ now = new DateTime(2023, 6, 1, 12, 0, 0);+ testRevisions =+ [+ CreateRevision(9, new DateTime(2023, 6, 1, 11, 0, 0)),+ CreateRevision(8, new DateTime(2023, 6, 1, 10, 0, 0)),+ CreateRevision(7, new DateTime(2023, 6, 1, 9, 0, 0)),+ CreateRevision(6, new DateTime(2023, 6, 1, 8, 0, 0)),+ CreateRevision(5, new DateTime(2023, 5, 31, 12, 0, 0)),+ CreateRevision(4, new DateTime(2023, 5, 31, 8, 0, 0)),+ CreateRevision(3, new DateTime(2023, 5, 28, 12, 0, 0)),+ CreateRevision(2, new DateTime(2023, 5, 20, 12, 0, 0)),+ CreateRevision(1, new DateTime(2023, 4, 1, 12, 0, 0)),+ ];+ }++ /// <summary>+ /// Test the bucket retention policy (as used by the V2 VaultController bucket write path) against a history+ /// containing multiple same-day writes.+ /// </summary>+ [Test]+ public void BucketRetentionPolicyTest()+ {+ var retentionPolicy = new RetentionPolicy+ {+ Rules =+ [+ new RevisionRetentionRule { RevisionsToKeep = 3 },+ new DailyRetentionRule { DaysToKeep = 7 },+ ],+ };++ var revisionsToDelete = VaultRetentionManager.ApplyRetention(retentionPolicy, testRevisions, now);+ var revisionsToKeep = new List<VaultDataBucketsHistory>(testRevisions);+ revisionsToKeep.RemoveAll(revisionsToDelete.Contains);++ // Expecting to keep:+ // - the 3 newest revisions (9, 8, 7)+ // - the last revision of each of the 7 most recent days that had writes: 9 (Jun 1), 5 (May 31), 3 (May 28),+ // 2 (May 20), 1 (Apr 1) - only 5 such days exist.+ // Deleted: 6 (superseded same-day write) and 4 (not the last write of May 31).+ Assert.Multiple(() =>+ {+ Assert.That(revisionsToKeep.Select(x => x.RevisionNumber), Is.EquivalentTo(new long[] { 9, 8, 7, 5, 3, 2, 1 }));+ Assert.That(revisionsToDelete.Select(x => x.RevisionNumber), Is.EquivalentTo(new long[] { 6, 4 }));+ });+ }++ /// <summary>+ /// Test that the generic time/revision based rules operate on bucket revisions.+ /// </summary>+ [Test]+ public void GenericRulesApplyToBucketRevisionsTest()+ {+ var revisionRule = new RevisionRetentionRule { RevisionsToKeep = 2 };+ var dailyRule = new DailyRetentionRule { DaysToKeep = 2 };++ Assert.Multiple(() =>+ {+ Assert.That(revisionRule.ApplyRule([.. testRevisions], now).Select(x => x.RevisionNumber), Is.EquivalentTo(new long[] { 9, 8 }));+ Assert.That(dailyRule.ApplyRule([.. testRevisions], now).Select(x => x.RevisionNumber), Is.EquivalentTo(new long[] { 9, 5 }));+ });+ }++ /// <summary>+ /// Test that manifest-specific rules keep nothing extra for bucket revisions, as buckets carry no db version or+ /// login credentials.+ /// </summary>+ [Test]+ public void ManifestOnlyRulesIgnoreBucketRevisionsTest()+ {+ var versionRule = new DbVersionRetentionRule { VersionsToKeep = 5 };+ var credentialRule = new LoginCredentialRetentionRule { CredentialsToKeep = 5 };++ Assert.Multiple(() =>+ {+ Assert.That(versionRule.ApplyRule([.. testRevisions], now), Is.Empty);+ Assert.That(credentialRule.ApplyRule([.. testRevisions], now), Is.Empty);+ });+ }++ /// <summary>+ /// Test that without any rules the newest bucket history revision is still always kept.+ /// </summary>+ [Test]+ public void NoRulesKeepsNewestBucketRevisionTest()+ {+ var revisionsToDelete = VaultRetentionManager.ApplyRetention(new RetentionPolicy(), testRevisions, now);++ Assert.Multiple(() =>+ {+ Assert.That(revisionsToDelete, Has.Count.EqualTo(testRevisions.Count - 1));+ Assert.That(revisionsToDelete.Select(x => x.RevisionNumber), Does.Not.Contain(9L));+ });+ }++ /// <summary>+ /// Test that a current bucket row passed as the current revision is taken into account by the rules but never+ /// returned as a deletion candidate.+ /// </summary>+ [Test]+ public void CurrentBucketRevisionNeverDeletedTest()+ {+ var retentionPolicy = new RetentionPolicy+ {+ Rules = [new RevisionRetentionRule { RevisionsToKeep = 2 }],+ };++ var currentRevision = new VaultDataBucket+ {+ OwnerUserId = "user-1",+ Category = VaultDataBucketCategory.Settings,+ EncryptedData = string.Empty,+ RevisionNumber = 10,+ UpdatedAt = new DateTime(2023, 6, 1, 12, 0, 0),+ };++ var revisionsToDelete = VaultRetentionManager.ApplyRetention(retentionPolicy, testRevisions, now, currentRevision);++ // The current revision occupies one of the 2 slots, so only history revision 9 survives alongside it.+ Assert.That(revisionsToDelete.Select(x => x.RevisionNumber), Is.EquivalentTo(new long[] { 8, 7, 6, 5, 4, 3, 2, 1 }));+ }++ private static VaultDataBucketsHistory CreateRevision(long revisionNumber, DateTime updatedAt)+ {+ return new VaultDataBucketsHistory+ {+ OwnerUserId = "user-1",+ Category = VaultDataBucketCategory.Settings,+ EncryptedData = string.Empty,+ RevisionNumber = revisionNumber,+ UpdatedAt = updatedAt,+ };+ }+}