Open-source eCommerce platform for C# and ASP.NET Core 10 with MongoDB. Build multi-store shops and multi-vendor marketplaces with B2B/B2C, REST API, plugins, Vue 3 and Docker.
C# aspnet-corecsharpdockerdotnetecommerceecommerce-platformmongodbmulti-storemulti-vendornosqlonline-shopshoppingcart
Multi-instance safety: Redis cache invalidation resilience & exactly-once scheduled tasks (#731)
Krzysztof Pajak506ab7c902df
diff --git a/Directory.Packages.props b/Directory.Packages.propsindex 8b8e6d5e6..a9f87a784 100644--- a/Directory.Packages.props+++ b/Directory.Packages.props@@ -65,6 +65,7 @@<PackageVersion Include="Verify.MSTest" Version="31.19.1" /><PackageVersion Include="Aspire.Hosting.AppHost" Version="13.4.3" /><PackageVersion Include="Aspire.Hosting.MongoDB" Version="13.4.3" />+ <PackageVersion Include="Aspire.Hosting.Redis" Version="13.4.3" /><PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.7.0" /><PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="10.7.0" /><PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.16.0" />diff --git a/src/Aspire/Aspire.AppHost/Aspire.AppHost.csproj b/src/Aspire/Aspire.AppHost/Aspire.AppHost.csprojindex 550a7d6b8..aef766843 100644--- a/src/Aspire/Aspire.AppHost/Aspire.AppHost.csproj+++ b/src/Aspire/Aspire.AppHost/Aspire.AppHost.csproj@@ -16,6 +16,7 @@<ItemGroup><PackageReference Include="Aspire.Hosting.AppHost" /><PackageReference Include="Aspire.Hosting.MongoDB" />+ <PackageReference Include="Aspire.Hosting.Redis" /></ItemGroup><ItemGroup><PackageReference Include="Aspire.Dashboard.Sdk.$(NETCoreSdkRuntimeIdentifier)" />diff --git a/src/Aspire/Aspire.AppHost/Program.cs b/src/Aspire/Aspire.AppHost/Program.csindex a575b90d6..efb79fa91 100644--- a/src/Aspire/Aspire.AppHost/Program.cs+++ b/src/Aspire/Aspire.AppHost/Program.cs@@ -4,6 +4,9 @@ var builder = DistributedApplication.CreateBuilder(args);var mongo = builder.AddMongoDB("mongo").WithLifetime(ContainerLifetime.Persistent);var mongodb = mongo.AddDatabase("Mongodb");-builder.ConfigureGrandWebProject(mongodb);++var redis = builder.AddRedis("redis").WithLifetime(ContainerLifetime.Persistent);++builder.ConfigureGrandWebProject(mongodb, redis);await builder.Build().RunAsync();diff --git a/src/Aspire/Aspire.AppHost/ProjectConfiguration.cs b/src/Aspire/Aspire.AppHost/ProjectConfiguration.csindex 5f8053133..90bb42205 100644--- a/src/Aspire/Aspire.AppHost/ProjectConfiguration.cs+++ b/src/Aspire/Aspire.AppHost/ProjectConfiguration.cs@@ -1,13 +1,30 @@-namespace Aspire.AppHost;+namespace Aspire.AppHost;public static class ProjectConfiguration{- public static void ConfigureGrandWebProject(this IDistributedApplicationBuilder builder, IResourceBuilder<MongoDBDatabaseResource> mongodb)+ public static void ConfigureGrandWebProject(this IDistributedApplicationBuilder builder,+ IResourceBuilder<MongoDBDatabaseResource> mongodb,+ IResourceBuilder<RedisResource> redis){+ //run two replicas of the same application (multi-pod simulation) sharing one database+ //and synchronizing their memory cache through the Redis pub/sub message bus.+ //replicas (not two separate AddProject resources) so Aspire builds the project once and+ //launches N processes - two independent builds would race over the same bin/ output and+ //crash a process with host error 0x80008081. Aspire assigns each replica its own port,+ //so no fixed WithHttpEndpoint here.builder.AddProject<Projects.Grand_Web>("grand-web")- .WithHttpEndpoint(80, name: "front")+ .WithReplicas(2).WithReference(mongodb)- .WaitFor(mongodb);+ .WaitFor(mongodb)+ .WaitFor(redis)+ //replicas share one content root - shadow copy would make them fight over the same+ //Plugins/bin folder (files locked by the other replica)+ .WithEnvironment("Extensions__PluginShadowCopy", "false")+ .WithEnvironment("Redis__RedisPubSubEnabled", "true")+ //show publish/receive debug logs of the cache message bus in the dashboard+ .WithEnvironment("Logging__LogLevel__Grand.Infrastructure.Caching.Redis", "Debug")+ .WithEnvironment("Redis__RedisPubSubChannel", "grandnode-cache")+ .WithEnvironment("Redis__RedisPubSubConnectionString", redis.Resource.ConnectionStringExpression);}-}\ No newline at end of file+}diff --git a/src/Business/Grand.Business.Core/Interfaces/System/ScheduleTasks/IScheduleTaskService.cs b/src/Business/Grand.Business.Core/Interfaces/System/ScheduleTasks/IScheduleTaskService.csindex 89f7d02d9..8c78591b9 100644--- a/src/Business/Grand.Business.Core/Interfaces/System/ScheduleTasks/IScheduleTaskService.cs+++ b/src/Business/Grand.Business.Core/Interfaces/System/ScheduleTasks/IScheduleTaskService.cs@@ -36,6 +36,18 @@ public interface IScheduleTaskService/// <param name="task">Task</param>Task UpdateTask(ScheduleTask task);+ /// <summary>+ /// Atomically claims a single run of the task, so that only one application instance+ /// executes it. The claim succeeds only if LastStartUtc still has the value the caller+ /// read (compare-and-set) - a concurrent claim by another instance makes it fail.+ /// </summary>+ /// <param name="taskId">Task identifier</param>+ /// <param name="expectedLastStartUtc">LastStartUtc value read before the claim</param>+ /// <param name="runStartUtc">New LastStartUtc value (start of the claimed run)</param>+ /// <param name="instanceId">Unique identifier of the claiming application instance</param>+ /// <returns>true if this instance won the claim and should execute the task</returns>+ Task<bool> TryClaimTaskRun(string taskId, DateTime? expectedLastStartUtc, DateTime runStartUtc, string instanceId);+/// <summary>/// Delete the task/// </summary>diff --git a/src/Core/Grand.Domain/Tasks/ScheduleTask.cs b/src/Core/Grand.Domain/Tasks/ScheduleTask.csindex d19979140..6d773f6f6 100644--- a/src/Core/Grand.Domain/Tasks/ScheduleTask.cs+++ b/src/Core/Grand.Domain/Tasks/ScheduleTask.cs@@ -10,6 +10,7 @@ public class ScheduleTask : BaseEntitypublic DateTime? LastSuccessUtc { get; set; }public string LeasedByMachineName { get; set; }public DateTime? LeasedUntilUtc { get; set; }+ public string LeasedByInstance { get; set; }public int TimeInterval { get; set; }public string StoreId { get; set; }}\ No newline at end of filediff --git a/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageBus.cs b/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageBus.csindex 7117afb2c..4e7a613c9 100644--- a/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageBus.cs+++ b/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageBus.cs@@ -1,66 +1,85 @@-using Grand.Infrastructure.Caching.Message;+using Grand.Infrastructure.Caching.Message;using Grand.Infrastructure.Configuration;using Microsoft.Extensions.DependencyInjection;+using Microsoft.Extensions.Hosting;+using Microsoft.Extensions.Logging;using StackExchange.Redis;-using System.Diagnostics;using System.Text.Json;namespace Grand.Infrastructure.Caching.Redis;-public sealed class RedisMessageBus : IMessageBus+public sealed class RedisMessageBus : IMessageBus, IHostedService, IDisposable{private static readonly string ClientId = Guid.NewGuid().ToString("N");++ private static readonly TimeSpan InitialRetryDelay = TimeSpan.FromSeconds(1);+ private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromSeconds(30);++ private readonly IConnectionMultiplexer _connection;private readonly RedisConfig _redisConfig;private readonly IServiceProvider _serviceProvider;private readonly ISubscriber _subscriber;+ private readonly ILogger<RedisMessageBus> _logger;+ private readonly CancellationTokenSource _cts = new();+ private Task _subscribeLoop;- public RedisMessageBus(ISubscriber subscriber, IServiceProvider serviceProvider, RedisConfig redisConfig)+ public RedisMessageBus(IConnectionMultiplexer connection, IServiceProvider serviceProvider,+ RedisConfig redisConfig, ILogger<RedisMessageBus> logger){- _subscriber = subscriber;+ _connection = connection;+ _subscriber = connection.GetSubscriber();_serviceProvider = serviceProvider;_redisConfig = redisConfig;- SubscribeAsync();+ _logger = logger;+ }++ public Task StartAsync(CancellationToken cancellationToken)+ {+ _connection.ConnectionFailed += OnConnectionFailed;+ _connection.ConnectionRestored += OnConnectionRestored;++ //keep the subscription attempt off the startup path since Redis may be unreachable,+ //and track the loop so it can be cancelled and awaited on shutdown+ _subscribeLoop = SubscribeWithRetryAsync(_cts.Token);+ return Task.CompletedTask;+ }++ public async Task StopAsync(CancellationToken cancellationToken)+ {+ _connection.ConnectionFailed -= OnConnectionFailed;+ _connection.ConnectionRestored -= OnConnectionRestored;+ await _cts.CancelAsync();+ if (_subscribeLoop != null)+ await _subscribeLoop.WaitAsync(cancellationToken);}public async Task PublishAsync<TMessage>(TMessage msg) where TMessage : IMessageEvent{+ var message = JsonSerializer.Serialize(new MessageEventClient {+ ClientId = ClientId,+ Key = msg.Key,+ MessageType = msg.MessageType+ });try{- var client = new MessageEventClient {- ClientId = ClientId,- Key = msg.Key,- MessageType = msg.MessageType- };- var message = JsonSerializer.Serialize(client);- await _subscriber.PublishAsync(RedisChannel.Literal(_redisConfig.RedisPubSubChannel), message);+ var receivers =+ await _subscriber.PublishAsync(RedisChannel.Literal(_redisConfig.RedisPubSubChannel), message);+ if (_logger.IsEnabled(LogLevel.Debug))+ _logger.LogDebug(+ "Published cache invalidation message (type: {MessageType}, key: {Key}), delivered to {Receivers} subscriber(s)",+ (MessageEventType)msg.MessageType, msg.Key, receivers);}catch (Exception ex){- Debug.WriteLine(ex.Message);+ _logger.LogError(ex,+ "Failed to publish cache invalidation message (type: {MessageType}, key: {Key}) - other instances may serve stale data until cache expiration",+ (MessageEventType)msg.MessageType, msg.Key);}}public Task SubscribeAsync(){- _ = _subscriber.SubscribeAsync(RedisChannel.Literal(_redisConfig.RedisPubSubChannel), (_, redisValue) =>- {- try- {- MessageEventClient message = null;- if (!redisValue.IsNull)- {- // Use the string overload explicitly to resolve ambiguity- message = JsonSerializer.Deserialize<MessageEventClient>(redisValue.ToString());- }- if (message != null && message.ClientId != ClientId)- OnSubscriptionChanged(message);- }- catch (Exception ex)- {- Debug.WriteLine(ex.Message);- }- });- return Task.CompletedTask;+ return _subscriber.SubscribeAsync(RedisChannel.Literal(_redisConfig.RedisPubSubChannel), OnMessage);}public void OnSubscriptionChanged(IMessageEvent message)@@ -80,4 +99,93 @@ public sealed class RedisMessageBus : IMessageBusbreak;}}-}\ No newline at end of file++ public void Dispose()+ {+ //event handlers are normally detached in StopAsync; detach again in case the+ //hosted-service lifecycle was skipped, and release the cancellation source+ _connection.ConnectionFailed -= OnConnectionFailed;+ _connection.ConnectionRestored -= OnConnectionRestored;+ if (!_cts.IsCancellationRequested)+ _cts.Cancel();+ _cts.Dispose();+ }++ private async Task SubscribeWithRetryAsync(CancellationToken cancellationToken)+ {+ var delay = InitialRetryDelay;+ while (!cancellationToken.IsCancellationRequested)+ {+ try+ {+ await SubscribeAsync();+ if (_logger.IsEnabled(LogLevel.Information))+ _logger.LogInformation("Subscribed to Redis pub/sub channel {Channel}",+ _redisConfig.RedisPubSubChannel);+ return;+ }+ catch (Exception ex)+ {+ _logger.LogError(ex,+ "Failed to subscribe to Redis pub/sub channel {Channel}, retrying in {Delay}s",+ _redisConfig.RedisPubSubChannel, delay.TotalSeconds);+ try+ {+ await Task.Delay(delay, cancellationToken);+ }+ catch (OperationCanceledException)+ {+ return;+ }+ delay = delay * 2 > MaxRetryDelay ? MaxRetryDelay : delay * 2;+ }+ }+ }++ private void OnMessage(RedisChannel channel, RedisValue redisValue)+ {+ try+ {+ if (redisValue.IsNull) return;+ var message = JsonSerializer.Deserialize<MessageEventClient>(redisValue.ToString());+ if (message != null && message.ClientId != ClientId)+ {+ if (_logger.IsEnabled(LogLevel.Debug))+ _logger.LogDebug(+ "Received cache invalidation message (type: {MessageType}, key: {Key}) from client {ClientId}",+ (MessageEventType)message.MessageType, message.Key, message.ClientId);+ OnSubscriptionChanged(message);+ }+ }+ catch (Exception ex)+ {+ _logger.LogError(ex, "Failed to process cache invalidation message from Redis");+ }+ }++ private void OnConnectionFailed(object sender, ConnectionFailedEventArgs e)+ {+ _logger.LogWarning(e.Exception,+ "Redis connection failed ({ConnectionType}, {FailureType}) - cache invalidation messages may be lost while disconnected",+ e.ConnectionType, e.FailureType);+ }++ private void OnConnectionRestored(object sender, ConnectionFailedEventArgs e)+ {+ //invalidation messages published while this instance was disconnected are lost+ //(Redis pub/sub has no replay), so the local cache can no longer be trusted+ if (e.ConnectionType != ConnectionType.Subscription) return;+ _logger.LogWarning(+ "Redis subscription connection restored - clearing local cache to drop entries that may have missed invalidation");+ try+ {+ using var scope = _serviceProvider.CreateScope();+ var cache = scope.ServiceProvider.GetRequiredService<ICacheBase>();+ _ = cache.Clear(false);+ }+ catch (Exception ex)+ {+ _logger.LogError(ex, "Failed to clear local cache after Redis connection was restored");+ }+ }+}diff --git a/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageCacheManager.cs b/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageCacheManager.csindex 6fc492267..954162361 100644--- a/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageCacheManager.cs+++ b/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageCacheManager.cs@@ -1,4 +1,4 @@-using Grand.Infrastructure.Caching.Message;+using Grand.Infrastructure.Caching.Message;using Grand.Infrastructure.Configuration;using MediatR;using Microsoft.Extensions.Caching.Memory;@@ -7,13 +7,11 @@ namespace Grand.Infrastructure.Caching.Redis;public class RedisMessageCacheManager : MemoryCacheBase, ICacheBase{- private readonly IMemoryCache _cache;private readonly IMessageBus _messageBus;public RedisMessageCacheManager(IMemoryCache cache, IMediator mediator, IMessageBus messageBus, CacheConfig config): base(cache, mediator, config){- _cache = cache;_messageBus = messageBus;}@@ -22,14 +20,13 @@ public class RedisMessageCacheManager : MemoryCacheBase, ICacheBase/// </summary>/// <param name="key">Key of cached item</param>/// <param name="publisher">Publisher</param>- public override Task RemoveAsync(string key, bool publisher = true)+ public override async Task RemoveAsync(string key, bool publisher = true){- _cache.Remove(key);+ await base.RemoveAsync(key, false);if (publisher)- _messageBus.PublishAsync(new MessageEvent { Key = key, MessageType = (int)MessageEventType.RemoveKey });-- return Task.CompletedTask;+ await _messageBus.PublishAsync(new MessageEvent+ { Key = key, MessageType = (int)MessageEventType.RemoveKey });}/// <summary>@@ -37,16 +34,13 @@ public class RedisMessageCacheManager : MemoryCacheBase, ICacheBase/// </summary>/// <param name="prefix">String prefix</param>/// <param name="publisher">publisher</param>- public override Task RemoveByPrefix(string prefix, bool publisher = true)+ public override async Task RemoveByPrefix(string prefix, bool publisher = true){- var entriesToRemove = CacheEntries.Where(x => x.Key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase));- foreach (var cacheEntries in entriesToRemove) _cache.Remove(cacheEntries.Key);+ await base.RemoveByPrefix(prefix, false);if (publisher)- _messageBus.PublishAsync(new MessageEvent+ await _messageBus.PublishAsync(new MessageEvent{ Key = prefix, MessageType = (int)MessageEventType.RemoveByPrefix });-- return Task.CompletedTask;}/// <summary>@@ -56,8 +50,9 @@ public class RedisMessageCacheManager : MemoryCacheBase, ICacheBasepublic override async Task Clear(bool publisher = true){await base.Clear(publisher);- if (publisher)- await _messageBus.PublishAsync(new MessageEvent { Key = "", MessageType = (int)MessageEventType.ClearCache });+ if (publisher)+ await _messageBus.PublishAsync(new MessageEvent+ { Key = "", MessageType = (int)MessageEventType.ClearCache });}-}\ No newline at end of file+}diff --git a/src/Modules/Grand.Module.ScheduledTasks/BackgroundServices/ScheduleTaskService.cs b/src/Modules/Grand.Module.ScheduledTasks/BackgroundServices/ScheduleTaskService.csindex c17036276..bc4670bb4 100644--- a/src/Modules/Grand.Module.ScheduledTasks/BackgroundServices/ScheduleTaskService.cs+++ b/src/Modules/Grand.Module.ScheduledTasks/BackgroundServices/ScheduleTaskService.cs@@ -78,6 +78,29 @@ public class ScheduleTaskService : IScheduleTaskServiceawait _taskRepository.UpdateAsync(task);}+ /// <summary>+ /// Atomically claims a single run of the task (compare-and-set on LastStartUtc)+ /// </summary>+ public virtual async Task<bool> TryClaimTaskRun(string taskId, DateTime? expectedLastStartUtc,+ DateTime runStartUtc, string instanceId)+ {+ //truncate to milliseconds so the value survives the database round-trip unchanged+ runStartUtc = new DateTime(runStartUtc.Ticks - runStartUtc.Ticks % TimeSpan.TicksPerMillisecond,+ DateTimeKind.Utc);++ //conditional update - matches only when no other instance has claimed the run+ //in the meantime; on MongoDB a single update is atomic, so exactly one instance wins+ await _taskRepository.UpdateOneAsync(+ x => x.Id == taskId && x.LastStartUtc == expectedLastStartUtc,+ UpdateBuilder<ScheduleTask>.Create()+ .Set(x => x.LastStartUtc, runStartUtc)+ .Set(x => x.LeasedByInstance, instanceId));++ //UpdateOneAsync does not report whether the filter matched - read back the winner+ var task = await GetTaskById(taskId);+ return task != null && task.LeasedByInstance == instanceId && task.LastStartUtc == runStartUtc;+ }+/// <summary>/// Delete the task/// </summary>diff --git a/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageBusTests.cs b/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageBusTests.csnew file mode 100644index 000000000..656c4fcbb--- /dev/null+++ b/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageBusTests.cs@@ -0,0 +1,169 @@+using Grand.Infrastructure.Caching;+using Grand.Infrastructure.Caching.Message;+using Grand.Infrastructure.Caching.Redis;+using Grand.Infrastructure.Configuration;+using Microsoft.Extensions.DependencyInjection;+using Microsoft.Extensions.Logging;+using Microsoft.VisualStudio.TestTools.UnitTesting;+using Moq;+using StackExchange.Redis;+using System.Net;+using System.Text.Json;++namespace Grand.Infrastructure.Tests.Caching.Redis;++[TestClass]+public class RedisMessageBusTests+{+ private Mock<IConnectionMultiplexer> _connectionMock;+ private Mock<ISubscriber> _subscriberMock;+ private Mock<ICacheBase> _cacheMock;+ private RedisConfig _config;+ private RedisMessageBus _bus;++ [TestInitialize]+ public void Init()+ {+ _subscriberMock = new Mock<ISubscriber>();+ _subscriberMock.Setup(s => s.SubscribeAsync(It.IsAny<RedisChannel>(),+ It.IsAny<Action<RedisChannel, RedisValue>>(), It.IsAny<CommandFlags>()))+ .Returns(Task.CompletedTask);++ _connectionMock = new Mock<IConnectionMultiplexer>();+ _connectionMock.Setup(c => c.GetSubscriber(It.IsAny<object>())).Returns(_subscriberMock.Object);++ _cacheMock = new Mock<ICacheBase>();+ _cacheMock.Setup(c => c.RemoveAsync(It.IsAny<string>(), It.IsAny<bool>())).Returns(Task.CompletedTask);+ _cacheMock.Setup(c => c.RemoveByPrefix(It.IsAny<string>(), It.IsAny<bool>())).Returns(Task.CompletedTask);+ _cacheMock.Setup(c => c.Clear(It.IsAny<bool>())).Returns(Task.CompletedTask);++ var services = new ServiceCollection();+ services.AddSingleton(_cacheMock.Object);+ var serviceProvider = services.BuildServiceProvider();++ _config = new RedisConfig { RedisPubSubChannel = "test-channel" };+ _bus = new RedisMessageBus(_connectionMock.Object, serviceProvider, _config,+ new Mock<ILogger<RedisMessageBus>>().Object);+ }++ [TestCleanup]+ public void Cleanup()+ {+ _bus.Dispose();+ }++ [TestMethod]+ public async Task StartAsync_SubscribesToConfiguredChannel()+ {+ await _bus.StartAsync(CancellationToken.None);++ //subscription runs on a tracked background loop - wait for it+ for (var i = 0; i < 100; i++)+ {+ try+ {+ _subscriberMock.Verify(s => s.SubscribeAsync(+ It.Is<RedisChannel>(c => c == RedisChannel.Literal("test-channel")),+ It.IsAny<Action<RedisChannel, RedisValue>>(), It.IsAny<CommandFlags>()), Times.AtLeastOnce);+ return;+ }+ catch (MockException)+ {+ await Task.Delay(20);+ }+ }++ Assert.Fail("SubscribeAsync was not called on the configured channel");+ }++ [TestMethod]+ public async Task StopAsync_CompletesAndStopsSubscribing()+ {+ await _bus.StartAsync(CancellationToken.None);+ await _bus.StopAsync(CancellationToken.None);+ }++ [TestMethod]+ public async Task PublishAsync_SendsSerializedMessageToConfiguredChannel()+ {+ RedisValue publishedValue = default;+ _subscriberMock.Setup(s =>+ s.PublishAsync(It.IsAny<RedisChannel>(), It.IsAny<RedisValue>(), It.IsAny<CommandFlags>()))+ .Callback<RedisChannel, RedisValue, CommandFlags>((_, value, _) => publishedValue = value)+ .ReturnsAsync(1);++ await _bus.PublishAsync(new MessageEvent { Key = "key", MessageType = (int)MessageEventType.RemoveKey });++ _subscriberMock.Verify(s => s.PublishAsync(+ It.Is<RedisChannel>(c => c == RedisChannel.Literal("test-channel")),+ It.IsAny<RedisValue>(), It.IsAny<CommandFlags>()), Times.Once);++ var message = JsonSerializer.Deserialize<MessageEventClient>(publishedValue.ToString());+ Assert.IsNotNull(message);+ Assert.AreEqual("key", message.Key);+ Assert.AreEqual((int)MessageEventType.RemoveKey, message.MessageType);+ Assert.IsFalse(string.IsNullOrEmpty(message.ClientId));+ }++ [TestMethod]+ public async Task PublishAsync_SubscriberThrows_DoesNotPropagateException()+ {+ _subscriberMock.Setup(s =>+ s.PublishAsync(It.IsAny<RedisChannel>(), It.IsAny<RedisValue>(), It.IsAny<CommandFlags>()))+ .ThrowsAsync(new RedisConnectionException(ConnectionFailureType.SocketFailure, "connection lost"));++ await _bus.PublishAsync(new MessageEvent { Key = "key", MessageType = (int)MessageEventType.RemoveKey });+ }++ [TestMethod]+ public void OnSubscriptionChanged_RemoveKey_RemovesFromLocalCacheWithoutRepublishing()+ {+ _bus.OnSubscriptionChanged(new MessageEventClient+ { ClientId = "other", Key = "key", MessageType = (int)MessageEventType.RemoveKey });++ _cacheMock.Verify(c => c.RemoveAsync("key", false), Times.Once);+ }++ [TestMethod]+ public void OnSubscriptionChanged_RemoveByPrefix_RemovesFromLocalCacheWithoutRepublishing()+ {+ _bus.OnSubscriptionChanged(new MessageEventClient+ { ClientId = "other", Key = "prefix", MessageType = (int)MessageEventType.RemoveByPrefix });++ _cacheMock.Verify(c => c.RemoveByPrefix("prefix", false), Times.Once);+ }++ [TestMethod]+ public void OnSubscriptionChanged_ClearCache_ClearsLocalCacheWithoutRepublishing()+ {+ _bus.OnSubscriptionChanged(new MessageEventClient+ { ClientId = "other", Key = "", MessageType = (int)MessageEventType.ClearCache });++ _cacheMock.Verify(c => c.Clear(false), Times.Once);+ }++ [TestMethod]+ public async Task ConnectionRestored_SubscriptionConnection_ClearsLocalCache()+ {+ await _bus.StartAsync(CancellationToken.None);++ //messages published while disconnected are lost - the local cache must be dropped+ _connectionMock.Raise(c => c.ConnectionRestored += null,+ new ConnectionFailedEventArgs(_connectionMock.Object, new DnsEndPoint("localhost", 6379),+ ConnectionType.Subscription, ConnectionFailureType.SocketClosed, null, "test"));++ _cacheMock.Verify(c => c.Clear(false), Times.Once);+ }++ [TestMethod]+ public async Task ConnectionRestored_InteractiveConnection_DoesNotClearLocalCache()+ {+ await _bus.StartAsync(CancellationToken.None);++ _connectionMock.Raise(c => c.ConnectionRestored += null,+ new ConnectionFailedEventArgs(_connectionMock.Object, new DnsEndPoint("localhost", 6379),+ ConnectionType.Interactive, ConnectionFailureType.SocketClosed, null, "test"));++ _cacheMock.Verify(c => c.Clear(It.IsAny<bool>()), Times.Never);+ }+}diff --git a/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageCacheManagerTests.cs b/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageCacheManagerTests.csnew file mode 100644index 000000000..cc2c7b064--- /dev/null+++ b/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageCacheManagerTests.cs@@ -0,0 +1,102 @@+using Grand.Infrastructure.Caching.Message;+using Grand.Infrastructure.Caching.Redis;+using Grand.Infrastructure.Configuration;+using MediatR;+using Microsoft.Extensions.Caching.Memory;+using Microsoft.Extensions.DependencyInjection;+using Microsoft.VisualStudio.TestTools.UnitTesting;+using Moq;++namespace Grand.Infrastructure.Tests.Caching.Redis;++[TestClass]+public class RedisMessageCacheManagerTests+{+ private Mock<IMessageBus> _messageBusMock;+ private IMemoryCache _memoryCache;+ private RedisMessageCacheManager _service;++ [TestInitialize]+ public void Init()+ {+ var services = new ServiceCollection();+ services.AddMemoryCache();+ _memoryCache = services.BuildServiceProvider().GetService<IMemoryCache>();+ _messageBusMock = new Mock<IMessageBus>();+ _service = new RedisMessageCacheManager(_memoryCache, new Mock<IMediator>().Object,+ _messageBusMock.Object, new CacheConfig { DefaultCacheTimeMinutes = 1 });+ }++ [TestMethod]+ public async Task RemoveAsync_RemovesKeyAndPublishesMessage()+ {+ _service.Get("key", () => "value");++ await _service.RemoveAsync("key");++ Assert.IsFalse(_memoryCache.TryGetValue("key", out _));+ _messageBusMock.Verify(m => m.PublishAsync(It.Is<MessageEvent>(e =>+ e.Key == "key" && e.MessageType == (int)MessageEventType.RemoveKey)), Times.Once);+ }++ [TestMethod]+ public async Task RemoveAsync_PublisherDisabled_DoesNotPublish()+ {+ _service.Get("key", () => "value");++ await _service.RemoveAsync("key", false);++ Assert.IsFalse(_memoryCache.TryGetValue("key", out _));+ _messageBusMock.Verify(m => m.PublishAsync(It.IsAny<MessageEvent>()), Times.Never);+ }++ [TestMethod]+ public async Task RemoveByPrefix_RemovesMatchingKeysAndPublishesMessage()+ {+ _service.Get("prefix-1", () => "value");+ _service.Get("prefix-2", () => "value");+ _service.Get("other", () => "value");++ await _service.RemoveByPrefix("prefix");++ Assert.IsFalse(_memoryCache.TryGetValue("prefix-1", out _));+ Assert.IsFalse(_memoryCache.TryGetValue("prefix-2", out _));+ Assert.IsTrue(_memoryCache.TryGetValue("other", out _));+ _messageBusMock.Verify(m => m.PublishAsync(It.Is<MessageEvent>(e =>+ e.Key == "prefix" && e.MessageType == (int)MessageEventType.RemoveByPrefix)), Times.Once);+ }++ [TestMethod]+ public async Task RemoveByPrefix_PublisherDisabled_DoesNotPublish()+ {+ _service.Get("prefix-1", () => "value");++ await _service.RemoveByPrefix("prefix", false);++ Assert.IsFalse(_memoryCache.TryGetValue("prefix-1", out _));+ _messageBusMock.Verify(m => m.PublishAsync(It.IsAny<MessageEvent>()), Times.Never);+ }++ [TestMethod]+ public async Task Clear_PublishesClearCacheMessage()+ {+ _service.Get("key", () => "value");++ await _service.Clear();++ Assert.IsFalse(_memoryCache.TryGetValue("key", out _));+ _messageBusMock.Verify(m => m.PublishAsync(It.Is<MessageEvent>(e =>+ e.MessageType == (int)MessageEventType.ClearCache)), Times.Once);+ }++ [TestMethod]+ public async Task Clear_PublisherDisabled_DoesNotPublish()+ {+ _service.Get("key", () => "value");++ await _service.Clear(false);++ Assert.IsFalse(_memoryCache.TryGetValue("key", out _));+ _messageBusMock.Verify(m => m.PublishAsync(It.IsAny<MessageEvent>()), Times.Never);+ }+}diff --git a/src/Tests/Grand.Modules.Tests/Grand.Modules.Tests.csproj b/src/Tests/Grand.Modules.Tests/Grand.Modules.Tests.csprojindex aa8e24bc4..dc6e1d9a2 100644--- a/src/Tests/Grand.Modules.Tests/Grand.Modules.Tests.csproj+++ b/src/Tests/Grand.Modules.Tests/Grand.Modules.Tests.csproj@@ -21,6 +21,7 @@<ProjectReference Include="..\..\Business\Grand.Business.Messages\Grand.Business.Messages.csproj" /><ProjectReference Include="..\..\Modules\Grand.Module.Migration\Grand.Module.Migration.csproj" /><ProjectReference Include="..\..\Modules\Grand.Module.ScheduledTasks\Grand.Module.ScheduledTasks.csproj" />+ <ProjectReference Include="..\Grand.Data.Tests\Grand.Data.Tests.csproj" /></ItemGroup></Project>diff --git a/src/Tests/Grand.Modules.Tests/Services/BackgroundService/ScheduleTaskServiceTests.cs b/src/Tests/Grand.Modules.Tests/Services/BackgroundService/ScheduleTaskServiceTests.csnew file mode 100644index 000000000..7663112f3--- /dev/null+++ b/src/Tests/Grand.Modules.Tests/Services/BackgroundService/ScheduleTaskServiceTests.cs@@ -0,0 +1,83 @@+using Grand.Data.Tests.MongoDb;+using Grand.Domain.Tasks;+using Grand.Module.ScheduledTasks.BackgroundServices;+using Microsoft.VisualStudio.TestTools.UnitTesting;++namespace Grand.Modules.Tests.Services.BackgroundService;++[TestClass]+public class ScheduleTaskServiceTests+{+ private MongoDBRepositoryTest<ScheduleTask> _repository;+ private ScheduleTaskService _service;++ [TestInitialize]+ public void Init()+ {+ _repository = new MongoDBRepositoryTest<ScheduleTask>();+ _service = new ScheduleTaskService(_repository);+ }++ private async Task<ScheduleTask> InsertTask(DateTime? lastStartUtc = null)+ {+ var task = new ScheduleTask {+ ScheduleTaskName = "TestTask",+ Enabled = true,+ TimeInterval = 1,+ LastStartUtc = lastStartUtc+ };+ await _service.InsertTask(task);+ //read back so LastStartUtc has the value as stored in the database (ms precision)+ return await _service.GetTaskById(task.Id);+ }++ [TestMethod]+ public async Task TryClaimTaskRun_FirstClaim_ReturnsTrueAndSetsLease()+ {+ var task = await InsertTask();+ var runStartUtc = DateTime.UtcNow;++ var claimed = await _service.TryClaimTaskRun(task.Id, task.LastStartUtc, runStartUtc, "instance-a");++ Assert.IsTrue(claimed);+ var updated = await _service.GetTaskById(task.Id);+ Assert.AreEqual("instance-a", updated.LeasedByInstance);+ Assert.IsNotNull(updated.LastStartUtc);+ }++ [TestMethod]+ public async Task TryClaimTaskRun_StaleExpectedValue_ReturnsFalseAndKeepsWinner()+ {+ var task = await InsertTask(DateTime.UtcNow.AddMinutes(-10));++ //instance A claims the run+ var claimedByA = await _service.TryClaimTaskRun(task.Id, task.LastStartUtc, DateTime.UtcNow, "instance-a");+ //instance B still holds the old LastStartUtc value - its claim must fail+ var claimedByB = await _service.TryClaimTaskRun(task.Id, task.LastStartUtc, DateTime.UtcNow, "instance-b");++ Assert.IsTrue(claimedByA);+ Assert.IsFalse(claimedByB);+ var updated = await _service.GetTaskById(task.Id);+ Assert.AreEqual("instance-a", updated.LeasedByInstance);+ }++ [TestMethod]+ public async Task TryClaimTaskRun_ConcurrentClaims_ExactlyOneWins()+ {+ var task = await InsertTask(DateTime.UtcNow.AddMinutes(-10));++ var claims = await Task.WhenAll(+ _service.TryClaimTaskRun(task.Id, task.LastStartUtc, DateTime.UtcNow, "instance-1"),+ _service.TryClaimTaskRun(task.Id, task.LastStartUtc, DateTime.UtcNow, "instance-2"));++ Assert.AreEqual(1, claims.Count(x => x));+ }++ [TestMethod]+ public async Task TryClaimTaskRun_NonExistingTask_ReturnsFalse()+ {+ var claimed = await _service.TryClaimTaskRun("000000000000000000000000", null, DateTime.UtcNow, "instance-a");++ Assert.IsFalse(claimed);+ }+}diff --git a/src/Tests/Grand.Web.Common.Tests/Infrastructure/BackgroundServiceTaskTests.cs b/src/Tests/Grand.Web.Common.Tests/Infrastructure/BackgroundServiceTaskTests.csnew file mode 100644index 000000000..eba0394a8--- /dev/null+++ b/src/Tests/Grand.Web.Common.Tests/Infrastructure/BackgroundServiceTaskTests.cs@@ -0,0 +1,145 @@+using Grand.Business.Core.Interfaces.System.ScheduleTasks;+using Grand.Domain.Tasks;+using Grand.Infrastructure;+using Grand.Web.Common.Infrastructure;+using Microsoft.Extensions.DependencyInjection;+using Microsoft.VisualStudio.TestTools.UnitTesting;+using Moq;++namespace Grand.Web.Common.Tests.Infrastructure;++[TestClass]+public class BackgroundServiceTaskTests+{+ private const string TaskName = "TestTask";++ private Mock<IScheduleTaskService> _scheduleTaskServiceMock;+ private Mock<IScheduleTask> _scheduleTaskMock;+ private ScheduleTask _task;+ private IServiceProvider _serviceProvider;++ [TestInitialize]+ public void Init()+ {+ _task = new ScheduleTask {+ Id = "1",+ ScheduleTaskName = TaskName,+ Enabled = true,+ TimeInterval = 60+ };++ _scheduleTaskServiceMock = new Mock<IScheduleTaskService>();+ _scheduleTaskServiceMock.Setup(s => s.GetTaskByName(TaskName)).ReturnsAsync(_task);+ _scheduleTaskServiceMock.Setup(s => s.UpdateTask(It.IsAny<ScheduleTask>()))+ .Returns(Task.CompletedTask);++ _scheduleTaskMock = new Mock<IScheduleTask>();+ _scheduleTaskMock.Setup(t => t.Execute()).Returns(Task.CompletedTask);++ var storeContextSetter = new Mock<IStoreContextSetter>();+ storeContextSetter.Setup(s => s.InitializeStoreContext(It.IsAny<string>()))+ .ReturnsAsync(Mock.Of<IStoreContext>());+ var workContextSetter = new Mock<IWorkContextSetter>();+ workContextSetter.Setup(s => s.InitializeWorkContext(It.IsAny<string>()))+ .ReturnsAsync(Mock.Of<IWorkContext>());++ var services = new ServiceCollection();+ services.AddLogging();+ services.AddSingleton(_scheduleTaskServiceMock.Object);+ services.AddKeyedSingleton(TaskName, _scheduleTaskMock.Object);+ services.AddSingleton(Mock.Of<IContextAccessor>());+ services.AddSingleton(storeContextSetter.Object);+ services.AddSingleton(workContextSetter.Object);+ _serviceProvider = services.BuildServiceProvider();+ }++ [TestMethod]+ public async Task ExecuteAsync_ClaimWon_ExecutesTaskAndPersistsResult()+ {+ _scheduleTaskServiceMock.Setup(s => s.TryClaimTaskRun(_task.Id, It.IsAny<DateTime?>(),+ It.IsAny<DateTime>(), It.IsAny<string>()))+ .ReturnsAsync(true);++ var service = new BackgroundServiceTask(TaskName, _serviceProvider);+ using var cts = new CancellationTokenSource();+ await service.StartAsync(cts.Token);++ await WaitFor(() => _scheduleTaskServiceMock.Invocations.Any(i =>+ i.Method.Name == nameof(IScheduleTaskService.UpdateTask)));+ cts.Cancel();++ _scheduleTaskMock.Verify(t => t.Execute(), Times.Once);+ _scheduleTaskServiceMock.Verify(s => s.UpdateTask(It.Is<ScheduleTask>(x =>+ x.LastSuccessUtc != null && x.LastStartUtc != null)), Times.Once);+ }++ [TestMethod]+ public async Task ExecuteAsync_ClaimLost_DoesNotExecuteAndDoesNotPersist()+ {+ _scheduleTaskServiceMock.Setup(s => s.TryClaimTaskRun(_task.Id, It.IsAny<DateTime?>(),+ It.IsAny<DateTime>(), It.IsAny<string>()))+ .ReturnsAsync(false);++ var service = new BackgroundServiceTask(TaskName, _serviceProvider);+ using var cts = new CancellationTokenSource();+ await service.StartAsync(cts.Token);++ await WaitFor(() => _scheduleTaskServiceMock.Invocations.Any(i =>+ i.Method.Name == nameof(IScheduleTaskService.TryClaimTaskRun)));+ //give the loop a moment to (incorrectly) execute the task if the guard is broken+ await Task.Delay(200);+ cts.Cancel();++ _scheduleTaskMock.Verify(t => t.Execute(), Times.Never);+ //losing instance must not overwrite the winner's data with a stale entity+ _scheduleTaskServiceMock.Verify(s => s.UpdateTask(It.IsAny<ScheduleTask>()), Times.Never);+ }++ [TestMethod]+ public async Task ExecuteAsync_TaskNotDueYet_DoesNotClaimNorExecute()+ {+ _task.LastStartUtc = DateTime.UtcNow;++ var service = new BackgroundServiceTask(TaskName, _serviceProvider);+ using var cts = new CancellationTokenSource();+ await service.StartAsync(cts.Token);++ await WaitFor(() => _scheduleTaskServiceMock.Invocations.Any(i =>+ i.Method.Name == nameof(IScheduleTaskService.GetTaskByName)));+ await Task.Delay(200);+ cts.Cancel();++ _scheduleTaskServiceMock.Verify(s => s.TryClaimTaskRun(It.IsAny<string>(), It.IsAny<DateTime?>(),+ It.IsAny<DateTime>(), It.IsAny<string>()), Times.Never);+ _scheduleTaskMock.Verify(t => t.Execute(), Times.Never);+ _scheduleTaskServiceMock.Verify(s => s.UpdateTask(It.IsAny<ScheduleTask>()), Times.Never);+ }++ [TestMethod]+ public async Task ExecuteAsync_TaskLeasedByOtherMachine_DoesNotExecute()+ {+ _task.LeasedByMachineName = "other-machine";++ var service = new BackgroundServiceTask(TaskName, _serviceProvider);+ using var cts = new CancellationTokenSource();+ await service.StartAsync(cts.Token);++ await WaitFor(() => _scheduleTaskServiceMock.Invocations.Any(i =>+ i.Method.Name == nameof(IScheduleTaskService.GetTaskByName)));+ await Task.Delay(200);+ cts.Cancel();++ _scheduleTaskMock.Verify(t => t.Execute(), Times.Never);+ }++ private static async Task WaitFor(Func<bool> condition)+ {+ for (var i = 0; i < 300; i++)+ {+ if (condition()) return;+ await Task.Delay(10);+ }++ Assert.Fail("Condition was not met within the timeout");+ }+}diff --git a/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.cs b/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.csindex dcc0e885e..cc03ed0d6 100644--- a/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.cs+++ b/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.cs@@ -9,6 +9,10 @@ namespace Grand.Web.Common.Infrastructure;public class BackgroundServiceTask : BackgroundService{+ //unique per process - pod/machine names are not stable across scaling, so a random+ //id generated at startup is all that is needed to tell instances apart+ private static readonly string InstanceId = Guid.NewGuid().ToString("N");+private readonly IServiceProvider _serviceProvider;private readonly string Name;@@ -40,6 +44,7 @@ public class BackgroundServiceTask : BackgroundServicemachineName == task.LeasedByMachineName)){+ var updateTask = false;var scheduleTask = serviceProvider.GetRequiredKeyedService<IScheduleTask>(task.ScheduleTaskName);if (scheduleTask != null){@@ -65,32 +70,57 @@ public class BackgroundServiceTask : BackgroundServiceif (runTask){- task.LastStartUtc = DateTime.UtcNow;- try- {- logger.LogInformation($"Task {Name} execute");- await scheduleTask.Execute();- task.LastSuccessUtc = DateTime.UtcNow;- task.LastNonSuccessEndUtc = null;+ //claim this run atomically - when several instances race,+ //only one wins and executes the task (no duplicated e-mails etc.)+ var runStartUtc = DateTime.UtcNow;+ var claimed = await scheduleTaskService.TryClaimTaskRun(task.Id, task.LastStartUtc,+ runStartUtc, InstanceId);+ if (claimed)+ {+ updateTask = true;+ task.LastStartUtc = runStartUtc;+ task.LeasedByInstance = InstanceId;+ try+ {+ logger.LogInformation("Task {TaskName} execute", Name);+ await scheduleTask.Execute();+ task.LastSuccessUtc = DateTime.UtcNow;+ task.LastNonSuccessEndUtc = null;+ }+ catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)+ {+ //application shutdown - do not classify as a task failure+ throw;+ }+ catch (Exception exc)+ {+ task.LastNonSuccessEndUtc = DateTime.UtcNow;+ task.Enabled = !task.StopOnError;+ logger.LogError(exc,+ "Error while running the \'{TaskScheduleTaskName}\' schedule task",+ task.ScheduleTaskName);+ }}- catch (Exception exc)+ else{- task.LastNonSuccessEndUtc = DateTime.UtcNow;- task.Enabled = !task.StopOnError;- logger.LogError(exc,- "Error while running the \'{TaskScheduleTaskName}\' schedule task",- task.ScheduleTaskName);+ //another instance executes this run - check again on the next interval+ if (logger.IsEnabled(LogLevel.Debug))+ logger.LogDebug("Task {TaskName} claimed by another instance, skipping", Name);}}}else{+ updateTask = true;task.Enabled = !task.StopOnError;task.LastNonSuccessEndUtc = DateTime.UtcNow;logger.LogError("Type {TaskName} is not registered", Name);}- await scheduleTaskService.UpdateTask(task);+ //persist only when this instance actually ran the task - an unconditional+ //write would overwrite the claim/results of the winning instance with stale data+ if (updateTask)+ await scheduleTaskService.UpdateTask(task);await Task.Delay(TimeSpan.FromMinutes(timeInterval), stoppingToken);}elsediff --git a/src/Web/Grand.Web.Common/Startup/StartupApplication.cs b/src/Web/Grand.Web.Common/Startup/StartupApplication.csindex 943280079..73ea28c19 100644--- a/src/Web/Grand.Web.Common/Startup/StartupApplication.cs+++ b/src/Web/Grand.Web.Common/Startup/StartupApplication.cs@@ -60,16 +60,24 @@ public class StartupApplication : IStartupApplicationvar config = new RedisConfig();configuration.GetSection("Redis").Bind(config);- serviceCollection.AddSingleton<ICacheBase, MemoryCacheBase>();-if (config.RedisPubSubEnabled){- var redis = ConnectionMultiplexer.Connect(config.RedisPubSubConnectionString);- serviceCollection.AddSingleton(_ => redis.GetSubscriber());- serviceCollection.AddSingleton<IMessageBus, RedisMessageBus>();+ //AbortOnConnectFail=false lets the multiplexer keep retrying in the background+ //instead of crashing the instance when Redis is temporarily unavailable at startup+ var options = ConfigurationOptions.Parse(config.RedisPubSubConnectionString);+ options.AbortOnConnectFail = false;+ var redis = ConnectionMultiplexer.Connect(options);+ serviceCollection.AddSingleton<IConnectionMultiplexer>(redis);+ //single instance exposed both as the message bus and as a hosted service that+ //owns the subscription lifecycle (start/stop) instead of the constructor+ serviceCollection.AddSingleton<RedisMessageBus>();+ serviceCollection.AddSingleton<IMessageBus>(sp => sp.GetRequiredService<RedisMessageBus>());+ serviceCollection.AddHostedService(sp => sp.GetRequiredService<RedisMessageBus>());serviceCollection.AddSingleton<ICacheBase, RedisMessageCacheManager>();return;}++ serviceCollection.AddSingleton<ICacheBase, MemoryCacheBase>();}private static void RegisterContextService(IServiceCollection serviceCollection)